compare and display package versions

scripts/README.md

@@ -0,0 +1,47 @@
+# 0.0.1-API-Parser
+
+Parser for fixing this: https://github.com/Homebrew/brew/issues/5725
+
+## Overview
+
+Homebrew is used to install software (packages). Homebrew uses 'formulae' to determine how a package is installed.
+This project will automatically check which packages have had newer versions released, whether the package has an open PR on homebrew, and display the results.
+
+## High-level Solution
+
+- Fetch latest package version information from [repology.org](https://repology.org/) and store on file system.
+- Fetch Homebrew Formulae information from [HomeBrew Formulae](https://formulae.brew.sh)
+- Compare Current Homebrew Formulae version numbers and those coming from Repology's API and Livecheck.
+- Determine whether package has open PR.
+- Display results.
+
+## Details
+
+- This project can be run automatically at set intervals via GitHub Actions.
+- Executing `ruby printPackageUpdates.rb` from the command line will query
+  both the Repology and Homebrew APIs. Homebrew's current version of each
+  package will be compared to the latest version of the package, per Repology's response.
+- Homebrew's livecheck is also queried for each package, and that data is parsed, if available.
+- Checks whether there is open PR for package.
+- Each outdated package will be displayed to the console like so:
+- Note that some packages will not be included in the Livecheck response.  Those will have a 'Livecheck latest:' value of 'Not found'.
+
+```
+Package: openclonk
+Brew current: 7.0
+Repology latest: 8.1
+Livecheck latest: 8.1
+Has Open PR?: true
+
+Package: openjdk
+Brew current: 13.0.2+8
+Repology latest: 15.0.0.0~14
+Livecheck latest: Not found.
+Has Open PR?: false
+
+Package: opentsdb
+Brew current: 2.3.1
+Repology latest: 2.4.0
+Livecheck latest: 2.4.0
+Has Open PR?: true
+```

scripts/bumpFormulae.rb

@@ -0,0 +1,20 @@
+require_relative 'helpers/parsed_file'
+require_relative 'helpers/brew_commands.rb'
+
+brew_commands = BrewCommands.new
+
+parsed_file = ParsedFile.new
+outdated_pckgs_to_update = parsed_file.get_latest_file("data/outdated_pckgs_to_update")
+
+File.foreach(outdated_pckgs_to_update) do |line|
+  line_hash = eval(line)
+  puts "\n bumping package: #{line_hash['name']} formula"
+
+  begin
+    bump_pr_response, bump_pr_status = brew_commands.bump_formula_pr(line_hash['name'], line_hash['download_url'], line_hash['checksum'])
+    puts "#{bump_pr_response}"
+  rescue
+    puts "- An error occured whilst bumping package #{line_hash['name']} \n"
+    return
+  end
+end

scripts/helpers/api_parser.rb

@@ -0,0 +1,133 @@
+require 'net/http'
+require 'json'
+
+require_relative 'brew_commands'
+require_relative 'homebrew_formula'
+
+class ApiParser
+  def call_api(url)
+    puts "- Calling API #{url}"
+    uri = URI(url)
+    response = Net::HTTP.get(uri)
+
+    puts "- Parsing response"
+    JSON.parse(response)
+  end
+
+  def query_repology_api(last_package_in_response = '')
+    url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1'
+
+    self.call_api(url)
+  end
+
+  def parse_repology_api()
+    puts "\n-------- Query outdated packages from Repology --------"
+    page_no = 1
+    puts "\n- Paginating repology api page: #{page_no}"
+
+    outdated_packages = self.query_repology_api('')
+    last_pacakge_index = outdated_packages.size - 1
+    response_size = outdated_packages.size
+
+    while response_size > 1  do
+      page_no += 1
+      puts "\n- Paginating repology api page: #{page_no}"
+
+      last_package_in_response = outdated_packages.keys[last_pacakge_index]
+      response = self.query_repology_api("#{last_package_in_response}/")
+
+      response_size = response.size
+      outdated_packages.merge!(response)
+      last_pacakge_index = outdated_packages.size - 1
+    end
+
+    puts "\n- #{outdated_packages.size} outdated pacakges identified by repology"
+    outdated_packages
+  end
+
+  def query_homebrew
+    puts "\n-------- Get Homebrew Formulas --------"
+    self.call_api('https://formulae.brew.sh/api/formula.json')
+  end
+
+  def parse_homebrew_formulas()
+    formulas = self.query_homebrew()
+    parsed_homebrew_formulas = {}
+
+    formulas.each do |formula|
+      parsed_homebrew_formulas[formula['name']] = {
+        "fullname" => formula["full_name"],
+        "oldname" => formula["oldname"],
+        "version" => formula["versions"]['stable'],
+        "download_url" => formula["urls"]['stable']['url'],
+      }
+    end
+
+    parsed_homebrew_formulas
+  end
+
+  def validate_packages(outdated_repology_packages, brew_formulas)
+    puts "\n-------- Verify Outdated Repology packages as Homebrew Formulas --------"
+    packages = {}
+
+    outdated_repology_packages.each do |package_name, repo_using_package|
+      # Identify homebrew repo
+      repology_homebrew_repo = repo_using_package.select { |repo| repo['repo'] == 'homebrew' }[0]
+      next if repology_homebrew_repo.empty?
+
+      latest_version = nil
+
+      # Identify latest version amongst repos
+      repo_using_package.each do |repo|
+        latest_version = repo['version'] if repo['status'] == 'newest'
+      end
+
+      repology_homebrew_repo['latest_version'] = latest_version if latest_version
+      homebrew_package_details = brew_formulas[repology_homebrew_repo['srcname']]
+      
+      # Format package
+      packages[repology_homebrew_repo['srcname']] = format_package(homebrew_package_details, repology_homebrew_repo)
+    end
+
+    packages
+  end
+
+
+  def format_package(homebrew_details, repology_details)
+    puts "- Formatting package: #{repology_details['srcname']}"
+
+    homebrew_formula = HomebrewFormula.new
+    new_download_url = homebrew_formula.generate_new_download_url(homebrew_details['download_url'], homebrew_details['version'], repology_details['latest_version'])
+
+    brew_commands = BrewCommands.new
+    livecheck_response = brew_commands.livecheck_check_formula(repology_details['srcname'])
+    has_open_pr = brew_commands.check_for_open_pr(repology_details['srcname'], new_download_url)
+   
+    formatted_package = {
+      'fullname'=> homebrew_details['fullname'],
+      'repology_version' => repology_details['latest_version'],
+      'homebrew_version' => homebrew_details['version'],
+      'livecheck_latest_version' => livecheck_response['livecheck_latest_version'],
+      'current_download_url' => homebrew_details['download_url'],
+      'latest_download_url' => new_download_url,
+      'repology_latest_version' => repology_details['latest_version'],
+      'has_open_pr' => has_open_pr
+    } 
+
+    formatted_package
+  end
+
+  def display_version_data(outdated_packages)
+    puts "==============Formatted outdated packages============\n"
+
+    outdated_packages.each do |package_name, package_details|
+      puts ""
+      puts "Package: #{package_name}"
+      puts "Brew current: #{package_details['homebrew_version']}"
+      puts "Repology latest: #{package_details['repology_version']}"
+      puts "Livecheck latest: #{package_details['livecheck_latest_version']}"
+      puts "Has Open PR?: #{package_details['has_open_pr']}"
+    end
+  end
+
+end

scripts/helpers/brew_commands.rb

@@ -0,0 +1,55 @@
+require "open3"
+
+class BrewCommands
+ 
+  def livecheck_check_formula(formula_name)
+    puts "- livecheck formula : #{formula_name}"
+    command_args = [
+      "brew",
+      "livecheck",
+      formula_name,
+      "--quiet",
+    ]
+
+    response = Open3.capture2e(*command_args)
+    self.parse_livecheck_response(response)
+  end
+
+  def parse_livecheck_response(livecheck_output)
+    livecheck_output = livecheck_output.first.gsub(' ', '').split(/:|==>|\n/)
+
+    # eg: ["burp", "2.2.18", "2.2.18"]
+    package_name, brew_version, latest_version = livecheck_output
+  
+    {'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version}
+  end
+
+  def bump_formula_pr(formula_name, url)
+    command_args = [
+      "brew",
+      "bump-formula-pr",
+      "--no-browse",
+      "--dry-run",
+      formula_name,
+      "--url=#{url}",
+    ]
+
+    response = Open3.capture2e(*command_args)
+    self.parse_formula_bump_response(response)
+  end
+
+  def parse_formula_bump_response(formula_bump_response)
+    response, status  = formula_bump_response
+    response    
+  end
+
+  def check_for_open_pr(formula_name, download_url)
+    puts "- Checking for open PRs for formula : #{formula_name}"
+
+    response =  bump_formula_pr(formula_name, download_url)
+
+    return true if !response.include? 'Error: These open pull requests may be duplicates'
+    false
+  end  
+
+end

scripts/helpers/homebrew_formula.rb

@@ -0,0 +1,27 @@
+require 'net/http'
+require 'open-uri'
+
+class HomebrewFormula
+
+  def generate_new_download_url(outdated_url, old_version, latest_version)
+      if [outdated_url, old_version, latest_version].include? nil
+        puts "\n- Could not generate download url"
+        nil
+      else
+        puts "\n- Generating download url"
+        outdated_url.gsub(old_version, latest_version)
+      end
+  end
+
+  def generate_checksum(new_url)
+    begin
+      puts "- Generating checksum for url: #{new_url}"
+      tempfile = URI.parse(new_url).open
+      tempfile.close
+      return Digest::SHA256.file(tempfile.path).hexdigest
+    rescue
+      puts "- Failed to generate Checksum \n"
+      return nil
+    end
+  end
+end

scripts/helpers/parsed_file.rb

@@ -0,0 +1,23 @@
+require 'fileutils'
+
+class ParsedFile
+
+  def get_latest_file(directory)
+    puts "- retrieving latest file in directory: #{directory}"
+    Dir.glob("#{directory}/*").max_by(1) {|f| File.mtime(f)}[0]
+  end
+
+  def save_to(directory, data)
+      # Create directory if does not exist
+      FileUtils.mkdir_p directory unless Dir.exists?(directory)
+
+      puts "- Generating datetime stamp"
+      #Include time to the filename for uniqueness when fetching multiple times a day
+      date_time = Time.new.strftime("%Y-%m-%dT%H_%M_%S")
+
+      # Writing parsed data to file
+      puts "- Writing data to file"
+      File.write("#{directory}/#{date_time}.txt", data)
+  end
+
+end

scripts/printPackageUpdates.rb

@@ -0,0 +1,10 @@
+require_relative 'helpers/api_parser'
+
+api_parser = ApiParser.new
+
+outdated_repology_packages = api_parser.parse_repology_api()
+brew_formulas = api_parser.parse_homebrew_formulas()
+
+formatted_outdated_packages = api_parser.validate_packages(outdated_repology_packages, brew_formulas)
+
+api_parser.display_version_data(formatted_outdated_packages)