diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000..491bbe9657 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,47 @@ +# 0.0.1-API-Parser + +Parser for fixing this: https://github.com/Homebrew/brew/issues/5725 + +## Overview + +Homebrew is used to install software (packages). Homebrew uses 'formulae' to determine how a package is installed. +This project will automatically check which packages have had newer versions released, whether the package has an open PR on homebrew, and display the results. + +## High-level Solution + +- Fetch latest package version information from [repology.org](https://repology.org/) and store on file system. +- Fetch Homebrew Formulae information from [HomeBrew Formulae](https://formulae.brew.sh) +- Compare Current Homebrew Formulae version numbers and those coming from Repology's API and Livecheck. +- Determine whether package has open PR. +- Display results. + +## Details + +- This project can be run automatically at set intervals via GitHub Actions. +- Executing `ruby printPackageUpdates.rb` from the command line will query + both the Repology and Homebrew APIs. Homebrew's current version of each + package will be compared to the latest version of the package, per Repology's response. +- Homebrew's livecheck is also queried for each package, and that data is parsed, if available. +- Checks whether there is open PR for package. +- Each outdated package will be displayed to the console like so: +- Note that some packages will not be included in the Livecheck response. Those will have a 'Livecheck latest:' value of 'Not found'. + +``` +Package: openclonk +Brew current: 7.0 +Repology latest: 8.1 +Livecheck latest: 8.1 +Has Open PR?: true + +Package: openjdk +Brew current: 13.0.2+8 +Repology latest: 15.0.0.0~14 +Livecheck latest: Not found. +Has Open PR?: false + +Package: opentsdb +Brew current: 2.3.1 +Repology latest: 2.4.0 +Livecheck latest: 2.4.0 +Has Open PR?: true +``` diff --git a/scripts/bumpFormulae.rb b/scripts/bumpFormulae.rb new file mode 100644 index 0000000000..5ccf0aeda3 --- /dev/null +++ b/scripts/bumpFormulae.rb @@ -0,0 +1,20 @@ +require_relative 'helpers/parsed_file' +require_relative 'helpers/brew_commands.rb' + +brew_commands = BrewCommands.new + +parsed_file = ParsedFile.new +outdated_pckgs_to_update = parsed_file.get_latest_file("data/outdated_pckgs_to_update") + +File.foreach(outdated_pckgs_to_update) do |line| + line_hash = eval(line) + puts "\n bumping package: #{line_hash['name']} formula" + + begin + bump_pr_response, bump_pr_status = brew_commands.bump_formula_pr(line_hash['name'], line_hash['download_url'], line_hash['checksum']) + puts "#{bump_pr_response}" + rescue + puts "- An error occured whilst bumping package #{line_hash['name']} \n" + return + end +end diff --git a/scripts/helpers/api_parser.rb b/scripts/helpers/api_parser.rb new file mode 100644 index 0000000000..9ab58a97e7 --- /dev/null +++ b/scripts/helpers/api_parser.rb @@ -0,0 +1,133 @@ +require 'net/http' +require 'json' + +require_relative 'brew_commands' +require_relative 'homebrew_formula' + +class ApiParser + def call_api(url) + puts "- Calling API #{url}" + uri = URI(url) + response = Net::HTTP.get(uri) + + puts "- Parsing response" + JSON.parse(response) + end + + def query_repology_api(last_package_in_response = '') + url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' + + self.call_api(url) + end + + def parse_repology_api() + puts "\n-------- Query outdated packages from Repology --------" + page_no = 1 + puts "\n- Paginating repology api page: #{page_no}" + + outdated_packages = self.query_repology_api('') + last_pacakge_index = outdated_packages.size - 1 + response_size = outdated_packages.size + + while response_size > 1 do + page_no += 1 + puts "\n- Paginating repology api page: #{page_no}" + + last_package_in_response = outdated_packages.keys[last_pacakge_index] + response = self.query_repology_api("#{last_package_in_response}/") + + response_size = response.size + outdated_packages.merge!(response) + last_pacakge_index = outdated_packages.size - 1 + end + + puts "\n- #{outdated_packages.size} outdated pacakges identified by repology" + outdated_packages + end + + def query_homebrew + puts "\n-------- Get Homebrew Formulas --------" + self.call_api('https://formulae.brew.sh/api/formula.json') + end + + def parse_homebrew_formulas() + formulas = self.query_homebrew() + parsed_homebrew_formulas = {} + + formulas.each do |formula| + parsed_homebrew_formulas[formula['name']] = { + "fullname" => formula["full_name"], + "oldname" => formula["oldname"], + "version" => formula["versions"]['stable'], + "download_url" => formula["urls"]['stable']['url'], + } + end + + parsed_homebrew_formulas + end + + def validate_packages(outdated_repology_packages, brew_formulas) + puts "\n-------- Verify Outdated Repology packages as Homebrew Formulas --------" + packages = {} + + outdated_repology_packages.each do |package_name, repo_using_package| + # Identify homebrew repo + repology_homebrew_repo = repo_using_package.select { |repo| repo['repo'] == 'homebrew' }[0] + next if repology_homebrew_repo.empty? + + latest_version = nil + + # Identify latest version amongst repos + repo_using_package.each do |repo| + latest_version = repo['version'] if repo['status'] == 'newest' + end + + repology_homebrew_repo['latest_version'] = latest_version if latest_version + homebrew_package_details = brew_formulas[repology_homebrew_repo['srcname']] + + # Format package + packages[repology_homebrew_repo['srcname']] = format_package(homebrew_package_details, repology_homebrew_repo) + end + + packages + end + + + def format_package(homebrew_details, repology_details) + puts "- Formatting package: #{repology_details['srcname']}" + + homebrew_formula = HomebrewFormula.new + new_download_url = homebrew_formula.generate_new_download_url(homebrew_details['download_url'], homebrew_details['version'], repology_details['latest_version']) + + brew_commands = BrewCommands.new + livecheck_response = brew_commands.livecheck_check_formula(repology_details['srcname']) + has_open_pr = brew_commands.check_for_open_pr(repology_details['srcname'], new_download_url) + + formatted_package = { + 'fullname'=> homebrew_details['fullname'], + 'repology_version' => repology_details['latest_version'], + 'homebrew_version' => homebrew_details['version'], + 'livecheck_latest_version' => livecheck_response['livecheck_latest_version'], + 'current_download_url' => homebrew_details['download_url'], + 'latest_download_url' => new_download_url, + 'repology_latest_version' => repology_details['latest_version'], + 'has_open_pr' => has_open_pr + } + + formatted_package + end + + def display_version_data(outdated_packages) + puts "==============Formatted outdated packages============\n" + + outdated_packages.each do |package_name, package_details| + puts "" + puts "Package: #{package_name}" + puts "Brew current: #{package_details['homebrew_version']}" + puts "Repology latest: #{package_details['repology_version']}" + puts "Livecheck latest: #{package_details['livecheck_latest_version']}" + puts "Has Open PR?: #{package_details['has_open_pr']}" + end + end + +end diff --git a/scripts/helpers/brew_commands.rb b/scripts/helpers/brew_commands.rb new file mode 100644 index 0000000000..526162095b --- /dev/null +++ b/scripts/helpers/brew_commands.rb @@ -0,0 +1,55 @@ +require "open3" + +class BrewCommands + + def livecheck_check_formula(formula_name) + puts "- livecheck formula : #{formula_name}" + command_args = [ + "brew", + "livecheck", + formula_name, + "--quiet", + ] + + response = Open3.capture2e(*command_args) + self.parse_livecheck_response(response) + end + + def parse_livecheck_response(livecheck_output) + livecheck_output = livecheck_output.first.gsub(' ', '').split(/:|==>|\n/) + + # eg: ["burp", "2.2.18", "2.2.18"] + package_name, brew_version, latest_version = livecheck_output + + {'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version} + end + + def bump_formula_pr(formula_name, url) + command_args = [ + "brew", + "bump-formula-pr", + "--no-browse", + "--dry-run", + formula_name, + "--url=#{url}", + ] + + response = Open3.capture2e(*command_args) + self.parse_formula_bump_response(response) + end + + def parse_formula_bump_response(formula_bump_response) + response, status = formula_bump_response + response + end + + def check_for_open_pr(formula_name, download_url) + puts "- Checking for open PRs for formula : #{formula_name}" + + response = bump_formula_pr(formula_name, download_url) + + return true if !response.include? 'Error: These open pull requests may be duplicates' + false + end + +end \ No newline at end of file diff --git a/scripts/helpers/homebrew_formula.rb b/scripts/helpers/homebrew_formula.rb new file mode 100644 index 0000000000..d99b7e8bde --- /dev/null +++ b/scripts/helpers/homebrew_formula.rb @@ -0,0 +1,27 @@ +require 'net/http' +require 'open-uri' + +class HomebrewFormula + + def generate_new_download_url(outdated_url, old_version, latest_version) + if [outdated_url, old_version, latest_version].include? nil + puts "\n- Could not generate download url" + nil + else + puts "\n- Generating download url" + outdated_url.gsub(old_version, latest_version) + end + end + + def generate_checksum(new_url) + begin + puts "- Generating checksum for url: #{new_url}" + tempfile = URI.parse(new_url).open + tempfile.close + return Digest::SHA256.file(tempfile.path).hexdigest + rescue + puts "- Failed to generate Checksum \n" + return nil + end + end +end diff --git a/scripts/helpers/parsed_file.rb b/scripts/helpers/parsed_file.rb new file mode 100644 index 0000000000..038a11a38d --- /dev/null +++ b/scripts/helpers/parsed_file.rb @@ -0,0 +1,23 @@ +require 'fileutils' + +class ParsedFile + + def get_latest_file(directory) + puts "- retrieving latest file in directory: #{directory}" + Dir.glob("#{directory}/*").max_by(1) {|f| File.mtime(f)}[0] + end + + def save_to(directory, data) + # Create directory if does not exist + FileUtils.mkdir_p directory unless Dir.exists?(directory) + + puts "- Generating datetime stamp" + #Include time to the filename for uniqueness when fetching multiple times a day + date_time = Time.new.strftime("%Y-%m-%dT%H_%M_%S") + + # Writing parsed data to file + puts "- Writing data to file" + File.write("#{directory}/#{date_time}.txt", data) + end + +end \ No newline at end of file diff --git a/scripts/printPackageUpdates.rb b/scripts/printPackageUpdates.rb new file mode 100644 index 0000000000..4b57abd95b --- /dev/null +++ b/scripts/printPackageUpdates.rb @@ -0,0 +1,10 @@ +require_relative 'helpers/api_parser' + +api_parser = ApiParser.new + +outdated_repology_packages = api_parser.parse_repology_api() +brew_formulas = api_parser.parse_homebrew_formulas() + +formatted_outdated_packages = api_parser.validate_packages(outdated_repology_packages, brew_formulas) + +api_parser.display_version_data(formatted_outdated_packages)