diff --git a/Library/Homebrew/dev-cmd/bump.rb b/Library/Homebrew/dev-cmd/bump.rb index ccf87e3202..f577f71add 100644 --- a/Library/Homebrew/dev-cmd/bump.rb +++ b/Library/Homebrew/dev-cmd/bump.rb @@ -17,7 +17,8 @@ module Homebrew def bump bump_args.parse - puts "command run" - # parse_repology_api() + # puts "command run" + outdated_repology_pacakges = RepologyParser.parse_api_response() + puts RepologyParser.validate__packages(outdated_repology_pacakges) end end diff --git a/Library/Homebrew/utils/livecheck.rb b/Library/Homebrew/utils/livecheck.rb index 2b420ef86c..4306793ac2 100644 --- a/Library/Homebrew/utils/livecheck.rb +++ b/Library/Homebrew/utils/livecheck.rb @@ -4,12 +4,12 @@ require "open3" module Livecheck def livecheck_formula_response(formula_name) - puts "- livecheck formula : #{formula_name}" + ohai "- livecheck formula : #{formula_name}" command_args = [ "brew", "livecheck", formula_name, - "--quiet" + "--quiet", ] response = Open3.capture2e(*command_args) @@ -17,11 +17,12 @@ module Livecheck end def parse_livecheck_response(response) - output = response.first.gsub(' ', '').split(/:|==>|\n/) + output = response.first.delete(" ").split(/:|==>|\n/) # eg: ["burp", "2.2.18", "2.2.18"] package_name, brew_version, latest_version = output - {'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version} + { "name" => package_name, "current_brew_version" => brew_version, + "livecheck_latest_version" => latest_version } end end diff --git a/Library/Homebrew/utils/repology.rb b/Library/Homebrew/utils/repology.rb index 80973956b1..0186d3da33 100644 --- a/Library/Homebrew/utils/repology.rb +++ b/Library/Homebrew/utils/repology.rb @@ -1,47 +1,75 @@ # frozen_string_literal: true -require "net/http" -require "json" +require "utils/curl" +require "formula_info" module RepologyParser - def call_api(url) - puts "- Calling API #{url}" - uri = URI(url) - response = Net::HTTP.get(uri) + module_function - puts "- Parsing response" - JSON.parse(response) - end - - def query_repology_api(last_package_in_response = "") + def query_api(last_package_in_response = "") url = "https://repology.org/api/v1/projects/#{last_package_in_response}?inrepo=homebrew&outdated=1" + ohai "Calling API #{url}" if Homebrew.args.verbose? - call_api(url) + output, errors, status = curl_output(url.to_s) + output = JSON.parse(output) end - def parse_repology_api - puts "\n-------- Query outdated packages from Repology --------" + def parse_api_response() + ohai "Querying outdated packages from Repology" page_no = 1 - puts "\n- Paginating repology api page: #{page_no}" + ohai "Paginating repology api page: #{page_no}" if Homebrew.args.verbose? - outdated_packages = query_repology_api("") + outdated_packages = query_api() last_pacakge_index = outdated_packages.size - 1 response_size = outdated_packages.size + page_limit = 15 - while response_size > 1 + while response_size > 1 && page_no <= page_limit page_no += 1 - puts "\n- Paginating repology api page: #{page_no}" + ohai "Paginating repology api page: #{page_no}" if Homebrew.args.verbose? last_package_in_response = outdated_packages.keys[last_pacakge_index] - response = query_repology_api("#{last_package_in_response}/") + response = query_api("#{last_package_in_response}/") response_size = response.size outdated_packages.merge!(response) last_pacakge_index = outdated_packages.size - 1 end - puts "\n- #{outdated_packages.size} outdated packages identified by repology" + ohai "#{outdated_packages.size} outdated packages identified" outdated_packages end + + def validate__packages(outdated_repology_packages) + ohai "Verifying outdated repology packages as Homebrew Formulae" + + packages = {} + outdated_repology_packages.each do |_name, repositories| + # identify homebrew repo + repology_homebrew_repo = repositories.find do |repo| + repo["repo"] == "homebrew" + end + + next if repology_homebrew_repo.empty? + latest_version = nil + + # identify latest version amongst repology repos + repositories.each do |repo| + latest_version = repo["version"] if repo["status"] == "newest" + end + + info = FormulaInfo.lookup(repology_homebrew_repo["srcname"]) + next unless info + current_version = info.pkg_version + + packages[repology_homebrew_repo["srcname"]] = { + "repology_latest_version" => latest_version, + "current_formula_version" => current_version.to_s + } + puts packages + end + # hash of hashes {"aacgain"=>{"repology_latest_version"=>"1.9", "current_formula_version"=>"1.8"}, ...} + packages + end end diff --git a/Library/Homebrew/utils/versions.rb b/Library/Homebrew/utils/versions.rb index 5da7f873ea..0155f1a0cf 100644 --- a/Library/Homebrew/utils/versions.rb +++ b/Library/Homebrew/utils/versions.rb @@ -22,15 +22,15 @@ module Versions parse_formula_bump_response(response) end - def parse_formula_bump_response(response) - response, status = formula_bump_response + def parse_formula_bump_response(formula_bump_response) + response, _status = formula_bump_response response end def check_for_open_pr(formula_name, download_url) - puts "- Checking for open PRs for formula : #{formula_name}" + ohai "- Checking for open PRs for formula : #{formula_name}" response = bump_formula_pr(formula_name, download_url) - !response.include? 'Error: These open pull requests may be duplicates' + !response.include? "Error: These open pull requests may be duplicates" end end diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 491bbe9657..0000000000 --- a/scripts/README.md +++ /dev/null @@ -1,47 +0,0 @@ -# 0.0.1-API-Parser - -Parser for fixing this: https://github.com/Homebrew/brew/issues/5725 - -## Overview - -Homebrew is used to install software (packages). Homebrew uses 'formulae' to determine how a package is installed. -This project will automatically check which packages have had newer versions released, whether the package has an open PR on homebrew, and display the results. - -## High-level Solution - -- Fetch latest package version information from [repology.org](https://repology.org/) and store on file system. -- Fetch Homebrew Formulae information from [HomeBrew Formulae](https://formulae.brew.sh) -- Compare Current Homebrew Formulae version numbers and those coming from Repology's API and Livecheck. -- Determine whether package has open PR. -- Display results. - -## Details - -- This project can be run automatically at set intervals via GitHub Actions. -- Executing `ruby printPackageUpdates.rb` from the command line will query - both the Repology and Homebrew APIs. Homebrew's current version of each - package will be compared to the latest version of the package, per Repology's response. -- Homebrew's livecheck is also queried for each package, and that data is parsed, if available. -- Checks whether there is open PR for package. -- Each outdated package will be displayed to the console like so: -- Note that some packages will not be included in the Livecheck response. Those will have a 'Livecheck latest:' value of 'Not found'. - -``` -Package: openclonk -Brew current: 7.0 -Repology latest: 8.1 -Livecheck latest: 8.1 -Has Open PR?: true - -Package: openjdk -Brew current: 13.0.2+8 -Repology latest: 15.0.0.0~14 -Livecheck latest: Not found. -Has Open PR?: false - -Package: opentsdb -Brew current: 2.3.1 -Repology latest: 2.4.0 -Livecheck latest: 2.4.0 -Has Open PR?: true -``` diff --git a/scripts/bumpFormulae.rb b/scripts/bumpFormulae.rb index 5ccf0aeda3..16678b61cc 100644 --- a/scripts/bumpFormulae.rb +++ b/scripts/bumpFormulae.rb @@ -1,5 +1,5 @@ -require_relative 'helpers/parsed_file' -require_relative 'helpers/brew_commands.rb' +require_relative "helpers/parsed_file" +require_relative "helpers/brew_commands.rb" brew_commands = BrewCommands.new @@ -11,10 +11,10 @@ File.foreach(outdated_pckgs_to_update) do |line| puts "\n bumping package: #{line_hash['name']} formula" begin - bump_pr_response, bump_pr_status = brew_commands.bump_formula_pr(line_hash['name'], line_hash['download_url'], line_hash['checksum']) + bump_pr_response, bump_pr_status = brew_commands.bump_formula_pr(line_hash["name"], line_hash["download_url"], line_hash["checksum"]) puts "#{bump_pr_response}" rescue - puts "- An error occured whilst bumping package #{line_hash['name']} \n" + puts "- An error occured whilst bumping package #{line_hash["name"]} \n" return end end diff --git a/scripts/helpers/api_parser.rb b/scripts/helpers/api_parser.rb deleted file mode 100644 index 9ab58a97e7..0000000000 --- a/scripts/helpers/api_parser.rb +++ /dev/null @@ -1,133 +0,0 @@ -require 'net/http' -require 'json' - -require_relative 'brew_commands' -require_relative 'homebrew_formula' - -class ApiParser - def call_api(url) - puts "- Calling API #{url}" - uri = URI(url) - response = Net::HTTP.get(uri) - - puts "- Parsing response" - JSON.parse(response) - end - - def query_repology_api(last_package_in_response = '') - url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' - - self.call_api(url) - end - - def parse_repology_api() - puts "\n-------- Query outdated packages from Repology --------" - page_no = 1 - puts "\n- Paginating repology api page: #{page_no}" - - outdated_packages = self.query_repology_api('') - last_pacakge_index = outdated_packages.size - 1 - response_size = outdated_packages.size - - while response_size > 1 do - page_no += 1 - puts "\n- Paginating repology api page: #{page_no}" - - last_package_in_response = outdated_packages.keys[last_pacakge_index] - response = self.query_repology_api("#{last_package_in_response}/") - - response_size = response.size - outdated_packages.merge!(response) - last_pacakge_index = outdated_packages.size - 1 - end - - puts "\n- #{outdated_packages.size} outdated pacakges identified by repology" - outdated_packages - end - - def query_homebrew - puts "\n-------- Get Homebrew Formulas --------" - self.call_api('https://formulae.brew.sh/api/formula.json') - end - - def parse_homebrew_formulas() - formulas = self.query_homebrew() - parsed_homebrew_formulas = {} - - formulas.each do |formula| - parsed_homebrew_formulas[formula['name']] = { - "fullname" => formula["full_name"], - "oldname" => formula["oldname"], - "version" => formula["versions"]['stable'], - "download_url" => formula["urls"]['stable']['url'], - } - end - - parsed_homebrew_formulas - end - - def validate_packages(outdated_repology_packages, brew_formulas) - puts "\n-------- Verify Outdated Repology packages as Homebrew Formulas --------" - packages = {} - - outdated_repology_packages.each do |package_name, repo_using_package| - # Identify homebrew repo - repology_homebrew_repo = repo_using_package.select { |repo| repo['repo'] == 'homebrew' }[0] - next if repology_homebrew_repo.empty? - - latest_version = nil - - # Identify latest version amongst repos - repo_using_package.each do |repo| - latest_version = repo['version'] if repo['status'] == 'newest' - end - - repology_homebrew_repo['latest_version'] = latest_version if latest_version - homebrew_package_details = brew_formulas[repology_homebrew_repo['srcname']] - - # Format package - packages[repology_homebrew_repo['srcname']] = format_package(homebrew_package_details, repology_homebrew_repo) - end - - packages - end - - - def format_package(homebrew_details, repology_details) - puts "- Formatting package: #{repology_details['srcname']}" - - homebrew_formula = HomebrewFormula.new - new_download_url = homebrew_formula.generate_new_download_url(homebrew_details['download_url'], homebrew_details['version'], repology_details['latest_version']) - - brew_commands = BrewCommands.new - livecheck_response = brew_commands.livecheck_check_formula(repology_details['srcname']) - has_open_pr = brew_commands.check_for_open_pr(repology_details['srcname'], new_download_url) - - formatted_package = { - 'fullname'=> homebrew_details['fullname'], - 'repology_version' => repology_details['latest_version'], - 'homebrew_version' => homebrew_details['version'], - 'livecheck_latest_version' => livecheck_response['livecheck_latest_version'], - 'current_download_url' => homebrew_details['download_url'], - 'latest_download_url' => new_download_url, - 'repology_latest_version' => repology_details['latest_version'], - 'has_open_pr' => has_open_pr - } - - formatted_package - end - - def display_version_data(outdated_packages) - puts "==============Formatted outdated packages============\n" - - outdated_packages.each do |package_name, package_details| - puts "" - puts "Package: #{package_name}" - puts "Brew current: #{package_details['homebrew_version']}" - puts "Repology latest: #{package_details['repology_version']}" - puts "Livecheck latest: #{package_details['livecheck_latest_version']}" - puts "Has Open PR?: #{package_details['has_open_pr']}" - end - end - -end diff --git a/scripts/helpers/brew_commands.rb b/scripts/helpers/brew_commands.rb index 380e64da48..c26401a7bd 100644 --- a/scripts/helpers/brew_commands.rb +++ b/scripts/helpers/brew_commands.rb @@ -16,12 +16,12 @@ class BrewCommands end def parse_livecheck_response(livecheck_output) - livecheck_output = livecheck_output.first.gsub(' ', '').split(/:|==>|\n/) + livecheck_output = livecheck_output.first.gsub(" ", "").split(/:|==>|\n/) # eg: ["burp", "2.2.18", "2.2.18"] package_name, brew_version, latest_version = livecheck_output - {'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version} + {"name" => package_name, "current_brew_version" => brew_version, "livecheck_latest_version" => latest_version} end def bump_formula_pr(formula_name, url) @@ -48,7 +48,7 @@ class BrewCommands response = bump_formula_pr(formula_name, download_url) - !response.include? 'Error: These open pull requests may be duplicates' + !response.include? "Error: These open pull requests may be duplicates" end end diff --git a/scripts/helpers/homebrew_formula.rb b/scripts/helpers/homebrew_formula.rb index d99b7e8bde..30358e424e 100644 --- a/scripts/helpers/homebrew_formula.rb +++ b/scripts/helpers/homebrew_formula.rb @@ -1,5 +1,5 @@ -require 'net/http' -require 'open-uri' +require "net/http" +require "open-uri" class HomebrewFormula diff --git a/scripts/helpers/parsed_file.rb b/scripts/helpers/parsed_file.rb deleted file mode 100644 index 038a11a38d..0000000000 --- a/scripts/helpers/parsed_file.rb +++ /dev/null @@ -1,23 +0,0 @@ -require 'fileutils' - -class ParsedFile - - def get_latest_file(directory) - puts "- retrieving latest file in directory: #{directory}" - Dir.glob("#{directory}/*").max_by(1) {|f| File.mtime(f)}[0] - end - - def save_to(directory, data) - # Create directory if does not exist - FileUtils.mkdir_p directory unless Dir.exists?(directory) - - puts "- Generating datetime stamp" - #Include time to the filename for uniqueness when fetching multiple times a day - date_time = Time.new.strftime("%Y-%m-%dT%H_%M_%S") - - # Writing parsed data to file - puts "- Writing data to file" - File.write("#{directory}/#{date_time}.txt", data) - end - -end \ No newline at end of file diff --git a/scripts/printPackageUpdates.rb b/scripts/printPackageUpdates.rb index 4b57abd95b..a304b95ba0 100644 --- a/scripts/printPackageUpdates.rb +++ b/scripts/printPackageUpdates.rb @@ -1,4 +1,4 @@ -require_relative 'helpers/api_parser' +require_relative "helpers/api_parser" api_parser = ApiParser.new