From 72481c08c3e572aba609f4697deb081d30040bde Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Thu, 25 Jun 2020 15:18:23 -0500 Subject: [PATCH 01/13] compare and display package versions --- scripts/README.md | 47 ++++++++++ scripts/bumpFormulae.rb | 20 +++++ scripts/helpers/api_parser.rb | 133 ++++++++++++++++++++++++++++ scripts/helpers/brew_commands.rb | 55 ++++++++++++ scripts/helpers/homebrew_formula.rb | 27 ++++++ scripts/helpers/parsed_file.rb | 23 +++++ scripts/printPackageUpdates.rb | 10 +++ 7 files changed, 315 insertions(+) create mode 100644 scripts/README.md create mode 100644 scripts/bumpFormulae.rb create mode 100644 scripts/helpers/api_parser.rb create mode 100644 scripts/helpers/brew_commands.rb create mode 100644 scripts/helpers/homebrew_formula.rb create mode 100644 scripts/helpers/parsed_file.rb create mode 100644 scripts/printPackageUpdates.rb diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000..491bbe9657 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,47 @@ +# 0.0.1-API-Parser + +Parser for fixing this: https://github.com/Homebrew/brew/issues/5725 + +## Overview + +Homebrew is used to install software (packages). Homebrew uses 'formulae' to determine how a package is installed. +This project will automatically check which packages have had newer versions released, whether the package has an open PR on homebrew, and display the results. + +## High-level Solution + +- Fetch latest package version information from [repology.org](https://repology.org/) and store on file system. +- Fetch Homebrew Formulae information from [HomeBrew Formulae](https://formulae.brew.sh) +- Compare Current Homebrew Formulae version numbers and those coming from Repology's API and Livecheck. +- Determine whether package has open PR. +- Display results. + +## Details + +- This project can be run automatically at set intervals via GitHub Actions. +- Executing `ruby printPackageUpdates.rb` from the command line will query + both the Repology and Homebrew APIs. Homebrew's current version of each + package will be compared to the latest version of the package, per Repology's response. +- Homebrew's livecheck is also queried for each package, and that data is parsed, if available. +- Checks whether there is open PR for package. +- Each outdated package will be displayed to the console like so: +- Note that some packages will not be included in the Livecheck response. Those will have a 'Livecheck latest:' value of 'Not found'. + +``` +Package: openclonk +Brew current: 7.0 +Repology latest: 8.1 +Livecheck latest: 8.1 +Has Open PR?: true + +Package: openjdk +Brew current: 13.0.2+8 +Repology latest: 15.0.0.0~14 +Livecheck latest: Not found. +Has Open PR?: false + +Package: opentsdb +Brew current: 2.3.1 +Repology latest: 2.4.0 +Livecheck latest: 2.4.0 +Has Open PR?: true +``` diff --git a/scripts/bumpFormulae.rb b/scripts/bumpFormulae.rb new file mode 100644 index 0000000000..5ccf0aeda3 --- /dev/null +++ b/scripts/bumpFormulae.rb @@ -0,0 +1,20 @@ +require_relative 'helpers/parsed_file' +require_relative 'helpers/brew_commands.rb' + +brew_commands = BrewCommands.new + +parsed_file = ParsedFile.new +outdated_pckgs_to_update = parsed_file.get_latest_file("data/outdated_pckgs_to_update") + +File.foreach(outdated_pckgs_to_update) do |line| + line_hash = eval(line) + puts "\n bumping package: #{line_hash['name']} formula" + + begin + bump_pr_response, bump_pr_status = brew_commands.bump_formula_pr(line_hash['name'], line_hash['download_url'], line_hash['checksum']) + puts "#{bump_pr_response}" + rescue + puts "- An error occured whilst bumping package #{line_hash['name']} \n" + return + end +end diff --git a/scripts/helpers/api_parser.rb b/scripts/helpers/api_parser.rb new file mode 100644 index 0000000000..9ab58a97e7 --- /dev/null +++ b/scripts/helpers/api_parser.rb @@ -0,0 +1,133 @@ +require 'net/http' +require 'json' + +require_relative 'brew_commands' +require_relative 'homebrew_formula' + +class ApiParser + def call_api(url) + puts "- Calling API #{url}" + uri = URI(url) + response = Net::HTTP.get(uri) + + puts "- Parsing response" + JSON.parse(response) + end + + def query_repology_api(last_package_in_response = '') + url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' + + self.call_api(url) + end + + def parse_repology_api() + puts "\n-------- Query outdated packages from Repology --------" + page_no = 1 + puts "\n- Paginating repology api page: #{page_no}" + + outdated_packages = self.query_repology_api('') + last_pacakge_index = outdated_packages.size - 1 + response_size = outdated_packages.size + + while response_size > 1 do + page_no += 1 + puts "\n- Paginating repology api page: #{page_no}" + + last_package_in_response = outdated_packages.keys[last_pacakge_index] + response = self.query_repology_api("#{last_package_in_response}/") + + response_size = response.size + outdated_packages.merge!(response) + last_pacakge_index = outdated_packages.size - 1 + end + + puts "\n- #{outdated_packages.size} outdated pacakges identified by repology" + outdated_packages + end + + def query_homebrew + puts "\n-------- Get Homebrew Formulas --------" + self.call_api('https://formulae.brew.sh/api/formula.json') + end + + def parse_homebrew_formulas() + formulas = self.query_homebrew() + parsed_homebrew_formulas = {} + + formulas.each do |formula| + parsed_homebrew_formulas[formula['name']] = { + "fullname" => formula["full_name"], + "oldname" => formula["oldname"], + "version" => formula["versions"]['stable'], + "download_url" => formula["urls"]['stable']['url'], + } + end + + parsed_homebrew_formulas + end + + def validate_packages(outdated_repology_packages, brew_formulas) + puts "\n-------- Verify Outdated Repology packages as Homebrew Formulas --------" + packages = {} + + outdated_repology_packages.each do |package_name, repo_using_package| + # Identify homebrew repo + repology_homebrew_repo = repo_using_package.select { |repo| repo['repo'] == 'homebrew' }[0] + next if repology_homebrew_repo.empty? + + latest_version = nil + + # Identify latest version amongst repos + repo_using_package.each do |repo| + latest_version = repo['version'] if repo['status'] == 'newest' + end + + repology_homebrew_repo['latest_version'] = latest_version if latest_version + homebrew_package_details = brew_formulas[repology_homebrew_repo['srcname']] + + # Format package + packages[repology_homebrew_repo['srcname']] = format_package(homebrew_package_details, repology_homebrew_repo) + end + + packages + end + + + def format_package(homebrew_details, repology_details) + puts "- Formatting package: #{repology_details['srcname']}" + + homebrew_formula = HomebrewFormula.new + new_download_url = homebrew_formula.generate_new_download_url(homebrew_details['download_url'], homebrew_details['version'], repology_details['latest_version']) + + brew_commands = BrewCommands.new + livecheck_response = brew_commands.livecheck_check_formula(repology_details['srcname']) + has_open_pr = brew_commands.check_for_open_pr(repology_details['srcname'], new_download_url) + + formatted_package = { + 'fullname'=> homebrew_details['fullname'], + 'repology_version' => repology_details['latest_version'], + 'homebrew_version' => homebrew_details['version'], + 'livecheck_latest_version' => livecheck_response['livecheck_latest_version'], + 'current_download_url' => homebrew_details['download_url'], + 'latest_download_url' => new_download_url, + 'repology_latest_version' => repology_details['latest_version'], + 'has_open_pr' => has_open_pr + } + + formatted_package + end + + def display_version_data(outdated_packages) + puts "==============Formatted outdated packages============\n" + + outdated_packages.each do |package_name, package_details| + puts "" + puts "Package: #{package_name}" + puts "Brew current: #{package_details['homebrew_version']}" + puts "Repology latest: #{package_details['repology_version']}" + puts "Livecheck latest: #{package_details['livecheck_latest_version']}" + puts "Has Open PR?: #{package_details['has_open_pr']}" + end + end + +end diff --git a/scripts/helpers/brew_commands.rb b/scripts/helpers/brew_commands.rb new file mode 100644 index 0000000000..526162095b --- /dev/null +++ b/scripts/helpers/brew_commands.rb @@ -0,0 +1,55 @@ +require "open3" + +class BrewCommands + + def livecheck_check_formula(formula_name) + puts "- livecheck formula : #{formula_name}" + command_args = [ + "brew", + "livecheck", + formula_name, + "--quiet", + ] + + response = Open3.capture2e(*command_args) + self.parse_livecheck_response(response) + end + + def parse_livecheck_response(livecheck_output) + livecheck_output = livecheck_output.first.gsub(' ', '').split(/:|==>|\n/) + + # eg: ["burp", "2.2.18", "2.2.18"] + package_name, brew_version, latest_version = livecheck_output + + {'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version} + end + + def bump_formula_pr(formula_name, url) + command_args = [ + "brew", + "bump-formula-pr", + "--no-browse", + "--dry-run", + formula_name, + "--url=#{url}", + ] + + response = Open3.capture2e(*command_args) + self.parse_formula_bump_response(response) + end + + def parse_formula_bump_response(formula_bump_response) + response, status = formula_bump_response + response + end + + def check_for_open_pr(formula_name, download_url) + puts "- Checking for open PRs for formula : #{formula_name}" + + response = bump_formula_pr(formula_name, download_url) + + return true if !response.include? 'Error: These open pull requests may be duplicates' + false + end + +end \ No newline at end of file diff --git a/scripts/helpers/homebrew_formula.rb b/scripts/helpers/homebrew_formula.rb new file mode 100644 index 0000000000..d99b7e8bde --- /dev/null +++ b/scripts/helpers/homebrew_formula.rb @@ -0,0 +1,27 @@ +require 'net/http' +require 'open-uri' + +class HomebrewFormula + + def generate_new_download_url(outdated_url, old_version, latest_version) + if [outdated_url, old_version, latest_version].include? nil + puts "\n- Could not generate download url" + nil + else + puts "\n- Generating download url" + outdated_url.gsub(old_version, latest_version) + end + end + + def generate_checksum(new_url) + begin + puts "- Generating checksum for url: #{new_url}" + tempfile = URI.parse(new_url).open + tempfile.close + return Digest::SHA256.file(tempfile.path).hexdigest + rescue + puts "- Failed to generate Checksum \n" + return nil + end + end +end diff --git a/scripts/helpers/parsed_file.rb b/scripts/helpers/parsed_file.rb new file mode 100644 index 0000000000..038a11a38d --- /dev/null +++ b/scripts/helpers/parsed_file.rb @@ -0,0 +1,23 @@ +require 'fileutils' + +class ParsedFile + + def get_latest_file(directory) + puts "- retrieving latest file in directory: #{directory}" + Dir.glob("#{directory}/*").max_by(1) {|f| File.mtime(f)}[0] + end + + def save_to(directory, data) + # Create directory if does not exist + FileUtils.mkdir_p directory unless Dir.exists?(directory) + + puts "- Generating datetime stamp" + #Include time to the filename for uniqueness when fetching multiple times a day + date_time = Time.new.strftime("%Y-%m-%dT%H_%M_%S") + + # Writing parsed data to file + puts "- Writing data to file" + File.write("#{directory}/#{date_time}.txt", data) + end + +end \ No newline at end of file diff --git a/scripts/printPackageUpdates.rb b/scripts/printPackageUpdates.rb new file mode 100644 index 0000000000..4b57abd95b --- /dev/null +++ b/scripts/printPackageUpdates.rb @@ -0,0 +1,10 @@ +require_relative 'helpers/api_parser' + +api_parser = ApiParser.new + +outdated_repology_packages = api_parser.parse_repology_api() +brew_formulas = api_parser.parse_homebrew_formulas() + +formatted_outdated_packages = api_parser.validate_packages(outdated_repology_packages, brew_formulas) + +api_parser.display_version_data(formatted_outdated_packages) From acc3610572a12d9de7d4e879a762bb8775d3ab76 Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Thu, 25 Jun 2020 15:18:23 -0500 Subject: [PATCH 02/13] compare and display package versions --- scripts/README.md | 47 ++++++++++ scripts/bumpFormulae.rb | 20 +++++ scripts/helpers/api_parser.rb | 133 ++++++++++++++++++++++++++++ scripts/helpers/brew_commands.rb | 55 ++++++++++++ scripts/helpers/homebrew_formula.rb | 27 ++++++ scripts/helpers/parsed_file.rb | 23 +++++ scripts/printPackageUpdates.rb | 10 +++ 7 files changed, 315 insertions(+) create mode 100644 scripts/README.md create mode 100644 scripts/bumpFormulae.rb create mode 100644 scripts/helpers/api_parser.rb create mode 100644 scripts/helpers/brew_commands.rb create mode 100644 scripts/helpers/homebrew_formula.rb create mode 100644 scripts/helpers/parsed_file.rb create mode 100644 scripts/printPackageUpdates.rb diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000000..491bbe9657 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,47 @@ +# 0.0.1-API-Parser + +Parser for fixing this: https://github.com/Homebrew/brew/issues/5725 + +## Overview + +Homebrew is used to install software (packages). Homebrew uses 'formulae' to determine how a package is installed. +This project will automatically check which packages have had newer versions released, whether the package has an open PR on homebrew, and display the results. + +## High-level Solution + +- Fetch latest package version information from [repology.org](https://repology.org/) and store on file system. +- Fetch Homebrew Formulae information from [HomeBrew Formulae](https://formulae.brew.sh) +- Compare Current Homebrew Formulae version numbers and those coming from Repology's API and Livecheck. +- Determine whether package has open PR. +- Display results. + +## Details + +- This project can be run automatically at set intervals via GitHub Actions. +- Executing `ruby printPackageUpdates.rb` from the command line will query + both the Repology and Homebrew APIs. Homebrew's current version of each + package will be compared to the latest version of the package, per Repology's response. +- Homebrew's livecheck is also queried for each package, and that data is parsed, if available. +- Checks whether there is open PR for package. +- Each outdated package will be displayed to the console like so: +- Note that some packages will not be included in the Livecheck response. Those will have a 'Livecheck latest:' value of 'Not found'. + +``` +Package: openclonk +Brew current: 7.0 +Repology latest: 8.1 +Livecheck latest: 8.1 +Has Open PR?: true + +Package: openjdk +Brew current: 13.0.2+8 +Repology latest: 15.0.0.0~14 +Livecheck latest: Not found. +Has Open PR?: false + +Package: opentsdb +Brew current: 2.3.1 +Repology latest: 2.4.0 +Livecheck latest: 2.4.0 +Has Open PR?: true +``` diff --git a/scripts/bumpFormulae.rb b/scripts/bumpFormulae.rb new file mode 100644 index 0000000000..5ccf0aeda3 --- /dev/null +++ b/scripts/bumpFormulae.rb @@ -0,0 +1,20 @@ +require_relative 'helpers/parsed_file' +require_relative 'helpers/brew_commands.rb' + +brew_commands = BrewCommands.new + +parsed_file = ParsedFile.new +outdated_pckgs_to_update = parsed_file.get_latest_file("data/outdated_pckgs_to_update") + +File.foreach(outdated_pckgs_to_update) do |line| + line_hash = eval(line) + puts "\n bumping package: #{line_hash['name']} formula" + + begin + bump_pr_response, bump_pr_status = brew_commands.bump_formula_pr(line_hash['name'], line_hash['download_url'], line_hash['checksum']) + puts "#{bump_pr_response}" + rescue + puts "- An error occured whilst bumping package #{line_hash['name']} \n" + return + end +end diff --git a/scripts/helpers/api_parser.rb b/scripts/helpers/api_parser.rb new file mode 100644 index 0000000000..9ab58a97e7 --- /dev/null +++ b/scripts/helpers/api_parser.rb @@ -0,0 +1,133 @@ +require 'net/http' +require 'json' + +require_relative 'brew_commands' +require_relative 'homebrew_formula' + +class ApiParser + def call_api(url) + puts "- Calling API #{url}" + uri = URI(url) + response = Net::HTTP.get(uri) + + puts "- Parsing response" + JSON.parse(response) + end + + def query_repology_api(last_package_in_response = '') + url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' + + self.call_api(url) + end + + def parse_repology_api() + puts "\n-------- Query outdated packages from Repology --------" + page_no = 1 + puts "\n- Paginating repology api page: #{page_no}" + + outdated_packages = self.query_repology_api('') + last_pacakge_index = outdated_packages.size - 1 + response_size = outdated_packages.size + + while response_size > 1 do + page_no += 1 + puts "\n- Paginating repology api page: #{page_no}" + + last_package_in_response = outdated_packages.keys[last_pacakge_index] + response = self.query_repology_api("#{last_package_in_response}/") + + response_size = response.size + outdated_packages.merge!(response) + last_pacakge_index = outdated_packages.size - 1 + end + + puts "\n- #{outdated_packages.size} outdated pacakges identified by repology" + outdated_packages + end + + def query_homebrew + puts "\n-------- Get Homebrew Formulas --------" + self.call_api('https://formulae.brew.sh/api/formula.json') + end + + def parse_homebrew_formulas() + formulas = self.query_homebrew() + parsed_homebrew_formulas = {} + + formulas.each do |formula| + parsed_homebrew_formulas[formula['name']] = { + "fullname" => formula["full_name"], + "oldname" => formula["oldname"], + "version" => formula["versions"]['stable'], + "download_url" => formula["urls"]['stable']['url'], + } + end + + parsed_homebrew_formulas + end + + def validate_packages(outdated_repology_packages, brew_formulas) + puts "\n-------- Verify Outdated Repology packages as Homebrew Formulas --------" + packages = {} + + outdated_repology_packages.each do |package_name, repo_using_package| + # Identify homebrew repo + repology_homebrew_repo = repo_using_package.select { |repo| repo['repo'] == 'homebrew' }[0] + next if repology_homebrew_repo.empty? + + latest_version = nil + + # Identify latest version amongst repos + repo_using_package.each do |repo| + latest_version = repo['version'] if repo['status'] == 'newest' + end + + repology_homebrew_repo['latest_version'] = latest_version if latest_version + homebrew_package_details = brew_formulas[repology_homebrew_repo['srcname']] + + # Format package + packages[repology_homebrew_repo['srcname']] = format_package(homebrew_package_details, repology_homebrew_repo) + end + + packages + end + + + def format_package(homebrew_details, repology_details) + puts "- Formatting package: #{repology_details['srcname']}" + + homebrew_formula = HomebrewFormula.new + new_download_url = homebrew_formula.generate_new_download_url(homebrew_details['download_url'], homebrew_details['version'], repology_details['latest_version']) + + brew_commands = BrewCommands.new + livecheck_response = brew_commands.livecheck_check_formula(repology_details['srcname']) + has_open_pr = brew_commands.check_for_open_pr(repology_details['srcname'], new_download_url) + + formatted_package = { + 'fullname'=> homebrew_details['fullname'], + 'repology_version' => repology_details['latest_version'], + 'homebrew_version' => homebrew_details['version'], + 'livecheck_latest_version' => livecheck_response['livecheck_latest_version'], + 'current_download_url' => homebrew_details['download_url'], + 'latest_download_url' => new_download_url, + 'repology_latest_version' => repology_details['latest_version'], + 'has_open_pr' => has_open_pr + } + + formatted_package + end + + def display_version_data(outdated_packages) + puts "==============Formatted outdated packages============\n" + + outdated_packages.each do |package_name, package_details| + puts "" + puts "Package: #{package_name}" + puts "Brew current: #{package_details['homebrew_version']}" + puts "Repology latest: #{package_details['repology_version']}" + puts "Livecheck latest: #{package_details['livecheck_latest_version']}" + puts "Has Open PR?: #{package_details['has_open_pr']}" + end + end + +end diff --git a/scripts/helpers/brew_commands.rb b/scripts/helpers/brew_commands.rb new file mode 100644 index 0000000000..526162095b --- /dev/null +++ b/scripts/helpers/brew_commands.rb @@ -0,0 +1,55 @@ +require "open3" + +class BrewCommands + + def livecheck_check_formula(formula_name) + puts "- livecheck formula : #{formula_name}" + command_args = [ + "brew", + "livecheck", + formula_name, + "--quiet", + ] + + response = Open3.capture2e(*command_args) + self.parse_livecheck_response(response) + end + + def parse_livecheck_response(livecheck_output) + livecheck_output = livecheck_output.first.gsub(' ', '').split(/:|==>|\n/) + + # eg: ["burp", "2.2.18", "2.2.18"] + package_name, brew_version, latest_version = livecheck_output + + {'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version} + end + + def bump_formula_pr(formula_name, url) + command_args = [ + "brew", + "bump-formula-pr", + "--no-browse", + "--dry-run", + formula_name, + "--url=#{url}", + ] + + response = Open3.capture2e(*command_args) + self.parse_formula_bump_response(response) + end + + def parse_formula_bump_response(formula_bump_response) + response, status = formula_bump_response + response + end + + def check_for_open_pr(formula_name, download_url) + puts "- Checking for open PRs for formula : #{formula_name}" + + response = bump_formula_pr(formula_name, download_url) + + return true if !response.include? 'Error: These open pull requests may be duplicates' + false + end + +end \ No newline at end of file diff --git a/scripts/helpers/homebrew_formula.rb b/scripts/helpers/homebrew_formula.rb new file mode 100644 index 0000000000..d99b7e8bde --- /dev/null +++ b/scripts/helpers/homebrew_formula.rb @@ -0,0 +1,27 @@ +require 'net/http' +require 'open-uri' + +class HomebrewFormula + + def generate_new_download_url(outdated_url, old_version, latest_version) + if [outdated_url, old_version, latest_version].include? nil + puts "\n- Could not generate download url" + nil + else + puts "\n- Generating download url" + outdated_url.gsub(old_version, latest_version) + end + end + + def generate_checksum(new_url) + begin + puts "- Generating checksum for url: #{new_url}" + tempfile = URI.parse(new_url).open + tempfile.close + return Digest::SHA256.file(tempfile.path).hexdigest + rescue + puts "- Failed to generate Checksum \n" + return nil + end + end +end diff --git a/scripts/helpers/parsed_file.rb b/scripts/helpers/parsed_file.rb new file mode 100644 index 0000000000..038a11a38d --- /dev/null +++ b/scripts/helpers/parsed_file.rb @@ -0,0 +1,23 @@ +require 'fileutils' + +class ParsedFile + + def get_latest_file(directory) + puts "- retrieving latest file in directory: #{directory}" + Dir.glob("#{directory}/*").max_by(1) {|f| File.mtime(f)}[0] + end + + def save_to(directory, data) + # Create directory if does not exist + FileUtils.mkdir_p directory unless Dir.exists?(directory) + + puts "- Generating datetime stamp" + #Include time to the filename for uniqueness when fetching multiple times a day + date_time = Time.new.strftime("%Y-%m-%dT%H_%M_%S") + + # Writing parsed data to file + puts "- Writing data to file" + File.write("#{directory}/#{date_time}.txt", data) + end + +end \ No newline at end of file diff --git a/scripts/printPackageUpdates.rb b/scripts/printPackageUpdates.rb new file mode 100644 index 0000000000..4b57abd95b --- /dev/null +++ b/scripts/printPackageUpdates.rb @@ -0,0 +1,10 @@ +require_relative 'helpers/api_parser' + +api_parser = ApiParser.new + +outdated_repology_packages = api_parser.parse_repology_api() +brew_formulas = api_parser.parse_homebrew_formulas() + +formatted_outdated_packages = api_parser.validate_packages(outdated_repology_packages, brew_formulas) + +api_parser.display_version_data(formatted_outdated_packages) From 0cb06acf5ba7e234d3bf6aa3e6d0f88ea16bd8b5 Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:18:36 -0500 Subject: [PATCH 03/13] add repology util file --- Library/Homebrew/utils/repology.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Library/Homebrew/utils/repology.rb diff --git a/Library/Homebrew/utils/repology.rb b/Library/Homebrew/utils/repology.rb new file mode 100644 index 0000000000..e69de29bb2 From 198429ab259ff45fa130a513adf6d6f46d9acc99 Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:21:06 -0500 Subject: [PATCH 04/13] rename repology.rb to update.rb --- Library/Homebrew/utils/repology.rb | 0 Library/Homebrew/utils/update.rb | 6 ++++++ 2 files changed, 6 insertions(+) delete mode 100644 Library/Homebrew/utils/repology.rb create mode 100644 Library/Homebrew/utils/update.rb diff --git a/Library/Homebrew/utils/repology.rb b/Library/Homebrew/utils/repology.rb deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Library/Homebrew/utils/update.rb b/Library/Homebrew/utils/update.rb new file mode 100644 index 0000000000..84defcf33b --- /dev/null +++ b/Library/Homebrew/utils/update.rb @@ -0,0 +1,6 @@ +require 'net/http' +require 'json' + +module Utils + +end From d4bbffdce85bbe164f540897fd2cb3561966351d Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:26:27 -0500 Subject: [PATCH 05/13] place api_parser code into util module --- Library/Homebrew/utils/update.rb | 127 ++++++++++++++++++++++++++++++- 1 file changed, 126 insertions(+), 1 deletion(-) diff --git a/Library/Homebrew/utils/update.rb b/Library/Homebrew/utils/update.rb index 84defcf33b..9f84ee7e58 100644 --- a/Library/Homebrew/utils/update.rb +++ b/Library/Homebrew/utils/update.rb @@ -2,5 +2,130 @@ require 'net/http' require 'json' module Utils - + class ApiParser + def call_api(url) + puts "- Calling API #{url}" + uri = URI(url) + response = Net::HTTP.get(uri) + + puts "- Parsing response" + JSON.parse(response) + end + + def query_repology_api(last_package_in_response = '') + url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' + + self.call_api(url) + end + + def parse_repology_api() + puts "\n-------- Query outdated packages from Repology --------" + page_no = 1 + puts "\n- Paginating repology api page: #{page_no}" + + outdated_packages = self.query_repology_api('') + last_pacakge_index = outdated_packages.size - 1 + response_size = outdated_packages.size + + while response_size > 1 do + page_no += 1 + puts "\n- Paginating repology api page: #{page_no}" + + last_package_in_response = outdated_packages.keys[last_pacakge_index] + response = self.query_repology_api("#{last_package_in_response}/") + + response_size = response.size + outdated_packages.merge!(response) + last_pacakge_index = outdated_packages.size - 1 + end + + puts "\n- #{outdated_packages.size} outdated pacakges identified by repology" + outdated_packages + end + end + + def query_homebrew + puts "\n-------- Get Homebrew Formulas --------" + self.call_api('https://formulae.brew.sh/api/formula.json') + end + + def parse_homebrew_formulas() + formulas = self.query_homebrew() + parsed_homebrew_formulas = {} + + formulas.each do |formula| + parsed_homebrew_formulas[formula['name']] = { + "fullname" => formula["full_name"], + "oldname" => formula["oldname"], + "version" => formula["versions"]['stable'], + "download_url" => formula["urls"]['stable']['url'], + } + end + + parsed_homebrew_formulas + end + + def validate_packages(outdated_repology_packages, brew_formulas) + puts "\n-------- Verify Outdated Repology packages as Homebrew Formulas --------" + packages = {} + + outdated_repology_packages.each do |package_name, repo_using_package| + # Identify homebrew repo + repology_homebrew_repo = repo_using_package.select { |repo| repo['repo'] == 'homebrew' }[0] + next if repology_homebrew_repo.empty? + + latest_version = nil + + # Identify latest version amongst repos + repo_using_package.each do |repo| + latest_version = repo['version'] if repo['status'] == 'newest' + end + + repology_homebrew_repo['latest_version'] = latest_version if latest_version + homebrew_package_details = brew_formulas[repology_homebrew_repo['srcname']] + + # Format package + packages[repology_homebrew_repo['srcname']] = format_package(homebrew_package_details, repology_homebrew_repo) + end + + packages + end + + + def format_package(homebrew_details, repology_details) + puts "- Formatting package: #{repology_details['srcname']}" + + homebrew_formula = HomebrewFormula.new + new_download_url = homebrew_formula.generate_new_download_url(homebrew_details['download_url'], homebrew_details['version'], repology_details['latest_version']) + + brew_commands = BrewCommands.new + livecheck_response = brew_commands.livecheck_check_formula(repology_details['srcname']) + has_open_pr = brew_commands.check_for_open_pr(repology_details['srcname'], new_download_url) + + formatted_package = { + 'fullname'=> homebrew_details['fullname'], + 'repology_version' => repology_details['latest_version'], + 'homebrew_version' => homebrew_details['version'], + 'livecheck_latest_version' => livecheck_response['livecheck_latest_version'], + 'current_download_url' => homebrew_details['download_url'], + 'latest_download_url' => new_download_url, + 'repology_latest_version' => repology_details['latest_version'], + 'has_open_pr' => has_open_pr + } + + formatted_package + end + + def display_version_data(outdated_packages) + puts "==============Formatted outdated packages============\n" + + outdated_packages.each do |package_name, package_details| + puts "" + puts "Package: #{package_name}" + puts "Brew current: #{package_details['homebrew_version']}" + puts "Repology latest: #{package_details['repology_version']}" + puts "Livecheck latest: #{package_details['livecheck_latest_version']}" + puts "Has Open PR?: #{package_details['has_open_pr']}" + end + end end From 9382b1273a9522ccf51acf4c1880ddfa261d1d87 Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:27:42 -0500 Subject: [PATCH 06/13] update display text --- Library/Homebrew/utils/update.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Library/Homebrew/utils/update.rb b/Library/Homebrew/utils/update.rb index 9f84ee7e58..87b363bb08 100644 --- a/Library/Homebrew/utils/update.rb +++ b/Library/Homebrew/utils/update.rb @@ -121,11 +121,11 @@ module Utils outdated_packages.each do |package_name, package_details| puts "" - puts "Package: #{package_name}" - puts "Brew current: #{package_details['homebrew_version']}" - puts "Repology latest: #{package_details['repology_version']}" - puts "Livecheck latest: #{package_details['livecheck_latest_version']}" - puts "Has Open PR?: #{package_details['has_open_pr']}" + puts "Formula: #{package_name}" + puts "Current formula version: #{package_details['homebrew_version']}" + puts "Repology latest version: #{package_details['repology_version']}" + puts "Livecheck latest version: #{package_details['livecheck_latest_version']}" + puts "Open pull request: #{package_details['has_open_pr']}" end end end From 2f419f5c55114122f290a692490cebd74d877026 Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:33:24 -0500 Subject: [PATCH 07/13] remove unnecessary explicit return val --- Library/Homebrew/utils/update.rb | 3 +-- scripts/helpers/brew_commands.rb | 13 ++++++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Library/Homebrew/utils/update.rb b/Library/Homebrew/utils/update.rb index 87b363bb08..48a18daa10 100644 --- a/Library/Homebrew/utils/update.rb +++ b/Library/Homebrew/utils/update.rb @@ -120,8 +120,7 @@ module Utils puts "==============Formatted outdated packages============\n" outdated_packages.each do |package_name, package_details| - puts "" - puts "Formula: #{package_name}" + puts "\nFormula: #{package_name}" puts "Current formula version: #{package_details['homebrew_version']}" puts "Repology latest version: #{package_details['repology_version']}" puts "Livecheck latest version: #{package_details['livecheck_latest_version']}" diff --git a/scripts/helpers/brew_commands.rb b/scripts/helpers/brew_commands.rb index 526162095b..380e64da48 100644 --- a/scripts/helpers/brew_commands.rb +++ b/scripts/helpers/brew_commands.rb @@ -1,7 +1,7 @@ require "open3" class BrewCommands - + def livecheck_check_formula(formula_name) puts "- livecheck formula : #{formula_name}" command_args = [ @@ -20,7 +20,7 @@ class BrewCommands # eg: ["burp", "2.2.18", "2.2.18"] package_name, brew_version, latest_version = livecheck_output - + {'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version} end @@ -40,7 +40,7 @@ class BrewCommands def parse_formula_bump_response(formula_bump_response) response, status = formula_bump_response - response + response end def check_for_open_pr(formula_name, download_url) @@ -48,8 +48,7 @@ class BrewCommands response = bump_formula_pr(formula_name, download_url) - return true if !response.include? 'Error: These open pull requests may be duplicates' - false - end + !response.include? 'Error: These open pull requests may be duplicates' + end -end \ No newline at end of file +end From f060009424f5f7ab9dfcd7da97d73ff1f5a8f5a8 Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:37:50 -0500 Subject: [PATCH 08/13] initialize bump.rb file --- Library/Homebrew/dev-cmd/bump.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 Library/Homebrew/dev-cmd/bump.rb diff --git a/Library/Homebrew/dev-cmd/bump.rb b/Library/Homebrew/dev-cmd/bump.rb new file mode 100644 index 0000000000..e69de29bb2 From d67aea7f431e887b857321ac9c71c53525dcac7d Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:47:19 -0500 Subject: [PATCH 09/13] skeleton code bump dev-cmd file --- Library/Homebrew/dev-cmd/bump.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Library/Homebrew/dev-cmd/bump.rb b/Library/Homebrew/dev-cmd/bump.rb index e69de29bb2..89f9ede7b3 100644 --- a/Library/Homebrew/dev-cmd/bump.rb +++ b/Library/Homebrew/dev-cmd/bump.rb @@ -0,0 +1,19 @@ +require "cli/parser" + +module Homebrew + module_function + + def bump_args + Homebrew::CLI::Parser.new do + usage_banner <<~EOS + `bump` + + Display out-of-date brew formulae, the latest version available, and whether a pull request has been opened. + EOS + end + end + + def bump + bump_args.parse + end +end From 56cf5bafdbc06a4cd43f7b5b1596542571bf3ab9 Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:51:58 -0500 Subject: [PATCH 10/13] focus util module on repology API call --- Library/Homebrew/utils/repology.rb | 46 ++++++++++ Library/Homebrew/utils/update.rb | 130 ----------------------------- 2 files changed, 46 insertions(+), 130 deletions(-) create mode 100644 Library/Homebrew/utils/repology.rb delete mode 100644 Library/Homebrew/utils/update.rb diff --git a/Library/Homebrew/utils/repology.rb b/Library/Homebrew/utils/repology.rb new file mode 100644 index 0000000000..822dfac51a --- /dev/null +++ b/Library/Homebrew/utils/repology.rb @@ -0,0 +1,46 @@ +require 'net/http' +require 'json' + +module Utils + module RepologyParser + def call_api(url) + puts "- Calling API #{url}" + uri = URI(url) + response = Net::HTTP.get(uri) + + puts "- Parsing response" + JSON.parse(response) + end + + def query_repology_api(last_package_in_response = '') + url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' + + self.call_api(url) + end + + def parse_repology_api() + puts "\n-------- Query outdated packages from Repology --------" + page_no = 1 + puts "\n- Paginating repology api page: #{page_no}" + + outdated_packages = self.query_repology_api('') + last_pacakge_index = outdated_packages.size - 1 + response_size = outdated_packages.size + + while response_size > 1 do + page_no += 1 + puts "\n- Paginating repology api page: #{page_no}" + + last_package_in_response = outdated_packages.keys[last_pacakge_index] + response = self.query_repology_api("#{last_package_in_response}/") + + response_size = response.size + outdated_packages.merge!(response) + last_pacakge_index = outdated_packages.size - 1 + end + + puts "\n- #{outdated_packages.size} outdated pacakges identified by repology" + outdated_packages + end + end +end diff --git a/Library/Homebrew/utils/update.rb b/Library/Homebrew/utils/update.rb deleted file mode 100644 index 48a18daa10..0000000000 --- a/Library/Homebrew/utils/update.rb +++ /dev/null @@ -1,130 +0,0 @@ -require 'net/http' -require 'json' - -module Utils - class ApiParser - def call_api(url) - puts "- Calling API #{url}" - uri = URI(url) - response = Net::HTTP.get(uri) - - puts "- Parsing response" - JSON.parse(response) - end - - def query_repology_api(last_package_in_response = '') - url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' - - self.call_api(url) - end - - def parse_repology_api() - puts "\n-------- Query outdated packages from Repology --------" - page_no = 1 - puts "\n- Paginating repology api page: #{page_no}" - - outdated_packages = self.query_repology_api('') - last_pacakge_index = outdated_packages.size - 1 - response_size = outdated_packages.size - - while response_size > 1 do - page_no += 1 - puts "\n- Paginating repology api page: #{page_no}" - - last_package_in_response = outdated_packages.keys[last_pacakge_index] - response = self.query_repology_api("#{last_package_in_response}/") - - response_size = response.size - outdated_packages.merge!(response) - last_pacakge_index = outdated_packages.size - 1 - end - - puts "\n- #{outdated_packages.size} outdated pacakges identified by repology" - outdated_packages - end - end - - def query_homebrew - puts "\n-------- Get Homebrew Formulas --------" - self.call_api('https://formulae.brew.sh/api/formula.json') - end - - def parse_homebrew_formulas() - formulas = self.query_homebrew() - parsed_homebrew_formulas = {} - - formulas.each do |formula| - parsed_homebrew_formulas[formula['name']] = { - "fullname" => formula["full_name"], - "oldname" => formula["oldname"], - "version" => formula["versions"]['stable'], - "download_url" => formula["urls"]['stable']['url'], - } - end - - parsed_homebrew_formulas - end - - def validate_packages(outdated_repology_packages, brew_formulas) - puts "\n-------- Verify Outdated Repology packages as Homebrew Formulas --------" - packages = {} - - outdated_repology_packages.each do |package_name, repo_using_package| - # Identify homebrew repo - repology_homebrew_repo = repo_using_package.select { |repo| repo['repo'] == 'homebrew' }[0] - next if repology_homebrew_repo.empty? - - latest_version = nil - - # Identify latest version amongst repos - repo_using_package.each do |repo| - latest_version = repo['version'] if repo['status'] == 'newest' - end - - repology_homebrew_repo['latest_version'] = latest_version if latest_version - homebrew_package_details = brew_formulas[repology_homebrew_repo['srcname']] - - # Format package - packages[repology_homebrew_repo['srcname']] = format_package(homebrew_package_details, repology_homebrew_repo) - end - - packages - end - - - def format_package(homebrew_details, repology_details) - puts "- Formatting package: #{repology_details['srcname']}" - - homebrew_formula = HomebrewFormula.new - new_download_url = homebrew_formula.generate_new_download_url(homebrew_details['download_url'], homebrew_details['version'], repology_details['latest_version']) - - brew_commands = BrewCommands.new - livecheck_response = brew_commands.livecheck_check_formula(repology_details['srcname']) - has_open_pr = brew_commands.check_for_open_pr(repology_details['srcname'], new_download_url) - - formatted_package = { - 'fullname'=> homebrew_details['fullname'], - 'repology_version' => repology_details['latest_version'], - 'homebrew_version' => homebrew_details['version'], - 'livecheck_latest_version' => livecheck_response['livecheck_latest_version'], - 'current_download_url' => homebrew_details['download_url'], - 'latest_download_url' => new_download_url, - 'repology_latest_version' => repology_details['latest_version'], - 'has_open_pr' => has_open_pr - } - - formatted_package - end - - def display_version_data(outdated_packages) - puts "==============Formatted outdated packages============\n" - - outdated_packages.each do |package_name, package_details| - puts "\nFormula: #{package_name}" - puts "Current formula version: #{package_details['homebrew_version']}" - puts "Repology latest version: #{package_details['repology_version']}" - puts "Livecheck latest version: #{package_details['livecheck_latest_version']}" - puts "Open pull request: #{package_details['has_open_pr']}" - end - end -end From 05f9d640bc65a415a211d2a5fbbca60aaff41cce Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:57:21 -0500 Subject: [PATCH 11/13] typo --- Library/Homebrew/utils/repology.rb | 65 +++++++++++++++--------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/Library/Homebrew/utils/repology.rb b/Library/Homebrew/utils/repology.rb index 822dfac51a..0ee4fabc01 100644 --- a/Library/Homebrew/utils/repology.rb +++ b/Library/Homebrew/utils/repology.rb @@ -1,46 +1,45 @@ require 'net/http' require 'json' -module Utils - module RepologyParser - def call_api(url) - puts "- Calling API #{url}" - uri = URI(url) - response = Net::HTTP.get(uri) +module RepologyParser + def call_api(url) + puts "- Calling API #{url}" + uri = URI(url) + response = Net::HTTP.get(uri) - puts "- Parsing response" - JSON.parse(response) - end + puts "- Parsing response" + JSON.parse(response) + end - def query_repology_api(last_package_in_response = '') - url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' + def query_repology_api(last_package_in_response = '') + url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' - self.call_api(url) - end + call_api(url) + end - def parse_repology_api() - puts "\n-------- Query outdated packages from Repology --------" - page_no = 1 + def parse_repology_api() + puts "\n-------- Query outdated packages from Repology --------" + page_no = 1 + puts "\n- Paginating repology api page: #{page_no}" + + outdated_packages = query_repology_api('') + last_pacakge_index = outdated_packages.size - 1 + response_size = outdated_packages.size + + while response_size > 1 do + page_no += 1 puts "\n- Paginating repology api page: #{page_no}" - outdated_packages = self.query_repology_api('') + last_package_in_response = outdated_packages.keys[last_pacakge_index] + response = query_repology_api("#{last_package_in_response}/") + + response_size = response.size + outdated_packages.merge!(response) last_pacakge_index = outdated_packages.size - 1 - response_size = outdated_packages.size - - while response_size > 1 do - page_no += 1 - puts "\n- Paginating repology api page: #{page_no}" - - last_package_in_response = outdated_packages.keys[last_pacakge_index] - response = self.query_repology_api("#{last_package_in_response}/") - - response_size = response.size - outdated_packages.merge!(response) - last_pacakge_index = outdated_packages.size - 1 - end - - puts "\n- #{outdated_packages.size} outdated pacakges identified by repology" - outdated_packages end + + puts "\n- #{outdated_packages.size} outdated packages identified by repology" + + outdated_packages end end From fa2199f5a89bbc5c59bc41caa9915562b765c280 Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 09:57:54 -0500 Subject: [PATCH 12/13] remove extraneous file --- scripts/helpers/parsed_file.rb | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 scripts/helpers/parsed_file.rb diff --git a/scripts/helpers/parsed_file.rb b/scripts/helpers/parsed_file.rb deleted file mode 100644 index 038a11a38d..0000000000 --- a/scripts/helpers/parsed_file.rb +++ /dev/null @@ -1,23 +0,0 @@ -require 'fileutils' - -class ParsedFile - - def get_latest_file(directory) - puts "- retrieving latest file in directory: #{directory}" - Dir.glob("#{directory}/*").max_by(1) {|f| File.mtime(f)}[0] - end - - def save_to(directory, data) - # Create directory if does not exist - FileUtils.mkdir_p directory unless Dir.exists?(directory) - - puts "- Generating datetime stamp" - #Include time to the filename for uniqueness when fetching multiple times a day - date_time = Time.new.strftime("%Y-%m-%dT%H_%M_%S") - - # Writing parsed data to file - puts "- Writing data to file" - File.write("#{directory}/#{date_time}.txt", data) - end - -end \ No newline at end of file From 14d63f61a5d3c5ca41d89c670a4bd58cab84aa6c Mon Sep 17 00:00:00 2001 From: Elizabeth Tackett Date: Mon, 29 Jun 2020 10:16:58 -0500 Subject: [PATCH 13/13] fix brew style violations --- Library/Homebrew/dev-cmd/bump.rb | 2 ++ Library/Homebrew/utils/repology.rb | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Library/Homebrew/dev-cmd/bump.rb b/Library/Homebrew/dev-cmd/bump.rb index 89f9ede7b3..7a5c6b427e 100644 --- a/Library/Homebrew/dev-cmd/bump.rb +++ b/Library/Homebrew/dev-cmd/bump.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "cli/parser" module Homebrew diff --git a/Library/Homebrew/utils/repology.rb b/Library/Homebrew/utils/repology.rb index 0ee4fabc01..80973956b1 100644 --- a/Library/Homebrew/utils/repology.rb +++ b/Library/Homebrew/utils/repology.rb @@ -1,5 +1,7 @@ -require 'net/http' -require 'json' +# frozen_string_literal: true + +require "net/http" +require "json" module RepologyParser def call_api(url) @@ -11,22 +13,22 @@ module RepologyParser JSON.parse(response) end - def query_repology_api(last_package_in_response = '') - url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1' + def query_repology_api(last_package_in_response = "") + url = "https://repology.org/api/v1/projects/#{last_package_in_response}?inrepo=homebrew&outdated=1" call_api(url) end - def parse_repology_api() + def parse_repology_api puts "\n-------- Query outdated packages from Repology --------" page_no = 1 puts "\n- Paginating repology api page: #{page_no}" - outdated_packages = query_repology_api('') + outdated_packages = query_repology_api("") last_pacakge_index = outdated_packages.size - 1 response_size = outdated_packages.size - while response_size > 1 do + while response_size > 1 page_no += 1 puts "\n- Paginating repology api page: #{page_no}"