admin/brew
1
0
Fork 0
Code Issues Packages Projects Releases 6 Wiki Activity

Update Repology parser and bump command

Browse Source
This commit is contained in:
Baffour Adu Boampong 2020-07-01 02:50:21 +00:00
commit 3265650279
11 changed files with 70 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Library/Homebrew/dev-cmd/bump.rb
View File

@ -17,7 +17,8 @@ module Homebrew
def bump
bump_args.parse
puts "command run"
# parse_repology_api()
# puts "command run"
outdated_repology_pacakges = RepologyParser.parse_api_response()
puts RepologyParser.validate__packages(outdated_repology_pacakges)
end
end

9
Library/Homebrew/utils/livecheck.rb
View File

@ -4,12 +4,12 @@ require "open3"
module Livecheck
def livecheck_formula_response(formula_name)
puts "- livecheck formula : #{formula_name}"
ohai "- livecheck formula : #{formula_name}"
command_args = [
"brew",
"livecheck",
formula_name,
"--quiet"
"--quiet",
]
response = Open3.capture2e(*command_args)
@ -17,11 +17,12 @@ module Livecheck
end
def parse_livecheck_response(response)
output = response.first.gsub(' ', '').split(/:|==>|\n/)
output = response.first.delete(" ").split(/:|==>|\n/)
# eg: ["burp", "2.2.18", "2.2.18"]
package_name, brew_version, latest_version = output
{'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version}
{ "name" => package_name, "current_brew_version" => brew_version,
"livecheck_latest_version" => latest_version }
end
end

68
Library/Homebrew/utils/repology.rb
View File

@ -1,47 +1,75 @@
# frozen_string_literal: true
require "net/http"
require "json"
require "utils/curl"
require "formula_info"
module RepologyParser
def call_api(url)
puts "- Calling API #{url}"
uri = URI(url)
response = Net::HTTP.get(uri)
module_function
puts "- Parsing response"
JSON.parse(response)
end
def query_repology_api(last_package_in_response = "")
def query_api(last_package_in_response = "")
url = "https://repology.org/api/v1/projects/#{last_package_in_response}?inrepo=homebrew&outdated=1"
ohai "Calling API #{url}" if Homebrew.args.verbose?
call_api(url)
output, errors, status = curl_output(url.to_s)
output = JSON.parse(output)
end
def parse_repology_api
puts "\n-------- Query outdated packages from Repology --------"
def parse_api_response()
ohai "Querying outdated packages from Repology"
page_no = 1
puts "\n- Paginating repology api page: #{page_no}"
ohai "Paginating repology api page: #{page_no}" if Homebrew.args.verbose?
outdated_packages = query_repology_api("")
outdated_packages = query_api()
last_pacakge_index = outdated_packages.size - 1
response_size = outdated_packages.size
page_limit = 15
while response_size > 1
while response_size > 1 && page_no <= page_limit
page_no += 1
puts "\n- Paginating repology api page: #{page_no}"
ohai "Paginating repology api page: #{page_no}" if Homebrew.args.verbose?
last_package_in_response = outdated_packages.keys[last_pacakge_index]
response = query_repology_api("#{last_package_in_response}/")
response = query_api("#{last_package_in_response}/")
response_size = response.size
outdated_packages.merge!(response)
last_pacakge_index = outdated_packages.size - 1
end
puts "\n- #{outdated_packages.size} outdated packages identified by repology"
ohai "#{outdated_packages.size} outdated packages identified"
outdated_packages
end
def validate__packages(outdated_repology_packages)
ohai "Verifying outdated repology packages as Homebrew Formulae"
packages = {}
outdated_repology_packages.each do |_name, repositories|
# identify homebrew repo
repology_homebrew_repo = repositories.find do |repo|
repo["repo"] == "homebrew"
end
next if repology_homebrew_repo.empty?
latest_version = nil
# identify latest version amongst repology repos
repositories.each do |repo|
latest_version = repo["version"] if repo["status"] == "newest"
end
info = FormulaInfo.lookup(repology_homebrew_repo["srcname"])
next unless info
current_version = info.pkg_version
packages[repology_homebrew_repo["srcname"]] = {
"repology_latest_version" => latest_version,
"current_formula_version" => current_version.to_s
}
puts packages
end
# hash of hashes {"aacgain"=>{"repology_latest_version"=>"1.9", "current_formula_version"=>"1.8"}, ...}
packages
end
end

8
Library/Homebrew/utils/versions.rb
View File

@ -22,15 +22,15 @@ module Versions
parse_formula_bump_response(response)
end
def parse_formula_bump_response(response)
response, status = formula_bump_response
def parse_formula_bump_response(formula_bump_response)
response, _status = formula_bump_response
response
end
def check_for_open_pr(formula_name, download_url)
puts "- Checking for open PRs for formula : #{formula_name}"
ohai "- Checking for open PRs for formula : #{formula_name}"
response = bump_formula_pr(formula_name, download_url)
!response.include? 'Error: These open pull requests may be duplicates'
!response.include? "Error: These open pull requests may be duplicates"
end
end

47
scripts/README.md
View File

@ -1,47 +0,0 @@
# 0.0.1-API-Parser
Parser for fixing this: https://github.com/Homebrew/brew/issues/5725
## Overview
Homebrew is used to install software (packages). Homebrew uses 'formulae' to determine how a package is installed.
This project will automatically check which packages have had newer versions released, whether the package has an open PR on homebrew, and display the results.
## High-level Solution
- Fetch latest package version information from [repology.org](https://repology.org/) and store on file system.
- Fetch Homebrew Formulae information from [HomeBrew Formulae](https://formulae.brew.sh)
- Compare Current Homebrew Formulae version numbers and those coming from Repology's API and Livecheck.
- Determine whether package has open PR.
- Display results.
## Details
- This project can be run automatically at set intervals via GitHub Actions.
- Executing `ruby printPackageUpdates.rb` from the command line will query
both the Repology and Homebrew APIs. Homebrew's current version of each
package will be compared to the latest version of the package, per Repology's response.
- Homebrew's livecheck is also queried for each package, and that data is parsed, if available.
- Checks whether there is open PR for package.
- Each outdated package will be displayed to the console like so:
- Note that some packages will not be included in the Livecheck response. Those will have a 'Livecheck latest:' value of 'Not found'.
```
Package: openclonk
Brew current: 7.0
Repology latest: 8.1
Livecheck latest: 8.1
Has Open PR?: true
Package: openjdk
Brew current: 13.0.2+8
Repology latest: 15.0.0.0~14
Livecheck latest: Not found.
Has Open PR?: false
Package: opentsdb
Brew current: 2.3.1
Repology latest: 2.4.0
Livecheck latest: 2.4.0
Has Open PR?: true
```

8
scripts/bumpFormulae.rb
View File

@ -1,5 +1,5 @@
require_relative 'helpers/parsed_file'
require_relative 'helpers/brew_commands.rb'
require_relative "helpers/parsed_file"
require_relative "helpers/brew_commands.rb"
brew_commands = BrewCommands.new
@ -11,10 +11,10 @@ File.foreach(outdated_pckgs_to_update) do |line|
puts "\n bumping package: #{line_hash['name']} formula"
begin
bump_pr_response, bump_pr_status = brew_commands.bump_formula_pr(line_hash['name'], line_hash['download_url'], line_hash['checksum'])
bump_pr_response, bump_pr_status = brew_commands.bump_formula_pr(line_hash["name"], line_hash["download_url"], line_hash["checksum"])
puts "#{bump_pr_response}"
rescue
puts "- An error occured whilst bumping package #{line_hash['name']} \n"
puts "- An error occured whilst bumping package #{line_hash["name"]} \n"
return
end
end

133
scripts/helpers/api_parser.rb
View File

@ -1,133 +0,0 @@
require 'net/http'
require 'json'
require_relative 'brew_commands'
require_relative 'homebrew_formula'
class ApiParser
def call_api(url)
puts "- Calling API #{url}"
uri = URI(url)
response = Net::HTTP.get(uri)
puts "- Parsing response"
JSON.parse(response)
end
def query_repology_api(last_package_in_response = '')
url = 'https://repology.org/api/v1/projects/' + last_package_in_response + '?inrepo=homebrew&outdated=1'
self.call_api(url)
end
def parse_repology_api()
puts "\n-------- Query outdated packages from Repology --------"
page_no = 1
puts "\n- Paginating repology api page: #{page_no}"
outdated_packages = self.query_repology_api('')
last_pacakge_index = outdated_packages.size - 1
response_size = outdated_packages.size
while response_size > 1 do
page_no += 1
puts "\n- Paginating repology api page: #{page_no}"
last_package_in_response = outdated_packages.keys[last_pacakge_index]
response = self.query_repology_api("#{last_package_in_response}/")
response_size = response.size
outdated_packages.merge!(response)
last_pacakge_index = outdated_packages.size - 1
end
puts "\n- #{outdated_packages.size} outdated pacakges identified by repology"
outdated_packages
end
def query_homebrew
puts "\n-------- Get Homebrew Formulas --------"
self.call_api('https://formulae.brew.sh/api/formula.json')
end
def parse_homebrew_formulas()
formulas = self.query_homebrew()
parsed_homebrew_formulas = {}
formulas.each do |formula|
parsed_homebrew_formulas[formula['name']] = {
"fullname" => formula["full_name"],
"oldname" => formula["oldname"],
"version" => formula["versions"]['stable'],
"download_url" => formula["urls"]['stable']['url'],
}
end
parsed_homebrew_formulas
end
def validate_packages(outdated_repology_packages, brew_formulas)
puts "\n-------- Verify Outdated Repology packages as Homebrew Formulas --------"
packages = {}
outdated_repology_packages.each do |package_name, repo_using_package|
# Identify homebrew repo
repology_homebrew_repo = repo_using_package.select { |repo| repo['repo'] == 'homebrew' }[0]
next if repology_homebrew_repo.empty?
latest_version = nil
# Identify latest version amongst repos
repo_using_package.each do |repo|
latest_version = repo['version'] if repo['status'] == 'newest'
end
repology_homebrew_repo['latest_version'] = latest_version if latest_version
homebrew_package_details = brew_formulas[repology_homebrew_repo['srcname']]
# Format package
packages[repology_homebrew_repo['srcname']] = format_package(homebrew_package_details, repology_homebrew_repo)
end
packages
end
def format_package(homebrew_details, repology_details)
puts "- Formatting package: #{repology_details['srcname']}"
homebrew_formula = HomebrewFormula.new
new_download_url = homebrew_formula.generate_new_download_url(homebrew_details['download_url'], homebrew_details['version'], repology_details['latest_version'])
brew_commands = BrewCommands.new
livecheck_response = brew_commands.livecheck_check_formula(repology_details['srcname'])
has_open_pr = brew_commands.check_for_open_pr(repology_details['srcname'], new_download_url)
formatted_package = {
'fullname'=> homebrew_details['fullname'],
'repology_version' => repology_details['latest_version'],
'homebrew_version' => homebrew_details['version'],
'livecheck_latest_version' => livecheck_response['livecheck_latest_version'],
'current_download_url' => homebrew_details['download_url'],
'latest_download_url' => new_download_url,
'repology_latest_version' => repology_details['latest_version'],
'has_open_pr' => has_open_pr
}
formatted_package
end
def display_version_data(outdated_packages)
puts "==============Formatted outdated packages============\n"
outdated_packages.each do |package_name, package_details|
puts ""
puts "Package: #{package_name}"
puts "Brew current: #{package_details['homebrew_version']}"
puts "Repology latest: #{package_details['repology_version']}"
puts "Livecheck latest: #{package_details['livecheck_latest_version']}"
puts "Has Open PR?: #{package_details['has_open_pr']}"
end
end
end

6
scripts/helpers/brew_commands.rb
View File

@ -16,12 +16,12 @@ class BrewCommands
end
def parse_livecheck_response(livecheck_output)
livecheck_output = livecheck_output.first.gsub(' ', '').split(/:|==>|\n/)
livecheck_output = livecheck_output.first.gsub(" ", "").split(/:|==>|\n/)
# eg: ["burp", "2.2.18", "2.2.18"]
package_name, brew_version, latest_version = livecheck_output
{'name' => package_name, 'current_brew_version' => brew_version, 'livecheck_latest_version' => latest_version}
{"name" => package_name, "current_brew_version" => brew_version, "livecheck_latest_version" => latest_version}
end
def bump_formula_pr(formula_name, url)
@ -48,7 +48,7 @@ class BrewCommands
response = bump_formula_pr(formula_name, download_url)
!response.include? 'Error: These open pull requests may be duplicates'
!response.include? "Error: These open pull requests may be duplicates"
end
end

4
scripts/helpers/homebrew_formula.rb
View File

@ -1,5 +1,5 @@
require 'net/http'
require 'open-uri'
require "net/http"
require "open-uri"
class HomebrewFormula

23
scripts/helpers/parsed_file.rb
View File

@ -1,23 +0,0 @@
require 'fileutils'
class ParsedFile
def get_latest_file(directory)
puts "- retrieving latest file in directory: #{directory}"
Dir.glob("#{directory}/*").max_by(1) {|f| File.mtime(f)}[0]
end
def save_to(directory, data)
# Create directory if does not exist
FileUtils.mkdir_p directory unless Dir.exists?(directory)
puts "- Generating datetime stamp"
#Include time to the filename for uniqueness when fetching multiple times a day
date_time = Time.new.strftime("%Y-%m-%dT%H_%M_%S")
# Writing parsed data to file
puts "- Writing data to file"
File.write("#{directory}/#{date_time}.txt", data)
end
end

2
scripts/printPackageUpdates.rb
View File

@ -1,4 +1,4 @@
require_relative 'helpers/api_parser'
require_relative "helpers/api_parser"
api_parser = ApiParser.new