admin/brew
1
0
Fork 0
Code Issues Packages Projects Releases 6 Wiki Activity

Use JSON files for bottle upload data. (#166)

Browse Source 
This means that we do not need to read formulae or evaluate Ruby at
upload time.
This commit is contained in:
Mike McQuaid 2016-05-28 15:54:05 +01:00
parent 9cf2710dc9
commit 6ba466f5d8
4 changed files with 163 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
Library/Homebrew/cmd/bottle.rb
View File

@ -281,7 +281,8 @@ module Homebrew
bottle.prefix prefix bottle.prefix prefix
end end
bottle.revision bottle_revision bottle.revision bottle_revision
bottle.sha256 bottle_path.sha256 => Utils::Bottles.tag sha256 = bottle_path.sha256
bottle.sha256 sha256 => Utils::Bottles.tag
old_spec = f.bottle_specification old_spec = f.bottle_specification
if ARGV.include?("--keep-old") && !old_spec.checksums.empty? if ARGV.include?("--keep-old") && !old_spec.checksums.empty?
@ -300,74 +301,104 @@ module Homebrew
puts "./#{filename}" puts "./#{filename}"
puts output puts output
if ARGV.include? "--rb" if ARGV.include? "--json"
File.open("#{filename.prefix}.bottle.rb", "w") do |file| json = {
file.write("\# #{f.full_name}\n") f.full_name => {
file.write(output) "formula" => {
"pkg_version" => f.pkg_version.to_s,
"path" => f.path.to_s.strip_prefix("#{HOMEBREW_REPOSITORY}/"),
},
"bottle" => {
"root_url" => bottle.root_url,
"prefix" => bottle.prefix,
"cellar" => bottle.cellar.to_s,
"revision" => bottle.revision,
"tags" => {
Utils::Bottles.tag.to_s => {
"filename" => filename.to_s,
"sha256" => sha256,
},
}
},
"bintray" => {
"package" => Utils::Bottles::Bintray.package(f.full_name),
"repository" => Utils::Bottles::Bintray.repository(f.tap),
},
},
}
File.open("#{filename.prefix}.bottle.json", "w") do |file|
file.write Utils::JSON.dump json
end end
end end
end end
module BottleMerger
def bottle(&block)
instance_eval(&block)
end
end
def merge def merge
write = ARGV.include? "--write" write = ARGV.include? "--write"
keep_old = ARGV.include? "--keep-old"
merge_hash = {} bottles_hash = ARGV.named.reduce({}) do |hash, json_file|
ARGV.named.each do |argument| hash.merge! Utils::JSON.load(IO.read(json_file))
bottle_block = IO.read(argument)
formula_name = bottle_block.lines.first.sub(/^# /, "").chomp
merge_hash[formula_name] ||= []
merge_hash[formula_name] << bottle_block
end end
merge_hash.each do |formula_name, bottle_blocks| bottles_hash.each do |formula_name, bottle_hash|
ohai formula_name ohai formula_name
f = Formulary.factory(formula_name)
if f.bottle_disabled? bottle = BottleSpecification.new
ofail "Formula #{f.full_name} has disabled bottle" bottle.root_url bottle_hash["bottle"]["root_url"]
puts f.bottle_disable_reason cellar = bottle_hash["bottle"]["cellar"]
next if cellar == "any" || cellar == "any_skip_relocation"
cellar = cellar.to_sym
end end
bottle.cellar cellar
bottle = if keep_old bottle.prefix bottle_hash["bottle"]["prefix"]
f.bottle_specification.dup bottle.revision bottle_hash["bottle"]["revision"]
else bottle_hash["bottle"]["tags"].each do |tag, tag_hash|
BottleSpecification.new bottle.sha256 tag_hash["sha256"] => tag.to_sym
end
bottle.extend(BottleMerger)
bottle_blocks.each { |block| bottle.instance_eval(block) }
old_spec = f.bottle_specification
if keep_old && !old_spec.checksums.empty?
bad_fields = [:root_url, :prefix, :cellar, :revision].select do |field|
old_spec.send(field) != bottle.send(field)
end
bad_fields.delete(:cellar) if old_spec.cellar == :any && bottle.cellar == :any_skip_relocation
if bad_fields.any?
ofail "--keep-old is passed but there are changes in: #{bad_fields.join ", "}"
next
end
end end
output = bottle_output bottle output = bottle_output bottle
puts output
if write if write
path = Pathname.new("#{HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"]}")
update_or_add = nil update_or_add = nil
Utils::Inreplace.inreplace(f.path) do |s| Utils::Inreplace.inreplace(path) do |s|
if s.include? "bottle do" if s.include? "bottle do"
update_or_add = "update" update_or_add = "update"
if ARGV.include? "--keep-old"
mismatches = []
s =~ / bottle do(.+?)end\n/m
bottle_block_contents = $1
bottle_block_contents.lines.each do |line|
line = line.strip
next if line.empty?
key, value, _, tag = line.split " ", 4
value = value.to_s.delete ":'\""
tag = tag.to_s.delete ":"
if !tag.empty?
if !bottle_hash["bottle"]["tags"][tag].to_s.empty?
mismatches << "#{key} => #{tag}"
else
bottle.send(key, value => tag.to_sym)
end
next
end
old_value = bottle_hash["bottle"][key].to_s
next if key == "cellar" && old_value == "any" && value == "any_skip_relocation"
mismatches << key if old_value.empty? || value != old_value
end
if mismatches.any?
odie "--keep-old was passed but there were changes in #{mismatches.join(", ")}!"
end
output = bottle_output bottle
end
puts output
string = s.sub!(/ bottle do.+?end\n/m, output) string = s.sub!(/ bottle do.+?end\n/m, output)
odie "Bottle block update failed!" unless string odie "Bottle block update failed!" unless string
else else
odie "--keep-old was passed but there was no existing bottle block!"
puts output
update_or_add = "add" update_or_add = "add"
if s.include? "stable do" if s.include? "stable do"
indent = s.slice(/^ +stable do/).length - "stable do".length indent = s.slice(/^ +stable do/).length - "stable do".length
@ -392,12 +423,16 @@ module Homebrew
end end
unless ARGV.include? "--no-commit" unless ARGV.include? "--no-commit"
f.path.parent.cd do pkg_version = bottle_hash["formula"]["pkg_version"]
path.parent.cd do
safe_system "git", "commit", "--no-edit", "--verbose", safe_system "git", "commit", "--no-edit", "--verbose",
"--message=#{f.name}: #{update_or_add} #{f.pkg_version} bottle.", "--message=#{formula_name}: #{update_or_add} #{pkg_version} bottle.",
"--", f.path "--", path
end end
end end
else
puts output
end end
end end
end end

35
Library/Homebrew/cmd/pull.rb
View File

@ -119,7 +119,7 @@ module Homebrew
end end
patch_puller.apply_patch patch_puller.apply_patch
changed_formulae = [] changed_formulae_names = []
if tap if tap
Utils.popen_read( Utils.popen_read(
@ -128,7 +128,7 @@ module Homebrew
).each_line do |line| ).each_line do |line|
name = "#{tap.name}/#{File.basename(line.chomp, ".rb")}" name = "#{tap.name}/#{File.basename(line.chomp, ".rb")}"
begin begin
changed_formulae << Formula[name] changed_formulae_names << name
# Make sure we catch syntax errors. # Make sure we catch syntax errors.
rescue Exception rescue Exception
next next
@ -137,7 +137,10 @@ module Homebrew
end end
fetch_bottles = false fetch_bottles = false
changed_formulae.each do |f| changed_formulae_names.each do |name|
next if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
f = Formula[name]
if ARGV.include? "--bottle" if ARGV.include? "--bottle"
if f.bottle_unneeded? if f.bottle_unneeded?
ohai "#{f}: skipping unneeded bottle." ohai "#{f}: skipping unneeded bottle."
@ -164,12 +167,16 @@ module Homebrew
message += "\n#{close_message}" unless message.include? close_message message += "\n#{close_message}" unless message.include? close_message
end end
if changed_formulae.empty? if changed_formulae_names.empty?
odie "cannot bump: no changed formulae found after applying patch" if do_bump odie "cannot bump: no changed formulae found after applying patch" if do_bump
is_bumpable = false is_bumpable = false
end end
if is_bumpable && !ARGV.include?("--clean")
formula = changed_formulae.first is_bumpable = false if ARGV.include?("--clean")
is_bumpable = false if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
if is_bumpable
formula = Formula[changed_formulae_names.first]
new_versions = current_versions_from_info_external(patch_changes[:formulae].first) new_versions = current_versions_from_info_external(patch_changes[:formulae].first)
orig_subject = message.empty? ? "" : message.lines.first.chomp orig_subject = message.empty? ? "" : message.lines.first.chomp
bump_subject = subject_for_bump(formula, old_versions, new_versions) bump_subject = subject_for_bump(formula, old_versions, new_versions)
@ -219,7 +226,7 @@ module Homebrew
# Publish bottles on Bintray # Publish bottles on Bintray
unless ARGV.include? "--no-publish" unless ARGV.include? "--no-publish"
published = publish_changed_formula_bottles(tap, changed_formulae) published = publish_changed_formula_bottles(tap, changed_formulae_names)
bintray_published_formulae.concat(published) bintray_published_formulae.concat(published)
end end
end end
@ -239,11 +246,16 @@ module Homebrew
private private
def publish_changed_formula_bottles(tap, changed_formulae) def publish_changed_formula_bottles(tap, changed_formulae_names)
if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
raise "Need to load formulae to publish them!"
end
published = [] published = []
bintray_creds = { :user => ENV["BINTRAY_USER"], :key => ENV["BINTRAY_KEY"] } bintray_creds = { :user => ENV["BINTRAY_USER"], :key => ENV["BINTRAY_KEY"] }
if bintray_creds[:user] && bintray_creds[:key] if bintray_creds[:user] && bintray_creds[:key]
changed_formulae.each do |f| changed_formulae_names.each do |name|
f = Formula[name]
next if f.bottle_unneeded? || f.bottle_disabled? next if f.bottle_unneeded? || f.bottle_disabled?
ohai "Publishing on Bintray: #{f.name} #{f.pkg_version}" ohai "Publishing on Bintray: #{f.name} #{f.pkg_version}"
publish_bottle_file_on_bintray(f, bintray_creds) publish_bottle_file_on_bintray(f, bintray_creds)
@ -493,6 +505,11 @@ module Homebrew
# version of a formula. # version of a formula.
def verify_bintray_published(formulae_names) def verify_bintray_published(formulae_names)
return if formulae_names.empty? return if formulae_names.empty?
if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
raise "Need to load formulae to verify their publication!"
end
ohai "Verifying bottles published on Bintray" ohai "Verifying bottles published on Bintray"
formulae = formulae_names.map { |n| Formula[n] } formulae = formulae_names.map { |n| Formula[n] }
max_retries = 300 # shared among all bottles max_retries = 300 # shared among all bottles

93
Library/Homebrew/cmd/test-bot.rb
View File

@ -114,7 +114,7 @@ module Homebrew
end end
def command_short def command_short
(@command - %w[brew --force --retry --verbose --build-bottle --rb]).join(" ") (@command - %w[brew --force --retry --verbose --build-bottle --json]).join(" ")
end end
def passed? def passed?
@ -570,15 +570,15 @@ module Homebrew
test "brew", "audit", *audit_args test "brew", "audit", *audit_args
if install_passed if install_passed
if formula.stable? && !ARGV.include?("--fast") && !ARGV.include?("--no-bottle") && !formula.bottle_disabled? if formula.stable? && !ARGV.include?("--fast") && !ARGV.include?("--no-bottle") && !formula.bottle_disabled?
bottle_args = ["--verbose", "--rb", formula_name] bottle_args = ["--verbose", "--json", formula_name]
bottle_args << "--keep-old" if ARGV.include? "--keep-old" bottle_args << "--keep-old" if ARGV.include? "--keep-old"
test "brew", "bottle", *bottle_args test "brew", "bottle", *bottle_args
bottle_step = steps.last bottle_step = steps.last
if bottle_step.passed? && bottle_step.has_output? if bottle_step.passed? && bottle_step.has_output?
bottle_filename = bottle_filename =
bottle_step.output.gsub(/.*(\.\/\S+#{Utils::Bottles::native_regex}).*/m, '\1') bottle_step.output.gsub(/.*(\.\/\S+#{Utils::Bottles::native_regex}).*/m, '\1')
bottle_rb_filename = bottle_filename.gsub(/\.(\d+\.)?tar\.gz$/, ".rb") bottle_json_filename = bottle_filename.gsub(/\.(\d+\.)?tar\.gz$/, ".json")
bottle_merge_args = ["--merge", "--write", "--no-commit", bottle_rb_filename] bottle_merge_args = ["--merge", "--write", "--no-commit", bottle_json_filename]
bottle_merge_args << "--keep-old" if ARGV.include? "--keep-old" bottle_merge_args << "--keep-old" if ARGV.include? "--keep-old"
test "brew", "bottle", *bottle_merge_args test "brew", "bottle", *bottle_merge_args
test "brew", "uninstall", "--force", formula_name test "brew", "uninstall", "--force", formula_name
@ -768,6 +768,9 @@ module Homebrew
end end
def test_ci_upload(tap) def test_ci_upload(tap)
# Don't trust formulae we're uploading
ENV["HOMEBREW_DISABLE_LOAD_FORMULA"] = "1"
jenkins = ENV["JENKINS_HOME"] jenkins = ENV["JENKINS_HOME"]
job = ENV["UPSTREAM_JOB_NAME"] job = ENV["UPSTREAM_JOB_NAME"]
id = ENV["UPSTREAM_BUILD_ID"] id = ENV["UPSTREAM_BUILD_ID"]
@ -779,14 +782,13 @@ module Homebrew
raise "Missing BINTRAY_USER or BINTRAY_KEY variables!" raise "Missing BINTRAY_USER or BINTRAY_KEY variables!"
end end
# Don't pass keys/cookies to subprocesses.. # Don't pass keys/cookies to subprocesses
ENV["BINTRAY_KEY"] = nil ENV["BINTRAY_KEY"] = nil
ENV["HUDSON_SERVER_COOKIE"] = nil ENV["HUDSON_SERVER_COOKIE"] = nil
ENV["JENKINS_SERVER_COOKIE"] = nil ENV["JENKINS_SERVER_COOKIE"] = nil
ENV["HUDSON_COOKIE"] = nil ENV["HUDSON_COOKIE"] = nil
ARGV << "--verbose" ARGV << "--verbose"
ARGV << "--keep-old" if ENV["UPSTREAM_BOTTLE_KEEP_OLD"]
ARGV << "--legacy" if ENV["UPSTREAM_BOTTLE_LEGACY"] ARGV << "--legacy" if ENV["UPSTREAM_BOTTLE_LEGACY"]
bottles = Dir["#{jenkins}/jobs/#{job}/configurations/axis-version/*/builds/#{id}/archive/*.bottle*.*"] bottles = Dir["#{jenkins}/jobs/#{job}/configurations/axis-version/*/builds/#{id}/archive/*.bottle*.*"]
@ -817,53 +819,58 @@ module Homebrew
end end
end end
bottle_args = ["--merge", "--write", *Dir["*.bottle.rb"]] json_files = Dir.glob("*.bottle.json")
bottle_args << "--keep-old" if ARGV.include? "--keep-old" system "brew", "bottle", "--merge", "--write", *json_files
system "brew", "bottle", *bottle_args
remote = "git@github.com:BrewTestBot/homebrew-#{tap.repo}.git" remote = "git@github.com:BrewTestBot/homebrew-#{tap.repo}.git"
tag = pr ? "pr-#{pr}" : "testing-#{number}" git_tag = pr ? "pr-#{pr}" : "testing-#{number}"
safe_system "git", "push", "--force", remote, "master:master", ":refs/tags/#{git_tag}"
bintray_repo = Utils::Bottles::Bintray.repository(tap)
bintray_repo_url = "https://api.bintray.com/packages/homebrew/#{bintray_repo}"
formula_packaged = {} formula_packaged = {}
Dir.glob("*.bottle*.tar.gz") do |filename| bottles_hash = json_files.reduce({}) do |hash, json_file|
formula_name, canonical_formula_name = Utils::Bottles.resolve_formula_names filename hash.merge! Utils::JSON.load(IO.read(json_file))
formula = Formulary.factory canonical_formula_name end
version = formula.pkg_version
bintray_package = Utils::Bottles::Bintray.package formula_name
if system "curl", "-I", "--silent", "--fail", "--output", "/dev/null", bottles_hash.each do |formula_name, bottle_hash|
"#{BottleSpecification::DEFAULT_DOMAIN}/#{bintray_repo}/#{filename}" version = bottle_hash["formula"]["pkg_version"]
raise <<-EOS.undent bintray_package = bottle_hash["bintray"]["package"]
#{filename} is already published. Please remove it manually from bintray_repo = bottle_hash["bintray"]["repository"]
https://bintray.com/homebrew/#{bintray_repo}/#{bintray_package}/view#files bintray_repo_url = "https://api.bintray.com/packages/homebrew/#{bintray_repo}"
EOS
end
unless formula_packaged[formula_name] bottle_hash["bottle"]["tags"].each do |tag, tag_hash|
package_url = "#{bintray_repo_url}/#{bintray_package}" filename = tag_hash["filename"]
unless system "curl", "--silent", "--fail", "--output", "/dev/null", package_url if system "curl", "-I", "--silent", "--fail", "--output", "/dev/null",
package_blob = <<-EOS.undent "#{BottleSpecification::DEFAULT_DOMAIN}/#{bintray_repo}/#{filename}"
{"name": "#{bintray_package}", raise <<-EOS.undent
"public_download_numbers": true, #{filename} is already published. Please remove it manually from
"public_stats": true} https://bintray.com/homebrew/#{bintray_repo}/#{bintray_package}/view#files
EOS EOS
curl "--silent", "--fail", "-u#{bintray_user}:#{bintray_key}",
"-H", "Content-Type: application/json",
"-d", package_blob, bintray_repo_url
puts
end end
formula_packaged[formula_name] = true
end
content_url = "https://api.bintray.com/content/homebrew" unless formula_packaged[formula_name]
content_url += "/#{bintray_repo}/#{bintray_package}/#{version}/#{filename}" package_url = "#{bintray_repo_url}/#{bintray_package}"
content_url += "?override=1" unless system "curl", "--silent", "--fail", "--output", "/dev/null", package_url
curl "--silent", "--fail", "-u#{bintray_user}:#{bintray_key}", package_blob = <<-EOS.undent
"-T", filename, content_url {"name": "#{bintray_package}",
puts "public_download_numbers": true,
"public_stats": true}
EOS
curl "--silent", "--fail", "-u#{bintray_user}:#{bintray_key}",
"-H", "Content-Type: application/json",
"-d", package_blob, bintray_repo_url
puts
end
formula_packaged[formula_name] = true
end
content_url = "https://api.bintray.com/content/homebrew"
content_url += "/#{bintray_repo}/#{bintray_package}/#{version}/#{filename}"
content_url += "?override=1"
curl "--silent", "--fail", "-u#{bintray_user}:#{bintray_key}",
"-T", filename, content_url
puts
end
end end
safe_system "git", "tag", "--force", tag safe_system "git", "tag", "--force", tag

4
Library/Homebrew/formulary.rb
View File

@ -16,6 +16,10 @@ class Formulary
end end
def self.load_formula(name, path, contents, namespace) def self.load_formula(name, path, contents, namespace)
if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
raise "Formula loading disabled by HOMEBREW_DISABLE_LOAD_FORMULA!"
end
mod = Module.new mod = Module.new
const_set(namespace, mod) const_set(namespace, mod)
mod.module_eval(contents, path) mod.module_eval(contents, path)