mirror of
https://github.com/Homebrew/brew.git
synced 2025-07-14 16:09:03 +08:00
eff70115b5
If we make a PR in which we're building many formulae (e.g. https://github.com/Homebrew/homebrew-core/pull/9249) then we may want to pull those that built and publish their bottles and discard those that did not. Instead of saying bottles will be published when they won't and then blowing up just avoid publishing them and print a warning instead.
583 lines
20 KiB
Ruby
583 lines
20 KiB
Ruby
#: `pull` [`--bottle`] [`--bump`] [`--clean`] [`--ignore-whitespace`] [`--resolve`] [`--branch-okay`] [`--no-pbcopy`] [`--no-publish`] <patch-source> [<patch-source>]
|
|
#:
|
|
#: Gets a patch from a GitHub commit or pull request and applies it to Homebrew.
|
|
#: Optionally, installs the formulae changed by the patch.
|
|
#:
|
|
#: Each <patch-source> may be one of:
|
|
#: * The ID number of a PR (Pull Request) in the homebrew/core GitHub
|
|
#: repository
|
|
#: * The URL of a PR on GitHub, using either the web page or API URL
|
|
#: formats. In this form, the PR may be on Homebrew/brew,
|
|
#: Homebrew/homebrew-core or any tap.
|
|
#: * The URL of a commit on GitHub
|
|
#: * A "http://bot.brew.sh/job/..." string specifying a testing job ID
|
|
#:
|
|
#: If `--bottle` was passed, handle bottles, pulling the bottle-update
|
|
#: commit and publishing files on Bintray.
|
|
#: If `--bump` was passed, for one-formula PRs, automatically reword
|
|
#: commit message to our preferred format.
|
|
#: If `--clean` was passed, do not rewrite or otherwise modify the
|
|
#: commits found in the pulled PR.
|
|
#: If `--ignore-whitespace` was passed, silently ignore whitespace
|
|
#: discrepancies when applying diffs.
|
|
#: If `--resolve` was passed, when a patch fails to apply, leave in
|
|
#: progress and allow user to
|
|
#: resolve, instead of aborting.
|
|
#: If `--branch-okay` was passed, do not warn if pulling to a branch
|
|
#: besides master (useful for testing).
|
|
#: If `--no-pbcopy` was passed, do not copy anything to the system
|
|
# clipboard.
|
|
#: If `--no-publish` was passed, do not publish bottles to Bintray.
|
|
|
|
require "net/http"
|
|
require "net/https"
|
|
require "utils"
|
|
require "json"
|
|
require "formula"
|
|
require "formulary"
|
|
require "tap"
|
|
require "version"
|
|
require "pkg_version"
|
|
|
|
module Homebrew
|
|
module_function
|
|
|
|
def pull
|
|
odie "You meant `git pull --rebase`." if ARGV[0] == "--rebase"
|
|
|
|
if ARGV.named.empty?
|
|
odie "This command requires at least one argument containing a URL or pull request number"
|
|
end
|
|
|
|
do_bump = ARGV.include?("--bump") && !ARGV.include?("--clean")
|
|
|
|
# Formulae with affected bottles that were published
|
|
bintray_published_formulae = []
|
|
tap = nil
|
|
|
|
ARGV.named.each do |arg|
|
|
if arg.to_i > 0
|
|
issue = arg
|
|
url = "https://github.com/Homebrew/homebrew-core/pull/#{arg}"
|
|
tap = CoreTap.instance
|
|
elsif (testing_match = arg.match %r{/job/Homebrew.*Testing/(\d+)/})
|
|
tap = ARGV.value("tap")
|
|
tap = if tap && tap.start_with?("homebrew/")
|
|
Tap.fetch("homebrew", tap.strip_prefix("homebrew/"))
|
|
elsif tap
|
|
odie "Tap option did not start with \"homebrew/\": #{tap}"
|
|
else
|
|
CoreTap.instance
|
|
end
|
|
_, testing_job = *testing_match
|
|
url = "https://github.com/Homebrew/homebrew-#{tap.repo}/compare/master...BrewTestBot:testing-#{testing_job}"
|
|
odie "Testing URLs require `--bottle`!" unless ARGV.include?("--bottle")
|
|
elsif (api_match = arg.match HOMEBREW_PULL_API_REGEX)
|
|
_, user, repo, issue = *api_match
|
|
url = "https://github.com/#{user}/#{repo}/pull/#{issue}"
|
|
tap = Tap.fetch(user, repo) if repo.start_with?("homebrew-")
|
|
elsif (url_match = arg.match HOMEBREW_PULL_OR_COMMIT_URL_REGEX)
|
|
url, user, repo, issue = *url_match
|
|
tap = Tap.fetch(user, repo) if repo.start_with?("homebrew-")
|
|
else
|
|
odie "Not a GitHub pull request or commit: #{arg}"
|
|
end
|
|
|
|
if !testing_job && ARGV.include?("--bottle") && issue.nil?
|
|
odie "No pull request detected!"
|
|
end
|
|
|
|
if tap
|
|
tap.install unless tap.installed?
|
|
Dir.chdir tap.path
|
|
else
|
|
Dir.chdir HOMEBREW_REPOSITORY
|
|
end
|
|
|
|
# The cache directory seems like a good place to put patches.
|
|
HOMEBREW_CACHE.mkpath
|
|
|
|
# Store current revision and branch
|
|
orig_revision = `git rev-parse --short HEAD`.strip
|
|
branch = `git symbolic-ref --short HEAD`.strip
|
|
|
|
unless branch == "master" || ARGV.include?("--clean") || ARGV.include?("--branch-okay")
|
|
opoo "Current branch is #{branch}: do you need to pull inside master?"
|
|
end
|
|
|
|
patch_puller = PatchPuller.new(url)
|
|
patch_puller.fetch_patch
|
|
patch_changes = files_changed_in_patch(patch_puller.patchpath, tap)
|
|
|
|
is_bumpable = patch_changes[:formulae].length == 1 && patch_changes[:others].empty?
|
|
if do_bump
|
|
odie "No changed formulae found to bump" if patch_changes[:formulae].empty?
|
|
if patch_changes[:formulae].length > 1
|
|
odie "Can only bump one changed formula; bumped #{patch_changes[:formulae]}"
|
|
end
|
|
odie "Can not bump if non-formula files are changed" unless patch_changes[:others].empty?
|
|
end
|
|
if is_bumpable
|
|
old_versions = current_versions_from_info_external(patch_changes[:formulae].first)
|
|
end
|
|
patch_puller.apply_patch
|
|
|
|
changed_formulae_names = []
|
|
|
|
if tap
|
|
Utils.popen_read(
|
|
"git", "diff-tree", "-r", "--name-only",
|
|
"--diff-filter=AM", orig_revision, "HEAD", "--", tap.formula_dir.to_s
|
|
).each_line do |line|
|
|
next unless line.end_with? ".rb\n"
|
|
name = "#{tap.name}/#{File.basename(line.chomp, ".rb")}"
|
|
changed_formulae_names << name
|
|
end
|
|
end
|
|
|
|
fetch_bottles = false
|
|
changed_formulae_names.each do |name|
|
|
next if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
|
|
|
|
begin
|
|
f = Formula[name]
|
|
# Make sure we catch syntax errors.
|
|
rescue Exception
|
|
next
|
|
end
|
|
|
|
if ARGV.include? "--bottle"
|
|
if f.bottle_unneeded?
|
|
ohai "#{f}: skipping unneeded bottle."
|
|
elsif f.bottle_disabled?
|
|
ohai "#{f}: skipping disabled bottle: #{f.bottle_disable_reason}"
|
|
else
|
|
fetch_bottles = true
|
|
end
|
|
else
|
|
next unless f.bottle_defined?
|
|
opoo "#{f.full_name} has a bottle: do you need to update it with --bottle?"
|
|
end
|
|
end
|
|
|
|
orig_message = message = `git log HEAD^.. --format=%B`
|
|
if issue && !ARGV.include?("--clean")
|
|
ohai "Patch closes issue ##{issue}"
|
|
close_message = "Closes ##{issue}."
|
|
# If this is a pull request, append a close message.
|
|
message += "\n#{close_message}" unless message.include? close_message
|
|
end
|
|
|
|
if changed_formulae_names.empty?
|
|
odie "cannot bump: no changed formulae found after applying patch" if do_bump
|
|
is_bumpable = false
|
|
end
|
|
|
|
is_bumpable = false if ARGV.include?("--clean")
|
|
is_bumpable = false if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
|
|
|
|
if is_bumpable
|
|
formula = Formula[changed_formulae_names.first]
|
|
new_versions = current_versions_from_info_external(patch_changes[:formulae].first)
|
|
orig_subject = message.empty? ? "" : message.lines.first.chomp
|
|
bump_subject = subject_for_bump(formula, old_versions, new_versions)
|
|
if do_bump
|
|
odie "No version changes found for #{formula.name}" if bump_subject.nil?
|
|
unless orig_subject == bump_subject
|
|
ohai "New bump commit subject: #{bump_subject}"
|
|
pbcopy bump_subject unless ARGV.include? "--no-pbcopy"
|
|
message = "#{bump_subject}\n\n#{message}"
|
|
end
|
|
elsif bump_subject != orig_subject && !bump_subject.nil?
|
|
opoo "Nonstandard bump subject: #{orig_subject}"
|
|
opoo "Subject should be: #{bump_subject}"
|
|
end
|
|
end
|
|
|
|
if message != orig_message && !ARGV.include?("--clean")
|
|
safe_system "git", "commit", "--amend", "--signoff", "--allow-empty", "-q", "-m", message
|
|
end
|
|
|
|
# Bottles: Pull bottle block commit and publish bottle files on Bintray
|
|
if fetch_bottles
|
|
bottle_commit_url = if testing_job
|
|
bottle_branch = "testing-bottle-#{testing_job}"
|
|
url
|
|
else
|
|
bottle_branch = "pull-bottle-#{issue}"
|
|
"https://github.com/BrewTestBot/homebrew-#{tap.repo}/compare/homebrew:master...pr-#{issue}"
|
|
end
|
|
|
|
curl "--silent", "--fail", "-o", "/dev/null", "-I", bottle_commit_url
|
|
|
|
safe_system "git", "checkout", "--quiet", "-B", bottle_branch, orig_revision
|
|
pull_patch bottle_commit_url, "bottle commit"
|
|
safe_system "git", "rebase", "--quiet", branch
|
|
safe_system "git", "checkout", "--quiet", branch
|
|
safe_system "git", "merge", "--quiet", "--ff-only", "--no-edit", bottle_branch
|
|
safe_system "git", "branch", "--quiet", "-D", bottle_branch
|
|
|
|
# Publish bottles on Bintray
|
|
unless ARGV.include? "--no-publish"
|
|
published = publish_changed_formula_bottles(tap, changed_formulae_names)
|
|
bintray_published_formulae.concat(published)
|
|
end
|
|
end
|
|
|
|
ohai "Patch changed:"
|
|
safe_system "git", "diff-tree", "-r", "--stat", orig_revision, "HEAD"
|
|
end
|
|
|
|
# Verify bintray publishing after all patches have been applied
|
|
bintray_published_formulae.uniq!
|
|
verify_bintray_published(bintray_published_formulae)
|
|
end
|
|
|
|
def force_utf8!(str)
|
|
str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
|
|
end
|
|
|
|
def publish_changed_formula_bottles(_tap, changed_formulae_names)
|
|
if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
|
|
raise "Need to load formulae to publish them!"
|
|
end
|
|
|
|
published = []
|
|
bintray_creds = { user: ENV["BINTRAY_USER"], key: ENV["BINTRAY_KEY"] }
|
|
if bintray_creds[:user] && bintray_creds[:key]
|
|
changed_formulae_names.each do |name|
|
|
f = Formula[name]
|
|
next if f.bottle_unneeded? || f.bottle_disabled?
|
|
publish_bottle_file_on_bintray(f, bintray_creds)
|
|
published << f.full_name
|
|
end
|
|
else
|
|
opoo "You must set BINTRAY_USER and BINTRAY_KEY to add or update bottles on Bintray!"
|
|
end
|
|
published
|
|
end
|
|
|
|
def pull_patch(url, description = nil)
|
|
PatchPuller.new(url, description).pull_patch
|
|
end
|
|
|
|
class PatchPuller
|
|
attr_reader :base_url
|
|
attr_reader :patch_url
|
|
attr_reader :patchpath
|
|
|
|
def initialize(url, description = nil)
|
|
@base_url = url
|
|
# GitHub provides commits/pull-requests raw patches using this URL.
|
|
@patch_url = url + ".patch"
|
|
@patchpath = HOMEBREW_CACHE + File.basename(patch_url)
|
|
@description = description
|
|
end
|
|
|
|
def pull_patch
|
|
fetch_patch
|
|
apply_patch
|
|
end
|
|
|
|
def fetch_patch
|
|
extra_msg = @description ? "(#{@description})" : nil
|
|
ohai "Fetching patch #{extra_msg}"
|
|
puts "Patch: #{patch_url}"
|
|
curl patch_url, "-s", "-o", patchpath
|
|
end
|
|
|
|
def apply_patch
|
|
# Applies a patch previously downloaded with fetch_patch()
|
|
# Deletes the patch file as a side effect, regardless of success
|
|
|
|
ohai "Applying patch"
|
|
patch_args = []
|
|
# Normally we don't want whitespace errors, but squashing them can break
|
|
# patches so an option is provided to skip this step.
|
|
if ARGV.include?("--ignore-whitespace") || ARGV.include?("--clean")
|
|
patch_args << "--whitespace=nowarn"
|
|
else
|
|
patch_args << "--whitespace=fix"
|
|
end
|
|
|
|
# Fall back to three-way merge if patch does not apply cleanly
|
|
patch_args << "-3"
|
|
patch_args << patchpath
|
|
|
|
begin
|
|
safe_system "git", "am", *patch_args
|
|
rescue ErrorDuringExecution
|
|
if ARGV.include? "--resolve"
|
|
odie "Patch failed to apply: try to resolve it."
|
|
else
|
|
system "git", "am", "--abort"
|
|
odie "Patch failed to apply: aborted."
|
|
end
|
|
ensure
|
|
patchpath.unlink
|
|
end
|
|
end
|
|
end
|
|
|
|
# List files changed by a patch, partitioned in to those that are (probably)
|
|
# formula definitions, and those which aren't. Only applies to patches on
|
|
# Homebrew core or taps, based simply on relative pathnames of affected files.
|
|
def files_changed_in_patch(patchfile, tap)
|
|
files = []
|
|
formulae = []
|
|
others = []
|
|
File.foreach(patchfile) do |line|
|
|
files << $1 if line =~ %r{^\+\+\+ b/(.*)}
|
|
end
|
|
files.each do |file|
|
|
if tap && tap.formula_file?(file)
|
|
formula_name = File.basename(file, ".rb")
|
|
formulae << formula_name unless formulae.include?(formula_name)
|
|
else
|
|
others << file
|
|
end
|
|
end
|
|
{ files: files, formulae: formulae, others: others }
|
|
end
|
|
|
|
# Get current formula versions without loading formula definition in this process
|
|
# Returns info as a hash (type => version), for pull.rb's internal use
|
|
# Uses special key :nonexistent => true for nonexistent formulae
|
|
def current_versions_from_info_external(formula_name)
|
|
info = FormulaInfoFromJson.lookup(formula_name)
|
|
versions = {}
|
|
if info
|
|
[:stable, :devel, :head].each do |spec_type|
|
|
versions[spec_type] = info.version(spec_type)
|
|
end
|
|
else
|
|
versions[:nonexistent] = true
|
|
end
|
|
versions
|
|
end
|
|
|
|
def subject_for_bump(formula, old, new)
|
|
if old[:nonexistent]
|
|
# New formula
|
|
headline_ver = if new[:stable]
|
|
new[:stable]
|
|
elsif new[:devel]
|
|
new[:devel]
|
|
else
|
|
new[:head]
|
|
end
|
|
subject = "#{formula.name} #{headline_ver} (new formula)"
|
|
else
|
|
# Update to existing formula
|
|
subject_strs = []
|
|
formula_name_str = formula.name
|
|
if old[:stable] != new[:stable]
|
|
if new[:stable].nil?
|
|
subject_strs << "remove stable"
|
|
formula_name_str += ":" # just for cosmetics
|
|
else
|
|
subject_strs << formula.version.to_s
|
|
end
|
|
end
|
|
if old[:devel] != new[:devel]
|
|
if new[:devel].nil?
|
|
# Only bother mentioning if there's no accompanying stable change
|
|
if !new[:stable].nil? && old[:stable] == new[:stable]
|
|
subject_strs << "remove devel"
|
|
formula_name_str += ":" # just for cosmetics
|
|
end
|
|
else
|
|
subject_strs << "#{formula.devel.version} (devel)"
|
|
end
|
|
end
|
|
subject = subject_strs.empty? ? nil : "#{formula_name_str} #{subject_strs.join(", ")}"
|
|
end
|
|
subject
|
|
end
|
|
|
|
def pbcopy(text)
|
|
Utils.popen_write("pbcopy") { |io| io.write text }
|
|
end
|
|
|
|
# Publishes the current bottle files for a given formula to Bintray
|
|
def publish_bottle_file_on_bintray(f, creds)
|
|
repo = Utils::Bottles::Bintray.repository(f.tap)
|
|
package = Utils::Bottles::Bintray.package(f.name)
|
|
info = FormulaInfoFromJson.lookup(f.name)
|
|
if info.nil?
|
|
raise "Failed publishing bottle: failed reading formula info for #{f.full_name}"
|
|
end
|
|
unless info.bottle_info_any
|
|
opoo "No bottle defined in formula #{package}"
|
|
return
|
|
end
|
|
version = info.pkg_version
|
|
ohai "Publishing on Bintray: #{package} #{version}"
|
|
curl "-w", '\n', "--silent", "--fail",
|
|
"-u#{creds[:user]}:#{creds[:key]}", "-X", "POST",
|
|
"-H", "Content-Type: application/json",
|
|
"-d", '{"publish_wait_for_secs": 0}',
|
|
"https://api.bintray.com/content/homebrew/#{repo}/#{package}/#{version}/publish"
|
|
end
|
|
|
|
# Formula info drawn from an external "brew info --json" call
|
|
class FormulaInfoFromJson
|
|
# The whole info structure parsed from the JSON
|
|
attr_accessor :info
|
|
|
|
def initialize(info)
|
|
@info = info
|
|
end
|
|
|
|
# Looks up formula on disk and reads its info
|
|
# Returns nil if formula is absent or if there was an error reading it
|
|
def self.lookup(name)
|
|
json = Utils.popen_read(HOMEBREW_BREW_FILE, "info", "--json=v1", name)
|
|
|
|
return nil unless $?.success?
|
|
|
|
Homebrew.force_utf8!(json)
|
|
FormulaInfoFromJson.new(JSON.parse(json)[0])
|
|
end
|
|
|
|
def bottle_tags
|
|
return [] unless info["bottle"]["stable"]
|
|
info["bottle"]["stable"]["files"].keys
|
|
end
|
|
|
|
def bottle_info(my_bottle_tag = Utils::Bottles.tag)
|
|
tag_s = my_bottle_tag.to_s
|
|
return nil unless info["bottle"]["stable"]
|
|
btl_info = info["bottle"]["stable"]["files"][tag_s]
|
|
return nil unless btl_info
|
|
BottleInfo.new(btl_info["url"], btl_info["sha256"])
|
|
end
|
|
|
|
def bottle_info_any
|
|
bottle_info(any_bottle_tag)
|
|
end
|
|
|
|
def any_bottle_tag
|
|
tag = Utils::Bottles.tag
|
|
# Prefer native bottles as a convenience for download caching
|
|
bottle_tags.include?(tag) ? tag : bottle_tags.first
|
|
end
|
|
|
|
def version(spec_type)
|
|
version_str = info["versions"][spec_type.to_s]
|
|
version_str && Version.create(version_str)
|
|
end
|
|
|
|
def pkg_version(spec_type = :stable)
|
|
PkgVersion.new(version(spec_type), revision)
|
|
end
|
|
|
|
def revision
|
|
info["revision"]
|
|
end
|
|
end
|
|
|
|
# Bottle info as used internally by pull, with alternate platform support
|
|
class BottleInfo
|
|
# URL of bottle as string
|
|
attr_accessor :url
|
|
# Expected SHA256 as string
|
|
attr_accessor :sha256
|
|
|
|
def initialize(url, sha256)
|
|
@url = url
|
|
@sha256 = sha256
|
|
end
|
|
end
|
|
|
|
# Verifies that formulae have been published on Bintray by downloading a bottle file
|
|
# for each one. Blocks until the published files are available.
|
|
# Raises an error if the verification fails.
|
|
# This does not currently work for `brew pull`, because it may have cached the old
|
|
# version of a formula.
|
|
def verify_bintray_published(formulae_names)
|
|
return if formulae_names.empty?
|
|
|
|
if ENV["HOMEBREW_DISABLE_LOAD_FORMULA"]
|
|
raise "Need to load formulae to verify their publication!"
|
|
end
|
|
|
|
ohai "Verifying bottles published on Bintray"
|
|
formulae = formulae_names.map { |n| Formula[n] }
|
|
max_retries = 300 # shared among all bottles
|
|
poll_retry_delay_seconds = 2
|
|
|
|
HOMEBREW_CACHE.cd do
|
|
formulae.each do |f|
|
|
retry_count = 0
|
|
wrote_dots = false
|
|
# Choose arbitrary bottle just to get the host/port for Bintray right
|
|
jinfo = FormulaInfoFromJson.lookup(f.full_name)
|
|
unless jinfo
|
|
opoo "Cannot publish bottle: Failed reading info for formula #{f.full_name}"
|
|
next
|
|
end
|
|
bottle_info = jinfo.bottle_info(jinfo.bottle_tags.first)
|
|
unless bottle_info
|
|
opoo "No bottle defined in formula #{f.full_name}"
|
|
next
|
|
end
|
|
|
|
# Poll for publication completion using a quick partial HEAD, to avoid spurious error messages
|
|
# 401 error is normal while file is still in async publishing process
|
|
url = URI(bottle_info.url)
|
|
puts "Verifying bottle: #{File.basename(url.path)}"
|
|
http = Net::HTTP.new(url.host, url.port)
|
|
http.use_ssl = true
|
|
retry_count = 0
|
|
http.start do
|
|
loop do
|
|
req = Net::HTTP::Head.new bottle_info.url
|
|
req.initialize_http_header "User-Agent" => HOMEBREW_USER_AGENT_RUBY
|
|
res = http.request req
|
|
break if res.is_a?(Net::HTTPSuccess)
|
|
|
|
unless res.is_a?(Net::HTTPClientError)
|
|
raise "Failed to find published #{f} bottle at #{url} (#{res.code} #{res.message})!"
|
|
end
|
|
|
|
if retry_count >= max_retries
|
|
raise "Failed to find published #{f} bottle at #{url}!"
|
|
end
|
|
print(wrote_dots ? "." : "Waiting on Bintray.")
|
|
wrote_dots = true
|
|
sleep poll_retry_delay_seconds
|
|
retry_count += 1
|
|
end
|
|
end
|
|
|
|
# Actual download and verification
|
|
# We do a retry on this, too, because sometimes the external curl will fail even
|
|
# when the prior HEAD has succeeded.
|
|
puts "\n" if wrote_dots
|
|
filename = File.basename(url.path)
|
|
curl_retry_delay_seconds = 4
|
|
max_curl_retries = 1
|
|
retry_count = 0
|
|
# We're in the cache; make sure to force re-download
|
|
loop do
|
|
begin
|
|
curl url, "-o", filename
|
|
break
|
|
rescue
|
|
if retry_count >= max_curl_retries
|
|
raise "Failed to download #{f} bottle from #{url}!"
|
|
end
|
|
puts "curl download failed; retrying in #{curl_retry_delay_seconds} sec"
|
|
sleep curl_retry_delay_seconds
|
|
curl_retry_delay_seconds *= 2
|
|
retry_count += 1
|
|
end
|
|
end
|
|
checksum = Checksum.new(:sha256, bottle_info.sha256)
|
|
Pathname.new(filename).verify_checksum(checksum)
|
|
end
|
|
end
|
|
end
|
|
end
|