brew/Library/Homebrew/bintray.rb

226 lines
7.7 KiB
Ruby
Raw Normal View History

2020-10-10 14:16:11 +02:00
# typed: false
2020-03-30 19:35:54 +11:00
# frozen_string_literal: true
require "utils/curl"
require "json"
2020-08-14 02:02:31 +02:00
# Bintray API client.
#
# @api private
2020-03-30 19:35:54 +11:00
class Bintray
include Context
2020-03-30 19:35:54 +11:00
API_URL = "https://api.bintray.com"
class Error < RuntimeError
end
def inspect
"#<Bintray: org=#{@bintray_org}>"
2020-03-30 19:35:54 +11:00
end
def initialize(org: "homebrew")
@bintray_org = org
2020-03-30 19:35:54 +11:00
raise UsageError, "Must set a Bintray organisation!" unless @bintray_org
2020-03-30 19:35:54 +11:00
ENV["HOMEBREW_FORCE_HOMEBREW_ON_LINUX"] = "1" if @bintray_org == "homebrew" && !OS.mac?
end
def open_api(url, *extra_curl_args, auth: true)
args = extra_curl_args
if auth
bintray: Fix "uninitialized constant EnvConfig" errors - This fix was suggested by Sorbet when I ran `HOMEBREW_SORBET=1 bundle exec srb tc` on the latest `master` while playing around with the latest changes post-GSoC meeting. - Then I noticed it was actually a bug, introduced in adc36a05ffeadb54b94c87d86f62fba9dbb86795, found by us not being able to publish bottles for [this build of the `n` formula](https://github.com/Homebrew/homebrew-core/runs/910309641?check_suite_focus=true) in https://github.com/Homebrew/homebrew-core/pull/58606: ``` [master 31d32307bd] n: update 6.7.0 bottle. 1 file changed, 3 insertions(+), 3 deletions(-) curl: (22) The requested URL returned error: 404 Not Found Error: uninitialized constant Bintray::EnvConfig /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/bintray.rb:28:in `open_api' /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/bintray.rb:43:in `upload' /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/bintray.rb:186:in `block (2 levels) in upload_bottle_json' /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/bintray.rb:158:in `each' /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/bintray.rb:158:in `block in upload_bottle_json' /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/bintray.rb:153:in `each' /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/bintray.rb:153:in `upload_bottle_json' /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/dev-cmd/pr-upload.rb:54:in `pr_upload' /home/linuxbrew/.linuxbrew/Homebrew/Library/Homebrew/brew.rb:111:in `<main>' Error: Failure while executing; `/home/linuxbrew/.linuxbrew/bin/brew pr-upload --verbose --bintray-org=homebrew` exited with 1. ```
2020-07-25 22:05:38 +01:00
raise UsageError, "HOMEBREW_BINTRAY_USER is unset." unless (user = Homebrew::EnvConfig.bintray_user)
raise UsageError, "HOMEBREW_BINTRAY_KEY is unset." unless (key = Homebrew::EnvConfig.bintray_key)
args += ["--user", "#{user}:#{key}"]
end
2020-03-30 19:35:54 +11:00
curl(*args, url,
2020-09-08 22:30:22 +02:00
print_stdout: false,
secrets: key)
2020-03-30 19:35:54 +11:00
end
def upload(local_file, repo:, package:, version:, remote_file:, sha256: nil)
url = "#{API_URL}/content/#{@bintray_org}/#{repo}/#{package}/#{version}/#{remote_file}"
args = ["--fail", "--upload-file", local_file]
2020-03-30 19:35:54 +11:00
args += ["--header", "X-Checksum-Sha2: #{sha256}"] unless sha256.blank?
result = open_api url, *args
json = JSON.parse(result.stdout)
raise "Bottle upload failed: #{json["message"]}" if json["message"] != "success"
result
2020-03-30 19:35:54 +11:00
end
def publish(repo:, package:, version:, file_count:, warn_on_error: false)
2020-03-30 19:35:54 +11:00
url = "#{API_URL}/content/#{@bintray_org}/#{repo}/#{package}/#{version}/publish"
result = open_api url, "--request", "POST", "--fail"
json = JSON.parse(result.stdout)
if file_count.present? && json["files"] != file_count
message = "Bottle publish failed: expected #{file_count} bottles, but published #{json["files"]} instead."
raise message unless warn_on_error
opoo message
end
odebug "Published #{json["files"]} bottles"
result
2020-03-30 19:35:54 +11:00
end
def official_org?(org: @bintray_org)
%w[homebrew linuxbrew].include? org
end
def stable_mirrored?(url)
headers, = curl_output("--connect-timeout", "15", "--location", "--head", url)
status_code = headers.scan(%r{^HTTP/.* (\d+)}).last.first
status_code.start_with?("2")
end
def mirror_formula(formula, repo: "mirror", publish_package: false)
package = Utils::Bottles::Bintray.package formula.name
create_package(repo: repo, package: package) unless package_exists?(repo: repo, package: package)
formula.downloader.fetch
version = ERB::Util.url_encode(formula.pkg_version)
filename = ERB::Util.url_encode(formula.downloader.basename)
destination_url = "https://dl.bintray.com/#{@bintray_org}/#{repo}/#{filename}"
odebug "Uploading to #{destination_url}"
upload(
formula.downloader.cached_location,
repo: repo,
package: package,
version: version,
sha256: formula.stable.checksum,
remote_file: filename,
)
return destination_url unless publish_package
odebug "Publishing #{@bintray_org}/#{repo}/#{package}/#{version}"
publish(repo: repo, package: package, version: version, file_count: 1)
destination_url
end
2020-03-30 19:35:54 +11:00
def create_package(repo:, package:, **extra_data_args)
2020-04-21 14:21:34 +01:00
url = "#{API_URL}/packages/#{@bintray_org}/#{repo}"
2020-03-30 19:35:54 +11:00
data = { name: package, public_download_numbers: true }
data[:public_stats] = official_org?
data.merge! extra_data_args
2020-04-21 14:21:34 +01:00
open_api url, "--header", "Content-Type: application/json", "--request", "POST", "--data", data.to_json
2020-03-30 19:35:54 +11:00
end
def package_exists?(repo:, package:)
url = "#{API_URL}/packages/#{@bintray_org}/#{repo}/#{package}"
2020-04-21 14:21:34 +01:00
begin
open_api url, "--fail", "--silent", "--output", "/dev/null", auth: false
2020-04-21 14:21:34 +01:00
rescue ErrorDuringExecution => e
2020-04-22 19:23:41 +01:00
stderr = e.output
.select { |type,| type == :stderr }
2020-04-21 14:21:34 +01:00
.map { |_, line| line }
.join
raise if e.status.exitstatus != 22 && !stderr.include?("404 Not Found")
false
else
true
end
2020-03-30 19:35:54 +11:00
end
# Gets the SHA-256 checksum of the specified remote file.
# Returns the empty string if the file exists but doesn't have a checksum.
# Returns nil if the file doesn't exist.
def remote_checksum(repo:, remote_file:)
2020-03-30 19:35:54 +11:00
url = "https://dl.bintray.com/#{@bintray_org}/#{repo}/#{remote_file}"
result = curl_output "--fail", "--silent", "--head", url
if result.success?
result.stdout.match(/^X-Checksum-Sha2:\s+(\h{64})\b/i)&.values_at(1)&.first || ""
2020-03-30 19:35:54 +11:00
else
raise Error if result.status.exitstatus != 22 && !result.stderr.include?("404 Not Found")
nil
2020-03-30 19:35:54 +11:00
end
end
def file_delete_instructions(bintray_repo, bintray_package, filename)
<<~EOS
Remove this file manually in your web browser:
https://bintray.com/#{@bintray_org}/#{bintray_repo}/#{bintray_package}/view#files
Or run:
curl -X DELETE -u $HOMEBREW_BINTRAY_USER:$HOMEBREW_BINTRAY_KEY \\
https://api.bintray.com/content/#{@bintray_org}/#{bintray_repo}/#{filename}
EOS
end
def upload_bottles(bottles_hash, publish_package: false, warn_on_error: false)
2020-03-30 19:35:54 +11:00
formula_packaged = {}
bottles_hash.each do |formula_name, bottle_hash|
2020-05-10 00:30:32 +01:00
version = ERB::Util.url_encode(bottle_hash["formula"]["pkg_version"])
2020-03-30 19:35:54 +11:00
bintray_package = bottle_hash["bintray"]["package"]
bintray_repo = bottle_hash["bintray"]["repository"]
bottle_count = bottle_hash["bottle"]["tags"].length
2020-03-30 19:35:54 +11:00
bottle_hash["bottle"]["tags"].each do |_tag, tag_hash|
2020-05-10 00:30:32 +01:00
filename = tag_hash["filename"] # URL encoded in Bottle::Filename#bintray
2020-03-30 19:35:54 +11:00
sha256 = tag_hash["sha256"]
delete_instructions = file_delete_instructions(bintray_repo, bintray_package, filename)
2020-03-30 19:35:54 +11:00
odebug "Checking remote file #{@bintray_org}/#{bintray_repo}/#{filename}"
result = remote_checksum(repo: bintray_repo, remote_file: filename)
case result
when nil
# File doesn't exist.
if !formula_packaged[formula_name] && !package_exists?(repo: bintray_repo, package: bintray_package)
odebug "Creating package #{@bintray_org}/#{bintray_repo}/#{bintray_package}"
create_package repo: bintray_repo, package: bintray_package
formula_packaged[formula_name] = true
end
odebug "Uploading #{@bintray_org}/#{bintray_repo}/#{bintray_package}/#{version}/#{filename}"
upload(tag_hash["local_filename"],
repo: bintray_repo,
package: bintray_package,
version: version,
remote_file: filename,
sha256: sha256)
when sha256
# File exists, checksum matches.
odebug "#{filename} is already published with matching hash."
bottle_count -= 1
when ""
# File exists, but can't find checksum
failed_message = "#{filename} is already published!"
raise Error, "#{failed_message}\n#{delete_instructions}" unless warn_on_error
opoo failed_message
else
# File exists, but checksum either doesn't exist or is mismatched.
failed_message = <<~EOS
#{filename} is already published with a mismatched hash!
Expected: #{sha256}
Actual: #{result}
2020-03-30 19:35:54 +11:00
EOS
raise Error, "#{failed_message}#{delete_instructions}" unless warn_on_error
opoo failed_message
2020-03-30 19:35:54 +11:00
end
end
next unless publish_package
odebug "Publishing #{@bintray_org}/#{bintray_repo}/#{bintray_package}/#{version}"
publish(repo: bintray_repo,
package: bintray_package,
version: version,
file_count: bottle_count,
warn_on_error: warn_on_error)
2020-03-30 19:35:54 +11:00
end
end
end