From 521d6b51a67cde6e38a0a574434a4990b9f17658 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Wed, 21 Apr 2021 17:10:05 +0100 Subject: [PATCH] dev-cmd/{bottle,pr-upload}: generate, upload `all: $SHA256` bottles. Generate `all: $SHA256` bottles if: - all generated bottles have the same cellar - all generated bottles have the same checksum In this case, on `brew bottle --merge --write`: delete all the non-`all` bottles (and their JSON) and upload only the single one that's necessary. --- Library/Homebrew/dev-cmd/bottle.rb | 57 +++++++++++++++++++++++---- Library/Homebrew/dev-cmd/pr-upload.rb | 22 ++++++++--- 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/Library/Homebrew/dev-cmd/bottle.rb b/Library/Homebrew/dev-cmd/bottle.rb index 27e5f5b02b..2aaa054d4f 100644 --- a/Library/Homebrew/dev-cmd/bottle.rb +++ b/Library/Homebrew/dev-cmd/bottle.rb @@ -607,12 +607,6 @@ module Homebrew def merge(args:) bottles_hash = merge_json_files(parse_json_files(args.named)) - # TODO: deduplicate --no-json bottles by: - # 1. throwing away bottles for newer versions of macOS if their SHA256 is - # identical - # 2. generating `all: $SHA256` bottles that can be used on macOS and Linux - # i.e. need to be `any_skip_relocation` and contain no ELF/MachO files. - any_cellars = ["any", "any_skip_relocation"] bottles_hash.each do |formula_name, bottle_hash| ohai formula_name @@ -620,14 +614,63 @@ module Homebrew bottle = BottleSpecification.new bottle.root_url bottle_hash["bottle"]["root_url"] bottle.rebuild bottle_hash["bottle"]["rebuild"] + + # if all the cellars and checksums are the same: we can create an + # `all: $SHA256` bottle. + tag_hashes = bottle_hash["bottle"]["tags"].values + all_bottle = (tag_hashes.count > 1) && tag_hashes.uniq do |tag_hash| + "#{tag_hash["cellar"]}-#{tag_hash["sha256"]}" + end.count == 1 + bottle_hash["bottle"]["tags"].each do |tag, tag_hash| cellar = tag_hash["cellar"] cellar = cellar.to_sym if any_cellars.include?(cellar) - sha256_hash = { cellar: cellar, tag.to_sym => tag_hash["sha256"] } + + tag_sym = if all_bottle + :all + else + tag.to_sym + end + + sha256_hash = { cellar: cellar, tag_sym => tag_hash["sha256"] } bottle.sha256 sha256_hash + + break if all_bottle end if args.write? + if all_bottle + all_bottle_hash = nil + bottle_hash["bottle"]["tags"].each do |tag, tag_hash| + local_filename = tag_hash["local_filename"] + local_json_filename = local_filename.sub(/\.tar\.gz$/, ".json") + + if all_bottle_hash.nil? + all_filename = tag_hash["filename"].sub(tag, "all") + all_local_filename = local_filename.sub(tag, "all") + all_local_json_filename = local_json_filename.sub(tag, "all") + + all_bottle_tag_hash = tag_hash.dup + all_bottle_tag_hash["filename"] = all_filename + all_bottle_tag_hash["local_filename"] = all_local_filename + cellar = all_bottle_tag_hash.delete("cellar") + + all_bottle_formula_hash = bottle_hash.dup + all_bottle_formula_hash["bottle"]["cellar"] = cellar + all_bottle_formula_hash["bottle"]["tags"] = { all: all_bottle_tag_hash } + + all_bottle_hash = { formula_name => all_bottle_formula_hash } + + FileUtils.cp local_filename, all_local_filename + all_local_json_path = Pathname(all_local_json_filename) + all_local_json_path.unlink if all_local_json_path.exist? + all_local_json_path.write(JSON.pretty_generate(all_bottle_hash)) + end + + FileUtils.rm_f [local_filename, local_json_filename] + end + end + Homebrew.install_bundler_gems! require "utils/ast" diff --git a/Library/Homebrew/dev-cmd/pr-upload.rb b/Library/Homebrew/dev-cmd/pr-upload.rb index bdac43d158..2f3682d21b 100644 --- a/Library/Homebrew/dev-cmd/pr-upload.rb +++ b/Library/Homebrew/dev-cmd/pr-upload.rb @@ -45,7 +45,7 @@ module Homebrew end end - def check_bottled_formulae(bottles_hash) + def check_bottled_formulae!(bottles_hash) bottles_hash.each do |name, bottle_hash| formula_path = HOMEBREW_REPOSITORY/bottle_hash["formula"]["path"] formula_version = Formulary.factory(formula_path).pkg_version @@ -84,9 +84,7 @@ module Homebrew end end - def pr_upload - args = pr_upload_args.parse - + def bottles_hash_from_json_files(root_url) json_files = Dir["*.bottle.json"] odie "No bottle JSON files found in the current working directory" if json_files.empty? @@ -94,12 +92,20 @@ module Homebrew hash.deep_merge(JSON.parse(File.read(json_file))) end - if args.root_url + if root_url bottles_hash.each_value do |bottle_hash| bottle_hash["bottle"]["root_url"] = args.root_url end end + bottles_hash + end + + def pr_upload + args = pr_upload_args.parse + + bottles_hash = bottles_hash_from_json_files(args.root_url) + bottle_args = ["bottle", "--merge", "--write"] bottle_args << "--verbose" if args.verbose? bottle_args << "--debug" if args.debug? @@ -133,10 +139,14 @@ module Homebrew end end - check_bottled_formulae(bottles_hash) + check_bottled_formulae!(bottles_hash) safe_system HOMEBREW_BREW_FILE, *bottle_args + # Reload the JSON files (in case `brew bottle --merge` generated + # `all: $SHA256` bottles) + bottles_hash = bottles_hash_from_json_files(args.root_url) + # Check the bottle commits did not break `brew audit` unless args.no_commit? audit_args = ["audit", "--skip-style"]