mirrors/brew
1
0
Fork 0
mirror of https://github.com/Homebrew/brew.git synced 2025-07-14 16:09:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

utils/pypi: refactor package handling

Browse Source
This commit is contained in:
Rylan Polster 2020-11-22 15:23:43 -05:00
parent ee47b863c4
commit b2e25d12aa
2 changed files with 334 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
Library/Homebrew/test/utils/pypi_spec.rb Normal file
View File

@ -0,0 +1,171 @@
# typed: false
# frozen_string_literal: true
require "utils/pypi"
describe PyPI do
let(:package_url) do
"https://files.pythonhosted.org/packages/b0/3f/2e1dad67eb172b6443b5eb37eb885a054a55cfd733393071499514140282/"\
"snakemake-5.29.0.tar.gz"
end
let(:old_package_url) do
"https://files.pythonhosted.org/packages/6f/c4/da52bfdd6168ea46a0fe2b7c983b6c34c377a8733ec177cc00b197a96a9f/"\
"snakemake-5.28.0.tar.gz"
end
describe PyPI::Package do
let(:package_checksum) { "47417307d08ecb0707b3b29effc933bd63d8c8e3ab15509c62b685b7614c6568" }
let(:old_package_checksum) { "2367ce91baf7f8fa7738d33aff9670ffdf5410bbac49aeb209f73b45a3425046" }
let(:package) { described_class.new("snakemake") }
let(:package_with_version) { described_class.new("snakemake==5.28.0") }
let(:package_with_different_version) { described_class.new("snakemake==5.29.0") }
let(:package_with_extra) { described_class.new("snakemake[foo]") }
let(:package_with_extra_and_version) { described_class.new("snakemake[foo]==5.28.0") }
let(:package_from_url) { described_class.new(package_url, is_url: true) }
let(:other_package) { described_class.new("virtualenv==20.2.0") }
describe "initialize" do
it "initializes name" do
expect(described_class.new("foo").name).to eq "foo"
end
it "initializes name with extra" do
expect(described_class.new("foo[bar]").name).to eq "foo"
end
it "initializes extra" do
expect(described_class.new("foo[bar]").extras).to eq ["bar"]
end
it "initializes multiple extras" do
expect(described_class.new("foo[bar,baz]").extras).to eq ["bar", "baz"]
end
it "initializes name with version" do
expect(described_class.new("foo==1.2.3").name).to eq "foo"
end
it "initializes version" do
expect(described_class.new("foo==1.2.3").version).to eq "1.2.3"
end
it "initializes extra with version" do
expect(described_class.new("foo[bar]==1.2.3").extras).to eq ["bar"]
end
it "initializes multiple extras with version" do
expect(described_class.new("foo[bar,baz]==1.2.3").extras).to eq ["bar", "baz"]
end
it "initializes version with extra" do
expect(described_class.new("foo[bar]==1.2.3").version).to eq "1.2.3"
end
it "initializes version with multiple extras" do
expect(described_class.new("foo[bar,baz]==1.2.3").version).to eq "1.2.3"
end
it "initializes name from url" do
expect(described_class.new(package_url, is_url: true).name).to eq "snakemake"
end
it "initializes version from url" do
expect(described_class.new(package_url, is_url: true).version).to eq "5.29.0"
end
end
describe ".pypi_info", :needs_network do
it "gets pypi info from a package name" do
expect(package.pypi_info.first).to eq "snakemake"
end
it "gets pypi info from a package name and specified version" do
expect(package.pypi_info(version: "5.29.0")).to eq ["snakemake", package_url, package_checksum, "5.29.0"]
end
it "gets pypi info from a package name with extra" do
expect(package_with_extra.pypi_info.first).to eq "snakemake"
end
it "gets pypi info from a package name and version" do
expect(package_with_version.pypi_info).to eq ["snakemake", old_package_url, old_package_checksum, "5.28.0"]
end
it "gets pypi info from a package name with overriden version" do
expected_result = ["snakemake", package_url, package_checksum, "5.29.0"]
expect(package_with_version.pypi_info(version: "5.29.0")).to eq expected_result
end
it "gets pypi info from a package name, extras, and version" do
expected_result = ["snakemake", old_package_url, old_package_checksum, "5.28.0"]
expect(package_with_extra_and_version.pypi_info).to eq expected_result
end
it "gets pypi info from a url" do
expect(package_from_url.pypi_info).to eq ["snakemake", package_url, package_checksum, "5.29.0"]
end
it "gets pypi info from a url with overriden version" do
expected_result = ["snakemake", old_package_url, old_package_checksum, "5.28.0"]
expect(package_from_url.pypi_info(version: "5.28.0")).to eq expected_result
end
end
describe ".to_s" do
it "returns string representation of package name" do
expect(package.to_s).to eq "snakemake"
end
it "returns string representation of package with version" do
expect(package_with_version.to_s).to eq "snakemake==5.28.0"
end
it "returns string representation of package with extra" do
expect(package_with_extra.to_s).to eq "snakemake[foo]"
end
it "returns string representation of package with extra and version" do
expect(package_with_extra_and_version.to_s).to eq "snakemake[foo]==5.28.0"
end
it "returns string representation of package from url" do
expect(package_from_url.to_s).to eq "snakemake==5.29.0"
end
end
describe ".same_package?" do
it "returns false for different packages" do
expect(package.same_package?(other_package)).to eq false
end
it "returns true for the same package" do
expect(package.same_package?(package_with_version)).to eq true
end
it "returns true for the same package with different versions" do
expect(package_with_version.same_package?(package_with_different_version)).to eq true
end
end
describe "<=>" do
it "returns -1" do
expect(package <=> other_package).to eq((-1))
end
it "returns 0" do
expect(package <=> package_with_version).to eq 0
end
it "returns 1" do
expect(other_package <=> package_with_extra_and_version).to eq 1
end
end
end
describe "update_pypi_url", :needs_network do
it "updates url to new version" do
expect(described_class.update_pypi_url(old_package_url, "5.29.0")).to eq package_url
end
end
end

240
Library/Homebrew/utils/pypi.rb
View File

@ -5,6 +5,8 @@
#
# @api private
module PyPI
extend T::Sig
module_function
PYTHONHOSTED_URL_PREFIX = "https://files.pythonhosted.org/packages/"
@ -12,45 +14,119 @@ module PyPI
@pipgrip_installed = nil
def url_to_pypi_package_name(url)
return unless url.start_with? PYTHONHOSTED_URL_PREFIX
# PyPI Package
#
# @api private
class Package
extend T::Sig
File.basename(url).match(/^(.+)-[a-z\d.]+$/)[1]
attr_accessor :name
attr_accessor :extras
attr_accessor :version
sig { params(package_string: String, is_url: T::Boolean).void }
def initialize(package_string, is_url: false)
@pypi_info = nil
if is_url
unless package_string.start_with?(PYTHONHOSTED_URL_PREFIX) &&
match = File.basename(package_string).match(/^(.+)-([a-z\d.]+?)(?:.tar.gz|.zip)$/)
raise ArgumentError, "package should be a valid PyPI url"
end
@name = match[1]
@version = match[2]
return
end
@name = package_string
@name, @version = @name.split("==") if @name.include? "=="
return unless match = @name.match(/^(.*?)\[(.+)\]$/)
@name = match[1]
@extras = match[2].split ","
end
# Get name, URL, SHA-256 checksum, and latest version for a given PyPI package.
sig { params(version: T.nilable(T.any(String, Version))).returns(T.nilable(T::Array[String])) }
def pypi_info(version: nil)
return @pypi_info if @pypi_info.present? && version.blank?
version ||= @version
metadata_url = if version.present?
"https://pypi.org/pypi/#{@name}/#{version}/json"
else
"https://pypi.org/pypi/#{@name}/json"
end
out, _, status = curl_output metadata_url, "--location"
return unless status.success?
begin
json = JSON.parse out
rescue JSON::ParserError
return
end
sdist = json["urls"].find { |url| url["packagetype"] == "sdist" }
return json["info"]["name"] if sdist.nil?
@pypi_info = [json["info"]["name"], sdist["url"], sdist["digests"]["sha256"], json["info"]["version"]]
end
sig { returns(T::Boolean) }
def valid_pypi_package?
info = pypi_info
info.present? && info.is_a?(Array)
end
sig { returns(String) }
def to_s
out = @name
out += "[#{@extras.join(",")}]" if @extras.present?
out += "==#{@version}" if @version.present?
out
end
sig { params(other: Package).returns(T::Boolean) }
def same_package?(other)
@name.tr("_", "-") == other.name.tr("_", "-")
end
# Compare only names so we can use .include? on a Package array
sig { params(other: Package).returns(T::Boolean) }
def ==(other)
same_package?(other)
end
sig { params(other: Package).returns(T.nilable(Integer)) }
def <=>(other)
@name <=> other.name
end
end
sig { params(url: String, version: T.any(String, Version)).returns(T.nilable(String)) }
def update_pypi_url(url, version)
package = url_to_pypi_package_name url
return if package.nil?
package = Package.new url, is_url: true
_, url = get_pypi_info(package, version)
_, url = package.pypi_info(version: version)
url
end
# Get name, URL, SHA-256 checksum, and latest version for a given PyPI package.
def get_pypi_info(package, version = nil)
package = package.split("[").first
metadata_url = if version.present?
"https://pypi.org/pypi/#{package}/#{version}/json"
else
"https://pypi.org/pypi/#{package}/json"
end
out, _, status = curl_output metadata_url, "--location"
return unless status.success?
begin
json = JSON.parse out
rescue JSON::ParserError
return
end
sdist = json["urls"].find { |url| url["packagetype"] == "sdist" }
return json["info"]["name"] if sdist.nil?
[json["info"]["name"], sdist["url"], sdist["digests"]["sha256"], json["info"]["version"]]
end
# Return true if resources were checked (even if no change).
sig do
params(
formula: Formula,
version: T.nilable(String),
package_name: T.nilable(String),
extra_packages: T.nilable(T::Array[String]),
exclude_packages: T.nilable(T::Array[String]),
print_only: T::Boolean,
silent: T::Boolean,
ignore_non_pypi_packages: T::Boolean,
).returns(T.nilable(T::Boolean))
end
def update_python_resources!(formula, version: nil, package_name: nil, extra_packages: nil, exclude_packages: nil,
print_only: false, silent: false, ignore_non_pypi_packages: false)
@ -73,12 +149,17 @@ module PyPI
end
end
version ||= formula.version if package_name.blank?
package_name ||= url_to_pypi_package_name formula.stable.url
extra_packages ||= []
exclude_packages ||= []
main_package = if package_name.present?
Package.new(package_name)
else
begin
Package.new(formula.stable.url, is_url: true)
rescue ArgumentError
nil
end
end
if package_name.blank?
if main_package.blank?
return if ignore_non_pypi_packages
odie <<~EOS
@ -87,76 +168,81 @@ module PyPI
EOS
end
input_package_names = { package_name => version }
extra_packages.each do |extra|
extra_name, extra_version = extra.split "=="
unless main_package.valid_pypi_package?
return if ignore_non_pypi_packages
if input_package_names.key?(extra_name) && input_package_names[extra_name] != extra_version
odie "Conflicting versions specified for the `#{extra_name}` package: "\
"#{input_package_names[extra_name]}, #{extra_version}"
odie "\"#{main_package}\" is not available on PyPI."
end
main_package.version = version if version.present?
extra_packages = (extra_packages || []).map { |p| Package.new p }
exclude_packages = (exclude_packages || []).map { |p| Package.new p }
input_packages = [main_package]
extra_packages.each do |extra_package|
if !extra_package.valid_pypi_package? && !ignore_non_pypi_packages
odie "\"#{extra_package}\" is not available on PyPI."
end
input_package_names[extra_name] = extra_version
input_packages.each do |existing_package|
if existing_package.same_package?(extra_package) && existing_package.version != extra_package.version
odie "Conflicting versions specified for the `#{extra_package.name}` package: "\
"#{existing_package.version}, #{extra_package.version}"
end
end
input_packages << extra_package unless input_packages.include? extra_package
end
input_package_names.each do |name, package_version|
name = name.split("[").first
next if get_pypi_info(name, package_version).present?
version_string = " at version #{package_version}" if package_version.present?
odie "\"#{name}\"#{version_string} is not available on PyPI." unless ignore_non_pypi_packages
end
non_pypi_resources = formula.resources.reject do |resource|
resource.url.start_with? PYTHONHOSTED_URL_PREFIX
end
if non_pypi_resources.present? && !print_only
odie "\"#{formula.name}\" contains non-PyPI resources. Please update the resources manually."
formula.resources.each do |resource|
if !print_only && !resource.url.start_with?(PYTHONHOSTED_URL_PREFIX)
odie "\"#{formula.name}\" contains non-PyPI resources. Please update the resources manually."
end
end
@pipgrip_installed ||= Formula["pipgrip"].any_version_installed?
odie '"pipgrip" must be installed (`brew install pipgrip`)' unless @pipgrip_installed
found_packages = {}
input_package_names.each do |name, package_version|
pypi_package_string = if package_version.present?
"#{name}==#{package_version}"
else
name
end
ohai "Retrieving PyPI dependencies for \"#{pypi_package_string}\"..." if !print_only && !silent
pipgrip_output = Utils.popen_read Formula["pipgrip"].bin/"pipgrip", "--json", "--no-cache-dir",
pypi_package_string
found_packages = []
input_packages.each do |package|
ohai "Retrieving PyPI dependencies for \"#{package}\"..." if !print_only && !silent
pipgrip_output = Utils.popen_read Formula["pipgrip"].bin/"pipgrip", "--json", "--no-cache-dir", package.to_s
unless $CHILD_STATUS.success?
odie <<~EOS
Unable to determine dependencies for \"#{name}\" because of a failure when running
`pipgrip --json --no-cache-dir #{pypi_package_string}`.
Unable to determine dependencies for \"#{package}\" because of a failure when running
`pipgrip --json --no-cache-dir #{package}`.
Please update the resources for \"#{formula.name}\" manually.
EOS
end
found_packages.merge!(JSON.parse(pipgrip_output).to_h) do |conflicting_package, old_version, new_version|
next old_version if old_version == new_version
JSON.parse(pipgrip_output).to_h.each do |new_name, new_version|
new_package = Package.new("#{new_name}==#{new_version}")
odie "Conflicting versions found for the `#{conflicting_package}` resource: #{old_version}, #{new_version}"
found_packages.each do |existing_package|
if existing_package.same_package?(new_package) && existing_package.version != new_package.version
odie "Conflicting versions found for the `#{new_package.name}` resource: "\
"#{existing_package.version}, #{new_package.version}"
end
end
found_packages << new_package unless found_packages.include? new_package
end
end
# Remove extra packages that may be included in pipgrip output
exclude_list = %W[#{package_name.split("[").first.downcase} argparse pip setuptools wheel wsgiref]
exclude_list = %W[#{main_package.name} argparse pip setuptools wheel wsgiref].map { |p| Package.new p }
found_packages.delete_if { |package| exclude_list.include? package }
new_resource_blocks = ""
found_packages.sort.each do |package, package_version|
found_packages.sort.each do |package|
if exclude_packages.include? package
ohai "Excluding \"#{package}==#{package_version}\"" if !print_only && !silent
ohai "Excluding \"#{package}\"" if !print_only && !silent
next
end
ohai "Getting PyPI info for \"#{package}==#{package_version}\"" if !print_only && !silent
name, url, checksum = get_pypi_info package, package_version
ohai "Getting PyPI info for \"#{package}\"" if !print_only && !silent
name, url, checksum = package.pypi_info
# Fail if unable to find name, url or checksum for any resource
if name.blank?
odie "Unable to resolve some dependencies. Please update the resources for \"#{formula.name}\" manually."