Create Utils::Curl module and explicitly include it.

This commit is contained in:
Markus Reiter 2020-10-10 15:23:03 +02:00
parent 318091cccc
commit ff653571b1
5 changed files with 274 additions and 263 deletions

View File

@ -11,6 +11,8 @@ require "lock_file"
require "mechanize/version" require "mechanize/version"
require "mechanize/http/content_disposition_parser" require "mechanize/http/content_disposition_parser"
require "utils/curl"
# @abstract Abstract superclass for all download strategies. # @abstract Abstract superclass for all download strategies.
# #
# @api private # @api private
@ -328,6 +330,8 @@ end
# #
# @api public # @api public
class CurlDownloadStrategy < AbstractFileDownloadStrategy class CurlDownloadStrategy < AbstractFileDownloadStrategy
include Utils::Curl
attr_reader :mirrors attr_reader :mirrors
def initialize(url, name, version, **meta) def initialize(url, name, version, **meta)

View File

@ -1,15 +0,0 @@
# typed: strict
module SharedAudits
def github(user, repo)
end
def gitlab(user, repo)
end
def bitbucket(user, repo)
end
def curl_output(*args, secrets: [], **options)
end
end

View File

@ -3,7 +3,14 @@
require "open3" require "open3"
def curl_executable module Utils
# Helper function for interacting with `curl`.
#
# @api private
module Curl
module_function
def curl_executable
@curl ||= [ @curl ||= [
ENV["HOMEBREW_CURL"], ENV["HOMEBREW_CURL"],
which("curl"), which("curl"),
@ -12,9 +19,9 @@ def curl_executable
raise "no executable curl was found" unless @curl raise "no executable curl was found" unless @curl
@curl @curl
end end
def curl_args(*extra_args, show_output: false, user_agent: :default) def curl_args(*extra_args, show_output: false, user_agent: :default)
args = [] args = []
# do not load .curlrc unless requested (must be the first argument) # do not load .curlrc unless requested (must be the first argument)
@ -45,9 +52,11 @@ def curl_args(*extra_args, show_output: false, user_agent: :default)
args << "--retry" << Homebrew::EnvConfig.curl_retries args << "--retry" << Homebrew::EnvConfig.curl_retries
args + extra_args args + extra_args
end end
def curl_with_workarounds(*args, secrets: nil, print_stdout: nil, print_stderr: nil, verbose: nil, env: {}, **options) def curl_with_workarounds(
*args, secrets: nil, print_stdout: nil, print_stderr: nil, verbose: nil, env: {}, **options
)
command_options = { command_options = {
secrets: secrets, secrets: secrets,
print_stdout: print_stdout, print_stdout: print_stdout,
@ -83,15 +92,15 @@ def curl_with_workarounds(*args, secrets: nil, print_stdout: nil, print_stderr:
end end
result result
end end
def curl(*args, print_stdout: true, **options) def curl(*args, print_stdout: true, **options)
result = curl_with_workarounds(*args, print_stdout: print_stdout, **options) result = curl_with_workarounds(*args, print_stdout: print_stdout, **options)
result.assert_success! result.assert_success!
result result
end end
def curl_download(*args, to: nil, partial: true, **options) def curl_download(*args, to: nil, partial: true, **options)
destination = Pathname(to) destination = Pathname(to)
destination.dirname.mkpath destination.dirname.mkpath
@ -118,28 +127,30 @@ def curl_download(*args, to: nil, partial: true, **options)
0 0
end end
curl("--location", "--remote-time", "--continue-at", continue_at.to_s, "--output", destination, *args, **options) curl(
end "--location", "--remote-time", "--continue-at", continue_at.to_s, "--output", destination, *args, **options
)
end
def curl_output(*args, **options) def curl_output(*args, **options)
curl_with_workarounds(*args, print_stderr: false, show_output: true, **options) curl_with_workarounds(*args, print_stderr: false, show_output: true, **options)
end end
# Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io). # Check if a URL is protected by CloudFlare (e.g. badlion.net and jaxx.io).
def url_protected_by_cloudflare?(details) def url_protected_by_cloudflare?(details)
[403, 503].include?(details[:status].to_i) && [403, 503].include?(details[:status].to_i) &&
details[:headers].match?(/^Set-Cookie: __cfduid=/i) && details[:headers].match?(/^Set-Cookie: __cfduid=/i) &&
details[:headers].match?(/^Server: cloudflare/i) details[:headers].match?(/^Server: cloudflare/i)
end end
# Check if a URL is protected by Incapsula (e.g. corsair.com). # Check if a URL is protected by Incapsula (e.g. corsair.com).
def url_protected_by_incapsula?(details) def url_protected_by_incapsula?(details)
details[:status].to_i == 403 && details[:status].to_i == 403 &&
details[:headers].match?(/^Set-Cookie: visid_incap_/i) && details[:headers].match?(/^Set-Cookie: visid_incap_/i) &&
details[:headers].match?(/^Set-Cookie: incap_ses_/i) details[:headers].match?(/^Set-Cookie: incap_ses_/i)
end end
def curl_check_http_content(url, user_agents: [:default], check_content: false, strict: false) def curl_check_http_content(url, user_agents: [:default], check_content: false, strict: false)
return unless url.start_with? "http" return unless url.start_with? "http"
details = nil details = nil
@ -218,9 +229,9 @@ def curl_check_http_content(url, user_agents: [:default], check_content: false,
return unless (90..110).cover?(lenratio) return unless (90..110).cover?(lenratio)
"The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser." "The URL #{url} may be able to use HTTPS rather than HTTP. Please verify it in a browser."
end end
def curl_http_content_headers_and_checksum(url, hash_needed: false, user_agent: :default) def curl_http_content_headers_and_checksum(url, hash_needed: false, user_agent: :default)
file = Tempfile.new.tap(&:close) file = Tempfile.new.tap(&:close)
max_time = hash_needed ? "600" : "25" max_time = hash_needed ? "600" : "25"
@ -252,10 +263,15 @@ def curl_http_content_headers_and_checksum(url, hash_needed: false, user_agent:
file_hash: output_hash, file_hash: output_hash,
file: output, file: output,
} }
ensure ensure
file.unlink file.unlink
end
def http_status_ok?(status)
(100..299).cover?(status.to_i)
end
end
end end
def http_status_ok?(status) # FIXME: Include `Utils::Curl` explicitly everywhere it is used.
(100..299).cover?(status.to_i) include Utils::Curl # rubocop:disable Style/MixinUsage
end

View File

@ -7,6 +7,9 @@ require "utils/curl"
# #
# @api private # @api private
module SharedAudits module SharedAudits
include Utils::Curl
extend Utils::Curl
module_function module_function
def github_repo_data(user, repo) def github_repo_data(user, repo)

View File

@ -1,12 +1,15 @@
# typed: true # typed: true
# frozen_string_literal: true # frozen_string_literal: true
require "utils/curl"
require "utils/github" require "utils/github"
# Helper module for updating SPDX license data. # Helper module for updating SPDX license data.
# #
# @api private # @api private
module SPDX module SPDX
extend Utils::Curl
module_function module_function
DATA_PATH = (HOMEBREW_DATA_PATH/"spdx").freeze DATA_PATH = (HOMEBREW_DATA_PATH/"spdx").freeze