45978435e7
- Previously I thought that comments were fine to discourage people from wasting their time trying to bump things that used `undef` that Sorbet didn't support. But RuboCop is better at this since it'll complain if the comments are unnecessary. - Suggested in https://github.com/Homebrew/brew/pull/18018#issuecomment-2283369501. - I've gone for a mixture of `rubocop:disable` for the files that can't be `typed: strict` (use of undef, required before everything else, etc) and `rubocop:todo` for everything else that should be tried to make strictly typed. There's no functional difference between the two as `rubocop:todo` is `rubocop:disable` with a different name. - And I entirely disabled the cop for the docs/ directory since `typed: strict` isn't going to gain us anything for some Markdown linting config files. - This means that now it's easier to track what needs to be done rather than relying on checklists of files in our big Sorbet issue: ```shell $ git grep 'typed: true # rubocop:todo Sorbet/StrictSigil' | wc -l 268 ``` - And this is confirmed working for new files: ```shell $ git status On branch use-rubocop-for-sorbet-strict-sigils Untracked files: (use "git add <file>..." to include in what will be committed) Library/Homebrew/bad.rb Library/Homebrew/good.rb nothing added to commit but untracked files present (use "git add" to track) $ brew style Offenses: bad.rb:1:1: C: Sorbet/StrictSigil: Sorbet sigil should be at least strict got true. ^^^^^^^^^^^^^ 1340 files inspected, 1 offense detected ```
492 lines
17 KiB
Ruby
492 lines
17 KiB
Ruby
# typed: true # rubocop:todo Sorbet/StrictSigil
|
|
# frozen_string_literal: true
|
|
|
|
require "context"
|
|
require "erb"
|
|
require "settings"
|
|
require "extend/cachable"
|
|
|
|
module Utils
|
|
# Helper module for fetching and reporting analytics data.
|
|
module Analytics
|
|
INFLUX_BUCKET = "analytics"
|
|
INFLUX_TOKEN = "iVdsgJ_OjvTYGAA79gOfWlA_fX0QCuj4eYUNdb-qVUTrC3tp3JTWCADVNE9HxV0kp2ZjIK9tuthy_teX4szr9A=="
|
|
INFLUX_HOST = "https://eu-central-1-1.aws.cloud2.influxdata.com"
|
|
INFLUX_ORG = "d81a3e6d582d485f"
|
|
|
|
extend Cachable
|
|
|
|
class << self
|
|
include Context
|
|
|
|
sig {
|
|
params(measurement: Symbol,
|
|
tags: T::Hash[Symbol, T.any(T::Boolean, String)],
|
|
fields: T::Hash[Symbol, T.any(T::Boolean, String)]).void
|
|
}
|
|
def report_influx(measurement, tags, fields)
|
|
return if not_this_run? || disabled?
|
|
|
|
# Tags are always implicitly strings and must have low cardinality.
|
|
tags_string = tags.map { |k, v| "#{k}=#{v}" }
|
|
.join(",")
|
|
|
|
# Fields need explicitly wrapped with quotes and can have high cardinality.
|
|
fields_string = fields.compact
|
|
.map { |k, v| %Q(#{k}="#{v}") }
|
|
.join(",")
|
|
|
|
args = [
|
|
"--max-time", "3",
|
|
"--header", "Authorization: Token #{INFLUX_TOKEN}",
|
|
"--header", "Content-Type: text/plain; charset=utf-8",
|
|
"--header", "Accept: application/json",
|
|
"--data-binary", "#{measurement},#{tags_string} #{fields_string} #{Time.now.to_i}"
|
|
]
|
|
|
|
# Second precision is highest we can do and has the lowest performance cost.
|
|
url = "#{INFLUX_HOST}/api/v2/write?bucket=#{INFLUX_BUCKET}&precision=s"
|
|
deferred_curl(url, args)
|
|
end
|
|
|
|
sig { params(url: String, args: T::Array[String]).void }
|
|
def deferred_curl(url, args)
|
|
require "utils/curl"
|
|
|
|
curl = Utils::Curl.curl_executable
|
|
args = Utils::Curl.curl_args(*args, "--silent", "--output", "/dev/null", show_error: false)
|
|
if ENV["HOMEBREW_ANALYTICS_DEBUG"]
|
|
puts "#{curl} #{args.join(" ")} \"#{url}\""
|
|
puts Utils.popen_read(curl, *args, url)
|
|
else
|
|
pid = spawn curl, *args, url
|
|
Process.detach T.must(pid)
|
|
end
|
|
end
|
|
|
|
sig {
|
|
params(measurement: Symbol, package_name: String, tap_name: String,
|
|
on_request: T::Boolean, options: String).void
|
|
}
|
|
def report_package_event(measurement, package_name:, tap_name:, on_request: false, options: "")
|
|
return if not_this_run? || disabled?
|
|
|
|
# ensure on_request is a boolean
|
|
on_request = on_request ? true : false
|
|
|
|
# ensure options are removed (by `.compact` below) if empty
|
|
options = nil if options.blank?
|
|
|
|
# Tags must have low cardinality.
|
|
tags = default_package_tags.merge(on_request:)
|
|
|
|
# Fields can have high cardinality.
|
|
fields = default_package_fields.merge(package: package_name, tap_name:, options:)
|
|
.compact
|
|
|
|
report_influx(measurement, tags, fields)
|
|
end
|
|
|
|
sig { params(exception: BuildError).void }
|
|
def report_build_error(exception)
|
|
return if not_this_run? || disabled?
|
|
|
|
formula = exception.formula
|
|
return unless formula
|
|
|
|
tap = formula.tap
|
|
return unless tap
|
|
return unless tap.should_report_analytics?
|
|
|
|
options = exception.options.to_a.compact.map(&:to_s).sort.uniq.join(" ")
|
|
report_package_event(:build_error, package_name: formula.name, tap_name: tap.name, options:)
|
|
end
|
|
|
|
sig { params(command_instance: Homebrew::AbstractCommand).void }
|
|
def report_command_run(command_instance)
|
|
return if not_this_run? || disabled?
|
|
|
|
command = command_instance.class.command_name
|
|
|
|
options_array = command_instance.args.options_only.to_a.compact
|
|
|
|
# Strip out any flag values to reduce cardinality and preserve privacy.
|
|
options_array.map! { |option| option.sub(/=.*/m, "=") }
|
|
|
|
# Strip out --with-* and --without-* options
|
|
options_array.reject! { |option| option.match(/^--with(out)?-/) }
|
|
|
|
options = options_array.sort.uniq.join(" ")
|
|
|
|
# Tags must have low cardinality.
|
|
tags = {
|
|
command:,
|
|
ci: ENV["CI"].present?,
|
|
devcmdrun: Homebrew::EnvConfig.devcmdrun?,
|
|
developer: Homebrew::EnvConfig.developer?,
|
|
}
|
|
|
|
# Fields can have high cardinality.
|
|
fields = { options: }
|
|
|
|
report_influx(:command_run, tags, fields)
|
|
end
|
|
|
|
sig { params(step_command_short: String, passed: T::Boolean).void }
|
|
def report_test_bot_test(step_command_short, passed)
|
|
return if not_this_run? || disabled?
|
|
return if ENV["HOMEBREW_TEST_BOT_ANALYTICS"].blank?
|
|
|
|
# ensure passed is a boolean
|
|
passed = passed ? true : false
|
|
|
|
# Tags must have low cardinality.
|
|
tags = {
|
|
passed:,
|
|
arch: HOMEBREW_PHYSICAL_PROCESSOR,
|
|
os: HOMEBREW_SYSTEM,
|
|
}
|
|
|
|
# Strip out any flag values to reduce cardinality and preserve privacy.
|
|
# Sort options to ensure consistent ordering and improve readability.
|
|
command_and_package, options =
|
|
step_command_short.split
|
|
.map { |arg| arg.sub(/=.*/, "=") }
|
|
.partition { |arg| !arg.start_with?("-") }
|
|
command = (command_and_package + options.sort).join(" ")
|
|
|
|
# Fields can have high cardinality.
|
|
fields = { command: }
|
|
|
|
report_influx(:test_bot_test, tags, fields)
|
|
end
|
|
|
|
def influx_message_displayed?
|
|
config_true?(:influxanalyticsmessage)
|
|
end
|
|
|
|
def messages_displayed?
|
|
config_true?(:analyticsmessage) &&
|
|
config_true?(:caskanalyticsmessage) &&
|
|
influx_message_displayed?
|
|
end
|
|
|
|
def disabled?
|
|
return true if Homebrew::EnvConfig.no_analytics?
|
|
|
|
config_true?(:analyticsdisabled)
|
|
end
|
|
|
|
def not_this_run?
|
|
ENV["HOMEBREW_NO_ANALYTICS_THIS_RUN"].present?
|
|
end
|
|
|
|
def no_message_output?
|
|
# Used by Homebrew/install
|
|
ENV["HOMEBREW_NO_ANALYTICS_MESSAGE_OUTPUT"].present?
|
|
end
|
|
|
|
def messages_displayed!
|
|
Homebrew::Settings.write :analyticsmessage, true
|
|
Homebrew::Settings.write :caskanalyticsmessage, true
|
|
Homebrew::Settings.write :influxanalyticsmessage, true
|
|
end
|
|
|
|
def enable!
|
|
Homebrew::Settings.write :analyticsdisabled, false
|
|
delete_uuid!
|
|
messages_displayed!
|
|
end
|
|
|
|
def disable!
|
|
Homebrew::Settings.write :analyticsdisabled, true
|
|
delete_uuid!
|
|
end
|
|
|
|
def delete_uuid!
|
|
Homebrew::Settings.delete :analyticsuuid
|
|
end
|
|
|
|
def output(args:, filter: nil)
|
|
require "api"
|
|
|
|
days = args.days || "30"
|
|
category = args.category || "install"
|
|
begin
|
|
json = Homebrew::API::Analytics.fetch category, days
|
|
rescue ArgumentError
|
|
# Ignore failed API requests
|
|
return
|
|
end
|
|
return if json.blank? || json["items"].blank?
|
|
|
|
os_version = category == "os-version"
|
|
cask_install = category == "cask-install"
|
|
results = {}
|
|
json["items"].each do |item|
|
|
key = if os_version
|
|
item["os_version"]
|
|
elsif cask_install
|
|
item["cask"]
|
|
else
|
|
item["formula"]
|
|
end
|
|
next if filter.present? && key != filter && !key.start_with?("#{filter} ")
|
|
|
|
results[key] = item["count"].tr(",", "").to_i
|
|
end
|
|
|
|
if filter.present? && results.blank?
|
|
onoe "No results matching `#{filter}` found!"
|
|
return
|
|
end
|
|
|
|
table_output(category, days, results, os_version:, cask_install:)
|
|
end
|
|
|
|
def output_analytics(json, args:)
|
|
full_analytics = args.analytics? || verbose?
|
|
|
|
ohai "Analytics"
|
|
json["analytics"].each do |category, value|
|
|
category = category.tr("_", "-")
|
|
analytics = []
|
|
|
|
value.each do |days, results|
|
|
days = days.to_i
|
|
if full_analytics
|
|
next if args.days.present? && args.days&.to_i != days
|
|
next if args.category.present? && args.category != category
|
|
|
|
table_output(category, days, results)
|
|
else
|
|
total_count = results.values.inject("+")
|
|
analytics << "#{number_readable(total_count)} (#{days} days)"
|
|
end
|
|
end
|
|
|
|
puts "#{category}: #{analytics.join(", ")}" unless full_analytics
|
|
end
|
|
end
|
|
|
|
# This method is undocumented because it is not intended for general use.
|
|
# It relies on screen scraping some GitHub HTML that's not available as an API.
|
|
# This seems very likely to break in the future.
|
|
# That said, it's the only way to get the data we want right now.
|
|
def output_github_packages_downloads(formula, args:)
|
|
return unless args.github_packages_downloads?
|
|
return unless formula.core_formula?
|
|
|
|
require "utils/curl"
|
|
|
|
escaped_formula_name = GitHubPackages.image_formula_name(formula.name)
|
|
.gsub("/", "%2F")
|
|
formula_url_suffix = "container/core%2F#{escaped_formula_name}/"
|
|
formula_url = "https://github.com/Homebrew/homebrew-core/pkgs/#{formula_url_suffix}"
|
|
output = Utils::Curl.curl_output("--fail", formula_url)
|
|
return unless output.success?
|
|
|
|
formula_version_urls = output.stdout
|
|
.scan(%r{/orgs/Homebrew/packages/#{formula_url_suffix}\d+\?tag=[^"]+})
|
|
.map do |url|
|
|
url.sub("/orgs/Homebrew/packages/", "/Homebrew/homebrew-core/pkgs/")
|
|
end
|
|
return if formula_version_urls.empty?
|
|
|
|
thirty_day_download_count = 0
|
|
formula_version_urls.each do |formula_version_url_suffix|
|
|
formula_version_url = "https://github.com#{formula_version_url_suffix}"
|
|
output = Utils::Curl.curl_output("--fail", formula_version_url)
|
|
next unless output.success?
|
|
|
|
last_thirty_days_match = output.stdout.match(
|
|
%r{<span class="[\s\-a-z]*">Last 30 days</span>\s*<span class="[\s\-a-z]*">([\d.M,]+)</span>}m,
|
|
)
|
|
next if last_thirty_days_match.blank?
|
|
|
|
last_thirty_days_downloads = last_thirty_days_match.captures.first.tr(",", "")
|
|
thirty_day_download_count += if (millions_match = last_thirty_days_downloads.match(/(\d+\.\d+)M/).presence)
|
|
millions_match.captures.first.to_f * 1_000_000
|
|
else
|
|
last_thirty_days_downloads.to_i
|
|
end
|
|
end
|
|
|
|
ohai "GitHub Packages Downloads"
|
|
puts "#{number_readable(thirty_day_download_count)} (30 days)"
|
|
end
|
|
|
|
def formula_output(formula, args:)
|
|
return if Homebrew::EnvConfig.no_analytics? || Homebrew::EnvConfig.no_github_api?
|
|
|
|
require "api"
|
|
|
|
json = Homebrew::API::Formula.fetch formula.name
|
|
return if json.blank? || json["analytics"].blank?
|
|
|
|
output_analytics(json, args:)
|
|
output_github_packages_downloads(formula, args:)
|
|
rescue ArgumentError
|
|
# Ignore failed API requests
|
|
nil
|
|
end
|
|
|
|
def cask_output(cask, args:)
|
|
return if Homebrew::EnvConfig.no_analytics? || Homebrew::EnvConfig.no_github_api?
|
|
|
|
require "api"
|
|
|
|
json = Homebrew::API::Cask.fetch cask.token
|
|
return if json.blank? || json["analytics"].blank?
|
|
|
|
output_analytics(json, args:)
|
|
rescue ArgumentError
|
|
# Ignore failed API requests
|
|
nil
|
|
end
|
|
|
|
sig { returns(T::Hash[Symbol, String]) }
|
|
def default_package_tags
|
|
cache[:default_package_tags] ||= begin
|
|
# Only display default prefixes to reduce cardinality and improve privacy
|
|
prefix = Homebrew.default_prefix? ? HOMEBREW_PREFIX.to_s : "custom-prefix"
|
|
|
|
# Tags are always strings and must have low cardinality.
|
|
{
|
|
ci: ENV["CI"].present?,
|
|
prefix:,
|
|
default_prefix: Homebrew.default_prefix?,
|
|
developer: Homebrew::EnvConfig.developer?,
|
|
devcmdrun: Homebrew::EnvConfig.devcmdrun?,
|
|
arch: HOMEBREW_PHYSICAL_PROCESSOR,
|
|
os: HOMEBREW_SYSTEM,
|
|
}
|
|
end
|
|
end
|
|
|
|
# remove os_version starting with " or number
|
|
# remove macOS patch release
|
|
sig { returns(T::Hash[Symbol, String]) }
|
|
def default_package_fields
|
|
cache[:default_package_fields] ||= begin
|
|
version = if (match_data = HOMEBREW_VERSION.match(/^[\d.]+/))
|
|
suffix = "-dev" if HOMEBREW_VERSION.include?("-")
|
|
match_data[0] + suffix.to_s
|
|
else
|
|
">=4.1.22"
|
|
end
|
|
|
|
# Only include OS versions with an actual name.
|
|
os_name_and_version = if (os_version = OS_VERSION.presence) && os_version.downcase.match?(/^[a-z]/)
|
|
os_version
|
|
end
|
|
|
|
{
|
|
version:,
|
|
os_name_and_version:,
|
|
}
|
|
end
|
|
end
|
|
|
|
def table_output(category, days, results, os_version: false, cask_install: false)
|
|
oh1 "#{category} (#{days} days)"
|
|
total_count = results.values.inject("+")
|
|
formatted_total_count = format_count(total_count)
|
|
formatted_total_percent = format_percent(100)
|
|
|
|
index_header = "Index"
|
|
count_header = "Count"
|
|
percent_header = "Percent"
|
|
name_with_options_header = if os_version
|
|
"macOS Version"
|
|
elsif cask_install
|
|
"Token"
|
|
else
|
|
"Name (with options)"
|
|
end
|
|
|
|
total_index_footer = "Total"
|
|
max_index_width = results.length.to_s.length
|
|
index_width = [
|
|
index_header.length,
|
|
total_index_footer.length,
|
|
max_index_width,
|
|
].max
|
|
count_width = [
|
|
count_header.length,
|
|
formatted_total_count.length,
|
|
].max
|
|
percent_width = [
|
|
percent_header.length,
|
|
formatted_total_percent.length,
|
|
].max
|
|
name_with_options_width = Tty.width -
|
|
index_width -
|
|
count_width -
|
|
percent_width -
|
|
10 # spacing and lines
|
|
|
|
formatted_index_header =
|
|
format "%#{index_width}s", index_header
|
|
formatted_name_with_options_header =
|
|
format "%-#{name_with_options_width}s",
|
|
name_with_options_header[0..name_with_options_width-1]
|
|
formatted_count_header =
|
|
format "%#{count_width}s", count_header
|
|
formatted_percent_header =
|
|
format "%#{percent_width}s", percent_header
|
|
puts "#{formatted_index_header} | #{formatted_name_with_options_header} | " \
|
|
"#{formatted_count_header} | #{formatted_percent_header}"
|
|
|
|
columns_line = "#{"-"*index_width}:|-#{"-"*name_with_options_width}-|-" \
|
|
"#{"-"*count_width}:|-#{"-"*percent_width}:"
|
|
puts columns_line
|
|
|
|
index = 0
|
|
results.each do |name_with_options, count|
|
|
index += 1
|
|
formatted_index = format "%0#{max_index_width}d", index
|
|
formatted_index = format "%-#{index_width}s", formatted_index
|
|
formatted_name_with_options =
|
|
format "%-#{name_with_options_width}s",
|
|
name_with_options[0..name_with_options_width-1]
|
|
formatted_count = format "%#{count_width}s", format_count(count)
|
|
formatted_percent = if total_count.zero?
|
|
format "%#{percent_width}s", format_percent(0)
|
|
else
|
|
format "%#{percent_width}s",
|
|
format_percent((count.to_i * 100) / total_count.to_f)
|
|
end
|
|
puts "#{formatted_index} | #{formatted_name_with_options} | " \
|
|
"#{formatted_count} | #{formatted_percent}%"
|
|
next if index > 10
|
|
end
|
|
return if results.length <= 1
|
|
|
|
formatted_total_footer =
|
|
format "%-#{index_width}s", total_index_footer
|
|
formatted_blank_footer =
|
|
format "%-#{name_with_options_width}s", ""
|
|
formatted_total_count_footer =
|
|
format "%#{count_width}s", formatted_total_count
|
|
formatted_total_percent_footer =
|
|
format "%#{percent_width}s", formatted_total_percent
|
|
puts "#{formatted_total_footer} | #{formatted_blank_footer} | " \
|
|
"#{formatted_total_count_footer} | #{formatted_total_percent_footer}%"
|
|
end
|
|
|
|
def config_true?(key)
|
|
Homebrew::Settings.read(key) == "true"
|
|
end
|
|
|
|
def format_count(count)
|
|
count.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
|
|
end
|
|
|
|
def format_percent(percent)
|
|
format("%<percent>.2f", percent:)
|
|
end
|
|
end
|
|
end
|
|
end
|