analytics: switch to InfluxDB for logging

Library/Homebrew/cask/installer.rb

@@ -110,7 +110,7 @@ module Cask
 
       install_artifacts
 
-      ::Utils::Analytics.report_event("cask_install", @cask.token) unless @cask.tap&.private?
+      ::Utils::Analytics.report_event("cask_install", @cask.token, on_request: true) unless @cask.tap&.private?
 
       purge_backed_up_versioned_files
 

Library/Homebrew/extend/os/mac/utils/analytics.rb

@@ -6,15 +6,15 @@ module Utils
     class << self
       extend T::Sig
 
-      sig { returns(String) }
+      sig { params(verbose: T::Boolean).returns(String) }
-      def custom_prefix_label
+      def custom_prefix_label(verbose: false)
         return generic_custom_prefix_label if Hardware::CPU.arm?
 
         "non-/usr/local"
       end
 
-      sig { returns(String) }
+      sig { params(verbose: T::Boolean).returns(String) }
-      def arch_label
+      def arch_label(verbose: false)
         if Hardware::CPU.arm?
           "ARM"
         elsif Hardware::CPU.in_rosetta2?

Library/Homebrew/formula_installer.rb

@@ -416,9 +416,7 @@ class FormulaInstaller
 
     if formula.tap&.installed? && !formula.tap&.private?
       action = "#{formula.full_name} #{options}".strip
-      Utils::Analytics.report_event("install", action)
+      Utils::Analytics.report_event("install", action, on_request: installed_on_request?)
-
-      Utils::Analytics.report_event("install_on_request", action) if installed_on_request?
     end
 
     self.class.attempted << formula

Library/Homebrew/test/utils/analytics_spec.rb

@@ -17,61 +17,110 @@ describe Utils::Analytics do
         allow(Hardware::CPU).to receive(:type).and_return(:intel)
         allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
         allow(Homebrew).to receive(:default_prefix?).and_return(false)
-        expected = "#{OS_VERSION}, #{described_class.custom_prefix_label}#{ci}"
+        expect(described_class.os_arch_prefix_ci).to have_key(:prefix)
-        expect(described_class.os_arch_prefix_ci).to eq expected
+        expect(described_class.os_arch_prefix_ci[:prefix]).to eq described_class.custom_prefix_label
       end
 
       it "returns OS_VERSION, ARM and prefix when HOMEBREW_PREFIX is a custom prefix on arm" do
         allow(Hardware::CPU).to receive(:type).and_return(:arm)
         allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(false)
         allow(Homebrew).to receive(:default_prefix?).and_return(false)
-        expected = "#{OS_VERSION}, ARM, #{described_class.custom_prefix_label}#{ci}"
+        expect(described_class.os_arch_prefix_ci).to have_key(:arch)
-        expect(described_class.os_arch_prefix_ci).to eq expected
+        expect(described_class.os_arch_prefix_ci[:arch]).to eq described_class.arch_label
+        expect(described_class.os_arch_prefix_ci).to have_key(:prefix)
+        expect(described_class.os_arch_prefix_ci[:prefix]).to eq described_class.custom_prefix_label
       end
 
       it "returns OS_VERSION, Rosetta and prefix when HOMEBREW_PREFIX is a custom prefix on Rosetta", :needs_macos do
         allow(Hardware::CPU).to receive(:type).and_return(:intel)
         allow(Hardware::CPU).to receive(:in_rosetta2?).and_return(true)
         allow(Homebrew).to receive(:default_prefix?).and_return(false)
-        expected = "#{OS_VERSION}, Rosetta, #{described_class.custom_prefix_label}#{ci}"
+        expect(described_class.os_arch_prefix_ci).to have_key(:prefix)
-        expect(described_class.os_arch_prefix_ci).to eq expected
+        expect(described_class.os_arch_prefix_ci[:prefix]).to eq described_class.custom_prefix_label
       end
 
       it "does not include prefix when HOMEBREW_PREFIX is the default prefix" do
         allow(Homebrew).to receive(:default_prefix?).and_return(true)
-        expect(described_class.os_arch_prefix_ci).not_to include(described_class.custom_prefix_label)
+        expect(described_class.os_arch_prefix_ci).not_to have_key(:prefix)
       end
 
       it "includes CI when ENV['CI'] is set" do
         ENV["CI"] = "true"
-        expect(described_class.os_arch_prefix_ci).to include("CI")
+        expect(described_class.os_arch_prefix_ci).to have_key(:ci)
       end
     end
   end
 
   describe "::report_event" do
     let(:f) { formula { url "foo-1.0" } }
-    let(:options) { FormulaInstaller.new(f).display_options(f) }
+    let(:options) { ["--head"].join }
     let(:action)  { "#{f.full_name} #{options}".strip }
 
     context "when ENV vars is set" do
       it "returns nil when HOMEBREW_NO_ANALYTICS is true" do
         ENV["HOMEBREW_NO_ANALYTICS"] = "true"
-        expect(described_class.report_event("install", action)).to be_nil
+        expect(described_class).not_to receive(:report_google)
+        expect(described_class).not_to receive(:report_influx)
+        described_class.report_event("install", action)
       end
 
       it "returns nil when HOMEBREW_NO_ANALYTICS_THIS_RUN is true" do
         ENV["HOMEBREW_NO_ANALYTICS_THIS_RUN"] = "true"
-        expect(described_class.report_event("install", action)).to be_nil
+        expect(described_class).not_to receive(:report_google)
+        expect(described_class).not_to receive(:report_influx)
+        described_class.report_event("install", action)
       end
 
       it "returns nil when HOMEBREW_ANALYTICS_DEBUG is true" do
         ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
         ENV.delete("HOMEBREW_NO_ANALYTICS")
         ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
-        expect(described_class.report_event("install", action)).to be_nil
+        expect(described_class).to receive(:report_google)
+        expect(described_class).to receive(:report_influx)
+
+        described_class.report_event("install", action)
       end
     end
+
+    it "passes to the influxdb and google methods" do
+      ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
+      ENV.delete("HOMEBREW_NO_ANALYTICS")
+      ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+      expect(described_class).to receive(:report_google).with(:event, hash_including(ea: action)).once
+      expect(described_class).to receive(:report_influx).with(:install, "formula_name --head", false,
+                                                              hash_including(developer: false)).once
+      described_class.report_event(:install, action)
+    end
+
+    it "sends to google twice on request" do
+      ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
+      ENV.delete("HOMEBREW_NO_ANALYTICS")
+      ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+      expect(described_class).to receive(:report_google).with(:event, hash_including(ea: action, ec: :install)).once
+      expect(described_class).to receive(:report_google).with(:event,
+                                                              hash_including(ea: action,
+                                                                             ec: :install_on_request)).once
+      expect(described_class).to receive(:report_influx).with(:install, "formula_name --head", true,
+                                                              hash_including(developer: false)).once
+
+      described_class.report_event(:install, action, on_request: true)
+    end
+  end
+
+  describe "::report_influx" do
+    let(:f) { formula { url "foo-1.0" } }
+    let(:options) { ["--head"].join }
+    let(:action)  { "#{f.full_name} #{options}".strip }
+
+    it "outputs in debug mode" do
+      ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
+      ENV.delete("HOMEBREW_NO_ANALYTICS")
+      ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+      ENV["HOMEBREW_ANALYTICS_ENABLE_INFLUX"] = "true"
+      expect do
+        described_class.report_influx(:install, action, true, developer: true, CI: true)
+      end.to output(/--data-raw install,[a-zA-Z=,]*,action=formula_name\\ --head/).to_stdout
+    end
   end
 
   describe "::report_build_error" do
@@ -86,7 +135,9 @@ describe Utils::Analytics do
       end
 
       it "does not report event if BuildError raised for a formula with a private remote repository" do
-        expect(described_class.report_build_error(err)).to be_nil
+        allow_any_instance_of(Tap).to receive(:private?).and_return(true)
+        expect(described_class).not_to receive(:report_event)
+        described_class.report_build_error(err)
       end
     end
 
@@ -95,7 +146,8 @@ describe Utils::Analytics do
       let(:f) { double(Formula, name: "foo", path: "blah", tap: nil) }
 
       it "does not report event if BuildError is raised" do
-        expect(described_class.report_build_error(err)).to be_nil
+        expect(described_class).not_to receive(:report_event)
+        described_class.report_build_error(err)
       end
     end
 
@@ -105,7 +157,8 @@ describe Utils::Analytics do
 
       it "does not report event if BuildError is raised" do
         allow_any_instance_of(Pathname).to receive(:directory?).and_return(false)
-        expect(described_class.report_build_error(err)).to be_nil
+        expect(described_class).not_to receive(:report_event)
+        described_class.report_build_error(err)
       end
     end
   end

Library/Homebrew/utils/analytics.rb

@@ -16,9 +16,18 @@ module Utils
 
       include Context
 
-      def report(type, metadata = {})
+      INFLUX_BUCKET = "analytics"
-        return if not_this_run?
+      INFLUX_TOKEN = "y2JZsgE7glWT9V-S-nElETLp8oyH9PGh9JVa-kCdOdp7mEHIOws4BtdjsIe3HHpCBty7IQHLnmh0prqK2vBj9A=="
-        return if disabled?
+      INFLUX_HOST = "europe-west1-1.gcp.cloud2.influxdata.com"
+
+      sig { params(type: T.any(String, Symbol), metadata: T::Hash[Symbol, T.untyped]).void }
+      def report_google(type, metadata = {})
+        os = metadata[:el][:os]
+        arch = ", #{metadata[:el][:arch]}" if metadata[:el][:arch].present?
+        prefix = ", #{metadata[:el][:prefix]}" if metadata[:el][:prefix].present?
+        ci = ", CI" if metadata[:el][:CI] == true
+
+        metadata[:el] = "#{os}#{arch}#{prefix}#{ci}"
 
         analytics_ids = ENV.fetch("HOMEBREW_ANALYTICS_IDS", "").split(",")
         analytics_ids.each do |analytics_id|
@@ -61,7 +70,7 @@ module Utils
             pid = fork do
               exec curl, *args,
                    "--silent", "--output", "/dev/null",
-                   "https://www.google-analytics.com/collect"
+                   "https://www.google-analytqics.com/collect"
             end
             Process.detach T.must(pid)
           end
@@ -69,12 +78,63 @@ module Utils
         nil
       end
 
-      def report_event(category, action, label = os_arch_prefix_ci, value = nil)
+      sig {
-        report(:event,
+        params(category: T.any(String, Symbol), action: T.any(String, Symbol), on_request: T::Boolean,
-               ec: category,
+               additional_tags: T::Hash[Symbol, T.untyped]).void
-               ea: action,
+      }
-               el: label,
+      def report_influx(category, action, on_request, additional_tags = {})
-               ev: value)
+        return unless ENV["HOMEBREW_ANALYTICS_ENABLE_INFLUX"]
+
+        # Append general information to device information
+        tags = additional_tags.merge(action: action, on_request: !on_request.nil?)
+                              .compact_blank
+                              .map { |k, v| "#{k}=#{v.to_s.sub(" ", "\\ ")}" } # convert to key/value parameters
+                              .join(",")
+
+        args = [
+          "--max-time", "3",
+          "--header", "Content-Type: text/plain; charset=utf-8",
+          "--header", "Accept: application/json",
+          "--header", "Authorization: Token #{INFLUX_TOKEN}",
+          "--data-raw", "#{category},#{tags} count=1i #{Time.now.to_i}"
+        ]
+
+        curl = Utils::Curl.curl_executable
+        url = "https://#{INFLUX_HOST}/api/v2/write?bucket=#{INFLUX_BUCKET}&precision=s"
+        if ENV["HOMEBREW_ANALYTICS_DEBUG"]
+          puts "#{curl} #{args.join(" ")} \"#{url}\""
+          puts Utils.popen_read(curl, *args, url)
+        else
+          pid = fork do
+            exec curl, *args, "--silent", "--output", "/dev/null", url
+          end
+          Process.detach T.must(pid)
+        end
+      end
+
+      sig { params(category: T.any(String, Symbol), action: String, on_request: T::Boolean).void }
+      def report_event(category, action, on_request: false)
+        return if not_this_run?
+        return if disabled?
+
+        google_label = os_arch_prefix_ci(verbose: false)
+
+        report_google(:event,
+                      ec: category,
+                      ea: action,
+                      el: google_label,
+                      ev: nil)
+
+        if on_request
+          report_google(:event,
+                        ec: :install_on_request,
+                        ea: action,
+                        el: google_label,
+                        ev: nil)
+        end
+
+        influx_additional_data = os_arch_prefix_ci(verbose: true)
+        report_influx(category, action, on_request, influx_additional_data)
       end
 
       def report_build_error(exception)
@@ -216,14 +276,14 @@ module Utils
         nil
       end
 
-      sig { returns(String) }
+      sig { params(verbose: T::Boolean).returns(String) }
-      def custom_prefix_label
+      def custom_prefix_label(verbose: false)
         "custom-prefix"
       end
       alias generic_custom_prefix_label custom_prefix_label
 
-      sig { returns(String) }
+      sig { params(verbose: T::Boolean).returns(String) }
-      def arch_label
+      def arch_label(verbose: false)
         if Hardware::CPU.arm?
           "ARM"
         else
@@ -237,13 +297,24 @@ module Utils
         remove_instance_variable(:@os_arch_prefix_ci)
       end
 
-      def os_arch_prefix_ci
+      sig { params(verbose: T::Boolean).returns(T::Hash[Symbol, String]) }
+      def os_arch_prefix_ci(verbose: false)
         @os_arch_prefix_ci ||= begin
-          os = OS_VERSION
+          data = {
-          arch = ", #{arch_label}" if arch_label.present?
+            os:        OS_VERSION,
-          prefix = ", #{custom_prefix_label}" unless Homebrew.default_prefix?
+            developer: Homebrew::EnvConfig.developer?,
-          ci = ", CI" if ENV["CI"]
+            version:   HOMEBREW_VERSION,
-          "#{os}#{arch}#{prefix}#{ci}"
+            system:    HOMEBREW_SYSTEM,
+            ci:        ENV["CI"].present?,
+            arch:      arch_label(verbose: verbose),
+            prefix:    custom_prefix_label(verbose: verbose),
+          }
+          unless verbose
+            data.delete(:arch) if data[:arch].blank?
+            data.delete(:prefix) if Homebrew.default_prefix?
+          end
+
+          data
         end
       end