brew/Library/Homebrew/dev-cmd/formula-analytics.rb

# typed: strict
# frozen_string_literal: true

require "abstract_command"
require "fileutils"

module Homebrew
  module DevCmd
    class FormulaAnalytics < AbstractCommand
      cmd_args do
        usage_banner <<~EOS
          `formula-analytics`

          Query Homebrew's analytics.
        EOS
        flag   "--days-ago=",
               description: "Query from the specified days ago until the present. The default is 30 days."
        switch "--install",
               description: "Output the number of specifically requested installations or installation as " \
                            "dependencies of formulae. This is the default."
        switch "--install-on-request",
               description: "Output the number of specifically requested installations of formulae."
        switch "--cask-install",
               description: "Output the number of installations of casks."
        switch "--build-error",
               description: "Output the number of build errors for formulae."
        switch "--os-version",
               description: "Output the number of events by OS name and version."
        switch "--homebrew-devcmdrun-developer",
               description: "Output the number of devcmdrun/HOMEBREW_DEVELOPER events."
        switch "--homebrew-env-config",
               description: "Output rates of non-default Homebrew environment configuration variables."
        switch "--homebrew-os-arch-ci",
               description: "Output the number of OS/Architecture/CI events."
        switch "--homebrew-prefixes",
               description: "Output Homebrew prefixes."
        switch "--homebrew-versions",
               description: "Output Homebrew versions."
        switch "--brew-command-run",
               description: "Output `brew` commands run."
        switch "--brew-command-run-options",
               description: "Output `brew` commands run with options."
        switch "--brew-test-bot-test",
               description: "Output `brew test-bot` steps run."
        switch "--json",
               description: "Output JSON. This is required: plain text support has been removed."
        switch "--all-core-formulae-json",
               description: "Output a different JSON format containing the JSON data for all " \
                            "Homebrew/homebrew-core formulae."
        switch "--setup",
               description: "Install the necessary gems, require them and exit without running a query."

        conflicts "--install", "--cask-install", "--install-on-request", "--build-error", "--os-version",
                  "--homebrew-devcmdrun-developer", "--homebrew-env-config", "--homebrew-os-arch-ci",
                  "--homebrew-prefixes", "--homebrew-versions", "--brew-command-run", "--brew-command-run-options",
                  "--brew-test-bot-test"
        conflicts "--json", "--all-core-formulae-json", "--setup"

        named_args :none

        hide_from_man_page!
      end

      FIRST_INFLUXDB_ANALYTICS_DATE = Date.new(2023, 03, 27).freeze

      sig { override.void }
      def run
        Homebrew.install_bundler_gems!(groups: ["formula_analytics"])

        setup_python
        influx_analytics(args)
      end

      sig { void }
      def setup_python
        formula_analytics_root = HOMEBREW_LIBRARY/"Homebrew/formula-analytics"
        vendor_python =  Pathname.new("~/.brew-formula-analytics/vendor/python").expand_path
        python_version = (formula_analytics_root/".python-version").read.chomp

        which_python = which("python#{python_version}", ORIGINAL_PATHS)
        odie <<~EOS if which_python.nil?
          Python #{python_version} is required. Try:
            brew install python@#{python_version}
        EOS

        venv_root = vendor_python/python_version
        vendor_python.children.reject { |path| path == venv_root }.each(&:rmtree) if vendor_python.exist?
        venv_python = venv_root/"bin/python"

        repo_requirements = HOMEBREW_LIBRARY/"Homebrew/formula-analytics/requirements.txt"
        venv_requirements = venv_root/"requirements.txt"
        if !venv_requirements.exist? || !FileUtils.identical?(repo_requirements, venv_requirements)
          safe_system which_python, "-I", "-m", "venv", "--clear", venv_root, out: :err
          safe_system venv_python, "-m", "pip", "install",
                      "--disable-pip-version-check",
                      "--require-hashes",
                      "--requirement", repo_requirements,
                      out: :err
          FileUtils.cp repo_requirements, venv_requirements
        end

        ENV["PATH"] = "#{venv_root}/bin:#{ENV.fetch("PATH")}"
        ENV["__PYVENV_LAUNCHER__"] = venv_python.to_s # support macOS framework Pythons

        require "pycall"
        PyCall.init(venv_python)
        require formula_analytics_root/"pycall-setup"
      end

      sig { params(args: Homebrew::DevCmd::FormulaAnalytics::Args).void }
      def influx_analytics(args)
        require "utils/analytics"
        require "json"

        return if args.setup?

        odie "`$HOMEBREW_NO_ANALYTICS` is set!" if ENV["HOMEBREW_NO_ANALYTICS"]

        token = ENV.fetch("HOMEBREW_INFLUXDB_TOKEN", nil)
        odie "No InfluxDB credentials found in `$HOMEBREW_INFLUXDB_TOKEN`!" unless token

        client = InfluxDBClient3.new(
          token:,
          host:     URI.parse(Utils::Analytics::INFLUX_HOST).host,
          org:      Utils::Analytics::INFLUX_ORG,
          database: Utils::Analytics::INFLUX_BUCKET,
        )

        max_days_ago = (Date.today - FIRST_INFLUXDB_ANALYTICS_DATE).to_s.to_i
        days_ago = (args.days_ago || 30).to_i
        if days_ago > max_days_ago
          opoo "Analytics started #{FIRST_INFLUXDB_ANALYTICS_DATE}. `--days-ago` set to maximum value."
          days_ago = max_days_ago
        end
        if days_ago > 365
          opoo "Analytics are only retained for 1 year, setting `--days-ago=365`."
          days_ago = 365
        end

        all_core_formulae_json = args.all_core_formulae_json?

        categories = []
        categories << :build_error if args.build_error?
        categories << :cask_install if args.cask_install?
        categories << :formula_install if args.install?
        categories << :formula_install_on_request if args.install_on_request?
        categories << :homebrew_devcmdrun_developer if args.homebrew_devcmdrun_developer?
        categories << :homebrew_env_config if args.homebrew_env_config?
        categories << :homebrew_os_arch_ci if args.homebrew_os_arch_ci?
        categories << :homebrew_prefixes if args.homebrew_prefixes?
        categories << :homebrew_versions if args.homebrew_versions?
        categories << :os_versions if args.os_version?
        categories << :command_run if args.brew_command_run?
        categories << :command_run_options if args.brew_command_run_options?
        categories << :test_bot_test if args.brew_test_bot_test?

        category_matching_buckets = [:build_error, :cask_install, :command_run, :test_bot_test]

        categories.each do |category|
          additional_where = all_core_formulae_json ? " AND tap_name ~ '^homebrew/(core|cask)$'" : ""
          bucket = if category_matching_buckets.include?(category)
            category
          elsif [:command_run_options, :homebrew_env_config].include?(category)
            :command_run
          else
            :formula_install
          end

          case category
          when :homebrew_devcmdrun_developer
            dimension_key = "devcmdrun_developer"
            groups = [:devcmdrun, :developer]
          when :homebrew_env_config
            dimension_key = "env_config"
            groups = [:env_config, :env_config_state]
            # Events predating the user-set-aware `env_config_state` tag
            # counted brew's own exports as configuration, so drop them.
            additional_where += " AND env_config_state IS NOT NULL"
          when :homebrew_os_arch_ci
            dimension_key = "os_arch_ci"
            groups = [:os, :arch, :ci]
          when :homebrew_prefixes
            dimension_key = "prefix"
            groups = [:prefix, :os, :arch]
            standard_prefixes = %w[/opt/homebrew /usr/local /home/linuxbrew/.linuxbrew]
          when :homebrew_versions
            dimension_key = "version"
            groups = [:version]
          when :os_versions
            dimension_key = :os_version
            groups = [:os_name_and_version]
          when :command_run
            dimension_key = "command_run"
            groups = [:command]
          when :command_run_options
            dimension_key = "command_run_options"
            groups = [:command, :options, :devcmdrun, :developer]
          when :test_bot_test
            dimension_key = "test_bot_test"
            groups = [:command, :passed, :arch, :os]
          when :cask_install
            dimension_key = :cask
            groups = [:package, :tap_name]
          else
            dimension_key = :formula
            additional_where += " AND on_request = 'true'" if category == :formula_install_on_request
            groups = [:package, :tap_name, :options]
          end

          sql_groups = groups.map { |e| "\"#{e}\"" }.join(",")
          query = <<~EOS
            SELECT #{sql_groups}, COUNT(*) AS "count" FROM "#{bucket}" WHERE time >= now() - INTERVAL '#{days_ago} day'#{additional_where} GROUP BY #{sql_groups}
          EOS
          batches = begin
            client.query(query:, language: "sql").to_batches
          rescue PyCall::PyError => e
            if e.message.include?("message: unauthenticated")
              odie "Could not authenticate with InfluxDB! Please check your `$HOMEBREW_INFLUXDB_TOKEN`!"
            end
            raise
          end

          json = T.let({
            category:,
            total_items: 0,
            start_date:  Date.today - days_ago.to_i,
            end_date:    Date.today,
            total_count: 0,
            items:       [],
          }, T::Hash[Symbol, T.untyped])

          batches.each do |batch|
            batch.to_pylist.each do |record|
              if category == :homebrew_env_config
                state = record["env_config_state"]
                env_config_name = record["env_config"].to_s
                # Drop malformed events from non-standard clients and events
                # for variables Homebrew no longer supports.
                next if %w[unset default non_default].exclude?(state)
                next unless Homebrew::EnvConfig::ENVS.key?(env_config_name.to_sym)

                count = record["count"]
                json[:total_count] += count
                json[:items] << {
                  number: nil,
                  dimension_key => env_config_name,
                  count:,
                  non_default_count: (state == "non_default") ? count : 0,
                  set_default_count: (state == "default") ? count : 0,
                  unset_count:       (state == "unset") ? count : 0,
                }
                next
              end

              dimension = case category
              when :homebrew_devcmdrun_developer
                "devcmdrun=#{record["devcmdrun"]} HOMEBREW_DEVELOPER=#{record["developer"]}"
              when :homebrew_os_arch_ci
                if record["ci"] == "true"
                  "#{record["os"]} #{record["arch"]} (CI)"
                else
                  "#{record["os"]} #{record["arch"]}"
                end
              when :homebrew_prefixes
                prefix = record["prefix"].to_s
                if T.must(standard_prefixes).none? { |std| std.casecmp?(prefix) }
                  "custom-prefix (#{record["os"]} #{record["arch"]})"
                else
                  prefix
                end
              when :os_versions
                format_os_version_dimension(record["os_name_and_version"])
              when :command_run_options
                "#{record["command"]} #{record["options"].to_s.split.sort.join(" ")}"
              when :test_bot_test
                command_and_package, options = record["command"].split.partition { |arg| !arg.start_with?("-") }

                # Cleanup bad data before https://github.com/Homebrew/homebrew-test-bot/pull/1043
                # Can delete this code after 27th April 2025.
                next if %w[audit install linkage style test].exclude?(command_and_package.first)
                next if command_and_package.last.include?("/")
                next if options.include?("--tap=")
                next if options.include?("--only-dependencies")
                next if options.include?("--cached")

                command_and_options = (command_and_package + options.sort).join(" ")
                passed = (record["passed"] == "true") ? "PASSED" : "FAILED"

                "#{command_and_options} (#{record["os"]} #{record["arch"]}) (#{passed})"
              else
                record[groups.first.to_s]
              end
              next if dimension.blank?

              if (tap_name = record["tap_name"].presence) &&
                 ((tap_name != "homebrew/cask" && dimension_key == :cask) ||
                  (tap_name != "homebrew/core" && dimension_key == :formula))
                dimension = "#{tap_name}/#{dimension}"
              end

              if (all_core_formulae_json || category == :build_error) &&
                 (options = record["options"].presence)
                # homebrew/core formulae don't have non-HEAD options but they ended up in our analytics anyway.
                if all_core_formulae_json
                  options = options.split.include?("--HEAD") ? "--HEAD" : ""
                end
                dimension = "#{dimension} #{options}"
              end

              dimension = dimension.strip
              next if dimension.match?(/[<>]/)

              count = record["count"]

              json[:total_items] += 1
              json[:total_count] += count

              json[:items] << {
                number: nil,
                dimension_key => dimension,
                count:,
              }
            end
          end

          odie "No data returned" if json[:total_count].zero?

          # Combine identical values
          deduped_items = {}

          json[:items].each do |item|
            key = item[dimension_key]
            if deduped_items.key?(key)
              deduped_items[key][:count] += item[:count]
              if category == :homebrew_env_config
                deduped_items[key][:non_default_count] += item[:non_default_count]
                deduped_items[key][:set_default_count] += item[:set_default_count]
                deduped_items[key][:unset_count] += item[:unset_count]
              end
            else
              deduped_items[key] = item
            end
          end

          json[:items] = deduped_items.values
          json[:total_items] = json[:items].length if category == :homebrew_env_config

          if all_core_formulae_json
            core_formula_items = {}

            json[:items].each do |item|
              item.delete(:number)
              formula_name, = item[dimension_key].split.first
              next if formula_name.include?("/")

              core_formula_items[formula_name] ||= []
              core_formula_items[formula_name] << item
            end
            json.delete(:items)

            core_formula_items.each_value do |items|
              items.sort_by! { |item| -item[:count] }
              items.each do |item|
                item[:count] = format_count(item[:count])
              end
            end

            json[:formulae] = core_formula_items.sort_by { |name, _| name }.to_h
          else
            json[:items].sort_by! do |item|
              if category == :homebrew_env_config
                -item[:non_default_count].to_f / item[:count]
              else
                -item[:count]
              end
            end

            json[:items].each_with_index do |item, index|
              item[:number] = index + 1

              percent = if category == :homebrew_env_config
                (item[:non_default_count].to_f / item[:count]) * 100
              else
                (item[:count].to_f / json[:total_count]) * 100
              end
              item[:percent] = format_percent(percent)
              item[:count] = format_count(item[:count])
              next if category != :homebrew_env_config

              item[:non_default_count] = format_count(item[:non_default_count])
              item[:set_default_count] = format_count(item[:set_default_count])
              item[:unset_count] = format_count(item[:unset_count])
              item[:default_value] = Homebrew::EnvConfig.default_description(item[dimension_key].to_sym)
            end
          end

          puts JSON.pretty_generate json
        end
      end

      sig { params(count: Integer).returns(String) }
      def format_count(count)
        count.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
      end

      sig { params(percent: Float).returns(String) }
      def format_percent(percent)
        format("%<percent>.2f", percent:).gsub(/\.00$/, "")
      end

      sig { params(dimension: T.nilable(String)).returns(T.nilable(String)) }
      def format_os_version_dimension(dimension)
        return if dimension.blank?

        require "macos_version"
        require "utils/analytics"

        wsl = dimension.end_with?(Utils::Analytics::WSL_SUFFIX)
        dimension = dimension.delete_suffix(Utils::Analytics::WSL_SUFFIX)
        dimension = dimension.gsub(/^Intel ?/, "")
                             .gsub(/^macOS ?/, "")
                             .gsub(/ \(.+\)$/, "")

        begin
          macos_version = ::MacOSVersion.new(dimension)
          if macos_version.pretty_name.presence && macos_version.to_sym != :dunno
            return "macOS #{macos_version.pretty_name} (#{macos_version.strip_patch})"
          end
        rescue MacOSVersion::Error
          nil
        end

        formatted_dimension = case dimension
        when /Ubuntu(-Server)? (14|16|18|20|22|24)\.04/ then "Ubuntu #{Regexp.last_match(2)}.04 LTS"
        when /Ubuntu(-Server)? (\d+\.\d+).\d ?(LTS)?/
          "Ubuntu #{Regexp.last_match(2)} #{Regexp.last_match(3)}".strip
        when %r{Debian GNU/Linux (\d+)\.\d+} then "Debian #{Regexp.last_match(1)} #{Regexp.last_match(2)}"
        when /CentOS (\w+) (\d+)/ then "CentOS #{Regexp.last_match(1)} #{Regexp.last_match(2)}"
        when /Fedora Linux (\d+)[.\d]*/ then "Fedora Linux #{Regexp.last_match(1)}"
        when /KDE neon .*?([\d.]+)/ then "KDE neon #{Regexp.last_match(1)}"
        when /Amazon Linux (\d+)\.[.\d]*/ then "Amazon Linux #{Regexp.last_match(1)}"
        when /^Armbian\S*(?: OS)? (\d+)\.0?(\d+)\S*(?: (\w+))?/
          "Armbian #{Regexp.last_match(1)}.#{Regexp.last_match(2)} #{Regexp.last_match(3)&.downcase}".strip
        when /Fedora Linux Rawhide[.\dn]*/ then "Fedora Linux Rawhide"
        when /Red Hat Enterprise Linux CoreOS (\d+\.\d+)[-.\d]*/
          "Red Hat Enterprise Linux CoreOS #{Regexp.last_match(1)}"
        when /([A-Za-z ]+)\s+(\d+)\.\d{8}[.\d]*/ then "#{Regexp.last_match(1)} #{Regexp.last_match(2)}"
        # odisabled: add new entries when removing support, remove entries when no longer in the data
        when /^10\.14[.\d]*/ then "macOS Mojave (10.14)"
        when /^10\.13[.\d]*/ then "macOS High Sierra (10.13)"
        when /^10\.12[.\d]*/ then "macOS Sierra (10.12)"
        when /^10\.(\d+)/ then "macOS 10.#{Regexp.last_match(1)}"
        else dimension
        end

        Utils::Analytics.with_wsl_suffix_if_needed(formatted_dimension, wsl:)
      end
    end
  end
end