audit: flag self-submissions by commit email

- New cask and formula PRs for closed-source apps with little use are
  often self-submitted, but this is only caught by self-report or by a
  reviewer matching the commit author's email to the app's domain.
- Treat a PR commit author whose email domain matches a non-forge
  `homepage`, `url` or `head` domain as a self-submission, so the
  existing self-submission notability thresholds apply.
- Fail new packages self-submitted this way that have no repository the
  notability audits can assess. The message makes clear this is not an
  automatic rejection and that the pull request author must provide
  evidence of notability.
- Share the notability audits' repository regexes, anchored to the
  start of the URL, so "no repository" means exactly what those audits
  cannot measure.
- Add a `ci-skip-self-submission` label for casks so maintainers can
  skip the audit once they have judged notability themselves.
- Read every page of commit emails from the GitHub API using the pull
  request's `commits_url` from `GITHUB_EVENT_PATH`, so checkout depth
  does not matter.
This commit is contained in:
Bevan Kay committed 2026-10-17 14:38:50 +11:00
1 parent 19fe946b9a
commit 3853882809
8 files changed
+297 -34

No files matched your search

+21 -5
View File
@@ -1086,7 +1086,7 @@ module Cask
return unless new_cask?
return if (url = cask.url).nil?
user, repo = get_repo_data(%r{https?://github\.com/([^/]+)/([^/]+)/?.*})
user, repo = get_repo_data(SharedAudits::NOTABILITY_REPOSITORY_REGEXES.fetch(:github))
return if user.nil? || repo.nil?
odebug "Auditing GitHub repo"
@@ -1101,7 +1101,7 @@ module Cask
return unless new_cask?
return if (url = cask.url).nil?
user, repo = get_repo_data(%r{https?://gitlab\.com/([^/]+)/([^/]+)/?.*})
user, repo = get_repo_data(SharedAudits::NOTABILITY_REPOSITORY_REGEXES.fetch(:gitlab))
return if user.nil? || repo.nil?
odebug "Auditing GitLab repo"
@@ -1116,7 +1116,7 @@ module Cask
return unless new_cask?
return if (url = cask.url).nil?
user, repo = get_repo_data(%r{https?://bitbucket\.org/([^/]+)/([^/]+)/?.*})
user, repo = get_repo_data(SharedAudits::NOTABILITY_REPOSITORY_REGEXES.fetch(:bitbucket))
return if user.nil? || repo.nil?
odebug "Auditing Bitbucket repo"
@@ -1131,7 +1131,7 @@ module Cask
return unless new_cask?
return if (url = cask.url).nil?
user, repo = get_repo_data(%r{https?://codeberg\.org/([^/]+)/([^/]+)/?.*})
user, repo = get_repo_data(SharedAudits::NOTABILITY_REPOSITORY_REGEXES.fetch(:forgejo))
return if user.nil? || repo.nil?
odebug "Auditing Forgejo repo"
@@ -1200,6 +1200,16 @@ module Cask
add_error new_domain_problem if new_domain_problem
end
sig { void }
def audit_self_submission
return unless new_cask?
return unless online?
return unless cask.tap&.official?
self_submission_problem = SharedAudits.self_submission_problem(self_submission_urls)
add_error self_submission_problem if self_submission_problem
end
sig { void }
def audit_url_https_availability
return unless online?
@@ -1480,7 +1490,13 @@ module Cask
def self_submission?(repo_owner)
return false if repo_owner.empty?
SharedAudits.self_submission_for_repo_owner?(repo_owner)
SharedAudits.self_submission_for_repo_owner?(repo_owner) ||
!SharedAudits.commit_email_domain_match(self_submission_urls).nil?
end
sig { returns(T::Hash[Symbol, T.nilable(String)]) }
def self_submission_urls
{ homepage: cask.homepage, url: cask.url&.to_s }
end
sig {
@@ -271,6 +271,8 @@ module Homebrew
bitbucket_repository]
end
audit_exceptions << "self_submission" if labels.include?("ci-skip-self-submission")
audit_exceptions << %w[token_valid token_bad_words token_conflicts] if labels.include?("ci-skip-token")
audit_args << "--except" << audit_exceptions.join(",") if audit_exceptions.any?
+21 -5
View File
@@ -681,6 +681,16 @@ module Homebrew
problem new_domain_problem if new_domain_problem
end
sig { void }
def audit_self_submission
return unless @core_tap
return unless @new_formula
return unless @online
self_submission_problem = SharedAudits.self_submission_problem(self_submission_urls)
new_formula_problem self_submission_problem if self_submission_problem
end
sig { void }
def audit_duplicate_formula
return unless @core_tap
@@ -806,7 +816,7 @@ module Homebrew
sig { void }
def audit_github_repository
user, repo = get_repo_data(%r{https?://github\.com/([^/]+)/([^/]+)/?.*}) if @new_formula
user, repo = get_repo_data(SharedAudits::NOTABILITY_REPOSITORY_REGEXES.fetch(:github)) if @new_formula
return if user.blank? || repo.nil?
@@ -819,7 +829,7 @@ module Homebrew
sig { void }
def audit_gitlab_repository
user, repo = get_repo_data(%r{https?://gitlab\.com/([^/]+)/([^/]+)/?.*}) if @new_formula
user, repo = get_repo_data(SharedAudits::NOTABILITY_REPOSITORY_REGEXES.fetch(:gitlab)) if @new_formula
return if user.blank? || repo.nil?
self_submission = self_submission?(user)
@@ -831,7 +841,7 @@ module Homebrew
sig { void }
def audit_bitbucket_repository
user, repo = get_repo_data(%r{https?://bitbucket\.org/([^/]+)/([^/]+)/?.*}) if @new_formula
user, repo = get_repo_data(SharedAudits::NOTABILITY_REPOSITORY_REGEXES.fetch(:bitbucket)) if @new_formula
return if user.blank? || repo.nil?
self_submission = self_submission?(user)
@@ -843,7 +853,7 @@ module Homebrew
sig { void }
def audit_forgejo_repository
user, repo = get_repo_data(%r{https?://codeberg\.org/([^/]+)/([^/]+)/?.*}) if @new_formula
user, repo = get_repo_data(SharedAudits::NOTABILITY_REPOSITORY_REGEXES.fetch(:forgejo)) if @new_formula
return if user.blank? || repo.nil?
self_submission = self_submission?(user)
@@ -1362,7 +1372,13 @@ module Homebrew
def self_submission?(repo_owner)
return false if repo_owner.blank?
SharedAudits.self_submission_for_repo_owner?(repo_owner)
SharedAudits.self_submission_for_repo_owner?(repo_owner) ||
!SharedAudits.commit_email_domain_match(self_submission_urls).nil?
end
sig { returns(T::Hash[Symbol, T.nilable(String)]) }
def self_submission_urls
{ homepage: formula.homepage, url: formula.stable&.url, head: formula.head&.url }
end
sig { params(formula: Formula).returns(T::Boolean) }
+54
View File
@@ -1699,6 +1699,60 @@ RSpec.describe Cask::Audit, :cask do
end
end
describe "self-submission" do
let(:only) { ["self_submission"] }
let(:new_cask) { true }
let(:self_submission_problem) { "Self-submitted (a commit author's email address matches brew.sh)" }
let(:cask) do
Cask::Cask.new("self-submission", tap: CoreCaskTap.instance) do
version "1.0"
sha256 :no_check
url "https://brew.sh/self-submission.tar.gz"
homepage "https://brew.sh"
end
end
context "when a commit author's email address matches the homepage domain" do
before do
allow(SharedAudits).to receive(:self_submission_problem)
.with({ homepage: "https://brew.sh", url: "https://brew.sh/self-submission.tar.gz" })
.and_return(self_submission_problem)
end
it { is_expected.to error_with(self_submission_problem) }
end
context "when the cask is not new" do
let(:new_cask) { false }
it "does not check for self-submission" do
expect(SharedAudits).not_to receive(:self_submission_problem)
run
end
end
end
describe "GitHub repository notability" do
let(:only) { ["github_repository"] }
let(:new_cask) { true }
let(:cask) do
Cask::Cask.new("github-repository", tap: CoreCaskTap.instance) do
version "1.0"
sha256 :no_check
url "https://github.com/acme/app/releases/download/1.0/app.dmg"
homepage "https://acme.io"
end
end
it "applies the self-submission thresholds when a commit author's email address matches the homepage" do
allow(SharedAudits).to receive_messages(self_submission_for_repo_owner?: false,
pull_request_commit_email_domains: ["acme.io"])
expect(SharedAudits).to receive(:github).with("acme", "app", self_submission: true)
run
end
end
describe "GitHub repository archived check" do
let(:online) { true }
let(:only) { ["github_repository_archived"] }
+31 -9
View File
@@ -140,18 +140,18 @@ RSpec.describe Homebrew::FormulaAuditor do
end
end
def homepage_auditor(**options)
formula_auditor "foo", <<~RUBY, { core_tap: true, new_formula: true }.merge(options)
class Foo < Formula
homepage "https://brew.sh"
url "https://brew.sh/foo-1.0.tar.gz"
end
RUBY
end
describe "#audit_homepage_domain_age" do
let(:domain_problem) { "`homepage` domain `brew.sh` was registered on 2026-07-01" }
def homepage_auditor(**options)
formula_auditor "foo", <<~RUBY, { core_tap: true, new_formula: true }.merge(options)
class Foo < Formula
homepage "https://brew.sh"
url "https://brew.sh/foo-1.0.tar.gz"
end
RUBY
end
it "reports newly registered homepage domains" do
allow(SharedAudits).to receive(:new_domain_problem)
.with("https://brew.sh").and_return(domain_problem)
@@ -186,6 +186,28 @@ RSpec.describe Homebrew::FormulaAuditor do
end
end
describe "#audit_self_submission" do
let(:self_submission_problem) { "Self-submitted (a commit author's email address matches brew.sh)" }
it "reports self-submissions without a git forge repository" do
allow(SharedAudits).to receive(:self_submission_problem)
.with({ homepage: "https://brew.sh", url: "https://brew.sh/foo-1.0.tar.gz", head: nil })
.and_return(self_submission_problem)
fa = homepage_auditor online: true
fa.audit_self_submission
expect(fa.new_formula_problems.first[:message]).to eq(self_submission_problem)
end
it "does not check existing formulae" do
fa = homepage_auditor new_formula: false
expect(SharedAudits).not_to receive(:self_submission_problem)
fa.audit_self_submission
end
end
describe "#audit_license" do
let(:spdx_license_data) { SPDX.license_data }
let(:spdx_exception_data) { SPDX.exception_data }
@@ -245,6 +245,84 @@ RSpec.describe SharedAudits do
end
end
describe "::pull_request_commit_email_domains" do
let(:commits_url) { "https://api.github.com/repos/Homebrew/homebrew-cask/pulls/1/commits" }
before { described_class.pull_request_commit_email_domains = nil }
after { described_class.pull_request_commit_email_domains = nil }
it "returns the email domains of the pull request's commit authors" do
event_path = mktmpdir/"event.json"
event_path.write(JSON.generate({ pull_request: { commits_url: } }))
ENV["GITHUB_EVENT_PATH"] = event_path.to_s
first_page = [{ "commit" => { "author" => { "email" => "Jo@Acme.io" } } }]
second_page = [{ "commit" => { "author" => { "email" => "x@brew.sh" } } }]
allow(GitHub::API).to receive(:paginate_rest).with(commits_url)
.and_yield(first_page, 1).and_yield(second_page, 2)
expect(described_class.pull_request_commit_email_domains).to eq(["acme.io", "brew.sh"])
end
it "returns no domains outside a pull request" do
ENV.delete("GITHUB_EVENT_PATH")
expect(described_class.pull_request_commit_email_domains).to be_empty
end
end
describe "::commit_email_domain_match" do
before { allow(described_class).to receive(:pull_request_commit_email_domains).and_return(["acme.io"]) }
it "returns the stanza and domain at a commit author's email domain" do
expect(described_class.commit_email_domain_match({ homepage: "https://www.acme.io/app" }))
.to eq([:homepage, "acme.io"])
end
it "matches a subdomain of a commit author's email domain" do
urls = { homepage: "https://acmeapp.com", url: "https://dl.acme.io/app.dmg" }
expect(described_class.commit_email_domain_match(urls)).to eq([:url, "dl.acme.io"])
end
it "ignores other domains" do
expect(described_class.commit_email_domain_match({ homepage: "https://notacme.io" })).to be_nil
end
it "ignores git forge domains" do
allow(described_class).to receive(:pull_request_commit_email_domains).and_return(["github.com"])
expect(described_class.commit_email_domain_match({ homepage: "https://github.com/acme/app" })).to be_nil
end
end
describe "::self_submission_problem" do
before { allow(described_class).to receive(:pull_request_commit_email_domains).and_return(["acme.io"]) }
it "reports self-submissions without a repository to assess" do
urls = { homepage: "https://acmeapp.com", url: "https://dl.acme.io/app.dmg" }
expect(described_class.self_submission_problem(urls)).to include("matches the `url` domain `dl.acme.io`")
end
it "reports self-submissions whose git forge pages have no repository to assess" do
urls = { homepage: "https://acme.github.io", url: "https://acme.io/app.dmg" }
expect(described_class.self_submission_problem(urls)).to include("matches the `url` domain `acme.io`")
end
it "ignores git forge URLs embedded in other URLs" do
urls = { homepage: "https://acme.io", url: "https://dl.acme.io/app.dmg?source=https://github.com/acme/app" }
expect(described_class.self_submission_problem(urls)).to include("matches the `homepage` domain `acme.io`")
end
it "leaves self-submissions with a repository to the notability audits" do
urls = { homepage: "https://acme.io", url: "https://github.com/acme/app/releases/download/v1.0/app.dmg" }
expect(described_class.self_submission_problem(urls)).to be_nil
end
end
describe "::github_tag_from_url" do
it "finds tags in archive urls" do
url = "https://github.com/a/b/archive/refs/tags/v1.2.3.tar.gz"
+88 -14
View File
@@ -13,6 +13,12 @@ module SharedAudits
GITLAB_NOTABILITY_THRESHOLDS = T.let({ forks: 30, stars: 75 }.freeze, T::Hash[Symbol, Integer])
BITBUCKET_NOTABILITY_THRESHOLDS = T.let({ forks: 30, watchers: 75 }.freeze, T::Hash[Symbol, Integer])
FORGEJO_NOTABILITY_THRESHOLDS = T.let({ forks: 30, watchers: 30, stars: 75 }.freeze, T::Hash[Symbol, Integer])
NOTABILITY_REPOSITORY_REGEXES = T.let({
github: %r{\Ahttps?://github\.com/([^/]+)/([^/]+)/?.*},
gitlab: %r{\Ahttps?://gitlab\.com/([^/]+)/([^/]+)/?.*},
bitbucket: %r{\Ahttps?://bitbucket\.org/([^/]+)/([^/]+)/?.*},
forgejo: %r{\Ahttps?://codeberg\.org/([^/]+)/([^/]+)/?.*},
}.freeze, T::Hash[Symbol, Regexp])
NEW_DOMAIN_THRESHOLD_DAYS = 30
GIT_FORGE_DOMAINS = %w[
bitbucket.org
@@ -34,9 +40,16 @@ module SharedAudits
@pull_request_author = T.let(nil, T.nilable(String))
@pull_request_author_computed = T.let(false, T::Boolean)
@self_submission_cache = T.let({}, T::Hash[String, T::Boolean])
@pull_request_commit_email_domains = T.let(nil, T.nilable(T::Array[String]))
@rdap_services = T.let(nil, T.nilable(RdapServices))
class << self
sig {
params(pull_request_commit_email_domains: T.nilable(T::Array[String]))
.returns(T.nilable(T::Array[String]))
}
attr_writer :pull_request_commit_email_domains
sig { params(rdap_services: T.nilable(RdapServices)).returns(T.nilable(RdapServices)) }
attr_writer :rdap_services
end
@@ -51,15 +64,8 @@ module SharedAudits
sig { params(homepage: String).returns(T.nilable(String)) }
def self.new_domain_problem(homepage)
host = begin
URI(homepage).host
rescue URI::InvalidURIError
nil
end
return if host.blank?
domain = host.downcase.delete_prefix("www.")
return if GIT_FORGE_DOMAINS.any? { |forge| domain == forge || domain.end_with?(".#{forge}") }
domain = url_domain(homepage)
return if domain.nil? || git_forge_domain?(domain)
@rdap_services ||= begin
result = Utils::Curl.curl_output(RDAP_BOOTSTRAP_URL, max_time: RDAP_TIMEOUT_SECONDS)
@@ -145,12 +151,50 @@ module SharedAudits
return @pull_request_author if @pull_request_author_computed
@pull_request_author_computed = true
github_event_path = ENV.fetch("GITHUB_EVENT_PATH", nil)
return @pull_request_author = nil if github_event_path.blank?
@pull_request_author = github_event_pull_request_value("user", "login")
end
@pull_request_author = JSON.parse(File.read(github_event_path)).dig("pull_request", "user", "login")
rescue Errno::ENOENT, JSON::ParserError
@pull_request_author = nil
sig { returns(T::Array[String]) }
def self.pull_request_commit_email_domains
@pull_request_commit_email_domains ||= begin
commits_url = github_event_pull_request_value("commits_url")
commits = []
if commits_url.present?
GitHub::API.paginate_rest(commits_url) do |page_commits, _page|
commits.concat(page_commits)
end
end
commits.filter_map { |commit| commit.dig("commit", "author", "email").to_s.split("@").last&.downcase }
rescue GitHub::API::Error
[]
end
end
sig { params(urls: T::Hash[Symbol, T.nilable(String)]).returns(T.nilable([Symbol, String])) }
def self.commit_email_domain_match(urls)
urls.each do |stanza, url|
next unless url
domain = url_domain(url)
next if domain.nil? || git_forge_domain?(domain)
next if pull_request_commit_email_domains.none? { |email_domain| domain_or_subdomain?(domain, email_domain) }
return [stanza, domain]
end
nil
end
sig { params(urls: T::Hash[Symbol, T.nilable(String)]).returns(T.nilable(String)) }
def self.self_submission_problem(urls)
return if urls.values.compact.grep(Regexp.union(NOTABILITY_REPOSITORY_REGEXES.values)).any?
match = commit_email_domain_match(urls)
return if match.nil?
stanza, domain = match
"Self-submitted (a commit author's email address matches the `#{stanza}` domain `#{domain}`) " \
"without a GitHub, GitLab, Bitbucket or Codeberg repository to assess its notability. " \
"This is not an automatic rejection, but the pull request author must provide evidence that it is notable."
end
sig { params(submitter: T.nilable(String), repo_owner: String).returns(T::Boolean) }
@@ -181,6 +225,36 @@ module SharedAudits
thresholds.transform_values { |value| value * SELF_SUBMISSION_THRESHOLD_MULTIPLIER }
end
sig { params(keys: String).returns(T.nilable(String)) }
private_class_method def self.github_event_pull_request_value(*keys)
github_event_path = ENV.fetch("GITHUB_EVENT_PATH", nil)
return if github_event_path.blank?
JSON.parse(File.read(github_event_path)).dig("pull_request", *keys)
rescue Errno::ENOENT, JSON::ParserError
nil
end
sig { params(url: String).returns(T.nilable(String)) }
private_class_method def self.url_domain(url)
host = begin
URI(url).host
rescue URI::InvalidURIError
nil
end
host.downcase.delete_prefix("www.") if host.present?
end
sig { params(domain: String).returns(T::Boolean) }
private_class_method def self.git_forge_domain?(domain)
GIT_FORGE_DOMAINS.any? { |forge| domain_or_subdomain?(domain, forge) }
end
sig { params(domain: String, parent: String).returns(T::Boolean) }
private_class_method def self.domain_or_subdomain?(domain, parent)
domain == parent || domain.end_with?(".#{parent}")
end
sig { params(product: String, cycle: String).returns(T.nilable(T::Hash[String, T.untyped])) }
def self.eol_data(product, cycle)
@eol_data ||= T.let({}, T.nilable(T::Hash[String, T.untyped]))
+2 -1
View File
@@ -29,9 +29,10 @@ A new package must demonstrate public interest beyond its author.
A GitHub project normally satisfies this requirement by meeting one of these thresholds:
* at least 30 forks, 30 watchers or 75 stars.
* at least 90 forks, 90 watchers or 225 stars for a self-submission by the repository owner.
* at least 90 forks, 90 watchers or 225 stars for a self-submission by the repository owner or by a commit author whose email address is at the package's `homepage`, `url` or `head` domain.
Equivalent public evidence may be considered for software hosted elsewhere.
A self-submission without a public code repository is not automatically rejected, but its author must provide evidence of notability in the pull request.
The metrics apply to the canonical upstream repository, not an unendorsed mirror or code-hosting fork.
A code repository less than 30 days old is normally not eligible.
Repository-specific exceptions may apply when these metrics do not represent the software's actual use or maintenance prospects.