Add partial benchmarks in CI (#8387)

Related to: https://github.com/open-telemetry/community/issues/3402

This makes benchmarks run only for modules where their Go files have
been changed.

This PR also removes the `-benchtime=500ms`, so it now uses CodSpeed's
default benchtime, which seems to be around 3 seconds, by my
observation. This would stabilize the benchmark result, and it takes 6
times the original benchmark time. But since we will use partial
benchmarks, I think this is fine.

This is a comparison of several benchmark results.
https://codspeed.io/open-telemetry/opentelemetry-go/branches/XSAM%3Afeat%2Faffected-benchmarks?utm_source=github&utm_medium=comment-v2&utm_content=button

<img width="858" height="1590" alt="CleanShot 2026-05-29 at 22 58 50"
src="https://github.com/user-attachments/assets/4918eaba-9858-4ac0-baf4-fd478b5cbc37"
/>

You can see the delta is at most around 9% after removing `benchtime`.

---

It also adds a CI to test `./internal/tools` folder. The
`./internal/tools` folder has a different Go version that can't pass the
regular CI test matrix.

.github/workflows/benchmark.yml

@@ -10,6 +10,11 @@ on:
       - 'Makefile'
       - '.github/workflows/benchmark.yml'
   workflow_dispatch:
+    inputs:
+      full:
+        description: "Run benchmarks for every module, ignoring the affected-module diff."
+        type: boolean
+        default: false
 
 # Declare default permissions as read only.
 permissions:
@@ -28,18 +33,46 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+          # Need history so the affected-module detector can diff
+          # against the prior commit.
+          fetch-depth: 0
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version: ${{ env.DEFAULT_GO_VERSION }}
+          check-latest: true
+          cache-dependency-path: "**/go.sum"
       - id: sharding
+        env:
+          # We want to diff HEAD against whatever "before this change"
+          # means for the current event.
+          #
+          # For a PR, that's the commit on main the PR branched from --
+          # we want every commit in the PR, not just the latest push.
+          # (Tempting to use github.event.before here, but on PRs it's
+          # the previous tip of the PR branch, so it'd miss earlier
+          # commits in the same PR.)
+          #
+          # For a push to main, it's the commit that was at the tip
+          # before this push -- exactly the new work this push brings.
+          #
+          # For a manual workflow_dispatch run, neither field is set,
+          # and the Makefile's default base ref kicks in.
+          BASE_REF: ${{ github.event.pull_request.base.sha || github.event.before || 'origin/main' }}
+          ARGS: ${{ inputs.full && '-all' || '' }}
         run: |
-          echo "shards=$(make print-sharded-benchmarks)" >> $GITHUB_OUTPUT
+          shards=$(make print-affected-benchmarks)
+          echo "shards=${shards}" >> "$GITHUB_OUTPUT"
 
   benchmark:
     needs: sharding-benchmark
+    # Skip the matrix entirely when nothing benchmark-relevant changed.
+    if: needs.sharding-benchmark.outputs.shards != '[]'
     permissions:
       contents: write # required for pushing to gh-pages branch
     name: Benchmarks
     runs-on: oracle-bare-metal-64cpu-1024gb-x86-64-ubuntu-24
     container:
-      image: ubuntu:26.04@sha256:7c2884fd32770fc6c173b78e0dc2278a2851d89f5447919edbc45475ac55dd6a
+      image: ubuntu:24.04@sha256:52df9b1ee71626e0088f7d400d5c6b5f7bb916f8f0c82b474289a4ece6cf3faf
       # CodSpeed's profiling feature needs elevated permissions.
       options: --cap-add=PERFMON
     strategy:
@@ -61,7 +94,14 @@ jobs:
         with:
           go-version: ${{ env.DEFAULT_GO_VERSION }}
           check-latest: true
-          cache-dependency-path: "**/go.sum"
+          # Disable setup-go's GOMODCACHE/GOCACHE caching on the bare-metal
+          # runner. The runner is persistent, so ~/go/pkg/mod and
+          # ~/.cache/go-build already survive between jobs locally. The
+          # Actions cache for this key reaches multiple GB across the matrix,
+          # which makes every shard pay ~1-2 min on either the restore
+          # (download + tar -xf) or the post-step save (tar + upload), with
+          # no benefit over the on-disk caches.
+          cache: false
       - name: Run the benchmarks
         env:
           # Job containers do not run systemd. Without this override,
@@ -73,10 +113,4 @@ jobs:
           allow-empty: true
           # CodSpeed runs extremely slow when there are 128 processes involved, which is the number
           # of CPUs on the bare metal machine. Limiting the number of processes to 4 helps mitigate this. 4 also matches the number of CPUs on GitHub runners.
-          #
-          # CodSpeed overrides the default benchtime if we don't explicitly specify it.
-          # Having this would avoid running benchmark for more than 1 hour.
-          #
-          # The benchtime is adjusted to 500 ms to speed up the benchmark time.
-          # Per https://github.com/open-telemetry/community/issues/2331#issuecomment-2356403352
-          run: GOMAXPROCS=4 make benchmark/${{matrix.shard}} ARGS=-benchtime=500ms
+          run: GOMAXPROCS=4 make benchmark/${{matrix.shard}}

.github/workflows/ci.yml

@@ -116,6 +116,25 @@ jobs:
       - name: Run benchmarks to check functionality
         run: make test-bench
 
+  test-tools:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Setup Environment
+        run: |
+          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+      - name: Install Go
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version: ${{ env.DEFAULT_GO_VERSION }}
+          cache-dependency-path: "**/go.sum"
+      - name: Run tests for internal tools
+        run: make test-tools
+
   test-race:
     runs-on: ubuntu-latest
     steps:

Makefile

@@ -69,8 +69,13 @@ $(GORELEASE): PACKAGE=golang.org/x/exp/cmd/gorelease
 GOVULNCHECK = $(TOOLS)/govulncheck
 $(TOOLS)/govulncheck: PACKAGE=golang.org/x/vuln/cmd/govulncheck
 
+AFFECTEDMODS = $(TOOLS)/affectedmods
+AFFECTEDMODS_FILES := $(sort $(shell find $(TOOLS_MOD_DIR)/affectedmods -type f))
+$(TOOLS)/affectedmods: PACKAGE=go.opentelemetry.io/otel/$(TOOLS_MOD_DIR)/affectedmods
+$(TOOLS)/affectedmods: $(AFFECTEDMODS_FILES)
+
 .PHONY: tools
-tools: $(CROSSLINK) $(GOLANGCI_LINT) $(MISSPELL) $(GOCOVMERGE) $(STRINGER) $(VERIFYREADMES) $(MULTIMOD) $(SEMCONVKIT) $(GOTMPL) $(GORELEASE)
+tools: $(CROSSLINK) $(GOLANGCI_LINT) $(MISSPELL) $(GOCOVMERGE) $(STRINGER) $(VERIFYREADMES) $(MULTIMOD) $(SEMCONVKIT) $(GOTMPL) $(GORELEASE) $(AFFECTEDMODS)
 
 # Virtualized python tools via docker
 
@@ -165,6 +170,9 @@ test/%:
 		| grep -v third_party \
 		| xargs $(GO) test -timeout $(TIMEOUT)s $(ARGS)
 
+.PHONY: test-tools
+test-tools: test/$(TOOLS_MOD_DIR)
+
 COVERAGE_MODE    = atomic
 COVERAGE_PROFILE = coverage.out
 .PHONY: test-coverage
@@ -189,15 +197,23 @@ benchmark/%:
 
 # sdk/metric is split into two shards to work around CodSpeed limitations.
 # See https://github.com/CodSpeedHQ/codspeed-go/issues/56
-BENCHMARK_SHARDS := $(filter-out ./sdk/metric,$(OTEL_GO_MOD_DIRS)) ./sdk/metric/root ./sdk/metric/internal
-benchmark/./sdk/metric/root:
+BENCHMARK_SHARDS := $(filter-out ./sdk/metric,$(OTEL_GO_MOD_DIRS)) ./sdk/metric/. ./sdk/metric/internal
+benchmark/./sdk/metric/.:
 	cd ./sdk/metric && $(GO) test -run='^$$' -bench=. $(ARGS) . ./exemplar/...
-benchmark/./sdk/metric/internal:
-	cd ./sdk/metric && $(GO) test -run='^$$' -bench=. $(ARGS) ./internal/...
 
 print-sharded-benchmarks:
 	@echo $(BENCHMARK_SHARDS) | jq -cR 'split(" ")'
 
+# Print the JSON list of benchmark shards whose code changed since
+# BASE_REF (default: main). Filters print-sharded-benchmarks
+# down to shards whose underlying module contains a changed *.go file.
+# Non-Go changes (docs, workflows, Makefile, go.mod/go.sum, tooling) emit [].
+# Override the diff base via BASE_REF, or pass ARGS=-all to emit the full list.
+BASE_REF ?= main
+.PHONY: print-affected-benchmarks
+print-affected-benchmarks:
+	@$(MAKE) -s print-sharded-benchmarks | $(GO) -C $(TOOLS_MOD_DIR) run ./affectedmods $(if $(strip $(BASE_REF)),-base=$(BASE_REF),) $(ARGS)
+
 .PHONY: golangci-lint golangci-lint-fix
 golangci-lint-fix: ARGS=--fix
 golangci-lint-fix: golangci-lint

internal/tools/affectedmods/main.go

@@ -0,0 +1,132 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// Affectedmods filters a list of benchmark shards to those whose code
+// changed in the current diff.
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"sort"
+	"strings"
+)
+
+var (
+	baseRef  = flag.String("base", "main", "git ref to diff against")
+	forceAll = flag.Bool("all", false, "emit every input shard without diffing")
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetPrefix("affectedmods: ")
+	flag.Parse()
+
+	shards, err := readShards(os.Stdin)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if *forceAll {
+		emit(shards)
+		return
+	}
+
+	changed, err := changedFiles(*baseRef)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if len(changed) == 0 {
+		emit(nil)
+		return
+	}
+
+	// Sort longest-first so owningModule returns the first prefix match.
+	sort.Slice(shards, func(i, j int) bool { return len(shards[i]) > len(shards[j]) })
+
+	affected := map[string]bool{}
+	for _, f := range changed {
+		if !strings.HasSuffix(f, ".go") {
+			continue
+		}
+		if strings.HasPrefix(f, "internal/tools/") {
+			continue // tooling has its own go.mod, never a benchmark target
+		}
+		if shard := owningModule(f, shards); shard != "" {
+			affected[shard] = true
+		}
+	}
+
+	out := make([]string, 0, len(affected))
+	for s := range affected {
+		out = append(out, s)
+	}
+	sort.Strings(out)
+	emit(out)
+}
+
+// readShards decodes a JSON array of shard labels from stdin.
+func readShards(r *os.File) ([]string, error) {
+	var shards []string
+	if err := json.NewDecoder(r).Decode(&shards); err != nil {
+		return nil, fmt.Errorf("decoding shards: %w", err)
+	}
+	if len(shards) == 0 {
+		return nil, fmt.Errorf("no shards on stdin")
+	}
+	return shards, nil
+}
+
+func changedFiles(base string) ([]string, error) {
+	if err := exec.Command("git", "rev-parse", "--verify", base).Run(); err != nil {
+		return nil, fmt.Errorf("base ref %q not found", base)
+	}
+	out, err := exec.Command("git", "diff", "--name-only", base+"...HEAD").Output()
+	if err != nil {
+		return nil, fmt.Errorf("git diff: %w", err)
+	}
+	var files []string
+	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
+		if line != "" {
+			files = append(files, line)
+		}
+	}
+	return files, nil
+}
+
+// owningModule returns the longest shard-prefix containing p. Inputs
+// are like "./sdk/log" or "./sdk/metric/." and must be sorted
+// longest-first. A trailing "/." on a shard label is stripped for
+// matching -- it's a marker for the catch-all sub-shard of a split
+// module. If no nested shard matches, the root module "." is returned
+// when present in modules.
+func owningModule(p string, modules []string) string {
+	root := ""
+	for _, m := range modules {
+		if m == "." {
+			root = "."
+			continue
+		}
+		bare := strings.TrimPrefix(m, "./")
+		bare = strings.TrimSuffix(bare, "/.")
+		if p == bare || strings.HasPrefix(p, bare+"/") {
+			return m
+		}
+	}
+	return root
+}
+
+func emit(shards []string) {
+	if shards == nil {
+		shards = []string{}
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(shards); err != nil {
+		log.Fatal(err)
+	}
+}

internal/tools/affectedmods/main_test.go

@@ -0,0 +1,110 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"sort"
+	"testing"
+)
+
+func TestOwningModule(t *testing.T) {
+	// Mirrors the shape of `make print-sharded-benchmarks`: "." for the
+	// root module, "./<dir>" for nested ones, and "./<mod>/." marking the
+	// catch-all sub-shard of a split module. Sorted longest-first, as
+	// owningModule expects.
+	mods := []string{
+		".",
+		"./sdk",
+		"./sdk/log",
+		"./sdk/log/logtest",
+		"./sdk/metric/.",
+		"./sdk/metric/internal",
+		"./bridge/opencensus",
+		"./bridge/opencensus/test",
+	}
+	sort.Slice(mods, func(i, j int) bool { return len(mods[i]) > len(mods[j]) })
+
+	tests := []struct {
+		name    string
+		path    string
+		modules []string
+		want    string
+	}{
+		{
+			name: "direct hit on nested module",
+			path: "sdk/log/exporter.go", modules: mods,
+			want: "./sdk/log",
+		},
+		{
+			name: "nested file picks longest prefix",
+			path: "sdk/log/logtest/recorder.go", modules: mods,
+			want: "./sdk/log/logtest",
+		},
+		{
+			name: "shorter prefix when no longer match exists",
+			path: "sdk/trace/span.go", modules: mods,
+			want: "./sdk",
+		},
+		{
+			name: "exact module path (no trailing slash)",
+			path: "sdk", modules: mods,
+			want: "./sdk",
+		},
+		{
+			name: "top-level file falls through to root",
+			path: "handler.go", modules: mods,
+			want: ".",
+		},
+		{
+			name: "nested file with no nested-module match falls through to root",
+			path: "propagation/baggage.go", modules: mods,
+			want: ".",
+		},
+		{
+			name: "module-name prefix without slash boundary falls through to root",
+			path: "sdk-extra/x.go", modules: mods,
+			want: ".",
+		},
+		{
+			name: "deep nesting under unrelated tree",
+			path: "bridge/opencensus/test/exporter.go", modules: mods,
+			want: "./bridge/opencensus/test",
+		},
+		{
+			name: "split-module catch-all matches files at module root",
+			path: "sdk/metric/meter.go", modules: mods,
+			want: "./sdk/metric/.",
+		},
+		{
+			name: "split-module catch-all matches files in non-internal subdir",
+			path: "sdk/metric/exemplar/x.go", modules: mods,
+			want: "./sdk/metric/.",
+		},
+		{
+			name: "split-module sub-shard wins for its subtree",
+			path: "sdk/metric/internal/x.go", modules: mods,
+			want: "./sdk/metric/internal",
+		},
+		{
+			name: "fallback empty when root not in list and no nested match",
+			path: "handler.go", modules: []string{"./sdk"},
+			want: "",
+		},
+		{
+			name: "empty modules returns empty",
+			path: "sdk/log/exporter.go", modules: nil,
+			want: "",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := owningModule(tt.path, tt.modules)
+			if got != tt.want {
+				t.Errorf("owningModule(%q, %v) = %q, want %q",
+					tt.path, tt.modules, got, tt.want)
+			}
+		})
+	}
+}

internal/tools/semconvkit/main_test.go

@@ -15,6 +15,15 @@ func TestNewMigrationIgnoresUnexportedDeclarations(t *testing.T) {
 	t.Parallel()
 
 	root := t.TempDir()
+	goMod := filepath.Join(root, "go.mod")
+	if err := os.WriteFile(
+		goMod,
+		[]byte("module example.com/semconvtest\n\ngo 1.25.0\n"),
+		0o600,
+	); err != nil {
+		t.Fatalf("WriteFile(%q) error = %v", goMod, err)
+	}
+
 	prev := filepath.Join(root, "v1.40.0")
 	cur := filepath.Join(root, "v1.41.0")