From 8e609892dbca4e8c2933e81abdd3cd5d77ff7bdd Mon Sep 17 00:00:00 2001 From: Bartek Plotka Date: Tue, 27 Oct 2026 21:31:49 +0000 Subject: [PATCH] storage/semconv: remove legacy 1.0/1.1 schemas, embedded registry, and testdata YAMLs Drops the inherited 1.0.0/1.1.0 schema parser, __semconv_url__ corroboration engine, embedded toy registry fixtures (storage/semconv/registry/{1.0.0,1.1.0,1.2.0,registry.yaml}), and upstream testdata YAML dumps, keeping only transform/2.0 Telemetry Policy translation. Signed-off-by: Bartek Plotka --- storage/semconv/bench_test.go | 318 --- storage/semconv/engine.go | 2024 +---------------- storage/semconv/engine_test.go | 1144 ---------- storage/semconv/otel_schema.go | 799 +------ storage/semconv/otel_schema_test.go | 1046 --------- storage/semconv/promql_rewrite.go | 2 +- storage/semconv/registry.go | 44 +- storage/semconv/registry/1.0.0 | 30 - storage/semconv/registry/1.1.0 | 30 - storage/semconv/registry/1.2.0 | 30 - storage/semconv/registry/registry.yaml | 22 - storage/semconv/registry_test.go | 131 +- storage/semconv/storage.go | 2 +- storage/semconv/storage_contract_test.go | 879 ------- storage/semconv/storage_fanout_test.go | 1102 --------- storage/semconv/storage_inheritance_test.go | 171 -- storage/semconv/storage_labels_test.go | 123 - .../semconv/storage_materialization_test.go | 183 -- storage/semconv/storage_registry_test.go | 221 -- .../semconv/storage_rename_validation_test.go | 947 -------- .../semconv/storage_select_contract_test.go | 1023 --------- storage/semconv/storage_test.go | 1127 +-------- storage/semconv/storage_topk_test.go | 83 - storage/semconv/storage_traversal_test.go | 131 -- storage/semconv/testdata/otel.yaml | 40 - .../testdata/otel_unsupported_format.yaml | 7 - .../testdata/otel_with_all_section.yaml | 17 - .../testdata/otel_with_chained_renames.yaml | 20 - .../testdata/otel_with_metric_renames.yaml | 10 - .../testdata/upstream/schema-1.44.0.yaml | 766 ------- .../testdata/upstream/semconv-1.21.0.yaml | 293 --- .../testdata/upstream/semconv-1.22.0.yaml | 239 -- .../testdata/upstream/semconv-1.43.0.yaml | 13 - .../testdata/upstream/semconv-1.44.0.yaml | 13 - storage/semconv/transform_policy_test.go | 7 - storage/semconv/upstream_golden_test.go | 168 -- 36 files changed, 163 insertions(+), 13042 deletions(-) delete mode 100644 storage/semconv/bench_test.go delete mode 100644 storage/semconv/engine_test.go delete mode 100644 storage/semconv/otel_schema_test.go delete mode 100644 storage/semconv/registry/1.0.0 delete mode 100644 storage/semconv/registry/1.1.0 delete mode 100644 storage/semconv/registry/1.2.0 delete mode 100644 storage/semconv/registry/registry.yaml delete mode 100644 storage/semconv/storage_contract_test.go delete mode 100644 storage/semconv/storage_fanout_test.go delete mode 100644 storage/semconv/storage_inheritance_test.go delete mode 100644 storage/semconv/storage_labels_test.go delete mode 100644 storage/semconv/storage_materialization_test.go delete mode 100644 storage/semconv/storage_registry_test.go delete mode 100644 storage/semconv/storage_rename_validation_test.go delete mode 100644 storage/semconv/storage_select_contract_test.go delete mode 100644 storage/semconv/storage_topk_test.go delete mode 100644 storage/semconv/storage_traversal_test.go delete mode 100644 storage/semconv/testdata/otel.yaml delete mode 100644 storage/semconv/testdata/otel_unsupported_format.yaml delete mode 100644 storage/semconv/testdata/otel_with_all_section.yaml delete mode 100644 storage/semconv/testdata/otel_with_chained_renames.yaml delete mode 100644 storage/semconv/testdata/otel_with_metric_renames.yaml delete mode 100644 storage/semconv/testdata/upstream/schema-1.44.0.yaml delete mode 100644 storage/semconv/testdata/upstream/semconv-1.21.0.yaml delete mode 100644 storage/semconv/testdata/upstream/semconv-1.22.0.yaml delete mode 100644 storage/semconv/testdata/upstream/semconv-1.43.0.yaml delete mode 100644 storage/semconv/testdata/upstream/semconv-1.44.0.yaml delete mode 100644 storage/semconv/upstream_golden_test.go diff --git a/storage/semconv/bench_test.go b/storage/semconv/bench_test.go deleted file mode 100644 index 325f55c70c..0000000000 --- a/storage/semconv/bench_test.go +++ /dev/null @@ -1,318 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/storage/semconv" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/util/annotations" - "github.com/prometheus/prometheus/util/teststorage" -) - -// benchSemconv declares the metric under test with one attribute, plus bench.solo, -// which no schema below renames. A query for bench.solo therefore fans out to a -// single variant with nothing rewritten but __name__, which is the common shape. -const benchSemconv = ` -groups: - - id: registry.attrs - type: attribute_group - brief: "Attributes" - attributes: - - id: %[2]s - type: string - stability: stable - brief: "An attribute" - examples: ["a"] - - id: metric.%[1]s - type: metric - brief: "A metric" - stability: stable - metric_name: %[1]s - instrument: counter - unit: By - attributes: - - ref: %[2]s - - id: metric.bench.solo - type: metric - brief: "A metric no schema renames" - stability: stable - metric_name: bench.solo - instrument: counter - unit: By - attributes: - - ref: attr.solo -` - -// benchMetricRenameSchema renames only the metric, so nothing a returned series -// carries is rewritten apart from __name__. -const benchMetricRenameSchema = ` -file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - bench.old: bench.new -` - -// benchAttributeRenameSchema renames the metric and one of its attributes, which -// is what forces a variant to be buffered and re-sorted. -const benchAttributeRenameSchema = ` -file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - bench.old: bench.new - - rename_attributes: - attribute_map: - attr.old: attr.new - apply_to_metrics: - - bench.new -` - -// benchAttrOf gives each benchmark metric the attribute name its era declares. -var benchAttrOf = map[string]string{ - "bench.old": "attr.old", - "bench.new": "attr.new", - "bench.solo": "attr.solo", -} - -// benchStorage builds a storage holding seriesCount series, spread evenly over the -// given metric names. -func benchStorage(b *testing.B, schema string, seriesCount int, metrics ...string) storage.Storage { - b.Helper() - wrapped, err := semconv.AwareStorageWithRegistry(teststorage.New(b), map[string][]byte{ - "registry.yaml": []byte(schema), - "1.0.0": fmt.Appendf(nil, benchSemconv, "bench.old", "attr.old"), - "1.1.0": fmt.Appendf(nil, benchSemconv, "bench.new", "attr.new"), - }) - require.NoError(b, err) - - app := wrapped.Appender(context.Background()) - for i := range seriesCount { - metric := metrics[i%len(metrics)] - attr := benchAttrOf[metric] - // Zero-padded so the stored order is stable and the series are spread - // across both eras rather than clustered. - lset := labels.FromStrings(model.MetricNameLabel, metric, attr, fmt.Sprintf("v%05d", i)) - _, err := app.Append(0, lset, 1, float64(i)) - require.NoError(b, err) - } - require.NoError(b, app.Commit()) - return wrapped -} - -// BenchmarkSelectFanOut measures a fanned-out Select, drained the way PromQL -// drains one. Metric-only variants stream without querying label metadata; -// attribute renames still require buffering and sorting. -func BenchmarkSelectFanOut(b *testing.B) { - for _, sc := range []struct { - name string - schema string - query string - metrics []string - }{ - // One variant, with only the canonical __name__ rewrite. - {"no rename", benchMetricRenameSchema, "bench.solo", []string{"bench.solo"}}, - // Two variants, still with only __name__ rewritten. - {"metric rename only", benchMetricRenameSchema, "bench.new", []string{"bench.old", "bench.new"}}, - // Two variants with an attribute rewritten as well. - {"attribute rename", benchAttributeRenameSchema, "bench.new", []string{"bench.old", "bench.new"}}, - } { - for _, seriesCount := range []int{100, 10000} { - b.Run(fmt.Sprintf("%s/%d series", sc.name, seriesCount), func(b *testing.B) { - s := benchStorage(b, sc.schema, seriesCount, sc.metrics...) - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, sc.query), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - - b.ReportAllocs() - b.ResetTimer() - for b.Loop() { - q, err := s.Querier(0, 10) - require.NoError(b, err) - - var series, samples int - set := q.Select(context.Background(), false, nil, matchers...) - for set.Next() { - series++ - it := set.At().Iterator(nil) - for it.Next() == chunkenc.ValFloat { - samples++ - } - } - require.NoError(b, set.Err()) - require.NoError(b, q.Close()) - if series != seriesCount { - b.Fatalf("got %d series, want %d", series, seriesCount) - } - } - }) - } - } -} - -const benchLatencyVariants = 32 - -type latencyStorage struct { - storage.Storage - delay time.Duration -} - -func (s latencyStorage) Querier(mint, maxt int64) (storage.Querier, error) { - q, err := s.Storage.Querier(mint, maxt) - if err != nil { - return nil, err - } - return &latencyQuerier{Querier: q, delay: s.delay}, nil -} - -func (s latencyStorage) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { - q, err := s.Storage.ChunkQuerier(mint, maxt) - if err != nil { - return nil, err - } - return &latencyChunkQuerier{ChunkQuerier: q, delay: s.delay}, nil -} - -type latencyQuerier struct { - storage.Querier - delay time.Duration -} - -func (q *latencyQuerier) Select(context.Context, bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet { - time.Sleep(q.delay) - return storage.NoopSeriesSet() -} - -func (q *latencyQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) { - time.Sleep(q.delay) - return nil, nil, nil -} - -func (q *latencyQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) { - time.Sleep(q.delay) - return nil, nil, nil -} - -type latencyChunkQuerier struct { - storage.ChunkQuerier - delay time.Duration -} - -func (q *latencyChunkQuerier) Select(context.Context, bool, *storage.SelectHints, ...*labels.Matcher) storage.ChunkSeriesSet { - time.Sleep(q.delay) - return storage.NoopChunkedSeriesSet() -} - -func benchLatencyRegistry() map[string][]byte { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: -`) - for i := range benchLatencyVariants - 1 { - schema = fmt.Appendf(schema, " bench.old.%02d: bench.current\n", i) - } - return map[string][]byte{ - "registry.yaml": schema, - "1.1.0": fmt.Appendf(nil, benchSemconv, "bench.current", "attr.current"), - } -} - -func benchLatencyStorage(b *testing.B) storage.Storage { - b.Helper() - wrapped, err := semconv.AwareStorageWithRegistry( - latencyStorage{Storage: teststorage.New(b), delay: time.Millisecond}, - benchLatencyRegistry(), - ) - require.NoError(b, err) - return wrapped -} - -func benchLatencyMatchers() []*labels.Matcher { - return []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "bench.current"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } -} - -// BenchmarkFanOutCallLatency measures the storage-call portion of schema -// fan-out independently of series decoding and iteration. -func BenchmarkFanOutCallLatency(b *testing.B) { - for _, operation := range []string{"select", "chunk select", "label names", "label values"} { - b.Run(operation, func(b *testing.B) { - s := benchLatencyStorage(b) - matchers := benchLatencyMatchers() - b.ReportAllocs() - b.ResetTimer() - - for b.Loop() { - switch operation { - case "select": - q, err := s.Querier(0, 10) - require.NoError(b, err) - set := q.Select(context.Background(), false, nil, matchers...) - for set.Next() { - } - require.NoError(b, set.Err()) - require.NoError(b, q.Close()) - case "chunk select": - q, err := s.ChunkQuerier(0, 10) - require.NoError(b, err) - set := q.Select(context.Background(), false, nil, matchers...) - for set.Next() { - } - require.NoError(b, set.Err()) - require.NoError(b, q.Close()) - case "label names": - q, err := s.Querier(0, 10) - require.NoError(b, err) - _, _, err = q.LabelNames(context.Background(), nil, matchers...) - require.NoError(b, err) - require.NoError(b, q.Close()) - case "label values": - q, err := s.Querier(0, 10) - require.NoError(b, err) - _, _, err = q.LabelValues(context.Background(), "attr.current", nil, matchers...) - require.NoError(b, err) - require.NoError(b, q.Close()) - } - } - }) - } -} diff --git a/storage/semconv/engine.go b/storage/semconv/engine.go index a120239f66..5e21fd8efb 100644 --- a/storage/semconv/engine.go +++ b/storage/semconv/engine.go @@ -38,9 +38,11 @@ const ( ) var ( + errMetricNameAnchor = errors.New("schema-aware query requires a non-empty equality matcher on __name__") errSchemaExpansion = errors.New("semconv schema expansion limit exceeded") errAmbiguousSchemaRename = errors.New("semconv schema rename is ambiguous") errUnsafeSchemaMatcher = errors.New("semconv schema matcher cannot be expanded safely") + errSemconvURLRequired = errors.New("semconvURL is required") ) func schemaExpansionError(kind string) error { @@ -107,561 +109,32 @@ func (b *schemaExpansionBudget) remainingKeyBytes() uint64 { return b.limits.keyBytes - b.keyBytes } -// metricOwnershipTracker follows metric identities through schema-version -// boundaries. Identity unions model explicit convergence, while reused records -// a disconnected identity claiming a previously occupied surface name. -type metricOwnershipTracker struct { - budget *schemaExpansionBudget - - parent []int - size []uint32 - occupied map[string]int - firstOwner map[string]int - reused map[string]struct{} -} - -func newMetricOwnershipTracker(budget *schemaExpansionBudget) *metricOwnershipTracker { - return &metricOwnershipTracker{ - budget: budget, - occupied: map[string]int{}, - firstOwner: map[string]int{}, - reused: map[string]struct{}{}, - } -} - -func (t *metricOwnershipTracker) newIdentity() (int, error) { - if err := t.budget.reserveWork(1); err != nil { - return 0, err - } - id := len(t.parent) - t.parent = append(t.parent, id) - t.size = append(t.size, 1) - return id, nil -} - -func (t *metricOwnershipTracker) root(id int) (int, error) { - for t.parent[id] != id { - if err := t.budget.reserveWork(1); err != nil { - return 0, err - } - id = t.parent[id] - } - return id, nil -} - -func (t *metricOwnershipTracker) merge(a, b int) (int, error) { - a, err := t.root(a) - if err != nil { - return 0, err - } - b, err = t.root(b) - if err != nil { - return 0, err - } - if a == b { - return a, nil - } - if t.size[a] < t.size[b] || (t.size[a] == t.size[b] && b < a) { - a, b = b, a - } - t.parent[b] = a - t.size[a] += t.size[b] - return a, nil -} - -func (t *metricOwnershipTracker) recordBoundary(name string, identity int) error { - identity, err := t.root(identity) - if err != nil { - return err - } - first, exists := t.firstOwner[name] - if !exists { - t.firstOwner[name] = identity - return nil - } - first, err = t.root(first) - if err != nil { - return err - } - if first != identity { - t.reused[name] = struct{}{} - } - return nil -} - -func sortedMapKeysWithBudget[V any](m map[string]V, budget *schemaExpansionBudget) ([]string, error) { - if err := budget.reserveWork(uint64(len(m))); err != nil { - return nil, err - } - keys := make([]string, 0, len(m)) - for key := range m { - keys = append(keys, key) - } - slices.Sort(keys) - return keys, nil -} - -// reusedMetricNamesWithBudget reports surface names that distinct metric -// identities occupy at schema-version boundaries. Changes within one revision -// remain ordered, but each individual rename map is applied atomically. -func reusedMetricNamesWithBudget(revisions []schemaRevision, budget *schemaExpansionBudget) (map[string]struct{}, error) { - tracker := newMetricOwnershipTracker(budget) - for _, revision := range revisions { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - targets, err := metricTargetsByFirstChangeWithBudget(revision, budget) - if err != nil { - return nil, err - } - - seeded := map[string]int{} - for changeIndex, change := range revision.changes { - if change.metricRenames == nil { - continue - } - sources, err := sortedMapKeysWithBudget(change.metricRenames.forward, budget) - if err != nil { - return nil, err - } - for _, source := range sources { - if _, exists := tracker.occupied[source]; exists { - continue - } - if producedByEarlierChange(targets, changeIndex, source) { - continue - } - // Schemas may repeat a retired spelling as a compatibility input in a - // later convergence. Reuse its known identity rather than inventing one. - identity, known := tracker.firstOwner[source] - if known { - identity, err = tracker.root(identity) - if err != nil { - return nil, err - } - } else { - identity, err = tracker.newIdentity() - if err != nil { - return nil, err - } - } - tracker.occupied[source] = identity - seeded[source] = identity - } - } - seededNames, err := sortedMapKeysWithBudget(seeded, budget) - if err != nil { - return nil, err - } - for _, name := range seededNames { - if err := tracker.recordBoundary(name, seeded[name]); err != nil { - return nil, err - } - } - - touched := map[string]struct{}{} - for _, change := range revision.changes { - if change.metricRenames == nil { - continue - } - if err := tracker.applyRenameMap(change.metricRenames, touched); err != nil { - return nil, err - } - } - touchedNames, err := sortedMapKeysWithBudget(touched, budget) - if err != nil { - return nil, err - } - for _, name := range touchedNames { - identity, exists := tracker.occupied[name] - if !exists { - continue - } - if err := tracker.recordBoundary(name, identity); err != nil { - return nil, err - } - } - } - return tracker.reused, nil -} - -func metricTargetsByFirstChangeWithBudget(revision schemaRevision, budget *schemaExpansionBudget) (map[string]int, error) { - targets := map[string]int{} - for i, change := range revision.changes { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if change.metricRenames == nil { - continue - } - names, err := sortedMapKeysWithBudget(change.metricRenames.reverse, budget) - if err != nil { - return nil, err - } - for _, name := range names { - if _, exists := targets[name]; !exists { - targets[name] = i - } - } - } - return targets, nil -} - -func schemaHasAttributeRenamesWithBudget(revisions []schemaRevision, budget *schemaExpansionBudget) (bool, error) { - for _, revision := range revisions { - for _, change := range revision.changes { - if err := budget.reserveWork(1); err != nil { - return false, err - } - if change.attributeRenames != nil { - return true, nil - } - } - } - return false, nil -} - -func (t *metricOwnershipTracker) applyRenameMap(renames *directedRenames, touched map[string]struct{}) error { - sources, err := sortedMapKeysWithBudget(renames.forward, t.budget) - if err != nil { - return err - } - incoming := map[string][]int{} - stationaryTargets := map[string]int{} - for _, source := range sources { - target := renames.forward[source] - touched[source] = struct{}{} - touched[target] = struct{}{} - identity, exists := t.occupied[source] - if exists { - incoming[target] = append(incoming[target], identity) - } - if _, targetMoves := renames.forward[target]; targetMoves { - continue - } - if identity, occupied := t.occupied[target]; occupied { - stationaryTargets[target] = identity - } - } - for _, source := range sources { - delete(t.occupied, source) - } - targetNames, err := sortedMapKeysWithBudget(incoming, t.budget) - if err != nil { - return err - } - for _, target := range targetNames { - identities := incoming[target] - identity := identities[0] - for _, other := range identities[1:] { - identity, err = t.merge(identity, other) - if err != nil { - return err - } - } - if stationary, exists := stationaryTargets[target]; exists { - identity, err = t.merge(identity, stationary) - if err != nil { - return err - } - } - t.occupied[target] = identity - } - return nil -} - // registrySource provides the raw bytes of registry files addressed by their -// registry/ path. The embedded registry (embed.FS) satisfies it directly; -// an operator-provided registry is adapted to it via newRegistrySource. +// registry/ path. type registrySource interface { ReadFile(name string) ([]byte, error) } type schemaEngine struct { - registry registrySource - limits schemaExpansionLimits - - otelSchemaCache *staticCache[otelSchema] - semconvCache *staticCache[semconv] - schemaSafetyCache *staticCache[schemaSafetyAnalysis] + registry registrySource + limits schemaExpansionLimits + otelSchemaCache *staticCache[otelSchema] } func newSchemaEngine(registry registrySource) *schemaEngine { return &schemaEngine{ - registry: registry, - limits: productionSchemaExpansionLimits(), - otelSchemaCache: newStaticCache[otelSchema](), - semconvCache: newStaticCache[semconv](), - schemaSafetyCache: newStaticCache[schemaSafetyAnalysis](), + registry: registry, + limits: productionSchemaExpansionLimits(), + otelSchemaCache: newStaticCache[otelSchema](), } } -type schemaSafetyAnalysis struct { - reused map[string]struct{} - hasAttributeRenames bool - work uint64 - err error +type matcherVariant struct { + matchers []*labels.Matcher + mapping *labelMapping + canonicalMatchers []*labels.Matcher } -// schemaSafetyAnalysis returns immutable lifecycle metadata for one cached -// schema. The work remains charged to every query even when the scan is cached. -func (e *schemaEngine) schemaSafetyAnalysis(url string, schema *otelSchema) schemaSafetyAnalysis { - if analysis, ok := e.schemaSafetyCache.get(url); ok { - return analysis - } - budget := newSchemaExpansionBudget(productionSchemaExpansionLimits()) - reused, err := reusedMetricNamesWithBudget(schema.revisions, budget) - hasAttributeRenames := false - if err == nil { - hasAttributeRenames, err = schemaHasAttributeRenamesWithBudget(schema.revisions, budget) - } - analysis := schemaSafetyAnalysis{ - reused: reused, - hasAttributeRenames: hasAttributeRenames, - work: budget.work, - err: err, - } - e.schemaSafetyCache.set(url, analysis) - return analysis -} - -type metricLookupStatus uint8 - -const ( - metricFileMissing metricLookupStatus = iota - metricUndeclared - metricAmbiguous - metricDeclared -) - -// metricLookup resolves what a semconv version declares for a metric name. A -// registry need not ship every version referenced by its schema, so an absent -// file is distinct from a file that omits or ambiguously declares the name. -type metricLookup func(version, name string) (metricDef, metricLookupStatus) - -// newMetricLookup resolves sibling semconv versions in semconvURL's registry -// directory and memoizes both successful and failed loads for one query. -func (e *schemaEngine) newMetricLookup(semconvURL string) metricLookup { - dir, _ := path.Split(semconvURL) - // Memo distinguishes "loaded" from "tried and failed" so an absent semconv - // is not re-fetched for every hop that consults it. - memo := map[string]*semconv{} - return func(version, name string) (metricDef, metricLookupStatus) { - sc, tried := memo[version] - if !tried { - if loaded, err := e.getSemconv(dir + version); err == nil { - sc = &loaded - } - memo[version] = sc - } - if sc == nil { - return metricDef{}, metricFileMissing - } - if slices.Contains(sc.ambiguousMetrics, name) { - return metricDef{}, metricAmbiguous - } - def, declared := sc.metrics[name] - if !declared { - return metricDef{}, metricUndeclared - } - return def, metricDeclared - } -} - -// renameValidator corroborates resolved metric lineages against the semconv -// files at schema-version boundaries. -// -// The OTel schema format names metrics only by their surface name, so the rename -// graph alone cannot distinguish a genuine rename of one metric from an edge -// that happens to join two unrelated metrics sharing a name at different -// versions. The semconv files can: each version states the unit and instrument -// of the metric it declares under that name, and upstream semantic conventions -// forbid a stable metric from changing either (policies/compatibility.rego). A -// disagreement therefore means the two names denote different metrics. -// -// A group's id is of no use here even though it looks like a stable identifier: -// upstream lints it to be exactly "metric." -// (policies/yaml_schema.rego), so it is renamed in lockstep with the metric and -// comparing ids only restates whether the name changed. -// Checks are made against the metric the query asked for — the anchor — and not -// between an edge's own two endpoints. Schema safety rejects proven disconnected -// name reuse before this validator runs. For histories where lifecycle ownership -// remains unproven, directional traversal supplies the boundary; unit and -// instrument reject a branch only when semconv data positively identifies a -// different metric. -type renameValidator struct { - lookup metricLookup - - // anchorName/anchorVersion/anchorDef describe the metric the query asked for. - // anchorVersion is the queried semconv version where it declares the name, and - // otherwise the version the identity was recovered from; see - // identityFromOwnEra. anchorKnown is false only when no version settles what - // the name denotes, leaving no identity to compare hops against. - // anchorDeclared distinguishes the requested version from recovered identity. - anchorName string - anchorVersion string - anchorDef metricDef - anchorDeclared bool - anchorKnown bool - - warnings []string - seen map[string]struct{} -} - -func newRenameValidator(schema *otelSchema, lookup metricLookup, anchor semconv, anchorName string, budget *schemaExpansionBudget) (*renameValidator, error) { - def, known := anchor.metrics[anchorName] - declared := known - version := anchor.version - if !known { - // The anchor semconv does not declare the queried name, which is an - // ordinary thing to query: the walk deliberately supports asking for a name - // the anchor version has already renamed away. Rather than give up on - // checking anything, take the metric's identity from a version where the - // name is the current one. - // version stays the queried one when this fails, so the warning that - // follows names the semconv the user actually asked for. - eraDef, eraVersion, eraKnown, err := identityFromOwnEra(schema, lookup, anchorName, budget) - if err != nil { - return nil, err - } - if eraKnown { - def, version, known = eraDef, eraVersion, true - } - } - return &renameValidator{ - lookup: lookup, - anchorName: anchorName, - anchorVersion: version, - anchorDef: def, - anchorDeclared: declared, - anchorKnown: known, - seen: map[string]struct{}{}, - }, nil -} - -// identityFromOwnEra resolves what name meant at a version where it is the current -// name, for use when the queried semconv version does not declare it. -// -// It declines to answer when the schema retires the name and later reintroduces it -// and the two eras disagree on what the metric is, because then there is no single -// answer to what the queried name denotes and picking one would corroborate hops -// against an identity the user never asked for. -func identityFromOwnEra(schema *otelSchema, lookup metricLookup, name string, budget *schemaExpansionBudget) (metricDef, string, bool, error) { - var ( - found metricDef - version string - ok bool - ) - eras, err := schema.eraVersionsOfWithBudget(name, budget) - if err != nil { - return metricDef{}, "", false, err - } - for _, era := range eras { - if err := budget.reserveWork(1); err != nil { - return metricDef{}, "", false, err - } - def, status := lookup(era, name) - switch status { - case metricAmbiguous: - return metricDef{}, "", false, nil - case metricDeclared: - default: - continue - } - if ok && !def.sameMetricAs(found) { - return metricDef{}, "", false, nil - } - if !ok { - found, version, ok = def, era, true - } - } - return found, version, ok, nil -} - -func (rv *renameValidator) warn(format string, args ...any) { - msg := fmt.Sprintf(format, args...) - if _, dup := rv.seen[msg]; dup { - return - } - rv.seen[msg] = struct{}{} - rv.warnings = append(rv.warnings, msg) -} - -// allowRevision reports whether a branch that changed from fromName to toName while -// replaying revision may continue at destinationVersion. Validation happens -// after the whole revision: names between two ordered transformations are not -// schema-version endpoints and therefore cannot be looked up meaningfully. -func (rv *renameValidator) allowRevision(revision, destinationVersion, fromName, toName string) bool { - if rv == nil || rv.lookup == nil || destinationVersion == "" { - return true - } - - def, status := rv.lookup(destinationVersion, toName) - if status == metricAmbiguous { - rv.warn("schema version %s resolves metric %q to %q, but semconv %s declares %q more than once; not following this ambiguous rename branch", - revision, fromName, toName, destinationVersion, toName) - return false - } - if status == metricFileMissing { - rv.warn("version %s is unavailable while checking metric renames at schema revision %s. Prometheus is following explicit schema renames at that boundary without corroboration", - destinationVersion, revision) - return true - } - if !rv.anchorKnown { - // No semconv version declares the queried name, or the ones that do - // disagree on what it is. Every hop is then traversed unchecked, which is - // reported rather than left silent: this is the case the corroboration - // exists for, so a caller should know it did not run. Reported only here, - // where a metric rename is actually being followed, so a query that crosses - // no rename says nothing. - rv.warn("metric %q is not declared by semconv %s, and no other version of it settles what the metric is; schema renames of it are being followed without corroboration and may merge unrelated series", - rv.anchorName, rv.anchorVersion) - return true - } - - switch status { - case metricUndeclared: - rv.warn("schema version %s resolves metric %q to %q, but semconv %s does not declare %q as a metric; the rename could not be corroborated", - revision, fromName, toName, destinationVersion, toName) - return true - case metricDeclared: - if def.sameMetricAs(rv.anchorDef) { - return true - } - if !def.stablyContradicts(rv.anchorDef) { - rv.warn("queried metric %q is declared as %s/%s with %s stability by semconv %s, while schema version %s resolves it to %q, declared as %s/%s with %s stability by semconv %s. Prometheus is following the explicit schema rename because the metadata does not prove the metrics are different", - rv.anchorName, describeOrUnspecified(rv.anchorDef.unit), describeOrUnspecified(rv.anchorDef.instrument), describeStability(rv.anchorDef.stability), rv.anchorVersion, - revision, toName, describeOrUnspecified(def.unit), describeOrUnspecified(def.instrument), describeStability(def.stability), destinationVersion) - return true - } - rv.warn("queried metric %q is declared as %s/%s by semconv %s, but schema version %s resolves it to %q, which semconv %s declares as %s/%s; treating them as different metrics and not merging their series", - rv.anchorName, describeOrUnspecified(rv.anchorDef.unit), describeOrUnspecified(rv.anchorDef.instrument), rv.anchorVersion, - revision, toName, destinationVersion, describeOrUnspecified(def.unit), describeOrUnspecified(def.instrument)) - return false - default: - rv.warn("version %s returned unknown metric lookup status %d while checking schema revision %s. Prometheus is following the explicit schema rename without corroboration", - destinationVersion, status, revision) - return true - } -} - -// describeOrUnspecified renders an empty unit or instrument explicitly in warnings. -func describeOrUnspecified(s string) string { - if s == "" { - return "(unspecified)" - } - return s -} - -func describeStability(s string) string { - if s == "" { - return "unspecified" - } - return s -} - -var errMetricNameAnchor = errors.New("schema-aware query requires a non-empty equality matcher on __name__") - func extractMetricName(matchers []*labels.Matcher) (string, error) { hasMetricMatcher := false for _, m := range matchers { @@ -717,794 +190,6 @@ func normalizeMetricMatchers(matchers []*labels.Matcher) (string, []*labels.Matc return metricName, out, true, nil } -type traversalDirection uint8 - -const ( - traverseBackward traversalDirection = iota - traverseForward -) - -func (d traversalDirection) String() string { - if d == traverseForward { - return "forward" - } - return "backward" -} - -// revisionPartitionWithBudget returns the number of revisions whose version is at or -// before targetVersion. Returning a count, rather than an anchor index, lets an -// anchor predating every revision correctly have an empty backward range. -func revisionPartitionWithBudget(revisions []schemaRevision, targetVersion string, budget *schemaExpansionBudget) (int, error) { - target := strings.TrimPrefix(targetVersion, "v") - for i, revision := range revisions { - if err := budget.reserveWork(1); err != nil { - return 0, err - } - if compareSemver(revision.version, target) > 0 { - return i, nil - } - } - return len(revisions), nil -} - -// traversalPartitionWithBudget returns the revision boundary from which both -// directions should be walked. When an anchor is the revision that retires an -// undeclared historical name, that revision belongs to the forward walk. A later -// anchor must stop at the retirement because the surface name may have been reused. -func traversalPartitionWithBudget(revisions []schemaRevision, targetVersion, metricName string, anchorDeclared bool, budget *schemaExpansionBudget) (int, error) { - partition, err := revisionPartitionWithBudget(revisions, targetVersion, budget) - if err != nil { - return 0, err - } - if anchorDeclared { - return partition, nil - } - - anchorVersion := strings.TrimPrefix(targetVersion, "v") - for i := partition - 1; i >= 0; i-- { - if err := budget.reserveWork(1); err != nil { - return 0, err - } - before, after, mentioned, err := revisions[i].metricBoundaryWithBudget(metricName, budget) - if err != nil { - return 0, err - } - if !mentioned || (!before && !after) { - continue - } - if before && !after && compareSemver(revisions[i].version, anchorVersion) == 0 { - return i, nil - } - break - } - return partition, nil -} - -type attributeOrigin uint8 - -const ( - attributeOriginResolved attributeOrigin = iota - attributeOriginPending -) - -type attributeLineage map[string]map[string]attributeOrigin - -func newAttributeLineageWithBudget(canonical []string, budget *schemaExpansionBudget) (attributeLineage, error) { - if len(canonical) == 0 { - return nil, nil - } - lineage := make(attributeLineage, min(len(canonical), maxSchemaExpansion)) - for _, name := range canonical { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if _, exists := lineage[name]; exists { - continue - } - if len(lineage) >= maxSchemaExpansion { - return nil, schemaExpansionError("attribute lineage") - } - lineage[name] = map[string]attributeOrigin{name: attributeOriginResolved} - } - return lineage, nil -} - -type lineageState struct { - matchers []*labels.Matcher - metric string - attrs attributeLineage - - // renamedFrom is set to the metric name at the start of the first metric - // transformation in a revision. It is cleared at revision boundaries. - renamedFrom string - - // metricProduced is set while walking a revision forward when an earlier - // transformation produced metric. metricOriginPending marks a temporary - // identity branch while walking a converging revision backward. - metricProduced bool - metricOriginPending bool - - // pendingAttributeMatchers identifies matcher positions retained as - // temporary identity branches while walking a converging revision backward. - pendingAttributeMatchers map[int]struct{} -} - -type matcherVariant struct { - matchers []*labels.Matcher - mapping *labelMapping - canonicalMatchers []*labels.Matcher -} - -type attributeAlias struct { - alias string - canonical string -} - -type variantAccumulator struct { - anchorMetric string - variants []matcherVariant - byMatchers map[string]int - - attributeIdentities map[string]map[string]struct{} - attributeAliases map[string]map[string]string - attributeParents map[string]map[string]string - attributeSizes map[string]map[string]uint32 - trackAttrIdentities bool - ambiguousMetrics map[string]struct{} - conflicts []map[string]struct{} - validator *renameValidator - - budget *schemaExpansionBudget - attributeSlots int -} - -func newVariantAccumulatorWithBudget(anchorMetric string, canonicalAttrs []string, rv *renameValidator, trackAttrIdentities bool, budget *schemaExpansionBudget) (*variantAccumulator, error) { - acc := &variantAccumulator{ - anchorMetric: anchorMetric, - byMatchers: map[string]int{}, - trackAttrIdentities: trackAttrIdentities, - validator: rv, - budget: budget, - } - if !trackAttrIdentities { - return acc, nil - } - canonical := make(map[string]struct{}, min(len(canonicalAttrs), maxSchemaExpansion)) - for _, name := range canonicalAttrs { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - canonical[name] = struct{}{} - } - acc.attributeIdentities = map[string]map[string]struct{}{anchorMetric: canonical} - acc.attributeAliases = map[string]map[string]string{} - acc.attributeParents = map[string]map[string]string{anchorMetric: {}} - acc.attributeSizes = map[string]map[string]uint32{anchorMetric: {}} - for name := range canonical { - acc.attributeParents[anchorMetric][name] = name - acc.attributeSizes[anchorMetric][name] = 1 - } - return acc, nil -} - -// add folds states with identical underlying matcher queries together. Their -// compatible label aliases can be normalised by one query. A physical metric -// or attribute name owned by distinct identities makes the whole query unsafe. -func (a *variantAccumulator) add(state lineageState) error { - if _, ambiguous := a.ambiguousMetrics[state.metric]; ambiguous { - return fmt.Errorf("%w: metric name %q belongs to distinct schema lineages", errAmbiguousSchemaRename, state.metric) - } - key, err := matcherKeyWithBudget(state.matchers, a.budget) - if err != nil { - return err - } - idx, exists := a.byMatchers[key] - if !exists { - if len(a.variants) >= maxSchemaExpansion { - return schemaExpansionError("matcher variants") - } - idx = len(a.variants) - a.byMatchers[key] = idx - a.variants = append(a.variants, matcherVariant{ - matchers: state.matchers, - mapping: buildLabelMapping(a.anchorMetric, nil), - }) - a.conflicts = append(a.conflicts, map[string]struct{}{}) - } - return a.mergeAttributeAliases(idx, state.metric, state.attrs) -} - -func (a *variantAccumulator) mergeAttributeAliases(idx int, metric string, lineage attributeLineage) error { - aliases, err := attributeAliasesWithBudget(lineage, a.budget) - if err != nil { - return err - } - - if a.trackAttrIdentities { - owners, err := resolvedAttributeOwnersWithBudget(lineage, a.budget) - if err != nil { - return err - } - - knownIdentities := a.attributeIdentities[metric] - knownAliases := a.attributeAliases[metric] - stagedIdentities := make(map[string]struct{}, len(owners)) - currentOwners := make(map[string][]string, len(owners)) - for _, owner := range owners { - if err := a.ensureAttributeIdentity(metric, owner.canonical); err != nil { - return err - } - stagedIdentities[owner.canonical] = struct{}{} - currentOwners[owner.current] = append(currentOwners[owner.current], owner.canonical) - } - - // Co-occupancy at one schema boundary is an explicit convergence, not name - // reuse. Merge those identities before comparing the boundary with history. - currentNames, err := sortedMapKeysWithBudget(currentOwners, a.budget) - if err != nil { - return err - } - for _, current := range currentNames { - identities := currentOwners[current] - for _, identity := range identities[1:] { - if err := a.mergeAttributeIdentities(metric, identities[0], identity); err != nil { - return err - } - } - } - - stagedNames, err := sortedMapKeysWithBudget(stagedIdentities, a.budget) - if err != nil { - return err - } - for _, identity := range stagedNames { - if canonical, exists := knownAliases[identity]; exists { - same, err := a.sameAttributeIdentity(metric, identity, canonical) - if err != nil { - return err - } - if same { - continue - } - return ambiguousAttributeRenameError(metric, identity, identity, canonical) - } - } - for _, pair := range aliases { - if err := a.budget.reserveWork(1); err != nil { - return err - } - if _, identity := knownIdentities[pair.alias]; identity { - same, err := a.sameAttributeIdentity(metric, pair.alias, pair.canonical) - if err != nil { - return err - } - if !same { - return ambiguousAttributeRenameError(metric, pair.alias, pair.alias, pair.canonical) - } - } - if _, identity := stagedIdentities[pair.alias]; identity { - same, err := a.sameAttributeIdentity(metric, pair.alias, pair.canonical) - if err != nil { - return err - } - if !same { - return ambiguousAttributeRenameError(metric, pair.alias, pair.alias, pair.canonical) - } - } - } - - if knownIdentities == nil { - knownIdentities = map[string]struct{}{} - a.attributeIdentities[metric] = knownIdentities - } - for identity := range stagedIdentities { - knownIdentities[identity] = struct{}{} - } - if knownAliases == nil { - knownAliases = map[string]string{} - a.attributeAliases[metric] = knownAliases - } - for _, pair := range aliases { - if _, exists := knownAliases[pair.alias]; !exists { - knownAliases[pair.alias] = pair.canonical - } - } - } - - mapping := a.variants[idx].mapping - if len(aliases) > 0 && mapping.translatedLabels == nil { - mapping.translatedLabels = map[string]string{} - } - for _, pair := range aliases { - if err := a.budget.reserveWork(1); err != nil { - return err - } - if _, conflicted := a.conflicts[idx][pair.alias]; conflicted { - continue - } - if existing, exists := mapping.translatedLabels[pair.alias]; exists && existing != pair.canonical { - if err := a.markAttributeConflict(idx, pair.alias, existing, pair.canonical); err != nil { - return err - } - continue - } - if _, exists := mapping.translatedLabels[pair.alias]; !exists { - if err := a.reserveAttributeSlot(); err != nil { - return err - } - } - mapping.translatedLabels[pair.alias] = pair.canonical - } - if len(mapping.translatedLabels) == 0 { - mapping.translatedLabels = nil - } - return nil -} - -func (a *variantAccumulator) mergeExistingAttributeAliases(matchers []*labels.Matcher, metric string, lineage attributeLineage) error { - key, err := matcherKeyWithBudget(matchers, a.budget) - if err != nil { - return err - } - idx, exists := a.byMatchers[key] - if !exists { - return nil - } - return a.mergeAttributeAliases(idx, metric, lineage) -} - -func ambiguousAttributeRenameError(metric, alias, first, second string) error { - return fmt.Errorf("%w: attribute name %q resolves to both %q and %q for metric %q", errAmbiguousSchemaRename, alias, first, second, metric) -} - -func (a *variantAccumulator) ensureAttributeIdentity(metric, identity string) error { - parents := a.attributeParents[metric] - if parents == nil { - parents = map[string]string{} - a.attributeParents[metric] = parents - a.attributeSizes[metric] = map[string]uint32{} - } - if _, exists := parents[identity]; exists { - return nil - } - if err := a.budget.reserveWork(1); err != nil { - return err - } - parents[identity] = identity - a.attributeSizes[metric][identity] = 1 - return nil -} - -func (a *variantAccumulator) attributeIdentityRoot(metric, identity string) (string, error) { - if err := a.ensureAttributeIdentity(metric, identity); err != nil { - return "", err - } - parents := a.attributeParents[metric] - for parents[identity] != identity { - if err := a.budget.reserveWork(1); err != nil { - return "", err - } - identity = parents[identity] - } - return identity, nil -} - -func (a *variantAccumulator) mergeAttributeIdentities(metric, first, second string) error { - first, err := a.attributeIdentityRoot(metric, first) - if err != nil { - return err - } - second, err = a.attributeIdentityRoot(metric, second) - if err != nil { - return err - } - if first == second { - return nil - } - sizes := a.attributeSizes[metric] - if sizes[first] < sizes[second] || (sizes[first] == sizes[second] && second < first) { - first, second = second, first - } - a.attributeParents[metric][second] = first - sizes[first] += sizes[second] - return nil -} - -func (a *variantAccumulator) sameAttributeIdentity(metric, first, second string) (bool, error) { - first, err := a.attributeIdentityRoot(metric, first) - if err != nil { - return false, err - } - second, err = a.attributeIdentityRoot(metric, second) - if err != nil { - return false, err - } - return first == second, nil -} - -func (a *variantAccumulator) reserveAttributeSlot() error { - if a.attributeSlots >= maxSchemaExpansion { - return schemaExpansionError("attribute mappings") - } - a.attributeSlots++ - return nil -} - -func (a *variantAccumulator) markAttributeConflict(idx int, alias, first, second string) error { - if _, mapped := a.variants[idx].mapping.translatedLabels[alias]; !mapped { - if err := a.reserveAttributeSlot(); err != nil { - return err - } - } - delete(a.variants[idx].mapping.translatedLabels, alias) - a.conflicts[idx][alias] = struct{}{} - if a.validator != nil { - a.validator.warn("attribute name %q resolves to both %q and %q for metric %q; leaving it unmodified rather than merging distinct labels", - alias, first, second, a.anchorMetric) - } - return nil -} - -type attributeOwner struct { - canonical string - current string -} - -func resolvedAttributeOwnersWithBudget(lineage attributeLineage, budget *schemaExpansionBudget) ([]attributeOwner, error) { - canonicalNames, err := sortedMapKeysWithBudget(lineage, budget) - if err != nil { - return nil, err - } - resolved := make([]attributeOwner, 0, len(canonicalNames)) - for _, canonical := range canonicalNames { - currentNames, err := sortedMapKeysWithBudget(lineage[canonical], budget) - if err != nil { - return nil, err - } - for _, current := range currentNames { - if lineage[canonical][current] == attributeOriginResolved { - resolved = append(resolved, attributeOwner{canonical: canonical, current: current}) - } - } - } - return resolved, nil -} - -func attributeAliasesWithBudget(lineage attributeLineage, budget *schemaExpansionBudget) ([]attributeAlias, error) { - var aliases []attributeAlias - for canonical, currentNames := range lineage { - for current, origin := range currentNames { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if origin == attributeOriginResolved && current != canonical { - aliases = append(aliases, attributeAlias{alias: current, canonical: canonical}) - } - } - } - slices.SortFunc(aliases, func(a, b attributeAlias) int { - if c := strings.Compare(a.alias, b.alias); c != 0 { - return c - } - return strings.Compare(a.canonical, b.canonical) - }) - return aliases, nil -} - -// generateMatcherVariantsWithBudget resolves ordered schema transformations in both -// directions from version. Metric renames may branch; each returned matcher -// set carries only the attribute aliases valid for its accepted lineages. -func generateMatcherVariantsWithBudget(version string, schema *otelSchema, matchers []*labels.Matcher, canonicalAttrs []string, rv *renameValidator, budget *schemaExpansionBudget) ([]matcherVariant, error) { - return generateMatcherVariantsWithAnalysis(version, schema, matchers, canonicalAttrs, rv, nil, budget) -} - -func generateMatcherVariantsWithAnalysis(version string, schema *otelSchema, matchers []*labels.Matcher, canonicalAttrs []string, rv *renameValidator, analysis *schemaSafetyAnalysis, budget *schemaExpansionBudget) ([]matcherVariant, error) { - metricName, normalizedMatchers, satisfiable, err := normalizeMetricMatchers(matchers) - if err != nil { - return nil, err - } - if metricName == "" { - return nil, errMetricNameAnchor - } - if !satisfiable { - return []matcherVariant{{matchers: matchers}}, nil - } - matchers = normalizedMatchers - - var ambiguousMetrics map[string]struct{} - trackAttrIdentities := false - if analysis == nil { - ambiguousMetrics, err = reusedMetricNamesWithBudget(schema.revisions, budget) - if err != nil { - return nil, err - } - trackAttrIdentities, err = schemaHasAttributeRenamesWithBudget(schema.revisions, budget) - if err != nil { - return nil, err - } - } else { - ambiguousMetrics = analysis.reused - trackAttrIdentities = analysis.hasAttributeRenames - } - - attrs, err := newAttributeLineageWithBudget(canonicalAttrs, budget) - if err != nil { - return nil, err - } - - anchor := lineageState{ - matchers: matchers, - metric: metricName, - attrs: attrs, - } - acc, err := newVariantAccumulatorWithBudget(metricName, canonicalAttrs, rv, trackAttrIdentities, budget) - if err != nil { - return nil, err - } - acc.ambiguousMetrics = ambiguousMetrics - if err := acc.add(anchor); err != nil { - return nil, err - } - - anchorDeclared := rv == nil || rv.anchorDeclared - partition, err := traversalPartitionWithBudget(schema.revisions, version, metricName, anchorDeclared, budget) - if err != nil { - return nil, err - } - backwardEnd, forwardStart := partition, partition - if err := walkSchemaRevisions(schema.revisions[:backwardEnd], anchor, traverseBackward, schema, rv, acc, budget); err != nil { - return nil, err - } - if err := walkSchemaRevisions(schema.revisions[forwardStart:], anchor, traverseForward, schema, rv, acc, budget); err != nil { - return nil, err - } - return finalizeMatcherVariants(acc.variants, matchers, budget) -} - -type revisionTargets struct { - metric map[string]int - attribute map[string]int -} - -func targetsByFirstChange(revision schemaRevision, budget *schemaExpansionBudget) (revisionTargets, error) { - targets := revisionTargets{ - metric: map[string]int{}, - attribute: map[string]int{}, - } - for i, change := range revision.changes { - if err := budget.reserveWork(1); err != nil { - return revisionTargets{}, err - } - var targetMap map[string]int - var renames *directedRenames - switch { - case change.metricRenames != nil: - targetMap = targets.metric - renames = change.metricRenames - case change.attributeRenames != nil: - targetMap = targets.attribute - renames = change.attributeRenames.renames - } - if renames == nil { - continue - } - for target := range renames.reverse { - if err := budget.reserveWork(1); err != nil { - return revisionTargets{}, err - } - if _, exists := targetMap[target]; !exists { - targetMap[target] = i - } - } - } - return targets, nil -} - -func producedByEarlierChange(targets map[string]int, changeIndex int, name string) bool { - first, exists := targets[name] - return exists && first < changeIndex -} - -type lineageStateSet struct { - states []lineageState - seen map[string]struct{} - budget *schemaExpansionBudget -} - -func newLineageStateSet(capacity int, budget *schemaExpansionBudget) *lineageStateSet { - if capacity > maxSchemaExpansion { - capacity = maxSchemaExpansion - } - return &lineageStateSet{ - states: make([]lineageState, 0, capacity), - seen: make(map[string]struct{}, capacity), - budget: budget, - } -} - -func (s *lineageStateSet) add(state lineageState) error { - key, err := lineageStateKeyWithBudget(state, s.budget) - if err != nil { - return err - } - if _, exists := s.seen[key]; exists { - return nil - } - if len(s.states) >= maxSchemaExpansion { - return schemaExpansionError("lineage states") - } - s.seen[key] = struct{}{} - s.states = append(s.states, state) - return nil -} - -func walkSchemaRevisions(revisions []schemaRevision, anchor lineageState, direction traversalDirection, schema *otelSchema, rv *renameValidator, acc *variantAccumulator, budget *schemaExpansionBudget) error { - states := []lineageState{anchor} - for step := 0; step < len(revisions) && len(states) > 0; step++ { - if err := budget.reserveWork(1); err != nil { - return err - } - revisionIndex := step - if direction == traverseBackward { - revisionIndex = len(revisions) - 1 - step - } - revision := revisions[revisionIndex] - targets, err := targetsByFirstChange(revision, budget) - if err != nil { - return err - } - for i := range states { - if err := budget.reserveWork(1); err != nil { - return err - } - states[i].renamedFrom = "" - states[i].metricProduced = false - states[i].metricOriginPending = false - states[i].pendingAttributeMatchers = nil - } - - for changeStep := 0; changeStep < len(revision.changes) && len(states) > 0; changeStep++ { - if err := budget.reserveWork(1); err != nil { - return err - } - changeIndex := changeStep - if direction == traverseBackward { - changeIndex = len(revision.changes) - 1 - changeStep - } - change := revision.changes[changeIndex] - var err error - switch { - case change.metricRenames != nil: - states, err = applyMetricRenames(states, change.metricRenames, direction, revision.version, changeIndex, targets.metric, rv, budget) - case change.attributeRenames != nil: - states, err = applyAttributeRenames(states, change.attributeRenames, direction, changeIndex, targets.attribute, acc, budget) - } - if err != nil { - return err - } - } - - destinationVersion := revision.version - if direction == traverseBackward { - var found bool - destinationVersion, found, err = schema.predecessorOfWithBudget(revision.version, budget) - if err != nil { - return err - } - if !found { - destinationVersion = "" - } - } - validated := newLineageStateSet(len(states), budget) - for _, state := range states { - if err := budget.reserveWork(1); err != nil { - return err - } - if state.metricOriginPending || len(state.pendingAttributeMatchers) > 0 { - continue - } - state.attrs, err = prunePendingAttributeOrigins(state.attrs, budget) - if err != nil { - return err - } - if state.renamedFrom != "" && !rv.allowRevision(revision.version, destinationVersion, state.renamedFrom, state.metric) { - continue - } - state.renamedFrom = "" - state.metricProduced = false - state.metricOriginPending = false - state.pendingAttributeMatchers = nil - if err := validated.add(state); err != nil { - return err - } - } - states = validated.states - for _, state := range states { - if err := acc.add(state); err != nil { - return err - } - } - } - return nil -} - -func (r *directedRenames) targets(name string, direction traversalDirection) (targets []string, renamed, wrongSide bool) { - if direction == traverseForward { - if target, ok := r.forward[name]; ok { - return []string{target}, true, false - } - _, wrongSide = r.reverse[name] - return nil, false, wrongSide - } - if targets, ok := r.reverse[name]; ok { - return targets, true, false - } - _, wrongSide = r.forward[name] - return nil, false, wrongSide -} - -func applyMetricRenames(states []lineageState, renames *directedRenames, direction traversalDirection, revision string, changeIndex int, earlierTargets map[string]int, rv *renameValidator, budget *schemaExpansionBudget) ([]lineageState, error) { - out := newLineageStateSet(len(states), budget) - for _, state := range states { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - targets, renamed, wrongSide := renames.targets(state.metric, direction) - if !renamed { - if wrongSide { - if direction == traverseForward && state.metricProduced { - if err := out.add(state); err != nil { - return nil, err - } - continue - } - if rv != nil { - rv.warn("schema version %s encounters metric %q only on the wrong side of a rename while traversing %s; stopping at this metric lifecycle boundary", - revision, state.metric, direction) - } - continue - } - if err := out.add(state); err != nil { - return nil, err - } - continue - } - - for _, target := range targets { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - next := state - if next.renamedFrom == "" { - next.renamedFrom = state.metric - } - next.metric = target - var err error - next.matchers, err = replaceMetricMatchersWithBudget(state.matchers, state.metric, target, budget) - if err != nil { - return nil, err - } - next.metricOriginPending = false - if direction == traverseForward { - next.metricProduced = true - } - if err := out.add(next); err != nil { - return nil, err - } - } - - _, isSource := renames.forward[state.metric] - if direction == traverseBackward && !isSource && producedByEarlierChange(earlierTargets, changeIndex, state.metric) { - next := state - next.metricOriginPending = true - if err := out.add(next); err != nil { - return nil, err - } - } - } - return out.states, nil -} - func replaceMetricMatchersWithBudget(matchers []*labels.Matcher, currentMetric, targetMetric string, budget *schemaExpansionBudget) ([]*labels.Matcher, error) { if err := budget.reserveWork(uint64(len(matchers))); err != nil { return nil, err @@ -1518,504 +203,6 @@ func replaceMetricMatchersWithBudget(matchers []*labels.Matcher, currentMetric, return out, nil } -func applyAttributeRenames(states []lineageState, step *attributeRenameStep, direction traversalDirection, changeIndex int, earlierTargets map[string]int, acc *variantAccumulator, budget *schemaExpansionBudget) ([]lineageState, error) { - out := newLineageStateSet(len(states), budget) - for _, state := range states { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if !step.appliesTo(state.metric) { - if err := out.add(state); err != nil { - return nil, err - } - continue - } - - lineage, err := seedAttributeLineage(state.attrs, step.renames, direction, budget) - if err != nil { - return nil, err - } - lineage, err = transformAttributeLineage(lineage, step.renames, direction, changeIndex, earlierTargets, budget) - if err != nil { - return nil, err - } - if err := acc.mergeExistingAttributeAliases(state.matchers, state.metric, lineage); err != nil { - return nil, err - } - state.attrs = lineage - matcherStates, err := transformAttributeMatcherStatesWithBudget(state, step.renames, direction, changeIndex, earlierTargets, budget) - if err != nil { - return nil, err - } - for _, next := range matcherStates { - if err := out.add(next); err != nil { - return nil, err - } - } - } - return out.states, nil -} - -func seedAttributeLineage(lineage attributeLineage, renames *directedRenames, direction traversalDirection, budget *schemaExpansionBudget) (attributeLineage, error) { - out := make(attributeLineage, len(lineage)) - entries := 0 - seenCurrent := map[string]struct{}{} - canonicalNames := make([]string, 0, len(lineage)) - for canonical := range lineage { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - canonicalNames = append(canonicalNames, canonical) - } - slices.Sort(canonicalNames) - for _, canonical := range canonicalNames { - currentNames := make([]string, 0, len(lineage[canonical])) - for current := range lineage[canonical] { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - currentNames = append(currentNames, current) - } - slices.Sort(currentNames) - for _, current := range currentNames { - if err := addAttributeOrigin(out, canonical, current, lineage[canonical][current], &entries, budget); err != nil { - return nil, err - } - seenCurrent[current] = struct{}{} - } - } - - var currentSide map[string][]string - if direction == traverseForward { - currentSide = make(map[string][]string, len(renames.forward)) - for source := range renames.forward { - currentSide[source] = nil - } - } else { - currentSide = renames.reverse - } - names := make([]string, 0, len(currentSide)) - for name := range currentSide { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - names = append(names, name) - } - slices.Sort(names) - for _, name := range names { - if _, exists := seenCurrent[name]; exists { - continue - } - if err := addAttributeOrigin(out, name, name, attributeOriginResolved, &entries, budget); err != nil { - return nil, err - } - } - return out, nil -} - -func transformAttributeLineage(lineage attributeLineage, renames *directedRenames, direction traversalDirection, changeIndex int, earlierTargets map[string]int, budget *schemaExpansionBudget) (attributeLineage, error) { - if len(lineage) == 0 { - return nil, nil - } - out := make(attributeLineage, len(lineage)) - entries := 0 - canonicalNames := make([]string, 0, len(lineage)) - for canonical := range lineage { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - canonicalNames = append(canonicalNames, canonical) - } - slices.Sort(canonicalNames) - for _, canonical := range canonicalNames { - currentNames := make([]string, 0, len(lineage[canonical])) - for current := range lineage[canonical] { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - currentNames = append(currentNames, current) - } - slices.Sort(currentNames) - for _, current := range currentNames { - origin := lineage[canonical][current] - targets, renamed, _ := renames.targets(current, direction) - if !renamed { - if err := addAttributeOrigin(out, canonical, current, origin, &entries, budget); err != nil { - return nil, err - } - continue - } - for _, target := range targets { - if err := addAttributeOrigin(out, canonical, target, attributeOriginResolved, &entries, budget); err != nil { - return nil, err - } - } - - _, isSource := renames.forward[current] - if direction == traverseBackward && !isSource && producedByEarlierChange(earlierTargets, changeIndex, current) { - if err := addAttributeOrigin(out, canonical, current, attributeOriginPending, &entries, budget); err != nil { - return nil, err - } - } - } - } - return out, nil -} - -func addAttributeOrigin(lineage attributeLineage, canonical, current string, origin attributeOrigin, entries *int, budget *schemaExpansionBudget) error { - if err := budget.reserveWork(1); err != nil { - return err - } - currentNames := lineage[canonical] - if currentNames == nil { - currentNames = map[string]attributeOrigin{} - lineage[canonical] = currentNames - } - if existing, exists := currentNames[current]; exists { - if origin < existing { - currentNames[current] = origin - } - return nil - } - if *entries >= maxSchemaExpansion { - return schemaExpansionError("attribute lineage") - } - currentNames[current] = origin - (*entries)++ - return nil -} - -type attributeMatcherGroup struct { - name string - indexes []int -} - -func attributeMatcherGroups(matchers []*labels.Matcher, budget *schemaExpansionBudget) ([]attributeMatcherGroup, error) { - var groups []attributeMatcherGroup - byName := map[string]int{} - for i, matcher := range matchers { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if matcher.Name == model.MetricNameLabel { - continue - } - idx, exists := byName[matcher.Name] - if !exists { - idx = len(groups) - byName[matcher.Name] = idx - groups = append(groups, attributeMatcherGroup{name: matcher.Name}) - } - groups[idx].indexes = append(groups[idx].indexes, i) - } - return groups, nil -} - -func finalizeMatcherVariants(variants []matcherVariant, canonicalMatchers []*labels.Matcher, budget *schemaExpansionBudget) ([]matcherVariant, error) { - renamedCanonical := map[string]struct{}{} - for _, variant := range variants { - if variant.mapping == nil { - continue - } - for alias, canonical := range variant.mapping.translatedLabels { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if alias != canonical { - renamedCanonical[canonical] = struct{}{} - } - } - } - if len(renamedCanonical) == 0 { - return variants, nil - } - - groups, err := attributeMatcherGroups(canonicalMatchers, budget) - if err != nil { - return nil, err - } - for _, group := range groups { - if _, renamed := renamedCanonical[group.name]; !renamed { - continue - } - matchesEmpty := true - for _, index := range group.indexes { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if !canonicalMatchers[index].Matches("") { - matchesEmpty = false - break - } - } - if matchesEmpty { - return nil, fmt.Errorf("%w: renamed attribute %q matcher conjunction matches an absent label", errUnsafeSchemaMatcher, group.name) - } - } - - for i := range variants { - if variants[i].mapping != nil && len(variants[i].mapping.translatedLabels) > 0 { - variants[i].canonicalMatchers = canonicalMatchers - } - } - return variants, nil -} - -func transformAttributeMatcherStatesWithBudget(state lineageState, renames *directedRenames, direction traversalDirection, changeIndex int, earlierTargets map[string]int, budget *schemaExpansionBudget) ([]lineageState, error) { - states := []lineageState{state} - groups, err := attributeMatcherGroups(state.matchers, budget) - if err != nil { - return nil, err - } - for _, group := range groups { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - targets, renamed, _ := renames.targets(group.name, direction) - if !renamed { - continue - } - - nextStates := newLineageStateSet(len(states), budget) - for _, current := range states { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - for _, target := range targets { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - next := current - next.matchers = slices.Clone(current.matchers) - for _, i := range group.indexes { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - matcher := current.matchers[i] - next.matchers[i] = labels.MustNewMatcher(matcher.Type, target, matcher.Value) - } - next.pendingAttributeMatchers, err = updatePendingAttributeMatchers(current.pendingAttributeMatchers, group.indexes, false, budget) - if err != nil { - return nil, err - } - if err := nextStates.add(next); err != nil { - return nil, err - } - } - - _, isSource := renames.forward[group.name] - if direction == traverseBackward && !isSource && producedByEarlierChange(earlierTargets, changeIndex, group.name) { - next := current - next.pendingAttributeMatchers, err = updatePendingAttributeMatchers(current.pendingAttributeMatchers, group.indexes, true, budget) - if err != nil { - return nil, err - } - if err := nextStates.add(next); err != nil { - return nil, err - } - } - } - states = nextStates.states - } - return states, nil -} - -func updatePendingAttributeMatchers(pending map[int]struct{}, indexes []int, add bool, budget *schemaExpansionBudget) (map[int]struct{}, error) { - updated := make(map[int]struct{}, len(pending)+len(indexes)) - for i := range pending { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - updated[i] = struct{}{} - } - for _, i := range indexes { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if add { - updated[i] = struct{}{} - } else { - delete(updated, i) - } - } - if len(updated) == 0 { - return nil, nil - } - return updated, nil -} - -func prunePendingAttributeOrigins(lineage attributeLineage, budget *schemaExpansionBudget) (attributeLineage, error) { - if len(lineage) == 0 { - return nil, nil - } - out := make(attributeLineage, len(lineage)) - for canonical, currentNames := range lineage { - for current, origin := range currentNames { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - if origin != attributeOriginResolved { - continue - } - if out[canonical] == nil { - out[canonical] = map[string]attributeOrigin{} - } - out[canonical][current] = origin - } - } - if len(out) == 0 { - return nil, nil - } - return out, nil -} - -func deduplicateLineageStatesWithBudget(states []lineageState, budget *schemaExpansionBudget) ([]lineageState, error) { - out := newLineageStateSet(len(states), budget) - for _, state := range states { - if err := out.add(state); err != nil { - return nil, err - } - } - return out.states, nil -} - -func lineageStateKeyWithBudget(state lineageState, budget *schemaExpansionBudget) (string, error) { - remaining := budget.remainingKeyBytes() - var size uint64 - addSize := func(n uint64) error { - if n > remaining-size { - return schemaExpansionLimitError("deduplication key bytes", budget.limits.keyBytes) - } - size += n - return nil - } - addPart := func(value string) error { - return addSize(keyPartSize(uint64(len(value)))) - } - addInt := func(value int) error { - return addSize(keyIntSize(value)) - } - - if err := budget.reserveWork(1); err != nil { - return "", err - } - if err := addInt(len(state.matchers)); err != nil { - return "", err - } - for _, matcher := range state.matchers { - if err := budget.reserveWork(1); err != nil { - return "", err - } - if err := addSize(1); err != nil { - return "", err - } - if err := addPart(matcher.Name); err != nil { - return "", err - } - if err := addPart(matcher.Value); err != nil { - return "", err - } - } - if err := addPart(state.metric); err != nil { - return "", err - } - if err := addPart(state.renamedFrom); err != nil { - return "", err - } - if err := addSize(2); err != nil { - return "", err - } - if err := addInt(len(state.pendingAttributeMatchers)); err != nil { - return "", err - } - for i := range state.pendingAttributeMatchers { - if err := budget.reserveWork(1); err != nil { - return "", err - } - if err := addInt(i); err != nil { - return "", err - } - } - if err := addInt(len(state.attrs)); err != nil { - return "", err - } - for canonical, currentNames := range state.attrs { - if err := budget.reserveWork(1); err != nil { - return "", err - } - if err := addPart(canonical); err != nil { - return "", err - } - if err := addInt(len(currentNames)); err != nil { - return "", err - } - for current := range currentNames { - if err := budget.reserveWork(1); err != nil { - return "", err - } - if err := addPart(current); err != nil { - return "", err - } - if err := addSize(1); err != nil { - return "", err - } - } - } - if err := budget.reserveKeyBytes(size); err != nil { - return "", err - } - - var b strings.Builder - b.Grow(int(size)) - writeKeyInt(&b, len(state.matchers)) - for _, matcher := range state.matchers { - b.WriteByte(byte(matcher.Type)) - writeKeyPart(&b, matcher.Name) - writeKeyPart(&b, matcher.Value) - } - writeKeyPart(&b, state.metric) - writeKeyPart(&b, state.renamedFrom) - if state.metricProduced { - b.WriteByte(1) - } else { - b.WriteByte(0) - } - if state.metricOriginPending { - b.WriteByte(1) - } else { - b.WriteByte(0) - } - pendingMatcherIndexes := make([]int, 0, len(state.pendingAttributeMatchers)) - for i := range state.pendingAttributeMatchers { - pendingMatcherIndexes = append(pendingMatcherIndexes, i) - } - slices.Sort(pendingMatcherIndexes) - writeKeyInt(&b, len(pendingMatcherIndexes)) - for _, i := range pendingMatcherIndexes { - writeKeyInt(&b, i) - } - canonicalNames := make([]string, 0, len(state.attrs)) - for canonical := range state.attrs { - canonicalNames = append(canonicalNames, canonical) - } - slices.Sort(canonicalNames) - writeKeyInt(&b, len(canonicalNames)) - for _, canonical := range canonicalNames { - writeKeyPart(&b, canonical) - currentNames := make([]string, 0, len(state.attrs[canonical])) - for current := range state.attrs[canonical] { - currentNames = append(currentNames, current) - } - slices.Sort(currentNames) - writeKeyInt(&b, len(currentNames)) - for _, current := range currentNames { - writeKeyPart(&b, current) - b.WriteByte(byte(state.attrs[canonical][current])) - } - } - return b.String(), nil -} - func matcherKeyWithBudget(matchers []*labels.Matcher, budget *schemaExpansionBudget) (string, error) { if err := budget.reserveWork(1); err != nil { return "", err @@ -2079,27 +266,9 @@ func writeKeyInt(b *strings.Builder, value int) { } type queryContext struct { - // warnings holds resolution problems that make the answer incomplete or - // potentially ambiguous. They are surfaced through Warnings(). warnings []string } -// getSemconv returns the semconv parsed from url, fetching it via the -// embedded registry on a cache miss. -func (e *schemaEngine) getSemconv(url string) (semconv, error) { - if sc, ok := e.semconvCache.get(url); ok { - return sc, nil - } - sc, err := e.fetchSemconv(url) - if err != nil { - return semconv{}, err - } - e.semconvCache.set(url, sc) - return sc, nil -} - -// getOTelSchema returns the OTel schema parsed from url, fetching it via the -// embedded registry on a cache miss. func (e *schemaEngine) getOTelSchema(url string) (otelSchema, error) { if s, ok := e.otelSchemaCache.get(url); ok { return s, nil @@ -2137,18 +306,16 @@ func (e *schemaEngine) getTransformSchemasByTarget(targetURL string) []*otelSche return matched } -var errSemconvURLRequired = errors.New("semconvURL is required") - // findMatcherVariants returns all variants to match for a single schematized -// metric selection. For 1.0.0/1.1.0 schemas, semconvURL points to a semantic -// conventions file and is required. For transform/2.0 policy files, semconvURL -// is optional because the transformation file is self-contained and directional. +// metric selection using the transform/2.0 policy referenced by schemaURL. func (e *schemaEngine) findMatcherVariants(semconvURL, schemaURL string, originalMatchers []*labels.Matcher) ([]matcherVariant, queryContext, error) { - if semconvURL == "" && schemaURL == "" { + if schemaURL == "" { return nil, queryContext{}, errSemconvURLRequired } + if semconvURL != "" && !registryURLRe.MatchString(semconvURL) { + return nil, queryContext{}, fmt.Errorf("invalid semconv URL %q: must match %s", semconvURL, registryURLRe.String()) + } - // Filter out the wrapper's reserved matchers. matchers := stripReservedLabels(originalMatchers) metricName, normalizedMatchers, satisfiable, err := normalizeMetricMatchers(matchers) @@ -2163,123 +330,39 @@ func (e *schemaEngine) findMatcherVariants(semconvURL, schemaURL string, origina } matchers = normalizedMatchers - if schemaURL != "" { - schema, err := e.getOTelSchema(schemaURL) - if err != nil { - // If schemaURL is a target schema URL or registry/ rather than - // a direct .yaml filename, check if one or more transform/2.0 policies target it. - if targetSchemas := e.getTransformSchemasByTarget(schemaURL); len(targetSchemas) > 0 { - if semconvURL != "" && !registryURLRe.MatchString(semconvURL) { - return nil, queryContext{}, fmt.Errorf("invalid semconv URL %q: must match %s", semconvURL, registryURLRe.String()) - } - budget := newSchemaExpansionBudget(e.limits) - variants, warnings, genErr := generateTransformPolicyVariantsMulti(targetSchemas, matchers, budget) - if genErr != nil { - return nil, queryContext{}, genErr - } - return variants, queryContext{warnings: warnings}, nil - } - return nil, queryContext{}, err - } - if schema.FileFormat == transformFileFormat { - if semconvURL != "" && !registryURLRe.MatchString(semconvURL) { - return nil, queryContext{}, fmt.Errorf("invalid semconv URL %q: must match %s", semconvURL, registryURLRe.String()) - } + schema, err := e.getOTelSchema(schemaURL) + if err != nil { + // If schemaURL is a target schema URL or registry/ rather than + // a direct .yaml filename, check if one or more transform/2.0 policies target it. + if targetSchemas := e.getTransformSchemasByTarget(schemaURL); len(targetSchemas) > 0 { budget := newSchemaExpansionBudget(e.limits) - variants, warnings, err := generateTransformPolicyVariants(&schema, matchers, budget) - if err != nil { - return nil, queryContext{}, err + variants, warnings, genErr := generateTransformPolicyVariantsMulti(targetSchemas, matchers, budget) + if genErr != nil { + return nil, queryContext{}, genErr } return variants, queryContext{warnings: warnings}, nil } - if semconvURL == "" { - return nil, queryContext{}, errSemconvURLRequired - } - } - - // Fetch semantic conventions for the anchor version (also validates the URL). - sc, err := e.getSemconv(semconvURL) - if err != nil { return nil, queryContext{}, err } - var warnings []string - ambiguousAnchor := slices.Contains(sc.ambiguousMetrics, metricName) - if ambiguousAnchor { - warnings = append(warnings, fmt.Sprintf( - "metric %q is declared by more than one group in semconv %s; its attributes and unit are ambiguous and results may fuse distinct metrics", - metricName, sc.version, - )) + budget := newSchemaExpansionBudget(e.limits) + variants, warnings, err := generateTransformPolicyVariants(&schema, matchers, budget) + if err != nil { + return nil, queryContext{}, err } - - // Generate schema-version rename variants. In production schemaURL is always - // set (classifyMatchers gates fan-out on it); the empty case is reached only - // by direct unit tests and falls through to the unmodified matchers. - allVariants := []matcherVariant{{ - matchers: matchers, - mapping: buildLabelMapping(metricName, nil), - }} - if schemaURL != "" { - schema, err := e.getOTelSchema(schemaURL) - if err != nil { - return nil, queryContext{}, err - } - if ambiguousAnchor { - // There is no unique anchor definition with which to validate a - // lineage or canonicalise its attributes. Query the surface name - // directly and report the ambiguity instead of choosing one group. - return allVariants, queryContext{warnings: warnings}, nil - } - - // Corroborate each rename against the semconv of the versions it - // connects, so a schema edge that joins two unrelated metrics sharing a - // surface name is reported rather than silently merged. - budget := newSchemaExpansionBudget(e.limits) - analysis := e.schemaSafetyAnalysis(schemaURL, &schema) - if analysis.err != nil { - return nil, queryContext{}, analysis.err - } - if err := budget.reserveWork(analysis.work); err != nil { - return nil, queryContext{}, err - } - rv, err := newRenameValidator(&schema, e.newMetricLookup(semconvURL), sc, metricName, budget) - if err != nil { - return nil, queryContext{}, err - } - allVariants, err = generateMatcherVariantsWithAnalysis(sc.version, &schema, matchers, sc.attributesOf(metricName), rv, &analysis, budget) - if err != nil { - return nil, queryContext{}, err - } - warnings = append(warnings, rv.warnings...) - } - - return allVariants, queryContext{warnings: warnings}, nil + return variants, queryContext{warnings: warnings}, nil } -// labelMapping rewrites a returned series' names to the queried semantic- -// conventions version: translatedMetric is the queried (anchor) metric name -// that every variant collapses to, and translatedLabels maps each historical -// attribute alias back to its anchor-version name. type labelMapping struct { - translatedLabels map[string]string // historical attribute alias → anchor name, e.g. "user" -> "tenant" + translatedLabels map[string]string translatedMetric string policyTransform *policySeriesTransformer } -// buildLabelMapping creates the mapping used to rewrite result labels back to -// the requested semantic-conventions version: the result metric name maps to -// the queried (anchor) name, and translatedLabels maps each historical -// attribute alias back to its anchor-version name (nil/empty when no attribute -// was renamed). func buildLabelMapping(metricName string, translatedLabels map[string]string) *labelMapping { return &labelMapping{translatedMetric: metricName, translatedLabels: translatedLabels} } -// aliasesOf returns name together with every historical alias that maps to it, -// i.e. the set of label names a returned series may carry for the canonical -// name. It is the inverse of translatedLabels and is used to fan LabelValues -// out across a renamed attribute's historical names. The metric name has no -// attribute aliases, so it is returned unchanged. func (m *labelMapping) aliasesOf(name string) []string { aliases := []string{name} for alias, canonical := range m.translatedLabels { @@ -2291,52 +374,11 @@ func (m *labelMapping) aliasesOf(name string) []string { return aliases } -// transformOTelSchemaLabels transforms series labels to the current semantic -// conventions version using the label mapping. func transformOTelSchemaLabels(originalLabels labels.Labels, mapping *labelMapping) (labels.Labels, error) { if mapping != nil && mapping.policyTransform != nil { return mapping.policyTransform.transformLabels(originalLabels) } - type mappedLabel struct { - source string - value string - } - - builder := labels.NewScratchBuilder(originalLabels.Len()) - mapped := make(map[string]mappedLabel, originalLabels.Len()) - var transformErr error - originalLabels.Range(func(l labels.Label) { - if transformErr != nil { - return - } - - name := l.Name - value := l.Value - switch l.Name { - case semconvURLLabel, schemaURLLabel: - return - case model.MetricNameLabel: - value = mapping.translatedMetric - default: - if canonical, ok := mapping.translatedLabels[l.Name]; ok { - name = canonical - } - } - - if existing, ok := mapped[name]; ok { - if existing.value != value { - transformErr = fmt.Errorf("semconv label transformation maps %q and %q to %q with conflicting values", existing.source, l.Name, name) - } - return - } - mapped[name] = mappedLabel{source: l.Name, value: value} - builder.Add(name, value) - }) - if transformErr != nil { - return labels.EmptyLabels(), transformErr - } - builder.Sort() - return builder.Labels(), nil + return originalLabels, nil } func labelMappingChangesLabels(lbls labels.Labels, mapping *labelMapping) bool { diff --git a/storage/semconv/engine_test.go b/storage/semconv/engine_test.go deleted file mode 100644 index 1cb1191ebf..0000000000 --- a/storage/semconv/engine_test.go +++ /dev/null @@ -1,1144 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv - -import ( - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" -) - -func metricChange(renames map[string]string) schemaChange { - return schemaChange{metricRenames: newDirectedRenames(renames)} -} - -func attributeChange(renames map[string]string, metrics ...string) schemaChange { - step := &attributeRenameStep{renames: newDirectedRenames(renames)} - if len(metrics) > 0 { - step.scopeSpecified = true - step.applyToMetrics = make(map[string]struct{}, len(metrics)) - for _, metric := range metrics { - step.applyToMetrics[metric] = struct{}{} - } - } - return schemaChange{attributeRenames: step} -} - -func testSchema(revisions ...schemaRevision) *otelSchema { - return &otelSchema{revisions: revisions} -} - -func testSchemaWithVersions(allVersions []string, revisions ...schemaRevision) *otelSchema { - return &otelSchema{allVersions: allVersions, revisions: revisions} -} - -func equalMatchers(metric string, attrs ...string) []*labels.Matcher { - matchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, metric)} - for _, attr := range attrs { - matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, attr, "value")) - } - return matchers -} - -func requireMatcherVariants(t *testing.T, version string, schema *otelSchema, matchers []*labels.Matcher, canonicalAttrs []string, rv *renameValidator) []matcherVariant { - t.Helper() - variants, err := generateMatcherVariantsWithBudget(version, schema, matchers, canonicalAttrs, rv, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - return variants -} - -func requireVariantAccumulator(t *testing.T, anchorMetric string, canonicalAttrs []string, rv *renameValidator) *variantAccumulator { - t.Helper() - acc, err := newVariantAccumulatorWithBudget(anchorMetric, canonicalAttrs, rv, true, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - return acc -} - -func requireLineageStateKey(t *testing.T, state lineageState) string { - t.Helper() - key, err := lineageStateKeyWithBudget(state, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - return key -} - -func variantNames(variants []matcherVariant) []string { - names := make([]string, 0, len(variants)) - for _, variant := range variants { - name, err := extractMetricName(variant.matchers) - if err == nil { - names = append(names, name) - } - } - return names -} - -func variantMetricAndAttribute(variant matcherVariant) string { - metric, attr := "", "" - for _, matcher := range variant.matchers { - if matcher.Name == labels.MetricName { - metric = matcher.Value - } else { - attr = matcher.Name - } - } - return metric + "/" + attr -} - -func TestFindMatcherVariants_RequiresSemconvURL(t *testing.T) { - e := newSchemaEngine(embeddedRegistry) - _, _, err := e.findMatcherVariants("", "", equalMatchers("http.server.duration")) - require.Error(t, err) - require.Contains(t, err.Error(), "semconvURL is required") -} - -func TestFindMatcherVariants_RequiresMetricNameAnchorBeforeRegistryLookup(t *testing.T) { - e := newSchemaEngine(embeddedRegistry) - _, _, err := e.findMatcherVariants("not-a-registry-path", "also-invalid", []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, "job", "api"), - }) - require.ErrorIs(t, err, errMetricNameAnchor) -} - -func TestRevisionPartition(t *testing.T) { - revisions := []schemaRevision{{version: "1.0.0"}, {version: "1.1.0"}, {version: "1.2.0"}} - tests := []struct { - version string - want int - }{ - {version: "0.9.0", want: 0}, - {version: "1.0.0", want: 1}, - {version: "1.0.5", want: 1}, - {version: "v1.1.0", want: 2}, - {version: "2.0.0", want: 3}, - } - for _, tc := range tests { - t.Run(tc.version, func(t *testing.T) { - got, err := revisionPartitionWithBudget(revisions, tc.version, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - require.Equal(t, tc.want, got) - }) - } - got, err := revisionPartitionWithBudget(nil, "1.0.0", newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - require.Zero(t, got) -} - -func TestNormalizeMetricMatchers(t *testing.T) { - t.Run("normalizes compatible constraints around an equality", func(t *testing.T) { - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, `metric\.(current|old)`), - labels.MustNewMatcher(labels.MatchEqual, "attribute", "value"), - labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, "metric.current"), - labels.MustNewMatcher(labels.MatchNotEqual, labels.MetricName, "metric.old"), - } - - name, got, satisfiable, err := normalizeMetricMatchers(matchers) - require.NoError(t, err) - require.True(t, satisfiable) - require.Equal(t, "metric.current", name) - require.Len(t, got, 2) - require.Equal(t, labels.MatchEqual, got[0].Type) - require.Equal(t, labels.MetricName, got[0].Name) - require.Equal(t, "metric.current", got[0].Value) - require.Same(t, matchers[1], got[1]) - }) - - t.Run("keeps contradictory constraints for direct evaluation", func(t *testing.T) { - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, "metric.current"), - labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, "metric.old"), - } - - name, got, satisfiable, err := normalizeMetricMatchers(matchers) - require.NoError(t, err) - require.False(t, satisfiable) - require.Equal(t, "metric.current", name) - require.Same(t, matchers[0], got[0]) - require.Same(t, matchers[1], got[1]) - }) - - t.Run("finds a non-empty equality independent of matcher order", func(t *testing.T) { - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, ""), - labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, "metric.current"), - } - - name, got, satisfiable, err := normalizeMetricMatchers(matchers) - require.NoError(t, err) - require.False(t, satisfiable) - require.Equal(t, "metric.current", name) - require.Same(t, matchers[0], got[0]) - require.Same(t, matchers[1], got[1]) - }) - - t.Run("requires a non-empty exact anchor", func(t *testing.T) { - for _, matcher := range []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, "metric.*"), - labels.MustNewMatcher(labels.MatchNotEqual, labels.MetricName, "metric.old"), - labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, ""), - } { - _, _, _, err := normalizeMetricMatchers([]*labels.Matcher{matcher}) - require.ErrorIs(t, err, errMetricNameAnchor) - require.ErrorContains(t, err, "non-empty equality matcher") - } - }) -} - -func TestGenerateMatcherVariants(t *testing.T) { - t.Run("walks chained revisions in both directions", func(t *testing.T) { - schema := testSchema( - schemaRevision{version: "1.1.0", changes: []schemaChange{ - metricChange(map[string]string{"metric.v1": "metric.v2"}), - attributeChange(map[string]string{"attr.v1": "attr.v2"}), - }}, - schemaRevision{version: "1.2.0", changes: []schemaChange{ - metricChange(map[string]string{"metric.v2": "metric.v3"}), - attributeChange(map[string]string{"attr.v2": "attr.v3"}), - }}, - ) - - backward := requireMatcherVariants(t, "1.2.0", schema, equalMatchers("metric.v3", "attr.v3"), []string{"attr.v3"}, nil) - require.Equal(t, []string{"metric.v3/attr.v3", "metric.v2/attr.v2", "metric.v1/attr.v1"}, []string{ - variantMetricAndAttribute(backward[0]), - variantMetricAndAttribute(backward[1]), - variantMetricAndAttribute(backward[2]), - }) - - forward := requireMatcherVariants(t, "1.0.0", schema, equalMatchers("metric.v1", "attr.v1"), []string{"attr.v1"}, nil) - require.Equal(t, []string{"metric.v1/attr.v1", "metric.v2/attr.v2", "metric.v3/attr.v3"}, []string{ - variantMetricAndAttribute(forward[0]), - variantMetricAndAttribute(forward[1]), - variantMetricAndAttribute(forward[2]), - }) - }) - - t.Run("emits only revision boundaries", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - metricChange(map[string]string{"metric.a": "metric.b"}), - attributeChange(map[string]string{"attr.a": "attr.b"}), - metricChange(map[string]string{"metric.b": "metric.c"}), - attributeChange(map[string]string{"attr.b": "attr.c"}), - }, - }) - variants := requireMatcherVariants(t, "1.1.0", schema, equalMatchers("metric.c", "attr.c"), []string{"attr.c"}, nil) - require.Len(t, variants, 2) - require.Equal(t, "metric.c/attr.c", variantMetricAndAttribute(variants[0])) - require.Equal(t, "metric.a/attr.a", variantMetricAndAttribute(variants[1])) - }) - - t.Run("validates only the final name of an ordered revision", func(t *testing.T) { - schema := testSchemaWithVersions([]string{"1.0.0", "1.1.0"}, schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - metricChange(map[string]string{"metric.a": "metric.intermediate"}), - metricChange(map[string]string{"metric.intermediate": "metric.current"}), - }, - }) - rv := &renameValidator{ - anchorName: "metric.current", - anchorVersion: "1.1.0", - anchorDef: metricDef{unit: "s", instrument: "histogram"}, - anchorKnown: true, - seen: map[string]struct{}{}, - lookup: func(version, name string) (metricDef, metricLookupStatus) { - if version == "1.0.0" && name == "metric.a" { - return metricDef{unit: "s", instrument: "histogram"}, metricDeclared - } - return metricDef{}, metricUndeclared - }, - } - variants := requireMatcherVariants(t, "1.1.0", schema, equalMatchers("metric.current"), nil, rv) - require.Equal(t, []string{"metric.current", "metric.a"}, variantNames(variants)) - require.Empty(t, rv.warnings, "the intermediate name is not a semconv-version endpoint") - }) - - t.Run("branches many-to-one renames deterministically", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{metricChange(map[string]string{ - "metric.b": "metric.current", - "metric.a": "metric.current", - })}, - }) - variants := requireMatcherVariants(t, "1.1.0", schema, equalMatchers("metric.current"), nil, nil) - require.Equal(t, []string{"metric.current", "metric.a", "metric.b"}, variantNames(variants)) - }) - - t.Run("preserves convergence across ordered metric changes", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - metricChange(map[string]string{"metric.a": "metric.current"}), - metricChange(map[string]string{"metric.b": "metric.current"}), - }, - }) - - backward := requireMatcherVariants(t, "1.1.0", schema, equalMatchers("metric.current"), nil, nil) - require.ElementsMatch(t, []string{"metric.current", "metric.a", "metric.b"}, variantNames(backward)) - - for _, predecessor := range []string{"metric.a", "metric.b"} { - forward := requireMatcherVariants(t, "1.0.0", schema, equalMatchers(predecessor), nil, nil) - require.ElementsMatch(t, []string{predecessor, "metric.current"}, variantNames(forward)) - } - }) - - t.Run("preserves convergence across ordered attribute changes", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - attributeChange(map[string]string{"attr.a": "attr.current"}), - attributeChange(map[string]string{"attr.b": "attr.current"}), - }, - }) - - backward := requireMatcherVariants(t, "1.1.0", schema, equalMatchers("metric", "attr.current"), []string{"attr.current"}, nil) - require.ElementsMatch(t, []string{"metric/attr.current", "metric/attr.a", "metric/attr.b"}, []string{ - variantMetricAndAttribute(backward[0]), - variantMetricAndAttribute(backward[1]), - variantMetricAndAttribute(backward[2]), - }) - for _, variant := range backward[1:] { - require.Equal(t, map[string]string{ - "attr.a": "attr.current", - "attr.b": "attr.current", - }, variant.mapping.translatedLabels) - } - - for _, predecessor := range []string{"attr.a", "attr.b"} { - forward := requireMatcherVariants(t, "1.0.0", schema, equalMatchers("metric", predecessor), []string{predecessor}, nil) - require.ElementsMatch(t, []string{"metric/" + predecessor, "metric/attr.current"}, []string{ - variantMetricAndAttribute(forward[0]), - variantMetricAndAttribute(forward[1]), - }) - } - }) - - t.Run("an anchor before every revision has no backward step", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.0.0", - changes: []schemaChange{metricChange(map[string]string{"metric.old": "metric.new"})}, - }) - variants := requireMatcherVariants(t, "0.9.0", schema, equalMatchers("metric.old"), nil, nil) - require.Equal(t, []string{"metric.old", "metric.new"}, variantNames(variants)) - }) - - t.Run("no revisions returns the direct variant", func(t *testing.T) { - variants := requireMatcherVariants(t, "1.0.0", &otelSchema{}, equalMatchers("metric"), nil, nil) - require.Len(t, variants, 1) - require.Equal(t, "metric", variantNames(variants)[0]) - }) -} - -func TestRenameValidatorLookupFailures(t *testing.T) { - t.Run("deduplicates an unavailable semconv at one boundary", func(t *testing.T) { - rv := &renameValidator{ - anchorKnown: true, - seen: map[string]struct{}{}, - lookup: func(string, string) (metricDef, metricLookupStatus) { - return metricDef{}, metricFileMissing - }, - } - - require.True(t, rv.allowRevision("1.1.0", "1.0.0", "metric.current", "metric.a")) - require.True(t, rv.allowRevision("1.1.0", "1.0.0", "metric.current", "metric.b")) - require.Len(t, rv.warnings, 1) - require.Contains(t, rv.warnings[0], "version 1.0.0 is unavailable") - }) - - t.Run("reports an unavailable semconv before an unknown anchor", func(t *testing.T) { - rv := &renameValidator{ - anchorName: "metric.current", - anchorVersion: "1.1.0", - seen: map[string]struct{}{}, - lookup: func(string, string) (metricDef, metricLookupStatus) { - return metricDef{}, metricFileMissing - }, - } - - require.True(t, rv.allowRevision("1.1.0", "1.0.0", "metric.current", "metric.old")) - require.Len(t, rv.warnings, 1) - require.Contains(t, rv.warnings[0], "version 1.0.0 is unavailable") - require.NotContains(t, rv.warnings[0], "no other version") - }) - - t.Run("fails open for an unknown lookup status", func(t *testing.T) { - rv := &renameValidator{ - anchorKnown: true, - seen: map[string]struct{}{}, - lookup: func(string, string) (metricDef, metricLookupStatus) { - return metricDef{}, metricLookupStatus(255) - }, - } - - require.True(t, rv.allowRevision("1.1.0", "1.0.0", "metric.current", "metric.old")) - require.Len(t, rv.warnings, 1) - require.Contains(t, rv.warnings[0], "unknown metric lookup status 255") - }) -} - -func TestMetricLifecycleBoundaries(t *testing.T) { - t.Run("stops a corroborated retired name from a later anchor", func(t *testing.T) { - schema := testSchemaWithVersions([]string{"1.0.0", "1.1.0", "1.2.0"}, schemaRevision{ - version: "1.1.0", - changes: []schemaChange{metricChange(map[string]string{"metric.old": "metric.current"})}, - }) - rv := &renameValidator{anchorKnown: true, seen: map[string]struct{}{}} - - variants := requireMatcherVariants(t, "1.2.0", schema, equalMatchers("metric.old"), nil, rv) - require.Equal(t, []string{"metric.old"}, variantNames(variants)) - require.Len(t, rv.warnings, 1) - require.Contains(t, rv.warnings[0], "metric lifecycle boundary") - }) - - t.Run("stops a reused old-side name even without identity evidence", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "2.0.0", - changes: []schemaChange{metricChange(map[string]string{"foo": "bar"})}, - }) - rv := &renameValidator{seen: map[string]struct{}{}} - variants := requireMatcherVariants(t, "5.0.0", schema, equalMatchers("foo"), nil, rv) - require.Equal(t, []string{"foo"}, variantNames(variants)) - require.Len(t, rv.warnings, 1) - require.Contains(t, rv.warnings[0], "metric lifecycle boundary") - }) - - t.Run("rejects a name reused within one revision", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - metricChange(map[string]string{"metric.a": "metric.b"}), - metricChange(map[string]string{"metric.c": "metric.a"}), - }, - }) - - rv := &renameValidator{anchorDeclared: true, anchorKnown: true, seen: map[string]struct{}{}} - _, err := generateMatcherVariantsWithBudget("1.1.0", schema, equalMatchers("metric.a"), nil, rv, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.ErrorIs(t, err, errAmbiguousSchemaRename) - require.ErrorContains(t, err, `metric name "metric.a"`) - }) - - t.Run("rejects a later identity claiming a retired name", func(t *testing.T) { - schema := testSchema( - schemaRevision{version: "1.1.0", changes: []schemaChange{metricChange(map[string]string{"foo": "bar"})}}, - schemaRevision{version: "1.2.0", changes: []schemaChange{metricChange(map[string]string{"baz": "foo"})}}, - ) - - for _, metric := range []string{"bar", "foo"} { - t.Run(metric, func(t *testing.T) { - _, err := generateMatcherVariantsWithBudget("1.2.0", schema, equalMatchers(metric), nil, nil, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.ErrorIs(t, err, errAmbiguousSchemaRename) - require.ErrorContains(t, err, `metric name "foo"`) - }) - } - }) - - t.Run("later convergence does not erase earlier reuse", func(t *testing.T) { - schema := testSchema( - schemaRevision{version: "1.1.0", changes: []schemaChange{metricChange(map[string]string{"foo": "bar"})}}, - schemaRevision{version: "1.2.0", changes: []schemaChange{metricChange(map[string]string{"baz": "foo"})}}, - schemaRevision{version: "1.3.0", changes: []schemaChange{metricChange(map[string]string{"foo": "bar"})}}, - ) - - _, err := generateMatcherVariantsWithBudget("1.3.0", schema, equalMatchers("bar"), nil, nil, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.ErrorIs(t, err, errAmbiguousSchemaRename) - require.ErrorContains(t, err, `metric name "foo"`) - }) - - t.Run("allows a repeated historical source to converge", func(t *testing.T) { - schema := testSchema( - schemaRevision{version: "1.1.0", changes: []schemaChange{metricChange(map[string]string{"metric.long": "metric.short"})}}, - schemaRevision{version: "1.2.0", changes: []schemaChange{metricChange(map[string]string{ - "metric.long": "metric.current", - "metric.short": "metric.current", - })}}, - ) - - variants := requireMatcherVariants(t, "1.2.0", schema, equalMatchers("metric.current"), nil, nil) - require.ElementsMatch(t, []string{"metric.current", "metric.long", "metric.short"}, variantNames(variants)) - }) - - t.Run("applies one rename map atomically", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{metricChange(map[string]string{"metric.a": "metric.b", "metric.b": "metric.a"})}, - }) - - _, err := generateMatcherVariantsWithBudget("1.1.0", schema, equalMatchers("metric.a"), nil, nil, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.ErrorIs(t, err, errAmbiguousSchemaRename) - }) - - t.Run("does not turn a transient name into a version era", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - metricChange(map[string]string{"metric.c": "metric.transient"}), - metricChange(map[string]string{"metric.transient": "metric.d"}), - }, - }) - rv := &renameValidator{anchorKnown: true, seen: map[string]struct{}{}} - - variants := requireMatcherVariants(t, "1.2.0", schema, equalMatchers("metric.transient"), nil, rv) - require.Equal(t, []string{"metric.transient"}, variantNames(variants)) - }) - - t.Run("allows a legitimate rename back", func(t *testing.T) { - schema := testSchema( - schemaRevision{version: "2.0.0", changes: []schemaChange{metricChange(map[string]string{"foo": "bar"})}}, - schemaRevision{version: "3.0.0", changes: []schemaChange{metricChange(map[string]string{"bar": "foo"})}}, - ) - rv := &renameValidator{seen: map[string]struct{}{}} - variants := requireMatcherVariants(t, "3.0.0", schema, equalMatchers("foo"), nil, rv) - require.Equal(t, []string{"foo", "bar"}, variantNames(variants)) - require.Empty(t, rv.warnings) - }) -} - -func TestManyToOneMetricRenames(t *testing.T) { - t.Run("follows each predecessor's history", func(t *testing.T) { - schema := testSchema( - schemaRevision{version: "1.1.0", changes: []schemaChange{ - metricChange(map[string]string{"metric.b.older": "metric.b"}), - }}, - schemaRevision{version: "1.2.0", changes: []schemaChange{ - metricChange(map[string]string{"metric.a": "metric.merged", "metric.b": "metric.merged"}), - }}, - ) - - variants := requireMatcherVariants(t, "1.2.0", schema, equalMatchers("metric.merged"), nil, nil) - require.ElementsMatch(t, - []string{"metric.merged", "metric.a", "metric.b", "metric.b.older"}, - variantNames(variants)) - }) - - t.Run("upstream schema collapses two spellings deterministically", func(t *testing.T) { - b, err := os.ReadFile("./testdata/upstream/schema-1.44.0.yaml") - require.NoError(t, err) - schema, err := loadOTelSchema(b) - require.NoError(t, err) - - variants := requireMatcherVariants(t, "1.44.0", &schema, - equalMatchers("k8s.replicationcontroller.pod.available"), nil, nil) - require.Equal(t, []string{ - "k8s.replicationcontroller.pod.available", - "k8s.replication_controller.available_pods", - "k8s.replicationcontroller.available_pods", - }, variantNames(variants)) - }) -} - -func TestAttributeScopingFollowsMetricBranches(t *testing.T) { - t.Run("scope is evaluated at its transformation position", func(t *testing.T) { - beforeRename := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - attributeChange(map[string]string{"user": "tenant"}, "metric.old"), - metricChange(map[string]string{"metric.old": "metric.new"}), - }, - }) - variants := requireMatcherVariants(t, "1.1.0", beforeRename, equalMatchers("metric.new", "tenant"), []string{"tenant"}, nil) - require.Equal(t, []string{"metric.new/tenant", "metric.old/user"}, []string{ - variantMetricAndAttribute(variants[0]), variantMetricAndAttribute(variants[1]), - }) - - afterRename := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - metricChange(map[string]string{"metric.old": "metric.new"}), - attributeChange(map[string]string{"user": "tenant"}, "metric.old"), - }, - }) - variants = requireMatcherVariants(t, "1.1.0", afterRename, equalMatchers("metric.new", "tenant"), []string{"tenant"}, nil) - require.Equal(t, []string{"metric.new/tenant", "metric.old/tenant"}, []string{ - variantMetricAndAttribute(variants[0]), variantMetricAndAttribute(variants[1]), - }) - }) - - t.Run("a scope applies only to its many-to-one predecessor", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - attributeChange(map[string]string{"user": "tenant"}, "metric.a"), - metricChange(map[string]string{"metric.a": "metric.current", "metric.b": "metric.current"}), - }, - }) - variants := requireMatcherVariants(t, "1.1.0", schema, equalMatchers("metric.current", "tenant"), []string{"tenant"}, nil) - require.Len(t, variants, 3) - require.Equal(t, "metric.current/tenant", variantMetricAndAttribute(variants[0])) - require.Equal(t, "metric.a/user", variantMetricAndAttribute(variants[1])) - require.Equal(t, map[string]string{"user": "tenant"}, variants[1].mapping.translatedLabels) - require.Equal(t, "metric.b/tenant", variantMetricAndAttribute(variants[2])) - require.Empty(t, variants[2].mapping.translatedLabels) - }) - - t.Run("an inapplicable earlier scope does not resolve a temporary origin", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{ - attributeChange(map[string]string{"attr.a": "attr.current"}, "metric.a"), - attributeChange(map[string]string{"attr.b": "attr.current"}, "metric.b"), - }, - }) - - variants := requireMatcherVariants(t, "1.1.0", schema, equalMatchers("metric.b", "attr.current"), []string{"attr.current"}, nil) - require.Equal(t, []string{"metric.b/attr.current", "metric.b/attr.b"}, []string{ - variantMetricAndAttribute(variants[0]), - variantMetricAndAttribute(variants[1]), - }) - require.Equal(t, map[string]string{"attr.b": "attr.current"}, variants[1].mapping.translatedLabels) - }) -} - -func TestAttributeMatcherExpansion(t *testing.T) { - t.Run("moves duplicate matchers together", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{attributeChange(map[string]string{ - "attr.a": "attr.current", - "attr.b": "attr.current", - "attr.c": "attr.current", - })}, - }) - matchers := equalMatchers("metric") - for range 13 { - matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, "attr.current", "value")) - } - - variants := requireMatcherVariants(t, "1.1.0", schema, matchers, []string{"attr.current"}, nil) - require.Len(t, variants, 4) - for _, variant := range variants { - var attributeName string - for _, matcher := range variant.matchers[1:] { - if attributeName == "" { - attributeName = matcher.Name - } - require.Equal(t, attributeName, matcher.Name) - } - } - }) - - t.Run("caps products across distinct attributes", func(t *testing.T) { - for _, tc := range []struct { - groups int - wantLen int - wantErr bool - }{ - {groups: 8, wantLen: 256}, - {groups: 9, wantErr: true}, - } { - t.Run(fmt.Sprintf("%d groups", tc.groups), func(t *testing.T) { - renamed := map[string]string{} - state := lineageState{metric: "metric", matchers: equalMatchers("metric")} - for i := range tc.groups { - current := fmt.Sprintf("attr.current.%d", i) - renamed[fmt.Sprintf("attr.a.%d", i)] = current - renamed[fmt.Sprintf("attr.b.%d", i)] = current - state.matchers = append(state.matchers, labels.MustNewMatcher(labels.MatchEqual, current, "value")) - } - - states, err := transformAttributeMatcherStatesWithBudget(state, newDirectedRenames(renamed), traverseBackward, 0, nil, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - if tc.wantErr { - require.ErrorIs(t, err, errSchemaExpansion) - return - } - require.NoError(t, err) - require.Len(t, states, tc.wantLen) - }) - } - }) -} - -func TestReplaceMetricMatchers(t *testing.T) { - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, "metric.current"), - labels.MustNewMatcher(labels.MatchRegexp, labels.MetricName, "metric.current"), - labels.MustNewMatcher(labels.MatchNotEqual, labels.MetricName, "metric.current"), - labels.MustNewMatcher(labels.MatchEqual, labels.MetricName, "metric.other"), - labels.MustNewMatcher(labels.MatchEqual, "attribute", "value"), - } - got, err := replaceMetricMatchersWithBudget(matchers, "metric.current", "metric.old", newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - - require.Equal(t, labels.MatchEqual, got[0].Type) - require.Equal(t, "metric.old", got[0].Value) - require.Same(t, matchers[1], got[1], "non-equality constraints are normalized before traversal") - require.Same(t, matchers[2], got[2], "non-equality constraints are normalized before traversal") - require.Same(t, matchers[3], got[3], "a different metric constraint must remain untouched") - require.Same(t, matchers[4], got[4], "a non-metric constraint must remain untouched") -} - -func TestVariantAccumulatorMergesAttributeAliases(t *testing.T) { - t.Run("compatible mappings", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{attributeChange(map[string]string{"user": "tenant"})}, - }) - variants := requireMatcherVariants(t, "1.1.0", schema, equalMatchers("metric"), []string{"tenant"}, nil) - require.Len(t, variants, 1) - require.Equal(t, map[string]string{"user": "tenant"}, variants[0].mapping.translatedLabels) - }) - - t.Run("conflicting mappings", func(t *testing.T) { - rv := &renameValidator{seen: map[string]struct{}{}} - acc := requireVariantAccumulator(t, "metric", []string{"tenant", "account"}, rv) - matchers := equalMatchers("metric") - require.NoError(t, acc.add(lineageState{matchers: matchers, metric: "metric", attrs: attributeLineage{ - "tenant": {"user": attributeOriginResolved}, - }})) - require.NoError(t, acc.add(lineageState{matchers: matchers, metric: "metric", attrs: attributeLineage{ - "account": {"user": attributeOriginResolved}, - }})) - - require.Len(t, acc.variants, 1) - require.NotContains(t, acc.variants[0].mapping.translatedLabels, "user") - require.Len(t, rv.warnings, 1) - require.Contains(t, rv.warnings[0], `attribute name "user" resolves to both "tenant" and "account"`) - }) - - t.Run("identity reuse across matcher variants", func(t *testing.T) { - acc := requireVariantAccumulator(t, "metric", nil, nil) - require.NoError(t, acc.add(lineageState{ - matchers: equalMatchers("metric", "first"), - metric: "metric", - attrs: attributeLineage{ - "user": {"account": attributeOriginResolved}, - }, - })) - err := acc.add(lineageState{ - matchers: equalMatchers("metric", "second"), - metric: "metric", - attrs: attributeLineage{ - "tenant": {"user": attributeOriginResolved}, - }, - }) - require.ErrorIs(t, err, errAmbiguousSchemaRename) - require.ErrorContains(t, err, `attribute name "user"`) - }) - - t.Run("identities stay scoped to a physical metric", func(t *testing.T) { - acc := requireVariantAccumulator(t, "metric.current", nil, nil) - require.NoError(t, acc.add(lineageState{ - matchers: equalMatchers("metric.old"), - metric: "metric.old", - attrs: attributeLineage{ - "user": {"account": attributeOriginResolved}, - }, - })) - require.NoError(t, acc.add(lineageState{ - matchers: equalMatchers("metric.new"), - metric: "metric.new", - attrs: attributeLineage{ - "tenant": {"user": attributeOriginResolved}, - }, - })) - }) -} - -func TestAttributeLifecycleReuse(t *testing.T) { - t.Run("rejects a distinct identity claiming an alias", func(t *testing.T) { - schema := testSchema( - schemaRevision{version: "1.1.0", changes: []schemaChange{attributeChange(map[string]string{"user": "tenant"})}}, - schemaRevision{version: "1.2.0", changes: []schemaChange{attributeChange(map[string]string{"account": "user"})}}, - ) - - _, err := generateMatcherVariantsWithBudget("1.2.0", schema, equalMatchers("metric", "user"), nil, nil, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.ErrorIs(t, err, errAmbiguousSchemaRename) - require.ErrorContains(t, err, `attribute name "user"`) - }) - - t.Run("allows a legitimate rename back", func(t *testing.T) { - schema := testSchema( - schemaRevision{version: "1.1.0", changes: []schemaChange{attributeChange(map[string]string{"user": "tenant"})}}, - schemaRevision{version: "1.2.0", changes: []schemaChange{attributeChange(map[string]string{"tenant": "user"})}}, - ) - - variants := requireMatcherVariants(t, "1.2.0", schema, equalMatchers("metric", "user"), nil, nil) - require.NotEmpty(t, variants) - }) - - t.Run("allows explicit convergence into an occupied identity", func(t *testing.T) { - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{attributeChange(map[string]string{ - "tls.client.server_name": "server.address", - })}, - }) - - variants := requireMatcherVariants(t, "1.0.0", schema, equalMatchers("metric"), []string{"server.address"}, nil) - require.NotEmpty(t, variants) - }) -} - -func TestSchemaExpansionLimits(t *testing.T) { - t.Run("attribute lineage includes canonical definitions", func(t *testing.T) { - for _, tc := range []struct { - attributes int - wantLen int - wantErr bool - }{ - {attributes: maxSchemaExpansion, wantLen: maxSchemaExpansion}, - {attributes: maxSchemaExpansion + 1, wantErr: true}, - } { - t.Run(fmt.Sprintf("%d attributes", tc.attributes), func(t *testing.T) { - attributes := make([]string, 0, tc.attributes) - for i := range tc.attributes { - attributes = append(attributes, fmt.Sprintf("attr.%03d", i)) - } - lineage, err := newAttributeLineageWithBudget(attributes, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - if tc.wantErr { - require.ErrorIs(t, err, errSchemaExpansion) - return - } - require.NoError(t, err) - require.Len(t, lineage, tc.wantLen) - }) - } - }) - - t.Run("matcher variants include the anchor", func(t *testing.T) { - for _, tc := range []struct { - sources int - wantLen int - wantErr bool - }{ - {sources: maxSchemaExpansion - 1, wantLen: maxSchemaExpansion}, - {sources: maxSchemaExpansion, wantErr: true}, - } { - t.Run(fmt.Sprintf("%d sources", tc.sources), func(t *testing.T) { - renamed := make(map[string]string, tc.sources) - for i := range tc.sources { - renamed[fmt.Sprintf("metric.old.%03d", i)] = "metric.current" - } - schema := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{metricChange(renamed)}, - }) - - variants, err := generateMatcherVariantsWithBudget("1.1.0", schema, equalMatchers("metric.current"), nil, nil, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - if tc.wantErr { - require.ErrorIs(t, err, errSchemaExpansion) - return - } - require.NoError(t, err) - require.Len(t, variants, tc.wantLen) - }) - } - }) - - t.Run("attribute mappings are bounded across states", func(t *testing.T) { - acc := requireVariantAccumulator(t, "metric", []string{"attr.current"}, nil) - matchers := equalMatchers("metric") - for i := range maxSchemaExpansion { - err := acc.add(lineageState{matchers: matchers, metric: "metric", attrs: attributeLineage{ - "attr.current": {fmt.Sprintf("attr.old.%03d", i): attributeOriginResolved}, - }}) - require.NoError(t, err) - } - err := acc.add(lineageState{matchers: matchers, metric: "metric", attrs: attributeLineage{ - "attr.current": {"attr.overflow": attributeOriginResolved}, - }}) - require.ErrorIs(t, err, errSchemaExpansion) - }) - - t.Run("label value jobs include the canonical name", func(t *testing.T) { - for _, tc := range []struct { - aliases int - wantLen int - wantErr bool - }{ - {aliases: maxSchemaExpansion - 1, wantLen: maxSchemaExpansion}, - {aliases: maxSchemaExpansion, wantErr: true}, - } { - t.Run(fmt.Sprintf("%d aliases", tc.aliases), func(t *testing.T) { - mapping := make(map[string]string, tc.aliases) - for i := range tc.aliases { - mapping[fmt.Sprintf("attr.old.%03d", i)] = "attr.current" - } - jobs, err := buildLabelValueJobs([]matcherVariant{{ - matchers: equalMatchers("metric"), - mapping: buildLabelMapping("metric", mapping), - }}, "attr.current") - if tc.wantErr { - require.ErrorIs(t, err, errSchemaExpansion) - return - } - require.NoError(t, err) - require.Len(t, jobs, tc.wantLen) - }) - } - }) -} - -func TestSchemaExpansionBudget(t *testing.T) { - state := lineageState{ - matchers: equalMatchers("metric", "tenant"), - metric: "metric", - attrs: attributeLineage{ - "tenant": {"tenant": attributeOriginResolved, "user": attributeOriginResolved}, - }, - } - - t.Run("find variants returns no partial result", func(t *testing.T) { - files := map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: -`), - "1.1.0": []byte(`groups: - - id: metric.metric - type: metric - metric_name: metric - unit: s - instrument: histogram -`), - } - for name, tc := range map[string]struct { - limits schemaExpansionLimits - detail string - }{ - "work": { - limits: schemaExpansionLimits{work: 1, keyBytes: 1_000}, - detail: "resolver work", - }, - "key bytes": { - limits: schemaExpansionLimits{work: 1_000, keyBytes: 1}, - detail: "deduplication key bytes", - }, - } { - t.Run(name, func(t *testing.T) { - engine := newSchemaEngine(newRegistrySource(files)) - engine.limits = tc.limits - variants, _, err := engine.findMatcherVariants( - "registry/1.1.0", - "registry/registry.yaml", - equalMatchers("metric"), - ) - require.ErrorIs(t, err, errSchemaExpansion) - require.ErrorContains(t, err, tc.detail) - require.Nil(t, variants) - }) - } - }) - - t.Run("metric ownership analysis returns no partial result", func(t *testing.T) { - revisions := []schemaRevision{{ - version: "1.1.0", - changes: []schemaChange{metricChange(map[string]string{"metric.old": "metric.current"})}, - }} - reused, err := reusedMetricNamesWithBudget(revisions, newSchemaExpansionBudget(schemaExpansionLimits{work: 1, keyBytes: 1_000})) - require.ErrorIs(t, err, errSchemaExpansion) - require.Nil(t, reused) - }) - - t.Run("cached metric ownership work is charged to every query", func(t *testing.T) { - files := map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - metric.old: metric.current -`), - "1.1.0": []byte(`groups: - - id: metric.metric.current - type: metric - metric_name: metric.current - unit: s - instrument: histogram -`), - } - engine := newSchemaEngine(newRegistrySource(files)) - schema, err := engine.getOTelSchema("registry/registry.yaml") - require.NoError(t, err) - analysis := engine.schemaSafetyAnalysis("registry/registry.yaml", &schema) - require.NoError(t, analysis.err) - require.Positive(t, analysis.work) - - engine.limits = schemaExpansionLimits{work: analysis.work - 1, keyBytes: 1_000_000} - variants, _, err := engine.findMatcherVariants( - "registry/1.1.0", - "registry/registry.yaml", - equalMatchers("metric.current"), - ) - require.ErrorIs(t, err, errSchemaExpansion) - require.Nil(t, variants) - }) - - t.Run("preflights keys before allocation and state mutation", func(t *testing.T) { - keyBytes := uint64(len(requireLineageStateKey(t, state))) - require.Positive(t, keyBytes) - - budget := newSchemaExpansionBudget(schemaExpansionLimits{work: 1_000, keyBytes: keyBytes - 1}) - states := newLineageStateSet(1, budget) - err := states.add(state) - require.ErrorIs(t, err, errSchemaExpansion) - require.ErrorContains(t, err, "deduplication key bytes") - require.Empty(t, states.states) - require.Empty(t, states.seen) - require.Zero(t, budget.keyBytes) - - budget = newSchemaExpansionBudget(schemaExpansionLimits{work: 1_000, keyBytes: keyBytes}) - states = newLineageStateSet(1, budget) - require.NoError(t, states.add(state)) - require.Len(t, states.states, 1) - require.Equal(t, keyBytes, budget.keyBytes) - }) - - t.Run("charges duplicate attempts", func(t *testing.T) { - probe := newSchemaExpansionBudget(schemaExpansionLimits{work: 1_000, keyBytes: 1_000}) - require.NoError(t, newLineageStateSet(1, probe).add(state)) - require.Positive(t, probe.work) - require.Positive(t, probe.keyBytes) - - budget := newSchemaExpansionBudget(schemaExpansionLimits{ - work: probe.work*2 - 1, - keyBytes: probe.keyBytes * 2, - }) - states := newLineageStateSet(1, budget) - require.NoError(t, states.add(state)) - err := states.add(state) - require.ErrorIs(t, err, errSchemaExpansion) - require.ErrorContains(t, err, "resolver work") - require.Len(t, states.states, 1) - require.Len(t, states.seen, 1) - require.Equal(t, probe.keyBytes, budget.keyBytes) - }) - - t.Run("starts before identity recovery", func(t *testing.T) { - schema := testSchemaWithVersions([]string{"1.0.0", "1.1.0"}, schemaRevision{ - version: "1.1.0", - changes: []schemaChange{metricChange(map[string]string{"metric.old": "metric.current"})}, - }) - anchor := semconv{version: "1.1.0", metrics: map[string]metricDef{}} - lookup := func(string, string) (metricDef, metricLookupStatus) { - return metricDef{unit: "s", instrument: "histogram"}, metricDeclared - } - budget := newSchemaExpansionBudget(schemaExpansionLimits{work: 1, keyBytes: 1_000}) - - _, err := newRenameValidator(schema, lookup, anchor, "metric.old", budget) - require.ErrorIs(t, err, errSchemaExpansion) - require.ErrorContains(t, err, "resolver work") - }) - - t.Run("shares one budget across traversal directions", func(t *testing.T) { - backward := testSchema(schemaRevision{ - version: "1.1.0", - changes: []schemaChange{metricChange(map[string]string{"metric.old": "metric.current"})}, - }) - forward := testSchema(schemaRevision{ - version: "1.2.0", - changes: []schemaChange{metricChange(map[string]string{"metric.current": "metric.new"})}, - }) - combined := testSchema( - schemaRevision{ - version: "1.1.0", - changes: []schemaChange{metricChange(map[string]string{"metric.old": "metric.current"})}, - }, - schemaRevision{ - version: "1.2.0", - changes: []schemaChange{metricChange(map[string]string{"metric.current": "metric.new"})}, - }, - ) - - measure := func(t *testing.T, schema *otelSchema, wantVariants int) uint64 { - t.Helper() - budget := newSchemaExpansionBudget(schemaExpansionLimits{work: 1_000, keyBytes: 1_000_000}) - variants, err := generateMatcherVariantsWithBudget("1.1.0", schema, equalMatchers("metric.current"), nil, nil, budget) - require.NoError(t, err) - require.Len(t, variants, wantVariants) - return budget.work - } - - backwardWork := measure(t, backward, 2) - forwardWork := measure(t, forward, 2) - combinedWork := measure(t, combined, 3) - limit := max(backwardWork, forwardWork) - require.Greater(t, combinedWork, limit) - - for name, schema := range map[string]*otelSchema{"backward": backward, "forward": forward} { - t.Run(name+" fits independently", func(t *testing.T) { - budget := newSchemaExpansionBudget(schemaExpansionLimits{work: limit, keyBytes: 1_000_000}) - _, err := generateMatcherVariantsWithBudget("1.1.0", schema, equalMatchers("metric.current"), nil, nil, budget) - require.NoError(t, err) - }) - } - - budget := newSchemaExpansionBudget(schemaExpansionLimits{work: limit, keyBytes: 1_000_000}) - variants, err := generateMatcherVariantsWithBudget("1.1.0", combined, equalMatchers("metric.current"), nil, nil, budget) - require.ErrorIs(t, err, errSchemaExpansion) - require.ErrorContains(t, err, "resolver work") - require.Nil(t, variants, "overflow must not return partial variants") - }) -} - -func BenchmarkMetricOwnershipAnalysisUpstream(b *testing.B) { - raw, err := os.ReadFile("./testdata/upstream/schema-1.44.0.yaml") - require.NoError(b, err) - schema, err := loadOTelSchema(raw) - require.NoError(b, err) - - b.ReportAllocs() - b.ResetTimer() - for b.Loop() { - reused, err := reusedMetricNamesWithBudget(schema.revisions, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - if err != nil { - b.Fatal(err) - } - if reused == nil { - b.Fatal("ownership analysis returned a nil result") - } - } -} - -func TestDeduplicateLineageStatesPreservesCanonicalIdentity(t *testing.T) { - matchers := equalMatchers("metric") - states := []lineageState{ - {matchers: matchers, metric: "metric", attrs: attributeLineage{"tenant": {"tenant": attributeOriginResolved, "user": attributeOriginResolved}}}, - {matchers: matchers, metric: "metric", attrs: attributeLineage{"tenant": {"user": attributeOriginResolved}}}, - {matchers: matchers, metric: "metric", attrs: attributeLineage{"tenant": {"user": attributeOriginResolved}, "user": {"tenant": attributeOriginResolved}}}, - } - - deduplicated, err := deduplicateLineageStatesWithBudget(states, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - require.Len(t, deduplicated, 3) - require.NotEqual(t, - requireLineageStateKey(t, lineageState{matchers: matchers, attrs: attributeLineage{"a": {"b": attributeOriginResolved}, "c": {"d": attributeOriginResolved}}}), - requireLineageStateKey(t, lineageState{matchers: matchers, attrs: attributeLineage{"a": {"b": attributeOriginResolved, "c": attributeOriginResolved, "d": attributeOriginResolved}}}), - ) - require.NotEqual(t, - requireLineageStateKey(t, lineageState{matchers: matchers, metric: "metric", renamedFrom: "metric.old"}), - requireLineageStateKey(t, lineageState{matchers: matchers, metric: "metric", renamedFrom: "metric.other"}), - ) - require.NotEqual(t, - requireLineageStateKey(t, lineageState{matchers: matchers, metric: "metric"}), - requireLineageStateKey(t, lineageState{matchers: matchers, metric: "metric", metricOriginPending: true}), - ) - require.NotEqual(t, - requireLineageStateKey(t, lineageState{matchers: matchers, metric: "metric"}), - requireLineageStateKey(t, lineageState{matchers: matchers, metric: "metric", pendingAttributeMatchers: map[int]struct{}{1: {}}}), - ) - require.NotEqual(t, - requireLineageStateKey(t, lineageState{matchers: matchers, attrs: attributeLineage{"a": {"b": attributeOriginResolved}}}), - requireLineageStateKey(t, lineageState{matchers: matchers, attrs: attributeLineage{"a": {"b": attributeOriginPending}}}), - ) -} diff --git a/storage/semconv/otel_schema.go b/storage/semconv/otel_schema.go index 1689f00e2b..77cd60a813 100644 --- a/storage/semconv/otel_schema.go +++ b/storage/semconv/otel_schema.go @@ -14,663 +14,29 @@ package semconv import ( - "embed" "fmt" "maps" - "path" "slices" - "strconv" - "strings" "sync" "github.com/grafana/regexp" "go.yaml.in/yaml/v3" ) -// embeddedRegistry holds the bundled semconv and OTel schema files served as -// the default source from which __semconv_url__ and __schema_url__ values are -// resolved. Operators may replace it with their own registry via configuration -// (see AwareStorageWithRegistry); either way, queries can only ever name files -// inside the registry namespace. Accepting arbitrary HTTP URLs or local -// filesystem paths from the matchers themselves would expose a server-side -// fetch primitive to anyone able to issue a PromQL query, which is why the -// matcher values are gated by registryURLRe rather than treated as locations. -// -//go:embed registry/* -var embeddedRegistry embed.FS - -// registryURLRe matches the only accepted shape of __semconv_url__ / -// __schema_url__ values: a single non-empty path segment under registry/. -// Path traversal (..) and absolute paths are rejected. +// registryURLRe matches the only accepted shape of __schema_url__ values: +// a single non-empty path segment under registry/. Path traversal (..) and +// absolute paths are rejected. var registryURLRe = regexp.MustCompile(`^registry/[^/.][^/]*$`) -// semverRe matches plain MAJOR.MINOR.PATCH version strings. Pre-release and -// build metadata are intentionally not accepted; semconv versions in the -// registry are plain dotted integers. -var semverRe = regexp.MustCompile(`^\d+\.\d+\.\d+$`) - -// validateSemver returns an error if v is not a plain MAJOR.MINOR.PATCH string. -func validateSemver(v string) error { - if !semverRe.MatchString(v) { - return fmt.Errorf("invalid semver %q: expected MAJOR.MINOR.PATCH", v) - } - return nil -} - -// semconv represents a semantic conventions file containing metric definitions. -// See: https://github.com/open-telemetry/semantic-conventions -// -// Example semconv YAML: -// -// groups: -// - id: metric.http.server.duration -// type: metric -// metric_name: http.server.duration -// instrument: histogram -// unit: s -type semconv struct { - Groups []semconvGroup `yaml:"groups"` - - version string - - // metrics indexes this version's metric groups by their metric_name - // (populated by loadSemconv). It answers both "is this name a metric at this - // version" and "what does it declare", which is what validating a resolved - // rename lineage needs. - metrics map[string]metricDef - - // ambiguousMetrics lists metric names declared by more than one group at - // this version, sorted. Such a name has no single definition, so anything - // resolved through it is unreliable and is reported to the caller rather - // than silently resolved to whichever group happened to be parsed last. - ambiguousMetrics []string -} - -// metricDef is the part of a semconv metric group that describes what the metric -// measures, as opposed to what it is currently called. Upstream semantic -// conventions forbid a stable metric from changing unit or instrument (see -// policies/compatibility.rego in open-telemetry/semantic-conventions), so a -// mismatch is evidence against a schema rename branch. Equality alone is not -// identity evidence because unrelated metrics may share both fields. -// -// Note that a group's id is deliberately not part of this: upstream lints id to -// be exactly "metric." (policies/yaml_schema.rego), so it is a pure -// function of the surface name and changes in lockstep with every rename. It -// therefore carries no identity information a rename edge does not already have. -type metricDef struct { - unit string - instrument string - stability string - attributes []string -} - -// sameMetricAs reports whether d and other have no unit or instrument -// contradiction. Directional schema traversal, not this comparison, establishes -// the candidate lineage. -func (d metricDef) sameMetricAs(other metricDef) bool { - return d.unit == other.unit && d.instrument == other.instrument -} - -// stablyContradicts reports whether both definitions are explicitly stable and -// disagree on identity metadata that both of them specify. -func (d metricDef) stablyContradicts(other metricDef) bool { - if d.stability != "stable" || other.stability != "stable" { - return false - } - return specifiedDifference(d.unit, other.unit) || specifiedDifference(d.instrument, other.instrument) -} - -func specifiedDifference(a, b string) bool { - return a != "" && b != "" && a != b -} - -// attributesOf returns the attributes the named metric declares at this semconv -// version, or nil if the name is not a metric here. -func (s semconv) attributesOf(name string) []string { - return s.metrics[name].attributes -} - -// otelSchema represents an OpenTelemetry schema file. -// See: https://opentelemetry.io/docs/specs/otel/schemas/file_format_v1.1.0/ -// -// Example schema YAML: -// -// file_format: 1.1.0 -// schema_url: https://example.com/schemas/1.0.0 -// -// versions: -// 1.0.0: -// metrics: -// changes: -// - rename_attributes: -// attribute_map: -// http.method: http.request.method -// apply_to_metrics: -// - http.server.duration +// otelSchema represents an OpenTelemetry Telemetry Policy transformation schema +// file (file_format: transform/2.0, see open-telemetry/weaver#1850). type otelSchema struct { - FileFormat string `yaml:"file_format"` - SchemaURL string `yaml:"schema_url"` - Versions map[string]otelSchemaVersion `yaml:"versions"` + FileFormat string `yaml:"file_format"` + SchemaURL string `yaml:"schema_url"` - // revisions holds the schema revisions that contain transformations, sorted - // by version. Transformations retain their file order because applying a - // revision backwards requires replaying them in the exact reverse order. - revisions []schemaRevision - - // allVersions lists every declared schema version in ascending order, - // including versions without transformations. - allVersions []string - - // transform holds the parsed policy definition when FileFormat is transform/2.0. transform *transformFile } -// predecessorOfWithBudget returns the version immediately before v in the -// complete schema history. -func (s *otelSchema) predecessorOfWithBudget(v string, budget *schemaExpansionBudget) (string, bool, error) { - for i, have := range s.allVersions { - if err := budget.reserveWork(1); err != nil { - return "", false, err - } - if have == v { - if i == 0 { - return "", false, nil - } - return s.allVersions[i-1], true, nil - } - } - return "", false, nil -} - -// eraVersionsOfWithBudget returns schema boundaries where name is current. -// Names used only between transformations in one revision have no boundary. -func (s *otelSchema) eraVersionsOfWithBudget(name string, budget *schemaExpansionBudget) ([]string, error) { - seen := map[string]struct{}{} - var versions []string - add := func(version string) error { - if version == "" { - return nil - } - if _, exists := seen[version]; exists { - return nil - } - if err := budget.reserveWork(1); err != nil { - return err - } - seen[version] = struct{}{} - versions = append(versions, version) - return nil - } - - for _, revision := range s.revisions { - if err := budget.reserveWork(1); err != nil { - return nil, err - } - before, after, _, err := revision.metricBoundaryWithBudget(name, budget) - if err != nil { - return nil, err - } - if before { - predecessor, ok, err := s.predecessorOfWithBudget(revision.version, budget) - if err != nil { - return nil, err - } - if ok { - if err := add(predecessor); err != nil { - return nil, err - } - } - } - if after { - if err := add(revision.version); err != nil { - return nil, err - } - } - } - slices.SortFunc(versions, compareSemver) - return versions, nil -} - -// metricBoundaryWithBudget reports whether name exists immediately before and -// after this revision. A name used only between ordered transformations is -// mentioned but absent at both version boundaries. -func (r schemaRevision) metricBoundaryWithBudget(name string, budget *schemaExpansionBudget) (before, after, mentioned bool, err error) { - for _, change := range r.changes { - if err := budget.reserveWork(1); err != nil { - return false, false, false, err - } - if change.metricRenames == nil { - continue - } - if _, source := change.metricRenames.forward[name]; source { - before = true - mentioned = true - break - } - if _, target := change.metricRenames.reverse[name]; target { - mentioned = true - break - } - } - for _, change := range slices.Backward(r.changes) { - if err := budget.reserveWork(1); err != nil { - return false, false, false, err - } - if change.metricRenames == nil { - continue - } - if _, target := change.metricRenames.reverse[name]; target { - after = true - break - } - if _, source := change.metricRenames.forward[name]; source { - break - } - } - return before, after, mentioned, nil -} - -// schemaRevision describes the ordered transformations that convert the -// preceding schema version to version. Revisions without transformations are -// omitted; allVersions retains them for predecessor lookup. -type schemaRevision struct { - version string - changes []schemaChange -} - -type schemaChange struct { - metricRenames *directedRenames - attributeRenames *attributeRenameStep -} - -// directedRenames preserves the direction declared by the schema. reverse may -// contain several predecessors because real schemas contain many-to-one metric -// and attribute renames. -type directedRenames struct { - forward map[string]string - reverse map[string][]string -} - -func newDirectedRenames(forward map[string]string) *directedRenames { - if len(forward) == 0 { - return nil - } - r := &directedRenames{ - forward: make(map[string]string, len(forward)), - reverse: make(map[string][]string), - } - for oldName, newName := range forward { - r.forward[oldName] = newName - r.reverse[newName] = append(r.reverse[newName], oldName) - } - for newName := range r.reverse { - slices.Sort(r.reverse[newName]) - } - return r -} - -type attributeRenameStep struct { - renames *directedRenames - applyToMetrics map[string]struct{} - scopeSpecified bool -} - -func newAttributeRenameStep(rename *otelRenameAttributes, scoped bool) *attributeRenameStep { - if rename == nil || len(rename.AttributeMap) == 0 { - return nil - } - step := &attributeRenameStep{renames: newDirectedRenames(rename.AttributeMap)} - if scoped && rename.ApplyToMetrics != nil { - step.scopeSpecified = true - step.applyToMetrics = make(map[string]struct{}, len(*rename.ApplyToMetrics)) - for _, name := range *rename.ApplyToMetrics { - step.applyToMetrics[name] = struct{}{} - } - } - return step -} - -func (s *attributeRenameStep) appliesTo(metricName string) bool { - if !s.scopeSpecified { - return true - } - _, ok := s.applyToMetrics[metricName] - return ok -} - -// collectSchemaRevision preserves the transformation order defined by the -// schema format: all-section changes precede metric-section changes, and each -// section is processed top-to-bottom. -func collectSchemaRevision(versionStr string, version otelSchemaVersion) (*schemaRevision, error) { - revision := &schemaRevision{version: versionStr} - if version.All != nil { - for i, change := range version.All.Changes { - transformation, err := validateSchemaChange(versionStr, "all", i+1, change) - if err != nil { - return nil, err - } - if transformation != "rename_attributes" { - return nil, fmt.Errorf("schema version %q all change %d contains unsupported transformation %q", versionStr, i+1, transformation) - } - if err := validateRenameAttributes(versionStr, "all", i+1, change.RenameAttributes); err != nil { - return nil, err - } - if step := newAttributeRenameStep(change.RenameAttributes, false); step != nil { - revision.changes = append(revision.changes, schemaChange{attributeRenames: step}) - } - } - } - if version.Metrics != nil { - for i, change := range version.Metrics.Changes { - transformation, err := validateSchemaChange(versionStr, "metrics", i+1, change) - if err != nil { - return nil, err - } - switch transformation { - case "split": - return nil, fmt.Errorf("schema version %q contains unsupported metric split transformation", versionStr) - case "rename_attributes": - if err := validateRenameAttributes(versionStr, "metrics", i+1, change.RenameAttributes); err != nil { - return nil, err - } - if step := newAttributeRenameStep(change.RenameAttributes, true); step != nil { - revision.changes = append(revision.changes, schemaChange{attributeRenames: step}) - } - case "rename_metrics": - if renames := newDirectedRenames(change.RenameMetrics); renames != nil { - revision.changes = append(revision.changes, schemaChange{metricRenames: renames}) - } - default: - return nil, fmt.Errorf("schema version %q metrics change %d contains unsupported transformation %q", versionStr, i+1, transformation) - } - } - } - if len(revision.changes) == 0 { - return nil, nil - } - return revision, nil -} - -func validateSchemaChange(versionStr, section string, index int, change otelSchemaChange) (string, error) { - if len(change.transformationKeys) != 1 { - return "", fmt.Errorf("schema version %q %s change %d must contain exactly one transformation, found %q", versionStr, section, index, change.transformationKeys) - } - return change.transformationKeys[0], nil -} - -func validateRenameAttributes(versionStr, section string, index int, rename *otelRenameAttributes) error { - if rename == nil { - return nil - } - for _, field := range rename.fieldKeys { - if field == "attribute_map" || section == "metrics" && field == "apply_to_metrics" { - continue - } - return fmt.Errorf("schema version %q %s change %d rename_attributes contains unsupported field %q", versionStr, section, index, field) - } - return nil -} - -// semconvGroup represents a semantic conventions group definition. -type semconvGroup struct { - ID string `yaml:"id"` - Type string `yaml:"type"` // "metric", "attribute", "span", etc. - Extends string `yaml:"extends"` - Prefix string `yaml:"prefix"` - Stability string `yaml:"stability"` - MetricName string `yaml:"metric_name"` // Only for type="metric" - Unit string `yaml:"unit"` // Only for type="metric", e.g. "s", "By" - Instrument string `yaml:"instrument"` // Only for type="metric", e.g. "histogram", "counter" - Attributes []semconvAttribute `yaml:"attributes,omitempty"` -} - -type semconvAttribute struct { - ID string `yaml:"id"` - Ref string `yaml:"ref"` -} - -const ( - // maxSemconvFileAttributeSlots bounds the cumulative capacity of flattened - // attribute slices materialized while loading one semconv file. - maxSemconvFileAttributeSlots uint64 = maxSchemaExpansionWork - - // maxSemconvRegistryAttributeSlots bounds that capacity across every semconv - // file in an operator-provided registry. - maxSemconvRegistryAttributeSlots uint64 = 1 << 20 -) - -type semconvRegistryMaterializationBudget struct { - attributeSlots uint64 -} - -type semconvMaterializationBudget struct { - attributeSlots uint64 - registry *semconvRegistryMaterializationBudget -} - -func (b *semconvMaterializationBudget) reserveAttributeSlots(n int) error { - slots := uint64(n) - if b.attributeSlots > maxSemconvFileAttributeSlots || slots > maxSemconvFileAttributeSlots-b.attributeSlots { - return schemaExpansionLimitError("semconv file attribute slots", maxSemconvFileAttributeSlots) - } - if b.registry != nil && (b.registry.attributeSlots > maxSemconvRegistryAttributeSlots || slots > maxSemconvRegistryAttributeSlots-b.registry.attributeSlots) { - return schemaExpansionLimitError("semconv registry attribute slots", maxSemconvRegistryAttributeSlots) - } - b.attributeSlots += slots - if b.registry != nil { - b.registry.attributeSlots += slots - } - return nil -} - -type semconvAttributeResolver struct { - groups map[string]*semconvGroup - resolved map[string]resolvedSemconvGroup - resolving map[string]int - stack []string - budget *semconvMaterializationBudget -} - -type resolvedSemconvGroup struct { - attributes []string -} - -func newSemconvAttributeResolver(groups []semconvGroup, registryBudget *semconvRegistryMaterializationBudget) (*semconvAttributeResolver, error) { - byID := make(map[string]*semconvGroup, len(groups)) - for i := range groups { - if groups[i].ID == "" { - continue - } - if _, exists := byID[groups[i].ID]; exists { - return nil, fmt.Errorf("duplicate semconv group id %q", groups[i].ID) - } - byID[groups[i].ID] = &groups[i] - } - return &semconvAttributeResolver{ - groups: byID, - resolved: map[string]resolvedSemconvGroup{}, - resolving: map[string]int{}, - budget: &semconvMaterializationBudget{registry: registryBudget}, - }, nil -} - -// resolve returns a group's deduplicated attributes, following extends -// transitively. Full transitivity follows the language contract even though -// historical build tools sometimes made schemas re-list grandparent references. -func (r *semconvAttributeResolver) resolve(group *semconvGroup) (resolvedSemconvGroup, error) { - if group.ID != "" { - if resolved, ok := r.resolved[group.ID]; ok { - return resolved, nil - } - if start, ok := r.resolving[group.ID]; ok { - cycle := append(slices.Clone(r.stack[start:]), group.ID) - return resolvedSemconvGroup{}, fmt.Errorf("semconv group inheritance cycle: %s", strings.Join(cycle, " -> ")) - } - r.resolving[group.ID] = len(r.stack) - r.stack = append(r.stack, group.ID) - defer func() { - delete(r.resolving, group.ID) - r.stack = r.stack[:len(r.stack)-1] - }() - } - - var inherited resolvedSemconvGroup - if group.Extends != "" { - parent, ok := r.groups[group.Extends] - if !ok { - return resolvedSemconvGroup{}, fmt.Errorf("semconv group %q extends unknown group %q", semconvGroupName(group), group.Extends) - } - var err error - inherited, err = r.resolve(parent) - if err != nil { - return resolvedSemconvGroup{}, err - } - } - - capacity := min(len(inherited.attributes)+len(group.Attributes), maxSchemaExpansion) - if err := r.budget.reserveAttributeSlots(capacity); err != nil { - return resolvedSemconvGroup{}, err - } - attributes := make([]string, 0, capacity) - seen := make(map[string]struct{}, capacity) - add := func(name string) error { - if _, exists := seen[name]; exists { - return nil - } - if len(attributes) >= maxSchemaExpansion { - return schemaExpansionError("semconv group attributes") - } - seen[name] = struct{}{} - attributes = append(attributes, name) - return nil - } - for _, name := range inherited.attributes { - if err := add(name); err != nil { - return resolvedSemconvGroup{}, err - } - } - for _, attribute := range group.Attributes { - name, err := semconvAttributeName(*group, group.Prefix, attribute) - if err != nil { - return resolvedSemconvGroup{}, err - } - if err := add(name); err != nil { - return resolvedSemconvGroup{}, err - } - } - - resolved := resolvedSemconvGroup{attributes: attributes} - if group.ID != "" { - r.resolved[group.ID] = resolved - } - return resolved, nil -} - -func semconvGroupName(group *semconvGroup) string { - if group.ID != "" { - return group.ID - } - if group.MetricName != "" { - return group.MetricName - } - return "(unnamed)" -} - -func semconvAttributeName(group semconvGroup, prefix string, attribute semconvAttribute) (string, error) { - switch { - case attribute.ID != "" && attribute.Ref != "": - return "", fmt.Errorf("semconv group %q attribute declares both id %q and ref %q", semconvGroupName(&group), attribute.ID, attribute.Ref) - case attribute.Ref != "": - return attribute.Ref, nil - case attribute.ID != "": - if prefix == "" { - return attribute.ID, nil - } - return prefix + "." + attribute.ID, nil - default: - return "", fmt.Errorf("semconv group %q attribute declares neither id nor ref", semconvGroupName(&group)) - } -} - -type otelSchemaVersion struct { - All *otelSchemaSection `yaml:"all,omitempty"` - Metrics *otelSchemaSection `yaml:"metrics,omitempty"` -} - -type otelSchemaSection struct { - Changes []otelSchemaChange `yaml:"changes,omitempty"` -} - -type otelSchemaChange struct { - RenameAttributes *otelRenameAttributes `yaml:"rename_attributes,omitempty"` - - // RenameMetrics maps each old metric name to its new name directly, with no - // intervening key. This is asymmetric with RenameAttributes, which nests its - // mapping under attribute_map, but it is what the file format specifies: - // - // metrics: - // changes: - // - rename_metrics: - // http.server.duration: http.server.request.duration - // - // See the rename_metrics transformation in - // https://opentelemetry.io/docs/specs/otel/schemas/file_format_v1.1.0/. - RenameMetrics map[string]string `yaml:"rename_metrics,omitempty"` - - // transformationKeys contains the sorted effective mapping keys, including - // keys introduced by YAML merges. - transformationKeys []string -} - -type otelRenameAttributes struct { - AttributeMap map[string]string `yaml:"attribute_map,omitempty"` - ApplyToMetrics *[]string `yaml:"apply_to_metrics,omitempty"` - - // fieldKeys contains the sorted effective mapping keys, including keys - // introduced by YAML merges. - fieldKeys []string -} - -func (c *otelSchemaChange) UnmarshalYAML(value *yaml.Node) error { - fields, keys, err := decodeEffectiveYAMLMapping(value) - if err != nil { - return err - } - c.transformationKeys = keys - if len(keys) != 1 { - return nil - } - - switch keys[0] { - case "rename_attributes": - field := fields[keys[0]] - return field.Decode(&c.RenameAttributes) - case "rename_metrics": - field := fields[keys[0]] - return field.Decode(&c.RenameMetrics) - } - return nil -} - -func (r *otelRenameAttributes) UnmarshalYAML(value *yaml.Node) error { - fields, keys, err := decodeEffectiveYAMLMapping(value) - if err != nil { - return err - } - r.fieldKeys = keys - if field, ok := fields["attribute_map"]; ok { - if err := field.Decode(&r.AttributeMap); err != nil { - return err - } - } - if field, ok := fields["apply_to_metrics"]; ok { - if err := field.Decode(&r.ApplyToMetrics); err != nil { - return err - } - } - return nil -} - func decodeEffectiveYAMLMapping(value *yaml.Node) (map[string]yaml.Node, []string, error) { var fields map[string]yaml.Node if err := value.Decode(&fields); err != nil { @@ -680,14 +46,9 @@ func decodeEffectiveYAMLMapping(value *yaml.Node) (map[string]yaml.Node, []strin } // staticCache is a generic, goroutine-safe cache keyed by URL for static -// (immutable, bounded) values. Entries never expire and are not evicted: -// the resolver only accepts paths inside the embedded registry, whose -// contents are bounded and immutable, so TTL- and size-based eviction would -// only add complexity for no benefit. Two callers racing on a cold key may -// both invoke the loader and Store the same value; the result is identical -// either way. +// (immutable, bounded) values. type staticCache[T any] struct { - m sync.Map // url → T + m sync.Map // url -> T } func newStaticCache[T any]() *staticCache[T] { @@ -721,153 +82,29 @@ func (e *schemaEngine) fetchOTelSchema(url string) (otelSchema, error) { return s, nil } -// loadOTelSchema parses raw OTel schema YAML bytes, validates its versions, and -// collects its ordered transformations. It is the parsing core of -// fetchOTelSchema and is directly callable from tests. +// loadOTelSchema parses raw OTel transform/2.0 schema YAML bytes. func loadOTelSchema(b []byte) (otelSchema, error) { var s otelSchema if err := yaml.Unmarshal(b, &s); err != nil { return otelSchema{}, fmt.Errorf("unmarshal: %w", err) } - if s.FileFormat == transformFileFormat { - tf, err := loadTransformFile(b) - if err != nil { - return otelSchema{}, err - } - s.transform = tf - if s.SchemaURL == "" { - s.SchemaURL = tf.targetURL - } - return s, nil + if s.FileFormat != transformFileFormat { + return otelSchema{}, fmt.Errorf("unsupported OTel schema file format %q (expected %s)", s.FileFormat, transformFileFormat) } - if s.FileFormat != "1.1.0" && s.FileFormat != "1.0.0" { - return otelSchema{}, fmt.Errorf("unsupported OTel schema file format %q (expected 1.0.0, 1.1.0, or transform/2.0)", s.FileFormat) - } - - s.revisions = make([]schemaRevision, 0, len(s.Versions)) - s.allVersions = make([]string, 0, len(s.Versions)) - - for versionStr, version := range s.Versions { - if err := validateSemver(versionStr); err != nil { - return otelSchema{}, err - } - s.allVersions = append(s.allVersions, versionStr) - revision, err := collectSchemaRevision(versionStr, version) - if err != nil { - return otelSchema{}, err - } - if revision != nil { - s.revisions = append(s.revisions, *revision) - } - } - - slices.SortFunc(s.revisions, func(a, b schemaRevision) int { - return compareSemver(a.version, b.version) - }) - slices.SortFunc(s.allVersions, compareSemver) - - return s, nil -} - -// compareSemver compares two MAJOR.MINOR.PATCH version strings. -// Returns -1 if a < b, 0 if a == b, 1 if a > b. Both inputs must have -// already passed validateSemver; otherwise behaviour is undefined. -func compareSemver(a, b string) int { - partsA := strings.Split(a, ".") - partsB := strings.Split(b, ".") - for i := range partsA { - numA, _ := strconv.Atoi(partsA[i]) - numB, _ := strconv.Atoi(partsB[i]) - if numA != numB { - if numA < numB { - return -1 - } - return 1 - } - } - return 0 -} - -// fetchSemconv reads a semconv file from the engine's registry source and parses -// it. The version is derived from the last path segment of url; the URL must -// satisfy registryURLRe and the derived version must satisfy validateSemver. -func (e *schemaEngine) fetchSemconv(url string) (semconv, error) { - b, err := e.readRegistryFile(url) + tf, err := loadTransformFile(b) if err != nil { - return semconv{}, fmt.Errorf("fetch semconv %q: %w", url, err) + return otelSchema{}, err } - _, version := path.Split(url) - s, err := loadSemconv(b, version) - if err != nil { - return semconv{}, fmt.Errorf("parse semconv %q: %w", url, err) - } - return s, nil -} - -// loadSemconv parses raw semconv YAML bytes and post-processes them. The -// version is supplied by the caller (semconv files do not record their own -// version inside the YAML) and must satisfy validateSemver. -func loadSemconv(b []byte, version string) (semconv, error) { - return loadSemconvWithBudget(b, version, nil) -} - -func loadSemconvWithBudget(b []byte, version string, registryBudget *semconvRegistryMaterializationBudget) (semconv, error) { - if err := validateSemver(version); err != nil { - return semconv{}, err - } - var s semconv - if err := yaml.Unmarshal(b, &s); err != nil { - return semconv{}, fmt.Errorf("unmarshal: %w", err) - } - s.version = version - s.metrics = make(map[string]metricDef) - resolver, err := newSemconvAttributeResolver(s.Groups, registryBudget) - if err != nil { - return semconv{}, err - } - resolvedGroups := make([]resolvedSemconvGroup, len(s.Groups)) - for i := range s.Groups { - group := &s.Groups[i] - resolved, err := resolver.resolve(group) - if err != nil { - return semconv{}, fmt.Errorf("resolve attributes for semconv group %q: %w", semconvGroupName(group), err) - } - resolvedGroups[i] = resolved - } - // A metric name declared by two groups has no single definition. Keeping the - // first declaration rather than the last makes the outcome depend on file - // order in one direction only, but either choice is arbitrary, so the name is - // also recorded as ambiguous and reported to the querier as a warning. - var ambiguous map[string]struct{} - for i := range s.Groups { - group := &s.Groups[i] - if group.Type != "metric" || group.MetricName == "" { - continue - } - if _, dup := s.metrics[group.MetricName]; dup { - if ambiguous == nil { - ambiguous = map[string]struct{}{} - } - ambiguous[group.MetricName] = struct{}{} - continue - } - s.metrics[group.MetricName] = metricDef{ - unit: group.Unit, - instrument: group.Instrument, - stability: group.Stability, - attributes: resolvedGroups[i].attributes, - } - } - if len(ambiguous) > 0 { - s.ambiguousMetrics = slices.Sorted(maps.Keys(ambiguous)) + s.transform = tf + if s.SchemaURL == "" { + s.SchemaURL = tf.targetURL } return s, nil } // readRegistryFile reads a file from the engine's registry source. The path must // satisfy registryURLRe — HTTP URLs and arbitrary local files are rejected to -// keep the __semconv_url__/__schema_url__ matchers from acting as a server-side -// fetch primitive. The gate applies to every source, embedded or operator-provided. +// keep the __schema_url__ matcher from acting as a server-side fetch primitive. func (e *schemaEngine) readRegistryFile(url string) ([]byte, error) { if !registryURLRe.MatchString(url) { return nil, fmt.Errorf("invalid registry URL %q: only registry paths (registry/) are accepted", url) diff --git a/storage/semconv/otel_schema_test.go b/storage/semconv/otel_schema_test.go deleted file mode 100644 index dfbf42c015..0000000000 --- a/storage/semconv/otel_schema_test.go +++ /dev/null @@ -1,1046 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv - -import ( - "fmt" - "os" - "strconv" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - "go.yaml.in/yaml/v3" - - "github.com/prometheus/prometheus/model/labels" -) - -// testEngine returns a schemaEngine backed by the embedded registry, for tests -// that exercise the engine's fetch/read methods directly. -func testEngine() *schemaEngine { - return newSchemaEngine(embeddedRegistry) -} - -// loadOTelSchemaFile is a test helper that reads a YAML fixture from disk and -// parses it via the same code path fetchOTelSchema uses. Tests use this rather -// than fetchOTelSchema directly because fetch* is restricted to the embedded -// registry. -func loadOTelSchemaFile(t *testing.T, path string) otelSchema { - t.Helper() - b, err := os.ReadFile(path) - require.NoError(t, err) - s, err := loadOTelSchema(b) - require.NoError(t, err) - return s -} - -func TestLoadOTelSchema(t *testing.T) { - schemaWithChange := func(section, change string) []byte { - return []byte(fmt.Sprintf(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - %s: - changes: -%s -`, section, change)) - } - - t.Run("rejects unsupported file format", func(t *testing.T) { - b, err := os.ReadFile("./testdata/otel_unsupported_format.yaml") - require.NoError(t, err) - _, err = loadOTelSchema(b) - require.Error(t, err) - require.Contains(t, err.Error(), "unsupported OTel schema file format") - }) - - t.Run("rejects unsupported metric splits", func(t *testing.T) { - _, err := loadOTelSchema([]byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - split: - apply_to_metric: http.server.duration - by_attribute: http.request.method - metrics_from_attributes: - http.server.get.duration: GET -`)) - require.EqualError(t, err, `schema version "1.1.0" contains unsupported metric split transformation`) - }) - - for _, tc := range []struct { - name string - section string - change string - wantErr string - }{ - { - name: "misspelled metric rename", - section: "metrics", - change: " - rename_metric: {}", - wantErr: `schema version "1.1.0" metrics change 1 contains unsupported transformation "rename_metric"`, - }, - { - name: "misspelled split", - section: "metrics", - change: " - splitt: {}", - wantErr: `schema version "1.1.0" metrics change 1 contains unsupported transformation "splitt"`, - }, - { - name: "unknown transformation", - section: "metrics", - change: " - unknown: {}", - wantErr: `schema version "1.1.0" metrics change 1 contains unsupported transformation "unknown"`, - }, - { - name: "split in all section", - section: "all", - change: " - split: {}", - wantErr: `schema version "1.1.0" all change 1 contains unsupported transformation "split"`, - }, - { - name: "multiple transformations", - section: "metrics", - change: ` - rename_attributes: {} - rename_metrics: {}`, - wantErr: `schema version "1.1.0" metrics change 1 must contain exactly one transformation, found ["rename_attributes" "rename_metrics"]`, - }, - { - name: "empty change", - section: "metrics", - change: " - {}", - wantErr: `schema version "1.1.0" metrics change 1 must contain exactly one transformation, found []`, - }, - { - name: "misspelled attribute map", - section: "metrics", - change: ` - rename_attributes: - attribute_maps: {}`, - wantErr: `schema version "1.1.0" metrics change 1 rename_attributes contains unsupported field "attribute_maps"`, - }, - { - name: "misspelled metric scope", - section: "metrics", - change: ` - rename_attributes: - apply_to_metric: []`, - wantErr: `schema version "1.1.0" metrics change 1 rename_attributes contains unsupported field "apply_to_metric"`, - }, - { - name: "metric scope in all section", - section: "all", - change: ` - rename_attributes: - apply_to_metrics: []`, - wantErr: `schema version "1.1.0" all change 1 rename_attributes contains unsupported field "apply_to_metrics"`, - }, - { - name: "empty split", - section: "metrics", - change: " - split: {}", - wantErr: `schema version "1.1.0" contains unsupported metric split transformation`, - }, - { - name: "null split", - section: "metrics", - change: " - split:", - wantErr: `schema version "1.1.0" contains unsupported metric split transformation`, - }, - } { - t.Run("rejects "+tc.name, func(t *testing.T) { - _, err := loadOTelSchema(schemaWithChange(tc.section, tc.change)) - require.EqualError(t, err, tc.wantErr) - }) - } - - t.Run("accepts recognized empty transformations", func(t *testing.T) { - schema, err := loadOTelSchema([]byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - rename_attributes: {} - - rename_attributes: - attribute_map: {} - - rename_attributes: - apply_to_metrics: [] - - rename_metrics: {} -`)) - require.NoError(t, err) - require.Empty(t, schema.revisions) - }) - - t.Run("resolves merged transformations", func(t *testing.T) { - schema, err := loadOTelSchema([]byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -attributes: &attributes - attribute_map: - old.name: new.name -change: &change - rename_attributes: - <<: *attributes -versions: - 1.1.0: - metrics: - changes: - - <<: *change -`)) - require.NoError(t, err) - require.Len(t, schema.revisions, 1) - require.Equal(t, "new.name", schema.revisions[0].changes[0].attributeRenames.renames.forward["old.name"]) - }) - - t.Run("rejects merged multiple transformations", func(t *testing.T) { - _, err := loadOTelSchema([]byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -change: &change - rename_attributes: {} -versions: - 1.1.0: - metrics: - changes: - - <<: *change - rename_metrics: {} -`)) - require.EqualError(t, err, `schema version "1.1.0" metrics change 1 must contain exactly one transformation, found ["rename_attributes" "rename_metrics"]`) - }) - - t.Run("collects renames from the all section", func(t *testing.T) { - schema := loadOTelSchemaFile(t, "./testdata/otel_with_all_section.yaml") - require.Len(t, schema.revisions, 1) - require.Len(t, schema.revisions[0].changes, 2) - - global := schema.revisions[0].changes[0].attributeRenames - require.Equal(t, "global.new", global.renames.forward["global.old"]) - require.Equal(t, []string{"global.old"}, global.renames.reverse["global.new"]) - require.True(t, global.appliesTo("any.metric")) - - scoped := schema.revisions[0].changes[1].attributeRenames - require.Equal(t, "metric.new", scoped.renames.forward["metric.old"]) - require.True(t, scoped.appliesTo("my.metric")) - require.False(t, scoped.appliesTo("other.metric")) - }) - - t.Run("collects per-version metric renames", func(t *testing.T) { - schema := loadOTelSchemaFile(t, "./testdata/otel_with_metric_renames.yaml") - require.Len(t, schema.revisions, 1) - renames := schema.revisions[0].changes[0].metricRenames - require.Equal(t, "new.metric.name", renames.forward["old.metric.name"]) - require.Equal(t, []string{"old.metric.name"}, renames.reverse["new.metric.name"]) - require.Equal(t, "another.new.metric", renames.forward["another.old.metric"]) - require.Equal(t, []string{"another.old.metric"}, renames.reverse["another.new.metric"]) - }) - - t.Run("scopes per-version attribute renames to apply_to_metrics", func(t *testing.T) { - schema := loadOTelSchemaFile(t, "./testdata/otel.yaml") - require.Len(t, schema.revisions, 1) - require.Len(t, schema.revisions[0].changes, 2) - http := schema.revisions[0].changes[0].attributeRenames - for _, metric := range []string{"http.server.duration", "http.server.request.count"} { - require.True(t, http.appliesTo(metric)) - } - require.False(t, http.appliesTo("process.cpu.time")) - require.Equal(t, "http.request.method", http.renames.forward["http.method"]) - - cpu := schema.revisions[0].changes[1].attributeRenames - require.True(t, cpu.appliesTo("process.cpu.time")) - require.False(t, cpu.appliesTo("http.server.duration")) - require.Equal(t, "cpu.mode", cpu.renames.forward["process.cpu.state"]) - }) - - t.Run("preserves empty metric scope", func(t *testing.T) { - schema, err := loadOTelSchema([]byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - omitted.old: omitted.new - - rename_attributes: - attribute_map: - empty.old: empty.new - apply_to_metrics: [] - - rename_attributes: - attribute_map: - scoped.old: scoped.new - apply_to_metrics: - - selected.metric -`)) - require.NoError(t, err) - require.Len(t, schema.revisions, 1) - changes := schema.revisions[0].changes - require.Len(t, changes, 3) - - require.True(t, changes[0].attributeRenames.appliesTo("any.metric")) - require.False(t, changes[1].attributeRenames.appliesTo("any.metric")) - require.True(t, changes[2].attributeRenames.appliesTo("selected.metric")) - require.False(t, changes[2].attributeRenames.appliesTo("other.metric")) - }) - - t.Run("collects renames from multiple versions", func(t *testing.T) { - schema := loadOTelSchemaFile(t, "./testdata/otel_with_chained_renames.yaml") - require.Len(t, schema.revisions, 2) - require.Equal(t, "metric.v2", schema.revisions[0].changes[0].metricRenames.forward["metric.v1"]) - require.Equal(t, "metric.v3", schema.revisions[1].changes[0].metricRenames.forward["metric.v2"]) - }) - - t.Run("sorts versions by semver", func(t *testing.T) { - schema := loadOTelSchemaFile(t, "./testdata/otel_with_chained_renames.yaml") - require.Len(t, schema.revisions, 2) - require.Equal(t, "1.0.0", schema.revisions[0].version) - require.Equal(t, "1.1.0", schema.revisions[1].version) - }) - - t.Run("keeps every predecessor of a real many-to-one rename", func(t *testing.T) { - schema := loadOTelSchemaFile(t, "./testdata/upstream/schema-1.44.0.yaml") - var renames *directedRenames - for _, revision := range schema.revisions { - if revision.version != "1.38.0" { - continue - } - for _, change := range revision.changes { - if change.metricRenames != nil { - renames = change.metricRenames - break - } - } - } - require.NotNil(t, renames) - require.Equal(t, []string{ - "k8s.replication_controller.available_pods", - "k8s.replicationcontroller.available_pods", - }, renames.reverse["k8s.replicationcontroller.pod.available"]) - - variants := requireMatcherVariants(t, "1.38.0", &schema, - equalMatchers("k8s.replicationcontroller.pod.available"), nil, nil) - require.ElementsMatch(t, []string{ - "k8s.replicationcontroller.pod.available", - "k8s.replication_controller.available_pods", - "k8s.replicationcontroller.available_pods", - }, variantNames(variants)) - }) -} - -func TestCompareSemver(t *testing.T) { - tests := []struct { - a, b string - expected int - }{ - {"1.0.0", "1.0.0", 0}, - {"1.0.0", "1.0.1", -1}, - {"1.0.1", "1.0.0", 1}, - {"1.0.0", "1.1.0", -1}, - {"1.1.0", "1.0.0", 1}, - {"1.0.0", "2.0.0", -1}, - {"2.0.0", "1.0.0", 1}, - {"1.10.0", "1.9.0", 1}, // Numeric comparison, not string. - {"10.0.0", "9.0.0", 1}, // Double-digit major. - } - for _, tc := range tests { - t.Run(tc.a+"_vs_"+tc.b, func(t *testing.T) { - result := compareSemver(tc.a, tc.b) - require.Equal(t, tc.expected, result) - }) - } -} - -func TestValidateSemver(t *testing.T) { - for _, v := range []string{"1.0.0", "10.20.30", "0.0.0"} { - require.NoErrorf(t, validateSemver(v), "expected %q to be accepted", v) - } - for _, v := range []string{"", "1", "1.0", "1.0.0.0", "1.0.x", "1.0.0-rc1", "v1.0.0"} { - require.Errorf(t, validateSemver(v), "expected %q to be rejected", v) - } -} - -func TestFetchSemconv(t *testing.T) { - t.Run("registry: loads embedded version file", func(t *testing.T) { - sc, err := testEngine().fetchSemconv("registry/1.0.0") - require.NoError(t, err) - require.Equal(t, "1.0.0", sc.version) - }) - - t.Run("registry: rejects path traversal", func(t *testing.T) { - e := testEngine() - for _, url := range []string{ - "registry/../etc/passwd", - "registry/..", - "../etc/passwd", - "/etc/passwd", - "http://example.com/x.yaml", - "https://example.com/x.yaml", - "./testdata/otel.yaml", - "registry/", - "", - } { - _, err := e.fetchSemconv(url) - require.Errorf(t, err, "expected %q to be rejected", url) - } - }) - - t.Run("registry: rejects non-semver version segment", func(t *testing.T) { - // registry.yaml passes the URL regex but is not a semver-named file, - // so version derivation fails. - _, err := testEngine().fetchSemconv("registry/registry.yaml") - require.Error(t, err) - require.Contains(t, err.Error(), "invalid semver") - }) -} - -func TestPredecessorOfWithBudget(t *testing.T) { - schema := loadOTelSchemaFile(t, "./testdata/otel_with_chained_renames.yaml") - require.Equal(t, []string{"1.0.0", "1.1.0"}, schema.allVersions) - - for _, tc := range []struct { - version string - expected string - found bool - }{ - {version: "1.1.0", expected: "1.0.0", found: true}, - {version: "1.0.0"}, - {version: "9.9.9"}, - {version: ""}, - } { - t.Run(tc.version, func(t *testing.T) { - got, found, err := schema.predecessorOfWithBudget(tc.version, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - require.Equal(t, tc.found, found) - require.Equal(t, tc.expected, got) - }) - } -} - -func TestEraVersionsOfWithBudget(t *testing.T) { - eraVersions := func(t *testing.T, schema *otelSchema, name string) []string { - t.Helper() - versions, err := schema.eraVersionsOfWithBudget(name, newSchemaExpansionBudget(productionSchemaExpansionLimits())) - require.NoError(t, err) - return versions - } - - schema, err := loadOTelSchema([]byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - old.name: new.name - 1.2.0: - metrics: - changes: - - rename_metrics: - other.name: old.name -`)) - require.NoError(t, err) - require.Equal(t, []string{"1.0.0", "1.2.0"}, eraVersions(t, &schema, "old.name")) - require.Equal(t, []string{"1.1.0"}, eraVersions(t, &schema, "new.name")) - require.Equal(t, []string{"1.1.0"}, eraVersions(t, &schema, "other.name")) - require.Empty(t, eraVersions(t, &schema, "unrelated.name")) - - ordered, err := loadOTelSchema([]byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - old.name: intermediate.name - - rename_metrics: - intermediate.name: current.name -`)) - require.NoError(t, err) - require.Equal(t, []string{"1.0.0"}, eraVersions(t, &ordered, "old.name")) - require.Empty(t, eraVersions(t, &ordered, "intermediate.name"), "a name used only between ordered changes is not a version boundary") - require.Equal(t, []string{"1.1.0"}, eraVersions(t, &ordered, "current.name")) -} - -func TestLoadSemconv(t *testing.T) { - t.Run("indexes metric groups with unit and instrument", func(t *testing.T) { - sc, err := loadSemconv([]byte(` -groups: - - id: metric.http.server.request.duration - type: metric - metric_name: http.server.request.duration - stability: stable - unit: s - instrument: histogram - attributes: - - ref: http.request.method -`), "1.0.0") - require.NoError(t, err) - require.Equal(t, metricDef{ - unit: "s", - instrument: "histogram", - stability: "stable", - attributes: []string{"http.request.method"}, - }, sc.metrics["http.server.request.duration"]) - require.Empty(t, sc.ambiguousMetrics) - }) - - t.Run("resolves transitive inheritance and group-local inline prefixes", func(t *testing.T) { - sc, err := loadSemconv([]byte(` -groups: - - id: metric.queue.depth - type: metric - metric_name: queue.depth - unit: "{item}" - instrument: updowncounter - extends: attributes.queue - prefix: metric - attributes: - - id: local - - ref: queue.name - - ref: queue.name - - id: metric.service.info - type: metric - metric_name: service.info - extends: attributes.common - - id: attributes.common - type: attribute_group - prefix: service - attributes: - - id: name - - id: attributes.queue - type: attribute_group - extends: attributes.queue.base - prefix: queue - attributes: - - id: capacity - - ref: queue.name - - id: attributes.queue.base - type: attribute_group - extends: attributes.common - prefix: queue - attributes: - - id: priority -`), "1.0.0") - require.NoError(t, err) - require.Equal(t, - []string{"service.name", "queue.priority", "queue.capacity", "queue.name", "metric.local"}, - sc.attributesOf("queue.depth"), - ) - require.Equal(t, []string{"service.name"}, sc.attributesOf("service.info"), - "resolving a child must not mutate its memoized parent") - }) - - t.Run("does not inherit prefixes for inline attributes", func(t *testing.T) { - sc, err := loadSemconv([]byte(` -groups: - - id: attributes.parent - type: attribute_group - prefix: parent - attributes: - - id: inherited - - id: metric.child.absent - type: metric - metric_name: child.absent - extends: attributes.parent - attributes: - - id: local - - id: metric.child.empty - type: metric - metric_name: child.empty - extends: attributes.parent - prefix: "" - attributes: - - id: local - - id: metric.child.explicit - type: metric - metric_name: child.explicit - extends: attributes.parent - prefix: child - attributes: - - id: local -`), "1.0.0") - require.NoError(t, err) - require.Equal(t, []string{"parent.inherited", "local"}, sc.attributesOf("child.absent")) - require.Equal(t, []string{"parent.inherited", "local"}, sc.attributesOf("child.empty")) - require.Equal(t, []string{"parent.inherited", "child.local"}, sc.attributesOf("child.explicit")) - }) - - for _, tc := range []struct { - name string - yaml string - wantErr string - }{ - { - name: "unused group with missing parent", - yaml: ` -groups: - - id: attributes.queue - type: attribute_group - extends: attributes.missing -`, - wantErr: `semconv group "attributes.queue" extends unknown group "attributes.missing"`, - }, - { - name: "unused inheritance cycle", - yaml: ` -groups: - - id: attributes.a - type: attribute_group - extends: attributes.b - - id: attributes.b - type: attribute_group - extends: attributes.a -`, - wantErr: "attributes.a -> attributes.b -> attributes.a", - }, - { - name: "duplicate group id", - yaml: ` -groups: - - id: attributes.queue - type: attribute_group - - id: attributes.queue - type: attribute_group -`, - wantErr: `duplicate semconv group id "attributes.queue"`, - }, - { - name: "unused group attribute has id and ref", - yaml: ` -groups: - - id: attributes.queue - type: attribute_group - attributes: - - id: queue.name - ref: queue.name -`, - wantErr: "declares both id", - }, - { - name: "unused group attribute has neither id nor ref", - yaml: ` -groups: - - id: attributes.queue - type: attribute_group - attributes: - - requirement_level: recommended -`, - wantErr: "declares neither id nor ref", - }, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := loadSemconv([]byte(tc.yaml), "1.0.0") - require.ErrorContains(t, err, tc.wantErr) - }) - } - - t.Run("rejects a malformed discarded metric definition", func(t *testing.T) { - _, err := loadSemconv([]byte(` -groups: - - id: metric.shared.first - type: metric - metric_name: shared - attributes: - - ref: queue.name - - id: metric.shared.second - type: metric - metric_name: shared - attributes: - - requirement_level: recommended -`), "1.0.0") - require.ErrorContains(t, err, `semconv group "metric.shared.second" attribute declares neither id nor ref`) - }) - - for _, tc := range []struct { - name string - metricAttrs int - wantErrContains string - }{ - {name: "accepts 256 resolved attributes", metricAttrs: 6}, - {name: "rejects 257 resolved attributes", metricAttrs: 7, wantErrContains: "semconv group attributes would exceed 256"}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := loadSemconv(inheritedAttributeSemconv(t, 250, tc.metricAttrs), "1.0.0") - if tc.wantErrContains != "" { - require.ErrorContains(t, err, tc.wantErrContains) - return - } - require.NoError(t, err) - }) - } - - t.Run("rejects 257 resolved attributes in an unused group", func(t *testing.T) { - _, err := loadSemconv(marshalSemconv(t, []semconvGroup{ - { - ID: "attributes.unused", - Type: "attribute_group", - Attributes: semconvAttributeRefs("unused.", maxSchemaExpansion+1), - }, - }), "1.0.0") - require.ErrorContains(t, err, "semconv group attributes would exceed 256") - }) - - t.Run("deduplicates before enforcing the resolved attribute limit", func(t *testing.T) { - _, err := loadSemconv(marshalSemconv(t, []semconvGroup{ - { - ID: "attributes.base", - Type: "attribute_group", - Attributes: semconvAttributeRefs("base.", maxSchemaExpansion), - }, - { - ID: "metric.queue.depth", - Type: "metric", - MetricName: "queue.depth", - Extends: "attributes.base", - Attributes: []semconvAttribute{{Ref: "base.0"}}, - }, - }), "1.0.0") - require.NoError(t, err) - }) - - exactFanoutChildren := int(maxSemconvFileAttributeSlots)/maxSchemaExpansion - 1 - for _, tc := range []struct { - name string - children int - wantErrContains string - }{ - {name: "accepts exact file attribute slot limit", children: exactFanoutChildren}, - { - name: "rejects file attribute slots above limit", - children: exactFanoutChildren + 1, - wantErrContains: "semconv file attribute slots would exceed 65536", - }, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := loadSemconv(inheritedAttributeFanoutSemconv(t, tc.children), "1.0.0") - if tc.wantErrContains != "" { - require.ErrorContains(t, err, tc.wantErrContains) - return - } - require.NoError(t, err) - }) - } - - t.Run("charges allocated capacity before deduplication", func(t *testing.T) { - groupsAtLimit := int(maxSemconvFileAttributeSlots) / maxSchemaExpansion - _, err := loadSemconv(duplicateHeavySemconv(t, groupsAtLimit+1), "1.0.0") - require.ErrorContains(t, err, "semconv file attribute slots would exceed 65536") - }) - - t.Run("indexes a metric group that declares no attributes", func(t *testing.T) { - // Such a group is still a metric, so it must be visible to the - // existence check that validates rename edges, even though it - // contributes nothing to attribute-rename normalisation. - sc, err := loadSemconv([]byte(` -groups: - - id: metric.queue.depth - type: metric - metric_name: queue.depth - unit: "{item}" - instrument: updowncounter -`), "1.0.0") - require.NoError(t, err) - require.Contains(t, sc.metrics, "queue.depth") - require.Empty(t, sc.attributesOf("queue.depth")) - }) - - t.Run("reports a metric name declared by more than one group", func(t *testing.T) { - // Two groups, same surface name, different semantics. Previously the - // second silently overwrote the first, so the queue.depth attributes - // were dropped and its unit was reported as the HTTP metric's. - sc, err := loadSemconv([]byte(` -groups: - - id: metric.shared.name - type: metric - metric_name: shared.name - unit: s - instrument: histogram - attributes: - - ref: http.request.method - - id: metric.shared.name.other - type: metric - metric_name: shared.name - unit: "{item}" - instrument: updowncounter - attributes: - - ref: queue.name -`), "1.0.0") - require.NoError(t, err) - require.Equal(t, []string{"shared.name"}, sc.ambiguousMetrics) - // Resolution is deterministic (first declaration wins) rather than - // dependent on which group happened to be parsed last. - require.Equal(t, []string{"http.request.method"}, sc.attributesOf("shared.name")) - }) -} - -func inheritedAttributeSemconv(t *testing.T, parentAttrs, metricAttrs int) []byte { - t.Helper() - return marshalSemconv(t, []semconvGroup{ - { - ID: "attributes.base", - Type: "attribute_group", - Attributes: semconvAttributeRefs("base.", parentAttrs), - }, - { - ID: "metric.queue.depth", - Type: "metric", - MetricName: "queue.depth", - Extends: "attributes.base", - Attributes: semconvAttributeRefs("metric.", metricAttrs), - }, - }) -} - -func inheritedAttributeFanoutSemconv(t *testing.T, children int) []byte { - t.Helper() - groups := make([]semconvGroup, 0, children+1) - groups = append(groups, semconvGroup{ - ID: "attributes.base", - Type: "attribute_group", - Attributes: semconvAttributeRefs("base.", maxSchemaExpansion), - }) - for i := range children { - groups = append(groups, semconvGroup{ - ID: "attributes.child." + strconv.Itoa(i), - Type: "attribute_group", - Extends: "attributes.base", - }) - } - return marshalSemconv(t, groups) -} - -func duplicateHeavySemconv(t *testing.T, groupCount int) []byte { - t.Helper() - attributes := make([]semconvAttribute, maxSchemaExpansion) - for i := range attributes { - attributes[i].Ref = "shared" - } - groups := make([]semconvGroup, groupCount) - for i := range groups { - groups[i] = semconvGroup{ - ID: "attributes.duplicates." + strconv.Itoa(i), - Type: "attribute_group", - Attributes: attributes, - } - } - return marshalSemconv(t, groups) -} - -func semconvAttributeRefs(prefix string, count int) []semconvAttribute { - result := make([]semconvAttribute, count) - for i := range count { - result[i].Ref = prefix + strconv.Itoa(i) - } - return result -} - -func marshalSemconv(t *testing.T, groups []semconvGroup) []byte { - t.Helper() - b, err := yaml.Marshal(semconv{Groups: groups}) - require.NoError(t, err) - return b -} - -func TestTransformOTelSchemaLabels(t *testing.T) { - t.Run("transforms metric and label names", func(t *testing.T) { - lbls := labels.FromStrings( - model.MetricNameLabel, "http_server_duration_seconds", - "http_method", "GET", - "http_status_code", "200", - "instance", "localhost:8080", - ) - - mapping := &labelMapping{ - translatedMetric: "http.server.duration", - translatedLabels: map[string]string{ - "http_method": "http.method", - "http_status_code": "http.status_code", - }, - } - - result, err := transformOTelSchemaLabels(lbls, mapping) - require.NoError(t, err) - - require.Equal(t, "http.server.duration", result.Get(model.MetricNameLabel)) - require.Equal(t, "GET", result.Get("http.method")) - require.Equal(t, "200", result.Get("http.status_code")) - require.Equal(t, "localhost:8080", result.Get("instance")) - require.Empty(t, result.Get("http_method")) - require.Empty(t, result.Get("http_status_code")) - }) - - t.Run("removes __schema_url__", func(t *testing.T) { - lbls := labels.FromStrings( - model.MetricNameLabel, "http_server_duration_seconds", - schemaURLLabel, "https://example.com/otel.yaml", - "http_method", "GET", - ) - - mapping := &labelMapping{ - translatedMetric: "http.server.duration", - translatedLabels: map[string]string{}, - } - - result, err := transformOTelSchemaLabels(lbls, mapping) - require.NoError(t, err) - - require.Empty(t, result.Get(schemaURLLabel)) - require.Equal(t, "http.server.duration", result.Get(model.MetricNameLabel)) - }) - - t.Run("sorts labels after a rename", func(t *testing.T) { - lbls := labels.FromStrings( - model.MetricNameLabel, "jvm.thread.count", - "service.name", "api", - "thread.daemon", "true", - ) - mapping := &labelMapping{ - translatedMetric: "jvm.thread.count", - translatedLabels: map[string]string{"thread.daemon": "jvm.thread.daemon"}, - } - - result, err := transformOTelSchemaLabels(lbls, mapping) - require.NoError(t, err) - require.Equal(t, labels.FromStrings( - model.MetricNameLabel, "jvm.thread.count", - "jvm.thread.daemon", "true", - "service.name", "api", - ), result) - require.Equal(t, "true", result.Get("jvm.thread.daemon")) - }) - - t.Run("collapses aliases with equal values", func(t *testing.T) { - lbls := labels.FromStrings( - model.MetricNameLabel, "jvm.thread.count", - "jvm.thread.daemon", "true", - "thread.daemon", "true", - ) - mapping := &labelMapping{ - translatedMetric: "jvm.thread.count", - translatedLabels: map[string]string{"thread.daemon": "jvm.thread.daemon"}, - } - - result, err := transformOTelSchemaLabels(lbls, mapping) - require.NoError(t, err) - require.Equal(t, labels.FromStrings( - model.MetricNameLabel, "jvm.thread.count", - "jvm.thread.daemon", "true", - ), result) - }) - - t.Run("rejects aliases with conflicting values", func(t *testing.T) { - lbls := labels.FromStrings( - model.MetricNameLabel, "jvm.thread.count", - "jvm.thread.daemon", "false", - "thread.daemon", "true", - ) - mapping := &labelMapping{ - translatedMetric: "jvm.thread.count", - translatedLabels: map[string]string{"thread.daemon": "jvm.thread.daemon"}, - } - - _, err := transformOTelSchemaLabels(lbls, mapping) - require.ErrorContains(t, err, `maps "jvm.thread.daemon" and "thread.daemon" to "jvm.thread.daemon" with conflicting values`) - }) - - t.Run("handles many-to-one mappings", func(t *testing.T) { - mapping := &labelMapping{ - translatedMetric: "metric.current", - translatedLabels: map[string]string{ - "legacy.a": "current", - "legacy.b": "current", - }, - } - - result, err := transformOTelSchemaLabels(labels.FromStrings( - model.MetricNameLabel, "metric.old", - "legacy.a", "same", - "legacy.b", "same", - ), mapping) - require.NoError(t, err) - require.Equal(t, labels.FromStrings( - model.MetricNameLabel, "metric.current", - "current", "same", - ), result) - - _, err = transformOTelSchemaLabels(labels.FromStrings( - model.MetricNameLabel, "metric.old", - "legacy.a", "one", - "legacy.b", "two", - ), mapping) - require.ErrorContains(t, err, `maps "legacy.a" and "legacy.b" to "current" with conflicting values`) - }) -} - -func TestReadRegistryFile(t *testing.T) { - t.Run("loads embedded registry entry", func(t *testing.T) { - b, err := testEngine().readRegistryFile("registry/1.0.0") - require.NoError(t, err) - require.NotEmpty(t, b) - }) - - t.Run("rejects HTTP, absolute paths, traversal, and non-registry paths", func(t *testing.T) { - e := testEngine() - for _, url := range []string{ - "http://example.com/x.yaml", - "/etc/passwd", - "registry/../etc/passwd", - "registry/..", - "./testdata/otel.yaml", - "registry/", - } { - _, err := e.readRegistryFile(url) - require.Errorf(t, err, "expected %q to be rejected", url) - } - }) -} - -// TestUpstreamSemconvAttributes pins inherited metric attributes from the real -// v1.21.0 and v1.22.0 HTTP semantic-convention groups. -func TestUpstreamSemconvAttributes(t *testing.T) { - for _, tc := range []struct { - version string - metric string - attributes []string - }{ - { - version: "1.21.0", - metric: "http.server.duration", - attributes: []string{ - "http.route", "server.address", "server.port", "url.scheme", - "http.request.method", "http.response.status_code", - "network.protocol.name", "network.protocol.version", - }, - }, - { - version: "1.22.0", - metric: "http.server.request.duration", - attributes: []string{ - "http.request.method", "http.response.status_code", "error.type", - "network.protocol.name", "network.protocol.version", "http.route", - "server.address", "server.port", "url.scheme", - }, - }, - } { - t.Run(tc.version, func(t *testing.T) { - b, err := os.ReadFile("./testdata/upstream/semconv-" + tc.version + ".yaml") - require.NoError(t, err) - sc, err := loadSemconv(b, tc.version) - require.NoError(t, err) - - require.Equal(t, tc.attributes, sc.attributesOf(tc.metric)) - require.Equal(t, "s", sc.metrics[tc.metric].unit) - require.Equal(t, "histogram", sc.metrics[tc.metric].instrument) - }) - } -} diff --git a/storage/semconv/promql_rewrite.go b/storage/semconv/promql_rewrite.go index ce91a85408..84c407452a 100644 --- a/storage/semconv/promql_rewrite.go +++ b/storage/semconv/promql_rewrite.go @@ -33,7 +33,7 @@ import ( // matchers, label_replace / label_join wrappers, and sample value filters so the // resulting query can be executed against an unmodified Prometheus engine. func RewritePromQLQuery(query, defaultSchemaURL string) (string, annotations.Annotations, error) { - return rewritePromQLWithEngine(newSchemaEngine(embeddedRegistry), query, defaultSchemaURL) + return rewritePromQLWithEngine(newSchemaEngine(newRegistrySource(nil)), query, defaultSchemaURL) } // RewritePromQLQueryWithRegistry is RewritePromQLQuery backed by an operator-provided diff --git a/storage/semconv/registry.go b/storage/semconv/registry.go index ebd6c25487..caa64dec9b 100644 --- a/storage/semconv/registry.go +++ b/storage/semconv/registry.go @@ -85,26 +85,12 @@ func newRegistrySource(files map[string][]byte) registrySource { // listRegistryYAMLFiles returns the sorted registry/.yaml paths available // in src so transform/2.0 policies can be discovered by target schema_url. func listRegistryYAMLFiles(src registrySource) []string { - switch s := src.(type) { - case fs.ReadDirFS: - entries, err := s.ReadDir(registryPrefix) - if err != nil { - return nil - } - var names []string - for _, e := range entries { - if !e.IsDir() && strings.HasSuffix(e.Name(), ".yaml") { - names = append(names, registryPrefix+"/"+e.Name()) - } - } - slices.Sort(names) - return names - case prefixedRegistry: - if rf, ok := s.inner.(registryFiles); ok { + if p, ok := src.(prefixedRegistry); ok { + if rf, ok := p.inner.(registryFiles); ok { var names []string for k := range rf { if strings.HasSuffix(k, ".yaml") { - names = append(names, s.prefix+"/"+k) + names = append(names, p.prefix+"/"+k) } } slices.Sort(names) @@ -123,14 +109,7 @@ func cloneRegistryFiles(files map[string][]byte) map[string][]byte { } // validateRegistryFiles reports whether files form a usable registry: a non-empty -// set in which every semver-named file (e.g. "1.0.0") parses as a semconv file, -// every other file (e.g. "registry.yaml") parses as an OTel schema, and at least -// one OTel schema is present. Semconv materialization is bounded cumulatively -// across the registry. It rejects invalid input at startup instead of letting it -// fail only at query time. Only versions queried as __semconv_url__ anchors -// require a semconv file; a missing sibling version disables corroboration for -// that boundary and produces a query warning. It reuses loadSemconv/loadOTelSchema -// so validation matches what queries rely on. +// set in which every file parses as a valid transform/2.0 schema. func validateRegistryFiles(files map[string][]byte) error { if len(files) == 0 { return errors.New("registry is empty") @@ -140,23 +119,10 @@ func validateRegistryFiles(files map[string][]byte) error { names = append(names, name) } slices.Sort(names) - registryBudget := &semconvRegistryMaterializationBudget{} - hasSchema := false for _, name := range names { - b := files[name] - if semverRe.MatchString(name) { - if _, err := loadSemconvWithBudget(b, name, registryBudget); err != nil { - return fmt.Errorf("registry semconv %q: %w", name, err) - } - continue - } - if _, err := loadOTelSchema(b); err != nil { + if _, err := loadOTelSchema(files[name]); err != nil { return fmt.Errorf("registry schema %q: %w", name, err) } - hasSchema = true - } - if !hasSchema { - return errors.New("registry has no OTel schema file") } return nil } diff --git a/storage/semconv/registry/1.0.0 b/storage/semconv/registry/1.0.0 deleted file mode 100644 index 6f981ba02e..0000000000 --- a/storage/semconv/registry/1.0.0 +++ /dev/null @@ -1,30 +0,0 @@ -groups: - - id: registry.http - type: attribute_group - brief: "HTTP attributes" - attributes: - - id: http.response.status_code - type: int - stability: stable - brief: "HTTP response status code" - examples: [200, 404, 500] - - id: registry.user - type: attribute_group - brief: "User attributes" - attributes: - - id: user - type: string - stability: stable - brief: "The username or user identifier" - examples: ["alice", "bob"] - - id: metric.test.counter - type: metric - brief: "Counts the number of iterations in the test application" - stability: development - metric_name: test.counter - instrument: counter - unit: By - attributes: - - ref: http.response.status_code - requirement_level: recommended - - ref: user diff --git a/storage/semconv/registry/1.1.0 b/storage/semconv/registry/1.1.0 deleted file mode 100644 index e97ade346c..0000000000 --- a/storage/semconv/registry/1.1.0 +++ /dev/null @@ -1,30 +0,0 @@ -groups: - - id: registry.http - type: attribute_group - brief: "HTTP attributes" - attributes: - - id: http.response.status_code - type: int - stability: stable - brief: "HTTP response status code" - examples: [200, 404, 500] - - id: registry.tenant - type: attribute_group - brief: "Tenant attributes" - attributes: - - id: tenant - type: string - stability: stable - brief: "The tenant identifier" - examples: ["alice", "bob"] - - id: metric.test - type: metric - brief: "Counts the number of iterations in the test application" - stability: stable - metric_name: test - instrument: counter - unit: By - attributes: - - ref: http.response.status_code - requirement_level: recommended - - ref: tenant diff --git a/storage/semconv/registry/1.2.0 b/storage/semconv/registry/1.2.0 deleted file mode 100644 index 86a41dea71..0000000000 --- a/storage/semconv/registry/1.2.0 +++ /dev/null @@ -1,30 +0,0 @@ -groups: - - id: registry.http - type: attribute_group - brief: "HTTP attributes" - attributes: - - id: http.response.status_code - type: int - stability: stable - brief: "HTTP response status code" - examples: [200, 404, 500] - - id: registry.tenant - type: attribute_group - brief: "Tenant attributes" - attributes: - - id: tenant - type: string - stability: stable - brief: "The tenant identifier" - examples: ["alice", "bob"] - - id: metric.test - type: metric - brief: "Counts the number of iterations in the test application" - stability: stable - metric_name: test.v2 - instrument: counter - unit: By - attributes: - - ref: http.response.status_code - requirement_level: recommended - - ref: tenant diff --git a/storage/semconv/registry/registry.yaml b/storage/semconv/registry/registry.yaml deleted file mode 100644 index e5aaf33359..0000000000 --- a/storage/semconv/registry/registry.yaml +++ /dev/null @@ -1,22 +0,0 @@ -file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 - -versions: - 1.0.0: - - 1.1.0: - metrics: - changes: - - rename_metrics: - test.counter: test - - rename_attributes: - attribute_map: - user: tenant - apply_to_metrics: - - test - - 1.2.0: - metrics: - changes: - - rename_metrics: - test: test.v2 diff --git a/storage/semconv/registry_test.go b/storage/semconv/registry_test.go index c217b47010..7feb9591c8 100644 --- a/storage/semconv/registry_test.go +++ b/storage/semconv/registry_test.go @@ -29,24 +29,23 @@ import ( "github.com/stretchr/testify/require" ) -// embeddedRegistryFiles reads the embedded registry into a base-name → bytes -// map, the shape an operator-provided registry takes. Each call returns a fresh -// map so tests may mutate it. -func embeddedRegistryFiles(t *testing.T) map[string][]byte { - t.Helper() - entries, err := embeddedRegistry.ReadDir(registryPrefix) - require.NoError(t, err) - files := map[string][]byte{} - for _, e := range entries { - if e.IsDir() { - continue - } - b, err := embeddedRegistry.ReadFile(registryPrefix + "/" + e.Name()) - require.NoError(t, err) - files[e.Name()] = b +func sampleRegistryFiles() map[string][]byte { + return map[string][]byte{ + "policy.yaml": []byte(`file_format: transform/2.0 +source: + schema_url: https://example.com/schemas/1.0.0 +target: + schema_url: https://example.com/schemas/2.0.0 +policies: + - id: rename-test + metric_transform: + matches: + - target: {descriptor_field: METRIC_DESCRIPTOR_FIELD_NAME} + predicate: {equals: test.counter} + add: + - {target: {descriptor_field: METRIC_DESCRIPTOR_FIELD_NAME}, value: test, upsert: true} +`), } - require.NotEmpty(t, files) - return files } // tarGz builds a gzip-compressed tar of files (keyed by entry name) for tests. @@ -66,110 +65,46 @@ func tarGz(t *testing.T, files map[string][]byte) []byte { } func TestNewRegistrySource(t *testing.T) { - files := embeddedRegistryFiles(t) + files := sampleRegistryFiles() e := newSchemaEngine(newRegistrySource(files)) t.Run("resolves a registry/ path", func(t *testing.T) { - b, err := e.readRegistryFile("registry/1.0.0") + b, err := e.readRegistryFile("registry/policy.yaml") require.NoError(t, err) - require.Equal(t, files["1.0.0"], b) - }) - - t.Run("resolves the schema index", func(t *testing.T) { - b, err := e.readRegistryFile("registry/registry.yaml") - require.NoError(t, err) - require.NotEmpty(t, b) + require.Equal(t, files["policy.yaml"], b) }) t.Run("rejects unprefixed, traversal, and remote paths", func(t *testing.T) { - for _, bad := range []string{"1.0.0", "registry/..", "registry/../etc/passwd", "/etc/passwd", "http://example.com/x"} { + for _, bad := range []string{"policy.yaml", "registry/..", "registry/../etc/passwd", "/etc/passwd", "http://example.com/x"} { _, err := e.readRegistryFile(bad) require.Errorf(t, err, "expected %q to be rejected", bad) } }) t.Run("reports a missing entry", func(t *testing.T) { - _, err := e.readRegistryFile("registry/9.9.9") + _, err := e.readRegistryFile("registry/missing.yaml") require.Error(t, err) }) } func TestValidateRegistryFiles(t *testing.T) { - t.Run("accepts the embedded layout", func(t *testing.T) { - require.NoError(t, validateRegistryFiles(embeddedRegistryFiles(t))) - }) - - registryWithFanoutVersions := func(t *testing.T, versionCount int) map[string][]byte { - t.Helper() - exactFanoutChildren := int(maxSemconvFileAttributeSlots)/maxSchemaExpansion - 1 - semconvFile := inheritedAttributeFanoutSemconv(t, exactFanoutChildren) - files := map[string][]byte{"registry.yaml": []byte("file_format: 1.1.0\n")} - for i := range versionCount { - files[fmt.Sprintf("1.%d.0", i)] = semconvFile - } - return files - } - - registryVersionsAtLimit := int(maxSemconvRegistryAttributeSlots / maxSemconvFileAttributeSlots) - t.Run("accepts exact registry attribute slot limit", func(t *testing.T) { - require.NoError(t, validateRegistryFiles(registryWithFanoutVersions(t, registryVersionsAtLimit))) - }) - - t.Run("rejects registry attribute slots above limit", func(t *testing.T) { - err := validateRegistryFiles(registryWithFanoutVersions(t, registryVersionsAtLimit+1)) - require.ErrorContains(t, err, "semconv registry attribute slots would exceed 1048576") + t.Run("accepts a valid transform/2.0 registry", func(t *testing.T) { + require.NoError(t, validateRegistryFiles(sampleRegistryFiles())) }) t.Run("rejects an empty registry", func(t *testing.T) { require.Error(t, validateRegistryFiles(nil)) }) - t.Run("rejects an unparseable schema", func(t *testing.T) { - require.Error(t, validateRegistryFiles(map[string][]byte{"registry.yaml": []byte("file_format: 9.9.9\n")})) - }) - - t.Run("rejects a registry with no schema", func(t *testing.T) { - files := embeddedRegistryFiles(t) - for name := range files { - if !semverRe.MatchString(name) { - delete(files, name) - } - } - require.Error(t, validateRegistryFiles(files)) - }) - - t.Run("accepts a registry shipping a subset of referenced versions", func(t *testing.T) { - // The schema references 1.0.0/1.1.0/1.2.0; shipping only some is valid — - // operators provide the versions they query as anchors, not the whole history. - files := embeddedRegistryFiles(t) - delete(files, "1.2.0") - require.NoError(t, validateRegistryFiles(files)) - }) - - t.Run("rejects an unparseable semconv version file", func(t *testing.T) { - files := embeddedRegistryFiles(t) - files["1.0.0"] = []byte("groups: [unterminated") - require.Error(t, validateRegistryFiles(files)) - }) - - t.Run("validates files in name order", func(t *testing.T) { - files := map[string][]byte{ - "registry.yaml": []byte("file_format: 1.1.0\n"), - "2.0.0": []byte("groups: [second"), - "1.0.0": []byte("groups: [first"), - } - err := validateRegistryFiles(files) - require.ErrorContains(t, err, `registry semconv "1.0.0"`) + t.Run("rejects an unparseable or unsupported schema format", func(t *testing.T) { + require.Error(t, validateRegistryFiles(map[string][]byte{"policy.yaml": []byte("file_format: 9.9.9\n")})) + require.Error(t, validateRegistryFiles(map[string][]byte{"policy.yaml": []byte("file_format: 1.1.0\n")})) }) } func TestUntargz(t *testing.T) { t.Run("flattens entry paths to base names", func(t *testing.T) { - want := map[string][]byte{ - "registry.yaml": []byte("file_format: 1.1.0\n"), - "1.0.0": []byte("groups: []\n"), - } - // Lay entries under a top-level directory to exercise flattening. + want := sampleRegistryFiles() nested := map[string][]byte{} for name, b := range want { nested["registry/"+name] = b @@ -180,7 +115,7 @@ func TestUntargz(t *testing.T) { }) t.Run("rejects a duplicate base name", func(t *testing.T) { - archive := tarGz(t, map[string][]byte{"a/1.0.0": []byte("x"), "b/1.0.0": []byte("y")}) + archive := tarGz(t, map[string][]byte{"a/policy.yaml": []byte("x"), "b/policy.yaml": []byte("y")}) _, err := untargz(bytes.NewReader(archive)) require.Error(t, err) require.Contains(t, err.Error(), "duplicate") @@ -202,7 +137,7 @@ func TestUntargz(t *testing.T) { tw := tar.NewWriter(gz) size := int64(maxRegistryDecompressedBytes) + (1 << 20) require.NoError(t, tw.WriteHeader(&tar.Header{Name: "big", Typeflag: tar.TypeReg, Mode: 0o644, Size: size})) - chunk := make([]byte, 1<<20) // Zeros compress to almost nothing. + chunk := make([]byte, 1<<20) for written := int64(0); written < size; { n := int64(len(chunk)) if rem := size - written; rem < n { @@ -224,7 +159,7 @@ func TestUntargz(t *testing.T) { func TestLoadRegistryFiles(t *testing.T) { t.Run("reads matched files keyed by base name", func(t *testing.T) { dir := t.TempDir() - want := embeddedRegistryFiles(t) + want := sampleRegistryFiles() for name, b := range want { require.NoError(t, os.WriteFile(filepath.Join(dir, name), b, 0o644)) } @@ -240,8 +175,8 @@ func TestLoadRegistryFiles(t *testing.T) { t.Run("rejects a duplicate base name across globs", func(t *testing.T) { dirA, dirB := t.TempDir(), t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dirA, "1.0.0"), []byte("a"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dirB, "1.0.0"), []byte("b"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dirA, "policy.yaml"), []byte("a"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dirB, "policy.yaml"), []byte("b"), 0o644)) _, err := LoadRegistryFiles([]string{filepath.Join(dirA, "*"), filepath.Join(dirB, "*")}) require.Error(t, err) require.Contains(t, err.Error(), "duplicate") @@ -249,7 +184,7 @@ func TestLoadRegistryFiles(t *testing.T) { } func TestFetchRegistry(t *testing.T) { - want := embeddedRegistryFiles(t) + want := sampleRegistryFiles() t.Run("downloads and unpacks an archive", func(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/storage/semconv/storage.go b/storage/semconv/storage.go index ff950ee354..ce42b972e9 100644 --- a/storage/semconv/storage.go +++ b/storage/semconv/storage.go @@ -64,7 +64,7 @@ func schemaWarning(msg string) error { // canonical naming. Queries without those matchers are passed through // unchanged. func AwareStorage(s storage.Storage) storage.Storage { - return newAwareStorage(s, newSchemaEngine(embeddedRegistry)) + return newAwareStorage(s, newSchemaEngine(newRegistrySource(nil))) } // AwareStorageWithRegistry behaves like AwareStorage but resolves __semconv_url__ diff --git a/storage/semconv/storage_contract_test.go b/storage/semconv/storage_contract_test.go deleted file mode 100644 index 03598c6239..0000000000 --- a/storage/semconv/storage_contract_test.go +++ /dev/null @@ -1,879 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv - -import ( - "context" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/util/annotations" -) - -func TestStorageFanOutContracts(t *testing.T) { - t.Run("canonical labels are valid", func(t *testing.T) { - mapping := buildLabelMapping("metric.new", map[string]string{"old": "new"}) - got, err := transformOTelSchemaLabels(labels.FromStrings( - model.MetricNameLabel, "metric.old", - semconvURLLabel, "registry/1.0.0", - "old", "same", - "new", "same", - ), mapping) - require.NoError(t, err) - require.Equal(t, labels.FromStrings(model.MetricNameLabel, "metric.new", "new", "same"), got) - }) - - t.Run("conflicting canonical labels fail", func(t *testing.T) { - mapping := buildLabelMapping("metric.new", map[string]string{"old": "new"}) - _, err := transformOTelSchemaLabels(labels.FromStrings("old", "first", "new", "second"), mapping) - require.ErrorContains(t, err, "conflicting values") - }) - - t.Run("only attribute mappings require resorting", func(t *testing.T) { - require.False(t, mappingNeedsResort(buildLabelMapping("metric.new", nil))) - require.True(t, mappingNeedsResort(buildLabelMapping("metric.new", map[string]string{"old": "new"}))) - }) - - t.Run("identity mappings preserve underlying objects", func(t *testing.T) { - mapping := buildLabelMapping("metric", nil) - series := storage.NewListSeries(labels.FromStrings(model.MetricNameLabel, "metric", "instance", "a"), nil) - seriesSet := &awareSeriesSet{ - SeriesSet: &singleSeriesSet{series: series}, - mapping: mapping, - } - require.True(t, seriesSet.Next()) - require.Same(t, series, seriesSet.At()) - - chunkSeries := storage.NewSeriesToChunkEncoder(series) - chunkSet := &awareChunkSeriesSet{ - ChunkSeriesSet: &singleChunkSeriesSet{series: chunkSeries}, - mapping: mapping, - } - require.True(t, chunkSet.Next()) - require.Same(t, chunkSeries, chunkSet.At()) - }) - - t.Run("select hints are cloned", func(t *testing.T) { - original := &storage.SelectHints{ - Start: 1, - End: 2, - Limit: 3, - Grouping: []string{"group"}, - ProjectionLabels: []string{"project"}, - } - cloned := cloneSelectHints(original) - cloned.Limit = 0 - cloned.Grouping[0] = "mutated group" - cloned.ProjectionLabels[0] = "mutated projection" - require.Equal(t, 3, original.Limit) - require.Equal(t, []string{"group"}, original.Grouping) - require.Equal(t, []string{"project"}, original.ProjectionLabels) - }) - - t.Run("canonical materialization is bounded", func(t *testing.T) { - budget := canonicalSeriesBudget{kind: "series", limit: 2, remaining: 2} - require.NoError(t, budget.take()) - require.NoError(t, budget.take()) - require.ErrorIs(t, budget.take(), errCanonicalSeriesMaterialization) - }) - - t.Run("canonical post-filter skips nonmatches and charges input", func(t *testing.T) { - matcher := labels.MustNewMatcher(labels.MatchEqual, "job", "keep") - filtered := storage.NewListSeries(labels.FromStrings(model.MetricNameLabel, "metric", "job", "drop"), nil) - matched := storage.NewListSeries(labels.FromStrings(model.MetricNameLabel, "metric", "job", "keep"), nil) - - seriesBudget := newCanonicalSeriesBudget("series", 2) - seriesSet := &awareSeriesSet{ - SeriesSet: &sliceSeriesSet{series: []storage.Series{filtered, matched}, index: -1}, - mapping: buildLabelMapping("metric", nil), - canonicalMatchers: []*labels.Matcher{matcher}, - budget: seriesBudget, - } - require.True(t, seriesSet.Next()) - require.Same(t, matched, seriesSet.At()) - require.False(t, seriesSet.Next()) - require.NoError(t, seriesSet.Err()) - require.Zero(t, seriesBudget.remaining) - - chunkBudget := newCanonicalSeriesBudget("chunks", 2) - matchedChunk := storage.NewSeriesToChunkEncoder(matched) - chunkSet := &awareChunkSeriesSet{ - ChunkSeriesSet: &sliceChunkSeriesSet{ - series: []storage.ChunkSeries{storage.NewSeriesToChunkEncoder(filtered), matchedChunk}, - index: -1, - }, - mapping: buildLabelMapping("metric", nil), - canonicalMatchers: []*labels.Matcher{matcher}, - budget: chunkBudget, - } - require.True(t, chunkSet.Next()) - require.Same(t, matchedChunk, chunkSet.At()) - require.False(t, chunkSet.Next()) - require.NoError(t, chunkSet.Err()) - require.Zero(t, chunkBudget.remaining) - }) - - t.Run("label value fan-out is bounded", func(t *testing.T) { - variants := []matcherVariant{ - {mapping: buildLabelMapping("metric", map[string]string{"old": "new"})}, - {mapping: buildLabelMapping("metric", nil)}, - } - jobs, err := buildLabelValueJobs(variants, "new") - require.NoError(t, err) - require.Len(t, jobs, 3) - - _, err = buildLabelValueJobsUpTo(variants, "new", 2) - require.ErrorIs(t, err, errSchemaExpansion) - }) -} - -type singleSeriesSet struct { - series storage.Series - next bool -} - -func (s *singleSeriesSet) Next() bool { - if s.next { - return false - } - s.next = true - return true -} - -func (s *singleSeriesSet) At() storage.Series { return s.series } -func (*singleSeriesSet) Err() error { return nil } -func (*singleSeriesSet) Warnings() annotations.Annotations { return nil } - -type singleChunkSeriesSet struct { - series storage.ChunkSeries - next bool -} - -func (s *singleChunkSeriesSet) Next() bool { - if s.next { - return false - } - s.next = true - return true -} - -func (s *singleChunkSeriesSet) At() storage.ChunkSeries { return s.series } -func (*singleChunkSeriesSet) Err() error { return nil } -func (*singleChunkSeriesSet) Warnings() annotations.Annotations { return nil } - -type sliceSeriesSet struct { - series []storage.Series - index int -} - -func (s *sliceSeriesSet) Next() bool { - s.index++ - return s.index < len(s.series) -} - -func (s *sliceSeriesSet) At() storage.Series { return s.series[s.index] } -func (*sliceSeriesSet) Err() error { return nil } -func (*sliceSeriesSet) Warnings() annotations.Annotations { return nil } - -type sliceChunkSeriesSet struct { - series []storage.ChunkSeries - index int -} - -func (s *sliceChunkSeriesSet) Next() bool { - s.index++ - return s.index < len(s.series) -} - -func (s *sliceChunkSeriesSet) At() storage.ChunkSeries { return s.series[s.index] } -func (*sliceChunkSeriesSet) Err() error { return nil } -func (*sliceChunkSeriesSet) Warnings() annotations.Annotations { return nil } - -type resolverBudgetCallCounts struct { - series int - chunk int - labelNames int - labelValues int -} - -type optionalSearchStorage struct { - storage.Storage - - querier storage.Querier - chunkQuerier storage.ChunkQuerier -} - -func (s *optionalSearchStorage) Querier(int64, int64) (storage.Querier, error) { - return s.querier, nil -} - -func (s *optionalSearchStorage) ChunkQuerier(int64, int64) (storage.ChunkQuerier, error) { - return s.chunkQuerier, nil -} - -type nonSearchQuerier struct { - storage.Querier - closes *int -} - -func (q *nonSearchQuerier) Close() error { - *q.closes++ - return nil -} - -type nonSearchChunkQuerier struct { - storage.ChunkQuerier - closes *int -} - -func (q *nonSearchChunkQuerier) Close() error { - *q.closes++ - return nil -} - -type searchCall struct { - kind string - ctx context.Context - name string - hints *storage.SearchHints - matchers []*labels.Matcher -} - -type searchRecorder struct { - calls []searchCall - result storage.SearchResultSet -} - -func (r *searchRecorder) labelNames(ctx context.Context, hints *storage.SearchHints, matchers ...*labels.Matcher) storage.SearchResultSet { - r.calls = append(r.calls, searchCall{kind: "names", ctx: ctx, hints: hints, matchers: matchers}) - return r.result -} - -func (r *searchRecorder) labelValues(ctx context.Context, name string, hints *storage.SearchHints, matchers ...*labels.Matcher) storage.SearchResultSet { - r.calls = append(r.calls, searchCall{kind: "values", ctx: ctx, name: name, hints: hints, matchers: matchers}) - return r.result -} - -type searchableQuerier struct { - *nonSearchQuerier - recorder *searchRecorder -} - -func (q *searchableQuerier) SearchLabelNames(ctx context.Context, hints *storage.SearchHints, matchers ...*labels.Matcher) storage.SearchResultSet { - return q.recorder.labelNames(ctx, hints, matchers...) -} - -func (q *searchableQuerier) SearchLabelValues(ctx context.Context, name string, hints *storage.SearchHints, matchers ...*labels.Matcher) storage.SearchResultSet { - return q.recorder.labelValues(ctx, name, hints, matchers...) -} - -type searchableChunkQuerier struct { - *nonSearchChunkQuerier - recorder *searchRecorder -} - -func (q *searchableChunkQuerier) SearchLabelNames(ctx context.Context, hints *storage.SearchHints, matchers ...*labels.Matcher) storage.SearchResultSet { - return q.recorder.labelNames(ctx, hints, matchers...) -} - -func (q *searchableChunkQuerier) SearchLabelValues(ctx context.Context, name string, hints *storage.SearchHints, matchers ...*labels.Matcher) storage.SearchResultSet { - return q.recorder.labelValues(ctx, name, hints, matchers...) -} - -type recordingSearchResultSet struct { - closes int -} - -func (*recordingSearchResultSet) Next() bool { return false } -func (*recordingSearchResultSet) At() storage.SearchResult { return storage.SearchResult{} } -func (*recordingSearchResultSet) Warnings() annotations.Annotations { return nil } -func (*recordingSearchResultSet) Err() error { return nil } - -func (s *recordingSearchResultSet) Close() error { - s.closes++ - return nil -} - -func TestAwareStoragePreservesOptionalSearcher(t *testing.T) { - assertDelegates := func(t *testing.T, searcher storage.Searcher, recorder *searchRecorder) { - t.Helper() - type contextKey struct{} - ctx := context.WithValue(t.Context(), contextKey{}, "sentinel") - hints := &storage.SearchHints{Limit: 7} - matchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "job", "api")} - - result := searcher.SearchLabelNames(ctx, hints, matchers...) - require.Same(t, recorder.result, result) - require.NoError(t, result.Close()) - result = searcher.SearchLabelValues(ctx, "instance", hints, matchers...) - require.Same(t, recorder.result, result) - require.NoError(t, result.Close()) - - require.Equal(t, []searchCall{ - {kind: "names", ctx: ctx, hints: hints, matchers: matchers}, - {kind: "values", ctx: ctx, name: "instance", hints: hints, matchers: matchers}, - }, recorder.calls) - } - assertRejectsReserved := func(t *testing.T, searcher storage.Searcher, recorder *searchRecorder) { - t.Helper() - calls := len(recorder.calls) - for _, reserved := range []string{semconvURLLabel, schemaURLLabel} { - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, "job", "api"), - labels.MustNewMatcher(labels.MatchEqual, reserved, "registry/test"), - } - for _, result := range []storage.SearchResultSet{ - searcher.SearchLabelNames(t.Context(), nil, matchers...), - searcher.SearchLabelValues(t.Context(), "instance", nil, matchers...), - } { - require.False(t, result.Next()) - require.ErrorIs(t, result.Err(), errSchemaAwareSearchUnsupported) - require.NoError(t, result.Close()) - } - } - require.Len(t, recorder.calls, calls) - } - - t.Run("does not advertise absent capability", func(t *testing.T) { - querierCloses, chunkCloses := 0, 0 - wrapped := AwareStorage(&optionalSearchStorage{ - querier: &nonSearchQuerier{Querier: storage.NoopQuerier(), closes: &querierCloses}, - chunkQuerier: &nonSearchChunkQuerier{ChunkQuerier: storage.NoopChunkedQuerier(), closes: &chunkCloses}, - }) - - querier, err := wrapped.Querier(0, 1) - require.NoError(t, err) - _, ok := querier.(storage.Searcher) - require.False(t, ok) - require.NoError(t, querier.Close()) - - chunkQuerier, err := wrapped.ChunkQuerier(0, 1) - require.NoError(t, err) - _, ok = chunkQuerier.(storage.Searcher) - require.False(t, ok) - require.NoError(t, chunkQuerier.Close()) - require.Equal(t, 1, querierCloses) - require.Equal(t, 1, chunkCloses) - }) - - t.Run("delegates present capability", func(t *testing.T) { - querierCloses, chunkCloses := 0, 0 - querierResult := &recordingSearchResultSet{} - chunkResult := &recordingSearchResultSet{} - querierRecorder := &searchRecorder{result: querierResult} - chunkRecorder := &searchRecorder{result: chunkResult} - wrapped := AwareStorage(&optionalSearchStorage{ - querier: &searchableQuerier{ - nonSearchQuerier: &nonSearchQuerier{Querier: storage.NoopQuerier(), closes: &querierCloses}, - recorder: querierRecorder, - }, - chunkQuerier: &searchableChunkQuerier{ - nonSearchChunkQuerier: &nonSearchChunkQuerier{ChunkQuerier: storage.NoopChunkedQuerier(), closes: &chunkCloses}, - recorder: chunkRecorder, - }, - }) - - querier, err := wrapped.Querier(0, 1) - require.NoError(t, err) - searcher, ok := querier.(storage.Searcher) - require.True(t, ok) - assertDelegates(t, searcher, querierRecorder) - assertRejectsReserved(t, searcher, querierRecorder) - require.NoError(t, querier.Close()) - - chunkQuerier, err := wrapped.ChunkQuerier(0, 1) - require.NoError(t, err) - searcher, ok = chunkQuerier.(storage.Searcher) - require.True(t, ok) - assertDelegates(t, searcher, chunkRecorder) - assertRejectsReserved(t, searcher, chunkRecorder) - require.NoError(t, chunkQuerier.Close()) - - require.Equal(t, 2, querierResult.closes) - require.Equal(t, 2, chunkResult.closes) - require.Equal(t, 1, querierCloses) - require.Equal(t, 1, chunkCloses) - }) -} - -type resolverBudgetQuerier struct { - storage.Querier - calls *resolverBudgetCallCounts -} - -func (q *resolverBudgetQuerier) Select(context.Context, bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet { - q.calls.series++ - return storage.NoopSeriesSet() -} - -func (q *resolverBudgetQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.calls.labelNames++ - return nil, nil, nil -} - -func (q *resolverBudgetQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.calls.labelValues++ - return nil, nil, nil -} - -type resolverBudgetChunkQuerier struct { - storage.ChunkQuerier - calls *resolverBudgetCallCounts -} - -func (q *resolverBudgetChunkQuerier) Select(context.Context, bool, *storage.SelectHints, ...*labels.Matcher) storage.ChunkSeriesSet { - q.calls.chunk++ - return storage.NoopChunkedSeriesSet() -} - -func (q *resolverBudgetChunkQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.calls.labelNames++ - return nil, nil, nil -} - -func (q *resolverBudgetChunkQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.calls.labelValues++ - return nil, nil, nil -} - -func TestSchemaExpansionFailsBeforeStorage(t *testing.T) { - engine := newSchemaEngine(embeddedRegistry) - engine.limits = schemaExpansionLimits{work: 1, keyBytes: 1_000} - calls := &resolverBudgetCallCounts{} - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, semconvURLLabel, "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, schemaURLLabel, "registry/registry.yaml"), - } - - querier := &awareQuerier{ - Querier: &resolverBudgetQuerier{Querier: storage.NoopQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - series := querier.Select(t.Context(), true, nil, matchers...) - require.False(t, series.Next()) - require.ErrorIs(t, series.Err(), errSchemaExpansion) - - chunkQuerier := &awareChunkQuerier{ - ChunkQuerier: &resolverBudgetChunkQuerier{ChunkQuerier: storage.NoopChunkedQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - chunks := chunkQuerier.Select(t.Context(), true, nil, matchers...) - require.False(t, chunks.Next()) - require.ErrorIs(t, chunks.Err(), errSchemaExpansion) - - _, _, err := querier.LabelNames(t.Context(), nil, matchers...) - require.ErrorIs(t, err, errSchemaExpansion) - _, _, err = querier.LabelValues(t.Context(), "tenant", nil, matchers...) - require.ErrorIs(t, err, errSchemaExpansion) - - require.Equal(t, &resolverBudgetCallCounts{}, calls) -} - -func TestUnsafeSchemaMatcherFailsBeforeStorage(t *testing.T) { - registry := map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - attr.old: attr.current -`), - "1.1.0": []byte(`groups: - - id: metric.metric - type: metric - metric_name: metric - instrument: counter - unit: "1" -`), - } - engine := newSchemaEngine(newRegistrySource(registry)) - calls := &resolverBudgetCallCounts{} - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric"), - labels.MustNewMatcher(labels.MatchNotEqual, "attr.current", "excluded"), - labels.MustNewMatcher(labels.MatchEqual, semconvURLLabel, "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, schemaURLLabel, "registry/registry.yaml"), - } - - querier := &awareQuerier{ - Querier: &resolverBudgetQuerier{Querier: storage.NoopQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - series := querier.Select(t.Context(), true, nil, matchers...) - require.False(t, series.Next()) - require.ErrorIs(t, series.Err(), errUnsafeSchemaMatcher) - - chunkQuerier := &awareChunkQuerier{ - ChunkQuerier: &resolverBudgetChunkQuerier{ChunkQuerier: storage.NoopChunkedQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - chunks := chunkQuerier.Select(t.Context(), true, nil, matchers...) - require.False(t, chunks.Next()) - require.ErrorIs(t, chunks.Err(), errUnsafeSchemaMatcher) - - _, _, err := querier.LabelNames(t.Context(), nil, matchers...) - require.ErrorIs(t, err, errUnsafeSchemaMatcher) - _, _, err = querier.LabelValues(t.Context(), "attr.current", nil, matchers...) - require.ErrorIs(t, err, errUnsafeSchemaMatcher) - require.Equal(t, &resolverBudgetCallCounts{}, calls) -} - -func TestAmbiguousSchemaReuseFailsBeforeStorage(t *testing.T) { - metricReuseSchema := `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.2.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - foo: bar - 1.2.0: - metrics: - changes: - - rename_metrics: - baz: foo -` - for _, tc := range []struct { - name string - metric string - attribute string - labelValue string - schema string - }{ - { - name: "historical metric alias", - metric: "bar", - labelValue: model.MetricNameLabel, - schema: metricReuseSchema, - }, - { - name: "reused metric anchor", - metric: "foo", - labelValue: model.MetricNameLabel, - schema: metricReuseSchema, - }, - { - name: "reused attribute alias", - metric: "metric", - attribute: "user", - labelValue: "user", - schema: `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.2.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - user: tenant - 1.2.0: - metrics: - changes: - - rename_attributes: - attribute_map: - account: user -`, - }, - } { - t.Run(tc.name, func(t *testing.T) { - registry := map[string][]byte{ - "registry.yaml": []byte(tc.schema), - "1.2.0": []byte(`groups: - - id: metric.metric - type: metric - metric_name: metric - instrument: counter - unit: "1" - - id: metric.foo - type: metric - metric_name: foo - instrument: counter - unit: "1" - - id: metric.bar - type: metric - metric_name: bar - instrument: counter - unit: "1" -`), - } - engine := newSchemaEngine(newRegistrySource(registry)) - calls := &resolverBudgetCallCounts{} - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, tc.metric), - labels.MustNewMatcher(labels.MatchEqual, semconvURLLabel, "registry/1.2.0"), - labels.MustNewMatcher(labels.MatchEqual, schemaURLLabel, "registry/registry.yaml"), - } - if tc.attribute != "" { - matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, tc.attribute, "value")) - } - - querier := &awareQuerier{ - Querier: &resolverBudgetQuerier{Querier: storage.NoopQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - series := querier.Select(t.Context(), true, nil, matchers...) - require.False(t, series.Next()) - require.ErrorIs(t, series.Err(), errAmbiguousSchemaRename) - - chunkQuerier := &awareChunkQuerier{ - ChunkQuerier: &resolverBudgetChunkQuerier{ChunkQuerier: storage.NoopChunkedQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - chunks := chunkQuerier.Select(t.Context(), true, nil, matchers...) - require.False(t, chunks.Next()) - require.ErrorIs(t, chunks.Err(), errAmbiguousSchemaRename) - - _, _, err := querier.LabelNames(t.Context(), nil, matchers...) - require.ErrorIs(t, err, errAmbiguousSchemaRename) - _, _, err = querier.LabelValues(t.Context(), tc.labelValue, nil, matchers...) - require.ErrorIs(t, err, errAmbiguousSchemaRename) - require.Equal(t, &resolverBudgetCallCounts{}, calls) - }) - } -} - -func TestForwardMetricConvergenceFansOutToStorage(t *testing.T) { - registry := map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - rename_metrics: - metric.old.one: metric.current - metric.old.two: metric.current -`), - "1.0.0": []byte(`groups: - - id: metric.metric.old.one - type: metric - metric_name: metric.old.one - instrument: counter - unit: "1" -`), - } - engine := newSchemaEngine(newRegistrySource(registry)) - calls := &resolverBudgetCallCounts{} - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric.old.one"), - labels.MustNewMatcher(labels.MatchEqual, semconvURLLabel, "registry/1.0.0"), - labels.MustNewMatcher(labels.MatchEqual, schemaURLLabel, "registry/registry.yaml"), - } - - querier := &awareQuerier{ - Querier: &resolverBudgetQuerier{Querier: storage.NoopQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - series := querier.Select(t.Context(), true, nil, matchers...) - require.False(t, series.Next()) - require.NoError(t, series.Err()) - - chunkQuerier := &awareChunkQuerier{ - ChunkQuerier: &resolverBudgetChunkQuerier{ChunkQuerier: storage.NoopChunkedQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - chunks := chunkQuerier.Select(t.Context(), true, nil, matchers...) - require.False(t, chunks.Next()) - require.NoError(t, chunks.Err()) - - _, _, err := querier.LabelNames(t.Context(), nil, matchers...) - require.NoError(t, err) - _, _, err = querier.LabelValues(t.Context(), model.MetricNameLabel, nil, matchers...) - require.NoError(t, err) - require.Positive(t, calls.series) - require.Positive(t, calls.chunk) - require.Positive(t, calls.labelNames) - require.Positive(t, calls.labelValues) -} - -func TestSchemaConvergenceFansOutToStorage(t *testing.T) { - for _, tc := range []struct { - name string - metric string - attribute string - schema string - }{ - { - name: "within one change", - metric: "metric", - attribute: "attr.current", - schema: `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - attr.old.one: attr.current - attr.old.two: attr.current -`, - }, - { - name: "across ordered changes", - metric: "metric", - attribute: "attr.current", - schema: `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - attr.old.one: attr.current - - rename_attributes: - attribute_map: - attr.old.two: attr.current -`, - }, - { - name: "through an intermediate predecessor", - metric: "metric", - attribute: "attr.current", - schema: `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - attr.old: attr.current - - rename_attributes: - attribute_map: - attr.old: attr.middle - - rename_attributes: - attribute_map: - attr.middle: attr.current -`, - }, - { - name: "attribute convergence across revisions", - metric: "metric", - attribute: "attr.current", - schema: `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - metrics: - changes: - - rename_attributes: - attribute_map: - attr.old.one: attr.current - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - attr.old.two: attr.current -`, - }, - { - name: "metric convergence across revisions", - metric: "metric.current", - schema: `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - metrics: - changes: - - rename_metrics: - metric.old.one: metric.current - 1.1.0: - metrics: - changes: - - rename_metrics: - metric.old.two: metric.current -`, - }, - } { - t.Run(tc.name, func(t *testing.T) { - registry := map[string][]byte{ - "registry.yaml": []byte(tc.schema), - "1.1.0": []byte(`groups: - - id: metric.metric - type: metric - metric_name: metric - instrument: counter - unit: "1" - - id: metric.metric.current - type: metric - metric_name: metric.current - instrument: counter - unit: "1" -`), - } - engine := newSchemaEngine(newRegistrySource(registry)) - calls := &resolverBudgetCallCounts{} - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, tc.metric), - labels.MustNewMatcher(labels.MatchEqual, semconvURLLabel, "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, schemaURLLabel, "registry/registry.yaml"), - } - if tc.attribute != "" { - matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, tc.attribute, "value")) - } - - querier := &awareQuerier{ - Querier: &resolverBudgetQuerier{Querier: storage.NoopQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - series := querier.Select(t.Context(), true, nil, matchers...) - require.False(t, series.Next()) - require.NoError(t, series.Err()) - - chunkQuerier := &awareChunkQuerier{ - ChunkQuerier: &resolverBudgetChunkQuerier{ChunkQuerier: storage.NoopChunkedQuerier(), calls: calls}, - engine: engine, - canonicalSeriesLimit: maxCanonicalSeriesMaterialization, - } - chunks := chunkQuerier.Select(t.Context(), true, nil, matchers...) - require.False(t, chunks.Next()) - require.NoError(t, chunks.Err()) - - _, _, err := querier.LabelNames(t.Context(), nil, matchers...) - require.NoError(t, err) - labelName := tc.attribute - if labelName == "" { - labelName = model.MetricNameLabel - } - _, _, err = querier.LabelValues(t.Context(), labelName, nil, matchers...) - require.NoError(t, err) - require.Positive(t, calls.series) - require.Positive(t, calls.chunk) - require.Positive(t, calls.labelNames) - require.Positive(t, calls.labelValues) - }) - } -} diff --git a/storage/semconv/storage_fanout_test.go b/storage/semconv/storage_fanout_test.go deleted file mode 100644 index 15dd05c824..0000000000 --- a/storage/semconv/storage_fanout_test.go +++ /dev/null @@ -1,1102 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "fmt" - "sync" - "testing" - "time" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/storage/semconv" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/prometheus/prometheus/util/annotations" - "github.com/prometheus/prometheus/util/teststorage" -) - -// TestFanOutResultIsSorted checks that a variant whose labels were rewritten is -// re-sorted before it reaches the merge. -// -// Each variant is queried sorted by the names stored for its own era, and rewriting -// an attribute to its anchor-version name reorders it: here "ttt" sorts after "user" -// but before "tenant". storage.NewMergeSeriesSet assumes every input reports -// strictly increasing labels, so feeding it the rewritten set unsorted let it emit -// series out of order and, where two eras rewrote to the same labels, twice — a -// duplicate labelset PromQL rejects. -// -// The samples are asserted, not just the labels. A merged series computes its labels -// eagerly and its samples only when iterated, so a chain built over a slice the -// merge function still holds looks perfectly well-formed until something reads it. -func TestFanOutResultIsSorted(t *testing.T) { - // Stored under the 1.0.0 name, so all of these come back in one variant. - // "user" is rewritten to "tenant" at the 1.1.0 anchor, which moves it before - // "ttt"; the two "1" series collide on the rewritten labels. - appendAll := func(t *testing.T, s storage.Storage) { - t.Helper() - appendSeries(t, s, "test.counter", 1, 1.0, "ttt", "2") - appendSeries(t, s, "test.counter", 1, 2.0, "user", "1") - appendSeries(t, s, "test.counter", 2, 3.0, "tenant", "1") - } - // The two colliding series chain into one, so it carries both their samples. - want := []seriesWithSamples{ - { - lset: labels.FromStrings(model.MetricNameLabel, "test", "tenant", "1"), - samples: []sample{{t: 1, v: 2.0}, {t: 2, v: 3.0}}, - }, - { - lset: labels.FromStrings(model.MetricNameLabel, "test", "ttt", "2"), - samples: []sample{{t: 1, v: 1.0}}, - }, - } - - t.Run("querier", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendAll(t, wrapped) - - set := selectAt(t, wrapped, "1.1.0", "test") - - got := collectWithSamples(t, set) - require.Equal(t, want, got, "expected strictly increasing labels with no duplicates") - }) - - t.Run("chunk querier", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendAll(t, wrapped) - - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - - set := cq.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - - got := collectWithSamples(t, storage.NewSeriesSetFromChunkSeriesSet(set)) - require.Equal(t, want, got, "expected strictly increasing labels with no duplicates") - }) -} - -func TestFanOutCanonicalLabelsRemainValid(t *testing.T) { - for _, tc := range []struct { - name string - labels []string - want labels.Labels - wantErr string - }{ - { - name: "renamed label is reordered", - labels: []string{"service.name", "api", "thread.daemon", "true"}, - want: labels.FromStrings( - model.MetricNameLabel, "jvm.thread.count", - "jvm.thread.daemon", "true", - "service.name", "api", - ), - }, - { - name: "equal aliases collapse", - labels: []string{"jvm.thread.daemon", "true", "thread.daemon", "true"}, - want: labels.FromStrings( - model.MetricNameLabel, "jvm.thread.count", - "jvm.thread.daemon", "true", - ), - }, - { - name: "conflicting aliases fail", - labels: []string{"jvm.thread.daemon", "false", "thread.daemon", "true"}, - wantErr: `maps "jvm.thread.daemon" and "thread.daemon" to "jvm.thread.daemon" with conflicting values`, - }, - } { - for _, query := range []string{"series", "chunks"} { - t.Run(tc.name+"/"+query, func(t *testing.T) { - wrapped := newCanonicalLabelStorage(t, tc.labels...) - if query == "series" { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, canonicalLabelMatchers()...) - if tc.wantErr != "" { - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), tc.wantErr) - return - } - require.True(t, set.Next()) - require.Equal(t, tc.want, set.At().Labels()) - require.Equal(t, tc.want.Get("jvm.thread.daemon"), set.At().Labels().Get("jvm.thread.daemon")) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - return - } - - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, canonicalLabelMatchers()...) - if tc.wantErr != "" { - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), tc.wantErr) - return - } - require.True(t, set.Next()) - require.Equal(t, tc.want, set.At().Labels()) - require.Equal(t, tc.want.Get("jvm.thread.daemon"), set.At().Labels().Get("jvm.thread.daemon")) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - }) - } - } -} - -func newCanonicalLabelStorage(t *testing.T, kv ...string) storage.Storage { - t.Helper() - wrapped, err := semconv.AwareStorageWithRegistry(teststorage.New(t), canonicalLabelRegistry()) - require.NoError(t, err) - appendSeries(t, wrapped, "jvm.thread.count", 1, 1, kv...) - return wrapped -} - -func canonicalLabelRegistry() map[string][]byte { - return map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - all: - changes: - - rename_attributes: - attribute_map: - thread.daemon: jvm.thread.daemon -`), - "1.1.0": []byte(`groups: - - id: metric.jvm.thread.count - type: metric - metric_name: jvm.thread.count - unit: "{thread}" - instrument: updowncounter - attributes: - - ref: jvm.thread.daemon -`), - } -} - -func canonicalLabelMatchers() []*labels.Matcher { - return []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "jvm.thread.count"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } -} - -// TestFanOutWithoutAttributeRenameIsSorted guards the assumption that lets a -// variant stream instead of being buffered and sorted: when the schema renames only -// the metric, every series in a variant matches the same equality matcher on -// __name__, so they all get the same replacement and the order the underlying -// querier returned them in survives. Output must be sorted with no variant sorted -// after the fact. -func TestFanOutWithoutAttributeRenameIsSorted(t *testing.T) { - wrapped, err := semconv.AwareStorageWithRegistry(teststorage.New(t), map[string][]byte{ - "registry.yaml": []byte(benchMetricRenameSchema), - "1.0.0": fmt.Appendf(nil, benchSemconv, "bench.old", "attr.old"), - "1.1.0": fmt.Appendf(nil, benchSemconv, "bench.new", "attr.new"), - }) - require.NoError(t, err) - - // attr.old and attr.new are distinct labels that the schema does not rename, so - // nothing rewrites them and the eras interleave in the merged order. - appendSeries(t, wrapped, "bench.old", 1, 1.0, "attr.old", "zzz") - appendSeries(t, wrapped, "bench.old", 1, 2.0, "attr.old", "aaa") - appendSeries(t, wrapped, "bench.new", 1, 3.0, "attr.new", "mmm") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "bench.new"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - - got := collectWithSamples(t, set) - require.Equal(t, []seriesWithSamples{ - { - lset: labels.FromStrings(model.MetricNameLabel, "bench.new", "attr.new", "mmm"), - samples: []sample{{t: 1, v: 3.0}}, - }, - { - lset: labels.FromStrings(model.MetricNameLabel, "bench.new", "attr.old", "aaa"), - samples: []sample{{t: 1, v: 2.0}}, - }, - { - lset: labels.FromStrings(model.MetricNameLabel, "bench.new", "attr.old", "zzz"), - samples: []sample{{t: 1, v: 1.0}}, - }, - }, got) -} - -type sample struct { - t int64 - v float64 -} - -type seriesWithSamples struct { - lset labels.Labels - samples []sample -} - -// collectWithSamples drains a series set in order, reading each series' samples. -// Unlike collectSeries it keeps the order and every sample, which is what a set -// fed to a merge has to be checked on: the merge computes a series' labels -// eagerly and its samples lazily, so a malformed chain shows only when read. -func collectWithSamples(t *testing.T, set storage.SeriesSet) []seriesWithSamples { - t.Helper() - var out []seriesWithSamples - for set.Next() { - s := set.At() - got := seriesWithSamples{lset: s.Labels()} - it := s.Iterator(nil) - for it.Next() == chunkenc.ValFloat { - ts, v := it.At() - got.samples = append(got.samples, sample{t: ts, v: v}) - require.LessOrEqual(t, len(got.samples), 16, "runaway iteration on %s", got.lset) - } - require.NoError(t, it.Err()) - out = append(out, got) - } - require.NoError(t, set.Err()) - return out -} - -// ctxCheckingStorage returns series sets that fail if the context they were -// selected with has been cancelled by the time they are read, which is what an -// underlying querier streaming from a remote does implicitly. -type ctxCheckingStorage struct { - storage.Storage -} - -func (s ctxCheckingStorage) Querier(mint, maxt int64) (storage.Querier, error) { - q, err := s.Storage.Querier(mint, maxt) - if err != nil { - return nil, err - } - return ctxCheckingQuerier{Querier: q}, nil -} - -func (s ctxCheckingStorage) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { - q, err := s.Storage.ChunkQuerier(mint, maxt) - if err != nil { - return nil, err - } - return ctxCheckingChunkQuerier{ChunkQuerier: q}, nil -} - -type ctxCheckingQuerier struct { - storage.Querier -} - -func (q ctxCheckingQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - return &ctxCheckingSeriesSet{ - SeriesSet: q.Querier.Select(ctx, sortSeries, hints, matchers...), - ctx: ctx, - } -} - -type ctxCheckingSeriesSet struct { - storage.SeriesSet - - ctx context.Context - err error -} - -func (s *ctxCheckingSeriesSet) Next() bool { - if err := s.ctx.Err(); err != nil { - s.err = err - return false - } - return s.SeriesSet.Next() -} - -func (s *ctxCheckingSeriesSet) Err() error { - if s.err != nil { - return s.err - } - return s.SeriesSet.Err() -} - -type ctxCheckingChunkQuerier struct { - storage.ChunkQuerier -} - -func (q ctxCheckingChunkQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.ChunkSeriesSet { - return &ctxCheckingChunkSeriesSet{ - ChunkSeriesSet: q.ChunkQuerier.Select(ctx, sortSeries, hints, matchers...), - ctx: ctx, - } -} - -type ctxCheckingChunkSeriesSet struct { - storage.ChunkSeriesSet - - ctx context.Context - err error -} - -func (s *ctxCheckingChunkSeriesSet) Next() bool { - if err := s.ctx.Err(); err != nil { - s.err = err - return false - } - return s.ChunkSeriesSet.Next() -} - -func (s *ctxCheckingChunkSeriesSet) Err() error { - if s.err != nil { - return s.err - } - return s.ChunkSeriesSet.Err() -} - -// TestFanOutKeepsSelectContextAlive checks that the context handed to the -// underlying Select outlives fan-out scheduling. -// -// A Select is lazy, so its context has to stay valid until the result is read. -// A derived context canceled when workers finish would expire before anything -// reads a variant and abort a streaming querier mid-read. The in-memory storage -// rarely notices, so this asserts it through a querier that checks every Next. -func TestFanOutKeepsSelectContextAlive(t *testing.T) { - wrapped, err := semconv.AwareStorageWithRegistry( - ctxCheckingStorage{Storage: teststorage.New(t)}, - map[string][]byte{ - "registry.yaml": []byte(benchMetricRenameSchema), - "1.0.0": fmt.Appendf(nil, benchSemconv, "bench.old", "attr.old"), - "1.1.0": fmt.Appendf(nil, benchSemconv, "bench.new", "attr.new"), - }, - ) - require.NoError(t, err) - - appendSeries(t, wrapped, "bench.old", 1, 1.0, "attr.old", "a") - appendSeries(t, wrapped, "bench.new", 1, 2.0, "attr.new", "b") - - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "bench.new"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - - t.Run("series", func(t *testing.T) { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, matchers...) - var got int - for set.Next() { - got++ - } - require.NoError(t, set.Err(), "the variants were read with a cancelled context") - require.Equal(t, 2, got, "both eras must be returned") - }) - - t.Run("chunks", func(t *testing.T) { - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, matchers...) - var got int - for set.Next() { - got++ - } - require.NoError(t, set.Err(), "the variants were read with a cancelled context") - require.Equal(t, 2, got, "both eras must be returned") - }) -} - -const fanOutProbeVariants = 17 - -type fanOutSerialProbe struct { - mu sync.Mutex - - totalCalls int - activeCalls int - maxActiveCalls int - activeByQuerier map[int]int - querierOverlap bool - matcherMutation bool - hintMutation bool - undrainedByQuerier map[int]int - selectedBeforeDrain bool - replacement *labels.Matcher - nextQuerierID int - openedQueriers int - closedQueriers map[int]int - cancelOnFirstCall context.CancelFunc -} - -func newFanOutSerialProbe() *fanOutSerialProbe { - return &fanOutSerialProbe{ - activeByQuerier: map[int]int{}, - undrainedByQuerier: map[int]int{}, - replacement: labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "mutated.by.storage"), - closedQueriers: map[int]int{}, - } -} - -func (p *fanOutSerialProbe) openQuerier() int { - p.mu.Lock() - defer p.mu.Unlock() - id := p.nextQuerierID - p.nextQuerierID++ - p.openedQueriers++ - return id -} - -func (p *fanOutSerialProbe) closeQuerier(id int) { - p.mu.Lock() - p.closedQueriers[id]++ - p.mu.Unlock() -} - -func (p *fanOutSerialProbe) mutateSelectHints(hints *storage.SelectHints) { - if hints == nil { - return - } - p.mu.Lock() - if hints.Limit == -1 || len(hints.Grouping) > 0 && hints.Grouping[0] == "mutated.by.storage" { - p.hintMutation = true - } - p.mu.Unlock() - hints.Limit = -1 - if len(hints.Grouping) > 0 { - hints.Grouping[0] = "mutated.by.storage" - } -} - -func (p *fanOutSerialProbe) mutateLabelHints(hints *storage.LabelHints) { - if hints == nil { - return - } - p.mu.Lock() - if hints.Limit == -1 { - p.hintMutation = true - } - p.mu.Unlock() - hints.Limit = -1 -} - -func (p *fanOutSerialProbe) call(querierID int, matchers []*labels.Matcher, returnsSet bool) bool { - p.mu.Lock() - p.activeCalls++ - p.maxActiveCalls = max(p.maxActiveCalls, p.activeCalls) - p.activeByQuerier[querierID]++ - if p.activeByQuerier[querierID] > 1 { - p.querierOverlap = true - } - requiresDrain := returnsSet - for _, matcher := range matchers { - if matcher == p.replacement { - p.matcherMutation = true - } - if matcher.Name == model.MetricNameLabel && matcher.Value == "metric.current" { - requiresDrain = false - } - } - p.totalCalls++ - var cancel context.CancelFunc - if p.totalCalls == 1 { - cancel = p.cancelOnFirstCall - } - if requiresDrain { - if p.undrainedByQuerier[querierID] > 0 { - p.selectedBeforeDrain = true - } - p.undrainedByQuerier[querierID]++ - } - p.mu.Unlock() - if cancel != nil { - cancel() - } - - if len(matchers) > 0 { - // Storage adapters may mutate the variadic matcher slice. - matchers[0] = p.replacement - } - time.Sleep(time.Millisecond) - p.mu.Lock() - p.activeCalls-- - p.activeByQuerier[querierID]-- - p.mu.Unlock() - return requiresDrain -} - -func (p *fanOutSerialProbe) drainedResortSet(querierID int) { - p.mu.Lock() - p.undrainedByQuerier[querierID]-- - p.mu.Unlock() -} - -func (p *fanOutSerialProbe) snapshot() (total, maxActive int, querierOverlap, matcherMutation, selectedBeforeDrain bool) { - p.mu.Lock() - defer p.mu.Unlock() - return p.totalCalls, p.maxActiveCalls, p.querierOverlap, p.matcherMutation, p.selectedBeforeDrain -} - -func (p *fanOutSerialProbe) closeSnapshot() (opened, closed int, duplicateClose bool) { - p.mu.Lock() - defer p.mu.Unlock() - for _, count := range p.closedQueriers { - closed += count - if count > 1 { - duplicateClose = true - } - } - return p.openedQueriers, closed, duplicateClose -} - -func (p *fanOutSerialProbe) hintsWereShared() bool { - p.mu.Lock() - defer p.mu.Unlock() - return p.hintMutation -} - -type fanOutProbeStorage struct { - storage.Storage - probe *fanOutSerialProbe -} - -func (s fanOutProbeStorage) Querier(mint, maxt int64) (storage.Querier, error) { - q, err := s.Storage.Querier(mint, maxt) - if err != nil { - return nil, err - } - return &fanOutProbeQuerier{Querier: q, probe: s.probe, id: s.probe.openQuerier()}, nil -} - -func (s fanOutProbeStorage) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { - q, err := s.Storage.ChunkQuerier(mint, maxt) - if err != nil { - return nil, err - } - return &fanOutProbeChunkQuerier{ChunkQuerier: q, probe: s.probe, id: s.probe.openQuerier()}, nil -} - -type fanOutProbeQuerier struct { - storage.Querier - probe *fanOutSerialProbe - id int -} - -func (q *fanOutProbeQuerier) Select(_ context.Context, _ bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - q.probe.mutateSelectHints(hints) - requiresDrain := q.probe.call(q.id, matchers, true) - return &fanOutProbeSeriesSet{SeriesSet: storage.NoopSeriesSet(), probe: q.probe, querierID: q.id, requiresDrain: requiresDrain} -} - -func (q *fanOutProbeQuerier) LabelNames(_ context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.probe.mutateLabelHints(hints) - q.probe.call(q.id, matchers, false) - return nil, nil, nil -} - -func (q *fanOutProbeQuerier) LabelValues(_ context.Context, _ string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.probe.mutateLabelHints(hints) - q.probe.call(q.id, matchers, false) - return nil, nil, nil -} - -func (q *fanOutProbeQuerier) Close() error { - q.probe.closeQuerier(q.id) - return q.Querier.Close() -} - -type fanOutProbeChunkQuerier struct { - storage.ChunkQuerier - probe *fanOutSerialProbe - id int -} - -func (q *fanOutProbeChunkQuerier) Select(_ context.Context, _ bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.ChunkSeriesSet { - q.probe.mutateSelectHints(hints) - requiresDrain := q.probe.call(q.id, matchers, true) - return &fanOutProbeChunkSeriesSet{ChunkSeriesSet: storage.NoopChunkedSeriesSet(), probe: q.probe, querierID: q.id, requiresDrain: requiresDrain} -} - -func (q *fanOutProbeChunkQuerier) LabelNames(_ context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.probe.mutateLabelHints(hints) - q.probe.call(q.id, matchers, false) - return nil, nil, nil -} - -func (q *fanOutProbeChunkQuerier) LabelValues(_ context.Context, _ string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.probe.mutateLabelHints(hints) - q.probe.call(q.id, matchers, false) - return nil, nil, nil -} - -func (q *fanOutProbeChunkQuerier) Close() error { - q.probe.closeQuerier(q.id) - return q.ChunkQuerier.Close() -} - -type fanOutProbeSeriesSet struct { - storage.SeriesSet - probe *fanOutSerialProbe - querierID int - requiresDrain bool -} - -func (s *fanOutProbeSeriesSet) Next() bool { - if s.requiresDrain { - s.probe.drainedResortSet(s.querierID) - s.requiresDrain = false - } - return false -} - -type fanOutProbeChunkSeriesSet struct { - storage.ChunkSeriesSet - probe *fanOutSerialProbe - querierID int - requiresDrain bool -} - -func (s *fanOutProbeChunkSeriesSet) Next() bool { - if s.requiresDrain { - s.probe.drainedResortSet(s.querierID) - s.requiresDrain = false - } - return false -} - -func fanOutProbeRegistry() map[string][]byte { - return fanOutRegistry(fanOutProbeVariants) -} - -func fanOutRegistry(variants int) map[string][]byte { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - all: - changes: - - rename_attributes: - attribute_map: - http.status_old: http.response.status_code - metrics: - changes: - - rename_metrics: -`) - previousSemconv := []byte("groups:\n") - for i := range variants - 1 { - schema = fmt.Appendf(schema, " metric.old.%02d: metric.current\n", i) - previousSemconv = fmt.Appendf(previousSemconv, ` - id: metric.metric.old.%02d - type: metric - metric_name: metric.old.%02d - unit: s - instrument: histogram - stability: stable -`, i, i) - } - return map[string][]byte{ - "registry.yaml": schema, - "1.0.0": previousSemconv, - "1.1.0": metricSemconv("metric.current", "s", "histogram"), - } -} - -func newFanOutProbeStorage(t *testing.T, probe *fanOutSerialProbe) storage.Storage { - return newFanOutProbeStorageWithRegistry(t, probe, fanOutProbeRegistry()) -} - -func newFanOutProbeStorageWithRegistry(t *testing.T, probe *fanOutSerialProbe, registry map[string][]byte) storage.Storage { - t.Helper() - wrapped, err := semconv.AwareStorageWithRegistry( - fanOutProbeStorage{Storage: teststorage.New(t), probe: probe}, - registry, - ) - require.NoError(t, err) - return wrapped -} - -func fanOutProbeMatchers() []*labels.Matcher { - return fanOutProbeMatchersFor("metric.current") -} - -func requireSerialFanOut(t *testing.T, probe *fanOutSerialProbe, wantCalls int, checkScheduling bool) { - t.Helper() - total, maxActive, querierOverlap, matcherMutation, selectedBeforeDrain := probe.snapshot() - require.Equal(t, wantCalls, total) - require.Equal(t, 1, maxActive, "storage calls must be serial") - require.False(t, querierOverlap, "calls on one underlying querier must not overlap") - require.False(t, matcherMutation, "storage mutation must not escape one matcher slice") - require.False(t, probe.hintsWereShared(), "storage mutation must not escape one hints value") - opened, _, _ := probe.closeSnapshot() - require.Equal(t, 1, opened, "fan-out must retain the outer query's storage snapshot") - if checkScheduling { - require.True(t, selectedBeforeDrain, "variant selects must be scheduled before iteration starts") - } -} - -func TestFanOutUsesOneQuerierAndIsolatesInputs(t *testing.T) { - t.Run("series", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - hints := &storage.SelectHints{Limit: 1, Grouping: []string{"job"}} - set := q.Select(context.Background(), false, hints, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - require.Equal(t, &storage.SelectHints{Limit: 1, Grouping: []string{"job"}}, hints) - requireSerialFanOut(t, probe, fanOutProbeVariants, true) - }) - - t.Run("chunks", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - hints := &storage.SelectHints{Limit: 1, Grouping: []string{"job"}} - set := q.Select(context.Background(), false, hints, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - require.Equal(t, &storage.SelectHints{Limit: 1, Grouping: []string{"job"}}, hints) - requireSerialFanOut(t, probe, fanOutProbeVariants, true) - }) - - t.Run("label names", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - hints := &storage.LabelHints{Limit: 1} - _, _, err = q.LabelNames(context.Background(), hints, fanOutProbeMatchers()...) - require.NoError(t, err) - require.Equal(t, &storage.LabelHints{Limit: 1}, hints) - requireSerialFanOut(t, probe, fanOutProbeVariants, false) - }) - - t.Run("label values", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - hints := &storage.LabelHints{Limit: 1} - _, _, err = q.LabelValues(context.Background(), model.MetricNameLabel, hints, fanOutProbeMatchers()...) - require.NoError(t, err) - require.Equal(t, &storage.LabelHints{Limit: 1}, hints) - requireSerialFanOut(t, probe, fanOutProbeVariants, false) - }) -} - -func TestStorageFanOutLimitFailsClosed(t *testing.T) { - assertNoCalls := func(t *testing.T, probe *fanOutSerialProbe) { - t.Helper() - total, maxActive, querierOverlap, matcherMutation, _ := probe.snapshot() - require.Zero(t, total) - require.Zero(t, maxActive) - require.False(t, querierOverlap) - require.False(t, matcherMutation) - } - - t.Run("allows 32 jobs", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorageWithRegistry(t, probe, fanOutRegistry(32)) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - require.Empty(t, warningStrings(set.Warnings())) - requireSerialFanOut(t, probe, 32, true) - }) - - t.Run("series", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorageWithRegistry(t, probe, fanOutRegistry(33)) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), "schema expansion limit exceeded") - require.Empty(t, warningStrings(set.Warnings())) - assertNoCalls(t, probe) - }) - - t.Run("chunks", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorageWithRegistry(t, probe, fanOutRegistry(33)) - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), "schema expansion limit exceeded") - require.Empty(t, warningStrings(set.Warnings())) - assertNoCalls(t, probe) - }) - - t.Run("label names", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorageWithRegistry(t, probe, fanOutRegistry(33)) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - _, anns, err := q.LabelNames(context.Background(), nil, fanOutProbeMatchers()...) - require.ErrorContains(t, err, "schema expansion limit exceeded") - require.Empty(t, warningStrings(anns)) - assertNoCalls(t, probe) - }) - - t.Run("label values", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - _, anns, err := q.LabelValues(context.Background(), "http.response.status_code", nil, fanOutProbeMatchers()...) - require.ErrorContains(t, err, "schema expansion limit exceeded") - require.Empty(t, warningStrings(anns)) - assertNoCalls(t, probe) - }) -} - -func TestFanOutOwnsOneQuerier(t *testing.T) { - for _, query := range []string{"series", "chunks"} { - t.Run(query, func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - if query == "series" { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - set := q.Select(context.Background(), false, nil, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - require.NoError(t, q.Close()) - } else { - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - set := q.Select(context.Background(), false, nil, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - require.NoError(t, q.Close()) - } - - opened, closed, duplicateClose := probe.closeSnapshot() - require.Equal(t, 1, opened) - require.Equal(t, 1, closed) - require.False(t, duplicateClose) - }) - } -} - -func TestCanceledFanOutDoesNotStartStorageCalls(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - t.Run("series", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(ctx, false, nil, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.ErrorIs(t, set.Err(), context.Canceled) - total, _, _, _, _ := probe.snapshot() - require.Equal(t, 0, total) - opened, _, _ := probe.closeSnapshot() - require.Equal(t, 1, opened) - }) - - t.Run("chunks", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(ctx, false, nil, fanOutProbeMatchers()...) - require.False(t, set.Next()) - require.ErrorIs(t, set.Err(), context.Canceled) - total, _, _, _, _ := probe.snapshot() - require.Equal(t, 0, total) - opened, _, _ := probe.closeSnapshot() - require.Equal(t, 1, opened) - }) - - t.Run("labels", func(t *testing.T) { - probe := newFanOutSerialProbe() - wrapped := newFanOutProbeStorage(t, probe) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - _, _, err = q.LabelNames(ctx, nil, fanOutProbeMatchers()...) - require.ErrorIs(t, err, context.Canceled) - total, _, _, _, _ := probe.snapshot() - require.Equal(t, 0, total) - opened, _, _ := probe.closeSnapshot() - require.Equal(t, 1, opened) - }) -} - -func TestCanceledFanOutStopsBeforeNextStorageCall(t *testing.T) { - for _, operation := range []string{"series", "chunks", "label names", "label values"} { - t.Run(operation, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - probe := newFanOutSerialProbe() - probe.cancelOnFirstCall = cancel - wrapped := newFanOutProbeStorage(t, probe) - - var queryErr error - switch operation { - case "series": - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(ctx, false, nil, fanOutProbeMatchers()...) - for set.Next() { - } - queryErr = set.Err() - case "chunks": - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(ctx, false, nil, fanOutProbeMatchers()...) - for set.Next() { - } - queryErr = set.Err() - case "label names": - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - _, _, queryErr = q.LabelNames(ctx, nil, fanOutProbeMatchers()...) - case "label values": - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - _, _, queryErr = q.LabelValues(ctx, model.MetricNameLabel, nil, fanOutProbeMatchers()...) - } - - require.ErrorIs(t, queryErr, context.Canceled) - total, maxActive, querierOverlap, _, _ := probe.snapshot() - require.Equal(t, 1, total) - require.Equal(t, 1, maxActive) - require.False(t, querierOverlap) - opened, _, _ := probe.closeSnapshot() - require.Equal(t, 1, opened) - }) - } -} - -// A TSDB querier captures append isolation when it is created. Schema fan-out -// must not open later queriers that can observe commits made after that point. -func TestFanOutPreservesTSDBSnapshot(t *testing.T) { - want := []seriesWithSamples{ - { - lset: labels.FromStrings(model.MetricNameLabel, "test", "identity", "current"), - samples: []sample{{t: 1, v: 1}}, - }, - { - lset: labels.FromStrings(model.MetricNameLabel, "test", "identity", "historical"), - samples: []sample{{t: 1, v: 2}}, - }, - } - - for _, query := range []string{"series", "chunks"} { - t.Run(query, func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test", 1, 1, "identity", "current") - appendSeries(t, wrapped, "test.counter", 1, 2, "identity", "historical") - - var got []seriesWithSamples - if query == "series" { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - appendSeries(t, wrapped, "test", 2, 11, "identity", "current") - appendSeries(t, wrapped, "test.counter", 2, 12, "identity", "historical") - got = collectWithSamples(t, q.Select(context.Background(), false, nil, fanOutProbeMatchersFor("test")...)) - } else { - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - appendSeries(t, wrapped, "test", 2, 11, "identity", "current") - appendSeries(t, wrapped, "test.counter", 2, 12, "identity", "historical") - set := q.Select(context.Background(), false, nil, fanOutProbeMatchersFor("test")...) - got = collectWithSamples(t, storage.NewSeriesSetFromChunkSeriesSet(set)) - } - - require.Equal(t, want, got) - }) - } -} - -// TestFanOutTSDBRangeSelect exercises the shared head chunk reader used when -// range-query hints enable its mutable chunk cache. The underlying querier does -// not permit concurrent Select calls. -func TestFanOutTSDBRangeSelect(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test.counter", 1, 1, "user", "a") - appendSeries(t, wrapped, "test", 1, 2, "tenant", "b") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, &storage.SelectHints{Start: 0, End: 10, Step: 1}, - fanOutProbeMatchersFor("test")..., - ) - require.Len(t, collectSeries(t, set), 2) -} - -func fanOutProbeMatchersFor(metricName string) []*labels.Matcher { - return []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, metricName), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } -} diff --git a/storage/semconv/storage_inheritance_test.go b/storage/semconv/storage_inheritance_test.go deleted file mode 100644 index 090d8dd39b..0000000000 --- a/storage/semconv/storage_inheritance_test.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage/semconv" - "github.com/prometheus/prometheus/util/teststorage" -) - -func inheritedAttributeRegistry() map[string][]byte { - return map[string][]byte{ - "registry.yaml": []byte(` -file_format: 1.1.0 -schema_url: https://example.com/schemas/1.0.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - legacy.user: tenant.id - legacy.partition: tenant.partition - apply_to_metrics: - - queue.depth -`), - "1.0.0": []byte(` -groups: - - id: attributes.identity - type: attribute_group - prefix: legacy - attributes: - - id: user - - id: attributes.queue.base - type: attribute_group - extends: attributes.identity - - id: attributes.queue - type: attribute_group - extends: attributes.queue.base - prefix: legacy - attributes: - - id: partition - - id: metric.queue.depth - type: metric - metric_name: queue.depth - instrument: updowncounter - unit: "{item}" - extends: attributes.queue -`), - "1.1.0": []byte(` -groups: - - id: metric.queue.depth - type: metric - metric_name: queue.depth - instrument: updowncounter - unit: "{item}" - extends: attributes.queue - - id: attributes.queue - type: attribute_group - extends: attributes.queue.base - prefix: tenant - attributes: - - id: partition - - id: attributes.identity - type: attribute_group - prefix: tenant - attributes: - - id: id - - id: attributes.queue.base - type: attribute_group - extends: attributes.identity -`), - } -} - -func TestInheritedMetricAttributesAreCanonicalized(t *testing.T) { - wrapped, err := semconv.AwareStorageWithRegistry(teststorage.New(t), inheritedAttributeRegistry()) - require.NoError(t, err) - appendSeries(t, wrapped, "queue.depth", 1, 7, "legacy.user", "alice", "legacy.partition", "primary") - - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "queue.depth"), - labels.MustNewMatcher(labels.MatchEqual, "tenant.id", "alice"), - labels.MustNewMatcher(labels.MatchEqual, "tenant.partition", "primary"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, matchers...) - series := collectSeries(t, set) - require.Len(t, series, 1) - for key := range series { - require.Contains(t, key, `"tenant.id"="alice"`) - require.Contains(t, key, `"tenant.partition"="primary"`) - require.NotContains(t, key, "legacy.user") - require.NotContains(t, key, "legacy.partition") - } - require.Empty(t, warningStrings(set.Warnings())) - - names, warnings, err := q.LabelNames(context.Background(), nil, matchers...) - require.NoError(t, err) - require.Contains(t, names, "tenant.id") - require.Contains(t, names, "tenant.partition") - require.NotContains(t, names, "legacy.user") - require.NotContains(t, names, "legacy.partition") - require.Empty(t, warningStrings(warnings)) - - values, warnings, err := q.LabelValues(context.Background(), "tenant.id", nil, matchers...) - require.NoError(t, err) - require.Equal(t, []string{"alice"}, values) - require.Empty(t, warningStrings(warnings)) - - values, warnings, err = q.LabelValues(context.Background(), "tenant.partition", nil, matchers...) - require.NoError(t, err) - require.Equal(t, []string{"primary"}, values) - require.Empty(t, warningStrings(warnings)) - - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - - chunkSet := cq.Select(context.Background(), false, nil, matchers...) - require.True(t, chunkSet.Next()) - require.Equal(t, "alice", chunkSet.At().Labels().Get("tenant.id")) - require.Equal(t, "primary", chunkSet.At().Labels().Get("tenant.partition")) - require.Empty(t, chunkSet.At().Labels().Get("legacy.user")) - require.Empty(t, chunkSet.At().Labels().Get("legacy.partition")) - require.False(t, chunkSet.Next()) - require.NoError(t, chunkSet.Err()) - require.Empty(t, warningStrings(chunkSet.Warnings())) - - names, warnings, err = cq.LabelNames(context.Background(), nil, matchers...) - require.NoError(t, err) - require.Contains(t, names, "tenant.id") - require.Contains(t, names, "tenant.partition") - require.NotContains(t, names, "legacy.user") - require.NotContains(t, names, "legacy.partition") - require.Empty(t, warningStrings(warnings)) - - values, warnings, err = cq.LabelValues(context.Background(), "tenant.id", nil, matchers...) - require.NoError(t, err) - require.Equal(t, []string{"alice"}, values) - require.Empty(t, warningStrings(warnings)) - - values, warnings, err = cq.LabelValues(context.Background(), "tenant.partition", nil, matchers...) - require.NoError(t, err) - require.Equal(t, []string{"primary"}, values) - require.Empty(t, warningStrings(warnings)) -} diff --git a/storage/semconv/storage_labels_test.go b/storage/semconv/storage_labels_test.go deleted file mode 100644 index b27081ff42..0000000000 --- a/storage/semconv/storage_labels_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" -) - -// TestLabelValuesCanonicalisesTheQueriedName asks for the values of a renamed -// attribute under its pre-rename name. The name is canonicalised first, so the -// fan-out covers the same aliases as asking under the anchor-version name; -// otherwise only the era that happens to store the attribute under the name as -// given contributes any values. -func TestLabelValuesCanonicalisesTheQueriedName(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test.counter", 1, 1.0, "user", "a") - appendSeries(t, wrapped, "test", 1, 2.0, "tenant", "b") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - // "user" is the 1.0.0 name of the attribute the 1.1.0 anchor calls "tenant". - values, _, err := q.LabelValues(context.Background(), "user", nil, matchers...) - require.NoError(t, err) - require.Equal(t, []string{"a", "b"}, values) -} - -// TestLabelNamesHidesReservedLabels checks that the reserved matcher labels are -// not reported as label names. Select strips them from the series it returns, so -// reporting them would advertise labels no series carries. -func TestLabelNamesHidesReservedLabels(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test", 1, 1.0, "tenant", "a", "__schema_url__", "registry/registry.yaml") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - names, _, err := q.LabelNames(context.Background(), nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - require.NoError(t, err) - require.Equal(t, []string{model.MetricNameLabel, "tenant"}, names) -} - -func TestLabelValuesHidesReservedLabels(t *testing.T) { - wrapped, counts := newCountingAwareStorage(t, canonicalLabelRegistry()) - appendSeries(t, wrapped, "jvm.thread.count", 1, 1.0, - "__schema_url__", "stored-schema", - "__semconv_url__", "stored-semconv", - ) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - - for _, tc := range []struct { - name string - query func(string) ([]string, error) - }{ - { - name: "querier", - query: func(name string) ([]string, error) { - values, _, err := q.LabelValues(context.Background(), name, nil, canonicalLabelMatchers()...) - return values, err - }, - }, - { - name: "chunk querier", - query: func(name string) ([]string, error) { - values, _, err := cq.LabelValues(context.Background(), name, nil, canonicalLabelMatchers()...) - return values, err - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - for _, name := range []string{"__schema_url__", "__semconv_url__"} { - counts.reset() - values, err := tc.query(name) - require.NoError(t, err) - require.NotNil(t, values) - require.Empty(t, values) - require.Zero(t, counts.total(), "reserved values must not schedule storage jobs") - } - }) - } - - counts.reset() - values, _, err := q.LabelValues(context.Background(), "__schema_url__", nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "jvm.thread.count"), - ) - require.NoError(t, err) - require.Equal(t, []string{"stored-schema"}, values, "ordinary queries must retain passthrough behavior") - require.Equal(t, int64(1), counts.total()) -} diff --git a/storage/semconv/storage_materialization_test.go b/storage/semconv/storage_materialization_test.go deleted file mode 100644 index 8fb870ec5e..0000000000 --- a/storage/semconv/storage_materialization_test.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv - -import ( - "context" - "fmt" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/util/teststorage" -) - -var materializationTestRegistry = map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - budget.old: budget.current - - rename_attributes: - attribute_map: - attr.old: attr.current - apply_to_metrics: - - budget.current - 1.2.0: - metrics: - changes: - - rename_metrics: - budget.current: budget.future - - rename_attributes: - attribute_map: - attr.current: attr.future - apply_to_metrics: - - budget.future -`), - "1.0.0": []byte(`groups: - - id: registry.attr.old - type: attribute_group - brief: Old attribute. - attributes: - - id: attr.old - type: string - brief: Old attribute. - examples: [value] - - id: metric.budget.old - type: metric - brief: Old metric. - metric_name: budget.old - instrument: counter - unit: "1" - attributes: - - ref: attr.old -`), - "1.1.0": []byte(`groups: - - id: registry.attr.current - type: attribute_group - brief: Current attribute. - attributes: - - id: attr.current - type: string - brief: Current attribute. - examples: [value] - - id: metric.budget.current - type: metric - brief: Current metric. - metric_name: budget.current - instrument: counter - unit: "1" - attributes: - - ref: attr.current -`), -} - -func newMaterializationTestStorage(t *testing.T, limit, oldSeries, futureSeries int) *awareStorage { - t.Helper() - underlying := teststorage.New(t) - wrapped := newAwareStorage(underlying, newSchemaEngine(newRegistrySource(materializationTestRegistry))) - wrapped.canonicalSeriesLimit = limit - app := wrapped.Appender(context.Background()) - appendSeries := func(metric, attribute string, index int) { - labelValues := []string{ - model.MetricNameLabel, metric, - attribute, "value", - "instance", fmt.Sprintf("%03d", index), - } - _, err := app.Append(0, labels.FromStrings(labelValues...), 1, float64(index)) - require.NoError(t, err) - } - for i := range oldSeries { - appendSeries("budget.old", "attr.old", i) - } - for i := range futureSeries { - appendSeries("budget.future", "attr.future", oldSeries+i) - } - require.NoError(t, app.Commit()) - variants, _, err := wrapped.engine.findMatcherVariants("registry/1.1.0", "registry/registry.yaml", materializationTestMatchers()) - require.NoError(t, err) - require.Len(t, variants, 3) - var names []string - for _, variant := range variants { - name, err := extractMetricName(variant.matchers) - require.NoError(t, err) - names = append(names, name) - } - require.ElementsMatch(t, []string{"budget.old", "budget.current", "budget.future"}, names) - return wrapped -} - -func materializationTestMatchers() []*labels.Matcher { - return []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "budget.current"), - labels.MustNewMatcher(labels.MatchEqual, semconvURLLabel, "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, schemaURLLabel, "registry/registry.yaml"), - } -} - -func TestCanonicalSeriesMaterializationBudget(t *testing.T) { - for _, query := range []string{"series", "chunks"} { - t.Run(query+" allows exact limit", func(t *testing.T) { - wrapped := newMaterializationTestStorage(t, 2, 1, 1) - if query == "series" { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(context.Background(), false, nil, materializationTestMatchers()...) - for set.Next() { - } - require.NoError(t, set.Err()) - return - } - - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(context.Background(), false, nil, materializationTestMatchers()...) - for set.Next() { - } - require.NoError(t, set.Err()) - }) - - t.Run(query+" rejects the next cross-variant input", func(t *testing.T) { - wrapped := newMaterializationTestStorage(t, 2, 2, 1) - hints := &storage.SelectHints{Start: 0, End: 10, Limit: 1} - if query == "series" { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(context.Background(), false, hints, materializationTestMatchers()...) - require.False(t, set.Next()) - require.ErrorIs(t, set.Err(), errCanonicalSeriesMaterialization) - require.ErrorContains(t, set.Err(), "more than 2 input series") - return - } - - q, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(context.Background(), false, hints, materializationTestMatchers()...) - require.False(t, set.Next()) - require.ErrorIs(t, set.Err(), errCanonicalSeriesMaterialization) - require.ErrorContains(t, set.Err(), "more than 2 input series") - }) - } -} diff --git a/storage/semconv/storage_registry_test.go b/storage/semconv/storage_registry_test.go deleted file mode 100644 index e393e70439..0000000000 --- a/storage/semconv/storage_registry_test.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "os" - "path/filepath" - "strconv" - "strings" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage/semconv" - "github.com/prometheus/prometheus/util/teststorage" -) - -// readRegistryDir loads an on-disk registry directory into a base-name → bytes -// map, mirroring how an operator supplies a registry via configuration. -func readRegistryDir(t *testing.T, dir string) map[string][]byte { - t.Helper() - entries, err := os.ReadDir(dir) - require.NoError(t, err) - files := map[string][]byte{} - for _, e := range entries { - if e.IsDir() { - continue - } - b, err := os.ReadFile(filepath.Join(dir, e.Name())) - require.NoError(t, err) - files[e.Name()] = b - } - require.NotEmpty(t, files) - return files -} - -func registryWithSemconv(contents []byte) map[string][]byte { - return map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.0.0 -versions: - 1.0.0: -`), - "1.0.0": contents, - } -} - -func writeAttributeRefs(b *strings.Builder, prefix string, count int) { - for i := range count { - b.WriteString(" - ref: ") - b.WriteString(prefix) - b.WriteString(strconv.Itoa(i)) - b.WriteByte('\n') - } -} - -func inheritedAttributeLimitRegistry(parentAttrs, metricAttrs int) map[string][]byte { - var b strings.Builder - b.WriteString(`groups: - - id: attributes.base - type: attribute_group - attributes: -`) - writeAttributeRefs(&b, "base.", parentAttrs) - b.WriteString(` - id: metric.queue.depth - type: metric - metric_name: queue.depth - extends: attributes.base - attributes: -`) - writeAttributeRefs(&b, "metric.", metricAttrs) - return registryWithSemconv([]byte(b.String())) -} - -func inheritedAttributeFanoutRegistry(children int) map[string][]byte { - var b strings.Builder - b.WriteString(`groups: - - id: attributes.base - type: attribute_group - attributes: -`) - writeAttributeRefs(&b, "base.", 256) - for i := range children { - b.WriteString(" - id: attributes.child.") - b.WriteString(strconv.Itoa(i)) - b.WriteString("\n type: attribute_group\n extends: attributes.base\n") - } - return registryWithSemconv([]byte(b.String())) -} - -func TestAwareStorageWithRegistry(t *testing.T) { - t.Run("rejects an invalid registry", func(t *testing.T) { - _, err := semconv.AwareStorageWithRegistry(teststorage.New(t), map[string][]byte{ - "registry.yaml": []byte("file_format: 9.9.9\n"), - }) - require.Error(t, err) - }) - - t.Run("rejects unsupported metric splits", func(t *testing.T) { - _, err := semconv.AwareStorageWithRegistry(teststorage.New(t), map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - split: - apply_to_metric: http.server.duration - by_attribute: http.request.method - metrics_from_attributes: - http.server.get.duration: GET -`), - }) - require.EqualError(t, err, `registry schema "registry.yaml": schema version "1.1.0" contains unsupported metric split transformation`) - }) - - t.Run("rejects unknown schema transformations", func(t *testing.T) { - _, err := semconv.AwareStorageWithRegistry(teststorage.New(t), map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.1.0: - metrics: - changes: - - rename_metric: {} -`), - }) - require.EqualError(t, err, `registry schema "registry.yaml": schema version "1.1.0" metrics change 1 contains unsupported transformation "rename_metric"`) - }) - - t.Run("rejects an empty registry", func(t *testing.T) { - _, err := semconv.AwareStorageWithRegistry(teststorage.New(t), nil) - require.Error(t, err) - }) - - t.Run("rejects invalid unused semconv metadata", func(t *testing.T) { - _, err := semconv.AwareStorageWithRegistry(teststorage.New(t), registryWithSemconv([]byte(` -groups: - - id: attributes.unused - type: attribute_group - extends: attributes.missing -`))) - require.ErrorContains(t, err, `semconv group "attributes.unused" extends unknown group "attributes.missing"`) - }) - - for _, tc := range []struct { - name string - metricAttrs int - wantErrContains string - }{ - {name: "accepts 256 resolved attributes", metricAttrs: 6}, - {name: "rejects 257 resolved attributes", metricAttrs: 7, wantErrContains: "semconv group attributes would exceed 256"}, - } { - t.Run(tc.name, func(t *testing.T) { - _, err := semconv.AwareStorageWithRegistry(teststorage.New(t), inheritedAttributeLimitRegistry(250, tc.metricAttrs)) - if tc.wantErrContains != "" { - require.ErrorContains(t, err, tc.wantErrContains) - return - } - require.NoError(t, err) - }) - } - - t.Run("rejects excessive cumulative inherited attributes", func(t *testing.T) { - files := inheritedAttributeFanoutRegistry(256) - require.Less(t, len(files["1.0.0"]), 16<<20) - _, err := semconv.AwareStorageWithRegistry(teststorage.New(t), files) - require.ErrorContains(t, err, "semconv file attribute slots would exceed 65536") - }) - - // An operator registry that mirrors the embedded one must drive the same - // schema-version rename fan-out, proving the injected source behaves - // identically to the embedded default. - t.Run("an operator registry mirroring the embedded one resolves identically", func(t *testing.T) { - files := readRegistryDir(t, "registry") - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, files) - require.NoError(t, err) - // The wrapper must retain its validated copy, not aliases owned by the caller. - for name, b := range files { - clear(b) - files[name] = []byte("mutated after construction") - } - - // Written under the semconv 1.0.0 name; semconv 1.1.0 renamed it to "test". - appendSeries(t, wrapped, "test.counter", 1, 7.0, "http.response.status_code", "200") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.NotEmpty(t, got, "expected the historical name to surface via the operator registry") - var found bool - for k := range got { - if strings.Contains(k, `__name__="test"`) { - found = true - } - } - require.True(t, found, "expected the renamed metric under its 1.1.0 name in: %v", got) - }) -} diff --git a/storage/semconv/storage_rename_validation_test.go b/storage/semconv/storage_rename_validation_test.go deleted file mode 100644 index 37c922aa3d..0000000000 --- a/storage/semconv/storage_rename_validation_test.go +++ /dev/null @@ -1,947 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "fmt" - "strings" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage/semconv" - "github.com/prometheus/prometheus/util/teststorage" -) - -// renameSchema is a two-version schema renaming old→new at 1.1.0, the shape -// every case below varies the semconv files around. -const renameSchema = `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - %s: %s -` - -// metricSemconv renders a semconv file declaring a single metric group. The -// group id follows the "metric." form that upstream semantic -// conventions lint for, so it necessarily changes whenever the metric name does. -func metricSemconv(metricName, unit, instrument string) []byte { - return metricSemconvWithStability(metricName, unit, instrument, "stable") -} - -func metricSemconvWithStability(metricName, unit, instrument, stability string) []byte { - stabilityLine := "" - if stability != "" { - stabilityLine = fmt.Sprintf(" stability: %s\n", stability) - } - return fmt.Appendf(nil, `groups: - - id: metric.%s - type: metric - metric_name: %s - unit: %q - instrument: %s -%s attributes: - - ref: http.response.status_code -`, metricName, metricName, unit, instrument, stabilityLine) -} - -// selectRenamed queries metricName anchored at semconv 1.1.0 over the given -// registry and returns the series found plus any warnings raised. -func selectRenamed(t *testing.T, files map[string][]byte, appendUnder, metricName string) (map[string]float64, []string) { - t.Helper() - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, files) - require.NoError(t, err) - - appendSeries(t, wrapped, appendUnder, 1, 7.0, "http.response.status_code", "200") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, metricName), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - return got, warningStrings(set.Warnings()) -} - -func selectRenamedError(t *testing.T, files map[string][]byte, appendUnder, metricName string) error { - t.Helper() - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, files) - require.NoError(t, err) - - appendSeries(t, wrapped, appendUnder, 1, 7.0, "http.response.status_code", "200") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, metricName), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - require.False(t, set.Next()) - return set.Err() -} - -func TestRenameEdgeValidation(t *testing.T) { - // A legitimate rename: the metric is the same measurement under a new name, - // so unit and instrument agree and the historical series must still merge. - // - // This is also the case that rules out using the semconv group id as a - // cross-version identity: the ids here are metric.old.name and - // metric.new.name, so requiring them to be equal across the edge would - // reject this rename even though it is exactly the rename the schema exists - // to describe. - t.Run("merges a rename whose unit and instrument agree", func(t *testing.T) { - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.0.0": metricSemconv("old.name", "s", "histogram"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }, "old.name", "new.name") - - require.Len(t, got, 1, "expected the pre-rename series under the queried name, got %v", got) - for k := range got { - require.Contains(t, k, `__name__="new.name"`) - } - require.Empty(t, warnings, "a corroborated rename must not warn") - }) - - // The case the schema format cannot express and name traversal cannot - // detect: two unrelated metrics that happen to share a surface name at - // different versions. Their units disagree, so merging them would average - // seconds with a queue depth. - t.Run("does not merge a rename whose unit disagrees", func(t *testing.T) { - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.0.0": metricSemconv("old.name", "{item}", "updowncounter"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }, "old.name", "new.name") - - require.Empty(t, got, "series of a differently-united metric must not be merged in, got %v", got) - requireWarningsContain(t, warnings, "treating them as different metrics") - requireWarningsContain(t, warnings, `resolves it to "old.name"`) - }) - - // Same unit, different instrument: a histogram and a counter are not the - // same metric even when they measure in the same unit. - t.Run("does not merge a rename whose instrument disagrees", func(t *testing.T) { - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.0.0": metricSemconv("old.name", "s", "counter"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }, "old.name", "new.name") - - require.Empty(t, got, "got %v", got) - requireWarningsContain(t, warnings, "treating them as different metrics") - }) - - for _, tc := range []struct { - name string - oldStability string - newStability string - }{ - {name: "development definitions", oldStability: "development", newStability: "development"}, - {name: "one stable definition", oldStability: "development", newStability: "stable"}, - {name: "unspecified definitions"}, - } { - t.Run("follows a metadata-changing rename for "+tc.name, func(t *testing.T) { - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.0.0": metricSemconvWithStability("old.name", "{item}", "updowncounter", tc.oldStability), - "1.1.0": metricSemconvWithStability("new.name", "s", "histogram", tc.newStability), - }, "old.name", "new.name") - - require.Len(t, got, 1, "an explicit non-stable rename must still surface its historical series, got %v", got) - requireWarningsContain(t, warnings, "following the explicit schema rename") - }) - } - - // A registry need not ship a semconv for every version its schema - // references, so an absent file means "cannot verify" and must leave the - // existing name-traversal behaviour untouched while warning the caller. - t.Run("warns but traverses an unverifiable rename when a semconv is absent", func(t *testing.T) { - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }, "old.name", "new.name") - - require.Len(t, got, 1, "expected the pre-rename series to still merge, got %v", got) - require.Len(t, warnings, 1) - requireWarningsContain(t, warnings, "version 1.0.0 is unavailable") - requireWarningsContain(t, warnings, "without corroboration") - }) - - // A name the semconv does not declare as a metric is a strong hint of a - // mis-authored edge, but a trimmed registry is a legitimate shape, so this - // is reported without dropping series. - t.Run("warns but still merges when a name is not declared as a metric", func(t *testing.T) { - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "typo.name", "new.name"), - "1.0.0": metricSemconv("old.name", "s", "histogram"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }, "typo.name", "new.name") - - require.Len(t, got, 1, "expected the series to still merge, got %v", got) - requireWarningsContain(t, warnings, "could not be corroborated") - }) -} - -// TestRecycledMetricName covers the case the schema format cannot express: a -// metric name that was renamed away and later reused by an unrelated metric. -// -// 1.0.0 foo exists, counting bytes -// 2.0.0 schema renames foo to bar -// 5.0.0 a new, unrelated metric claims the name foo, measuring seconds -// -// Querying foo at 5.0.0 means the new metric. The 2.0.0 rename edge concerns the -// old foo and must not drag bar's series in. Walking backwards encounters the -// queried name only on the old side of that edge, which marks the lifecycle -// boundary even when the old and new metrics have identical signatures. -func TestRecycledMetricName(t *testing.T) { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/5.0.0 -versions: - 1.0.0: - 2.0.0: - metrics: - changes: - - rename_metrics: - foo: bar - 5.0.0: -`) - - for _, tc := range []struct { - name string - unit string - instrument string - }{ - {name: "different signature", unit: "s", instrument: "histogram"}, - {name: "same signature", unit: "By", instrument: "counter"}, - } { - t.Run(tc.name, func(t *testing.T) { - // 5.0.0 declares both the reused name and the metric the old foo became. - currentSemconv := fmt.Appendf(nil, `groups: - - id: metric.foo - type: metric - metric_name: foo - unit: %q - instrument: %s - attributes: - - ref: http.response.status_code - - id: metric.bar - type: metric - metric_name: bar - unit: "By" - instrument: counter - attributes: - - ref: http.response.status_code -`, tc.unit, tc.instrument) - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": schema, - "1.0.0": metricSemconv("foo", "By", "counter"), - "2.0.0": metricSemconv("bar", "By", "counter"), - "5.0.0": currentSemconv, - }) - require.NoError(t, err) - - // Series of the metric the old foo became. Querying today's foo must - // not pick these up, even when identity fields happen to match. - appendSeries(t, wrapped, "bar", 1, 7.0, "http.response.status_code", "200") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/5.0.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.Empty(t, got, "an unrelated metric that once held this name must not be merged in, got %v", got) - requireWarningsContain(t, warningStrings(set.Warnings()), "metric lifecycle boundary") - }) - } - - t.Run("trimmed anchor omits the reused name", func(t *testing.T) { - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": schema, - "1.0.0": metricSemconv("foo", "By", "counter"), - "2.0.0": metricSemconv("bar", "By", "counter"), - "5.0.0": metricSemconv("unrelated", "s", "histogram"), - }) - require.NoError(t, err) - - appendSeries(t, wrapped, "bar", 1, 7.0, "http.response.status_code", "200") - appendSeries(t, wrapped, "foo", 1, 9.0, "http.response.status_code", "500") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/5.0.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.Equal(t, map[string]float64{ - labels.FromStrings(model.MetricNameLabel, "foo", "http.response.status_code", "500").String(): 9, - }, got) - requireWarningsContain(t, warningStrings(set.Warnings()), "metric lifecycle boundary") - }) -} - -// TestAttributeRenameScope checks that an attribute rename restricted by -// apply_to_metrics does not rewrite the attributes of other metrics. The schema -// scopes user→tenant to scoped.metric only, so a series of other.metric that -// carries user must keep it. -func TestAttributeRenameScope(t *testing.T) { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - user: tenant - apply_to_metrics: - - scoped.metric -`) - semconv110 := []byte(`groups: - - id: metric.scoped.metric - type: metric - metric_name: scoped.metric - unit: "s" - instrument: histogram - attributes: - - ref: tenant - - id: metric.other.metric - type: metric - metric_name: other.metric - unit: "s" - instrument: histogram - attributes: - - ref: tenant -`) - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": schema, - "1.1.0": semconv110, - }) - require.NoError(t, err) - - appendSeries(t, wrapped, "scoped.metric", 1, 1.0, "user", "alice") - appendSeries(t, wrapped, "other.metric", 1, 2.0, "user", "bob") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - selectMetric := func(name string) map[string]float64 { - return collectSeries(t, q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, name), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - )) - } - - // The metric the rename is scoped to still has its historical attribute - // normalised to the anchor version's name. - got := selectMetric("scoped.metric") - require.Len(t, got, 1, "got %v", got) - for k := range got { - require.Contains(t, k, `tenant="alice"`, "the scoped metric must be normalised") - } - - // Any other metric must be left alone. Before apply_to_metrics was honoured, - // this attribute was rewritten to tenant as well. - got = selectMetric("other.metric") - require.Len(t, got, 1, "got %v", got) - for k := range got { - require.Contains(t, k, `user="bob"`, "a rename scoped to another metric must not apply here") - require.NotContains(t, k, "tenant=") - } -} - -func TestRejectedLineageDoesNotLeakAttributeScope(t *testing.T) { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/5.0.0 -versions: - 1.0.0: - 2.0.0: - metrics: - changes: - - rename_metrics: - foo: bar - - rename_attributes: - attribute_map: - user: tenant - apply_to_metrics: - - bar - 5.0.0: -`) - semconv500 := []byte(`groups: - - id: metric.foo - type: metric - metric_name: foo - unit: "s" - instrument: histogram - attributes: - - ref: tenant -`) - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": schema, - "5.0.0": semconv500, - }) - require.NoError(t, err) - appendSeries(t, wrapped, "foo", 1, 1.0, "user", "alice") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "foo"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/5.0.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.Len(t, got, 1) - for key := range got { - require.Contains(t, key, `user="alice"`) - require.NotContains(t, key, `tenant=`) - } - requireWarningsContain(t, warningStrings(set.Warnings()), "metric lifecycle boundary") -} - -func TestManyToOneAttributeScopeStaysBranchLocal(t *testing.T) { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - user: tenant - apply_to_metrics: - - metric.a - - rename_metrics: - metric.a: metric.current - metric.b: metric.current -`) - semconv100 := []byte(`groups: - - id: metric.metric.a - type: metric - metric_name: metric.a - unit: "s" - instrument: histogram - attributes: - - ref: user - - id: metric.metric.b - type: metric - metric_name: metric.b - unit: "s" - instrument: histogram - attributes: - - ref: user -`) - semconv110 := []byte(`groups: - - id: metric.metric.current - type: metric - metric_name: metric.current - unit: "s" - instrument: histogram - attributes: - - ref: tenant -`) - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": schema, - "1.0.0": semconv100, - "1.1.0": semconv110, - }) - require.NoError(t, err) - appendSeries(t, wrapped, "metric.a", 1, 1.0, "user", "alice") - appendSeries(t, wrapped, "metric.b", 1, 2.0, "user", "bob") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - baseMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric.current"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - - set := q.Select(context.Background(), false, nil, baseMatchers...) - got := collectSeries(t, set) - require.Len(t, got, 2, "got %v", got) - var sawScoped, sawUnscoped bool - for key := range got { - sawScoped = sawScoped || strings.Contains(key, `tenant="alice"`) - sawUnscoped = sawUnscoped || strings.Contains(key, `user="bob"`) - } - require.True(t, sawScoped, "metric.a's scoped alias was not canonicalised: %v", got) - require.True(t, sawUnscoped, "metric.a's scope leaked into metric.b: %v", got) - require.Empty(t, warningStrings(set.Warnings())) - - matched := q.Select(context.Background(), false, nil, append(baseMatchers, - labels.MustNewMatcher(labels.MatchEqual, "tenant", "alice"), - )...) - require.Len(t, collectSeries(t, matched), 1) - - names, anns, err := q.LabelNames(context.Background(), nil, baseMatchers...) - require.NoError(t, err) - require.Empty(t, warningStrings(anns)) - require.Contains(t, names, "tenant") - require.Contains(t, names, "user") - - values, anns, err := q.LabelValues(context.Background(), "tenant", nil, baseMatchers...) - require.NoError(t, err) - require.Empty(t, warningStrings(anns)) - require.Equal(t, []string{"alice"}, values) - - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - chunks := cq.Select(context.Background(), false, nil, baseMatchers...) - var chunkLabels []string - for chunks.Next() { - chunkLabels = append(chunkLabels, chunks.At().Labels().String()) - } - require.NoError(t, chunks.Err()) - require.Len(t, chunkLabels, 2) - require.Condition(t, func() bool { - return strings.Contains(strings.Join(chunkLabels, "\n"), `tenant="alice"`) && - strings.Contains(strings.Join(chunkLabels, "\n"), `user="bob"`) - }, "chunk variants used the wrong mappings: %v", chunkLabels) -} - -func TestOrderedConvergingMetricRenames(t *testing.T) { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - metric.a: metric.current - - rename_metrics: - metric.b: metric.current -`) - semconv100 := []byte(`groups: - - id: metric.metric.a - type: metric - metric_name: metric.a - unit: s - instrument: histogram - - id: metric.metric.b - type: metric - metric_name: metric.b - unit: s - instrument: histogram -`) - semconv110 := metricSemconv("metric.current", "s", "histogram") - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": schema, - "1.0.0": semconv100, - "1.1.0": semconv110, - }) - require.NoError(t, err) - appendSeries(t, wrapped, "metric.a", 1, 1.0, "source", "a") - appendSeries(t, wrapped, "metric.b", 1, 2.0, "source", "b") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric.current"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.Len(t, got, 2, "both ordered predecessors must resolve to the current metric: %v", got) - require.Empty(t, warningStrings(set.Warnings())) -} - -func TestOrderedConvergingAttributeRenames(t *testing.T) { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: - attr.a: attr.current - - rename_attributes: - attribute_map: - attr.b: attr.current -`) - semconv110 := []byte(`groups: - - id: metric.metric.current - type: metric - metric_name: metric.current - unit: s - instrument: histogram - attributes: - - ref: attr.current -`) - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": schema, - "1.1.0": semconv110, - }) - require.NoError(t, err) - appendSeries(t, wrapped, "metric.current", 1, 1.0, "attr.a", "match", "source", "a") - appendSeries(t, wrapped, "metric.current", 1, 2.0, "attr.b", "match", "source", "b") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric.current"), - labels.MustNewMatcher(labels.MatchEqual, "attr.current", "match"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - set := q.Select(context.Background(), false, nil, matchers...) - got := collectSeries(t, set) - require.Len(t, got, 2, "both ordered attribute predecessors must be queried: %v", got) - for key := range got { - require.Contains(t, key, `"attr.current"="match"`) - require.NotContains(t, key, `"attr.a"=`) - require.NotContains(t, key, `"attr.b"=`) - } - require.Empty(t, warningStrings(set.Warnings())) -} - -func TestAmbiguousHistoricalBranchIsRejectedIndependently(t *testing.T) { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - old.bad: metric.current - old.good: metric.current -`) - semconv100 := []byte(`groups: - - id: metric.old.bad.one - type: metric - metric_name: old.bad - unit: "s" - instrument: histogram - - id: metric.old.bad.two - type: metric - metric_name: old.bad - unit: "By" - instrument: counter - - id: metric.old.good - type: metric - metric_name: old.good - unit: "s" - instrument: histogram -`) - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": schema, - "1.0.0": semconv100, - "1.1.0": metricSemconv("metric.current", "s", "histogram"), - }) - require.NoError(t, err) - appendSeries(t, wrapped, "old.bad", 1, 1.0, "source", "bad") - appendSeries(t, wrapped, "old.good", 1, 2.0, "source", "good") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric.current"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.Len(t, got, 1, "the ambiguous sibling branch must be excluded: %v", got) - for key := range got { - require.Contains(t, key, `source="good"`) - } - requireWarningsContain(t, warningStrings(set.Warnings()), "ambiguous rename branch") -} - -func TestAmbiguousMetricNameWarns(t *testing.T) { - // Two groups declaring the same metric_name: the collision that previously - // resolved to whichever group was parsed last, with no warning. - ambiguous := []byte(`groups: - - id: metric.shared.name - type: metric - metric_name: shared.name - unit: "s" - instrument: histogram - attributes: - - ref: http.response.status_code - - id: metric.shared.name.other - type: metric - metric_name: shared.name - unit: "{item}" - instrument: updowncounter - attributes: - - ref: queue.name -`) - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "shared.name"), - "1.1.0": ambiguous, - }) - require.NoError(t, err) - - appendSeries(t, wrapped, "shared.name", 1, 7.0) - appendSeries(t, wrapped, "old.name", 1, 9.0) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "shared.name"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - - t.Run("Select", func(t *testing.T) { - set := q.Select(context.Background(), false, nil, matchers...) - got := collectSeries(t, set) - require.Len(t, got, 1, "an ambiguous anchor must not traverse to old.name: %v", got) - for _, value := range got { - require.Equal(t, 7.0, value) - } - requireWarningsContain(t, warningStrings(set.Warnings()), "declared by more than one group") - }) - - t.Run("LabelNames", func(t *testing.T) { - _, anns, err := q.LabelNames(context.Background(), nil, matchers...) - require.NoError(t, err) - requireWarningsContain(t, warningStrings(anns), "declared by more than one group") - }) - - t.Run("LabelValues", func(t *testing.T) { - _, anns, err := q.LabelValues(context.Background(), "http.response.status_code", nil, matchers...) - require.NoError(t, err) - requireWarningsContain(t, warningStrings(anns), "declared by more than one group") - }) - - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - - t.Run("ChunkSelect", func(t *testing.T) { - set := cq.Select(context.Background(), false, nil, matchers...) - var n int - for set.Next() { - n++ - } - require.NoError(t, set.Err()) - require.Equal(t, 1, n, "an ambiguous anchor must use only the direct chunk variant") - requireWarningsContain(t, warningStrings(set.Warnings()), "declared by more than one group") - }) - - t.Run("ChunkLabelNames", func(t *testing.T) { - _, anns, err := cq.LabelNames(context.Background(), nil, matchers...) - require.NoError(t, err) - requireWarningsContain(t, warningStrings(anns), "declared by more than one group") - }) - - t.Run("ChunkLabelValues", func(t *testing.T) { - _, anns, err := cq.LabelValues(context.Background(), "http.response.status_code", nil, matchers...) - require.NoError(t, err) - requireWarningsContain(t, warningStrings(anns), "declared by more than one group") - }) -} - -// TestChunkQuerierSurfacesWarnings checks the ChunkQuerier path annotates too, -// since it fans out through the same resolver as Querier. -func TestChunkQuerierSurfacesWarnings(t *testing.T) { - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.0.0": metricSemconv("old.name", "{item}", "updowncounter"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }) - require.NoError(t, err) - appendSeries(t, wrapped, "new.name", 1, 7.0) - - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - - set := cq.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "new.name"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - var n int - for set.Next() { - n++ - } - require.NoError(t, set.Err()) - require.Equal(t, 1, n, "the queried metric's own series must still be returned") - - var found bool - for _, w := range warningStrings(set.Warnings()) { - if strings.Contains(w, "treating them as different metrics") { - found = true - } - } - require.True(t, found, "expected the chunk querier to surface the mis-linked rename warning") -} - -// recycledSchema retires old.name at 1.1.0 and hands the name to an unrelated -// metric at 1.2.0, so the name has two eras and they need not mean the same thing. -const recycledSchema = `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - old.name: new.name - 1.2.0: - metrics: - changes: - - rename_metrics: - other.name: old.name -` - -// TestRenameValidationWithoutAnchorDeclaration covers querying a name the anchor -// semconv does not declare, which is ordinary input here: the fan-out deliberately -// supports asking for a name the anchor version has already renamed away. -// -// Corroboration used to switch itself off silently for exactly that query, which is -// the case it exists for. The identity of the queried metric is instead taken from a -// version where the name is the current one, and only where no version settles it is -// the check skipped — and then said so. -func TestRenameValidationWithoutAnchorDeclaration(t *testing.T) { - t.Run("corroborates against the era that declares the queried name", func(t *testing.T) { - // Semconv 1.1.0 knows only new.name, so the queried old.name is resolved - // against 1.0.0, where it is current. The two disagree on unit and - // instrument, so the rename joins unrelated metrics and must not merge. - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.0.0": metricSemconv("old.name", "{item}", "updowncounter"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }, "new.name", "old.name") - - require.Empty(t, got, "the contradicted rename must not pull in the other metric's series, got %v", got) - requireWarningsContain(t, warnings, "treating them as different metrics") - }) - - t.Run("merges when the era that declares the queried name agrees", func(t *testing.T) { - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.0.0": metricSemconv("old.name", "s", "histogram"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }, "new.name", "old.name") - - require.Len(t, got, 1, "expected the renamed series under the queried name, got %v", got) - for k := range got { - require.Contains(t, k, `__name__="old.name"`) - } - require.Empty(t, warnings, "a corroborated rename must not warn") - }) - - t.Run("warns when no version declares the queried name", func(t *testing.T) { - // Nothing declares old.name anywhere, so there is no identity to check - // hops against. The series still merge, but the caller is told the - // corroboration did not run rather than left to assume it passed. - got, warnings := selectRenamed(t, map[string][]byte{ - "registry.yaml": fmt.Appendf(nil, renameSchema, "old.name", "new.name"), - "1.0.0": metricSemconv("unrelated.name", "s", "histogram"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - }, "new.name", "old.name") - - require.Len(t, got, 1, "an unverifiable rename is still followed, got %v", got) - requireWarningsContain(t, warnings, "without corroboration") - }) - - t.Run("rejects when a disconnected identity reuses the queried name", func(t *testing.T) { - // old.name is current at 1.0.0 and again at 1.2.0, as two different - // metrics. Neither era can stand in for the other, so picking one would - // check hops against an identity the caller never asked for. - err := selectRenamedError(t, map[string][]byte{ - "registry.yaml": []byte(recycledSchema), - "1.0.0": metricSemconv("old.name", "{item}", "updowncounter"), - "1.1.0": metricSemconv("new.name", "s", "histogram"), - "1.2.0": metricSemconv("old.name", "s", "histogram"), - }, "new.name", "old.name") - - require.ErrorContains(t, err, "semconv schema rename is ambiguous") - }) - - t.Run("rejects reuse before checking an ambiguous era declaration", func(t *testing.T) { - ambiguous := []byte(`groups: - - id: metric.old.name.one - type: metric - metric_name: old.name - unit: s - instrument: histogram - - id: metric.old.name.two - type: metric - metric_name: old.name - unit: By - instrument: counter -`) - err := selectRenamedError(t, map[string][]byte{ - "registry.yaml": []byte(recycledSchema), - "1.0.0": ambiguous, - "1.1.0": metricSemconv("new.name", "s", "histogram"), - "1.2.0": metricSemconv("old.name", "s", "histogram"), - }, "new.name", "old.name") - - require.ErrorContains(t, err, "semconv schema rename is ambiguous") - }) -} diff --git a/storage/semconv/storage_select_contract_test.go b/storage/semconv/storage_select_contract_test.go deleted file mode 100644 index 88e7bade34..0000000000 --- a/storage/semconv/storage_select_contract_test.go +++ /dev/null @@ -1,1023 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "errors" - "fmt" - "slices" - "strconv" - "sync" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/storage/semconv" - "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/util/annotations" - "github.com/prometheus/prometheus/util/teststorage" -) - -func schemaReadMatchers() []*labels.Matcher { - return []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } -} - -func TestSchemaAwareSelectSchedulesBeforeIteration(t *testing.T) { - primary := teststorage.New(t) - secondary := teststorage.New(t) - appendSeries(t, secondary, "test.counter", 1, 1, "user", "acme") - appendSeries(t, secondary, "up", 1, 2, "instance", "secondary") - wrapper := semconv.AwareStorage(storage.NewFanout(nil, primary, secondary)) - - t.Run("series", func(t *testing.T) { - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - - schemaSet := q.Select(t.Context(), true, nil, schemaReadMatchers()...) - secondSchemaSet := q.Select(t.Context(), true, nil, schemaReadMatchers()...) - plainSet := q.Select(t.Context(), true, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "up")) - - schemaSeries := collectSeries(t, schemaSet) - require.Len(t, schemaSeries, 1) - for got := range schemaSeries { - require.Contains(t, got, `__name__="test"`) - require.Contains(t, got, `tenant="acme"`) - } - require.Len(t, collectSeries(t, secondSchemaSet), 1) - require.Len(t, collectSeries(t, plainSet), 1) - }) - - t.Run("chunk series", func(t *testing.T) { - q, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - - schemaSet := q.Select(t.Context(), true, nil, schemaReadMatchers()...) - secondSchemaSet := q.Select(t.Context(), true, nil, schemaReadMatchers()...) - plainSet := q.Select(t.Context(), true, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "up")) - - schemaSeries := collectSeries(t, storage.NewSeriesSetFromChunkSeriesSet(schemaSet)) - require.Len(t, schemaSeries, 1) - for got := range schemaSeries { - require.Contains(t, got, `__name__="test"`) - require.Contains(t, got, `tenant="acme"`) - } - require.Len(t, collectSeries(t, storage.NewSeriesSetFromChunkSeriesSet(secondSchemaSet)), 1) - require.Len(t, collectSeries(t, storage.NewSeriesSetFromChunkSeriesSet(plainSet)), 1) - }) -} - -type contractHintStorage struct { - storage.Storage - - mu sync.Mutex - seriesHints []*storage.SelectHints - chunkHints []*storage.SelectHints - seriesSort []bool - chunkSort []bool - labelNamesHints []*storage.LabelHints - labelValuesHints []*storage.LabelHints - seriesNextCalls int - chunkNextCalls int - labelNamesCalls int - labelValuesCalls int - labelNamesErr error -} - -type contractStorageCalls struct { - seriesSelects int - chunkSelects int - seriesNext int - chunkNext int - labelNames int - labelValues int -} - -func copyRecordedHints(hints *storage.SelectHints) *storage.SelectHints { - if hints == nil { - return nil - } - cloned := *hints - cloned.Grouping = slices.Clone(hints.Grouping) - cloned.ProjectionLabels = slices.Clone(hints.ProjectionLabels) - return &cloned -} - -func copyRecordedLabelHints(hints *storage.LabelHints) *storage.LabelHints { - if hints == nil { - return nil - } - cloned := *hints - return &cloned -} - -func (s *contractHintStorage) recordSeries(sortSeries bool, hints *storage.SelectHints) { - s.mu.Lock() - defer s.mu.Unlock() - s.seriesSort = append(s.seriesSort, sortSeries) - s.seriesHints = append(s.seriesHints, copyRecordedHints(hints)) -} - -func (s *contractHintStorage) recordChunk(sortSeries bool, hints *storage.SelectHints) { - s.mu.Lock() - defer s.mu.Unlock() - s.chunkSort = append(s.chunkSort, sortSeries) - s.chunkHints = append(s.chunkHints, copyRecordedHints(hints)) -} - -func (s *contractHintStorage) recordedHints() (series, chunks []*storage.SelectHints) { - s.mu.Lock() - defer s.mu.Unlock() - return slices.Clone(s.seriesHints), slices.Clone(s.chunkHints) -} - -func (s *contractHintStorage) recordedSort() (series, chunks []bool) { - s.mu.Lock() - defer s.mu.Unlock() - return slices.Clone(s.seriesSort), slices.Clone(s.chunkSort) -} - -func (s *contractHintStorage) recordedLabelHints() (names, values []*storage.LabelHints) { - s.mu.Lock() - defer s.mu.Unlock() - return slices.Clone(s.labelNamesHints), slices.Clone(s.labelValuesHints) -} - -func enforceRecordedLabelLimit(values []string, hints *storage.LabelHints) []string { - if hints != nil && hints.Limit > 0 && len(values) > hints.Limit { - return values[:hints.Limit] - } - return values -} - -func (s *contractHintStorage) calls() contractStorageCalls { - s.mu.Lock() - defer s.mu.Unlock() - return contractStorageCalls{ - seriesSelects: len(s.seriesHints), - chunkSelects: len(s.chunkHints), - seriesNext: s.seriesNextCalls, - chunkNext: s.chunkNextCalls, - labelNames: s.labelNamesCalls, - labelValues: s.labelValuesCalls, - } -} - -func (s *contractHintStorage) Querier(mint, maxt int64) (storage.Querier, error) { - q, err := s.Storage.Querier(mint, maxt) - if err != nil { - return nil, err - } - return &contractHintQuerier{Querier: q, storage: s}, nil -} - -func (s *contractHintStorage) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { - q, err := s.Storage.ChunkQuerier(mint, maxt) - if err != nil { - return nil, err - } - return &contractHintChunkQuerier{ChunkQuerier: q, storage: s}, nil -} - -type contractHintQuerier struct { - storage.Querier - storage *contractHintStorage -} - -func (q *contractHintQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - q.storage.recordSeries(sortSeries, hints) - return &contractHintSeriesSet{ - SeriesSet: q.Querier.Select(ctx, sortSeries, hints, matchers...), - hints: copyRecordedHints(hints), - storage: q.storage, - } -} - -func (q *contractHintQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.storage.mu.Lock() - q.storage.labelNamesCalls++ - q.storage.labelNamesHints = append(q.storage.labelNamesHints, copyRecordedLabelHints(hints)) - err := q.storage.labelNamesErr - q.storage.mu.Unlock() - if err != nil { - return nil, nil, err - } - names, anns, err := q.Querier.LabelNames(ctx, hints, matchers...) - return enforceRecordedLabelLimit(names, hints), anns, err -} - -func (q *contractHintQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.storage.mu.Lock() - q.storage.labelValuesCalls++ - q.storage.labelValuesHints = append(q.storage.labelValuesHints, copyRecordedLabelHints(hints)) - q.storage.mu.Unlock() - values, anns, err := q.Querier.LabelValues(ctx, name, hints, matchers...) - return enforceRecordedLabelLimit(values, hints), anns, err -} - -type contractHintChunkQuerier struct { - storage.ChunkQuerier - storage *contractHintStorage -} - -func (q *contractHintChunkQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.ChunkSeriesSet { - q.storage.recordChunk(sortSeries, hints) - return &contractHintChunkSeriesSet{ - ChunkSeriesSet: q.ChunkQuerier.Select(ctx, sortSeries, hints, matchers...), - hints: copyRecordedHints(hints), - storage: q.storage, - } -} - -func (q *contractHintChunkQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.storage.mu.Lock() - q.storage.labelNamesCalls++ - q.storage.labelNamesHints = append(q.storage.labelNamesHints, copyRecordedLabelHints(hints)) - err := q.storage.labelNamesErr - q.storage.mu.Unlock() - if err != nil { - return nil, nil, err - } - names, anns, err := q.ChunkQuerier.LabelNames(ctx, hints, matchers...) - return enforceRecordedLabelLimit(names, hints), anns, err -} - -func (q *contractHintChunkQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.storage.mu.Lock() - q.storage.labelValuesCalls++ - q.storage.labelValuesHints = append(q.storage.labelValuesHints, copyRecordedLabelHints(hints)) - q.storage.mu.Unlock() - values, anns, err := q.ChunkQuerier.LabelValues(ctx, name, hints, matchers...) - return enforceRecordedLabelLimit(values, hints), anns, err -} - -type contractHintSeriesSet struct { - storage.SeriesSet - hints *storage.SelectHints - storage *contractHintStorage - seen int - at storage.Series -} - -func (s *contractHintSeriesSet) Next() bool { - s.storage.mu.Lock() - s.storage.seriesNextCalls++ - s.storage.mu.Unlock() - if s.hints != nil && s.hints.Limit > 0 && s.seen >= s.hints.Limit { - return false - } - if !s.SeriesSet.Next() { - return false - } - s.seen++ - series := s.SeriesSet.At() - s.at = &contractHintSeries{Series: series, labels: projectLabels(series.Labels(), s.hints)} - return true -} - -func (s *contractHintSeriesSet) At() storage.Series { - return s.at -} - -type contractHintSeries struct { - storage.Series - labels labels.Labels -} - -func (s *contractHintSeries) Labels() labels.Labels { - return s.labels -} - -type contractHintChunkSeriesSet struct { - storage.ChunkSeriesSet - hints *storage.SelectHints - storage *contractHintStorage - seen int - at storage.ChunkSeries -} - -func (s *contractHintChunkSeriesSet) Next() bool { - s.storage.mu.Lock() - s.storage.chunkNextCalls++ - s.storage.mu.Unlock() - if s.hints != nil && s.hints.Limit > 0 && s.seen >= s.hints.Limit { - return false - } - if !s.ChunkSeriesSet.Next() { - return false - } - s.seen++ - series := s.ChunkSeriesSet.At() - s.at = &contractHintChunkSeries{ChunkSeries: series, labels: projectLabels(series.Labels(), s.hints)} - return true -} - -func (s *contractHintChunkSeriesSet) At() storage.ChunkSeries { - return s.at -} - -type contractHintChunkSeries struct { - storage.ChunkSeries - labels labels.Labels -} - -func (s *contractHintChunkSeries) Labels() labels.Labels { - return s.labels -} - -func projectLabels(full labels.Labels, hints *storage.SelectHints) labels.Labels { - if hints == nil || len(hints.ProjectionLabels) == 0 { - return full - } - projected := make(map[string]struct{}, len(hints.ProjectionLabels)) - for _, name := range hints.ProjectionLabels { - projected[name] = struct{}{} - } - - builder := labels.NewScratchBuilder(full.Len() + 1) - full.Range(func(label labels.Label) { - _, listed := projected[label.Name] - if listed == hints.ProjectionInclude { - builder.Add(label.Name, label.Value) - } - }) - builder.Add("__series_hash__", strconv.FormatUint(labels.StableHash(full), 10)) - builder.Sort() - return builder.Labels() -} - -func shardFixtureValues(t *testing.T) (excluded, included string) { - t.Helper() - for i := range 1000 { - value := fmt.Sprintf("value-%03d", i) - physical := labels.FromStrings(model.MetricNameLabel, "test.counter", "user", value) - canonical := labels.FromStrings(model.MetricNameLabel, "test", "tenant", value) - if labels.StableHash(physical)%2 != 0 { - continue - } - switch labels.StableHash(canonical) % 2 { - case 1: - if excluded == "" { - excluded = value - } - case 0: - if excluded != "" { - return excluded, value - } - } - } - t.Fatal("could not construct stable shard fixture") - return "", "" -} - -func TestSchemaAwareSelectAppliesHintsToCanonicalLabels(t *testing.T) { - t.Run("clears unsafe physical hints", func(t *testing.T) { - excluded, included := shardFixtureValues(t) - underlying := teststorage.New(t) - contractStorage := &contractHintStorage{Storage: underlying} - wrapper := semconv.AwareStorage(contractStorage) - appendSeries(t, underlying, "test.counter", 1, 1, "user", excluded) - appendSeries(t, underlying, "test.counter", 1, 2, "user", included) - - hints := &storage.SelectHints{ - Start: 0, - End: 10, - Limit: 1, - Func: "sum", - Grouping: []string{"tenant"}, - By: true, - ShardCount: 2, - ShardIndex: 0, - ProjectionLabels: []string{"tenant"}, - ProjectionInclude: true, - } - wantHints := copyRecordedHints(hints) - - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - gotSeries := collectSeries(t, q.Select(t.Context(), true, hints, schemaReadMatchers()...)) - require.Len(t, gotSeries, 1) - for got := range gotSeries { - require.Contains(t, got, `__name__="test"`) - require.Contains(t, got, `tenant="`+included+`"`) - require.NotContains(t, got, excluded) - require.NotContains(t, got, "__series_hash__") - } - - cq, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, cq.Close()) }) - gotChunks := collectSeries(t, storage.NewSeriesSetFromChunkSeriesSet( - cq.Select(t.Context(), true, hints, schemaReadMatchers()...), - )) - require.Len(t, gotChunks, 1) - for got := range gotChunks { - require.Contains(t, got, `tenant="`+included+`"`) - require.NotContains(t, got, "__series_hash__") - } - - require.Equal(t, wantHints, hints, "schema fan-out must not mutate caller-owned hints") - seriesHints, chunkHints := contractStorage.recordedHints() - for _, recorded := range append(seriesHints, chunkHints...) { - require.NotNil(t, recorded) - require.Zero(t, recorded.Limit) - require.Zero(t, recorded.ShardCount) - require.Zero(t, recorded.ShardIndex) - require.Nil(t, recorded.ProjectionLabels) - require.False(t, recorded.ProjectionInclude) - require.Empty(t, recorded.Func) - require.Nil(t, recorded.Grouping) - require.False(t, recorded.By) - } - }) - - t.Run("preserves metadata-only series token", func(t *testing.T) { - underlying := teststorage.New(t) - contractStorage := &contractHintStorage{Storage: underlying} - wrapper := semconv.AwareStorage(contractStorage) - appendSeries(t, underlying, "test.counter", 1, 1, "user", "acme") - - hints := &storage.SelectHints{ - Start: 0, - End: 10, - Func: "series", - Grouping: []string{"tenant"}, - By: true, - ProjectionLabels: []string{"tenant"}, - ProjectionInclude: true, - } - wantHints := copyRecordedHints(hints) - - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - _ = q.Select(t.Context(), true, hints, schemaReadMatchers()...) - - cq, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, cq.Close()) }) - _ = cq.Select(t.Context(), true, hints, schemaReadMatchers()...) - - require.Equal(t, wantHints, hints, "schema fan-out must not mutate caller-owned hints") - seriesHints, chunkHints := contractStorage.recordedHints() - require.Greater(t, len(seriesHints), 1, "series metadata lookup must fan out") - require.Greater(t, len(chunkHints), 1, "chunk metadata lookup must fan out") - for _, recorded := range append(seriesHints, chunkHints...) { - require.NotNil(t, recorded) - require.Equal(t, "series", recorded.Func) - require.Nil(t, recorded.Grouping) - require.False(t, recorded.By) - require.Nil(t, recorded.ProjectionLabels) - require.False(t, recorded.ProjectionInclude) - } - }) -} - -func contractFanOutRegistry(variants int) map[string][]byte { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/current -versions: - 1.0.0: -`) - for i := 1; i < variants; i++ { - oldName := fmt.Sprintf("metric.old.%02d", i-1) - newName := fmt.Sprintf("metric.old.%02d", i) - if i == variants-1 { - newName = "metric.current" - } - schema = fmt.Appendf(schema, ` 1.%d.0: - metrics: - changes: - - rename_metrics: - %s: %s -`, i, oldName, newName) - } - version := fmt.Sprintf("1.%d.0", variants-1) - return map[string][]byte{ - "registry.yaml": schema, - version: []byte(`groups: - - id: metric.metric.current - type: metric - brief: "A metric" - stability: stable - metric_name: metric.current - instrument: counter - unit: "1" -`), - } -} - -func contractFanOutMatchers(variants int) []*labels.Matcher { - return []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric.current"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", fmt.Sprintf("registry/1.%d.0", variants-1)), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } -} - -func newContractFanOutStorage(t testing.TB, variants int) (storage.Storage, *contractHintStorage) { - t.Helper() - underlying := &contractHintStorage{Storage: teststorage.New(t)} - wrapper, err := semconv.AwareStorageWithRegistry(underlying, contractFanOutRegistry(variants)) - require.NoError(t, err) - return wrapper, underlying -} - -func newCanonicalLimitStorage(t *testing.T, storedControlLabels bool) (storage.Storage, *contractHintStorage) { - t.Helper() - underlying := &contractHintStorage{Storage: teststorage.New(t)} - appendSeries(t, underlying, "test.counter", 1, 1, "trace", "x") - appendSeries(t, underlying, "test.counter", 1, 2, "user", "a") - if storedControlLabels { - appendSeries(t, underlying, "test.counter", 1, 3, - "user", "c", - "__schema_url__", "stored", - "__semconv_url__", "stored", - ) - } else { - appendSeries(t, underlying, "test.counter", 1, 3, "user", "c") - } - appendSeries(t, underlying, "test", 1, 4, "tenant", "b") - appendSeries(t, underlying, "test", 1, 5, "tenant", "z") - return semconv.AwareStorage(underlying), underlying -} - -func TestSchemaAwareLimitsApplyAfterCanonicalization(t *testing.T) { - for _, query := range []string{"series", "chunks"} { - t.Run(query, func(t *testing.T) { - wrapper, _ := newCanonicalLimitStorage(t, false) - hints := &storage.SelectHints{Start: 0, End: 10, Step: 5, Limit: 1} - wantHints := copyRecordedHints(hints) - wantLabels := labels.FromStrings(model.MetricNameLabel, "test", "tenant", "a") - - if query == "series" { - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - require.Equal(t, map[string]float64{wantLabels.String(): 2}, - collectSeries(t, q.Select(t.Context(), false, hints, schemaReadMatchers()...))) - } else { - q, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - set := q.Select(t.Context(), false, hints, schemaReadMatchers()...) - require.True(t, set.Next()) - require.Equal(t, wantLabels, set.At().Labels()) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - } - - require.Equal(t, wantHints, hints, "schema fan-out must not mutate caller-owned hints") - }) - } - - for _, query := range []string{"querier", "chunk querier"} { - t.Run("label names/"+query, func(t *testing.T) { - wrapper, underlying := newCanonicalLimitStorage(t, true) - hints := &storage.LabelHints{Limit: 2} - - var ( - names []string - err error - ) - if query == "querier" { - q, qerr := wrapper.Querier(0, 10) - require.NoError(t, qerr) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - names, _, err = q.LabelNames(t.Context(), hints, schemaReadMatchers()...) - } else { - q, qerr := wrapper.ChunkQuerier(0, 10) - require.NoError(t, qerr) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - names, _, err = q.LabelNames(t.Context(), hints, schemaReadMatchers()...) - } - - require.NoError(t, err) - require.Equal(t, []string{model.MetricNameLabel, "tenant"}, names) - require.Equal(t, 2, hints.Limit, "schema fan-out must not mutate caller-owned hints") - nameHints, _ := underlying.recordedLabelHints() - require.Greater(t, len(nameHints), 1) - for _, recorded := range nameHints { - require.Equal(t, &storage.LabelHints{}, recorded, - "label-name limits must be applied after reserved-label filtering and canonicalization") - } - }) - - t.Run("label values/"+query, func(t *testing.T) { - wrapper, underlying := newCanonicalLimitStorage(t, false) - hints := &storage.LabelHints{Limit: 1} - - var ( - values []string - err error - ) - if query == "querier" { - q, qerr := wrapper.Querier(0, 10) - require.NoError(t, qerr) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - values, _, err = q.LabelValues(t.Context(), "tenant", hints, schemaReadMatchers()...) - } else { - q, qerr := wrapper.ChunkQuerier(0, 10) - require.NoError(t, qerr) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - values, _, err = q.LabelValues(t.Context(), "tenant", hints, schemaReadMatchers()...) - } - - require.NoError(t, err) - require.Equal(t, []string{"a"}, values) - require.Equal(t, 1, hints.Limit, "schema fan-out must not mutate caller-owned hints") - _, valueHints := underlying.recordedLabelHints() - require.Greater(t, len(valueHints), 1) - for _, recorded := range valueHints { - require.Equal(t, hints, recorded, "safe per-alias limit pushdown should be retained") - } - }) - } - - t.Run("nil and zero limits stay disabled", func(t *testing.T) { - wrapper, _ := newCanonicalLimitStorage(t, false) - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - - require.Len(t, collectSeries(t, q.Select(t.Context(), false, nil, schemaReadMatchers()...)), 5) - zeroSelectHints := &storage.SelectHints{Start: 0, End: 10} - require.Len(t, collectSeries(t, q.Select(t.Context(), false, zeroSelectHints, schemaReadMatchers()...)), 5) - require.Zero(t, zeroSelectHints.Limit) - - names, _, err := q.LabelNames(t.Context(), nil, schemaReadMatchers()...) - require.NoError(t, err) - require.Equal(t, []string{model.MetricNameLabel, "tenant", "trace"}, names) - zeroLabelHints := &storage.LabelHints{} - values, _, err := q.LabelValues(t.Context(), "tenant", zeroLabelHints, schemaReadMatchers()...) - require.NoError(t, err) - require.Equal(t, []string{"a", "b", "c", "z"}, values) - require.Zero(t, zeroLabelHints.Limit) - }) -} - -func TestSchemaAwareStorageFanOutLimitFailsClosed(t *testing.T) { - for _, variants := range []int{32, 33} { - wantErr := variants > 32 - t.Run(fmt.Sprintf("%d variants", variants), func(t *testing.T) { - t.Run("series", func(t *testing.T) { - wrapper, underlying := newContractFanOutStorage(t, variants) - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - - set := q.Select(t.Context(), false, nil, contractFanOutMatchers(variants)...) - require.False(t, set.Next()) - if wantErr { - require.ErrorContains(t, set.Err(), "schema expansion limit exceeded") - require.Zero(t, underlying.calls().seriesSelects) - } else { - require.NoError(t, set.Err()) - require.Equal(t, variants, underlying.calls().seriesSelects) - } - require.Zero(t, underlying.calls().labelNames) - }) - - t.Run("chunks", func(t *testing.T) { - wrapper, underlying := newContractFanOutStorage(t, variants) - q, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - - set := q.Select(t.Context(), false, nil, contractFanOutMatchers(variants)...) - require.False(t, set.Next()) - if wantErr { - require.ErrorContains(t, set.Err(), "schema expansion limit exceeded") - require.Zero(t, underlying.calls().chunkSelects) - } else { - require.NoError(t, set.Err()) - require.Equal(t, variants, underlying.calls().chunkSelects) - } - require.Zero(t, underlying.calls().labelNames) - }) - - t.Run("label names", func(t *testing.T) { - wrapper, underlying := newContractFanOutStorage(t, variants) - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - - _, _, err = q.LabelNames(t.Context(), nil, contractFanOutMatchers(variants)...) - if wantErr { - require.ErrorContains(t, err, "schema expansion limit exceeded") - require.Zero(t, underlying.calls().labelNames) - } else { - require.NoError(t, err) - require.Equal(t, variants, underlying.calls().labelNames) - } - }) - - t.Run("label values", func(t *testing.T) { - wrapper, underlying := newContractFanOutStorage(t, variants) - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - - _, _, err = q.LabelValues(t.Context(), model.MetricNameLabel, nil, contractFanOutMatchers(variants)...) - if wantErr { - require.ErrorContains(t, err, "schema expansion limit exceeded") - require.Zero(t, underlying.calls().labelValues) - } else { - require.NoError(t, err) - require.Equal(t, variants, underlying.calls().labelValues) - } - }) - }) - } -} - -func appendContractSeries(t testing.TB, s storage.Storage, count int) { - appendContractSeriesForMetric(t, s, "metric.old.00", count) -} - -func appendContractSeriesForMetric(t testing.TB, s storage.Storage, metric string, count int) { - t.Helper() - app := s.Appender(t.Context()) - for i := range count { - _, err := app.Append(0, labels.FromStrings( - model.MetricNameLabel, metric, - "instance", fmt.Sprintf("instance-%05d", i), - ), 1, float64(i)) - require.NoError(t, err) - } - require.NoError(t, app.Commit()) -} - -func TestSchemaAwareMetricOnlySelectStreamsWithoutLabelMetadata(t *testing.T) { - const seriesCount = 100 - for _, query := range []string{"series", "chunks"} { - t.Run(query, func(t *testing.T) { - underlying := &contractHintStorage{ - Storage: teststorage.New(t), - labelNamesErr: errors.New("label names unavailable"), - } - appendContractSeries(t, underlying, seriesCount) - wrapper, err := semconv.AwareStorageWithRegistry(underlying, contractFanOutRegistry(2)) - require.NoError(t, err) - hints := &storage.SelectHints{Start: 0, End: 10, Limit: 1} - wantHints := copyRecordedHints(hints) - - if query == "series" { - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - set := q.Select(t.Context(), false, hints, contractFanOutMatchers(2)...) - require.True(t, set.Next()) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - calls := underlying.calls() - require.Zero(t, calls.labelNames) - require.Less(t, calls.seriesNext, seriesCount) - seriesHints, _ := underlying.recordedHints() - require.Len(t, seriesHints, 2) - for _, recorded := range seriesHints { - require.Equal(t, wantHints, recorded) - } - require.Equal(t, wantHints, hints, "schema fan-out must not mutate caller-owned hints") - return - } - - q, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - set := q.Select(t.Context(), false, hints, contractFanOutMatchers(2)...) - require.True(t, set.Next()) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - calls := underlying.calls() - require.Zero(t, calls.labelNames) - require.Less(t, calls.chunkNext, seriesCount) - _, chunkHints := underlying.recordedHints() - require.Len(t, chunkHints, 2) - for _, recorded := range chunkHints { - require.Equal(t, wantHints, recorded) - } - require.Equal(t, wantHints, hints, "schema fan-out must not mutate caller-owned hints") - }) - } -} - -func TestSchemaAwareSelectRejectsStoredControlLabels(t *testing.T) { - for _, labelName := range []string{"__semconv_url__", "__schema_url__"} { - for _, query := range []string{"series", "chunks"} { - t.Run(labelName+"/"+query, func(t *testing.T) { - underlying := teststorage.New(t) - appendSeries(t, underlying, "test.counter", 1, 1, "user", "acme", labelName, "stored") - wrapper := semconv.AwareStorage(underlying) - - if query == "series" { - q, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - set := q.Select(t.Context(), false, nil, schemaReadMatchers()...) - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), "encountered stored control label "+labelName) - return - } - - q, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, q.Close()) }) - set := q.Select(t.Context(), false, nil, schemaReadMatchers()...) - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), "encountered stored control label "+labelName) - }) - } - } -} - -func TestSchemaAwareIdentitySelectRejectsStoredControlLabels(t *testing.T) { - for _, labelName := range []string{"__semconv_url__", "__schema_url__"} { - for _, query := range []string{"series", "chunks"} { - t.Run(labelName+"/"+query, func(t *testing.T) { - underlying := &contractHintStorage{Storage: teststorage.New(t)} - appendSeries(t, underlying, "metric.current", 1, 1, "instance", "a", labelName, "stored") - wrapper, err := semconv.AwareStorageWithRegistry(underlying, contractFanOutRegistry(1)) - require.NoError(t, err) - matchers := contractFanOutMatchers(1) - hints := &storage.SelectHints{ - Start: 0, - End: 10, - ProjectionLabels: []string{"instance"}, - ProjectionInclude: true, - } - wantHints := copyRecordedHints(hints) - - if query == "series" { - querier, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set := querier.Select(t.Context(), false, hints, matchers...) - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), "encountered stored control label "+labelName) - seriesHints, _ := underlying.recordedHints() - require.Len(t, seriesHints, 1) - require.Nil(t, seriesHints[0].ProjectionLabels) - require.False(t, seriesHints[0].ProjectionInclude) - require.Equal(t, wantHints, hints) - return - } - - querier, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set := querier.Select(t.Context(), false, hints, matchers...) - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), "encountered stored control label "+labelName) - _, chunkHints := underlying.recordedHints() - require.Len(t, chunkHints, 1) - require.Nil(t, chunkHints[0].ProjectionLabels) - require.False(t, chunkHints[0].ProjectionInclude) - require.Equal(t, wantHints, hints) - }) - } - } -} - -func TestSchemaAwareIdentitySelectSanitizesUnsafeHints(t *testing.T) { - for _, funcCase := range []struct { - name string - function string - wantFunc string - }{ - {name: "aggregation", function: "sum"}, - {name: "metadata only", function: "series", wantFunc: "series"}, - } { - for _, query := range []string{"series", "chunks"} { - t.Run(funcCase.name+"/"+query, func(t *testing.T) { - underlying := &contractHintStorage{Storage: teststorage.New(t, func(opts *tsdb.Options) { - opts.EnableSharding = true - })} - appendSeries(t, underlying, "metric.current", 1, 1, "instance", "a") - appendSeries(t, underlying, "metric.current", 2, 2, "instance", "b") - wrapper, err := semconv.AwareStorageWithRegistry(underlying, contractFanOutRegistry(1)) - require.NoError(t, err) - hints := &storage.SelectHints{ - Start: 1, - End: 2, - Limit: 1, - Step: 3, - Func: funcCase.function, - Grouping: []string{"instance"}, - By: true, - Range: 4, - ShardCount: 1, - ShardIndex: 0, - DisableTrimming: true, - ProjectionLabels: []string{"instance"}, - ProjectionInclude: true, - } - originalHints := copyRecordedHints(hints) - wantHints := copyRecordedHints(hints) - wantHints.Func = funcCase.wantFunc - wantHints.Grouping = nil - wantHints.By = false - wantHints.ProjectionLabels = nil - wantHints.ProjectionInclude = false - - if query == "series" { - querier, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set := querier.Select(t.Context(), false, hints, contractFanOutMatchers(1)...) - require.True(t, set.Next()) - require.True(t, set.At().Labels().Has(model.MetricNameLabel)) - require.False(t, set.At().Labels().Has("__series_hash__")) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - seriesHints, _ := underlying.recordedHints() - require.Equal(t, []*storage.SelectHints{wantHints}, seriesHints) - seriesSort, _ := underlying.recordedSort() - require.Equal(t, []bool{false}, seriesSort) - require.Equal(t, originalHints, hints) - return - } - - querier, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set := querier.Select(t.Context(), false, hints, contractFanOutMatchers(1)...) - require.True(t, set.Next()) - require.True(t, set.At().Labels().Has(model.MetricNameLabel)) - require.False(t, set.At().Labels().Has("__series_hash__")) - require.False(t, set.Next()) - require.NoError(t, set.Err()) - _, chunkHints := underlying.recordedHints() - require.Equal(t, []*storage.SelectHints{wantHints}, chunkHints) - _, chunkSort := underlying.recordedSort() - require.Equal(t, []bool{false}, chunkSort) - require.Equal(t, originalHints, hints) - }) - } - } -} - -func BenchmarkSchemaAwareMetricOnlySelect10000(b *testing.B) { - for _, tc := range []struct { - name string - variants int - metric string - }{ - {name: "identity", variants: 1, metric: "metric.current"}, - {name: "rewrite", variants: 2, metric: "metric.old.00"}, - } { - b.Run(tc.name, func(b *testing.B) { - underlying := teststorage.New(b) - appendContractSeriesForMetric(b, underlying, tc.metric, 10000) - wrapper, err := semconv.AwareStorageWithRegistry(underlying, contractFanOutRegistry(tc.variants)) - require.NoError(b, err) - matchers := contractFanOutMatchers(tc.variants) - - b.ReportAllocs() - b.ResetTimer() - for b.Loop() { - q, err := wrapper.Querier(0, 10) - if err != nil { - b.Fatal(err) - } - set := q.Select(b.Context(), false, nil, matchers...) - count := 0 - for set.Next() { - count++ - } - if err := set.Err(); err != nil { - b.Fatal(err) - } - if err := q.Close(); err != nil { - b.Fatal(err) - } - if count != 10000 { - b.Fatalf("got %d series, want 10000", count) - } - } - }) - } -} diff --git a/storage/semconv/storage_test.go b/storage/semconv/storage_test.go index 60174b5f93..7476cb930d 100644 --- a/storage/semconv/storage_test.go +++ b/storage/semconv/storage_test.go @@ -15,11 +15,6 @@ package semconv_test import ( "context" - "errors" - "fmt" - "slices" - "strings" - "sync" "testing" "github.com/prometheus/common/model" @@ -33,198 +28,6 @@ import ( "github.com/prometheus/prometheus/util/teststorage" ) -// requireWarningsContain asserts that at least one warning in the slice -// contains substr. Annotations are returned as a map, so iteration order is -// non-deterministic — checking warnings[0] would be flaky. -func requireWarningsContain(t *testing.T, warnings []string, substr string) { - t.Helper() - for _, w := range warnings { - if strings.Contains(w, substr) { - return - } - } - t.Fatalf("expected a warning containing %q, got %v", substr, warnings) -} - -// newAwareStorage builds a TestStorage and wraps it with AwareStorage. It -// returns both so tests can append directly into the underlying storage. -func newAwareStorage(t *testing.T) (storage.Storage, *teststorage.TestStorage) { - t.Helper() - underlying := teststorage.New(t) - return semconv.AwareStorage(underlying), underlying -} - -// erroringStorage wraps a storage.Storage and replaces LabelNames / -// LabelValues responses with a configured error whenever the supplied -// matchers carry an explicit __name__ value found in errsByMetric. All other -// methods delegate. It is a test-only fake used to exercise the wrapper's -// multi-variant error aggregation paths. -type erroringStorage struct { - storage.Storage - errsByMetric map[string]error -} - -type queryCallCounts struct { - mu sync.Mutex - selectSeries int64 - selectChunks int64 - labelNames int64 - labelValues int64 - chunkLabelNames int64 - chunkLabelValues int64 -} - -func (c *queryCallCounts) reset() { - c.mu.Lock() - defer c.mu.Unlock() - c.selectSeries = 0 - c.selectChunks = 0 - c.labelNames = 0 - c.labelValues = 0 - c.chunkLabelNames = 0 - c.chunkLabelValues = 0 -} - -func (c *queryCallCounts) total() int64 { - c.mu.Lock() - defer c.mu.Unlock() - return c.selectSeries + c.selectChunks + c.labelNames + c.labelValues + c.chunkLabelNames + c.chunkLabelValues -} - -func (c *queryCallCounts) increment(counter *int64) { - c.mu.Lock() - defer c.mu.Unlock() - (*counter)++ -} - -type countingStorage struct { - storage.Storage - counts *queryCallCounts -} - -func (s *countingStorage) Querier(mint, maxt int64) (storage.Querier, error) { - q, err := s.Storage.Querier(mint, maxt) - if err != nil { - return nil, err - } - return &countingQuerier{Querier: q, counts: s.counts}, nil -} - -func (s *countingStorage) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { - q, err := s.Storage.ChunkQuerier(mint, maxt) - if err != nil { - return nil, err - } - return &countingChunkQuerier{ChunkQuerier: q, counts: s.counts}, nil -} - -type countingQuerier struct { - storage.Querier - counts *queryCallCounts -} - -func (q *countingQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { - q.counts.increment(&q.counts.selectSeries) - return q.Querier.Select(ctx, sortSeries, hints, matchers...) -} - -func (q *countingQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.counts.increment(&q.counts.labelNames) - return q.Querier.LabelNames(ctx, hints, matchers...) -} - -func (q *countingQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.counts.increment(&q.counts.labelValues) - return q.Querier.LabelValues(ctx, name, hints, matchers...) -} - -type countingChunkQuerier struct { - storage.ChunkQuerier - counts *queryCallCounts -} - -func (q *countingChunkQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.ChunkSeriesSet { - q.counts.increment(&q.counts.selectChunks) - return q.ChunkQuerier.Select(ctx, sortSeries, hints, matchers...) -} - -func (q *countingChunkQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.counts.increment(&q.counts.chunkLabelNames) - return q.ChunkQuerier.LabelNames(ctx, hints, matchers...) -} - -func (q *countingChunkQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - q.counts.increment(&q.counts.chunkLabelValues) - return q.ChunkQuerier.LabelValues(ctx, name, hints, matchers...) -} - -func (s *erroringStorage) Querier(mint, maxt int64) (storage.Querier, error) { - q, err := s.Storage.Querier(mint, maxt) - if err != nil { - return nil, err - } - return &erroringQuerier{Querier: q, errsByMetric: s.errsByMetric}, nil -} - -func (s *erroringStorage) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { - q, err := s.Storage.ChunkQuerier(mint, maxt) - if err != nil { - return nil, err - } - return &erroringChunkQuerier{ChunkQuerier: q, errsByMetric: s.errsByMetric}, nil -} - -// errsForMetric returns the configured error for the metric name carried in -// matchers, or nil if no matcher targets a metric in errsByMetric. -func errsForMetric(errsByMetric map[string]error, matchers []*labels.Matcher) error { - for _, m := range matchers { - if m.Name == model.MetricNameLabel { - if err, ok := errsByMetric[m.Value]; ok { - return err - } - } - } - return nil -} - -type erroringQuerier struct { - storage.Querier - errsByMetric map[string]error -} - -func (q *erroringQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - if err := errsForMetric(q.errsByMetric, matchers); err != nil { - return nil, nil, err - } - return q.Querier.LabelNames(ctx, hints, matchers...) -} - -func (q *erroringQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - if err := errsForMetric(q.errsByMetric, matchers); err != nil { - return nil, nil, err - } - return q.Querier.LabelValues(ctx, name, hints, matchers...) -} - -type erroringChunkQuerier struct { - storage.ChunkQuerier - errsByMetric map[string]error -} - -func (q *erroringChunkQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - if err := errsForMetric(q.errsByMetric, matchers); err != nil { - return nil, nil, err - } - return q.ChunkQuerier.LabelNames(ctx, hints, matchers...) -} - -func (q *erroringChunkQuerier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - if err := errsForMetric(q.errsByMetric, matchers); err != nil { - return nil, nil, err - } - return q.ChunkQuerier.LabelValues(ctx, name, hints, matchers...) -} - // appendSeries writes a single (name=metric, labels..., t, v) sample into s. // Labels alternate name/value pairs. func appendSeries(t *testing.T, s storage.Storage, metric string, ts int64, v float64, kv ...string) { @@ -237,8 +40,7 @@ func appendSeries(t *testing.T, s storage.Storage, metric string, ts int64, v fl require.NoError(t, app.Commit()) } -// collectSeries drains a SeriesSet into a slice of label-string -> values. -// Each value is the first sample's value (tests append a single sample each). +// collectSeries drains a SeriesSet into a map of label-string -> first sample value. func collectSeries(t *testing.T, set storage.SeriesSet) map[string]float64 { t.Helper() out := make(map[string]float64) @@ -268,252 +70,6 @@ func collectSeriesSampleCounts(t *testing.T, set storage.SeriesSet) map[string]i return out } -func undeclaredAttributeRegistry() map[string][]byte { - return map[string][]byte{ - "registry.yaml": []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.0.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: - m.old: m - - rename_attributes: - attribute_map: - svc.env: svc.environment - apply_to_metrics: - - m -`), - "1.0.0": []byte(`groups: - - id: metric.m.old - type: metric - metric_name: m.old - instrument: counter - unit: "1" -`), - "1.1.0": []byte(`groups: - - id: metric.m - type: metric - metric_name: m - instrument: counter - unit: "1" -`), - } -} - -func undeclaredAttributeMatchers(version, metric, attribute string, withAttribute bool) []*labels.Matcher { - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, metric), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/"+version), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - if withAttribute { - matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, attribute, "prod")) - } - return matchers -} - -func TestSchemaAttributeRenamesWithoutSemconvDeclarations(t *testing.T) { - underlying := teststorage.New(t) - wrapper, err := semconv.AwareStorageWithRegistry(underlying, undeclaredAttributeRegistry()) - require.NoError(t, err) - appendSeries(t, underlying, "m.old", 1, 1, "svc.env", "prod") - appendSeries(t, underlying, "m", 2, 2, "svc.environment", "prod") - - for _, anchor := range []struct { - name string - version string - metric string - attribute string - alias string - }{ - {name: "backward", version: "1.1.0", metric: "m", attribute: "svc.environment", alias: "svc.env"}, - {name: "forward", version: "1.0.0", metric: "m.old", attribute: "svc.env", alias: "svc.environment"}, - } { - t.Run(anchor.name, func(t *testing.T) { - for _, withAttribute := range []bool{false, true} { - t.Run(fmt.Sprintf("attribute matcher %t", withAttribute), func(t *testing.T) { - matchers := undeclaredAttributeMatchers(anchor.version, anchor.metric, anchor.attribute, withAttribute) - querier, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - got := collectSeries(t, querier.Select(t.Context(), true, nil, matchers...)) - require.Len(t, got, 1) - for labelSet := range got { - require.Contains(t, labelSet, `__name__="`+anchor.metric+`"`) - require.Contains(t, labelSet, `"`+anchor.attribute+`"="prod"`) - require.NotContains(t, labelSet, `"`+anchor.alias+`"=`) - } - - chunkQuerier, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, chunkQuerier.Close()) }) - chunks := chunkQuerier.Select(t.Context(), true, nil, matchers...) - require.Len(t, collectSeries(t, storage.NewSeriesSetFromChunkSeriesSet(chunks)), 1) - }) - } - - matchers := undeclaredAttributeMatchers(anchor.version, anchor.metric, anchor.attribute, false) - querier, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - names, _, err := querier.LabelNames(t.Context(), nil, matchers...) - require.NoError(t, err) - require.Contains(t, names, anchor.attribute) - require.NotContains(t, names, anchor.alias) - values, _, err := querier.LabelValues(t.Context(), anchor.attribute, nil, matchers...) - require.NoError(t, err) - require.Equal(t, []string{"prod"}, values) - }) - } - - valueUnderlying := teststorage.New(t) - valueWrapper, err := semconv.AwareStorageWithRegistry(valueUnderlying, undeclaredAttributeRegistry()) - require.NoError(t, err) - appendSeries(t, valueUnderlying, "m.old", 1, 1, "svc.env", "legacy") - appendSeries(t, valueUnderlying, "m", 2, 2, "svc.environment", "current") - for _, anchor := range []struct { - version string - metric string - attribute string - }{ - {version: "1.1.0", metric: "m", attribute: "svc.environment"}, - {version: "1.0.0", metric: "m.old", attribute: "svc.env"}, - } { - querier, err := valueWrapper.Querier(0, 10) - require.NoError(t, err) - values, _, err := querier.LabelValues( - t.Context(), - anchor.attribute, - nil, - undeclaredAttributeMatchers(anchor.version, anchor.metric, anchor.attribute, false)..., - ) - require.NoError(t, err) - require.NoError(t, querier.Close()) - require.Equal(t, []string{"current", "legacy"}, values) - } -} - -func TestSchemaMetricAndAttributeRevisionBoundaries(t *testing.T) { - for _, anchor := range []struct { - name string - version string - metric string - attribute string - alias string - }{ - {name: "backward", version: "1.1.0", metric: "m", attribute: "svc.environment", alias: "svc.env"}, - {name: "forward", version: "1.0.0", metric: "m.old", attribute: "svc.env", alias: "svc.environment"}, - } { - t.Run(anchor.name, func(t *testing.T) { - underlying := teststorage.New(t) - wrapper, err := semconv.AwareStorageWithRegistry(underlying, undeclaredAttributeRegistry()) - require.NoError(t, err) - appendSeries(t, underlying, "m.old", 1, 1, - "instance", "old-boundary", "svc.env", "prod") - appendSeries(t, underlying, "m", 2, 2, - "instance", "current-boundary", "svc.environment", "prod") - appendSeries(t, underlying, "m", 10, 10, - "instance", "dual", "svc.env", "prod", "svc.environment", "prod") - - matchers := undeclaredAttributeMatchers(anchor.version, anchor.metric, anchor.attribute, true) - for _, query := range []string{"series", "chunks"} { - t.Run(query, func(t *testing.T) { - var set storage.SeriesSet - if query == "series" { - querier, err := wrapper.Querier(0, 20) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set = querier.Select(t.Context(), true, nil, matchers...) - } else { - querier, err := wrapper.ChunkQuerier(0, 20) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set = storage.NewSeriesSetFromChunkSeriesSet(querier.Select(t.Context(), true, nil, matchers...)) - } - got := collectSeriesSampleCounts(t, set) - require.Len(t, got, 3) - for labelSet, sampleCount := range got { - require.Contains(t, labelSet, `__name__="`+anchor.metric+`"`) - require.Contains(t, labelSet, `"`+anchor.attribute+`"="prod"`) - require.NotContains(t, labelSet, `"`+anchor.alias+`"=`) - require.Equal(t, 1, sampleCount) - } - }) - } - - querier, err := wrapper.Querier(0, 20) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - baseMatchers := undeclaredAttributeMatchers(anchor.version, anchor.metric, anchor.attribute, false) - names, _, err := querier.LabelNames(t.Context(), nil, baseMatchers...) - require.NoError(t, err) - require.Contains(t, names, anchor.attribute) - require.NotContains(t, names, anchor.alias) - values, _, err := querier.LabelValues(t.Context(), anchor.attribute, nil, baseMatchers...) - require.NoError(t, err) - require.Equal(t, []string{"prod"}, values) - }) - } -} - -func TestSchemaLabelMigrationConflictsFailClosed(t *testing.T) { - for _, query := range []string{"series", "chunks"} { - t.Run(query, func(t *testing.T) { - t.Run("equal values coalesce on the anchor metric", func(t *testing.T) { - wrapper, underlying := newAwareStorage(t) - appendSeries(t, underlying, "test", 1, 1, "user", "acme", "tenant", "acme") - - if query == "series" { - querier, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - got := collectSeries(t, querier.Select(t.Context(), true, nil, schemaReadMatchers()...)) - require.Len(t, got, 1) - for labelSet := range got { - require.Contains(t, labelSet, `tenant="acme"`) - require.NotContains(t, labelSet, "user=") - } - return - } - - querier, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set := storage.NewSeriesSetFromChunkSeriesSet(querier.Select(t.Context(), true, nil, schemaReadMatchers()...)) - require.Len(t, collectSeries(t, set), 1) - }) - - t.Run("different values on the anchor metric fail the query", func(t *testing.T) { - wrapper, underlying := newAwareStorage(t) - appendSeries(t, underlying, "test", 1, 1, "instance", "conflict", "user", "legacy", "tenant", "current") - appendSeries(t, underlying, "test", 2, 2, "instance", "healthy", "user", "same") - - if query == "series" { - querier, err := wrapper.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set := querier.Select(t.Context(), true, nil, schemaReadMatchers()...) - for set.Next() { - } - require.ErrorContains(t, set.Err(), "conflicting values") - return - } - - querier, err := wrapper.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, querier.Close()) }) - set := querier.Select(t.Context(), true, nil, schemaReadMatchers()...) - for set.Next() { - } - require.ErrorContains(t, set.Err(), "conflicting values") - }) - }) - } -} - -// warningStrings flattens annotations into their string forms for assertion. func warningStrings(a map[string]error) []string { out := make([]string, 0, len(a)) for k := range a { @@ -522,626 +78,87 @@ func warningStrings(a map[string]error) []string { return out } -func metricVariantOverflowRegistry(sources int) map[string][]byte { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_metrics: -`) - for i := range sources { - schema = fmt.Appendf(schema, " metric.old.%03d: metric.current\n", i) - } - return map[string][]byte{ - "registry.yaml": schema, - "1.1.0": metricSemconv("metric.current", "s", "histogram"), - } -} +func TestAwareStoragePassthroughAndWarnings(t *testing.T) { + wrapped, underlying := newTransformTestStorage(t) + appendSeries(t, underlying, "test", 1, 1.0, "env", "prod") -func aggregateExpansionOverflowRegistry(attributes, sources int) map[string][]byte { - files := metricVariantOverflowRegistry(sources) - semconvFile := []byte(`groups: - - id: metric.metric.current - type: metric - metric_name: metric.current - unit: s - instrument: histogram - attributes: -`) - for i := range attributes { - name := "attr.current" - if i > 0 { - name = fmt.Sprintf("attr.current.%03d", i) - } - semconvFile = fmt.Appendf(semconvFile, " - ref: %s\n", name) - } - files["1.1.0"] = semconvFile - return files -} - -func labelValueJobOverflowRegistry(aliases int) map[string][]byte { - schema := []byte(`file_format: 1.1.0 -schema_url: https://example.com/schemas/1.1.0 -versions: - 1.0.0: - 1.1.0: - metrics: - changes: - - rename_attributes: - attribute_map: -`) - for i := range aliases { - schema = fmt.Appendf(schema, " attr.old.%03d: attr.current\n", i) - } - schema = append(schema, []byte(` - rename_metrics: - metric.a: metric.current - metric.b: metric.current -`)...) - return map[string][]byte{ - "registry.yaml": schema, - "1.0.0": []byte(`groups: - - id: metric.metric.a - type: metric - metric_name: metric.a - unit: s - instrument: histogram - stability: stable - - id: metric.metric.b - type: metric - metric_name: metric.b - unit: s - instrument: histogram - stability: stable -`), - "1.1.0": []byte(`groups: - - id: metric.metric.current - type: metric - metric_name: metric.current - unit: s - instrument: histogram - attributes: - - ref: attr.current -`), - } -} - -func newCountingAwareStorage(t *testing.T, files map[string][]byte) (storage.Storage, *queryCallCounts) { - t.Helper() - counts := &queryCallCounts{} - underlying := &countingStorage{Storage: teststorage.New(t), counts: counts} - wrapped, err := semconv.AwareStorageWithRegistry(underlying, files) + q, err := wrapped.Querier(0, 10) require.NoError(t, err) - return wrapped, counts + t.Cleanup(func() { require.NoError(t, q.Close()) }) + + t.Run("plain selector without __schema_url__ passes through without warnings", func(t *testing.T) { + set := q.Select(t.Context(), false, nil, + labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), + ) + got := collectSeries(t, set) + require.Empty(t, set.Warnings()) + require.Equal(t, map[string]float64{`{__name__="test", env="prod"}`: 1.0}, got) + }) + + t.Run("invalid __schema_url__ falls back to passthrough with ErrSchemaWarning", func(t *testing.T) { + set := q.Select(t.Context(), false, nil, + labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), + labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "http://example.com/x.yaml"), + ) + got := collectSeries(t, set) + require.Equal(t, map[string]float64{`{__name__="test", env="prod"}`: 1.0}, got) + warnings := set.Warnings() + require.NotEmpty(t, warnings) + for _, err := range warnings { + require.ErrorIs(t, err, semconv.ErrSchemaWarning) + require.ErrorIs(t, err, annotations.PromQLWarning) + } + }) + + t.Run("missing __name__ equality matcher with __schema_url__ fails query", func(t *testing.T) { + set := q.Select(t.Context(), false, nil, + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, "http.*"), + labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/http_1.20.0_to_1.23.1.yaml"), + ) + require.False(t, set.Next()) + require.ErrorContains(t, set.Err(), "requires a non-empty equality matcher on __name__") + }) } -func collectChunkSeriesCount(t *testing.T, set storage.ChunkSeriesSet) int { - t.Helper() - count := 0 - for set.Next() { - count++ - } - require.NoError(t, set.Err()) - return count -} - -// TestAwareStorage tests AwareStorage. -func TestAwareStorage(t *testing.T) { - t.Run("passes through without a special matcher", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "up", 1, 42, "instance", "a") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "up")) - - got := collectSeries(t, set) - require.Len(t, got, 1) - for k, v := range got { - require.Contains(t, k, "instance=\"a\"") - require.Equal(t, 42.0, v) - } - }) - - t.Run("warns and passes through on a duplicate __semconv_url__", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test", 1, 1.0) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.0.0"), - ) - - _ = collectSeries(t, set) - warnings := warningStrings(set.Warnings()) - require.NotEmpty(t, warnings) - requireWarningsContain(t, warnings, "used more than once") - }) - - t.Run("warns and passes through on a non-equal __semconv_url__", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test", 1, 1.0) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchRegexp, "__semconv_url__", "registry/1\\..*"), - ) - - _ = collectSeries(t, set) - warnings := warningStrings(set.Warnings()) - require.NotEmpty(t, warnings) - requireWarningsContain(t, warnings, "ambiguous") - }) - - t.Run("rejects http and traversal URLs", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test", 1, 1.0) - - for _, bad := range []string{ - "http://169.254.169.254/latest/meta-data/iam/info", - "https://example.com/etc/passwd", - "/etc/passwd", - "../../../etc/passwd", - "./testdata/otel.yaml", - "registry/../etc/passwd", - } { - t.Run(bad, func(t *testing.T) { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", bad), - // A valid __schema_url__ triggers fan-out so the bad - // __semconv_url__ is actually loaded and rejected. - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - // The set must complete cleanly (passthrough) and surface a warning. - _ = collectSeries(t, set) - warnings := warningStrings(set.Warnings()) - require.NotEmpty(t, warnings, "expected a warning for %q", bad) - requireWarningsContain(t, warnings, "schematization logic is skipped") - }) - } - }) - - t.Run("LabelValues passes through without a special matcher", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "up", 1, 1.0, "instance", "a") - appendSeries(t, wrapped, "up", 1, 1.0, "instance", "b") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - vals, _, err := q.LabelValues(context.Background(), "instance", nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "up")) - require.NoError(t, err) - require.ElementsMatch(t, []string{"a", "b"}, vals) - }) - - // Schema-version rename fan-out: a native-OTel producer's historical metric - // name surfaces under the requested version's canonical name via __schema_url__. - t.Run("schema version rename", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - // The producer wrote the metric under its semconv 1.0.0 name "test.counter" - // (native OTel names); semconv 1.1.0 renamed it to "test". - appendSeries(t, wrapped, "test.counter", 1, 7.0, "http.response.status_code", "200") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.NotEmpty(t, got, "expected the historical name to surface via __schema_url__") - var found bool - for k := range got { - if strings.Contains(k, `__name__="test"`) { - found = true - } - } - require.True(t, found, "expected the renamed metric under its 1.1.0 name in: %v", got) - }) - - t.Run("rewrites duplicate metric name matchers", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test.counter", 1, 7.0) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.Len(t, got, 1, "both canonical metric matchers must follow the historical lineage: %v", got) - for key := range got { - require.Contains(t, key, `__name__="test"`) - } - }) - - // When the fan-out probes multiple historical names, the wrapper surfaces - // every underlying failure via errors.Is rather than only the first. - t.Run("aggregates variant errors", func(t *testing.T) { - // Anchored at semconv 1.1.0, "test" fans out to its historical names: - // test.counter (1.0.0), test (1.1.0) and test.v2 (1.2.0). Each variant is - // probed concurrently; a per-variant error must propagate through the join. - errAnchor := errors.New("err-anchor") - errOld := errors.New("err-old") - errNew := errors.New("err-new") - +func TestAwareStorageWithRegistryValidationAndDefensiveCopy(t *testing.T) { + t.Run("rejects empty or invalid registry", func(t *testing.T) { underlying := teststorage.New(t) - wrapped := semconv.AwareStorage(&erroringStorage{ - Storage: underlying, - errsByMetric: map[string]error{ - "test": errAnchor, - "test.counter": errOld, - "test.v2": errNew, - }, + t.Cleanup(func() { require.NoError(t, underlying.Close()) }) + _, err := semconv.AwareStorageWithRegistry(underlying, nil) + require.Error(t, err) + _, err = semconv.AwareStorageWithRegistry(underlying, map[string][]byte{ + "bad.yaml": []byte("file_format: 9.9.9\n"), }) + require.Error(t, err) + }) + t.Run("defensively copies caller registry map and byte slices", func(t *testing.T) { + files := testTransformRegistry() + underlying := teststorage.New(t) + t.Cleanup(func() { require.NoError(t, underlying.Close()) }) + wrapped, err := semconv.AwareStorageWithRegistry(underlying, files) + require.NoError(t, err) + + for name, b := range files { + clear(b) + files[name] = []byte("mutated after construction") + } + + appendSeries(t, underlying, "http.server.duration", 1, 7.0, + "http.method", "GET", + "http.status_code", "200", + ) q, err := wrapped.Querier(0, 10) require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) + t.Cleanup(func() { require.NoError(t, q.Close()) }) - _, _, err = q.LabelNames(context.Background(), nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), + set := q.Select(t.Context(), true, nil, + labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "http.server.request.duration"), + labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/http_1.20.0_to_1.23.1.yaml"), ) - require.Error(t, err) - require.ErrorIs(t, err, errAnchor, "errors.Join should preserve every variant error") - require.ErrorIs(t, err, errOld, "errors.Join should preserve every variant error") - require.ErrorIs(t, err, errNew, "errors.Join should preserve every variant error") - }) - - // Attribute-rename normalisation: registry.yaml renames the attribute user - // (semconv 1.0.0) -> tenant (1.1.0) on metric test, alongside the metric - // rename test.counter -> test. A query at 1.1.0 must surface the 1.0.0-era - // series under the canonical attribute name. - t.Run("attribute rename", func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - // semconv 1.0.0 era: native name test.counter with the old attribute user. - appendSeries(t, wrapped, "test.counter", 1, 7.0, "user", "acme", "http.response.status_code", "200") - // semconv 1.1.0 era: renamed to test with the new attribute tenant. - appendSeries(t, wrapped, "test", 2, 8.0, "tenant", "acme", "http.response.status_code", "200") - - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - - t.Run("Select merges eras under the canonical attribute name", func(t *testing.T) { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), true, nil, matchers...) - got := collectSeries(t, set) - require.Len(t, got, 1, "the two eras should merge into a single series under tenant: %v", got) - for k := range got { - require.Contains(t, k, `__name__="test"`) - require.Contains(t, k, `tenant="acme"`) - require.NotContains(t, k, "user=", "historical attribute name must be normalised: %v", got) - } - }) - - t.Run("LabelNames reports the canonical attribute name", func(t *testing.T) { - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - names, _, err := q.LabelNames(context.Background(), nil, matchers...) - require.NoError(t, err) - require.Contains(t, names, "tenant") - require.Contains(t, names, "http.response.status_code") - require.NotContains(t, names, "user", "historical attribute name must be normalised in LabelNames: %v", names) - }) - - t.Run("LabelValues surfaces values from both eras", func(t *testing.T) { - // Distinct values per era: the assertion fails unless the 1.0.0 era, - // stored under "user", is also consulted when querying values of "tenant". - lvStore, _ := newAwareStorage(t) - appendSeries(t, lvStore, "test.counter", 1, 7.0, "user", "legacy") - appendSeries(t, lvStore, "test", 2, 8.0, "tenant", "current") - - q, err := lvStore.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - values, _, err := q.LabelValues(context.Background(), "tenant", nil, matchers...) - require.NoError(t, err) - require.ElementsMatch(t, []string{"legacy", "current"}, values) - }) + got := collectSeries(t, set) + require.Equal(t, map[string]float64{ + `{__name__="http.server.request.duration", "http.request.method"="GET", "http.response.status_code"="200"}`: 7.0, + }, got) }) } - -func TestMetricNameConstraintsKeepPromQLSemantics(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test.counter", 1, 1.0, "era", "old") - appendSeries(t, wrapped, "test", 1, 2.0, "era", "current") - - tests := []struct { - name string - nameMatchers []*labels.Matcher - wantSeries int - wantLabelName bool - wantValues []string - }{ - { - name: "compatible constraints apply to the canonical name", - nameMatchers: []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, `test(?:\.counter)?`), - labels.MustNewMatcher(labels.MatchNotEqual, model.MetricNameLabel, "test.counter"), - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - }, - wantSeries: 2, - wantLabelName: true, - wantValues: []string{"current", "old"}, - }, - { - name: "contradictory constraints remain unsatisfiable", - nameMatchers: []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, `test\.counter`), - }, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - matchers := append(slices.Clone(tc.nameMatchers), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, matchers...) - got := collectSeries(t, set) - require.Len(t, got, tc.wantSeries) - for key := range got { - require.Contains(t, key, `__name__="test"`) - } - - names, _, err := q.LabelNames(context.Background(), nil, matchers...) - require.NoError(t, err) - require.Equal(t, tc.wantLabelName, slices.Contains(names, "era")) - - values, _, err := q.LabelValues(context.Background(), "era", nil, matchers...) - require.NoError(t, err) - if len(tc.wantValues) == 0 { - require.Empty(t, values) - } else { - require.Equal(t, tc.wantValues, values) - } - - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - require.Equal(t, tc.wantSeries, collectChunkSeriesCount(t, - cq.Select(context.Background(), false, nil, matchers...))) - }) - } -} - -func TestSchemaAwareQueriesRequireExactMetricNameAnchor(t *testing.T) { - controlMatchers := func(metricMatcher *labels.Matcher) []*labels.Matcher { - matchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "job", "api")} - if metricMatcher != nil { - matchers = append(matchers, metricMatcher) - } - return append(matchers, - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - } - - for _, tc := range []struct { - name string - metricMatcher *labels.Matcher - }{ - {name: "missing"}, - {name: "regexp", metricMatcher: labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, "metric.*")}, - {name: "negative", metricMatcher: labels.MustNewMatcher(labels.MatchNotEqual, model.MetricNameLabel, "metric.old")}, - {name: "empty equality", metricMatcher: labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "")}, - } { - t.Run(tc.name, func(t *testing.T) { - wrapped, counts := newCountingAwareStorage(t, canonicalLabelRegistry()) - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - matchers := controlMatchers(tc.metricMatcher) - - counts.reset() - set := q.Select(context.Background(), false, nil, matchers...) - require.False(t, set.Next()) - require.ErrorContains(t, set.Err(), "requires a non-empty equality matcher on __name__") - require.Zero(t, counts.total()) - - counts.reset() - chunkSet := cq.Select(context.Background(), false, nil, matchers...) - require.False(t, chunkSet.Next()) - require.ErrorContains(t, chunkSet.Err(), "requires a non-empty equality matcher on __name__") - require.Zero(t, counts.total()) - - counts.reset() - _, _, err = q.LabelNames(context.Background(), nil, matchers...) - require.ErrorContains(t, err, "requires a non-empty equality matcher on __name__") - require.Zero(t, counts.total()) - - counts.reset() - _, _, err = cq.LabelValues(context.Background(), "job", nil, matchers...) - require.ErrorContains(t, err, "requires a non-empty equality matcher on __name__") - require.Zero(t, counts.total()) - }) - } -} - -func TestSchemaExpansionOverflowFailsBeforeStorage(t *testing.T) { - for _, tc := range []struct { - name string - files map[string][]byte - errorDetail string - }{ - { - name: "collection cardinality", - files: metricVariantOverflowRegistry(256), - errorDetail: "matcher variants would exceed 256", - }, - { - name: "aggregate resolver work", - files: aggregateExpansionOverflowRegistry(255, 255), - errorDetail: "resolver work would exceed 65536", - }, - } { - t.Run(tc.name, func(t *testing.T) { - testSchemaExpansionOverflowFailsBeforeStorage(t, tc.files, tc.errorDetail) - }) - } -} - -func testSchemaExpansionOverflowFailsBeforeStorage(t *testing.T, files map[string][]byte, errorDetail string) { - wrapped, counts := newCountingAwareStorage(t, files) - appendSeries(t, wrapped, "metric.current", 1, 7.0, "attr.current", "value") - matchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric.current"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - } - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - cq, err := wrapped.ChunkQuerier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = cq.Close() }) - - assertExpansionError := func(t *testing.T, err error) { - t.Helper() - require.ErrorContains(t, err, "schema expansion limit exceeded") - require.ErrorContains(t, err, errorDetail) - require.Zero(t, counts.total(), "overflow must fail before querying storage") - } - - t.Run("Select", func(t *testing.T) { - counts.reset() - set := q.Select(context.Background(), false, nil, matchers...) - require.False(t, set.Next()) - assertExpansionError(t, set.Err()) - require.Empty(t, set.Warnings()) - }) - - t.Run("LabelNames", func(t *testing.T) { - counts.reset() - names, anns, err := q.LabelNames(context.Background(), nil, matchers...) - assertExpansionError(t, err) - require.Nil(t, names) - require.Empty(t, anns) - }) - - t.Run("LabelValues", func(t *testing.T) { - counts.reset() - values, anns, err := q.LabelValues(context.Background(), "attr.current", nil, matchers...) - assertExpansionError(t, err) - require.Nil(t, values) - require.Empty(t, anns) - }) - - t.Run("ChunkSelect", func(t *testing.T) { - counts.reset() - set := cq.Select(context.Background(), false, nil, matchers...) - require.False(t, set.Next()) - assertExpansionError(t, set.Err()) - require.Empty(t, set.Warnings()) - }) - - t.Run("ChunkLabelNames", func(t *testing.T) { - counts.reset() - names, anns, err := cq.LabelNames(context.Background(), nil, matchers...) - assertExpansionError(t, err) - require.Nil(t, names) - require.Empty(t, anns) - }) - - t.Run("ChunkLabelValues", func(t *testing.T) { - counts.reset() - values, anns, err := cq.LabelValues(context.Background(), "attr.current", nil, matchers...) - assertExpansionError(t, err) - require.Nil(t, values) - require.Empty(t, anns) - }) -} - -func TestLabelValueJobOverflowFailsClosed(t *testing.T) { - wrapped, counts := newCountingAwareStorage(t, labelValueJobOverflowRegistry(128)) - appendSeries(t, wrapped, "metric.current", 1, 7.0, "attr.current", "value") - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - counts.reset() - values, anns, err := q.LabelValues(context.Background(), "attr.current", nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "metric.current"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.1.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - require.ErrorContains(t, err, "schema expansion limit exceeded") - require.Empty(t, values) - require.Empty(t, anns) - require.Zero(t, counts.total(), "job overflow must not query storage") -} - -func TestSchemaWarning_ClassifiedAsWarning(t *testing.T) { - // ErrSchemaWarning chains through annotations.PromQLWarning, so warnings - // emitted by the wrapper satisfy errors.Is for both sentinels and surface - // as PromQL warnings via util/annotations.AsStrings. - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test", 1, 1.0) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "test"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "http://example.com/x.yaml"), - ) - _ = collectSeries(t, set) - got := set.Warnings() - require.NotEmpty(t, got, "expected at least one SchemaWarning to be emitted") - for _, err := range got { - require.ErrorIs(t, err, semconv.ErrSchemaWarning, "warning %v should be a SchemaWarning", err) - require.ErrorIs(t, err, annotations.PromQLWarning, "warning %v should chain through PromQLWarning", err) - } -} diff --git a/storage/semconv/storage_topk_test.go b/storage/semconv/storage_topk_test.go deleted file mode 100644 index 18d2460bad..0000000000 --- a/storage/semconv/storage_topk_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv - -import ( - "math" - "testing" - - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" -) - -type topKTestSeries struct { - labels labels.Labels - contributors int -} - -func (s *topKTestSeries) Labels() labels.Labels { - return s.labels -} - -func collectTopKTestSeries(input []*topKTestSeries, limit int) ([]*topKTestSeries, int) { - position := -1 - got := collectAndChainTopK( - func() bool { - position++ - return position < len(input) - }, - func() *topKTestSeries { return input[position] }, - limit, - func(series ...*topKTestSeries) *topKTestSeries { - merged := &topKTestSeries{labels: series[0].labels} - for _, item := range series { - merged.contributors += item.contributors - } - return merged - }, - ) - return got, position -} - -func TestCollectAndChainTopK(t *testing.T) { - series := func(value string) *topKTestSeries { - return &topKTestSeries{ - labels: labels.FromStrings("instance", value), - contributors: 1, - } - } - - got, drained := collectTopKTestSeries([]*topKTestSeries{ - series("z"), series("a"), series("c"), series("a"), series("b"), series("d"), - }, 2) - require.Equal(t, 6, drained) - require.Equal(t, []labels.Labels{ - labels.FromStrings("instance", "a"), - labels.FromStrings("instance", "b"), - }, []labels.Labels{got[0].Labels(), got[1].Labels()}) - require.Equal(t, 2, got[0].contributors) - require.Equal(t, 1, got[1].contributors) -} - -func TestCollectAndChainTopKDoesNotPreallocateFromLimit(t *testing.T) { - input := []*topKTestSeries{{ - labels: labels.FromStrings("instance", "a"), - contributors: 1, - }} - got, drained := collectTopKTestSeries(input, math.MaxInt) - require.Equal(t, 1, drained) - require.Equal(t, input, got) - require.LessOrEqual(t, cap(got), len(input)) -} diff --git a/storage/semconv/storage_traversal_test.go b/storage/semconv/storage_traversal_test.go deleted file mode 100644 index 9c514fdd24..0000000000 --- a/storage/semconv/storage_traversal_test.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" -) - -// selectAt runs a fanned-out Select against the embedded registry, anchored at -// semconv version anchor. -func selectAt(t *testing.T, s storage.Storage, anchor, metricName string) storage.SeriesSet { - t.Helper() - q, err := s.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - return q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, metricName), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/"+anchor), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) -} - -// TestFanOutReachesEveryEraFromAnyAnchor walks the embedded registry's rename -// chain (test.counter → test at 1.1.0 → test.v2 at 1.2.0) from valid anchors. -// -// Anchoring at the newest version happens to work with a truncated walk, because -// then every rename is on the backward side; the oldest and middle anchors are what -// expose a walk that mis-slices the version history or fails to chain across the -// anchor. Each anchor must reach all three eras. -func TestFanOutReachesEveryEraFromAnyAnchor(t *testing.T) { - // eras names the attribute each era's series is expected to carry in the - // result. Ordered schema changes determine the anchor-era name even when the - // anchor semconv does not declare attributes for the queried metric. - eras := func(attrs ...string) []string { return attrs } - - for _, tc := range []struct { - name string - anchor string - queried string - attrs []string - description string - }{ - { - name: "oldest anchor", - anchor: "1.0.0", - queried: "test.counter", - attrs: eras("user", "user", "user"), - description: "no version is at or before 1.0.0, so the whole chain has to be walked forward", - }, - { - name: "middle anchor", - anchor: "1.1.0", - queried: "test", - attrs: eras("tenant", "tenant", "tenant"), - description: "1.1.0 is walked backward and 1.2.0 forward", - }, - { - name: "newest anchor", - anchor: "1.2.0", - queried: "test.v2", - attrs: eras("tenant", "tenant", "tenant"), - description: "every rename is on the backward side", - }, - { - name: "middle anchor with a name that version renamed away", - anchor: "1.1.0", - queried: "test.counter", - attrs: eras("user", "user", "user"), - description: "test.counter is the name 1.1.0 renamed, so the backward walk crosses " + - "that rename and the forward walk has to continue from where it landed", - }, - { - name: "newest anchor with the name that version renamed away", - anchor: "1.2.0", - queried: "test", - attrs: eras("tenant", "tenant", "tenant"), - description: "the retirement revision is oriented forward when it is the exact anchor", - }, - } { - t.Run(tc.name, func(t *testing.T) { - wrapped, _ := newAwareStorage(t) - // One series per naming era, distinguished by attribute value so the - // three stay distinct series rather than merging into one. - appendSeries(t, wrapped, "test.counter", 1, 1.0, "user", "a") - appendSeries(t, wrapped, "test", 1, 2.0, "tenant", "b") - appendSeries(t, wrapped, "test.v2", 1, 3.0, "tenant", "c") - - got := collectSeries(t, selectAt(t, wrapped, tc.anchor, tc.queried)) - - require.Len(t, got, 3, "%s: got %v", tc.description, got) - for i, value := range []string{"a", "b", "c"} { - require.Contains(t, got, - labels.FromStrings(model.MetricNameLabel, tc.queried, tc.attrs[i], value).String()) - } - }) - } -} - -func TestFanOutStopsAtRetirementBeforeLaterAnchor(t *testing.T) { - wrapped, _ := newAwareStorage(t) - appendSeries(t, wrapped, "test.counter", 1, 1.0, "user", "a") - appendSeries(t, wrapped, "test", 1, 2.0, "tenant", "b") - appendSeries(t, wrapped, "test.v2", 1, 3.0, "tenant", "c") - - set := selectAt(t, wrapped, "1.2.0", "test.counter") - got := collectSeries(t, set) - - require.Equal(t, map[string]float64{ - labels.FromStrings(model.MetricNameLabel, "test.counter", "user", "a").String(): 1, - }, got) - requireWarningsContain(t, warningStrings(set.Warnings()), "metric lifecycle boundary") -} diff --git a/storage/semconv/testdata/otel.yaml b/storage/semconv/testdata/otel.yaml deleted file mode 100644 index 252c48d561..0000000000 --- a/storage/semconv/testdata/otel.yaml +++ /dev/null @@ -1,40 +0,0 @@ -file_format: "1.1.0" -schema_url: https://example.com/schemas/1.0.0 - -versions: - 1.0.0: - metrics: - changes: - - rename_attributes: - attribute_map: - http.method: http.request.method - http.status_code: http.response.status_code - url.path: url.full - apply_to_metrics: - - http.server.duration - - http.server.request.count - - - rename_attributes: - attribute_map: - process.cpu.state: cpu.mode - apply_to_metrics: - - process.cpu.time - -groups: - - id: metric.http.server.duration - type: metric - metric_name: http.server.duration - instrument: histogram - unit: s - - - id: metric.http.server.request.count - type: metric - metric_name: http.server.request.count - instrument: counter - unit: "{request}" - - - id: metric.process.cpu.time - type: metric - metric_name: process.cpu.time - instrument: counter - unit: s diff --git a/storage/semconv/testdata/otel_unsupported_format.yaml b/storage/semconv/testdata/otel_unsupported_format.yaml deleted file mode 100644 index bb046a91a4..0000000000 --- a/storage/semconv/testdata/otel_unsupported_format.yaml +++ /dev/null @@ -1,7 +0,0 @@ -file_format: "2.0.0" -schema_url: https://example.com/schemas/2.0.0 - -versions: - 2.0.0: - metrics: - changes: [] diff --git a/storage/semconv/testdata/otel_with_all_section.yaml b/storage/semconv/testdata/otel_with_all_section.yaml deleted file mode 100644 index 0c89fde726..0000000000 --- a/storage/semconv/testdata/otel_with_all_section.yaml +++ /dev/null @@ -1,17 +0,0 @@ -file_format: "1.1.0" -schema_url: https://example.com/schemas/1.0.0 - -versions: - 1.0.0: - all: - changes: - - rename_attributes: - attribute_map: - global.old: global.new - metrics: - changes: - - rename_attributes: - attribute_map: - metric.old: metric.new - apply_to_metrics: - - my.metric diff --git a/storage/semconv/testdata/otel_with_chained_renames.yaml b/storage/semconv/testdata/otel_with_chained_renames.yaml deleted file mode 100644 index e951f1e4d4..0000000000 --- a/storage/semconv/testdata/otel_with_chained_renames.yaml +++ /dev/null @@ -1,20 +0,0 @@ -file_format: "1.1.0" -schema_url: https://example.com/schemas/1.1.0 - -versions: - 1.0.0: - metrics: - changes: - - rename_metrics: - metric.v1: metric.v2 - - rename_attributes: - attribute_map: - attr.v1: attr.v2 - 1.1.0: - metrics: - changes: - - rename_metrics: - metric.v2: metric.v3 - - rename_attributes: - attribute_map: - attr.v2: attr.v3 diff --git a/storage/semconv/testdata/otel_with_metric_renames.yaml b/storage/semconv/testdata/otel_with_metric_renames.yaml deleted file mode 100644 index 2cf0dadaec..0000000000 --- a/storage/semconv/testdata/otel_with_metric_renames.yaml +++ /dev/null @@ -1,10 +0,0 @@ -file_format: "1.1.0" -schema_url: https://example.com/schemas/1.0.0 - -versions: - 1.0.0: - metrics: - changes: - - rename_metrics: - old.metric.name: new.metric.name - another.old.metric: another.new.metric diff --git a/storage/semconv/testdata/upstream/schema-1.44.0.yaml b/storage/semconv/testdata/upstream/schema-1.44.0.yaml deleted file mode 100644 index 37024fa888..0000000000 --- a/storage/semconv/testdata/upstream/schema-1.44.0.yaml +++ /dev/null @@ -1,766 +0,0 @@ -file_format: 1.1.0 -schema_url: https://opentelemetry.io/schemas/1.44.0 -versions: - 1.44.0: - metrics: - changes: - - rename_metrics: - container.memory.paging.faults: container.paging.faults - k8s.node.memory.paging.faults: k8s.node.paging.faults - k8s.pod.memory.paging.faults: k8s.pod.paging.faults - 1.43.0: - 1.42.0: - metrics: - changes: - - rename_metrics: - v8js.memory.heap.limit: v8js.memory.heap.space.size - 1.41.1: - 1.41.0: - metrics: - changes: - - rename_metrics: - k8s.container.cpu.limit: k8s.container.cpu.limit.desired - k8s.container.cpu.limit_utilization: k8s.container.cpu.limit.utilization - k8s.container.cpu.request: k8s.container.cpu.request.desired - k8s.container.cpu.request_utilization: k8s.container.cpu.request.utilization - k8s.container.memory.limit: k8s.container.memory.limit.desired - k8s.container.memory.request: k8s.container.memory.request.desired - 1.40.0: - all: - changes: - - rename_attributes: - attribute_map: - feature_flag.evaluation.error.message: feature_flag.error.message - metrics: - changes: - - rename_metrics: - system.memory.shared: system.memory.linux.shared - 1.39.0: - all: - changes: - - rename_attributes: - attribute_map: - linux.memory.slab.state: system.memory.linux.slab.state - peer.service: service.peer.name - rpc.connect_rpc.error_code: rpc.response.status_code - rpc.connect_rpc.request.metadata: rpc.request.metadata - rpc.connect_rpc.response.metadata: rpc.response.metadata - rpc.grpc.request.metadata: rpc.request.metadata - rpc.grpc.response.metadata: rpc.response.metadata - rpc.jsonrpc.request_id: jsonrpc.request.id - rpc.jsonrpc.version: jsonrpc.protocol.version - rpc.system: rpc.system.name - metrics: - changes: - - rename_metrics: - process.open_file_descriptor.count: process.unix.file_descriptor.count - system.linux.memory.available: system.memory.linux.available - system.linux.memory.slab.usage: system.memory.linux.slab.usage - 1.38.0: - all: - changes: - - rename_attributes: - attribute_map: - process.context_switch_type: process.context_switch.type - process.paging.fault_type: system.paging.fault.type - system.cpu.logical_number: cpu.logical_number - system.paging.type: system.paging.fault.type - system.process.status: process.state - system.processes.status: process.state - metrics: - changes: - - rename_metrics: - k8s.cronjob.active_jobs: k8s.cronjob.job.active - k8s.daemonset.current_scheduled_nodes: k8s.daemonset.node.current_scheduled - k8s.daemonset.desired_scheduled_nodes: k8s.daemonset.node.desired_scheduled - k8s.daemonset.misscheduled_nodes: k8s.daemonset.node.misscheduled - k8s.daemonset.ready_nodes: k8s.daemonset.node.ready - k8s.deployment.available_pods: k8s.deployment.pod.available - k8s.deployment.desired_pods: k8s.deployment.pod.desired - k8s.hpa.current_pods: k8s.hpa.pod.current - k8s.hpa.desired_pods: k8s.hpa.pod.desired - k8s.hpa.max_pods: k8s.hpa.pod.max - k8s.hpa.min_pods: k8s.hpa.pod.min - k8s.job.active_pods: k8s.job.pod.active - k8s.job.desired_successful_pods: k8s.job.pod.desired_successful - k8s.job.failed_pods: k8s.job.pod.failed - k8s.job.max_parallel_pods: k8s.job.pod.max_parallel - k8s.job.successful_pods: k8s.job.pod.successful - k8s.node.allocatable.cpu: k8s.node.cpu.allocatable - k8s.node.allocatable.ephemeral_storage: k8s.node.ephemeral_storage.allocatable - k8s.node.allocatable.memory: k8s.node.memory.allocatable - k8s.node.allocatable.pods: k8s.node.pod.allocatable - k8s.replicaset.available_pods: k8s.replicaset.pod.available - k8s.replicaset.desired_pods: k8s.replicaset.pod.desired - k8s.replication_controller.available_pods: k8s.replicationcontroller.pod.available - k8s.replication_controller.desired_pods: k8s.replicationcontroller.pod.desired - k8s.replicationcontroller.available_pods: k8s.replicationcontroller.pod.available - k8s.replicationcontroller.desired_pods: k8s.replicationcontroller.pod.desired - k8s.statefulset.current_pods: k8s.statefulset.pod.current - k8s.statefulset.desired_pods: k8s.statefulset.pod.desired - k8s.statefulset.ready_pods: k8s.statefulset.pod.ready - k8s.statefulset.updated_pods: k8s.statefulset.pod.updated - v8js.heap.space.available_size: v8js.memory.heap.space.available_size - v8js.heap.space.physical_size: v8js.memory.heap.space.physical_size - 1.37.0: - all: - changes: - - rename_attributes: - attribute_map: - android.state: android.app.state - container.runtime: container.runtime.name - enduser.role: user.roles - gen_ai.openai.request.service_tier: openai.request.service_tier - gen_ai.openai.response.service_tier: openai.response.service_tier - gen_ai.openai.response.system_fingerprint: openai.response.system_fingerprint - gen_ai.system: gen_ai.provider.name - ios.state: ios.app.state - 1.36.0: - 1.35.0: - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1698 - - rename_attributes: - attribute_map: - az.namespace: azure.resource_provider.namespace - az.service_request_id: azure.service.request.id - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/issues/1800 - - rename_metrics: - system.network.connections: system.network.connection.count - 1.34.0: - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/2295 - - rename_metrics: - cpu.time: system.cpu.time - cpu.utilization: system.cpu.utilization - cpu.frequency: system.cpu.frequency - 1.33.0: - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1982 - - rename_attributes: - attribute_map: - feature_flag.provider_name: feature_flag.provider.name - # https://github.com/open-telemetry/semantic-conventions/pull/1994 - - rename_attributes: - attribute_map: - feature_flag.evaluation.error.message: error.message - 1.32.0: - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1989 - - rename_attributes: - attribute_map: - feature_flag.evaluation.reason: feature_flag.result.reason - feature_flag.variant: feature_flag.result.variant - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/2042 - - rename_metrics: - otel.sdk.span.live.count: otel.sdk.span.live - otel.sdk.span.ended.count: otel.sdk.span.ended - otel.sdk.processor.span.processed.count: otel.sdk.processor.span.processed - otel.sdk.exporter.span.inflight.count: otel.sdk.exporter.span.inflight - otel.sdk.exporter.span.exported.count: otel.sdk.exporter.span.exported - 1.31.0: - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1880 - - rename_attributes: - attribute_map: - android.state: android.app.state - io.state: ios.app.state - metrics: - changes: - - rename_metrics: - k8s.replication_controller.desired_pods: k8s.replicationcontroller.desired_pods - k8s.replication_controller.available_pods: k8s.replicationcontroller.available_pods - # https://github.com/open-telemetry/semantic-conventions/pull/1896 - - rename_metrics: - system.cpu.time: cpu.time - system.cpu.utilization: cpu.utilization - system.cpu.frequency: cpu.frequency - # https://github.com/open-telemetry/semantic-conventions/pull/1896 - - rename_attributes: - attribute_map: - system.cpu.logical_number: cpu.logical_number - 1.30.0: - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1632 - - rename_attributes: - attribute_map: - gen_ai.openai.request.seed: gen_ai.request.seed - system.network.state: network.connection.state - # https://github.com/open-telemetry/semantic-conventions/pull/1624 - - rename_attributes: - attribute_map: - code.function: code.function.name - code.filepath: code.file.path - code.lineno: code.line.number - code.column: code.column.number - # https://github.com/open-telemetry/semantic-conventions/pull/1734 - - rename_attributes: - attribute_map: - db.system: db.system.name - db.cassandra.coordinator.dc: cassandra.coordinator.dc - db.cassandra.coordinator.id: cassandra.coordinator.id - db.cassandra.consistency_level: cassandra.consistency.level - db.cassandra.idempotence: cassandra.query.idempotent - db.cassandra.page_size: cassandra.page.size - db.cassandra.speculative_execution_count: cassandra.speculative_execution.count - db.cosmosdb.client_id: azure.client.id - db.cosmosdb.connection_mode: azure.cosmosdb.connection.mode - db.cosmosdb.consistency_level: azure.cosmosdb.consistency.level - db.cosmosdb.request_charge: azure.cosmosdb.operation.request_charge - db.cosmosdb.request_content_length: azure.cosmosdb.request.body.size - db.cosmosdb.regions_contacted: azure.cosmosdb.operation.contacted_regions - db.cosmosdb.sub_status_code: azure.cosmosdb.response.sub_status_code - db.elasticsearch.node.name: elasticsearch.node.name - # db.elasticsearch.path_parts is a template attribute, schema transformation - # does not support it, adding as a comment for consistency - # db.elasticsearch.path_parts. -> db.operation.parameter. - metrics: - changes: - - rename_metrics: - db.client.cosmosdb.operation.request_charge: azure.cosmosdb.client.operation.request_charge - db.client.cosmosdb.active_instance.count: azure.cosmosdb.client.active_instance.count - - 1.29.0: - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1520 - - rename_attributes: - attribute_map: - process.executable.build_id.profiling: process.executable.build_id.htlhash - # https://github.com/open-telemetry/semantic-conventions/pull/1383 - - rename_attributes: - attribute_map: - vcs.repository.change.id: vcs.change.id - vcs.repository.change.title: vcs.change.title - vcs.repository.ref.name: vcs.ref.head.name - vcs.repository.ref.revision: vcs.ref.head.revision - vcs.repository.ref.type: vcs.ref.head.type - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1492 - - rename_attributes: - attribute_map: - system.device: network.interface.name - apply_to_metrics: - - container.network.io - - system.network.dropped - - system.network.errors - - system.network.io - - system.network.connections - 1.28.0: - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1422 - - rename_metrics: - messaging.client.published.messages: messaging.client.sent.messages - 1.27.0: - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1216 - - rename_attributes: - attribute_map: - tls.client.server_name: server.address - # https://github.com/open-telemetry/semantic-conventions/pull/1075 - - rename_attributes: - attribute_map: - deployment.environment: deployment.environment.name - # https://github.com/open-telemetry/semantic-conventions/pull/1245 - - rename_attributes: - attribute_map: - messaging.kafka.message.offset: messaging.kafka.offset - # https://github.com/open-telemetry/semantic-conventions/pull/815 - - rename_attributes: - attribute_map: - messaging.kafka.consumer.group: messaging.consumer.group.name - messaging.rocketmq.client_group: messaging.consumer.group.name - messaging.eventhubs.consumer.group: messaging.consumer.group.name - messaging.servicebus.destination.subscription_name: messaging.destination.subscription.name - # https://github.com/open-telemetry/semantic-conventions/pull/1200 - - rename_attributes: - attribute_map: - gen_ai.usage.completion_tokens: gen_ai.usage.output_tokens - gen_ai.usage.prompt_tokens: gen_ai.usage.input_tokens - spans: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1002 - - rename_attributes: - attribute_map: - db.elasticsearch.cluster.name: db.namespace - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/1125 - - rename_attributes: - attribute_map: - db.client.connections.state: db.client.connection.state - apply_to_metrics: - - db.client.connection.count - - rename_attributes: - attribute_map: - db.client.connections.pool.name: db.client.connection.pool.name - apply_to_metrics: - - db.client.connection.count - - db.client.connection.idle.max - - db.client.connection.idle.min - - db.client.connection.max - - db.client.connection.pending_requests - - db.client.connection.timeouts - - db.client.connection.create_time - - db.client.connection.wait_time - - db.client.connection.use_time - # https://github.com/open-telemetry/semantic-conventions/pull/1006 - - rename_metrics: - messaging.publish.messages: messaging.client.published.messages - # https://github.com/open-telemetry/semantic-conventions/pull/1026 - - rename_attributes: - attribute_map: - system.cpu.state: cpu.mode - process.cpu.state: cpu.mode - container.cpu.state: cpu.mode - apply_to_metrics: - - system.cpu.time - - system.cpu.utilization - - process.cpu.time - - process.cpu.utilization - - container.cpu.time - # https://github.com/open-telemetry/semantic-conventions/pull/1265 - - rename_metrics: - jvm.buffer.memory.usage: jvm.buffer.memory.used - 1.26.0: - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/966 - - rename_metrics: - db.client.connections.usage: db.client.connection.count - db.client.connections.idle.max: db.client.connection.idle.max - db.client.connections.idle.min: db.client.connection.idle.min - db.client.connections.max: db.client.connection.max - db.client.connections.pending_requests: db.client.connection.pending_requests - db.client.connections.timeouts: db.client.connection.timeouts - # https://github.com/open-telemetry/semantic-conventions/pull/948 - - rename_attributes: - attribute_map: - messaging.client_id: messaging.client.id - # https://github.com/open-telemetry/semantic-conventions/pull/909 - - rename_attributes: - attribute_map: - state: db.client.connections.state - apply_to_metrics: - - db.client.connections.usage - - rename_attributes: - attribute_map: - pool.name: db.client.connections.pool.name - apply_to_metrics: - - db.client.connections.usage - - db.client.connections.idle.max - - db.client.connections.idle.min - - db.client.connections.max - - db.client.connections.pending_requests - - db.client.connections.timeouts - - db.client.connections.create_time - - db.client.connections.wait_time - - db.client.connections.use_time - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/731 - - rename_attributes: - attribute_map: - enduser.id: user.id - - 1.25.0: - spans: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/911 - - rename_attributes: - attribute_map: - db.name: db.namespace - # https://github.com/open-telemetry/semantic-conventions/pull/870 - - rename_attributes: - attribute_map: - db.sql.table: db.collection.name - db.mongodb.collection: db.collection.name - db.cosmosdb.container: db.collection.name - db.cassandra.table: db.collection.name - # https://github.com/open-telemetry/semantic-conventions/pull/798 - - rename_attributes: - attribute_map: - messaging.kafka.destination.partition: messaging.destination.partition.id - # https://github.com/open-telemetry/semantic-conventions/pull/875 - - rename_attributes: - attribute_map: - db.operation: db.operation.name - # https://github.com/open-telemetry/semantic-conventions/pull/913 - - rename_attributes: - attribute_map: - messaging.operation: messaging.operation.type - # https://github.com/open-telemetry/semantic-conventions/pull/866 - - rename_attributes: - attribute_map: - db.statement: db.query.text - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/484 - - rename_attributes: - attribute_map: - system.processes.status: system.process.status - apply_to_metrics: - - system.processes.count - - rename_metrics: - system.processes.count: system.process.count - system.processes.created: system.process.created - # https://github.com/open-telemetry/semantic-conventions/pull/625 - - rename_attributes: - attribute_map: - container.labels: container.label - k8s.pod.labels: k8s.pod.label - # https://github.com/open-telemetry/semantic-conventions/pull/330 - - rename_metrics: - process.threads: process.thread.count - process.open_file_descriptors: process.open_file_descriptor.count - - rename_attributes: - attribute_map: - state: process.cpu.state - apply_to_metrics: - - process.cpu.time - - process.cpu.utilization - - rename_attributes: - attribute_map: - direction: disk.io.direction - apply_to_metrics: - - process.disk.io - - rename_attributes: - attribute_map: - type: process.context_switch_type - apply_to_metrics: - - process.context_switches - - rename_attributes: - attribute_map: - direction: network.io.direction - apply_to_metrics: - - process.network.io - - rename_attributes: - attribute_map: - type: process.paging.fault_type - apply_to_metrics: - - process.paging.faults - all: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/854 - - rename_attributes: - attribute_map: - message.type: rpc.message.type - message.id: rpc.message.id - message.compressed_size: rpc.message.compressed_size - message.uncompressed_size: rpc.message.uncompressed_size - - 1.24.0: - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/536 - - rename_metrics: - jvm.memory.usage: jvm.memory.used - jvm.memory.usage_after_last_gc: jvm.memory.used_after_last_gc - # https://github.com/open-telemetry/semantic-conventions/pull/530 - - rename_attributes: - attribute_map: - system.network.io.direction: network.io.direction - system.disk.io.direction: disk.io.direction - 1.23.1: - 1.23.0: - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/20 - - rename_attributes: - attribute_map: - thread.daemon: jvm.thread.daemon - apply_to_metrics: - - jvm.thread.count - 1.22.0: - spans: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/229 - - rename_attributes: - attribute_map: - messaging.message.payload_size_bytes: messaging.message.body.size - # https://github.com/open-telemetry/opentelemetry-specification/pull/374 - - rename_attributes: - attribute_map: - http.resend_count: http.request.resend_count - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/224 - - rename_metrics: - http.client.duration: http.client.request.duration - http.server.duration: http.server.request.duration - # https://github.com/open-telemetry/semantic-conventions/pull/241 - - rename_metrics: - process.runtime.jvm.memory.usage: jvm.memory.usage - process.runtime.jvm.memory.committed: jvm.memory.committed - process.runtime.jvm.memory.limit: jvm.memory.limit - process.runtime.jvm.memory.usage_after_last_gc: jvm.memory.usage_after_last_gc - process.runtime.jvm.gc.duration: jvm.gc.duration - # also https://github.com/open-telemetry/semantic-conventions/pull/252 - process.runtime.jvm.threads.count: jvm.thread.count - # also https://github.com/open-telemetry/semantic-conventions/pull/252 - process.runtime.jvm.classes.loaded: jvm.class.loaded - # also https://github.com/open-telemetry/semantic-conventions/pull/252 - process.runtime.jvm.classes.unloaded: jvm.class.unloaded - # also https://github.com/open-telemetry/semantic-conventions/pull/252 - # and https://github.com/open-telemetry/semantic-conventions/pull/60 - process.runtime.jvm.classes.current_loaded: jvm.class.count - process.runtime.jvm.cpu.time: jvm.cpu.time - process.runtime.jvm.cpu.recent_utilization: jvm.cpu.recent_utilization - process.runtime.jvm.memory.init: jvm.memory.init - process.runtime.jvm.system.cpu.utilization: jvm.system.cpu.utilization - process.runtime.jvm.system.cpu.load_1m: jvm.system.cpu.load_1m - # https://github.com/open-telemetry/semantic-conventions/pull/253 - process.runtime.jvm.buffer.usage: jvm.buffer.memory.usage - # https://github.com/open-telemetry/semantic-conventions/pull/253 - process.runtime.jvm.buffer.limit: jvm.buffer.memory.limit - process.runtime.jvm.buffer.count: jvm.buffer.count - # https://github.com/open-telemetry/semantic-conventions/pull/20 - - rename_attributes: - attribute_map: - type: jvm.memory.type - pool: jvm.memory.pool.name - apply_to_metrics: - - jvm.memory.usage - - jvm.memory.committed - - jvm.memory.limit - - jvm.memory.usage_after_last_gc - - jvm.memory.init - - rename_attributes: - attribute_map: - name: jvm.gc.name - action: jvm.gc.action - apply_to_metrics: - - jvm.gc.duration - - rename_attributes: - attribute_map: - daemon: thread.daemon - apply_to_metrics: - - jvm.threads.count - - rename_attributes: - attribute_map: - pool: jvm.buffer.pool.name - apply_to_metrics: - - jvm.buffer.memory.usage - - jvm.buffer.memory.limit - - jvm.buffer.count - # https://github.com/open-telemetry/semantic-conventions/pull/89 - - rename_attributes: - attribute_map: - state: system.cpu.state - cpu: system.cpu.logical_number - apply_to_metrics: - - system.cpu.time - - system.cpu.utilization - - rename_attributes: - attribute_map: - state: system.memory.state - apply_to_metrics: - - system.memory.usage - - system.memory.utilization - - rename_attributes: - attribute_map: - state: system.paging.state - apply_to_metrics: - - system.paging.usage - - system.paging.utilization - - rename_attributes: - attribute_map: - type: system.paging.type - direction: system.paging.direction - apply_to_metrics: - - system.paging.faults - - system.paging.operations - - rename_attributes: - attribute_map: - device: system.device - direction: system.disk.direction - apply_to_metrics: - - system.disk.io - - system.disk.operations - - system.disk.io_time - - system.disk.operation_time - - system.disk.merged - - rename_attributes: - attribute_map: - device: system.device - state: system.filesystem.state - type: system.filesystem.type - mode: system.filesystem.mode - mountpoint: system.filesystem.mountpoint - apply_to_metrics: - - system.filesystem.usage - - system.filesystem.utilization - - rename_attributes: - attribute_map: - device: system.device - direction: system.network.direction - protocol: network.protocol - state: system.network.state - apply_to_metrics: - - system.network.dropped - - system.network.packets - - system.network.errors - - system.network.io - - system.network.connections - - rename_attributes: - attribute_map: - status: system.processes.status - apply_to_metrics: - - system.processes.count - # https://github.com/open-telemetry/semantic-conventions/pull/247 - - rename_metrics: - http.server.request.size: http.server.request.body.size - http.server.response.size: http.server.response.body.size - resources: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/178 - - rename_attributes: - attribute_map: - telemetry.auto.version: telemetry.distro.version - 1.21.0: - spans: - changes: - # https://github.com/open-telemetry/opentelemetry-specification/pull/3336 - - rename_attributes: - attribute_map: - messaging.kafka.client_id: messaging.client_id - messaging.rocketmq.client_id: messaging.client_id - # https://github.com/open-telemetry/opentelemetry-specification/pull/3402 - - rename_attributes: - attribute_map: - # net.peer.(name|port) attributes were usually populated on client side - # so they should be usually translated to server.(address|port) - # net.host.* attributes were only populated on server side - net.host.name: server.address - net.host.port: server.port - # was only populated on client side - net.sock.peer.name: server.socket.domain - # net.sock.peer.(addr|port) mapping is not possible - # since they applied to both client and server side - # were only populated on server side - net.sock.host.addr: server.socket.address - net.sock.host.port: server.socket.port - http.client_ip: client.address - # https://github.com/open-telemetry/opentelemetry-specification/pull/3426 - - rename_attributes: - attribute_map: - net.protocol.name: network.protocol.name - net.protocol.version: network.protocol.version - net.host.connection.type: network.connection.type - net.host.connection.subtype: network.connection.subtype - net.host.carrier.name: network.carrier.name - net.host.carrier.mcc: network.carrier.mcc - net.host.carrier.mnc: network.carrier.mnc - net.host.carrier.icc: network.carrier.icc - # https://github.com/open-telemetry/opentelemetry-specification/pull/3355 - - rename_attributes: - attribute_map: - http.method: http.request.method - http.status_code: http.response.status_code - http.scheme: url.scheme - http.url: url.full - http.request_content_length: http.request.body.size - http.response_content_length: http.response.body.size - metrics: - changes: - # https://github.com/open-telemetry/semantic-conventions/pull/53 - - rename_metrics: - process.runtime.jvm.cpu.utilization: process.runtime.jvm.cpu.recent_utilization - 1.20.0: - spans: - changes: - # https://github.com/open-telemetry/opentelemetry-specification/pull/3272 - - rename_attributes: - attribute_map: - net.app.protocol.name: net.protocol.name - net.app.protocol.version: net.protocol.version - 1.19.0: - spans: - changes: - # https://github.com/open-telemetry/opentelemetry-specification/pull/3209 - - rename_attributes: - attribute_map: - faas.execution: faas.invocation_id - # https://github.com/open-telemetry/opentelemetry-specification/pull/3188 - - rename_attributes: - attribute_map: - faas.id: cloud.resource_id - # https://github.com/open-telemetry/opentelemetry-specification/pull/3190 - - rename_attributes: - attribute_map: - http.user_agent: user_agent.original - resources: - changes: - # https://github.com/open-telemetry/opentelemetry-specification/pull/3190 - - rename_attributes: - attribute_map: - browser.user_agent: user_agent.original - 1.18.0: - 1.17.0: - spans: - changes: - # https://github.com/open-telemetry/opentelemetry-specification/pull/2957 - - rename_attributes: - attribute_map: - messaging.consumer_id: messaging.consumer.id - messaging.protocol: net.app.protocol.name - messaging.protocol_version: net.app.protocol.version - messaging.destination: messaging.destination.name - messaging.temp_destination: messaging.destination.temporary - messaging.destination_kind: messaging.destination.kind - messaging.message_id: messaging.message.id - messaging.conversation_id: messaging.message.conversation_id - messaging.message_payload_size_bytes: messaging.message.payload_size_bytes - messaging.message_payload_compressed_size_bytes: messaging.message.payload_compressed_size_bytes - messaging.rabbitmq.routing_key: messaging.rabbitmq.destination.routing_key - messaging.kafka.message_key: messaging.kafka.message.key - messaging.kafka.partition: messaging.kafka.destination.partition - messaging.kafka.tombstone: messaging.kafka.message.tombstone - messaging.rocketmq.message_type: messaging.rocketmq.message.type - messaging.rocketmq.message_tag: messaging.rocketmq.message.tag - messaging.rocketmq.message_keys: messaging.rocketmq.message.keys - messaging.kafka.consumer_group: messaging.kafka.consumer.group - 1.16.0: - 1.15.0: - spans: - changes: - # https://github.com/open-telemetry/opentelemetry-specification/pull/2743 - - rename_attributes: - attribute_map: - http.retry_count: http.resend_count - 1.14.0: - 1.13.0: - spans: - changes: - # https://github.com/open-telemetry/opentelemetry-specification/pull/2614 - - rename_attributes: - attribute_map: - net.peer.ip: net.sock.peer.addr - net.host.ip: net.sock.host.addr - 1.12.0: - 1.11.0: - 1.10.0: - 1.9.0: - 1.8.0: - spans: - changes: - - rename_attributes: - attribute_map: - db.cassandra.keyspace: db.name - db.hbase.namespace: db.name - 1.7.0: - 1.6.1: - 1.5.0: - 1.4.0: diff --git a/storage/semconv/testdata/upstream/semconv-1.21.0.yaml b/storage/semconv/testdata/upstream/semconv-1.21.0.yaml deleted file mode 100644 index 577e8695df..0000000000 --- a/storage/semconv/testdata/upstream/semconv-1.21.0.yaml +++ /dev/null @@ -1,293 +0,0 @@ -# Verbatim excerpts of the HTTP attribute and metric groups from -# open-telemetry/semantic-conventions v1.21.0 : model/http-common.yaml -# open-telemetry/semantic-conventions v1.21.0 : model/metrics/http.yaml -# open-telemetry/semantic-conventions v1.21.0 : model/metrics/process-runtime-jvm-metrics-experimental.yaml -# Combined under one groups key so inheritance is exercised exactly as it is in -# the upstream registry. -groups: - - id: attributes.http.common - type: attribute_group - brief: "Describes HTTP attributes." - prefix: http - attributes: - - id: request.method - type: - allow_custom_values: true - members: - - id: connect - value: "CONNECT" - brief: 'CONNECT method.' - - id: delete - value: "DELETE" - brief: 'DELETE method.' - - id: get - value: "GET" - brief: 'GET method.' - - id: head - value: "HEAD" - brief: 'HEAD method.' - - id: options - value: "OPTIONS" - brief: 'OPTIONS method.' - - id: patch - value: "PATCH" - brief: 'PATCH method.' - - id: post - value: "POST" - brief: 'POST method.' - - id: put - value: "PUT" - brief: 'PUT method.' - - id: trace - value: "TRACE" - brief: 'TRACE method.' - - id: other - value: "_OTHER" - brief: 'Any HTTP method that the instrumentation has no prior knowledge of.' - requirement_level: required - brief: 'HTTP request method.' - examples: ["GET", "POST", "HEAD"] - note: | - HTTP request method value SHOULD be "known" to the instrumentation. - By default, this convention defines "known" methods as the ones listed in [RFC9110](https://www.rfc-editor.org/rfc/rfc9110.html#name-methods) - and the PATCH method defined in [RFC5789](https://www.rfc-editor.org/rfc/rfc5789.html). - - If the HTTP request method is not known to instrumentation, it MUST set the `http.request.method` attribute to `_OTHER` and, except if reporting a metric, MUST - set the exact method received in the request line as value of the `http.request.method_original` attribute. - - If the HTTP instrumentation could end up converting valid HTTP request methods to `_OTHER`, then it MUST provide a way to override - the list of known HTTP methods. If this override is done via environment variable, then the environment variable MUST be named - OTEL_INSTRUMENTATION_HTTP_KNOWN_METHODS and support a comma-separated list of case-sensitive known HTTP methods - (this list MUST be a full override of the default known method, it is not a list of known methods in addition to the defaults). - - HTTP method names are case-sensitive and `http.request.method` attribute value MUST match a known HTTP method name exactly. - Instrumentations for specific web frameworks that consider HTTP methods to be case insensitive, SHOULD populate a canonical equivalent. - Tracing instrumentations that do so, MUST also set `http.request.method_original` to the original value. - - id: response.status_code - type: int - requirement_level: - conditionally_required: If and only if one was received/sent. - brief: '[HTTP response status code](https://tools.ietf.org/html/rfc7231#section-6).' - examples: [200] - - ref: network.protocol.name - examples: ['http', 'spdy'] - requirement_level: - recommended: if not default (`http`). - - ref: network.protocol.version - examples: ['1.0', '1.1', '2.0'] - - - id: attributes.http.client - prefix: http - type: attribute_group - brief: 'HTTP Client attributes' - attributes: - - ref: server.address - requirement_level: required - brief: > - Host identifier of the ["URI origin"](https://www.rfc-editor.org/rfc/rfc9110.html#name-uri-origin) HTTP request is sent to. - note: | - Determined by using the first of the following that applies - - - Host identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form - - Host identifier of the `Host` header - - SHOULD NOT be set if capturing it would require an extra DNS lookup. - - ref: server.port - requirement_level: - conditionally_required: If not default (`80` for `http` scheme, `443` for `https`). - brief: > - Port identifier of the ["URI origin"](https://www.rfc-editor.org/rfc/rfc9110.html#name-uri-origin) HTTP request is sent to. - note: > - When [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) is absolute URI, `server.port` MUST match - URI port identifier, otherwise it MUST match `Host` header port identifier. - - - id: attributes.http.server - prefix: http - type: attribute_group - brief: 'HTTP Server attributes' - attributes: - - id: route - type: string - requirement_level: - conditionally_required: If and only if it's available - brief: > - The matched route (path template in the format used by the respective server framework). See note below - examples: ['/users/:userID?', '{controller}/{action}/{id?}'] - note: > - MUST NOT be populated when this is not supported by the HTTP server framework as the route attribute should have low-cardinality and the URI path can NOT substitute it. - - SHOULD include the [application root](/docs/http/http-spans.md#http-server-definitions) if there is one. - - ref: server.address - brief: > - Name of the local HTTP server that received the request. - note: | - Determined by using the first of the following that applies - - - The [primary server name](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. MUST only - include host identifier. - - Host identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Host identifier of the `Host` header - - SHOULD NOT be set if only IP address is available and capturing name would require a reverse DNS lookup. - - - ref: server.port - brief: > - Port of the local HTTP server that received the request. - note: | - Determined by using the first of the following that applies - - - Port identifier of the [primary server host](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. - - Port identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Port identifier of the `Host` header - - ref: url.scheme - requirement_level: required - examples: ["http", "https"] - - id: metric_attributes.http.server - type: attribute_group - brief: 'HTTP server attributes' - extends: attributes.http.server - attributes: - - ref: server.address - brief: > - Name of the local HTTP server that received the request. - requirement_level: opt_in - note: | - Determined by using the first of the following that applies - - - The [primary server name](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. MUST only - include host identifier. - - Host identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Host identifier of the `Host` header - - SHOULD NOT be set if only IP address is available and capturing name would require a reverse DNS lookup. - - - ref: server.port - brief: > - Port of the local HTTP server that received the request. - requirement_level: opt_in - note: | - Determined by using the first of the following that applies - - - Port identifier of the [primary server host](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. - - Port identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Port identifier of the `Host` header - # todo (lmolkova) build tools don't populate grandparent attributes - - ref: http.request.method - - ref: http.response.status_code - - ref: network.protocol.name - - ref: network.protocol.version - - - id: metric_attributes.http.client - type: attribute_group - brief: 'HTTP client attributes' - extends: attributes.http.client - attributes: - # todo (lmolkova) build tools don't populate grandparent attributes - - ref: http.request.method - - ref: http.response.status_code - - ref: network.protocol.name - - ref: network.protocol.version - - ref: server.socket.address - - - id: metric.http.server.duration - type: metric - metric_name: http.server.duration - brief: "Measures the duration of inbound HTTP requests." - instrument: histogram - unit: "s" - extends: metric_attributes.http.server - - - id: metric.http.server.active_requests - type: metric - metric_name: http.server.active_requests - brief: "Measures the number of concurrent HTTP requests that are currently in-flight." - instrument: updowncounter - unit: "{request}" - attributes: - - ref: http.request.method - - ref: url.scheme - requirement_level: required - examples: ["http", "https"] - - ref: server.address - requirement_level: opt_in - brief: > - Name of the local HTTP server that received the request. - note: | - Determined by using the first of the following that applies - - - The [primary server name](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. MUST only - include host identifier. - - Host identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Host identifier of the `Host` header - - SHOULD NOT be set if only IP address is available and capturing name would require a reverse DNS lookup. - - - ref: server.port - requirement_level: opt_in - brief: > - Port of the local HTTP server that received the request. - note: | - Determined by using the first of the following that applies - - - Port identifier of the [primary server host](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. - - Port identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Port identifier of the `Host` header - - - id: metric.http.server.request.size - type: metric - metric_name: http.server.request.size - brief: "Measures the size of HTTP request messages (compressed)." - instrument: histogram - unit: "By" - extends: metric_attributes.http.server - - - id: metric.http.server.response.size - type: metric - metric_name: http.server.response.size - brief: "Measures the size of HTTP response messages (compressed)." - instrument: histogram - unit: "By" - extends: metric_attributes.http.server - - - id: metric.http.client.duration - type: metric - metric_name: http.client.duration - brief: "Measures the duration of outbound HTTP requests." - instrument: histogram - unit: "s" - extends: metric_attributes.http.client - - - id: metric.http.client.request.size - type: metric - metric_name: http.client.request.size - brief: "Measures the size of HTTP request messages (compressed)." - instrument: histogram - unit: "By" - extends: metric_attributes.http.client - - - id: metric.http.client.response.size - type: metric - metric_name: http.client.response.size - brief: "Measures the size of HTTP response messages (compressed)." - instrument: histogram - unit: "By" - extends: metric_attributes.http.client - - - id: metric.process.runtime.jvm.system.cpu.load_1m - type: metric - metric_name: process.runtime.jvm.system.cpu.load_1m - brief: "Average CPU load of the whole system for the last minute as reported by the JVM." - note: > - The value range is [0,n], where n is the number of CPU cores - or a negative number if the value is not available. - This utilization is not defined as being for the specific interval since last measurement - (unlike `system.cpu.utilization`). - [Reference](https://docs.oracle.com/en/java/javase/17/docs/api/java.management/java/lang/management/OperatingSystemMXBean.html#getSystemLoadAverage()). - instrument: gauge - unit: "1" diff --git a/storage/semconv/testdata/upstream/semconv-1.22.0.yaml b/storage/semconv/testdata/upstream/semconv-1.22.0.yaml deleted file mode 100644 index c20164e89b..0000000000 --- a/storage/semconv/testdata/upstream/semconv-1.22.0.yaml +++ /dev/null @@ -1,239 +0,0 @@ -# Verbatim excerpts of the HTTP attribute and metric groups from -# open-telemetry/semantic-conventions v1.22.0 : model/http-common.yaml -# open-telemetry/semantic-conventions v1.22.0 : model/metrics/http.yaml -# open-telemetry/semantic-conventions v1.22.0 : model/metrics/jvm-metrics-experimental.yaml -# Combined under one groups key so inheritance is exercised exactly as it is in -# the upstream registry. -groups: - - id: attributes.http.common - type: attribute_group - brief: "Describes HTTP attributes." - attributes: - - ref: http.request.method - requirement_level: required - - ref: http.response.status_code - requirement_level: - conditionally_required: If and only if one was received/sent. - - ref: error.type - requirement_level: - conditionally_required: If request has ended with an error. - examples: ['timeout', 'name_resolution_error', '500'] - note: | - If the request fails with an error before response status code was sent or received, - `error.type` SHOULD be set to exception type or a component-specific low cardinality error code. - - If response status code was sent or received and status indicates an error according to [HTTP span status definition](/docs/http/http-spans.md), - `error.type` SHOULD be set to the status code number (represented as a string), an exception type (if thrown) or a component-specific error code. - - The `error.type` value SHOULD be predictable and SHOULD have low cardinality. - Instrumentations SHOULD document the list of errors they report. - - The cardinality of `error.type` within one instrumentation library SHOULD be low, but - telemetry consumers that aggregate data from multiple instrumentation libraries and applications - should be prepared for `error.type` to have high cardinality at query time, when no - additional filters are applied. - - If the request has completed successfully, instrumentations SHOULD NOT set `error.type`. - - ref: network.protocol.name - examples: ['http', 'spdy'] - requirement_level: - recommended: if not default (`http`). - - ref: network.protocol.version - examples: ['1.0', '1.1', '2', '3'] - - - id: attributes.http.client - type: attribute_group - brief: 'HTTP Client attributes' - extends: attributes.http.common - attributes: - - ref: server.address - requirement_level: required - brief: > - Host identifier of the ["URI origin"](https://www.rfc-editor.org/rfc/rfc9110.html#name-uri-origin) HTTP request is sent to. - note: | - Determined by using the first of the following that applies - - - Host identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form - - Host identifier of the `Host` header - - If an HTTP client request is explicitly made to an IP address, e.g. `http://x.x.x.x:8080`, then - `server.address` SHOULD be the IP address `x.x.x.x`. A DNS lookup SHOULD NOT be used. - - ref: server.port - requirement_level: - conditionally_required: If not default (`80` for `http` scheme, `443` for `https`). - brief: > - Port identifier of the ["URI origin"](https://www.rfc-editor.org/rfc/rfc9110.html#name-uri-origin) HTTP request is sent to. - note: > - When [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) is absolute URI, `server.port` MUST match - URI port identifier, otherwise it MUST match `Host` header port identifier. - - - id: attributes.http.server - type: attribute_group - brief: 'HTTP Server attributes' - extends: attributes.http.common - attributes: - - ref: http.route - requirement_level: - conditionally_required: If and only if it's available - - ref: server.address - brief: > - Name of the local HTTP server that received the request. - note: | - Determined by using the first of the following that applies - - - The [primary server name](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. MUST only - include host identifier. - - Host identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Host identifier of the `Host` header - - SHOULD NOT be set if only IP address is available and capturing name would require a reverse DNS lookup. - - - ref: server.port - brief: > - Port of the local HTTP server that received the request. - note: | - Determined by using the first of the following that applies - - - Port identifier of the [primary server host](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. - - Port identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Port identifier of the `Host` header - requirement_level: - recommended: If not default (`80` for `http` scheme, `443` for `https`). - - ref: url.scheme - requirement_level: required - examples: ["http", "https"] - - id: metric_attributes.http.server - type: attribute_group - brief: 'HTTP server attributes' - extends: attributes.http.server - attributes: - - ref: server.address - requirement_level: opt_in - - ref: server.port - requirement_level: opt_in - - - id: metric_attributes.http.client - type: attribute_group - brief: 'HTTP client attributes' - extends: attributes.http.client - attributes: - - ref: url.scheme - requirement_level: required - - - id: metric.http.server.request.duration - type: metric - metric_name: http.server.request.duration - brief: "Duration of HTTP server requests." - instrument: histogram - unit: "s" - extends: metric_attributes.http.server - - - id: metric.http.server.active_requests - type: metric - metric_name: http.server.active_requests - brief: "Number of active HTTP server requests." - instrument: updowncounter - unit: "{request}" - attributes: - - ref: http.request.method - requirement_level: required - - ref: url.scheme - requirement_level: required - examples: ["http", "https"] - - ref: server.address - requirement_level: opt_in - brief: > - Name of the local HTTP server that received the request. - note: | - Determined by using the first of the following that applies - - - The [primary server name](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. MUST only - include host identifier. - - Host identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Host identifier of the `Host` header - - SHOULD NOT be set if only IP address is available and capturing name would require a reverse DNS lookup. - - - ref: server.port - requirement_level: opt_in - brief: > - Port of the local HTTP server that received the request. - note: | - Determined by using the first of the following that applies - - - Port identifier of the [primary server host](/docs/http/http-spans.md#http-server-definitions) of the matched virtual host. - - Port identifier of the [request target](https://www.rfc-editor.org/rfc/rfc9110.html#target.resource) - if it's sent in absolute-form. - - Port identifier of the `Host` header - - - id: metric.http.server.request.body.size - type: metric - metric_name: http.server.request.body.size - brief: "Size of HTTP server request bodies." - instrument: histogram - unit: "By" - note: > - The size of the request payload body in bytes. This is the number of bytes transferred excluding headers and - is often, but not always, present as the [Content-Length](https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length) - header. For requests using transport encoding, this should be the compressed size. - extends: metric_attributes.http.server - - - id: metric.http.server.response.body.size - type: metric - metric_name: http.server.response.body.size - brief: "Size of HTTP server response bodies." - instrument: histogram - unit: "By" - note: > - The size of the response payload body in bytes. This is the number of bytes transferred excluding headers and - is often, but not always, present as the [Content-Length](https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length) - header. For requests using transport encoding, this should be the compressed size. - extends: metric_attributes.http.server - - - id: metric.http.client.request.duration - type: metric - metric_name: http.client.request.duration - brief: "Duration of HTTP client requests." - instrument: histogram - unit: "s" - extends: metric_attributes.http.client - - - id: metric.http.client.request.body.size - type: metric - metric_name: http.client.request.body.size - brief: "Size of HTTP client request bodies." - instrument: histogram - unit: "By" - note: > - The size of the request payload body in bytes. This is the number of bytes transferred excluding headers and - is often, but not always, present as the [Content-Length](https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length) - header. For requests using transport encoding, this should be the compressed size. - extends: metric_attributes.http.client - - - id: metric.http.client.response.body.size - type: metric - metric_name: http.client.response.body.size - brief: "Size of HTTP client response bodies." - instrument: histogram - unit: "By" - note: > - The size of the response payload body in bytes. This is the number of bytes transferred excluding headers and - is often, but not always, present as the [Content-Length](https://www.rfc-editor.org/rfc/rfc9110.html#field.content-length) - header. For requests using transport encoding, this should be the compressed size. - extends: metric_attributes.http.client - - - id: metric.jvm.system.cpu.load_1m - type: metric - metric_name: jvm.system.cpu.load_1m - brief: "Average CPU load of the whole system for the last minute as reported by the JVM." - note: > - The value range is [0,n], where n is the number of CPU cores - or a negative number if the value is not available. - This utilization is not defined as being for the specific interval since last measurement - (unlike `system.cpu.utilization`). - [Reference](https://docs.oracle.com/en/java/javase/17/docs/api/java.management/java/lang/management/OperatingSystemMXBean.html#getSystemLoadAverage()). - instrument: gauge - unit: "{run_queue_item}" diff --git a/storage/semconv/testdata/upstream/semconv-1.43.0.yaml b/storage/semconv/testdata/upstream/semconv-1.43.0.yaml deleted file mode 100644 index cce2b1a409..0000000000 --- a/storage/semconv/testdata/upstream/semconv-1.43.0.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Verbatim excerpt from open-telemetry/semantic-conventions v1.43.0 -# (commit 89aae438b3b3b0a8dd33003c9d70592baf7dbd0d): model/jvm/metrics.yaml. -groups: - - id: metric.jvm.class.loaded - type: metric - metric_name: jvm.class.loaded - annotations: - code_generation: - metric_value_type: int - brief: "Number of classes loaded since JVM start." - instrument: counter - unit: "{class}" - stability: stable diff --git a/storage/semconv/testdata/upstream/semconv-1.44.0.yaml b/storage/semconv/testdata/upstream/semconv-1.44.0.yaml deleted file mode 100644 index 49db786b21..0000000000 --- a/storage/semconv/testdata/upstream/semconv-1.44.0.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Verbatim excerpt from open-telemetry/semantic-conventions v1.44.0 -# (commit e10a930844c6951757a43b849d364f7d056ac32b): model/jvm/metrics.yaml. -groups: - - id: metric.jvm.cpu.count - type: metric - metric_name: jvm.cpu.count - annotations: - code_generation: - metric_value_type: int - brief: "Number of processors available to the Java virtual machine." - instrument: updowncounter - unit: "{cpu}" - stability: stable diff --git a/storage/semconv/transform_policy_test.go b/storage/semconv/transform_policy_test.go index ff689c2265..eeee31ee11 100644 --- a/storage/semconv/transform_policy_test.go +++ b/storage/semconv/transform_policy_test.go @@ -29,13 +29,6 @@ import ( // transform/2.0 features exercised by the policy and PromQL rewriter tests. func testTransformRegistry() map[string][]byte { return map[string][]byte{ - "1.0.0": []byte(`groups: - - id: metric.test.counter - type: metric - metric_name: test.counter - instrument: counter - unit: By -`), "http_1.20.0_to_1.23.1.yaml": []byte(`file_format: transform/2.0 source: schema_url: https://opentelemetry.io/schemas/1.20.0 diff --git a/storage/semconv/upstream_golden_test.go b/storage/semconv/upstream_golden_test.go deleted file mode 100644 index 651389cf03..0000000000 --- a/storage/semconv/upstream_golden_test.go +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package semconv_test - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage/semconv" - "github.com/prometheus/prometheus/util/teststorage" -) - -// The files under testdata/upstream combine a real v1.44.0 schema with verbatim -// HTTP and JVM semantic-convention groups from v1.21.0, v1.22.0, v1.43.0, and -// v1.44.0. Each fixture names its upstream source; the HTTP groups retain their -// inheritance structure. -// Hand-written fixtures are free to encode whatever shape the parser happens to -// expect, which is how rename_metrics came to be read from a name_map key that -// the file format does not have: every fixture agreed with the bug, so the tests -// passed while metric renames silently did nothing on real data. These tests -// exist so the real file format is what the parser is held to. -// -// Attribute-rename scoping uses a focused fixture elsewhere because the real -// schema's HTTP attribute renames in this version apply only to spans. -const ( - upstreamSchema = "./testdata/upstream/schema-1.44.0.yaml" - upstreamSemconv1_21_0 = "./testdata/upstream/semconv-1.21.0.yaml" - upstreamSemconv1_22_0 = "./testdata/upstream/semconv-1.22.0.yaml" - upstreamSemconv1_43_0 = "./testdata/upstream/semconv-1.43.0.yaml" - upstreamSemconv1_44_0 = "./testdata/upstream/semconv-1.44.0.yaml" -) - -// This edge is deliberately synthetic: its endpoints are unrelated stable -// metrics whose definitions below are verbatim upstream data. -const upstreamStableContradictionSchema = `file_format: 1.1.0 -schema_url: https://example.com/schemas/1.44.0 -versions: - 1.43.0: - 1.44.0: - metrics: - changes: - - rename_metrics: - jvm.class.loaded: jvm.cpu.count -` - -// upstreamRegistry assembles the real artefacts into a registry, keyed the way -// AwareStorageWithRegistry expects: semver base names for semconv files, anything -// else for the schema. -func upstreamRegistry(t *testing.T) map[string][]byte { - t.Helper() - read := func(path string) []byte { - b, err := os.ReadFile(filepath.Clean(path)) - require.NoError(t, err) - return b - } - return map[string][]byte{ - "registry.yaml": read(upstreamSchema), - "1.21.0": read(upstreamSemconv1_21_0), - "1.22.0": read(upstreamSemconv1_22_0), - "1.43.0": read(upstreamSemconv1_43_0), - "1.44.0": read(upstreamSemconv1_44_0), - } -} - -// TestUpstreamSchemaMetricRenames guards the file format itself. Reading -// rename_metrics from a nested name_map key yielded zero metric renames from -// every real schema while all fixtures still passed, so assert against the real -// artefact that metric renames are found at all. -func TestUpstreamSchemaMetricRenames(t *testing.T) { - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, upstreamRegistry(t)) - require.NoError(t, err) - - // http.server.duration was renamed to http.server.request.duration in - // semconv 1.22.0, the canonical example of a metric rename. Write the series - // under its pre-rename name and query it under the post-rename one. - appendSeries(t, wrapped, "http.server.duration", 1, 7.0, "http.response.status_code", "200") - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "http.server.request.duration"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.22.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.Len(t, got, 1, - "expected the pre-rename series to surface under the queried name; zero means rename_metrics did not parse, got %v", got) - for k := range got { - require.Contains(t, k, `__name__="http.server.request.duration"`) - } - - // Both versions declare the metric as s/histogram, so the metric rename is - // corroborated. Later schema revisions may independently report ambiguous - // attribute histories, but none of the warnings may concern metric lineage. - for _, warning := range warningStrings(set.Warnings()) { - require.Contains(t, warning, "attribute name", - "a rename corroborated by the real semconv files must not raise a metric warning") - } -} - -func TestUpstreamExperimentalMetricRenameMayChangeUnit(t *testing.T) { - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, upstreamRegistry(t)) - require.NoError(t, err) - - appendSeries(t, wrapped, "process.runtime.jvm.system.cpu.load_1m", 1, 7.0) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "jvm.system.cpu.load_1m"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.22.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/registry.yaml"), - ) - got := collectSeries(t, set) - require.Len(t, got, 1, "the upstream experimental rename must retain its historical series: %v", got) - for key := range got { - require.Contains(t, key, `__name__="jvm.system.cpu.load_1m"`) - } - requireWarningsContain(t, warningStrings(set.Warnings()), "following the explicit schema rename") -} - -func TestUpstreamStableMetricContradiction(t *testing.T) { - registry := upstreamRegistry(t) - registry["stable-contradiction.yaml"] = []byte(upstreamStableContradictionSchema) - - underlying := teststorage.New(t) - wrapped, err := semconv.AwareStorageWithRegistry(underlying, registry) - require.NoError(t, err) - - appendSeries(t, wrapped, "jvm.class.loaded", 1, 7.0) - - q, err := wrapped.Querier(0, 10) - require.NoError(t, err) - t.Cleanup(func() { _ = q.Close() }) - - set := q.Select(context.Background(), false, nil, - labels.MustNewMatcher(labels.MatchEqual, model.MetricNameLabel, "jvm.cpu.count"), - labels.MustNewMatcher(labels.MatchEqual, "__semconv_url__", "registry/1.44.0"), - labels.MustNewMatcher(labels.MatchEqual, "__schema_url__", "registry/stable-contradiction.yaml"), - ) - got := collectSeries(t, set) - require.Empty(t, got, "stable definitions with conflicting units and instruments must not merge: %v", got) - requireWarningsContain(t, warningStrings(set.Warnings()), "treating them as different metrics") - requireWarningsContain(t, warningStrings(set.Warnings()), `resolves it to "jvm.class.loaded"`) -}