From 610bd68f2bc4817da3be89d3d21f085f1987ad12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Desch=C3=AAnes?= Date: Fri, 20 Nov 2026 08:10:42 -0500 Subject: [PATCH] Use one help text per metric name across collectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each project collector builds its metric descriptors from its own project's metric descriptor. Projects can describe the same metric type differently, custom and auto-created metrics in particular, and the registry then rejects the gather with "has help X but should have Y", which fails the whole scrape. Add a MetricHelpStore that keeps the first help text seen for each metric name, for as long as that name keeps being reported. The Runtime creates one and passes it to every collector it builds through MonitoringCollectorOptions, so all collectors in an exposition serve the same help text. A collector built without a store uses one of its own. Names not reported for an hour are dropped, which bounds the store. Fixes #361 Signed-off-by: Antoine DeschĂȘnes Co-Authored-By: Claude Fable 5.2 --- collectors/metric_help.go | 100 +++++++++++++++++++++ collectors/monitoring_collector.go | 12 +++ collectors/monitoring_metrics.go | 7 +- collectors/monitoring_metrics_test.go | 124 +++++++++++++++++++++++++- collectors/runtime.go | 6 +- collectors/runtime_test.go | 23 +++++ 6 files changed, 268 insertions(+), 4 deletions(-) create mode 100644 collectors/metric_help.go diff --git a/collectors/metric_help.go b/collectors/metric_help.go new file mode 100644 index 00000000..0e3baa09 --- /dev/null +++ b/collectors/metric_help.go @@ -0,0 +1,100 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collectors + +import ( + "sync" + "sync/atomic" + "time" +) + +// metricHelpTTL is how long a help text is kept after its metric name was +// last reported. +const metricHelpTTL = time.Hour + +// MetricHelpStore holds the help text served for each metric name. Collectors +// gathered into the same exposition must share one store: projects can +// describe the same metric type differently (custom metrics in particular), +// while an exposition needs a single help text per metric family. +// +// A MetricHelpStore is safe for concurrent use, and its zero value is ready +// to use. +type MetricHelpStore struct { + mu sync.RWMutex + entries map[string]*metricHelpEntry + lastSweep time.Time + now func() time.Time +} + +type metricHelpEntry struct { + help string + lastUsed atomic.Int64 +} + +// NewMetricHelpStore returns an empty MetricHelpStore. +func NewMetricHelpStore() *MetricHelpStore { + return &MetricHelpStore{} +} + +// Help returns the help text for fqName. The description passed on the first +// call for a name is kept and returned by every later call for that name, +// until the name has not been asked for during metricHelpTTL. +func (s *MetricHelpStore) Help(fqName, description string) string { + now := time.Now() + if s.now != nil { + now = s.now() + } + + // lastUsed is updated under the read lock, so that sweep, which holds the + // write lock, never drops an entry a caller is about to return. + s.mu.RLock() + entry, ok := s.entries[fqName] + if ok { + entry.lastUsed.Store(now.UnixNano()) + } + s.mu.RUnlock() + if ok { + return entry.help + } + + s.mu.Lock() + defer s.mu.Unlock() + if entry, ok := s.entries[fqName]; ok { + entry.lastUsed.Store(now.UnixNano()) + return entry.help + } + s.sweep(now) + if s.entries == nil { + s.entries = make(map[string]*metricHelpEntry) + } + entry = &metricHelpEntry{help: description} + entry.lastUsed.Store(now.UnixNano()) + s.entries[fqName] = entry + return description +} + +// sweep drops the entries unused for metricHelpTTL, at most once per +// metricHelpTTL. The caller must hold the write lock. +func (s *MetricHelpStore) sweep(now time.Time) { + if now.Sub(s.lastSweep) < metricHelpTTL { + return + } + s.lastSweep = now + deadline := now.Add(-metricHelpTTL).UnixNano() + for fqName, entry := range s.entries { + if entry.lastUsed.Load() < deadline { + delete(s.entries, fqName) + } + } +} diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 7d5b138d..0a6a906d 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -82,6 +82,7 @@ type MonitoringCollector struct { histogramStore DeltaHistogramStore aggregateDeltas bool descriptorCache DescriptorCache + helpStore *MetricHelpStore } type MonitoringCollectorOptions struct { @@ -109,6 +110,10 @@ type MonitoringCollectorOptions struct { DescriptorCacheTTL time.Duration // DescriptorCacheOnlyGoogle decides whether only google specific descriptors should be cached or all DescriptorCacheOnlyGoogle bool + // MetricHelpStore keeps one help text per metric name. Pass the same store to every collector registered in + // the same registry, so that projects describing a metric type differently do not fail the gather. When nil, + // the collector uses a store of its own. + MetricHelpStore *MetricHelpStore } func isGoogleMetric(name string) bool { @@ -219,6 +224,11 @@ func NewMonitoringCollector(projectID string, monitoringService *monitoring.Serv } + helpStore := opts.MetricHelpStore + if helpStore == nil { + helpStore = NewMetricHelpStore() + } + monitoringCollector := &MonitoringCollector{ projectID: projectID, metricsTypePrefixes: opts.MetricTypePrefixes, @@ -240,6 +250,7 @@ func NewMonitoringCollector(projectID string, monitoringService *monitoring.Serv histogramStore: histogramStore, aggregateDeltas: opts.AggregateDeltas, descriptorCache: descriptorCache, + helpStore: helpStore, } return monitoringCollector, nil @@ -440,6 +451,7 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics( c.counterStore, c.histogramStore, c.aggregateDeltas, + c.helpStore, ) if err != nil { return fmt.Errorf("error creating the TimeSeriesMetrics %v", err) diff --git a/collectors/monitoring_metrics.go b/collectors/monitoring_metrics.go index 2241300b..43a9d583 100644 --- a/collectors/monitoring_metrics.go +++ b/collectors/monitoring_metrics.go @@ -60,6 +60,7 @@ type timeSeriesMetrics struct { counterStore DeltaCounterStore histogramStore DeltaHistogramStore aggregateDeltas bool + helpStore *MetricHelpStore } func newTimeSeriesMetrics(descriptor *monitoring.MetricDescriptor, @@ -67,7 +68,8 @@ func newTimeSeriesMetrics(descriptor *monitoring.MetricDescriptor, fillMissingLabels bool, counterStore DeltaCounterStore, histogramStore DeltaHistogramStore, - aggregateDeltas bool) (*timeSeriesMetrics, error) { + aggregateDeltas bool, + helpStore *MetricHelpStore) (*timeSeriesMetrics, error) { return &timeSeriesMetrics{ metricDescriptor: descriptor, @@ -78,13 +80,14 @@ func newTimeSeriesMetrics(descriptor *monitoring.MetricDescriptor, counterStore: counterStore, histogramStore: histogramStore, aggregateDeltas: aggregateDeltas, + helpStore: helpStore, }, nil } func (t *timeSeriesMetrics) newMetricDesc(fqName string, labelKeys []string) *prometheus.Desc { return prometheus.NewDesc( fqName, - t.metricDescriptor.Description, + t.helpStore.Help(fqName, t.metricDescriptor.Description), labelKeys, prometheus.Labels{}, ) diff --git a/collectors/monitoring_metrics_test.go b/collectors/monitoring_metrics_test.go index 06a447d9..3ee73ab2 100644 --- a/collectors/monitoring_metrics_test.go +++ b/collectors/monitoring_metrics_test.go @@ -13,7 +13,15 @@ package collectors -import "testing" +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/api/monitoring/v3" +) func TestNormalizeMetricName(t *testing.T) { t.Parallel() @@ -24,3 +32,117 @@ func TestNormalizeMetricName(t *testing.T) { t.Fatalf("normalizeMetricName() = %q, want %q", got, want) } } + +type collectFunc func(ch chan<- prometheus.Metric) + +func (f collectFunc) Describe(chan<- *prometheus.Desc) {} + +func (f collectFunc) Collect(ch chan<- prometheus.Metric) { f(ch) } + +func TestCustomMetricWithDifferentDescriptionsPerProjectGathers(t *testing.T) { + t.Parallel() + + const metricType = "custom.googleapis.com/spark/executor/bigquery-metrics-source/bytesRead" + projects := []struct { + id string + description string + }{ + {"project-a", "Auto-created custom metric."}, + {"project-b", ""}, + } + + helpStore := NewMetricHelpStore() + collector := collectFunc(func(ch chan<- prometheus.Metric) { + for _, p := range projects { + descriptor := &monitoring.MetricDescriptor{Type: metricType, Description: p.description} + tsm, err := newTimeSeriesMetrics(descriptor, ch, false, nil, nil, false, helpStore) + if err != nil { + t.Errorf("newTimeSeriesMetrics: %v", err) + return + } + ts := &monitoring.TimeSeries{ + Metric: &monitoring.Metric{Type: metricType}, + Resource: &monitoring.MonitoredResource{Type: "cloud_dataproc_batch"}, + } + tsm.CollectNewConstMetric(ts, time.Now(), []string{"project_id"}, prometheus.CounterValue, 1, []string{p.id}, "CUMULATIVE") + } + }) + + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + families, err := registry.Gather() + if err != nil { + t.Fatalf("Gather() error = %v", err) + } + if len(families) != 1 { + t.Fatalf("Gather() returned %d families, want 1", len(families)) + } + if got, want := families[0].GetHelp(), projects[0].description; got != want { + t.Fatalf("family %s has help %q, want the first description seen, %q", families[0].GetName(), got, want) + } + if got := len(families[0].GetMetric()); got != 2 { + t.Fatalf("family %s has %d metrics, want 2", families[0].GetName(), got) + } +} + +func TestMetricHelpStoreKeepsFirstDescriptionUnderConcurrency(t *testing.T) { + t.Parallel() + + store := NewMetricHelpStore() + const callers = 64 + got := make([]string, callers) + var wg sync.WaitGroup + for i := range callers { + wg.Add(1) + go func() { + defer wg.Done() + got[i] = store.Help("metric_name", fmt.Sprintf("description %d", i)) + }() + } + wg.Wait() + + for i, help := range got { + if help != got[0] { + t.Fatalf("caller %d got help %q, caller 0 got %q", i, help, got[0]) + } + } +} + +func TestMetricHelpStoreZeroValue(t *testing.T) { + t.Parallel() + + var store MetricHelpStore + if got := store.Help("metric_name", "first"); got != "first" { + t.Fatalf("Help() = %q, want %q", got, "first") + } + if got := store.Help("metric_name", "second"); got != "first" { + t.Fatalf("Help() = %q, want %q", got, "first") + } +} + +func TestMetricHelpStoreDropsUnusedNames(t *testing.T) { + t.Parallel() + + now := time.Unix(0, 0) + store := &MetricHelpStore{now: func() time.Time { return now }} + store.Help("live_metric", "live") + store.Help("gone_metric", "gone") + + // live_metric keeps being reported, gone_metric does not. + for range 4 { + now = now.Add(metricHelpTTL / 2) + store.Help("live_metric", "other") + } + // A new name triggers the sweep. + store.Help("new_metric", "new") + + if got := store.Help("live_metric", "other"); got != "live" { + t.Fatalf("Help(live_metric) = %q, want %q", got, "live") + } + if got := store.Help("gone_metric", "replaced"); got != "replaced" { + t.Fatalf("Help(gone_metric) = %q, want %q", got, "replaced") + } + if got := len(store.entries); got != 3 { + t.Fatalf("store holds %d names, want 3", got) + } +} diff --git a/collectors/runtime.go b/collectors/runtime.go index e7c6f92e..2354f1b2 100644 --- a/collectors/runtime.go +++ b/collectors/runtime.go @@ -45,6 +45,7 @@ type Runtime struct { counterStoreFactory CounterStoreFactory histogramStoreFactory HistogramStoreFactory cache *collectorCache + helpStore *MetricHelpStore discoverProjectIDs func(ctx context.Context, filter string) ([]string, error) } @@ -105,6 +106,7 @@ func NewRuntime(ctx context.Context, logger *slog.Logger, cfg *config.Config, co logger: logger, counterStoreFactory: counterFactory, histogramStoreFactory: histogramFactory, + helpStore: NewMetricHelpStore(), discoverProjectIDs: discoverProjectIDs, }, nil } @@ -209,10 +211,12 @@ func (r *Runtime) collectorFor(projectID string, prefixFilter []string) (*Monito func (r *Runtime) newCollector(projectID string, prefixFilter []string) (*MonitoringCollector, error) { filtered := r.filterMetricTypePrefixes(prefixFilter) + opts := monitoringCollectorOptionsForPrefixes(r.cfg, filtered) + opts.MetricHelpStore = r.helpStore return NewMonitoringCollector( projectID, r.service, - monitoringCollectorOptionsForPrefixes(r.cfg, filtered), + opts, r.logger, r.counterStoreFactory(r.logger, r.cfg.AggregateDeltasTTL), r.histogramStoreFactory(r.logger, r.cfg.AggregateDeltasTTL), diff --git a/collectors/runtime_test.go b/collectors/runtime_test.go index bbc31c51..9c4e51fa 100644 --- a/collectors/runtime_test.go +++ b/collectors/runtime_test.go @@ -421,3 +421,26 @@ func TestWithCacheSharesProjectIDsPointer(t *testing.T) { t.Fatalf("sibling.Collectors() returned %d collectors after original's projectIDs updated, want 2", len(cs)) } } + +func TestRuntimeCollectorsShareMetricHelpStore(t *testing.T) { + t.Parallel() + + cfg := &config.Config{MetricsPrefixes: []string{"custom.googleapis.com/"}} + r := newTestRuntime(t, cfg, []string{"project-a", "project-b"}, nil) + r.counterStoreFactory = func(_ *slog.Logger, _ time.Duration) DeltaCounterStore { return nil } + r.histogramStoreFactory = func(_ *slog.Logger, _ time.Duration) DeltaHistogramStore { return nil } + r.helpStore = NewMetricHelpStore() + + collectors, err := r.WithCache().Collectors() + if err != nil { + t.Fatalf("Collectors() error = %v", err) + } + if len(collectors) != 2 { + t.Fatalf("Collectors() returned %d collectors, want 2", len(collectors)) + } + for _, c := range collectors { + if c.helpStore != r.helpStore { + t.Fatalf("collector for %s does not use the runtime's help store", c.projectID) + } + } +}