diff --git a/collectors/metric_help.go b/collectors/metric_help.go new file mode 100644 index 0000000..0e3baa0 --- /dev/null +++ b/collectors/metric_help.go @@ -0,0 +1,100 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collectors + +import ( + "sync" + "sync/atomic" + "time" +) + +// metricHelpTTL is how long a help text is kept after its metric name was +// last reported. +const metricHelpTTL = time.Hour + +// MetricHelpStore holds the help text served for each metric name. Collectors +// gathered into the same exposition must share one store: projects can +// describe the same metric type differently (custom metrics in particular), +// while an exposition needs a single help text per metric family. +// +// A MetricHelpStore is safe for concurrent use, and its zero value is ready +// to use. +type MetricHelpStore struct { + mu sync.RWMutex + entries map[string]*metricHelpEntry + lastSweep time.Time + now func() time.Time +} + +type metricHelpEntry struct { + help string + lastUsed atomic.Int64 +} + +// NewMetricHelpStore returns an empty MetricHelpStore. +func NewMetricHelpStore() *MetricHelpStore { + return &MetricHelpStore{} +} + +// Help returns the help text for fqName. The description passed on the first +// call for a name is kept and returned by every later call for that name, +// until the name has not been asked for during metricHelpTTL. +func (s *MetricHelpStore) Help(fqName, description string) string { + now := time.Now() + if s.now != nil { + now = s.now() + } + + // lastUsed is updated under the read lock, so that sweep, which holds the + // write lock, never drops an entry a caller is about to return. + s.mu.RLock() + entry, ok := s.entries[fqName] + if ok { + entry.lastUsed.Store(now.UnixNano()) + } + s.mu.RUnlock() + if ok { + return entry.help + } + + s.mu.Lock() + defer s.mu.Unlock() + if entry, ok := s.entries[fqName]; ok { + entry.lastUsed.Store(now.UnixNano()) + return entry.help + } + s.sweep(now) + if s.entries == nil { + s.entries = make(map[string]*metricHelpEntry) + } + entry = &metricHelpEntry{help: description} + entry.lastUsed.Store(now.UnixNano()) + s.entries[fqName] = entry + return description +} + +// sweep drops the entries unused for metricHelpTTL, at most once per +// metricHelpTTL. The caller must hold the write lock. +func (s *MetricHelpStore) sweep(now time.Time) { + if now.Sub(s.lastSweep) < metricHelpTTL { + return + } + s.lastSweep = now + deadline := now.Add(-metricHelpTTL).UnixNano() + for fqName, entry := range s.entries { + if entry.lastUsed.Load() < deadline { + delete(s.entries, fqName) + } + } +} diff --git a/collectors/monitoring_collector.go b/collectors/monitoring_collector.go index 7d5b138..0a6a906 100644 --- a/collectors/monitoring_collector.go +++ b/collectors/monitoring_collector.go @@ -82,6 +82,7 @@ type MonitoringCollector struct { histogramStore DeltaHistogramStore aggregateDeltas bool descriptorCache DescriptorCache + helpStore *MetricHelpStore } type MonitoringCollectorOptions struct { @@ -109,6 +110,10 @@ type MonitoringCollectorOptions struct { DescriptorCacheTTL time.Duration // DescriptorCacheOnlyGoogle decides whether only google specific descriptors should be cached or all DescriptorCacheOnlyGoogle bool + // MetricHelpStore keeps one help text per metric name. Pass the same store to every collector registered in + // the same registry, so that projects describing a metric type differently do not fail the gather. When nil, + // the collector uses a store of its own. + MetricHelpStore *MetricHelpStore } func isGoogleMetric(name string) bool { @@ -219,6 +224,11 @@ func NewMonitoringCollector(projectID string, monitoringService *monitoring.Serv } + helpStore := opts.MetricHelpStore + if helpStore == nil { + helpStore = NewMetricHelpStore() + } + monitoringCollector := &MonitoringCollector{ projectID: projectID, metricsTypePrefixes: opts.MetricTypePrefixes, @@ -240,6 +250,7 @@ func NewMonitoringCollector(projectID string, monitoringService *monitoring.Serv histogramStore: histogramStore, aggregateDeltas: opts.AggregateDeltas, descriptorCache: descriptorCache, + helpStore: helpStore, } return monitoringCollector, nil @@ -440,6 +451,7 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics( c.counterStore, c.histogramStore, c.aggregateDeltas, + c.helpStore, ) if err != nil { return fmt.Errorf("error creating the TimeSeriesMetrics %v", err) diff --git a/collectors/monitoring_metrics.go b/collectors/monitoring_metrics.go index 2241300..43a9d58 100644 --- a/collectors/monitoring_metrics.go +++ b/collectors/monitoring_metrics.go @@ -60,6 +60,7 @@ type timeSeriesMetrics struct { counterStore DeltaCounterStore histogramStore DeltaHistogramStore aggregateDeltas bool + helpStore *MetricHelpStore } func newTimeSeriesMetrics(descriptor *monitoring.MetricDescriptor, @@ -67,7 +68,8 @@ func newTimeSeriesMetrics(descriptor *monitoring.MetricDescriptor, fillMissingLabels bool, counterStore DeltaCounterStore, histogramStore DeltaHistogramStore, - aggregateDeltas bool) (*timeSeriesMetrics, error) { + aggregateDeltas bool, + helpStore *MetricHelpStore) (*timeSeriesMetrics, error) { return &timeSeriesMetrics{ metricDescriptor: descriptor, @@ -78,13 +80,14 @@ func newTimeSeriesMetrics(descriptor *monitoring.MetricDescriptor, counterStore: counterStore, histogramStore: histogramStore, aggregateDeltas: aggregateDeltas, + helpStore: helpStore, }, nil } func (t *timeSeriesMetrics) newMetricDesc(fqName string, labelKeys []string) *prometheus.Desc { return prometheus.NewDesc( fqName, - t.metricDescriptor.Description, + t.helpStore.Help(fqName, t.metricDescriptor.Description), labelKeys, prometheus.Labels{}, ) diff --git a/collectors/monitoring_metrics_test.go b/collectors/monitoring_metrics_test.go index 06a447d..3ee73ab 100644 --- a/collectors/monitoring_metrics_test.go +++ b/collectors/monitoring_metrics_test.go @@ -13,7 +13,15 @@ package collectors -import "testing" +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/api/monitoring/v3" +) func TestNormalizeMetricName(t *testing.T) { t.Parallel() @@ -24,3 +32,117 @@ func TestNormalizeMetricName(t *testing.T) { t.Fatalf("normalizeMetricName() = %q, want %q", got, want) } } + +type collectFunc func(ch chan<- prometheus.Metric) + +func (f collectFunc) Describe(chan<- *prometheus.Desc) {} + +func (f collectFunc) Collect(ch chan<- prometheus.Metric) { f(ch) } + +func TestCustomMetricWithDifferentDescriptionsPerProjectGathers(t *testing.T) { + t.Parallel() + + const metricType = "custom.googleapis.com/spark/executor/bigquery-metrics-source/bytesRead" + projects := []struct { + id string + description string + }{ + {"project-a", "Auto-created custom metric."}, + {"project-b", ""}, + } + + helpStore := NewMetricHelpStore() + collector := collectFunc(func(ch chan<- prometheus.Metric) { + for _, p := range projects { + descriptor := &monitoring.MetricDescriptor{Type: metricType, Description: p.description} + tsm, err := newTimeSeriesMetrics(descriptor, ch, false, nil, nil, false, helpStore) + if err != nil { + t.Errorf("newTimeSeriesMetrics: %v", err) + return + } + ts := &monitoring.TimeSeries{ + Metric: &monitoring.Metric{Type: metricType}, + Resource: &monitoring.MonitoredResource{Type: "cloud_dataproc_batch"}, + } + tsm.CollectNewConstMetric(ts, time.Now(), []string{"project_id"}, prometheus.CounterValue, 1, []string{p.id}, "CUMULATIVE") + } + }) + + registry := prometheus.NewRegistry() + registry.MustRegister(collector) + families, err := registry.Gather() + if err != nil { + t.Fatalf("Gather() error = %v", err) + } + if len(families) != 1 { + t.Fatalf("Gather() returned %d families, want 1", len(families)) + } + if got, want := families[0].GetHelp(), projects[0].description; got != want { + t.Fatalf("family %s has help %q, want the first description seen, %q", families[0].GetName(), got, want) + } + if got := len(families[0].GetMetric()); got != 2 { + t.Fatalf("family %s has %d metrics, want 2", families[0].GetName(), got) + } +} + +func TestMetricHelpStoreKeepsFirstDescriptionUnderConcurrency(t *testing.T) { + t.Parallel() + + store := NewMetricHelpStore() + const callers = 64 + got := make([]string, callers) + var wg sync.WaitGroup + for i := range callers { + wg.Add(1) + go func() { + defer wg.Done() + got[i] = store.Help("metric_name", fmt.Sprintf("description %d", i)) + }() + } + wg.Wait() + + for i, help := range got { + if help != got[0] { + t.Fatalf("caller %d got help %q, caller 0 got %q", i, help, got[0]) + } + } +} + +func TestMetricHelpStoreZeroValue(t *testing.T) { + t.Parallel() + + var store MetricHelpStore + if got := store.Help("metric_name", "first"); got != "first" { + t.Fatalf("Help() = %q, want %q", got, "first") + } + if got := store.Help("metric_name", "second"); got != "first" { + t.Fatalf("Help() = %q, want %q", got, "first") + } +} + +func TestMetricHelpStoreDropsUnusedNames(t *testing.T) { + t.Parallel() + + now := time.Unix(0, 0) + store := &MetricHelpStore{now: func() time.Time { return now }} + store.Help("live_metric", "live") + store.Help("gone_metric", "gone") + + // live_metric keeps being reported, gone_metric does not. + for range 4 { + now = now.Add(metricHelpTTL / 2) + store.Help("live_metric", "other") + } + // A new name triggers the sweep. + store.Help("new_metric", "new") + + if got := store.Help("live_metric", "other"); got != "live" { + t.Fatalf("Help(live_metric) = %q, want %q", got, "live") + } + if got := store.Help("gone_metric", "replaced"); got != "replaced" { + t.Fatalf("Help(gone_metric) = %q, want %q", got, "replaced") + } + if got := len(store.entries); got != 3 { + t.Fatalf("store holds %d names, want 3", got) + } +} diff --git a/collectors/runtime.go b/collectors/runtime.go index e7c6f92..2354f1b 100644 --- a/collectors/runtime.go +++ b/collectors/runtime.go @@ -45,6 +45,7 @@ type Runtime struct { counterStoreFactory CounterStoreFactory histogramStoreFactory HistogramStoreFactory cache *collectorCache + helpStore *MetricHelpStore discoverProjectIDs func(ctx context.Context, filter string) ([]string, error) } @@ -105,6 +106,7 @@ func NewRuntime(ctx context.Context, logger *slog.Logger, cfg *config.Config, co logger: logger, counterStoreFactory: counterFactory, histogramStoreFactory: histogramFactory, + helpStore: NewMetricHelpStore(), discoverProjectIDs: discoverProjectIDs, }, nil } @@ -209,10 +211,12 @@ func (r *Runtime) collectorFor(projectID string, prefixFilter []string) (*Monito func (r *Runtime) newCollector(projectID string, prefixFilter []string) (*MonitoringCollector, error) { filtered := r.filterMetricTypePrefixes(prefixFilter) + opts := monitoringCollectorOptionsForPrefixes(r.cfg, filtered) + opts.MetricHelpStore = r.helpStore return NewMonitoringCollector( projectID, r.service, - monitoringCollectorOptionsForPrefixes(r.cfg, filtered), + opts, r.logger, r.counterStoreFactory(r.logger, r.cfg.AggregateDeltasTTL), r.histogramStoreFactory(r.logger, r.cfg.AggregateDeltasTTL), diff --git a/collectors/runtime_test.go b/collectors/runtime_test.go index bbc31c5..9c4e51f 100644 --- a/collectors/runtime_test.go +++ b/collectors/runtime_test.go @@ -421,3 +421,26 @@ func TestWithCacheSharesProjectIDsPointer(t *testing.T) { t.Fatalf("sibling.Collectors() returned %d collectors after original's projectIDs updated, want 2", len(cs)) } } + +func TestRuntimeCollectorsShareMetricHelpStore(t *testing.T) { + t.Parallel() + + cfg := &config.Config{MetricsPrefixes: []string{"custom.googleapis.com/"}} + r := newTestRuntime(t, cfg, []string{"project-a", "project-b"}, nil) + r.counterStoreFactory = func(_ *slog.Logger, _ time.Duration) DeltaCounterStore { return nil } + r.histogramStoreFactory = func(_ *slog.Logger, _ time.Duration) DeltaHistogramStore { return nil } + r.helpStore = NewMetricHelpStore() + + collectors, err := r.WithCache().Collectors() + if err != nil { + t.Fatalf("Collectors() error = %v", err) + } + if len(collectors) != 2 { + t.Fatalf("Collectors() returned %d collectors, want 2", len(collectors)) + } + for _, c := range collectors { + if c.helpStore != r.helpStore { + t.Fatalf("collector for %s does not use the runtime's help store", c.projectID) + } + } +}