Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 100 additions & 0 deletions collectors/metric_help.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collectors

import (
"sync"
"sync/atomic"
"time"
)

// metricHelpTTL is how long a help text is kept after its metric name was
// last reported.
const metricHelpTTL = time.Hour

// MetricHelpStore holds the help text served for each metric name. Collectors
// gathered into the same exposition must share one store: projects can
// describe the same metric type differently (custom metrics in particular),
// while an exposition needs a single help text per metric family.
//
// A MetricHelpStore is safe for concurrent use, and its zero value is ready
// to use.
type MetricHelpStore struct {
mu sync.RWMutex
entries map[string]*metricHelpEntry
lastSweep time.Time
now func() time.Time
}

type metricHelpEntry struct {
help string
lastUsed atomic.Int64
}

// NewMetricHelpStore returns an empty MetricHelpStore.
func NewMetricHelpStore() *MetricHelpStore {
return &MetricHelpStore{}
}

// Help returns the help text for fqName. The description passed on the first
// call for a name is kept and returned by every later call for that name,
// until the name has not been asked for during metricHelpTTL.
func (s *MetricHelpStore) Help(fqName, description string) string {
now := time.Now()
if s.now != nil {
now = s.now()
}

// lastUsed is updated under the read lock, so that sweep, which holds the
// write lock, never drops an entry a caller is about to return.
s.mu.RLock()
entry, ok := s.entries[fqName]
if ok {
entry.lastUsed.Store(now.UnixNano())
}
s.mu.RUnlock()
if ok {
return entry.help
}

s.mu.Lock()
defer s.mu.Unlock()
if entry, ok := s.entries[fqName]; ok {
entry.lastUsed.Store(now.UnixNano())
return entry.help
}
s.sweep(now)
if s.entries == nil {
s.entries = make(map[string]*metricHelpEntry)
}
entry = &metricHelpEntry{help: description}
entry.lastUsed.Store(now.UnixNano())
s.entries[fqName] = entry
return description
}

// sweep drops the entries unused for metricHelpTTL, at most once per
// metricHelpTTL. The caller must hold the write lock.
func (s *MetricHelpStore) sweep(now time.Time) {
if now.Sub(s.lastSweep) < metricHelpTTL {
return
}
s.lastSweep = now
deadline := now.Add(-metricHelpTTL).UnixNano()
for fqName, entry := range s.entries {
if entry.lastUsed.Load() < deadline {
delete(s.entries, fqName)
}
}
}
12 changes: 12 additions & 0 deletions collectors/monitoring_collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ type MonitoringCollector struct {
histogramStore DeltaHistogramStore
aggregateDeltas bool
descriptorCache DescriptorCache
helpStore *MetricHelpStore
}

type MonitoringCollectorOptions struct {
Expand Down Expand Up @@ -109,6 +110,10 @@ type MonitoringCollectorOptions struct {
DescriptorCacheTTL time.Duration
// DescriptorCacheOnlyGoogle decides whether only google specific descriptors should be cached or all
DescriptorCacheOnlyGoogle bool
// MetricHelpStore keeps one help text per metric name. Pass the same store to every collector registered in
// the same registry, so that projects describing a metric type differently do not fail the gather. When nil,
// the collector uses a store of its own.
MetricHelpStore *MetricHelpStore
}

func isGoogleMetric(name string) bool {
Expand Down Expand Up @@ -219,6 +224,11 @@ func NewMonitoringCollector(projectID string, monitoringService *monitoring.Serv

}

helpStore := opts.MetricHelpStore
if helpStore == nil {
helpStore = NewMetricHelpStore()
}

monitoringCollector := &MonitoringCollector{
projectID: projectID,
metricsTypePrefixes: opts.MetricTypePrefixes,
Expand All @@ -240,6 +250,7 @@ func NewMonitoringCollector(projectID string, monitoringService *monitoring.Serv
histogramStore: histogramStore,
aggregateDeltas: opts.AggregateDeltas,
descriptorCache: descriptorCache,
helpStore: helpStore,
}

return monitoringCollector, nil
Expand Down Expand Up @@ -440,6 +451,7 @@ func (c *MonitoringCollector) reportTimeSeriesMetrics(
c.counterStore,
c.histogramStore,
c.aggregateDeltas,
c.helpStore,
)
if err != nil {
return fmt.Errorf("error creating the TimeSeriesMetrics %v", err)
Expand Down
7 changes: 5 additions & 2 deletions collectors/monitoring_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,14 +60,16 @@ type timeSeriesMetrics struct {
counterStore DeltaCounterStore
histogramStore DeltaHistogramStore
aggregateDeltas bool
helpStore *MetricHelpStore
}

func newTimeSeriesMetrics(descriptor *monitoring.MetricDescriptor,
ch chan<- prometheus.Metric,
fillMissingLabels bool,
counterStore DeltaCounterStore,
histogramStore DeltaHistogramStore,
aggregateDeltas bool) (*timeSeriesMetrics, error) {
aggregateDeltas bool,
helpStore *MetricHelpStore) (*timeSeriesMetrics, error) {

return &timeSeriesMetrics{
metricDescriptor: descriptor,
Expand All @@ -78,13 +80,14 @@ func newTimeSeriesMetrics(descriptor *monitoring.MetricDescriptor,
counterStore: counterStore,
histogramStore: histogramStore,
aggregateDeltas: aggregateDeltas,
helpStore: helpStore,
}, nil
}

func (t *timeSeriesMetrics) newMetricDesc(fqName string, labelKeys []string) *prometheus.Desc {
return prometheus.NewDesc(
fqName,
t.metricDescriptor.Description,
t.helpStore.Help(fqName, t.metricDescriptor.Description),
labelKeys,
prometheus.Labels{},
)
Expand Down
124 changes: 123 additions & 1 deletion collectors/monitoring_metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,15 @@

package collectors

import "testing"
import (
"fmt"
"sync"
"testing"
"time"

"github.com/prometheus/client_golang/prometheus"
"google.golang.org/api/monitoring/v3"
)

func TestNormalizeMetricName(t *testing.T) {
t.Parallel()
Expand All @@ -24,3 +32,117 @@ func TestNormalizeMetricName(t *testing.T) {
t.Fatalf("normalizeMetricName() = %q, want %q", got, want)
}
}

type collectFunc func(ch chan<- prometheus.Metric)

func (f collectFunc) Describe(chan<- *prometheus.Desc) {}

func (f collectFunc) Collect(ch chan<- prometheus.Metric) { f(ch) }

func TestCustomMetricWithDifferentDescriptionsPerProjectGathers(t *testing.T) {
t.Parallel()

const metricType = "custom.googleapis.com/spark/executor/bigquery-metrics-source/bytesRead"
projects := []struct {
id string
description string
}{
{"project-a", "Auto-created custom metric."},
{"project-b", ""},
}

helpStore := NewMetricHelpStore()
collector := collectFunc(func(ch chan<- prometheus.Metric) {
for _, p := range projects {
descriptor := &monitoring.MetricDescriptor{Type: metricType, Description: p.description}
tsm, err := newTimeSeriesMetrics(descriptor, ch, false, nil, nil, false, helpStore)
if err != nil {
t.Errorf("newTimeSeriesMetrics: %v", err)
return
}
ts := &monitoring.TimeSeries{
Metric: &monitoring.Metric{Type: metricType},
Resource: &monitoring.MonitoredResource{Type: "cloud_dataproc_batch"},
}
tsm.CollectNewConstMetric(ts, time.Now(), []string{"project_id"}, prometheus.CounterValue, 1, []string{p.id}, "CUMULATIVE")
}
})

registry := prometheus.NewRegistry()
registry.MustRegister(collector)
families, err := registry.Gather()
if err != nil {
t.Fatalf("Gather() error = %v", err)
}
if len(families) != 1 {
t.Fatalf("Gather() returned %d families, want 1", len(families))
}
if got, want := families[0].GetHelp(), projects[0].description; got != want {
t.Fatalf("family %s has help %q, want the first description seen, %q", families[0].GetName(), got, want)
}
if got := len(families[0].GetMetric()); got != 2 {
t.Fatalf("family %s has %d metrics, want 2", families[0].GetName(), got)
}
}

func TestMetricHelpStoreKeepsFirstDescriptionUnderConcurrency(t *testing.T) {
t.Parallel()

store := NewMetricHelpStore()
const callers = 64
got := make([]string, callers)
var wg sync.WaitGroup
for i := range callers {
wg.Add(1)
go func() {
defer wg.Done()
got[i] = store.Help("metric_name", fmt.Sprintf("description %d", i))
}()
}
wg.Wait()

for i, help := range got {
if help != got[0] {
t.Fatalf("caller %d got help %q, caller 0 got %q", i, help, got[0])
}
}
}

func TestMetricHelpStoreZeroValue(t *testing.T) {
t.Parallel()

var store MetricHelpStore
if got := store.Help("metric_name", "first"); got != "first" {
t.Fatalf("Help() = %q, want %q", got, "first")
}
if got := store.Help("metric_name", "second"); got != "first" {
t.Fatalf("Help() = %q, want %q", got, "first")
}
}

func TestMetricHelpStoreDropsUnusedNames(t *testing.T) {
t.Parallel()

now := time.Unix(0, 0)
store := &MetricHelpStore{now: func() time.Time { return now }}
store.Help("live_metric", "live")
store.Help("gone_metric", "gone")

// live_metric keeps being reported, gone_metric does not.
for range 4 {
now = now.Add(metricHelpTTL / 2)
store.Help("live_metric", "other")
}
// A new name triggers the sweep.
store.Help("new_metric", "new")

if got := store.Help("live_metric", "other"); got != "live" {
t.Fatalf("Help(live_metric) = %q, want %q", got, "live")
}
if got := store.Help("gone_metric", "replaced"); got != "replaced" {
t.Fatalf("Help(gone_metric) = %q, want %q", got, "replaced")
}
if got := len(store.entries); got != 3 {
t.Fatalf("store holds %d names, want 3", got)
}
}
6 changes: 5 additions & 1 deletion collectors/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type Runtime struct {
counterStoreFactory CounterStoreFactory
histogramStoreFactory HistogramStoreFactory
cache *collectorCache
helpStore *MetricHelpStore
discoverProjectIDs func(ctx context.Context, filter string) ([]string, error)
}

Expand Down Expand Up @@ -105,6 +106,7 @@ func NewRuntime(ctx context.Context, logger *slog.Logger, cfg *config.Config, co
logger: logger,
counterStoreFactory: counterFactory,
histogramStoreFactory: histogramFactory,
helpStore: NewMetricHelpStore(),
discoverProjectIDs: discoverProjectIDs,
}, nil
}
Expand Down Expand Up @@ -209,10 +211,12 @@ func (r *Runtime) collectorFor(projectID string, prefixFilter []string) (*Monito

func (r *Runtime) newCollector(projectID string, prefixFilter []string) (*MonitoringCollector, error) {
filtered := r.filterMetricTypePrefixes(prefixFilter)
opts := monitoringCollectorOptionsForPrefixes(r.cfg, filtered)
opts.MetricHelpStore = r.helpStore
return NewMonitoringCollector(
projectID,
r.service,
monitoringCollectorOptionsForPrefixes(r.cfg, filtered),
opts,
r.logger,
r.counterStoreFactory(r.logger, r.cfg.AggregateDeltasTTL),
r.histogramStoreFactory(r.logger, r.cfg.AggregateDeltasTTL),
Expand Down
23 changes: 23 additions & 0 deletions collectors/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,3 +421,26 @@ func TestWithCacheSharesProjectIDsPointer(t *testing.T) {
t.Fatalf("sibling.Collectors() returned %d collectors after original's projectIDs updated, want 2", len(cs))
}
}

func TestRuntimeCollectorsShareMetricHelpStore(t *testing.T) {
t.Parallel()

cfg := &config.Config{MetricsPrefixes: []string{"custom.googleapis.com/"}}
r := newTestRuntime(t, cfg, []string{"project-a", "project-b"}, nil)
r.counterStoreFactory = func(_ *slog.Logger, _ time.Duration) DeltaCounterStore { return nil }
r.histogramStoreFactory = func(_ *slog.Logger, _ time.Duration) DeltaHistogramStore { return nil }
r.helpStore = NewMetricHelpStore()

collectors, err := r.WithCache().Collectors()
if err != nil {
t.Fatalf("Collectors() error = %v", err)
}
if len(collectors) != 2 {
t.Fatalf("Collectors() returned %d collectors, want 2", len(collectors))
}
for _, c := range collectors {
if c.helpStore != r.helpStore {
t.Fatalf("collector for %s does not use the runtime's help store", c.projectID)
}
}
}