From 0db83f3f840f2993ca2a2b9286e4d5128ca6839b Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Tue, 6 Oct 2026 23:15:28 -0700 Subject: [PATCH 1/2] feat: replicated tables --- .changeset/replicated-engine.md | 5 + .github/workflows/chart-test.yml | 10 +- charts/clickstack/README.md | 8 +- charts/clickstack/templates/_helpers.tpl | 78 ++++- .../clickstack/tests/app-configmap_test.yaml | 50 +++ .../tests/clickhouse-database_test.yaml | 286 ++++++++++++++++++ .../tests/clickhouse-promql_test.yaml | 2 +- .../tests/hyperdx-rollout-checksums_test.yaml | 10 +- charts/clickstack/values.yaml | 35 ++- docs/UPGRADE.md | 33 ++ integration-tests/README.md | 10 +- integration-tests/full-stack/assert.sh | 10 + integration-tests/run-suite.sh | 44 ++- scripts/smoke-test.sh | 39 ++- 14 files changed, 590 insertions(+), 30 deletions(-) create mode 100644 .changeset/replicated-engine.md create mode 100644 charts/clickstack/tests/clickhouse-database_test.yaml diff --git a/.changeset/replicated-engine.md b/.changeset/replicated-engine.md new file mode 100644 index 00000000..3a8e9e4c --- /dev/null +++ b/.changeset/replicated-engine.md @@ -0,0 +1,5 @@ +--- +"helm-charts": minor +--- + +feat: fresh installs with the bundled ClickHouse write ClickStack tables to a dedicated `otel` database, created as Replicated with `Replicated*` tables. `helm upgrade` keeps the database and engines of existing installs, and an external ClickHouse keeps `default`. `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE` now also drives the default sources and ClickHouse user grants. `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE` keeps existing installs writing to `default` when the chart is rendered without cluster access (ArgoCD, `helm template`). See docs/UPGRADE.md. diff --git a/.github/workflows/chart-test.yml b/.github/workflows/chart-test.yml index afd3c3b2..a6349a37 100644 --- a/.github/workflows/chart-test.yml +++ b/.github/workflows/chart-test.yml @@ -32,7 +32,7 @@ jobs: echo "Discovered suites: $SUITES" integration-test: - name: integration-test (${{ matrix.suite }}${{ matrix.mode == 'upgrade' && ', upgrade' || '' }}) + name: integration-test (${{ matrix.suite }}${{ matrix.mode != 'install' && format(', {0}', matrix.mode) || '' }}) needs: discover runs-on: ubuntu-latest timeout-minutes: 60 @@ -41,7 +41,8 @@ jobs: matrix: suite: ${{ fromJson(needs.discover.outputs.suites) }} # PRs also upgrade from the base branch, to catch changes that only break existing releases. - mode: ${{ github.event_name == 'pull_request' && fromJson('["install","upgrade"]') || fromJson('["install"]') }} + # upgrade-template applies the upgrade with helm template | kubectl apply, as Argo CD does. + mode: ${{ github.event_name == 'pull_request' && fromJson('["install","upgrade","upgrade-template"]') || fromJson('["install"]') }} env: SUITE: ${{ matrix.suite }} steps: @@ -51,7 +52,7 @@ jobs: persist-credentials: false - name: Checkout base branch - if: matrix.mode == 'upgrade' + if: matrix.mode != 'install' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.base.sha }} @@ -90,7 +91,8 @@ jobs: - name: Run integration test suite env: - UPGRADE_FROM: ${{ matrix.mode == 'upgrade' && format('{0}/.upgrade-base', github.workspace) || '' }} + UPGRADE_FROM: ${{ matrix.mode != 'install' && format('{0}/.upgrade-base', github.workspace) || '' }} + UPGRADE_METHOD: ${{ matrix.mode == 'upgrade-template' && 'template' || 'helm' }} run: | chmod +x integration-tests/run-suite.sh integration-tests/run-suite.sh "$SUITE" diff --git a/charts/clickstack/README.md b/charts/clickstack/README.md index 3a0f2d9c..296ae8b0 100644 --- a/charts/clickstack/README.md +++ b/charts/clickstack/README.md @@ -46,16 +46,16 @@ Documented values below. Undocumented keys (operator CRD passthrough fields, sub | Key | Type | Default | Description | |-----|------|---------|-------------| | additionalManifests | list | `[]` | Additional Kubernetes manifests deployed alongside the chart (supports tpl). See docs/ADDITIONAL-MANIFESTS.md for examples. | -| clickhouse | object | `{"cluster":{"spec":{"containerTemplate":{"image":{"repository":"clickhouse/clickhouse-server","tag":"26.8.37.168-alpine"},"resources":{"limits":{"memory":"2Gi"},"requests":{"cpu":"500m","memory":"2Gi"}}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"keeperClusterRef":{"name":"{{ include \"clickstack.clickhouse.keeper\" . }}"},"replicas":1,"settings":{"extraConfig":{"asynchronous_metric_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"keep_alive_timeout":64,"max_concurrent_queries":100,"max_connections":4096,"metric_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"part_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"query_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"text_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"}},"extraUsersConfig":{"users":{"app":{"grants":[{"query":"GRANT SHOW ON *.*, SELECT ON system.*, SELECT ON default.*, CREATE TEMPORARY TABLE ON *.*"}],"password":"{{ .Values.clickstack.secrets.CLICKHOUSE_APP_PASSWORD }}","profile":"default"},"otelcollector":{"grants":[{"query":"GRANT SELECT,INSERT,CREATE,SHOW ON default.*"}],"password":"{{ .Values.clickstack.secrets.CLICKHOUSE_PASSWORD }}","profile":"default"}}},"logger":{"count":10,"level":"information","size":"100M"}},"shards":1}},"enabled":true,"keeper":{"spec":{"containerTemplate":{"image":{"repository":"clickhouse/clickhouse-keeper","tag":"26.8.37.168-alpine"}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"5Gi"}}},"replicas":1}},"nativePort":9000,"port":8123,"prometheus":{"enabled":true,"port":9363}}` | In-cluster ClickHouse via ClickHouseCluster CR. Set enabled: false for external ClickHouse. | -| clickhouse.cluster | object | `{"spec":{"containerTemplate":{"image":{"repository":"clickhouse/clickhouse-server","tag":"26.8.37.168-alpine"},"resources":{"limits":{"memory":"2Gi"},"requests":{"cpu":"500m","memory":"2Gi"}}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"keeperClusterRef":{"name":"{{ include \"clickstack.clickhouse.keeper\" . }}"},"replicas":1,"settings":{"extraConfig":{"asynchronous_metric_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"keep_alive_timeout":64,"max_concurrent_queries":100,"max_connections":4096,"metric_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"part_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"query_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"text_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"}},"extraUsersConfig":{"users":{"app":{"grants":[{"query":"GRANT SHOW ON *.*, SELECT ON system.*, SELECT ON default.*, CREATE TEMPORARY TABLE ON *.*"}],"password":"{{ .Values.clickstack.secrets.CLICKHOUSE_APP_PASSWORD }}","profile":"default"},"otelcollector":{"grants":[{"query":"GRANT SELECT,INSERT,CREATE,SHOW ON default.*"}],"password":"{{ .Values.clickstack.secrets.CLICKHOUSE_PASSWORD }}","profile":"default"}}},"logger":{"count":10,"level":"information","size":"100M"}},"shards":1}}` | ClickHouseCluster CRD spec (rendered verbatim). See ClickHouse Operator docs. | +| clickhouse | object | `{"cluster":{"spec":{"containerTemplate":{"image":{"repository":"clickhouse/clickhouse-server","tag":"26.8.37.168-alpine"},"resources":{"limits":{"memory":"2Gi"},"requests":{"cpu":"500m","memory":"2Gi"}}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"keeperClusterRef":{"name":"{{ include \"clickstack.clickhouse.keeper\" . }}"},"replicas":1,"settings":{"extraConfig":{"asynchronous_metric_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"keep_alive_timeout":64,"max_concurrent_queries":100,"max_connections":4096,"metric_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"part_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"query_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"text_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"}},"extraUsersConfig":{"users":{"app":{"grants":[{"query":"GRANT SHOW ON *.*, SELECT ON system.*, {{ include \"clickstack.clickhouse.databaseGrants\" (dict \"privileges\" \"SELECT\" \"context\" .) }}, CREATE TEMPORARY TABLE ON *.*"}],"password":"{{ .Values.clickstack.secrets.CLICKHOUSE_APP_PASSWORD }}","profile":"default"},"otelcollector":{"grants":[{"query":"GRANT {{ include \"clickstack.clickhouse.databaseGrants\" (dict \"privileges\" \"SELECT,INSERT,CREATE,SHOW\" \"context\" .) }}"}],"password":"{{ .Values.clickstack.secrets.CLICKHOUSE_PASSWORD }}","profile":"default"}}},"logger":{"count":10,"level":"information","size":"100M"}},"shards":1}},"enabled":true,"keeper":{"spec":{"containerTemplate":{"image":{"repository":"clickhouse/clickhouse-keeper","tag":"26.8.37.168-alpine"}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"5Gi"}}},"replicas":1}},"nativePort":9000,"port":8123,"prometheus":{"enabled":true,"port":9363}}` | In-cluster ClickHouse via ClickHouseCluster CR. Set enabled: false for external ClickHouse. | +| clickhouse.cluster | object | `{"spec":{"containerTemplate":{"image":{"repository":"clickhouse/clickhouse-server","tag":"26.8.37.168-alpine"},"resources":{"limits":{"memory":"2Gi"},"requests":{"cpu":"500m","memory":"2Gi"}}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"10Gi"}}},"keeperClusterRef":{"name":"{{ include \"clickstack.clickhouse.keeper\" . }}"},"replicas":1,"settings":{"extraConfig":{"asynchronous_metric_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"keep_alive_timeout":64,"max_concurrent_queries":100,"max_connections":4096,"metric_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"part_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"query_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"},"text_log":{"ttl":"event_date + INTERVAL 30 DAY DELETE"}},"extraUsersConfig":{"users":{"app":{"grants":[{"query":"GRANT SHOW ON *.*, SELECT ON system.*, {{ include \"clickstack.clickhouse.databaseGrants\" (dict \"privileges\" \"SELECT\" \"context\" .) }}, CREATE TEMPORARY TABLE ON *.*"}],"password":"{{ .Values.clickstack.secrets.CLICKHOUSE_APP_PASSWORD }}","profile":"default"},"otelcollector":{"grants":[{"query":"GRANT {{ include \"clickstack.clickhouse.databaseGrants\" (dict \"privileges\" \"SELECT,INSERT,CREATE,SHOW\" \"context\" .) }}"}],"password":"{{ .Values.clickstack.secrets.CLICKHOUSE_PASSWORD }}","profile":"default"}}},"logger":{"count":10,"level":"information","size":"100M"}},"shards":1}}` | ClickHouseCluster CRD spec (rendered verbatim). See ClickHouse Operator docs. | | clickhouse.keeper | object | `{"spec":{"containerTemplate":{"image":{"repository":"clickhouse/clickhouse-keeper","tag":"26.8.37.168-alpine"}},"dataVolumeClaimSpec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"5Gi"}}},"replicas":1}}` | KeeperCluster CRD spec (rendered verbatim). See ClickHouse Operator docs. | | clickhouse.nativePort | int | `9000` | Native TCP port used in connection strings. | | clickhouse.port | int | `8123` | HTTP interface port for in-cluster connections. | | clickhouse.prometheus.enabled | bool | `true` | Expose ClickHouse Prometheus metrics endpoint. | | clickstack.autoscaling | object | `{"enabled":false,"spec":{}}` | HorizontalPodAutoscaler for HyperDX. spec is passed through verbatim (autoscaling/v2). Do not include scaleTargetRef — the chart auto-wires it. When enabled, omit deployment.replicas; disabling autoscaling reverts to deployment.replicas (default 1). | -| clickstack.config | object | `{"API_PORT":"8000","APP_PORT":"3000","CLICKHOUSE_ENDPOINT":"tcp://{{ include \"clickstack.clickhouse.svc\" . }}:{{ .Values.clickhouse.nativePort }}?dial_timeout=10s","CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT":"{{ include \"clickstack.clickhouse.svc\" . }}:{{ .Values.clickhouse.prometheus.port }}","CLICKHOUSE_SERVER_ENDPOINT":"{{ include \"clickstack.clickhouse.svc\" . }}:{{ .Values.clickhouse.nativePort }}","CLICKHOUSE_USER":"otelcollector","ENABLE_PROMQL":"{{ if .Values.clickstack.promql.enabled }}true{{ else }}false{{ end }}","FRONTEND_URL":"http://localhost:3000","HDX_EXPORTER_ENABLED":"{{ if .Values.clickstack.config.NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT }}true{{ else }}false{{ end }}","HYPERDX_API_PORT":"8000","HYPERDX_APP_PORT":"3000","HYPERDX_LOG_LEVEL":"info","HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE":"default","HYPERDX_OTEL_EXPORTER_TABLES_TTL":"720h","MONGO_URI":"mongodb://hyperdx:{{ .Values.clickstack.secrets.MONGODB_PASSWORD }}@{{ include \"clickstack.mongodb.svc\" . }}:27017/hyperdx?authSource=hyperdx","NEXT_PUBLIC_ENABLE_PROMQL":"{{ if .Values.clickstack.promql.enabled }}true{{ else }}false{{ end }}","OPAMP_PORT":"4320","OPAMP_SERVER_URL":"http://{{ include \"clickstack.hyperdx.fullname\" . }}:{{ .Values.clickstack.ports.opamp }}","OTEL_EXPORTER_OTLP_ENDPOINT":"http://{{ include \"clickstack.otel.fullname\" . }}:4318","OTEL_SERVICE_NAME":"hdx-oss-api","RUN_SCHEDULED_TASKS_EXTERNALLY":"false","USAGE_STATS_ENABLED":"true"}` | Shared non-sensitive environment variables (clickstack-config ConfigMap). Used by HyperDX and OTEL collector via envFrom. Values support tpl expressions. Override entries with plain strings to point at external services. Changes to the rendered ConfigMap roll the HyperDX Deployment; the otel-collector rolls on every helm upgrade. | +| clickstack.config | object | `{"API_PORT":"8000","APP_PORT":"3000","CLICKHOUSE_ENDPOINT":"tcp://{{ include \"clickstack.clickhouse.svc\" . }}:{{ .Values.clickhouse.nativePort }}?dial_timeout=10s","CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT":"{{ include \"clickstack.clickhouse.svc\" . }}:{{ .Values.clickhouse.prometheus.port }}","CLICKHOUSE_SERVER_ENDPOINT":"{{ include \"clickstack.clickhouse.svc\" . }}:{{ .Values.clickhouse.nativePort }}","CLICKHOUSE_USER":"otelcollector","ENABLE_PROMQL":"{{ if .Values.clickstack.promql.enabled }}true{{ else }}false{{ end }}","FRONTEND_URL":"http://localhost:3000","HDX_EXPORTER_ENABLED":"{{ if .Values.clickstack.config.NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT }}true{{ else }}false{{ end }}","HYPERDX_API_PORT":"8000","HYPERDX_APP_PORT":"3000","HYPERDX_LOG_LEVEL":"info","HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE":"{{ include \"clickstack.clickhouse.defaultDatabase\" . }}","HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE":"{{ include \"clickstack.clickhouse.defaultFallbackDatabase\" . }}","HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED":"{{ include \"clickstack.clickhouse.defaultReplicated\" . }}","HYPERDX_OTEL_EXPORTER_TABLES_TTL":"720h","MONGO_URI":"mongodb://hyperdx:{{ .Values.clickstack.secrets.MONGODB_PASSWORD }}@{{ include \"clickstack.mongodb.svc\" . }}:27017/hyperdx?authSource=hyperdx","NEXT_PUBLIC_ENABLE_PROMQL":"{{ if .Values.clickstack.promql.enabled }}true{{ else }}false{{ end }}","OPAMP_PORT":"4320","OPAMP_SERVER_URL":"http://{{ include \"clickstack.hyperdx.fullname\" . }}:{{ .Values.clickstack.ports.opamp }}","OTEL_EXPORTER_OTLP_ENDPOINT":"http://{{ include \"clickstack.otel.fullname\" . }}:4318","OTEL_SERVICE_NAME":"hdx-oss-api","RUN_SCHEDULED_TASKS_EXTERNALLY":"false","USAGE_STATS_ENABLED":"true"}` | Shared non-sensitive environment variables (clickstack-config ConfigMap). Used by HyperDX and OTEL collector via envFrom. Values support tpl expressions. Override entries with plain strings to point at external services. Changes to the rendered ConfigMap roll the HyperDX Deployment; the otel-collector rolls on every helm upgrade. | | clickstack.dashboards | object | `{"configMaps":{},"enabled":false,"includeReleaseNamespace":true,"namespaces":[],"rbac":{"create":true},"resources":{"limits":{"cpu":"50m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}},"sidecarImage":"kiwigrid/k8s-sidecar:2.10.1","sidecarPullPolicy":"IfNotPresent"}` | Dashboard provisioning: upserts dashboard JSON into MongoDB (matched by name, never deletes). Provide dashboards inline via configMaps, or label ConfigMaps hyperdx.io/dashboard: "true" for a k8s-sidecar watcher to discover. | -| clickstack.deployment | object | `{"annotations":{},"containerSecurityContext":{},"defaultConnections":"[\n {\n \"name\": \"Local ClickHouse\",\n \"host\": \"http://{{ include \"clickstack.clickhouse.svc\" . }}:8123\",\n \"port\": 8123,\n \"username\": \"app\",\n \"password\": \"{{ .Values.clickstack.secrets.CLICKHOUSE_APP_PASSWORD }}\"\n }\n]\n","defaultSources":"[\n {\n \"from\": {\n \"databaseName\": \"default\",\n \"tableName\": \"otel_logs\"\n },\n \"kind\": \"log\",\n \"timestampValueExpression\": \"Timestamp\",\n \"name\": \"Logs\",\n \"displayedTimestampValueExpression\": \"Timestamp\",\n \"implicitColumnExpression\": \"Body\",\n \"serviceNameExpression\": \"ServiceName\",\n \"bodyExpression\": \"Body\",\n \"eventAttributesExpression\": \"LogAttributes\",\n \"resourceAttributesExpression\": \"ResourceAttributes\",\n \"defaultTableSelectExpression\": \"Timestamp,ServiceName,SeverityText,Body\",\n \"severityTextExpression\": \"SeverityText\",\n \"traceIdExpression\": \"TraceId\",\n \"spanIdExpression\": \"SpanId\",\n \"connection\": \"Local ClickHouse\",\n \"traceSourceId\": \"Traces\",\n \"metricSourceId\": \"Metrics\"\n },\n {\n \"from\": {\n \"databaseName\": \"default\",\n \"tableName\": \"otel_traces\"\n },\n \"kind\": \"trace\",\n \"timestampValueExpression\": \"Timestamp\",\n \"name\": \"Traces\",\n \"displayedTimestampValueExpression\": \"Timestamp\",\n \"implicitColumnExpression\": \"SpanName\",\n \"serviceNameExpression\": \"ServiceName\",\n \"eventAttributesExpression\": \"SpanAttributes\",\n \"resourceAttributesExpression\": \"ResourceAttributes\",\n \"defaultTableSelectExpression\": \"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName\",\n \"traceIdExpression\": \"TraceId\",\n \"spanIdExpression\": \"SpanId\",\n \"durationExpression\": \"Duration\",\n \"durationPrecision\": 9,\n \"parentSpanIdExpression\": \"ParentSpanId\",\n \"spanNameExpression\": \"SpanName\",\n \"spanKindExpression\": \"SpanKind\",\n \"statusCodeExpression\": \"StatusCode\",\n \"statusMessageExpression\": \"StatusMessage\",\n \"spanEventsValueExpression\": \"Events\",\n \"spanLinksValueExpression\": \"Links\",\n \"connection\": \"Local ClickHouse\",\n \"logSourceId\": \"Logs\",\n \"sessionSourceId\": \"Sessions\",\n \"metricSourceId\": \"Metrics\"\n },\n {\n \"from\": {\n \"databaseName\": \"default\",\n \"tableName\": \"\"\n },\n \"kind\": \"metric\",\n \"timestampValueExpression\": \"TimeUnix\",\n \"name\": \"Metrics\",\n \"resourceAttributesExpression\": \"ResourceAttributes\",\n \"metricTables\": {\n \"gauge\": \"otel_metrics_gauge\",\n \"histogram\": \"otel_metrics_histogram\",\n \"sum\": \"otel_metrics_sum\",\n \"summary\": \"otel_metrics_summary\",\n \"exponential histogram\": \"otel_metrics_exponential_histogram\"\n },\n \"connection\": \"Local ClickHouse\",\n \"logSourceId\": \"Logs\"\n },\n {\n \"from\": {\n \"databaseName\": \"default\",\n \"tableName\": \"hyperdx_sessions\"\n },\n \"kind\": \"session\",\n \"timestampValueExpression\": \"TimestampTime\",\n \"name\": \"Sessions\",\n \"resourceAttributesExpression\": \"ResourceAttributes\",\n \"connection\": \"Local ClickHouse\",\n \"traceSourceId\": \"Traces\"\n },\n {\n \"from\": {\n \"databaseName\": \"default\",\n \"tableName\": \"metrics_ts\"\n },\n \"kind\": \"promql\",\n \"timestampValueExpression\": \"timestamp\",\n \"name\": \"PromQL\",\n \"connection\": \"Local ClickHouse\"\n }\n]\n","deploymentAnnotations":{},"env":[],"existingConfigConnectionsKey":"connections.json","existingConfigSecret":"","existingConfigSourcesKey":"sources.json","image":{"pullPolicy":"IfNotPresent","repository":"docker.hyperdx.io/hyperdx/hyperdx","tag":null},"initContainers":[],"labels":{},"livenessProbe":{"enabled":true,"failureThreshold":3,"initialDelaySeconds":10,"path":"/health","periodSeconds":30,"timeoutSeconds":5},"nodeSelector":{},"podAnnotations":{},"podLabels":{},"podSecurityContext":{},"priorityClassName":"","readinessProbe":{"enabled":true,"failureThreshold":3,"initialDelaySeconds":1,"path":"/ready","periodSeconds":10,"timeoutSeconds":5},"replicas":1,"resources":{},"tolerations":[],"topologySpreadConstraints":[],"useExistingConfigSecret":false,"volumeMounts":[],"volumes":[],"waitForMongodb":{"image":"busybox@sha256:1fcf5df59121b92d61e066df1788e8df0cc35623f5d62d9679a41e163b6a0cdb","pullPolicy":"IfNotPresent"}}` | HyperDX Deployment spec (image, probes, scheduling, init containers, volumes). defaultConnections/defaultSources JSON: set to "" to disable; use useExistingConfigSecret for external secrets. | +| clickstack.deployment | object | `{"annotations":{},"containerSecurityContext":{},"defaultConnections":"[\n {\n \"name\": \"Local ClickHouse\",\n \"host\": \"http://{{ include \"clickstack.clickhouse.svc\" . }}:8123\",\n \"port\": 8123,\n \"username\": \"app\",\n \"password\": \"{{ .Values.clickstack.secrets.CLICKHOUSE_APP_PASSWORD }}\"\n }\n]\n","defaultSources":"[\n {\n \"from\": {\n \"databaseName\": \"{{ include \"clickstack.clickhouse.database\" . }}\",\n \"tableName\": \"otel_logs\"\n },\n \"kind\": \"log\",\n \"timestampValueExpression\": \"Timestamp\",\n \"name\": \"Logs\",\n \"displayedTimestampValueExpression\": \"Timestamp\",\n \"implicitColumnExpression\": \"Body\",\n \"serviceNameExpression\": \"ServiceName\",\n \"bodyExpression\": \"Body\",\n \"eventAttributesExpression\": \"LogAttributes\",\n \"resourceAttributesExpression\": \"ResourceAttributes\",\n \"defaultTableSelectExpression\": \"Timestamp,ServiceName,SeverityText,Body\",\n \"severityTextExpression\": \"SeverityText\",\n \"traceIdExpression\": \"TraceId\",\n \"spanIdExpression\": \"SpanId\",\n \"connection\": \"Local ClickHouse\",\n \"traceSourceId\": \"Traces\",\n \"metricSourceId\": \"Metrics\"\n },\n {\n \"from\": {\n \"databaseName\": \"{{ include \"clickstack.clickhouse.database\" . }}\",\n \"tableName\": \"otel_traces\"\n },\n \"kind\": \"trace\",\n \"timestampValueExpression\": \"Timestamp\",\n \"name\": \"Traces\",\n \"displayedTimestampValueExpression\": \"Timestamp\",\n \"implicitColumnExpression\": \"SpanName\",\n \"serviceNameExpression\": \"ServiceName\",\n \"eventAttributesExpression\": \"SpanAttributes\",\n \"resourceAttributesExpression\": \"ResourceAttributes\",\n \"defaultTableSelectExpression\": \"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName\",\n \"traceIdExpression\": \"TraceId\",\n \"spanIdExpression\": \"SpanId\",\n \"durationExpression\": \"Duration\",\n \"durationPrecision\": 9,\n \"parentSpanIdExpression\": \"ParentSpanId\",\n \"spanNameExpression\": \"SpanName\",\n \"spanKindExpression\": \"SpanKind\",\n \"statusCodeExpression\": \"StatusCode\",\n \"statusMessageExpression\": \"StatusMessage\",\n \"spanEventsValueExpression\": \"Events\",\n \"spanLinksValueExpression\": \"Links\",\n \"connection\": \"Local ClickHouse\",\n \"logSourceId\": \"Logs\",\n \"sessionSourceId\": \"Sessions\",\n \"metricSourceId\": \"Metrics\"\n },\n {\n \"from\": {\n \"databaseName\": \"{{ include \"clickstack.clickhouse.database\" . }}\",\n \"tableName\": \"\"\n },\n \"kind\": \"metric\",\n \"timestampValueExpression\": \"TimeUnix\",\n \"name\": \"Metrics\",\n \"resourceAttributesExpression\": \"ResourceAttributes\",\n \"metricTables\": {\n \"gauge\": \"otel_metrics_gauge\",\n \"histogram\": \"otel_metrics_histogram\",\n \"sum\": \"otel_metrics_sum\",\n \"summary\": \"otel_metrics_summary\",\n \"exponential histogram\": \"otel_metrics_exponential_histogram\"\n },\n \"connection\": \"Local ClickHouse\",\n \"logSourceId\": \"Logs\"\n },\n {\n \"from\": {\n \"databaseName\": \"{{ include \"clickstack.clickhouse.database\" . }}\",\n \"tableName\": \"hyperdx_sessions\"\n },\n \"kind\": \"session\",\n \"timestampValueExpression\": \"TimestampTime\",\n \"name\": \"Sessions\",\n \"resourceAttributesExpression\": \"ResourceAttributes\",\n \"connection\": \"Local ClickHouse\",\n \"traceSourceId\": \"Traces\"\n },\n {\n \"from\": {\n \"databaseName\": \"{{ include \"clickstack.clickhouse.database\" . }}\",\n \"tableName\": \"metrics_ts\"\n },\n \"kind\": \"promql\",\n \"timestampValueExpression\": \"timestamp\",\n \"name\": \"PromQL\",\n \"connection\": \"Local ClickHouse\"\n }\n]\n","deploymentAnnotations":{},"env":[],"existingConfigConnectionsKey":"connections.json","existingConfigSecret":"","existingConfigSourcesKey":"sources.json","image":{"pullPolicy":"IfNotPresent","repository":"docker.hyperdx.io/hyperdx/hyperdx","tag":null},"initContainers":[],"labels":{},"livenessProbe":{"enabled":true,"failureThreshold":3,"initialDelaySeconds":10,"path":"/health","periodSeconds":30,"timeoutSeconds":5},"nodeSelector":{},"podAnnotations":{},"podLabels":{},"podSecurityContext":{},"priorityClassName":"","readinessProbe":{"enabled":true,"failureThreshold":3,"initialDelaySeconds":1,"path":"/ready","periodSeconds":10,"timeoutSeconds":5},"replicas":1,"resources":{},"tolerations":[],"topologySpreadConstraints":[],"useExistingConfigSecret":false,"volumeMounts":[],"volumes":[],"waitForMongodb":{"image":"busybox@sha256:1fcf5df59121b92d61e066df1788e8df0cc35623f5d62d9679a41e163b6a0cdb","pullPolicy":"IfNotPresent"}}` | HyperDX Deployment spec (image, probes, scheduling, init containers, volumes). defaultConnections/defaultSources JSON: set to "" to disable; use useExistingConfigSecret for external secrets. | | clickstack.deployment.env | list | `[]` | Extra environment variables for the HyperDX container, rendered verbatim (supports `valueFrom`, e.g. `secretKeyRef`). | | clickstack.deployment.existingConfigSecret | string | `""` | Secret containing connections and sources JSON when useExistingConfigSecret is true. | | clickstack.deployment.image.repository | string | `"docker.hyperdx.io/hyperdx/hyperdx"` | HyperDX container image repository. | diff --git a/charts/clickstack/templates/_helpers.tpl b/charts/clickstack/templates/_helpers.tpl index df8b67a3..24e12f7e 100644 --- a/charts/clickstack/templates/_helpers.tpl +++ b/charts/clickstack/templates/_helpers.tpl @@ -393,7 +393,7 @@ repeated because ClickHouse ignores its legacy prometheus.endpoint settings once handlers block exists. */}} {{- define "clickstack.clickhouse.promqlSettings" -}} -{{- $database := .Values.clickstack.config.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE | default "default" }} +{{- $database := include "clickstack.clickhouse.database" . }} extraConfig: prometheus: handlers: @@ -466,3 +466,79 @@ stringData: {{ $k }}: {{ tpl (toString $v) $ | quote }} {{- end }} {{- end }} + +{{/* +Whether the bundled ClickHouse is enabled. Accepts a bool or a "true"/"false" string +(e.g. from --set-string). Renders "true" or "". +*/}} +{{- define "clickstack.clickhouse.enabled" -}} +{{- if eq (lower (toString .Values.clickhouse.enabled)) "true" }}true{{- end -}} +{{- end }} + +{{/* +Default ClickStack database and replicated seed flag, as JSON with "database" and "replicated". +- External ClickHouse: default, not replicated. +- Upgrade: reuse the existing clickstack-config ConfigMap, so installs keep their database and + engines. A missing key means default or "false", matching charts that predate these keys. +- Fresh install, or a render without cluster access (helm template): Replicated otel database. +*/}} +{{- define "clickstack.clickhouse.seedDefaults" -}} +{{- $defaults := dict "database" "otel" "replicated" "true" -}} +{{- if not (include "clickstack.clickhouse.enabled" .) -}} +{{- $defaults = dict "database" "default" "replicated" "false" -}} +{{- else -}} +{{- $existing := lookup "v1" "ConfigMap" .Release.Namespace "clickstack-config" -}} +{{- if $existing -}} +{{- $data := default (dict) $existing.data -}} +{{- $defaults = dict + "database" (default "default" (get $data "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE")) + "replicated" (default "false" (get $data "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED")) -}} +{{- end -}} +{{- end -}} +{{- toJson $defaults -}} +{{- end }} + +{{/* +Default for clickstack.config.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE. See clickstack.clickhouse.seedDefaults. +*/}} +{{- define "clickstack.clickhouse.defaultDatabase" -}} +{{- (include "clickstack.clickhouse.seedDefaults" . | fromJson).database -}} +{{- end }} + +{{/* +Default for clickstack.config.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED. See clickstack.clickhouse.seedDefaults. +*/}} +{{- define "clickstack.clickhouse.defaultReplicated" -}} +{{- (include "clickstack.clickhouse.seedDefaults" . | fromJson).replicated -}} +{{- end }} + +{{/* +Default for clickstack.config.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: default for the +bundled ClickHouse, so renders without lookup keep existing installs on default, and empty otherwise. +*/}} +{{- define "clickstack.clickhouse.defaultFallbackDatabase" -}} +{{- if include "clickstack.clickhouse.enabled" . }}default{{- end -}} +{{- end }} + +{{/* +ClickHouse database for ClickStack tables: clickstack.config.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE +rendered through tpl, so the collector seed, default sources, PromQL handlers, and user grants always agree. +*/}} +{{- define "clickstack.clickhouse.database" -}} +{{- $config := default (dict) .Values.clickstack.config -}} +{{- tpl (toString (default "default" (index $config "HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE"))) . -}} +{{- end }} + +{{/* +ClickHouse grant elements for ClickStack tables. Always covers default.* so data written +before the dedicated database stays readable, plus the configured database when it differs. +Usage: include "clickstack.clickhouse.databaseGrants" (dict "privileges" "SELECT" "context" .) +*/}} +{{- define "clickstack.clickhouse.databaseGrants" -}} +{{- $database := include "clickstack.clickhouse.database" .context -}} +{{- $grants := list (printf "%s ON default.*" .privileges) -}} +{{- if ne $database "default" -}} +{{- $grants = append $grants (printf "%s ON %s.*" .privileges $database) -}} +{{- end -}} +{{- join ", " $grants -}} +{{- end }} diff --git a/charts/clickstack/tests/app-configmap_test.yaml b/charts/clickstack/tests/app-configmap_test.yaml index 3eaed896..28dd13bc 100644 --- a/charts/clickstack/tests/app-configmap_test.yaml +++ b/charts/clickstack/tests/app-configmap_test.yaml @@ -79,6 +79,56 @@ tests: path: data.NEXT_PUBLIC_ENABLE_PROMQL value: "false" + - it: should use the otel database for the bundled ClickHouse + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "otel" + + - it: should use the default database for an external ClickHouse + set: + clickhouse: + enabled: false + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "default" + + - it: should render a plain string database override + set: + clickstack: + config: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "clickstack" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "clickstack" + + - it: should enable replicated seeding for the bundled ClickHouse + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "true" + + - it: should disable replicated seeding for an external ClickHouse + set: + clickhouse: + enabled: false + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "false" + + - it: should allow opting out of replicated seeding + set: + clickstack: + config: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: "false" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "false" + - it: should fail when the removed 1.x usageStatsEnabled key is set set: hyperdx: diff --git a/charts/clickstack/tests/clickhouse-database_test.yaml b/charts/clickstack/tests/clickhouse-database_test.yaml new file mode 100644 index 00000000..60b6ef2c --- /dev/null +++ b/charts/clickstack/tests/clickhouse-database_test.yaml @@ -0,0 +1,286 @@ +suite: Test ClickStack Database Selection +templates: + - hyperdx/configmap.yaml + - hyperdx/deployment.yaml + - clickhouse/cluster.yaml +release: + namespace: clickstack-ns +# Each case checks the ConfigMap values, all five default sources, and the ClickHouse grants. +# The default sources define exactly five databaseName entries, so {5} matches means all of them. +tests: + - it: should use a Replicated otel database on a fresh install + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "otel" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "true" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE + value: "default" + template: hyperdx/configmap.yaml + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="DEFAULT_SOURCES")].value + pattern: '(?s)("databaseName": "otel".*){5}' + template: hyperdx/deployment.yaml + - equal: + path: spec.settings.extraUsersConfig.users.app.grants[0].query + value: "GRANT SHOW ON *.*, SELECT ON system.*, SELECT ON default.*, SELECT ON otel.*, CREATE TEMPORARY TABLE ON *.*" + template: clickhouse/cluster.yaml + - equal: + path: spec.settings.extraUsersConfig.users.otelcollector.grants[0].query + value: "GRANT SELECT,INSERT,CREATE,SHOW ON default.*, SELECT,INSERT,CREATE,SHOW ON otel.*" + template: clickhouse/cluster.yaml + + - it: should keep default and plain engines on upgrade from a chart without these keys + kubernetesProvider: + scheme: &configMapScheme + "v1/ConfigMap": + gvr: + version: "v1" + resource: "configmaps" + namespaced: true + objects: + - kind: ConfigMap + apiVersion: v1 + metadata: + name: clickstack-config + namespace: clickstack-ns + data: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "default" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "default" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "false" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE + value: "default" + template: hyperdx/configmap.yaml + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="DEFAULT_SOURCES")].value + pattern: '(?s)("databaseName": "default".*){5}' + template: hyperdx/deployment.yaml + # Same grants as charts before the otel database, so upgrades render no grant change. + - equal: + path: spec.settings.extraUsersConfig.users.app.grants[0].query + value: "GRANT SHOW ON *.*, SELECT ON system.*, SELECT ON default.*, CREATE TEMPORARY TABLE ON *.*" + template: clickhouse/cluster.yaml + - equal: + path: spec.settings.extraUsersConfig.users.otelcollector.grants[0].query + value: "GRANT SELECT,INSERT,CREATE,SHOW ON default.*" + template: clickhouse/cluster.yaml + + - it: should fall back to default when an existing ConfigMap has neither key + kubernetesProvider: + scheme: *configMapScheme + objects: + - kind: ConfigMap + apiVersion: v1 + metadata: + name: clickstack-config + namespace: clickstack-ns + data: + HYPERDX_LOG_LEVEL: "info" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "default" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "false" + template: hyperdx/configmap.yaml + + - it: should keep the otel database on upgrade from this chart + kubernetesProvider: + scheme: *configMapScheme + objects: + - kind: ConfigMap + apiVersion: v1 + metadata: + name: clickstack-config + namespace: clickstack-ns + data: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "otel" + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: "true" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "otel" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "true" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE + value: "default" + template: hyperdx/configmap.yaml + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="DEFAULT_SOURCES")].value + pattern: '(?s)("databaseName": "otel".*){5}' + template: hyperdx/deployment.yaml + - equal: + path: spec.settings.extraUsersConfig.users.app.grants[0].query + value: "GRANT SHOW ON *.*, SELECT ON system.*, SELECT ON default.*, SELECT ON otel.*, CREATE TEMPORARY TABLE ON *.*" + template: clickhouse/cluster.yaml + - equal: + path: spec.settings.extraUsersConfig.users.otelcollector.grants[0].query + value: "GRANT SELECT,INSERT,CREATE,SHOW ON default.*, SELECT,INSERT,CREATE,SHOW ON otel.*" + template: clickhouse/cluster.yaml + + - it: should ignore a clickstack-config ConfigMap in another namespace + kubernetesProvider: + scheme: *configMapScheme + objects: + - kind: ConfigMap + apiVersion: v1 + metadata: + name: clickstack-config + namespace: other-ns + data: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "default" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "otel" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "true" + template: hyperdx/configmap.yaml + + - it: should use default and plain engines for an external ClickHouse + set: + clickhouse: + enabled: false + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "default" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "false" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE + value: "" + template: hyperdx/configmap.yaml + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="DEFAULT_SOURCES")].value + pattern: '(?s)("databaseName": "default".*){5}' + template: hyperdx/deployment.yaml + - hasDocuments: + count: 0 + template: clickhouse/cluster.yaml + + - it: should treat a string "false" for clickhouse.enabled as an external ClickHouse + set: + clickhouse.enabled: "false" + kubernetesProvider: + scheme: *configMapScheme + objects: + - kind: ConfigMap + apiVersion: v1 + metadata: + name: clickstack-config + namespace: clickstack-ns + data: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "otel" + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: "true" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "default" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "false" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE + value: "" + template: hyperdx/configmap.yaml + + - it: should let explicit values win over an existing ConfigMap + set: + clickstack: + config: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "clickstack" + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: "false" + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: "" + kubernetesProvider: + scheme: *configMapScheme + objects: + - kind: ConfigMap + apiVersion: v1 + metadata: + name: clickstack-config + namespace: clickstack-ns + data: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "otel" + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: "true" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE + value: "clickstack" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED + value: "false" + template: hyperdx/configmap.yaml + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE + value: "" + template: hyperdx/configmap.yaml + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="DEFAULT_SOURCES")].value + pattern: '(?s)("databaseName": "clickstack".*){5}' + template: hyperdx/deployment.yaml + - equal: + path: spec.settings.extraUsersConfig.users.app.grants[0].query + value: "GRANT SHOW ON *.*, SELECT ON system.*, SELECT ON default.*, SELECT ON clickstack.*, CREATE TEMPORARY TABLE ON *.*" + template: clickhouse/cluster.yaml + - equal: + path: spec.settings.extraUsersConfig.users.otelcollector.grants[0].query + value: "GRANT SELECT,INSERT,CREATE,SHOW ON default.*, SELECT,INSERT,CREATE,SHOW ON clickstack.*" + template: clickhouse/cluster.yaml + + - it: should not duplicate the default grant when default is set explicitly + set: + clickstack: + config: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "default" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].env[?(@.name=="DEFAULT_SOURCES")].value + pattern: '(?s)("databaseName": "default".*){5}' + template: hyperdx/deployment.yaml + - equal: + path: spec.settings.extraUsersConfig.users.app.grants[0].query + value: "GRANT SHOW ON *.*, SELECT ON system.*, SELECT ON default.*, CREATE TEMPORARY TABLE ON *.*" + template: clickhouse/cluster.yaml + - equal: + path: spec.settings.extraUsersConfig.users.otelcollector.grants[0].query + value: "GRANT SELECT,INSERT,CREATE,SHOW ON default.*" + template: clickhouse/cluster.yaml + + - it: should render a plain string fallback database override + set: + clickstack: + config: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: "legacy" + asserts: + - equal: + path: data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE + value: "legacy" + template: hyperdx/configmap.yaml diff --git a/charts/clickstack/tests/clickhouse-promql_test.yaml b/charts/clickstack/tests/clickhouse-promql_test.yaml index 520a0708..9db75b67 100644 --- a/charts/clickstack/tests/clickhouse-promql_test.yaml +++ b/charts/clickstack/tests/clickhouse-promql_test.yaml @@ -12,7 +12,7 @@ tests: path: spec.settings.extraConfig.prometheus.handlers.remote_write_handler.handler value: type: remote_write - database: default + database: otel table: metrics_ts - equal: path: spec.settings.extraConfig.prometheus.handlers.remote_read_handler.url diff --git a/charts/clickstack/tests/hyperdx-rollout-checksums_test.yaml b/charts/clickstack/tests/hyperdx-rollout-checksums_test.yaml index 16d91cbf..f07dee93 100644 --- a/charts/clickstack/tests/hyperdx-rollout-checksums_test.yaml +++ b/charts/clickstack/tests/hyperdx-rollout-checksums_test.yaml @@ -20,7 +20,7 @@ tests: asserts: - equal: path: spec.template.metadata.annotations["checksum/clickstack-config"] - value: ab3037603573ead5d1c9f9878c4437fa49a4d615b4229dc9633b3c26c150a500 + value: 95a7bc6e790ae5206697add6a8ba440f29ecd856cf7572fdfa349aaf47c3541f - equal: path: spec.template.metadata.annotations["checksum/clickstack-secret"] value: 93200c574b88c22d434ac7d53d4f1105effa8d491a41f9102e131b7f5b919218 @@ -34,7 +34,7 @@ tests: asserts: - equal: path: spec.template.metadata.annotations["checksum/clickstack-config"] - value: bd473aedda4bbc33639c2e46f3948f8ad391c86204020d7e28dc42d511286afb + value: 58fbb3a60256066f2dbc54bb6e53215bb651eba85648121d76fcab20a32fae79 - equal: path: spec.template.metadata.annotations["checksum/clickstack-secret"] value: 93200c574b88c22d434ac7d53d4f1105effa8d491a41f9102e131b7f5b919218 @@ -48,7 +48,7 @@ tests: asserts: - equal: path: spec.template.metadata.annotations["checksum/clickstack-config"] - value: ab3037603573ead5d1c9f9878c4437fa49a4d615b4229dc9633b3c26c150a500 + value: 95a7bc6e790ae5206697add6a8ba440f29ecd856cf7572fdfa349aaf47c3541f - equal: path: spec.template.metadata.annotations["checksum/clickstack-secret"] value: 1adcdfb26c7114f9990031b28eb1ae1d7097dcf27c16487d16c8666da062a507 @@ -110,7 +110,7 @@ tests: value: preserved - equal: path: spec.template.metadata.annotations["checksum/clickstack-config"] - value: ab3037603573ead5d1c9f9878c4437fa49a4d615b4229dc9633b3c26c150a500 + value: 95a7bc6e790ae5206697add6a8ba440f29ecd856cf7572fdfa349aaf47c3541f - equal: path: spec.template.metadata.annotations["checksum/clickstack-secret"] value: 93200c574b88c22d434ac7d53d4f1105effa8d491a41f9102e131b7f5b919218 @@ -130,7 +130,7 @@ tests: asserts: - equal: path: spec.template.metadata.annotations["checksum/clickstack-config"] - value: ab3037603573ead5d1c9f9878c4437fa49a4d615b4229dc9633b3c26c150a500 + value: 95a7bc6e790ae5206697add6a8ba440f29ecd856cf7572fdfa349aaf47c3541f - equal: path: spec.template.metadata.annotations["checksum/clickstack-secret"] value: 93200c574b88c22d434ac7d53d4f1105effa8d491a41f9102e131b7f5b919218 diff --git a/charts/clickstack/values.yaml b/charts/clickstack/values.yaml index 5f537740..94416da2 100644 --- a/charts/clickstack/values.yaml +++ b/charts/clickstack/values.yaml @@ -124,7 +124,12 @@ clickstack: OTEL_SERVICE_NAME: "hdx-oss-api" USAGE_STATS_ENABLED: "true" OPAMP_PORT: "4320" - HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "default" + # Database for ClickStack tables. Fresh installs with the bundled ClickHouse use "otel", + # upgrades keep the database from the existing clickstack-config ConfigMap, and an external + # ClickHouse uses "default". A plain string override wins and also updates the bundled + # ClickHouse user grants and the default sources created for new teams. Sources of + # existing HyperDX teams are not changed. + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: '{{ include "clickstack.clickhouse.defaultDatabase" . }}' CLICKHOUSE_USER: "otelcollector" RUN_SCHEDULED_TASKS_EXTERNALLY: "false" # ClickHouse table retention (TTL). Go-duration string; "d" suffix allowed (e.g. "180d"). @@ -139,6 +144,18 @@ clickstack: # Apply TTL changes to tables that already exist, not just newly created ones (default false): # HYPERDX_OTEL_EXPORTER_RECONCILE_TABLE_TTL: "true" # Per-signal TTLs and reconcile need collector image 2.36.0 or newer. + # Create the database with the Replicated engine and seed Replicated* tables. Only takes + # effect for a database the seed creates: an existing non-Replicated database keeps plain + # engines. Same defaults as the database above: "true" for fresh installs with the bundled + # ClickHouse, the existing value on upgrades, "false" for an external ClickHouse. + # Needs collector image 2.47.0 or newer. + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: '{{ include "clickstack.clickhouse.defaultReplicated" . }}' + # If the database above does not exist yet and this one already has ClickStack tables, the + # collector keeps writing here instead of creating it. Keeps existing installs on "default" + # when rendered without cluster access (ArgoCD, helm template). "default" for the bundled + # ClickHouse, empty (off) otherwise. Set "" to move an existing install to the database above. + # Needs collector image 2.47.0 or newer. + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: '{{ include "clickstack.clickhouse.defaultFallbackDatabase" . }}' # Service endpoint defaults -- override to use external instances FRONTEND_URL: "http://localhost:3000" MONGO_URI: 'mongodb://hyperdx:{{ .Values.clickstack.secrets.MONGODB_PASSWORD }}@{{ include "clickstack.mongodb.svc" . }}:27017/hyperdx?authSource=hyperdx' @@ -242,7 +259,7 @@ clickstack: [ { "from": { - "databaseName": "default", + "databaseName": "{{ include "clickstack.clickhouse.database" . }}", "tableName": "otel_logs" }, "kind": "log", @@ -264,7 +281,7 @@ clickstack: }, { "from": { - "databaseName": "default", + "databaseName": "{{ include "clickstack.clickhouse.database" . }}", "tableName": "otel_traces" }, "kind": "trace", @@ -294,7 +311,7 @@ clickstack: }, { "from": { - "databaseName": "default", + "databaseName": "{{ include "clickstack.clickhouse.database" . }}", "tableName": "" }, "kind": "metric", @@ -313,7 +330,7 @@ clickstack: }, { "from": { - "databaseName": "default", + "databaseName": "{{ include "clickstack.clickhouse.database" . }}", "tableName": "hyperdx_sessions" }, "kind": "session", @@ -325,7 +342,7 @@ clickstack: }, { "from": { - "databaseName": "default", + "databaseName": "{{ include "clickstack.clickhouse.database" . }}", "tableName": "metrics_ts" }, "kind": "promql", @@ -461,6 +478,8 @@ clickhouse: requests: storage: 10Gi settings: + # Grants cover default.* (data written before the dedicated database) and + # the configured HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE. extraUsersConfig: users: app: @@ -469,12 +488,12 @@ clickhouse: grants: # The prometheus_api_v1 handler reads via the timeSeriesSelector() table function, # and ClickHouse gates every table function behind CREATE TEMPORARY TABLE. - - query: "GRANT SHOW ON *.*, SELECT ON system.*, SELECT ON default.*, CREATE TEMPORARY TABLE ON *.*" + - query: 'GRANT SHOW ON *.*, SELECT ON system.*, {{ include "clickstack.clickhouse.databaseGrants" (dict "privileges" "SELECT" "context" .) }}, CREATE TEMPORARY TABLE ON *.*' otelcollector: password: '{{ .Values.clickstack.secrets.CLICKHOUSE_PASSWORD }}' profile: default grants: - - query: "GRANT SELECT,INSERT,CREATE,SHOW ON default.*" + - query: 'GRANT {{ include "clickstack.clickhouse.databaseGrants" (dict "privileges" "SELECT,INSERT,CREATE,SHOW" "context" .) }}' # The operator defaults to level trace with 50 x 1000M log files, written # to the data volume. That fills the 10Gi default volume within days. logger: diff --git a/docs/UPGRADE.md b/docs/UPGRADE.md index 63f200b9..33fe07b6 100644 --- a/docs/UPGRADE.md +++ b/docs/UPGRADE.md @@ -2,6 +2,39 @@ Each section lists what changes when you upgrade to that chart version and anything you have to do first, newest first. A version that isn't listed needs nothing beyond `helm upgrade`. The full list of changes is in [CHANGELOG.md](../CHANGELOG.md). +## Unreleased + +- Fresh installs with the bundled ClickHouse write ClickStack tables to a dedicated, Replicated `otel` database. `helm upgrade` keeps existing installs on their current database and engines. If you render without cluster access (ArgoCD, `helm template`), read [Dedicated `otel` database](#dedicated-otel-database) before upgrading. + +### Dedicated `otel` database + +Fresh installs with the bundled ClickHouse (`clickhouse.enabled: true`) write ClickStack tables to a dedicated `otel` database, which the collector creates as Replicated with `Replicated*` tables. The replicated seed needs collector image `2.47.0` or newer. With an older image `otel` is created with plain engines and stays that way. External ClickHouse (`clickhouse.enabled: false`) keeps `default` with plain engines. + +On fresh installs, HyperDX sources, dashboards and alerts use `otel` automatically. A few things outside them still default to `default`: + +- Raw SQL with unqualified table names (SQL tiles, the MCP SQL tool) resolves to the `default` database. Qualify tables, for example `otel.otel_logs`. +- Custom `grants:` for the `app` or `otelcollector` users replace the chart's grants. Include `otel.*`, with `CREATE` for `otelcollector`. +- External tools and custom `defaultSources` that query `default.otel_*` should use `otel`. + +`helm upgrade` keeps the existing database and engines automatically: the chart reads `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE` and `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED` from the existing `clickstack-config` ConfigMap. There is nothing to do. + +Existing installs with data in `default` show `SchemaInSync=False` on the ClickHouseCluster permanently, because the operator does not convert a populated `default` to Replicated. This predates the `otel` database and is expected. + +Tools that render without cluster access (ArgoCD, `helm template | kubectl apply`) cannot see the existing ConfigMap and render `otel`. With collector image `2.47.0` or newer this is still safe for data: `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE` defaults to `default` for the bundled ClickHouse, so a collector that finds no `otel` database but ClickStack tables in `default` keeps writing to `default`. HyperDX teams created after such an upgrade still get default sources on `otel`. To keep everything consistent, pin the old values: + +```yaml +clickstack: + config: + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "default" + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: "false" +``` + +To deliberately move an existing install to `otel`, set `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: ""` and point the HyperDX sources at `otel`. Data already in `default` stays there and can be added as extra sources. + +The collector reads `clickstack-config` only at startup. `helm upgrade` rolls its pods on every upgrade, but renders without a release revision (ArgoCD, `helm template`) do not. After changing any of these keys there, run `kubectl rollout restart deployment/-otel-collector`. + +Older collector images have no fallback. There, an unpinned render sends new data to `otel` while HyperDX sources created earlier still read `default`, because HyperDX only creates default sources for a team that has none. Nothing is lost. Later `helm upgrade`s keep `otel`, since the chart reads it back from the ConfigMap. The simplest fix is to keep `otel` and point the HyperDX sources at it; older data stays in `default` and can be added as extra sources. Pinning back to `default` also drops the `otel.*` grants, so data written in between stays in `otel` and is unreadable unless you add grants and sources for it. + ## 3.8.0 - The top-level `hyperdx` values key is now `clickstack`. Values files and `--set hyperdx.*` flags still work and print a deprecation warning in the install notes; move them under `clickstack` before a future chart version removes `hyperdx`. Where both keys set the same value, `hyperdx` wins. Rendered resources don't change. `--reuse-values` keeps the release's stored `hyperdx` values, which then override anything you move under `clickstack`, so do the upgrade that moves them with `-f ` instead. diff --git a/integration-tests/README.md b/integration-tests/README.md index f46c4300..02fb77c5 100644 --- a/integration-tests/README.md +++ b/integration-tests/README.md @@ -51,6 +51,7 @@ All flags default to `false` and timeout defaults to `300` if omitted. | `NAMESPACE` | Kubernetes namespace | `default` | | `SUITE_DIR` | Absolute path to suite directory | `/path/to/integration-tests/my-new-suite` | | `REPO_ROOT` | Absolute path to repo root | `/path/to/ClickStack-helm-charts` | +| `PRE_UPGRADE_DATABASE` | After an upgrade, the ClickHouse database the base release wrote to (from its `clickstack-config`). Empty on a fresh install | `default` | The script should `exit 0` on success, `exit 1` on failure. @@ -79,18 +80,23 @@ On failure, the CI workflow collects pod status, events, logs, and resource desc On pull requests every suite also runs a second time as `integration-test (, upgrade)`. That run installs the charts and `values.yaml` from the PR's base branch and passes that branch's `assert.sh`. It then upgrades to the PR's charts and values with `--wait` and runs the PR's `assert.sh`. The `helm diff` of the upgrade is written to the job summary. A suite that doesn't exist on the base branch passes the upgrade run without testing anything. -To run it locally, point `UPGRADE_FROM` at a checkout of the older ref. The diff is skipped unless the `helm-diff` plugin is installed. +PRs also run each suite as `integration-test (, upgrade-template)`. It is the same as the upgrade run, except that the PR's chart is rendered with `helm template` and applied with `kubectl apply --server-side`, as Argo CD and other GitOps tools do. Without cluster access the chart's `lookup` calls return nothing, which is how an install that predates a new default (such as the `otel` database) would see it. The harness then restarts the release's Deployments, since `.Release.Revision` stays 1 and nothing else would roll the otel-collector. + +Before either upgrade the harness records the base release's ClickHouse database in `PRE_UPGRADE_DATABASE`. The `full-stack` assert checks that new data still lands in that database after the upgrade, and on a fresh install that the database and its tables are Replicated. + +To run it locally, point `UPGRADE_FROM` at a checkout of the older ref, and set `UPGRADE_METHOD=template` for the GitOps-style upgrade. The diff is skipped unless the `helm-diff` plugin is installed. ```bash git worktree add /tmp/clickstack-main origin/main UPGRADE_FROM=/tmp/clickstack-main ./integration-tests/run-suite.sh api-only +UPGRADE_FROM=/tmp/clickstack-main UPGRADE_METHOD=template ./integration-tests/run-suite.sh full-stack ``` ## Existing suites ### `full-stack` -Deploys the complete ClickStack platform with all subcharts (MongoDB, ClickHouse, OTEL collector). Runs the comprehensive smoke test (`scripts/smoke-test.sh`) which verifies OTEL ingestion into ClickHouse and runs Playwright e2e tests (user registration + log search). +Deploys the complete ClickStack platform with all subcharts (MongoDB, ClickHouse, OTEL collector). Runs the comprehensive smoke test (`scripts/smoke-test.sh`) which verifies OTEL ingestion into ClickHouse and runs Playwright e2e tests (user registration + log search). Fresh installs also check that the ClickStack database and its MergeTree tables use Replicated engines; upgrades check that new data still lands in the database the base release used. ### `api-only` diff --git a/integration-tests/full-stack/assert.sh b/integration-tests/full-stack/assert.sh index 579ea0f9..8450f293 100755 --- a/integration-tests/full-stack/assert.sh +++ b/integration-tests/full-stack/assert.sh @@ -30,6 +30,16 @@ echo "Final pod status:" kubectl get pods -o wide kubectl get services +if [ -n "${PRE_UPGRADE_DATABASE:-}" ]; then + # An upgrade must keep writing to the base release's database, whatever the new ConfigMap says: + # a render without cluster access sets the new default and relies on the collector's fallback. + echo "Upgraded from a release writing to '$PRE_UPGRADE_DATABASE'; checking new data still lands there" + export CLICKHOUSE_DATABASE="$PRE_UPGRADE_DATABASE" +else + # A fresh install with the bundled ClickHouse seeds a Replicated database. + export CLICKHOUSE_EXPECT_REPLICATED=true +fi + echo "Running smoke tests..." chmod +x "$REPO_ROOT/scripts/smoke-test.sh" TIMEOUT=300 RELEASE_NAME="$RELEASE_NAME" NAMESPACE="$NAMESPACE" "$REPO_ROOT/scripts/smoke-test.sh" diff --git a/integration-tests/run-suite.sh b/integration-tests/run-suite.sh index af915c5c..e4031271 100755 --- a/integration-tests/run-suite.sh +++ b/integration-tests/run-suite.sh @@ -29,12 +29,22 @@ TIMEOUT=$(read_config '.timeout' '300') # Optional checkout of an older ref: install its chart first, then upgrade to this checkout. UPGRADE_FROM="${UPGRADE_FROM:-}" +# How the upgrade is applied: "helm" runs helm upgrade; "template" pipes helm template into +# kubectl apply, as Argo CD and other GitOps tools do, so the chart's lookup calls see nothing. +UPGRADE_METHOD="${UPGRADE_METHOD:-helm}" +case "$UPGRADE_METHOD" in + helm|template) ;; + *) echo "ERROR: UPGRADE_METHOD must be 'helm' or 'template', got '$UPGRADE_METHOD'"; exit 1 ;; +esac BASE_ROOT="${UPGRADE_FROM:-$REPO_ROOT}" BASE_SUITE_DIR="$BASE_ROOT/integration-tests/$SUITE_NAME" if [ -n "$UPGRADE_FROM" ] && [ ! -f "$BASE_SUITE_DIR/values.yaml" ]; then echo "Suite '$SUITE_NAME' does not exist in $BASE_ROOT, nothing to upgrade from" exit 0 fi +# Database the base release writes ClickStack tables to, recorded before an upgrade. Empty on a +# fresh install. The post-upgrade assert.sh checks that new data still lands there. +export PRE_UPGRADE_DATABASE="" echo "=== Integration test suite: $SUITE_NAME ===" echo "Release: $RELEASE_NAME" @@ -43,6 +53,9 @@ echo "Operators: $NEEDS_OPERATORS" echo "Local storage: $NEEDS_LOCAL_STORAGE" echo "Timeout: ${TIMEOUT}s" echo "Upgrade from: ${UPGRADE_FROM:-none}" +if [ -n "$UPGRADE_FROM" ]; then + echo "Upgrade method: $UPGRADE_METHOD" +fi echo "" echo "--- Creating Kind cluster: $CLUSTER_NAME ---" @@ -112,6 +125,24 @@ show_upgrade_diff() { } >>"$out" } +# Upgrade the way GitOps tools do: render without cluster access and apply the manifests. +template_upgrade() { + local selector="app.kubernetes.io/instance=$RELEASE_NAME" + echo "--- helm template | kubectl apply ClickStack chart from $REPO_ROOT ---" + helm dependency build "$REPO_ROOT/charts/clickstack" + helm template "$RELEASE_NAME" "$REPO_ROOT/charts/clickstack" \ + -n "$NAMESPACE" -f "$SUITE_DIR/values.yaml" \ + | kubectl apply -n "$NAMESPACE" --server-side --force-conflicts -f - + # Pods whose template didn't change keep running with the old config, and .Release.Revision is + # always 1 here, so the otel-collector doesn't roll. Restart the release's Deployments, as a + # later image bump or node drain would, so the asserts see what the new config does. + kubectl rollout restart deployment -n "$NAMESPACE" -l "$selector" + for deployment in $(kubectl get deployment -n "$NAMESPACE" -l "$selector" -o name); do + kubectl rollout status "$deployment" -n "$NAMESPACE" --timeout="${TIMEOUT}s" + done + echo "" +} + if [ "$NEEDS_OPERATORS" = "true" ]; then deploy_operators "$BASE_ROOT" install fi @@ -127,12 +158,19 @@ deploy_chart "$BASE_ROOT" "$BASE_SUITE_DIR" install if [ -n "$UPGRADE_FROM" ]; then # Prove the old release works before upgrading it, so a failure after the upgrade is the upgrade's fault. run_assert "$UPGRADE_FROM" "$BASE_SUITE_DIR" + PRE_UPGRADE_DATABASE=$(kubectl get configmap clickstack-config -n "$NAMESPACE" \ + -o jsonpath='{.data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}' 2>/dev/null || true) + echo "Database before upgrade: ${PRE_UPGRADE_DATABASE:-none}" if [ "$NEEDS_OPERATORS" = "true" ]; then deploy_operators "$REPO_ROOT" upgrade fi - show_upgrade_diff - # --wait so assert.sh checks the new pods, not old ones that are still Ready. - deploy_chart "$REPO_ROOT" "$SUITE_DIR" upgrade --wait + if [ "$UPGRADE_METHOD" = "template" ]; then + template_upgrade + else + show_upgrade_diff + # --wait so assert.sh checks the new pods, not old ones that are still Ready. + deploy_chart "$REPO_ROOT" "$SUITE_DIR" upgrade --wait + fi fi run_assert "$REPO_ROOT" "$SUITE_DIR" diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 5cb41d71..398cc12f 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -10,7 +10,12 @@ TIMEOUT=${TIMEOUT:-300} CLICKHOUSE_SERVICE=${CLICKHOUSE_SERVICE:-$RELEASE_NAME-$CHART_NAME-clickhouse-clickhouse-headless} CLICKHOUSE_SECRET_NAME=${CLICKHOUSE_SECRET_NAME:-clickstack-secret} CLICKHOUSE_HTTP_USER=${CLICKHOUSE_HTTP_USER:-app} -CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:-default} +# Defaults to the database the chart configured for the collector. +CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:-$(kubectl get configmap clickstack-config -n "$NAMESPACE" \ + -o jsonpath='{.data.HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE}' 2>/dev/null || true)} +CLICKHOUSE_DATABASE=${CLICKHOUSE_DATABASE:-otel} +# Set to "true" to also check that the database uses the Replicated engine and its MergeTree tables are Replicated*. +CLICKHOUSE_EXPECT_REPLICATED=${CLICKHOUSE_EXPECT_REPLICATED:-false} CLICKHOUSE_TRACE_TABLE=${CLICKHOUSE_TRACE_TABLE:-otel_traces} CLICKHOUSE_LOG_TABLE=${CLICKHOUSE_LOG_TABLE:-otel_logs} INGESTION_POLL_INTERVAL=${INGESTION_POLL_INTERVAL:-5} @@ -206,6 +211,28 @@ wait_for_table_count_increase() { done } +check_replicated_engines() { + local engine="" + local plain_tables="" + + engine=$(run_clickhouse_query "SELECT engine FROM system.databases WHERE name = '${CLICKHOUSE_DATABASE}';") + engine=$(echo "$engine" | tr -d '[:space:]') + if [ "$engine" != "Replicated" ]; then + echo "ERROR: Database ${CLICKHOUSE_DATABASE} has engine '${engine}', expected Replicated" + return 1 + fi + echo "Database ${CLICKHOUSE_DATABASE}: OK (engine Replicated)" + + plain_tables=$(run_clickhouse_query "SELECT name || ' (' || engine || ')' FROM system.tables \ +WHERE database = '${CLICKHOUSE_DATABASE}' AND engine LIKE '%MergeTree' AND engine NOT LIKE 'Replicated%' ORDER BY name;") + if [ -n "$plain_tables" ]; then + echo "ERROR: MergeTree tables in ${CLICKHOUSE_DATABASE} without a Replicated engine:" + echo "$plain_tables" + return 1 + fi + echo "Tables in ${CLICKHOUSE_DATABASE}: OK (all MergeTree tables are Replicated)" +} + send_telemetrygen_signal() { local signal=$1 local count_flag=$2 @@ -308,6 +335,11 @@ echo "Waiting for traces/logs to land in ClickHouse..." wait_for_table_count_increase "$CLICKHOUSE_TRACE_TABLE" "$trace_baseline" "$TIMEOUT" wait_for_table_count_increase "$CLICKHOUSE_LOG_TABLE" "$log_baseline" "$TIMEOUT" +if [ "$CLICKHOUSE_EXPECT_REPLICATED" = "true" ]; then + echo "Verifying Replicated engines in ${CLICKHOUSE_DATABASE}..." + check_replicated_engines +fi + stop_port_forward "$otlp_http_pf_pid" stop_port_forward "$clickhouse_pf_pid" @@ -335,5 +367,8 @@ echo "- OTEL Collector metrics accessible" echo "- OTEL Collector Deployment available" echo "- ClickHouseCluster reconciled (Ready)" echo "- MongoDBCommunity reconciled (Running)" -echo "- OTEL traces and logs persisted to ClickHouse" +echo "- OTEL traces and logs persisted to ClickHouse (${CLICKHOUSE_DATABASE})" +if [ "$CLICKHOUSE_EXPECT_REPLICATED" = "true" ]; then + echo "- ${CLICKHOUSE_DATABASE} and its MergeTree tables use Replicated engines" +fi echo "- App registers user and displays logs via default connection" From 23828af48ba74d188faa9406590c46b7dccc8176 Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Fri, 20 Nov 2026 17:58:28 -0500 Subject: [PATCH 2/2] docs: add upgrade note --- docs/UPGRADE.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/UPGRADE.md b/docs/UPGRADE.md index 33fe07b6..323d28ad 100644 --- a/docs/UPGRADE.md +++ b/docs/UPGRADE.md @@ -4,7 +4,7 @@ Each section lists what changes when you upgrade to that chart version and anyth ## Unreleased -- Fresh installs with the bundled ClickHouse write ClickStack tables to a dedicated, Replicated `otel` database. `helm upgrade` keeps existing installs on their current database and engines. If you render without cluster access (ArgoCD, `helm template`), read [Dedicated `otel` database](#dedicated-otel-database) before upgrading. +- Fresh installs with the bundled ClickHouse write ClickStack tables to a dedicated, Replicated `otel` database. `helm upgrade` keeps existing installs on their current database and engines. If you render without cluster access (ArgoCD, `helm template`), pin `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: "default"` before upgrading; see [Dedicated `otel` database](#dedicated-otel-database). ### Dedicated `otel` database @@ -20,7 +20,7 @@ On fresh installs, HyperDX sources, dashboards and alerts use `otel` automatical Existing installs with data in `default` show `SchemaInSync=False` on the ClickHouseCluster permanently, because the operator does not convert a populated `default` to Replicated. This predates the `otel` database and is expected. -Tools that render without cluster access (ArgoCD, `helm template | kubectl apply`) cannot see the existing ConfigMap and render `otel`. With collector image `2.47.0` or newer this is still safe for data: `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE` defaults to `default` for the bundled ClickHouse, so a collector that finds no `otel` database but ClickStack tables in `default` keeps writing to `default`. HyperDX teams created after such an upgrade still get default sources on `otel`. To keep everything consistent, pin the old values: +Tools that render without cluster access (ArgoCD, `helm template | kubectl apply`) cannot see the existing ConfigMap and render `otel`. If you deploy this way, pin the old values before upgrading: ```yaml clickstack: @@ -29,6 +29,8 @@ clickstack: HYPERDX_OTEL_EXPORTER_CLICKHOUSE_REPLICATED: "false" ``` +Without the pin, collector image `2.47.0` or newer still keeps logs, traces and metrics in `default`: `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE` defaults to `default` for the bundled ClickHouse, so a collector that finds no `otel` database but ClickStack tables in `default` keeps writing to `default`. Two things still break. PromQL ingestion stops, because the chart points ClickHouse's Prometheus remote write handler at `otel.metrics_ts`, which the collector never creates. And HyperDX teams created after the upgrade get default sources on `otel`. + To deliberately move an existing install to `otel`, set `HYPERDX_OTEL_EXPORTER_CLICKHOUSE_FALLBACK_DATABASE: ""` and point the HyperDX sources at `otel`. Data already in `default` stays there and can be added as extra sources. The collector reads `clickstack-config` only at startup. `helm upgrade` rolls its pods on every upgrade, but renders without a release revision (ArgoCD, `helm template`) do not. After changing any of these keys there, run `kubectl rollout restart deployment/-otel-collector`.