From c56175ce70bd32866de65810ee5c17537a139282 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 00:16:36 +0300 Subject: [PATCH 01/10] refactor(env): one policy for data URLs and outbound keys Data clients read their URL through dataUrl(), which falls back to the local Docker services in development when unset. Outbound clients read their key from config.services instead of raw process.env, one place for Autumn, Dub, Resend, Slack, Axiom, Superlog, Supermemory, self-tracking and TCC. Removes duplicated loopback checks and Axiom gating. --- .env.example | 19 +++- apps/api/src/lib/evlog-api.ts | 7 +- apps/api/src/lib/tcc-otel.ts | 3 +- apps/basket/src/lib/blocked-traffic-alerts.ts | 8 +- apps/basket/src/lib/evlog-basket.ts | 7 +- apps/dashboard/app/(auth)/layout.tsx | 4 +- .../billing/actions/cancel-feedback-action.ts | 3 +- apps/dashboard/instrumentation.ts | 11 ++- apps/insights/src/ai-digest.ts | 4 +- apps/insights/src/lib/evlog-insights.ts | 7 +- apps/links/src/lib/logging.ts | 7 +- apps/slack/src/lib/evlog-slack.ts | 7 +- apps/uptime/src/lib/evlog-uptime.ts | 7 +- apps/uptime/src/uptime-transition-alerts.ts | 4 +- bun.lock | 1 + packages/ai/src/lib/databuddy.ts | 4 +- packages/ai/src/lib/supermemory.ts | 4 +- packages/auth/drizzle.config.ts | 3 +- packages/db/drizzle.config.ts | 3 +- packages/db/src/clickhouse/client.ts | 8 +- packages/db/src/clickhouse/verify.ts | 3 +- packages/db/src/client.ts | 3 +- packages/db/src/e2e-db-lifecycle.test.ts | 2 - packages/db/src/e2e-db-lifecycle.ts | 13 +-- packages/db/src/seed.ts | 6 +- packages/db/src/test-env.ts | 9 +- packages/env/src/app.test.ts | 92 ++++++++++++++++++- packages/env/src/app.ts | 50 ++++++++-- packages/env/src/boolean.ts | 42 ++++++++- packages/notifications/src/alarm-config.ts | 6 +- packages/redis/bullmq.ts | 6 +- packages/redis/redis-options.ts | 4 +- packages/services/src/business-memory.ts | 3 +- packages/services/src/feedback.ts | 3 +- packages/services/src/website-cache.ts | 3 +- packages/services/src/websites.ts | 3 +- packages/shared/package.json | 1 + packages/shared/src/evlog-superlog.ts | 3 +- 38 files changed, 274 insertions(+), 99 deletions(-) diff --git a/.env.example b/.env.example index 13291ca290..cc94ece2c6 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,16 @@ # Local development. For Docker self-hosting, use selfhost.env.example. -CLICKHOUSE_URL="http://default:@localhost:8123/databuddy_analytics" +# DATABASE_URL, REDIS_URL, BULLMQ_REDIS_URL and CLICKHOUSE_URL default to the +# docker-compose services in development. Set them only to point elsewhere. + # Required by DQL. This must use the separately provisioned dql_user account; # never point it at the application/admin ClickHouse credentials. CLICKHOUSE_DQL_URL="http://dql_user:databuddy_dql_dev_password@localhost:8123/databuddy_analytics" -DATABASE_URL="postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy" -REDIS_URL="redis://localhost:6379" -BULLMQ_REDIS_URL="redis://localhost:6379" + +# Redpanda (Kafka) ingestion. Blank writes straight to ClickHouse. +REDPANDA_BROKER="" +REDPANDA_USER="" +REDPANDA_PASSWORD="" # Public URLs. Local development uses built-in localhost defaults when blank. DASHBOARD_URL="" @@ -65,6 +69,8 @@ SLACK_SIGNING_SECRET="" SLACK_CLIENT_ID="" SLACK_CLIENT_SECRET="" SLACK_REDIRECT_URI="" +# Internal signup, billing, and feedback notifications +SLACK_WEBHOOK_URL="" # Not Necessary unless using blog MARBLE_API_KEY= @@ -77,6 +83,11 @@ NOTRA_API_KEY= # Superlog OTLP logs export. When set, evlog drains in api/basket/insights/slack/uptime # also ship wide events to https://intake.superlog.sh/v1/logs. SUPERLOG_API_KEY="" +AXIOM_TOKEN="" +TCC_API_KEY="" + +# Databuddy self-analytics for product events +DATABUDDY_API_KEY="" # S3-compatible object storage for status page logo and favicon uploads. # Both credentials are required together; the API refuses to boot with only one. diff --git a/apps/api/src/lib/evlog-api.ts b/apps/api/src/lib/evlog-api.ts index 412ad1e326..ecea68ebf7 100644 --- a/apps/api/src/lib/evlog-api.ts +++ b/apps/api/src/lib/evlog-api.ts @@ -1,6 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog"; import type { DrainContext, EnrichContext } from "evlog"; import { createAxiomDrain } from "evlog/axiom"; @@ -15,7 +15,7 @@ import { createDrainPipeline } from "evlog/pipeline"; const batchedAxiomDrain = createDrainPipeline({ batch: { size: 50, intervalMs: 5000 }, maxBufferSize: 2000, -})(createAxiomDrain({ apiKey: process.env.AXIOM_TOKEN })); +})(createAxiomDrain({ apiKey: config.services.axiomToken })); const batchedSuperlogDrain = createBatchedSuperlogDrain(); @@ -30,8 +30,7 @@ const devFsLogsDir = join( const useLocalEvlogFiles = process.env.NODE_ENV === "development" || readBooleanEnv("API_EVLOG_FS"); -const drainToAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const drainToAxiom = Boolean(config.services.axiomToken); const devFsDrain = useLocalEvlogFiles ? createFsDrain({ dir: devFsLogsDir, pretty: false }) diff --git a/apps/api/src/lib/tcc-otel.ts b/apps/api/src/lib/tcc-otel.ts index 78bf99db4b..69abbc12b8 100644 --- a/apps/api/src/lib/tcc-otel.ts +++ b/apps/api/src/lib/tcc-otel.ts @@ -1,4 +1,5 @@ import { TCCSpanProcessor } from "@contextcompany/otel"; +import { config } from "@databuddy/env/app"; import { resourceFromAttributes } from "@opentelemetry/resources"; import { NodeSDK } from "@opentelemetry/sdk-node"; import { @@ -9,7 +10,7 @@ import pkg from "../../package.json"; let sdk: NodeSDK | null = null; export function initTccTracing(): void { - if (sdk || !process.env.TCC_API_KEY) { + if (sdk || !config.services.tccApiKey) { return; } diff --git a/apps/basket/src/lib/blocked-traffic-alerts.ts b/apps/basket/src/lib/blocked-traffic-alerts.ts index 88e2f1e808..fe36df54c1 100644 --- a/apps/basket/src/lib/blocked-traffic-alerts.ts +++ b/apps/basket/src/lib/blocked-traffic-alerts.ts @@ -5,7 +5,7 @@ import { normalizeEmailNotificationSettings, type EmailNotificationSettings, } from "@databuddy/db"; -import { config } from "@databuddy/env/app"; +import { config, dataUrl } from "@databuddy/env/app"; import { BlockedTrafficAlertEmail, render } from "@databuddy/email"; import { redis } from "@databuddy/redis"; import { @@ -282,7 +282,7 @@ async function sendAlertEmail(input: { previousBlocked: number; windowBlockedCount: number; }): Promise { - const apiKey = requireBlockedTrafficEmailApiKey(config.email.resendApiKey); + const apiKey = requireBlockedTrafficEmailApiKey(config.services.resendApiKey); const siteLabel = input.context.websiteName || @@ -346,7 +346,7 @@ async function maybeSendBlockedTrafficAlertAsync( if (!context.ownerId) { return; } - if (!process.env.REDIS_URL) { + if (!dataUrl("REDIS_URL")) { return; } @@ -379,7 +379,7 @@ async function maybeSendBlockedTrafficAlertAsync( return; } - if (!config.email.resendApiKey) { + if (!config.services.resendApiKey) { warnBlockedTrafficEmailUnconfigured(); return; } diff --git a/apps/basket/src/lib/evlog-basket.ts b/apps/basket/src/lib/evlog-basket.ts index f750b1123b..05271b4189 100644 --- a/apps/basket/src/lib/evlog-basket.ts +++ b/apps/basket/src/lib/evlog-basket.ts @@ -1,6 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createBatchedAxiomDrain, downgradeClientHttpError, @@ -12,7 +12,7 @@ import { CLIENT_ERROR_MESSAGES } from "@lib/structured-errors"; import type { DrainContext, EnrichContext } from "evlog"; import { createFsDrain } from "evlog/fs"; -const batchedAxiomDrain = createBatchedAxiomDrain(process.env.AXIOM_TOKEN); +const batchedAxiomDrain = createBatchedAxiomDrain(config.services.axiomToken); const batchedSuperlogDrain = createBatchedSuperlogDrain(); @@ -27,8 +27,7 @@ const devFsLogsDir = join( const useLocalEvlogFiles = process.env.NODE_ENV === "development" || readBooleanEnv("BASKET_EVLOG_FS"); -const drainToAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const drainToAxiom = Boolean(config.services.axiomToken); const devFsDrain = useLocalEvlogFiles ? createFsDrain({ dir: devFsLogsDir, pretty: false }) diff --git a/apps/dashboard/app/(auth)/layout.tsx b/apps/dashboard/app/(auth)/layout.tsx index 7928c75300..7db1ce9407 100644 --- a/apps/dashboard/app/(auth)/layout.tsx +++ b/apps/dashboard/app/(auth)/layout.tsx @@ -1,4 +1,4 @@ -import { readBooleanEnv } from "@databuddy/env/boolean"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { connection } from "next/server"; import type { ReactNode } from "react"; import AuthLayout from "./auth-layout"; @@ -12,7 +12,7 @@ export default async function Layout({ children }: { children: ReactNode }) { ({ maxBufferSize: 2000, }); -const hasAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const hasAxiom = Boolean(config.services.axiomToken); const batchedAxiomDrain = hasAxiom ? pipeline( createAxiomDrain({ - apiKey: process.env.AXIOM_TOKEN, + apiKey: config.services.axiomToken, dataset: "insights", }) ) diff --git a/apps/links/src/lib/logging.ts b/apps/links/src/lib/logging.ts index eeaa507013..5bbd0d8351 100644 --- a/apps/links/src/lib/logging.ts +++ b/apps/links/src/lib/logging.ts @@ -1,6 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createDatabuddyEvlogEnv } from "@databuddy/shared/evlog-redaction"; import type { DrainContext, EnrichContext, WideEvent } from "evlog"; import { log } from "evlog"; @@ -20,7 +20,7 @@ type LogFields = Record; const batchedAxiomDrain = createDrainPipeline({ batch: { size: 50, intervalMs: 5000 }, maxBufferSize: 2000, -})(createAxiomDrain({ apiKey: process.env.AXIOM_TOKEN })); +})(createAxiomDrain({ apiKey: config.services.axiomToken })); const fsDrain = process.env.NODE_ENV === "development" || readBooleanEnv("LINKS_EVLOG_FS") @@ -36,8 +36,7 @@ const fsDrain = }) : null; -const drainToAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const drainToAxiom = Boolean(config.services.axiomToken); const DURATION_RE = /^([\d.]+)(ms|s)$/; diff --git a/apps/slack/src/lib/evlog-slack.ts b/apps/slack/src/lib/evlog-slack.ts index 578b172cb7..a836d53f02 100644 --- a/apps/slack/src/lib/evlog-slack.ts +++ b/apps/slack/src/lib/evlog-slack.ts @@ -2,7 +2,7 @@ import { isRecord } from "@/lib/guards"; import { AsyncLocalStorage } from "node:async_hooks"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog"; import type { DrainContext, RequestLogger } from "evlog"; import { createLogger, log } from "evlog"; @@ -15,15 +15,14 @@ type SlackLogFields = Record; const activeSlackLog = new AsyncLocalStorage(); -const hasAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const hasAxiom = Boolean(config.services.axiomToken); const batchedAxiomDrain = hasAxiom ? createDrainPipeline({ batch: { size: 50, intervalMs: 5000 }, maxBufferSize: 2000, })( createAxiomDrain({ - apiKey: process.env.AXIOM_TOKEN, + apiKey: config.services.axiomToken, dataset: "slack", }) ) diff --git a/apps/uptime/src/lib/evlog-uptime.ts b/apps/uptime/src/lib/evlog-uptime.ts index 6482fda0fe..2bfd58ee69 100644 --- a/apps/uptime/src/lib/evlog-uptime.ts +++ b/apps/uptime/src/lib/evlog-uptime.ts @@ -1,6 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createBatchedAxiomDrain, enrichHttpWideEvent, @@ -10,7 +10,7 @@ import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog"; import type { DrainContext, EnrichContext } from "evlog"; import { createFsDrain } from "evlog/fs"; -const batchedAxiomDrain = createBatchedAxiomDrain(process.env.AXIOM_TOKEN); +const batchedAxiomDrain = createBatchedAxiomDrain(config.services.axiomToken); const batchedSuperlogDrain = createBatchedSuperlogDrain(); const devFsLogsDir = join( @@ -24,8 +24,7 @@ const devFsLogsDir = join( const useLocalEvlogFiles = process.env.NODE_ENV === "development" || readBooleanEnv("UPTIME_EVLOG_FS"); -const drainToAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const drainToAxiom = Boolean(config.services.axiomToken); const devFsDrain = useLocalEvlogFiles ? createFsDrain({ dir: devFsLogsDir, pretty: false }) diff --git a/apps/uptime/src/uptime-transition-alerts.ts b/apps/uptime/src/uptime-transition-alerts.ts index d4421752a0..dd47bf6418 100644 --- a/apps/uptime/src/uptime-transition-alerts.ts +++ b/apps/uptime/src/uptime-transition-alerts.ts @@ -7,7 +7,7 @@ import { } from "@databuddy/db"; import { chQuery } from "@databuddy/db/clickhouse"; import { uptimeSchedules } from "@databuddy/db/schema"; -import { config } from "@databuddy/env/app"; +import { config, dataUrl } from "@databuddy/env/app"; import { NotificationClient, buildAlarmNotificationTargets, @@ -395,7 +395,7 @@ export async function writeMonitorState( } async function queryPreviousState(siteId: string): Promise { - if (!process.env.CLICKHOUSE_URL) { + if (!dataUrl("CLICKHOUSE_URL")) { return { kind: "missing" }; } try { diff --git a/bun.lock b/bun.lock index d6b8791a0b..af96f68afa 100644 --- a/bun.lock +++ b/bun.lock @@ -673,6 +673,7 @@ "name": "@databuddy/shared", "version": "0.0.1", "dependencies": { + "@databuddy/env": "workspace:*", "evlog": "catalog:", "ua-parser-js": "catalog:", "undici": "^8.2.0", diff --git a/packages/ai/src/lib/databuddy.ts b/packages/ai/src/lib/databuddy.ts index 30af4270fa..3cf206cae7 100644 --- a/packages/ai/src/lib/databuddy.ts +++ b/packages/ai/src/lib/databuddy.ts @@ -1,7 +1,7 @@ -import { readBooleanEnv } from "@databuddy/env/boolean"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { Databuddy } from "@databuddy/sdk/node"; -const apiKey = process.env.DATABUDDY_API_KEY; +const apiKey = config.services.databuddyApiKey; const websiteId = process.env.DATABUDDY_WEBSITE_ID; function createClient(source: string, namespace?: string) { diff --git a/packages/ai/src/lib/supermemory.ts b/packages/ai/src/lib/supermemory.ts index f565e11e99..9e5f75c2c1 100644 --- a/packages/ai/src/lib/supermemory.ts +++ b/packages/ai/src/lib/supermemory.ts @@ -1,8 +1,8 @@ +import { config } from "@databuddy/env/app"; import { getMemoryClient } from "@databuddy/services/business-memory"; export { getMemoryClient } from "@databuddy/services/business-memory"; import { stripHtmlTags } from "./sanitize"; -const apiKey = process.env.SUPERMEMORY_API_KEY; const MAX_MEMORY_LENGTH = 2000; const MEMORY_REQUEST_CLAUSE = /[^.!?;:,\n]+\??/g; const MEMORY_REQUEST_FILLER = @@ -46,7 +46,7 @@ const FORGET_ACKNOWLEDGEMENT = /^(?:(?:thanks|thank you|ok|okay|got it)[.!]\s*)+/i; export function isMemoryEnabled(): boolean { - return Boolean(apiKey); + return Boolean(config.services.supermemoryApiKey); } export function sanitizeMemoryContent( diff --git a/packages/auth/drizzle.config.ts b/packages/auth/drizzle.config.ts index e61d204043..5bb06bfd23 100644 --- a/packages/auth/drizzle.config.ts +++ b/packages/auth/drizzle.config.ts @@ -1,3 +1,4 @@ +import { dataUrl } from "@databuddy/env/app"; import { defineConfig } from "drizzle-kit"; export default defineConfig({ @@ -5,6 +6,6 @@ export default defineConfig({ schema: "./src/drizzle/schema.ts", dialect: "postgresql", dbCredentials: { - url: process.env.DATABASE_URL || "", + url: dataUrl("DATABASE_URL") || "", }, }); diff --git a/packages/db/drizzle.config.ts b/packages/db/drizzle.config.ts index dcb1caa703..b688a7798e 100644 --- a/packages/db/drizzle.config.ts +++ b/packages/db/drizzle.config.ts @@ -1,3 +1,4 @@ +import { dataUrl } from "@databuddy/env/app"; import { defineConfig } from "drizzle-kit"; export default defineConfig({ @@ -21,7 +22,7 @@ export default defineConfig({ ], dialect: "postgresql", dbCredentials: { - url: process.env.DATABASE_URL || "", + url: dataUrl("DATABASE_URL") || "", }, tablesFilter: ["!pg_stat_statements", "!pg_stat_statements_info"], }); diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts index 3bd83263cb..a52233a051 100644 --- a/packages/db/src/clickhouse/client.ts +++ b/packages/db/src/clickhouse/client.ts @@ -1,5 +1,6 @@ import { createClient, type ResponseJSON } from "@clickhouse/client"; import type { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config"; +import { dataUrl, isLocalHost } from "@databuddy/env/app"; import { finalizeDeliveryTables } from "./logical-reads"; export const TABLE_NAMES = { events: "analytics.events", @@ -82,10 +83,9 @@ function assertLoopbackForIntegrationTests(url: string | undefined): void { if (process.env.CLICKHOUSE_INTEGRATION_TESTS !== "true") { return; } - const hostname = url ? new URL(url).hostname : ""; - if (!["localhost", "127.0.0.1", "::1", "[::1]"].includes(hostname)) { + if (!(url && isLocalHost(url))) { throw new Error( - `ClickHouse integration tests only run against a loopback server; CLICKHOUSE_URL host is "${hostname || "unset"}"` + `ClickHouse integration tests only run against a loopback server; CLICKHOUSE_URL host is "${url ? new URL(url).hostname : "unset"}"` ); } } @@ -93,7 +93,7 @@ function assertLoopbackForIntegrationTests(url: string | undefined): void { assertLoopbackForIntegrationTests(process.env.CLICKHOUSE_URL); const baseClient = createClient({ - url: process.env.CLICKHOUSE_URL, + url: dataUrl("CLICKHOUSE_URL"), ...CLICKHOUSE_OPTIONS, }); diff --git a/packages/db/src/clickhouse/verify.ts b/packages/db/src/clickhouse/verify.ts index 13569de280..a31a036b9f 100644 --- a/packages/db/src/clickhouse/verify.ts +++ b/packages/db/src/clickhouse/verify.ts @@ -1,5 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { dataUrl } from "@databuddy/env/app"; import { CLICKHOUSE_OPTIONS } from "./client"; import { type ParsedTable, @@ -27,7 +28,7 @@ function dbNameOf(sql: string): string { } async function fetchLive(): Promise> { - const raw = process.env.CLICKHOUSE_READONLY_URL ?? process.env.CLICKHOUSE_URL; + const raw = process.env.CLICKHOUSE_READONLY_URL ?? dataUrl("CLICKHOUSE_URL"); if (!raw) { throw new Error("CLICKHOUSE_READONLY_URL or CLICKHOUSE_URL must be set"); } diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index dae1a171bf..876d79a66a 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -1,5 +1,6 @@ /** biome-ignore-all lint/performance/noNamespaceImport: "Required" */ +import { dataUrl } from "@databuddy/env/app"; import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; import { Pool } from "pg"; import { relations } from "./drizzle/schema/relations"; @@ -92,7 +93,7 @@ let _pool: Pool | null = null; function getDb(): DB { if (!_db) { - const databaseUrl = process.env.DATABASE_URL; + const databaseUrl = dataUrl("DATABASE_URL"); if (!databaseUrl) { throw new Error("DATABASE_URL is not set"); } diff --git a/packages/db/src/e2e-db-lifecycle.test.ts b/packages/db/src/e2e-db-lifecycle.test.ts index b4e7bcde74..8ea6d87674 100644 --- a/packages/db/src/e2e-db-lifecycle.test.ts +++ b/packages/db/src/e2e-db-lifecycle.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "bun:test"; import { deriveAdminDatabaseUrl, deriveDatabaseUrl, - isLocalDbHostname, parseLifecycleArgs, resolveE2EDatabaseName, resolveLifecycleConfig, @@ -69,7 +68,6 @@ describe("e2e db lifecycle helpers", () => { }); it("refuses non-local database hosts by default", () => { - expect(isLocalDbHostname("localhost")).toBe(true); expect(() => resolveLifecycleConfig({ allowNonLocal: false, diff --git a/packages/db/src/e2e-db-lifecycle.ts b/packages/db/src/e2e-db-lifecycle.ts index 3982e6aed1..6d2ce43ab1 100644 --- a/packages/db/src/e2e-db-lifecycle.ts +++ b/packages/db/src/e2e-db-lifecycle.ts @@ -1,11 +1,6 @@ +import { isLocalHost } from "@databuddy/env/app"; import { Client } from "pg"; -const LOCAL_DATABASE_HOSTS = new Set([ - "localhost", - "127.0.0.1", - "::1", - "[::1]", -]); const DEFAULT_E2E_DB_PREFIX = "databuddy_e2e"; const INVALID_DB_IDENTIFIER_PARTS = /[^A-Za-z0-9_]+/g; const REPEATED_UNDERSCORES = /_+/g; @@ -131,10 +126,6 @@ export function normalizeDatabaseUrl(databaseDsn: string): URL { return parsed; } -export function isLocalDbHostname(hostname: string): boolean { - return LOCAL_DATABASE_HOSTS.has(hostname.toLowerCase()); -} - export function deriveAdminDatabaseUrl(baseUrl: URL): URL { const url = new URL(baseUrl.href); url.pathname = "/postgres"; @@ -151,7 +142,7 @@ export function resolveLifecycleConfig( input: ParsedLifecycleArgs ): ResolvedLifecycleConfig { const baseUrl = normalizeDatabaseUrl(input.baseDsn); - if (!(input.allowNonLocal || isLocalDbHostname(baseUrl.hostname))) { + if (!(input.allowNonLocal || isLocalHost(baseUrl.href))) { throw new Error( `Refusing to manage E2E DB on non-local host '${baseUrl.hostname}'. Set --allow-non-local to override.` ); diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts index 96bd36bfca..651cd79a0c 100644 --- a/packages/db/src/seed.ts +++ b/packages/db/src/seed.ts @@ -1,12 +1,12 @@ +import { dataUrl, isLocalHost } from "@databuddy/env/app"; import { faker } from "@faker-js/faker"; import { clickHouse, TABLE_NAMES } from "./clickhouse/client"; import { db } from "./client"; -import { isLocalDbHostname } from "./e2e-db-lifecycle"; for (const name of ["CLICKHOUSE_URL", "DATABASE_URL"] as const) { - const url = process.env[name]; + const url = dataUrl(name); const hostname = url && URL.canParse(url) ? new URL(url).hostname : ""; - if (!isLocalDbHostname(hostname)) { + if (!(hostname && isLocalHost(hostname))) { throw new Error( `db:seed only runs against local databases; ${name} host is "${hostname || "unset"}"` ); diff --git a/packages/db/src/test-env.ts b/packages/db/src/test-env.ts index ae28c0396e..1c20377df9 100644 --- a/packages/db/src/test-env.ts +++ b/packages/db/src/test-env.ts @@ -1,17 +1,14 @@ -import { readBooleanEnv } from "@databuddy/env/app"; +import { isLocalHost, readBooleanEnv } from "@databuddy/env/app"; const useCiUrls = readBooleanEnv("CI"); const defaultDatabaseUrl = "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy_test"; const defaultRedisUrl = "redis://localhost:6379/1"; -const loopbackHosts = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]); - function localTestUrl(name: string, value: string): string { - const { hostname } = new URL(value); - if (!loopbackHosts.has(hostname)) { + if (!isLocalHost(value)) { throw new Error( - `${name} must point at a local test service. Refusing to run tests against ${hostname}.` + `${name} must point at a local test service. Refusing to run tests against ${new URL(value).hostname}.` ); } return value; diff --git a/packages/env/src/app.test.ts b/packages/env/src/app.test.ts index 0734dd9ccc..a713392dc5 100644 --- a/packages/env/src/app.test.ts +++ b/packages/env/src/app.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from "bun:test"; -import { assertConfigured, createConfig, readBooleanEnv } from "./app"; +import { + assertConfigured, + billingMode, + createConfig, + dataUrl, + isLocalHost, + readBooleanEnv, +} from "./app"; const HOSTED = { AUTUMN_SECRET_KEY: "am_sk_test", @@ -210,6 +217,89 @@ describe("assertConfigured", () => { }); }); +describe("isLocalHost", () => { + it("accepts loopback hosts and compose service names", () => { + for (const url of [ + "postgres://u:p@localhost:5432/databuddy", + "http://default:@127.0.0.1:8123", + "redis://[::1]:6379", + "redis://redis:6379", + "redpanda:9092", + ]) { + expect(isLocalHost(url)).toBe(true); + } + }); + + it("rejects dotted remote hosts", () => { + expect(isLocalHost("postgres://u:p@db.example.com:5432/x")).toBe(false); + expect(isLocalHost("broker.example.com:9092")).toBe(false); + }); +}); + +describe("dataUrl", () => { + const REMOTE = "postgres://u:p@db.example.com:5432/databuddy"; + + it("defaults unset URLs to the local Docker services in development", () => { + const env = { NODE_ENV: "development" }; + expect(dataUrl("DATABASE_URL", env)).toBe( + "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy" + ); + expect(dataUrl("BULLMQ_REDIS_URL", env)).toBe("redis://localhost:6379"); + expect(dataUrl("CLICKHOUSE_URL", env)).toBe( + "http://default:@localhost:8123/databuddy_analytics" + ); + }); + + it("uses an explicit URL as is in every environment", () => { + for (const NODE_ENV of ["development", "production", "test"]) { + expect(dataUrl("DATABASE_URL", { DATABASE_URL: REMOTE, NODE_ENV })).toBe( + REMOTE + ); + } + }); + + it("leaves unset URLs unset outside development", () => { + for (const NODE_ENV of ["production", "test", undefined]) { + expect(dataUrl("REDIS_URL", { NODE_ENV })).toBeUndefined(); + } + }); +}); + +describe("services", () => { + it("reads trimmed keys live and treats blanks as unset", () => { + const env: Record = { + NODE_ENV: "development", + RESEND_API_KEY: " ", + SLACK_WEBHOOK_URL: " https://hooks.slack.com/services/x ", + }; + const { services } = createConfig(env); + expect(services.slackWebhookUrl).toBe("https://hooks.slack.com/services/x"); + expect(services.resendApiKey).toBeUndefined(); + env.RESEND_API_KEY = "re_live"; + expect(services.resendApiKey).toBe("re_live"); + }); +}); + +describe("billingMode", () => { + it("keeps self-hosting unmetered even with a copied billing key", () => { + expect(billingMode({ ...HOSTED, SELFHOST: "true" })).toBe("selfhost"); + }); + + it("bills hosted production even when the key is missing", () => { + expect(billingMode(HOSTED)).toBe("live"); + expect(billingMode({ NODE_ENV: "production" })).toBe("live"); + }); + + it("bills elsewhere only when a billing key is set", () => { + for (const NODE_ENV of ["development", "test"]) { + expect(billingMode({ NODE_ENV })).toBe("disabled"); + expect(billingMode({ NODE_ENV, AUTUMN_SECRET_KEY: "am_sk" })).toBe( + "live" + ); + } + }); +}); + describe("readBooleanEnv", () => { it("only enables an explicit true value", () => { for (const value of [undefined, "", "false", "0", "1", "yes"]) { diff --git a/packages/env/src/app.ts b/packages/env/src/app.ts index 92b14e912f..d20bf61666 100644 --- a/packages/env/src/app.ts +++ b/packages/env/src/app.ts @@ -1,6 +1,6 @@ -import { readBooleanEnv } from "./boolean"; +import { LOOPBACK_HOSTS, readBooleanEnv } from "./boolean"; -export { readBooleanEnv } from "./boolean"; +export { dataUrl, isLocalHost, readBooleanEnv } from "./boolean"; // App-wide runtime config. // @@ -51,6 +51,18 @@ const EMAIL = { }, } as const; +const SERVICES = { + autumnSecretKey: "AUTUMN_SECRET_KEY", + axiomToken: "AXIOM_TOKEN", + databuddyApiKey: "DATABUDDY_API_KEY", + dubApiKey: "DUB_API_KEY", + resendApiKey: "RESEND_API_KEY", + slackWebhookUrl: "SLACK_WEBHOOK_URL", + superlogApiKey: "SUPERLOG_API_KEY", + supermemoryApiKey: "SUPERMEMORY_API_KEY", + tccApiKey: "TCC_API_KEY", +} as const; + const TRAILING_SLASH = /\/$/; type Env = Record; @@ -72,11 +84,11 @@ export interface Config { email: { alertsFrom: string; from: string; - resendApiKey?: string; }; integrations: { openAiAdsPixelId?: string; }; + services: Record; storage?: StorageConfig; urls: { api: string; @@ -91,12 +103,37 @@ export interface Config { const REQUIRED_IN_PRODUCTION = ["BETTER_AUTH_SECRET"] as const; const REQUIRED_IN_HOSTED_CLOUD = ["AUTUMN_SECRET_KEY"] as const; -const LOOPBACK_HOSTS = new Set(["0.0.0.0", "127.0.0.1", "[::1]", "localhost"]); + +function liveReads>( + currentEnv: () => Env, + keys: T +): Record { + return Object.defineProperties( + {} as Record, + Object.fromEntries( + Object.entries(keys).map(([name, key]) => [ + name, + { enumerable: true, get: () => readOptional(currentEnv(), key) }, + ]) + ) + ); +} function isHostedCloud(env: Env): boolean { return env.NODE_ENV === "production" && !readBooleanEnv("SELFHOST", env); } +export type BillingMode = "selfhost" | "live" | "disabled"; + +export function billingMode(env: Env = process.env): BillingMode { + if (readBooleanEnv("SELFHOST", env)) { + return "selfhost"; + } + return isHostedCloud(env) || readOptional(env, SERVICES.autumnSecretKey) + ? "live" + : "disabled"; +} + function defaultUrl(env: Env, setting: UrlConfig): string { return isHostedCloud(env) ? setting.cloud : setting.local; } @@ -168,7 +205,8 @@ function readStorage(env: Env): StorageConfig | undefined { }; } -export function createConfig(env: Env = process.env): Config { +export function createConfig(source?: Env): Config { + const env = source ?? process.env; const dashboardUrl = readUrl(env, URLS.dashboard); const apiUrl = readUrl(env, URLS.api); @@ -183,11 +221,11 @@ export function createConfig(env: Env = process.env): Config { email: { alertsFrom: readEmail(env, EMAIL.alertsFrom), from: readEmail(env, EMAIL.from), - resendApiKey: readOptional(env, "RESEND_API_KEY"), }, integrations: { openAiAdsPixelId: readOptional(env, "NEXT_PUBLIC_OPENAI_ADS_PIXEL_ID"), }, + services: liveReads(() => source ?? process.env, SERVICES), storage: readStorage(env), urls: { api: apiUrl, diff --git a/packages/env/src/boolean.ts b/packages/env/src/boolean.ts index 8fa1c655be..7570989f15 100644 --- a/packages/env/src/boolean.ts +++ b/packages/env/src/boolean.ts @@ -1,6 +1,46 @@ +type Env = Record; + +export const LOOPBACK_HOSTS = new Set([ + "0.0.0.0", + "127.0.0.1", + "[::1]", + "localhost", +]); + +const LOCAL_DATA_URLS = { + BULLMQ_REDIS_URL: "redis://localhost:6379", + CLICKHOUSE_URL: "http://default:@localhost:8123/databuddy_analytics", + DATABASE_URL: + "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy", + REDIS_URL: "redis://localhost:6379", +} as const; + export function readBooleanEnv( name: string, - environment: Record = process.env + environment: Env = process.env ): boolean { return environment[name]?.trim().toLowerCase() === "true"; } + +function hostnameOf(url: string): string { + return new URL( + url.includes("://") ? url : `tcp://${url}` + ).hostname.toLowerCase(); +} + +export function isLocalHost(url: string): boolean { + const hostname = hostnameOf(url); + return ( + LOOPBACK_HOSTS.has(hostname) || (hostname !== "" && !hostname.includes(".")) + ); +} + +export function dataUrl( + key: keyof typeof LOCAL_DATA_URLS, + env: Env = process.env +): string | undefined { + return ( + env[key] || + (env.NODE_ENV === "development" ? LOCAL_DATA_URLS[key] : undefined) + ); +} diff --git a/packages/notifications/src/alarm-config.ts b/packages/notifications/src/alarm-config.ts index 4d794e502e..6478ace983 100644 --- a/packages/notifications/src/alarm-config.ts +++ b/packages/notifications/src/alarm-config.ts @@ -1,4 +1,4 @@ -import { readBooleanEnv } from "@databuddy/env/boolean"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import type { NotificationClientConfig } from "./client"; import type { NotificationChannel } from "./types"; @@ -122,7 +122,7 @@ export function buildAlarmNotificationTargets( }, }); } else if (dest.type === "email") { - if (!process.env.RESEND_API_KEY) { + if (!config.services.resendApiKey) { warnAlarmEmailUnconfigured(); continue; } @@ -139,7 +139,7 @@ export function buildAlarmNotificationTargets( text?: string; }) => { const { Resend } = await import("resend"); - const apiKey = process.env.RESEND_API_KEY; + const apiKey = config.services.resendApiKey; if (!apiKey) { throw new Error("Email delivery is not configured"); } diff --git a/packages/redis/bullmq.ts b/packages/redis/bullmq.ts index e1a500ce1c..32c1587019 100644 --- a/packages/redis/bullmq.ts +++ b/packages/redis/bullmq.ts @@ -1,4 +1,4 @@ -import { readBooleanEnv } from "@databuddy/env/boolean"; +import { dataUrl, readBooleanEnv } from "@databuddy/env/boolean"; import type { RedisOptions } from "ioredis"; export interface BullMQConnectionConfig { @@ -10,11 +10,11 @@ function resolveBullMQRedisUrl(config: BullMQConnectionConfig = {}): string { ? `${config.envPrefix}_BULLMQ_REDIS_URL` : null; const prefixedUrl = prefixedName ? process.env[prefixedName]?.trim() : ""; - const fallbackUrl = process.env.BULLMQ_REDIS_URL?.trim(); + const fallbackUrl = dataUrl("BULLMQ_REDIS_URL")?.trim(); const redisUrl = prefixedUrl || fallbackUrl || - (readBooleanEnv("SELFHOST") ? process.env.REDIS_URL?.trim() : undefined); + (readBooleanEnv("SELFHOST") ? dataUrl("REDIS_URL")?.trim() : undefined); if (!redisUrl) { throw new Error( `${prefixedName ? `${prefixedName} or ` : ""}BULLMQ_REDIS_URL environment variable is required` diff --git a/packages/redis/redis-options.ts b/packages/redis/redis-options.ts index 0f49234ab6..37c48c254b 100644 --- a/packages/redis/redis-options.ts +++ b/packages/redis/redis-options.ts @@ -1,3 +1,5 @@ +import { dataUrl } from "@databuddy/env/app"; + export interface RedisConnectionOptions { commandTimeout: number; connectTimeout: number; @@ -15,7 +17,7 @@ export interface LinkCacheRedisConnectionOptions export type RateLimitRedisConnectionOptions = LinkCacheRedisConnectionOptions; export function getRedisUrl(): string { - const url = process.env.REDIS_URL; + const url = dataUrl("REDIS_URL"); if (!url) { throw new Error("REDIS_URL environment variable is required"); } diff --git a/packages/services/src/business-memory.ts b/packages/services/src/business-memory.ts index 2eb9dbbe66..68336b1014 100644 --- a/packages/services/src/business-memory.ts +++ b/packages/services/src/business-memory.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import type { db } from "@databuddy/db"; +import { config } from "@databuddy/env/app"; import Supermemory from "supermemory"; import { z } from "zod"; @@ -13,7 +14,7 @@ export interface BusinessScope { let client: Supermemory | null = null; export function getMemoryClient(): Supermemory | null { - const apiKey = process.env.SUPERMEMORY_API_KEY; + const apiKey = config.services.supermemoryApiKey; if (!apiKey) { return null; } diff --git a/packages/services/src/feedback.ts b/packages/services/src/feedback.ts index ff067600db..deb99d1490 100644 --- a/packages/services/src/feedback.ts +++ b/packages/services/src/feedback.ts @@ -1,7 +1,8 @@ import { db, eq, type Feedback, feedback, user } from "@databuddy/db"; +import { config } from "@databuddy/env/app"; import { randomUUIDv7 } from "bun"; -const SLACK_WEBHOOK_URL = process.env.SLACK_WEBHOOK_URL ?? ""; +const SLACK_WEBHOOK_URL = config.services.slackWebhookUrl ?? ""; const SLACK_TIMEOUT_MS = 10_000; const DESCRIPTION_PREVIEW_LIMIT = 500; diff --git a/packages/services/src/website-cache.ts b/packages/services/src/website-cache.ts index 4c9d664fa3..05c99b0cb5 100644 --- a/packages/services/src/website-cache.ts +++ b/packages/services/src/website-cache.ts @@ -1,5 +1,6 @@ import { RedisClient } from "bun"; import type { Website } from "@databuddy/db/schema"; +import { dataUrl } from "@databuddy/env/app"; type WebsiteCacheRecord = Omit< Website, @@ -44,7 +45,7 @@ export class WebsiteCache { return this.redis; } - const url = process.env.REDIS_URL; + const url = dataUrl("REDIS_URL"); if (!url) { this.isHealthyCached = false; return null; diff --git a/packages/services/src/websites.ts b/packages/services/src/websites.ts index 577ec1c53e..a60480c70c 100644 --- a/packages/services/src/websites.ts +++ b/packages/services/src/websites.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { db, eq, isUniqueViolationFor } from "@databuddy/db"; +import { dataUrl } from "@databuddy/env/app"; import { type WebsiteInsert, type Website, @@ -88,7 +89,7 @@ export class WebsiteService { } private async invalidateReadCaches(id: string): Promise { - if (!process.env.REDIS_URL) { + if (!dataUrl("REDIS_URL")) { return; } diff --git a/packages/shared/package.json b/packages/shared/package.json index dd3d39d3c0..2c7ada3b20 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -46,6 +46,7 @@ "bots:sync": "curl -fsSL https://raw.githubusercontent.com/arcjet/well-known-bots/main/well-known-bots.json -o src/utils/bot-detection/well-known-bots.json && bun test src/utils/bot-detection" }, "dependencies": { + "@databuddy/env": "workspace:*", "evlog": "catalog:", "ua-parser-js": "catalog:", "undici": "^8.2.0", diff --git a/packages/shared/src/evlog-superlog.ts b/packages/shared/src/evlog-superlog.ts index 3cb13b18c0..09d10853dd 100644 --- a/packages/shared/src/evlog-superlog.ts +++ b/packages/shared/src/evlog-superlog.ts @@ -1,9 +1,10 @@ +import { config } from "@databuddy/env/app"; import type { DrainContext } from "evlog"; import { createOTLPDrain } from "evlog/otlp"; import { createDrainPipeline } from "evlog/pipeline"; export function createBatchedSuperlogDrain() { - const apiKey = process.env.SUPERLOG_API_KEY; + const apiKey = config.services.superlogApiKey; if (!apiKey) { return null; } From 76d78adc6f3984514c0f6332ef99db51120df363 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 02:28:53 +0300 Subject: [PATCH 02/10] refactor(env): simplify service reads and preserve development logging Keep service keys live while suppressing Axiom logs in development. Includes the regression test from 669a8c2ce. --- packages/env/src/app.test.ts | 21 ++++++++++--- packages/env/src/app.ts | 58 ++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/packages/env/src/app.test.ts b/packages/env/src/app.test.ts index a713392dc5..de9bb610b7 100644 --- a/packages/env/src/app.test.ts +++ b/packages/env/src/app.test.ts @@ -272,11 +272,24 @@ describe("services", () => { RESEND_API_KEY: " ", SLACK_WEBHOOK_URL: " https://hooks.slack.com/services/x ", }; - const { services } = createConfig(env); - expect(services.slackWebhookUrl).toBe("https://hooks.slack.com/services/x"); - expect(services.resendApiKey).toBeUndefined(); + const config = createConfig(env); + expect(config.services.slackWebhookUrl).toBe( + "https://hooks.slack.com/services/x" + ); + expect(config.services.resendApiKey).toBeUndefined(); env.RESEND_API_KEY = "re_live"; - expect(services.resendApiKey).toBe("re_live"); + expect(config.services.resendApiKey).toBe("re_live"); + }); + + it("keeps development logs out of Axiom even with a token set", () => { + const env: Record = { + NODE_ENV: "development", + AXIOM_TOKEN: "xaat-dev", + }; + const config = createConfig(env); + expect(config.services.axiomToken).toBeUndefined(); + env.NODE_ENV = "production"; + expect(config.services.axiomToken).toBe("xaat-dev"); }); }); diff --git a/packages/env/src/app.ts b/packages/env/src/app.ts index d20bf61666..dc980288e4 100644 --- a/packages/env/src/app.ts +++ b/packages/env/src/app.ts @@ -51,18 +51,6 @@ const EMAIL = { }, } as const; -const SERVICES = { - autumnSecretKey: "AUTUMN_SECRET_KEY", - axiomToken: "AXIOM_TOKEN", - databuddyApiKey: "DATABUDDY_API_KEY", - dubApiKey: "DUB_API_KEY", - resendApiKey: "RESEND_API_KEY", - slackWebhookUrl: "SLACK_WEBHOOK_URL", - superlogApiKey: "SUPERLOG_API_KEY", - supermemoryApiKey: "SUPERMEMORY_API_KEY", - tccApiKey: "TCC_API_KEY", -} as const; - const TRAILING_SLASH = /\/$/; type Env = Record; @@ -88,7 +76,7 @@ export interface Config { integrations: { openAiAdsPixelId?: string; }; - services: Record; + services: ReturnType; storage?: StorageConfig; urls: { api: string; @@ -104,32 +92,17 @@ export interface Config { const REQUIRED_IN_PRODUCTION = ["BETTER_AUTH_SECRET"] as const; const REQUIRED_IN_HOSTED_CLOUD = ["AUTUMN_SECRET_KEY"] as const; -function liveReads>( - currentEnv: () => Env, - keys: T -): Record { - return Object.defineProperties( - {} as Record, - Object.fromEntries( - Object.entries(keys).map(([name, key]) => [ - name, - { enumerable: true, get: () => readOptional(currentEnv(), key) }, - ]) - ) - ); -} - function isHostedCloud(env: Env): boolean { return env.NODE_ENV === "production" && !readBooleanEnv("SELFHOST", env); } -export type BillingMode = "selfhost" | "live" | "disabled"; - -export function billingMode(env: Env = process.env): BillingMode { +export function billingMode( + env: Env = process.env +): "selfhost" | "live" | "disabled" { if (readBooleanEnv("SELFHOST", env)) { return "selfhost"; } - return isHostedCloud(env) || readOptional(env, SERVICES.autumnSecretKey) + return isHostedCloud(env) || readOptional(env, "AUTUMN_SECRET_KEY") ? "live" : "disabled"; } @@ -181,6 +154,23 @@ function readOrigins(values: Array): string[] { return [...new Set(values.flatMap(readList).map(normalizeOrigin))]; } +function readServices(env: Env) { + return { + autumnSecretKey: readOptional(env, "AUTUMN_SECRET_KEY"), + axiomToken: + env.NODE_ENV === "development" + ? undefined + : readOptional(env, "AXIOM_TOKEN"), + databuddyApiKey: readOptional(env, "DATABUDDY_API_KEY"), + dubApiKey: readOptional(env, "DUB_API_KEY"), + resendApiKey: readOptional(env, "RESEND_API_KEY"), + slackWebhookUrl: readOptional(env, "SLACK_WEBHOOK_URL"), + superlogApiKey: readOptional(env, "SUPERLOG_API_KEY"), + supermemoryApiKey: readOptional(env, "SUPERMEMORY_API_KEY"), + tccApiKey: readOptional(env, "TCC_API_KEY"), + }; +} + function readStorage(env: Env): StorageConfig | undefined { const accessKeyId = readOptional(env, "AWS_ACCESS_KEY_ID"); const secretAccessKey = readOptional(env, "AWS_SECRET_ACCESS_KEY"); @@ -225,7 +215,9 @@ export function createConfig(source?: Env): Config { integrations: { openAiAdsPixelId: readOptional(env, "NEXT_PUBLIC_OPENAI_ADS_PIXEL_ID"), }, - services: liveReads(() => source ?? process.env, SERVICES), + get services() { + return readServices(source ?? process.env); + }, storage: readStorage(env), urls: { api: apiUrl, From 3e4704da6c3ac02bab22a85e2ed4554737d3b893 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 02:16:28 +0300 Subject: [PATCH 03/10] fix(test): give the shared test env the CI Autumn key Since billing is off without AUTUMN_SECRET_KEY, local integration runs skipped plan gates that CI enforces. The test env now defaults the same dummy key CI sets, so both run the gates. --- packages/db/src/test-env.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/db/src/test-env.ts b/packages/db/src/test-env.ts index 1c20377df9..1f027e3ad4 100644 --- a/packages/db/src/test-env.ts +++ b/packages/db/src/test-env.ts @@ -32,4 +32,5 @@ process.env.CLICKHOUSE_URL = : "http://default:@localhost:8123"; process.env.BETTER_AUTH_SECRET ??= "test-auth-secret-for-integration"; process.env.BETTER_AUTH_URL ??= "http://localhost:3001"; +process.env.AUTUMN_SECRET_KEY ||= "test-autumn-secret-key"; process.env.NODE_ENV = "test"; From 074b6cd5348340784a80b90cdabc076aca908f57 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 03:44:17 +0300 Subject: [PATCH 04/10] fix(redis): pin NODE_ENV in BullMQ tests so the dev localhost fallback can't satisfy them dataUrl() falls back to localhost when NODE_ENV=development, which the root .env sets, so the 'requires BULLMQ_REDIS_URL' cases passed in the package and failed under the root test run and pre-push hook. --- packages/redis/bullmq.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/redis/bullmq.test.ts b/packages/redis/bullmq.test.ts index 5f2eae40e0..912e0fc9ef 100644 --- a/packages/redis/bullmq.test.ts +++ b/packages/redis/bullmq.test.ts @@ -9,6 +9,7 @@ const originalEnv = process.env; beforeEach(() => { process.env = { ...originalEnv, + NODE_ENV: "production", SELFHOST: "false", REDIS_URL: "rediss://shared:secret@shared.test:6380/2", BULLMQ_REDIS_URL: "", From fe2989b116e321b58611b1657d942c842d1811ec Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:12:11 +0300 Subject: [PATCH 05/10] fix(env): refuse remote IPv6 in local service guards --- packages/db/src/e2e-db-lifecycle.test.ts | 8 ++++++-- packages/env/src/app.test.ts | 18 +++++++++++++++--- packages/env/src/boolean.ts | 3 ++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/db/src/e2e-db-lifecycle.test.ts b/packages/db/src/e2e-db-lifecycle.test.ts index 8ea6d87674..9a716399e2 100644 --- a/packages/db/src/e2e-db-lifecycle.test.ts +++ b/packages/db/src/e2e-db-lifecycle.test.ts @@ -67,11 +67,15 @@ describe("e2e db lifecycle helpers", () => { ); }); - it("refuses non-local database hosts by default", () => { + it.each([ + "postgres://u:p@db.example.com:5432/databuddy", + "postgres://[2001:db8::1]:5432/databuddy", + "postgres://[::ffff:8.8.8.8]:5432/databuddy", + ])("refuses non-local database hosts by default: %s", (baseDsn) => { expect(() => resolveLifecycleConfig({ allowNonLocal: false, - baseDsn: "postgres://u:p@db.example.com:5432/databuddy", + baseDsn, command: "create", dbPrefix: "databuddy_e2e", }) diff --git a/packages/env/src/app.test.ts b/packages/env/src/app.test.ts index de9bb610b7..b572e9d3d8 100644 --- a/packages/env/src/app.test.ts +++ b/packages/env/src/app.test.ts @@ -222,7 +222,10 @@ describe("isLocalHost", () => { for (const url of [ "postgres://u:p@localhost:5432/databuddy", "http://default:@127.0.0.1:8123", + "http://0.0.0.0:8123", "redis://[::1]:6379", + "redis://[0:0:0:0:0:0:0:1]:6379", + "[::1]:6379", "redis://redis:6379", "redpanda:9092", ]) { @@ -230,9 +233,18 @@ describe("isLocalHost", () => { } }); - it("rejects dotted remote hosts", () => { - expect(isLocalHost("postgres://u:p@db.example.com:5432/x")).toBe(false); - expect(isLocalHost("broker.example.com:9092")).toBe(false); + it("rejects dotted remote hosts and non-loopback IPv6 literals", () => { + for (const url of [ + "postgres://u:p@db.example.com:5432/x", + "broker.example.com:9092", + "postgres://[2001:db8::1]:5432/x", + "http://[::ffff:8.8.8.8]:8123", + "[2001:db8::1]:9092", + ]) { + expect(isLocalHost(url)).toBe(false); + } + // Unbracketed IPv6 is invalid URL input and must also fail closed. + expect(() => isLocalHost("2001:db8::1")).toThrow(); }); }); diff --git a/packages/env/src/boolean.ts b/packages/env/src/boolean.ts index 7570989f15..c467aacbce 100644 --- a/packages/env/src/boolean.ts +++ b/packages/env/src/boolean.ts @@ -31,7 +31,8 @@ function hostnameOf(url: string): string { export function isLocalHost(url: string): boolean { const hostname = hostnameOf(url); return ( - LOOPBACK_HOSTS.has(hostname) || (hostname !== "" && !hostname.includes(".")) + LOOPBACK_HOSTS.has(hostname) || + (hostname !== "" && !hostname.includes(".") && !hostname.includes(":")) ); } From 67e468f225ec181872b442e1d0e6c2ade2d0099e Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:47:18 +0300 Subject: [PATCH 06/10] fix(env): preserve explicit URLs and strict loopback guards --- packages/db/src/clickhouse/client.test.ts | 37 +++++++++++++---- packages/db/src/clickhouse/client.ts | 9 +++-- packages/db/src/e2e-db-lifecycle.test.ts | 36 +++++++++++++++++ packages/db/src/e2e-db-lifecycle.ts | 4 +- packages/db/src/seed.ts | 4 +- packages/db/src/test-env.test.ts | 48 +++++++++++++++++++++++ packages/db/src/test-env.ts | 4 +- packages/env/src/app.test.ts | 23 +++++++++++ packages/env/src/app.ts | 7 +++- packages/env/src/boolean.ts | 5 +++ packages/redis/bullmq.test.ts | 34 ++++++++++++++++ packages/redis/bullmq.ts | 5 ++- 12 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 packages/db/src/test-env.test.ts diff --git a/packages/db/src/clickhouse/client.test.ts b/packages/db/src/clickhouse/client.test.ts index b2d331c55f..40675b5c41 100644 --- a/packages/db/src/clickhouse/client.test.ts +++ b/packages/db/src/clickhouse/client.test.ts @@ -177,27 +177,50 @@ describe("integration test loopback guard", () => { }); } - test("refuses a remote ClickHouse when integration tests are enabled", () => { + test.each([ + "clickhouse.example.test", + "staging", + "clickhouse", + "0.0.0.0", + ])("refuses non-loopback ClickHouse when integration tests are enabled: %s", (host) => { const result = importClient({ CLICKHOUSE_INTEGRATION_TESTS: "true", - CLICKHOUSE_URL: "http://default:@clickhouse.example.test:8123", + CLICKHOUSE_URL: `http://default:@${host}:8123`, }); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain( - 'only run against a loopback server; CLICKHOUSE_URL host is "clickhouse.example.test"' + `only run against a loopback server; CLICKHOUSE_URL host is "${host}"` ); }); - test("refuses an unset ClickHouse URL when integration tests are enabled", () => { - const result = importClient({ CLICKHOUSE_INTEGRATION_TESTS: "true" }); + test.each([ + "production", + "test", + ])("refuses an unset integration URL outside development: %s", (NODE_ENV) => { + const result = importClient({ + NODE_ENV, + CLICKHOUSE_INTEGRATION_TESTS: "true", + }); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain('host is "unset"'); }); - test("allows loopback when integration tests are enabled", () => { + test("allows the resolved development URL when CLICKHOUSE_URL is unset", () => { + const result = importClient({ + NODE_ENV: "development", + CLICKHOUSE_INTEGRATION_TESTS: "true", + }); + expect(result.exitCode, result.stderr.toString()).toBe(0); + }); + + test.each([ + "localhost", + "127.0.0.1", + "[::1]", + ])("allows loopback when integration tests are enabled: %s", (host) => { const result = importClient({ CLICKHOUSE_INTEGRATION_TESTS: "true", - CLICKHOUSE_URL: "http://default:@127.0.0.1:8123/databuddy_analytics", + CLICKHOUSE_URL: `http://default:@${host}:8123/databuddy_analytics`, }); expect(result.exitCode).toBe(0); }); diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts index a52233a051..0da2b8e0e7 100644 --- a/packages/db/src/clickhouse/client.ts +++ b/packages/db/src/clickhouse/client.ts @@ -1,6 +1,6 @@ import { createClient, type ResponseJSON } from "@clickhouse/client"; import type { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config"; -import { dataUrl, isLocalHost } from "@databuddy/env/app"; +import { dataUrl, isLoopbackHost } from "@databuddy/env/app"; import { finalizeDeliveryTables } from "./logical-reads"; export const TABLE_NAMES = { events: "analytics.events", @@ -83,17 +83,18 @@ function assertLoopbackForIntegrationTests(url: string | undefined): void { if (process.env.CLICKHOUSE_INTEGRATION_TESTS !== "true") { return; } - if (!(url && isLocalHost(url))) { + if (!(url && isLoopbackHost(url))) { throw new Error( `ClickHouse integration tests only run against a loopback server; CLICKHOUSE_URL host is "${url ? new URL(url).hostname : "unset"}"` ); } } -assertLoopbackForIntegrationTests(process.env.CLICKHOUSE_URL); +const clickHouseUrl = dataUrl("CLICKHOUSE_URL"); +assertLoopbackForIntegrationTests(clickHouseUrl); const baseClient = createClient({ - url: dataUrl("CLICKHOUSE_URL"), + url: clickHouseUrl, ...CLICKHOUSE_OPTIONS, }); diff --git a/packages/db/src/e2e-db-lifecycle.test.ts b/packages/db/src/e2e-db-lifecycle.test.ts index 9a716399e2..b4852abe2e 100644 --- a/packages/db/src/e2e-db-lifecycle.test.ts +++ b/packages/db/src/e2e-db-lifecycle.test.ts @@ -68,6 +68,9 @@ describe("e2e db lifecycle helpers", () => { }); it.each([ + "postgres://u:p@db:5432/databuddy", + "postgres://u:p@staging:5432/databuddy", + "postgres://u:p@0.0.0.0:5432/databuddy", "postgres://u:p@db.example.com:5432/databuddy", "postgres://[2001:db8::1]:5432/databuddy", "postgres://[::ffff:8.8.8.8]:5432/databuddy", @@ -82,6 +85,39 @@ describe("e2e db lifecycle helpers", () => { ).toThrow("Refusing to manage E2E DB on non-local host"); }); + it("refuses to drop a database on a single-label host without the override", () => { + const args = parseLifecycleArgs([ + "drop", + "--base-dsn", + "postgres://u:p@staging:5432/databuddy", + "--db-name", + "databuddy_e2e_run", + ]); + expect(() => resolveLifecycleConfig(args)).toThrow( + "Refusing to manage E2E DB on non-local host" + ); + expect( + resolveLifecycleConfig({ ...args, allowNonLocal: true }).adminDsn + ).toBe("postgres://u:p@staging:5432/postgres"); + }); + + it.each([ + "localhost", + "127.0.0.1", + "[::1]", + "[0:0:0:0:0:0:0:1]", + ])("allows an E2E database on loopback: %s", (host) => { + expect( + resolveLifecycleConfig({ + allowNonLocal: false, + baseDsn: `postgres://u:p@${host}:5432/databuddy`, + command: "create", + dbPrefix: "databuddy_e2e", + runId: "run", + }).dbName + ).toBe("databuddy_e2e_run"); + }); + it("prints shell-safe assignments", () => { expect( toShellAssignments({ DATABASE_URL: "postgres://u:p@localhost/db'quoted" }) diff --git a/packages/db/src/e2e-db-lifecycle.ts b/packages/db/src/e2e-db-lifecycle.ts index 6d2ce43ab1..0396aac8d7 100644 --- a/packages/db/src/e2e-db-lifecycle.ts +++ b/packages/db/src/e2e-db-lifecycle.ts @@ -1,4 +1,4 @@ -import { isLocalHost } from "@databuddy/env/app"; +import { isLoopbackHost } from "@databuddy/env/app"; import { Client } from "pg"; const DEFAULT_E2E_DB_PREFIX = "databuddy_e2e"; @@ -142,7 +142,7 @@ export function resolveLifecycleConfig( input: ParsedLifecycleArgs ): ResolvedLifecycleConfig { const baseUrl = normalizeDatabaseUrl(input.baseDsn); - if (!(input.allowNonLocal || isLocalHost(baseUrl.href))) { + if (!(input.allowNonLocal || isLoopbackHost(baseUrl.href))) { throw new Error( `Refusing to manage E2E DB on non-local host '${baseUrl.hostname}'. Set --allow-non-local to override.` ); diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts index 651cd79a0c..6f1557e7bd 100644 --- a/packages/db/src/seed.ts +++ b/packages/db/src/seed.ts @@ -1,4 +1,4 @@ -import { dataUrl, isLocalHost } from "@databuddy/env/app"; +import { dataUrl, isLoopbackHost } from "@databuddy/env/app"; import { faker } from "@faker-js/faker"; import { clickHouse, TABLE_NAMES } from "./clickhouse/client"; import { db } from "./client"; @@ -6,7 +6,7 @@ import { db } from "./client"; for (const name of ["CLICKHOUSE_URL", "DATABASE_URL"] as const) { const url = dataUrl(name); const hostname = url && URL.canParse(url) ? new URL(url).hostname : ""; - if (!(hostname && isLocalHost(hostname))) { + if (!(url && hostname && isLoopbackHost(url))) { throw new Error( `db:seed only runs against local databases; ${name} host is "${hostname || "unset"}"` ); diff --git a/packages/db/src/test-env.test.ts b/packages/db/src/test-env.test.ts new file mode 100644 index 0000000000..7a79c660ba --- /dev/null +++ b/packages/db/src/test-env.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from "bun:test"; + +describe("test environment service URLs", () => { + function importTestEnv(overrides: Record) { + return Bun.spawnSync({ + cmd: [process.execPath, "--no-env-file", "-e", 'import "./test-env.ts"'], + cwd: import.meta.dir, + env: { + ...process.env, + CI: "true", + DATABASE_URL: "postgres://u:p@localhost:5432/databuddy_test", + REDIS_URL: "redis://localhost:6379/1", + BULLMQ_REDIS_URL: "redis://localhost:6379/1", + CLICKHOUSE_URL: "http://localhost:8123", + ...overrides, + }, + stdout: "ignore", + stderr: "pipe", + }); + } + + test.each([ + ["DATABASE_URL", "postgres://u:p@staging:5432/databuddy_test"], + ["REDIS_URL", "redis://redis:6379/1"], + ["BULLMQ_REDIS_URL", "redis://queue:6379/1"], + ["CLICKHOUSE_URL", "http://clickhouse:8123"], + ])("refuses CI-supplied non-loopback %s", (name, url) => { + const result = importTestEnv({ [name]: url }); + expect(result.exitCode).not.toBe(0); + expect(result.stderr.toString()).toContain( + `${name} must point at a local test service` + ); + }); + + test.each([ + "localhost", + "127.0.0.1", + "[::1]", + ])("allows CI-supplied loopback services: %s", (host) => { + const result = importTestEnv({ + DATABASE_URL: `postgres://u:p@${host}:5432/databuddy_test`, + REDIS_URL: `redis://${host}:6379/1`, + BULLMQ_REDIS_URL: `redis://${host}:6379/1`, + CLICKHOUSE_URL: `http://${host}:8123`, + }); + expect(result.exitCode, result.stderr.toString()).toBe(0); + }); +}); diff --git a/packages/db/src/test-env.ts b/packages/db/src/test-env.ts index 1f027e3ad4..da77fedfba 100644 --- a/packages/db/src/test-env.ts +++ b/packages/db/src/test-env.ts @@ -1,4 +1,4 @@ -import { isLocalHost, readBooleanEnv } from "@databuddy/env/app"; +import { isLoopbackHost, readBooleanEnv } from "@databuddy/env/app"; const useCiUrls = readBooleanEnv("CI"); const defaultDatabaseUrl = @@ -6,7 +6,7 @@ const defaultDatabaseUrl = const defaultRedisUrl = "redis://localhost:6379/1"; function localTestUrl(name: string, value: string): string { - if (!isLocalHost(value)) { + if (!isLoopbackHost(value)) { throw new Error( `${name} must point at a local test service. Refusing to run tests against ${new URL(value).hostname}.` ); diff --git a/packages/env/src/app.test.ts b/packages/env/src/app.test.ts index b572e9d3d8..aab49ee019 100644 --- a/packages/env/src/app.test.ts +++ b/packages/env/src/app.test.ts @@ -5,6 +5,7 @@ import { createConfig, dataUrl, isLocalHost, + isLoopbackHost, readBooleanEnv, } from "./app"; @@ -248,6 +249,28 @@ describe("isLocalHost", () => { }); }); +describe("isLoopbackHost", () => { + it.each([ + "postgres://u:p@localhost:5432/databuddy", + "http://127.0.0.1:8123", + "redis://[::1]:6379", + "redis://[0:0:0:0:0:0:0:1]:6379", + ])("accepts concrete loopback URLs: %s", (url) => { + expect(isLoopbackHost(url)).toBe(true); + }); + + it.each([ + "postgres://u:p@db:5432/databuddy", + "postgres://u:p@staging:5432/databuddy", + "redis://redis:6379", + "http://0.0.0.0:8123", + "http://clickhouse.example.com:8123", + "http://[2001:db8::1]:8123", + ])("refuses non-loopback URLs at test/destructive boundaries: %s", (url) => { + expect(isLoopbackHost(url)).toBe(false); + }); +}); + describe("dataUrl", () => { const REMOTE = "postgres://u:p@db.example.com:5432/databuddy"; diff --git a/packages/env/src/app.ts b/packages/env/src/app.ts index dc980288e4..27508a129f 100644 --- a/packages/env/src/app.ts +++ b/packages/env/src/app.ts @@ -1,6 +1,11 @@ import { LOOPBACK_HOSTS, readBooleanEnv } from "./boolean"; -export { dataUrl, isLocalHost, readBooleanEnv } from "./boolean"; +export { + dataUrl, + isLocalHost, + isLoopbackHost, + readBooleanEnv, +} from "./boolean"; // App-wide runtime config. // diff --git a/packages/env/src/boolean.ts b/packages/env/src/boolean.ts index c467aacbce..1f1a7dbb98 100644 --- a/packages/env/src/boolean.ts +++ b/packages/env/src/boolean.ts @@ -36,6 +36,11 @@ export function isLocalHost(url: string): boolean { ); } +export function isLoopbackHost(url: string): boolean { + const hostname = hostnameOf(url); + return hostname !== "0.0.0.0" && LOOPBACK_HOSTS.has(hostname); +} + export function dataUrl( key: keyof typeof LOCAL_DATA_URLS, env: Env = process.env diff --git a/packages/redis/bullmq.test.ts b/packages/redis/bullmq.test.ts index 912e0fc9ef..b55f2421aa 100644 --- a/packages/redis/bullmq.test.ts +++ b/packages/redis/bullmq.test.ts @@ -96,6 +96,40 @@ assert.equal(getBullMQConnectionOptions().host, "queue.example.com"); expect(() => getBullMQConnectionOptions()).toThrow(); }); + it("prefers the self-hosted shared URL over development defaults", () => { + process.env.NODE_ENV = "development"; + process.env.SELFHOST = "true"; + delete process.env.BULLMQ_REDIS_URL; + + for (const resolve of [ + getBullMQConnectionOptions, + getBullMQWorkerConnectionOptions, + ]) { + expect(resolve({ envPrefix: "INSIGHTS" })).toMatchObject({ + host: "shared.test", + port: 6380, + db: 2, + tls: {}, + }); + process.env.BULLMQ_REDIS_URL = "redis://dedicated.test:6379/3"; + expect(resolve().host).toBe("dedicated.test"); + delete process.env.BULLMQ_REDIS_URL; + } + }); + + it.each([ + "false", + "true", + ])("keeps local development defaults when SELFHOST=%s and URLs are unset", (selfhost) => { + process.env.NODE_ENV = "development"; + process.env.SELFHOST = selfhost; + delete process.env.BULLMQ_REDIS_URL; + delete process.env.REDIS_URL; + + expect(getBullMQConnectionOptions().host).toBe("localhost"); + expect(getBullMQWorkerConnectionOptions().host).toBe("localhost"); + }); + it("parses redis URLs for queue producers", () => { process.env.BULLMQ_REDIS_URL = "redis://user:pass@example.test:6380/3"; diff --git a/packages/redis/bullmq.ts b/packages/redis/bullmq.ts index 32c1587019..8cc1ad00d2 100644 --- a/packages/redis/bullmq.ts +++ b/packages/redis/bullmq.ts @@ -10,11 +10,12 @@ function resolveBullMQRedisUrl(config: BullMQConnectionConfig = {}): string { ? `${config.envPrefix}_BULLMQ_REDIS_URL` : null; const prefixedUrl = prefixedName ? process.env[prefixedName]?.trim() : ""; - const fallbackUrl = dataUrl("BULLMQ_REDIS_URL")?.trim(); + const fallbackUrl = process.env.BULLMQ_REDIS_URL?.trim(); const redisUrl = prefixedUrl || fallbackUrl || - (readBooleanEnv("SELFHOST") ? dataUrl("REDIS_URL")?.trim() : undefined); + (readBooleanEnv("SELFHOST") ? dataUrl("REDIS_URL")?.trim() : undefined) || + dataUrl("BULLMQ_REDIS_URL")?.trim(); if (!redisUrl) { throw new Error( `${prefixedName ? `${prefixedName} or ` : ""}BULLMQ_REDIS_URL environment variable is required` From 1eed79e98622468dd1bdc837c550613380717923 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:13:12 +0300 Subject: [PATCH 07/10] fix(db): guard lifecycle overrides and local command defaults --- package.json | 4 +- packages/db/src/db-command-env.test.ts | 71 ++++++++++++++++++++++++ packages/db/src/e2e-db-lifecycle.test.ts | 35 ++++++++++++ packages/db/src/e2e-db-lifecycle.ts | 5 ++ 4 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 packages/db/src/db-command-env.test.ts diff --git a/package.json b/package.json index a3e2328c32..a761b7ca58 100644 --- a/package.json +++ b/package.json @@ -45,11 +45,11 @@ "format": "bunx ultracite fix", "check-types": "dotenv -- turbo run check-types", "generate-db": "turbo run generate --filter=@databuddy/db", - "db:studio": "dotenv -- turbo run db:studio", + "db:studio": "dotenv -- sh -c 'NODE_ENV=${NODE_ENV:-development} turbo run db:studio \"$@\"' --", "clickhouse:init": "dotenv -- turbo run clickhouse:init", "clickhouse:dql:init": "dotenv -- bun packages/db/src/clickhouse/dql.ts", "ch:verify": "dotenv -- bun packages/db/src/clickhouse/verify.ts", - "db:push": "dotenv -- turbo run db:push", + "db:push": "dotenv -- sh -c 'NODE_ENV=${NODE_ENV:-development} turbo run db:push \"$@\"' --", "db:migrate": "dotenv -- turbo run db:migrate", "db:deploy": "dotenv -- turbo run db:deploy", "db:seed": "dotenv -- sh -c 'cd packages/db && bun run db:seed \"$@\"' --", diff --git a/packages/db/src/db-command-env.test.ts b/packages/db/src/db-command-env.test.ts new file mode 100644 index 0000000000..6f4d828f14 --- /dev/null +++ b/packages/db/src/db-command-env.test.ts @@ -0,0 +1,71 @@ +import { chmodSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "bun:test"; +import manifest from "../../../package.json"; + +const dotenv = join(import.meta.dir, "../../../node_modules/dotenv-cli/cli.js"); +const node = Bun.which("node"); +if (!node) { + throw new Error("Node is required to run the dotenv CLI"); +} + +it.each([ + "db:push", + "db:studio", +] as const)("defaults %s after dotenv without changing explicit modes or arguments", async (command) => { + const directory = mkdtempSync(join(tmpdir(), "databuddy-db-command-")); + try { + await Bun.write( + join(directory, "dotenv"), + '#!/bin/sh\nexec "$DB_COMMAND_NODE" "$DB_COMMAND_DOTENV" "$@"\n' + ); + await Bun.write( + join(directory, "turbo"), + '#!/bin/sh\nexec "$DB_COMMAND_BUN" --no-env-file "$DB_COMMAND_PROBE" "$@"\n' + ); + await Bun.write( + join(directory, "probe.ts"), + "console.log(JSON.stringify({ mode: process.env.NODE_ENV, args: process.argv.slice(2) }));\n" + ); + chmodSync(join(directory, "dotenv"), 0o755); + chmodSync(join(directory, "turbo"), 0o755); + + for (const [file, inherited, expected] of [ + ["", undefined, "development"], + ["NODE_ENV=\n", undefined, "development"], + ["NODE_ENV=production\n", undefined, "production"], + ["NODE_ENV=test\n", undefined, "test"], + ["NODE_ENV=development\n", "production", "production"], + ] as const) { + await Bun.write(join(directory, ".env"), file); + const child = Bun.spawn( + [ + "/bin/sh", + "-c", + `${manifest.scripts[command]} --dry-run "argument with spaces"`, + ], + { + cwd: directory, + env: { + PATH: `${directory}:/usr/bin:/bin`, + DB_COMMAND_BUN: process.execPath, + DB_COMMAND_NODE: node, + DB_COMMAND_DOTENV: dotenv, + DB_COMMAND_PROBE: join(directory, "probe.ts"), + ...(inherited === undefined ? {} : { NODE_ENV: inherited }), + }, + stdout: "pipe", + stderr: "pipe", + } + ); + expect(await child.exited).toBe(0); + expect(await new Response(child.stdout).json()).toEqual({ + mode: expected, + args: ["run", command, "--dry-run", "argument with spaces"], + }); + } + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/db/src/e2e-db-lifecycle.test.ts b/packages/db/src/e2e-db-lifecycle.test.ts index b4852abe2e..3163955e83 100644 --- a/packages/db/src/e2e-db-lifecycle.test.ts +++ b/packages/db/src/e2e-db-lifecycle.test.ts @@ -101,6 +101,41 @@ describe("e2e db lifecycle helpers", () => { ).toBe("postgres://u:p@staging:5432/postgres"); }); + it.each([ + "host=remote.example", + "%68ost=remote.example", + "host=localhost&host=remote.example", + "host=%2Ftmp%2Fpostgres", + ])("refuses PostgreSQL host overrides before create or drop: %s", (query) => { + for (const command of ["create", "drop"] as const) { + expect(() => + resolveLifecycleConfig({ + allowNonLocal: false, + baseDsn: `postgres://u:p@localhost:5432/databuddy?${query}`, + command, + dbName: "databuddy_e2e_run", + dbPrefix: "databuddy_e2e", + }) + ).toThrow("Refusing to manage E2E DB with a PostgreSQL host override"); + } + }); + + it("preserves an explicitly allowed PostgreSQL host override", () => { + const config = resolveLifecycleConfig({ + allowNonLocal: true, + baseDsn: "postgres://u:p@localhost:5432/databuddy?host=remote.example", + command: "create", + dbName: "databuddy_e2e_run", + dbPrefix: "databuddy_e2e", + }); + expect(config.adminDsn).toBe( + "postgres://u:p@localhost:5432/postgres?host=remote.example" + ); + expect(config.dbDsn).toBe( + "postgres://u:p@localhost:5432/databuddy_e2e_run?host=remote.example" + ); + }); + it.each([ "localhost", "127.0.0.1", diff --git a/packages/db/src/e2e-db-lifecycle.ts b/packages/db/src/e2e-db-lifecycle.ts index 0396aac8d7..aa4a5abfb8 100644 --- a/packages/db/src/e2e-db-lifecycle.ts +++ b/packages/db/src/e2e-db-lifecycle.ts @@ -142,6 +142,11 @@ export function resolveLifecycleConfig( input: ParsedLifecycleArgs ): ResolvedLifecycleConfig { const baseUrl = normalizeDatabaseUrl(input.baseDsn); + if (!input.allowNonLocal && baseUrl.searchParams.has("host")) { + throw new Error( + "Refusing to manage E2E DB with a PostgreSQL host override. Set --allow-non-local to override." + ); + } if (!(input.allowNonLocal || isLoopbackHost(baseUrl.href))) { throw new Error( `Refusing to manage E2E DB on non-local host '${baseUrl.hostname}'. Set --allow-non-local to override.` From f81cd086a250e94d187894d278cc2abfd84376e1 Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:25:19 +0300 Subject: [PATCH 08/10] fix(db): resolve dotenv CLI across workspace installs --- packages/db/src/db-command-env.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/src/db-command-env.test.ts b/packages/db/src/db-command-env.test.ts index 6f4d828f14..e9179850d5 100644 --- a/packages/db/src/db-command-env.test.ts +++ b/packages/db/src/db-command-env.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { expect, it } from "bun:test"; import manifest from "../../../package.json"; -const dotenv = join(import.meta.dir, "../../../node_modules/dotenv-cli/cli.js"); +const dotenv = Bun.resolveSync("dotenv-cli/cli.js", import.meta.dir); const node = Bun.which("node"); if (!node) { throw new Error("Node is required to run the dotenv CLI"); From a476b16c890df64e97ecfaf5d7a40eb7b734c05c Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:13:56 +0000 Subject: [PATCH 09/10] fix(env): redact malformed service URLs from errors --- packages/env/src/app.test.ts | 19 +++++++++++++++++++ packages/env/src/boolean.ts | 10 +++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/packages/env/src/app.test.ts b/packages/env/src/app.test.ts index aab49ee019..809b5ff22e 100644 --- a/packages/env/src/app.test.ts +++ b/packages/env/src/app.test.ts @@ -244,6 +244,25 @@ describe("isLocalHost", () => { ]) { expect(isLocalHost(url)).toBe(false); } + }); + + it("rejects malformed service URLs without exposing credentials", () => { + const invalid = "postgres://synthetic-user:synthetic-private@[broken"; + for (const checkHost of [isLocalHost, isLoopbackHost]) { + let failure: unknown; + try { + checkHost(invalid); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + expect(String(failure)).toContain("configured service URL"); + expect(JSON.stringify(failure)).not.toContain("synthetic-private"); + expect(Bun.inspect(failure)).not.toContain("synthetic-private"); + } + }); + + it("rejects unbracketed IPv6", () => { // Unbracketed IPv6 is invalid URL input and must also fail closed. expect(() => isLocalHost("2001:db8::1")).toThrow(); }); diff --git a/packages/env/src/boolean.ts b/packages/env/src/boolean.ts index 1f1a7dbb98..f607139674 100644 --- a/packages/env/src/boolean.ts +++ b/packages/env/src/boolean.ts @@ -23,9 +23,13 @@ export function readBooleanEnv( } function hostnameOf(url: string): string { - return new URL( - url.includes("://") ? url : `tcp://${url}` - ).hostname.toLowerCase(); + try { + return new URL( + url.includes("://") ? url : `tcp://${url}` + ).hostname.toLowerCase(); + } catch { + throw new Error("Cannot parse host from the configured service URL"); + } } export function isLocalHost(url: string): boolean { From 2f397df35c3e929051061e1644e4785f50f2ef1a Mon Sep 17 00:00:00 2001 From: iza <59828082+izadoesdev@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:22:58 +0000 Subject: [PATCH 10/10] test(env): align service guards with repository rules --- packages/db/src/db-command-env.test.ts | 87 +++++++++++++++----------- packages/env/src/app.test.ts | 5 +- packages/redis/bullmq.test.ts | 10 +-- 3 files changed, 58 insertions(+), 44 deletions(-) diff --git a/packages/db/src/db-command-env.test.ts b/packages/db/src/db-command-env.test.ts index e9179850d5..cc8eb911f3 100644 --- a/packages/db/src/db-command-env.test.ts +++ b/packages/db/src/db-command-env.test.ts @@ -10,10 +10,29 @@ if (!node) { throw new Error("Node is required to run the dotenv CLI"); } -it.each([ - "db:push", - "db:studio", -] as const)("defaults %s after dotenv without changing explicit modes or arguments", async (command) => { +it.each( + (["db:push", "db:studio"] as const).flatMap((command) => + ( + [ + ["", undefined, "development"], + ["NODE_ENV=\n", undefined, "development"], + ["NODE_ENV=production\n", undefined, "production"], + ["NODE_ENV=test\n", undefined, "test"], + ["NODE_ENV=development\n", "production", "production"], + ] as const + ).map(([file, inherited, expected]) => ({ + command, + file, + inherited, + expected, + })) + ) +)("runs $command with $expected mode after dotenv and preserves arguments", async ({ + command, + file, + inherited, + expected, +}) => { const directory = mkdtempSync(join(tmpdir(), "databuddy-db-command-")); try { await Bun.write( @@ -31,40 +50,32 @@ it.each([ chmodSync(join(directory, "dotenv"), 0o755); chmodSync(join(directory, "turbo"), 0o755); - for (const [file, inherited, expected] of [ - ["", undefined, "development"], - ["NODE_ENV=\n", undefined, "development"], - ["NODE_ENV=production\n", undefined, "production"], - ["NODE_ENV=test\n", undefined, "test"], - ["NODE_ENV=development\n", "production", "production"], - ] as const) { - await Bun.write(join(directory, ".env"), file); - const child = Bun.spawn( - [ - "/bin/sh", - "-c", - `${manifest.scripts[command]} --dry-run "argument with spaces"`, - ], - { - cwd: directory, - env: { - PATH: `${directory}:/usr/bin:/bin`, - DB_COMMAND_BUN: process.execPath, - DB_COMMAND_NODE: node, - DB_COMMAND_DOTENV: dotenv, - DB_COMMAND_PROBE: join(directory, "probe.ts"), - ...(inherited === undefined ? {} : { NODE_ENV: inherited }), - }, - stdout: "pipe", - stderr: "pipe", - } - ); - expect(await child.exited).toBe(0); - expect(await new Response(child.stdout).json()).toEqual({ - mode: expected, - args: ["run", command, "--dry-run", "argument with spaces"], - }); - } + await Bun.write(join(directory, ".env"), file); + const child = Bun.spawn( + [ + "/bin/sh", + "-c", + `${manifest.scripts[command]} --dry-run "argument with spaces"`, + ], + { + cwd: directory, + env: { + PATH: `${directory}:/usr/bin:/bin`, + DB_COMMAND_BUN: process.execPath, + DB_COMMAND_NODE: node, + DB_COMMAND_DOTENV: dotenv, + DB_COMMAND_PROBE: join(directory, "probe.ts"), + ...(inherited === undefined ? {} : { NODE_ENV: inherited }), + }, + stdout: "pipe", + stderr: "pipe", + } + ); + expect(await child.exited).toBe(0); + expect(await new Response(child.stdout).json()).toEqual({ + mode: expected, + args: ["run", command, "--dry-run", "argument with spaces"], + }); } finally { rmSync(directory, { recursive: true, force: true }); } diff --git a/packages/env/src/app.test.ts b/packages/env/src/app.test.ts index 809b5ff22e..42ce4d34e9 100644 --- a/packages/env/src/app.test.ts +++ b/packages/env/src/app.test.ts @@ -249,10 +249,13 @@ describe("isLocalHost", () => { it("rejects malformed service URLs without exposing credentials", () => { const invalid = "postgres://synthetic-user:synthetic-private@[broken"; for (const checkHost of [isLocalHost, isLoopbackHost]) { - let failure: unknown; + let failure: Error | undefined; try { checkHost(invalid); } catch (error) { + if (!(error instanceof Error)) { + throw error; + } failure = error; } expect(failure).toBeInstanceOf(Error); diff --git a/packages/redis/bullmq.test.ts b/packages/redis/bullmq.test.ts index b55f2421aa..13371af98f 100644 --- a/packages/redis/bullmq.test.ts +++ b/packages/redis/bullmq.test.ts @@ -58,7 +58,7 @@ assert.equal(getBullMQConnectionOptions().host, "queue.example.com"); }); it("requires BULLMQ_REDIS_URL", () => { - delete process.env.BULLMQ_REDIS_URL; + Reflect.deleteProperty(process.env, "BULLMQ_REDIS_URL"); expect(() => getBullMQConnectionOptions()).toThrow( "BULLMQ_REDIS_URL environment variable is required" @@ -99,7 +99,7 @@ assert.equal(getBullMQConnectionOptions().host, "queue.example.com"); it("prefers the self-hosted shared URL over development defaults", () => { process.env.NODE_ENV = "development"; process.env.SELFHOST = "true"; - delete process.env.BULLMQ_REDIS_URL; + Reflect.deleteProperty(process.env, "BULLMQ_REDIS_URL"); for (const resolve of [ getBullMQConnectionOptions, @@ -113,7 +113,7 @@ assert.equal(getBullMQConnectionOptions().host, "queue.example.com"); }); process.env.BULLMQ_REDIS_URL = "redis://dedicated.test:6379/3"; expect(resolve().host).toBe("dedicated.test"); - delete process.env.BULLMQ_REDIS_URL; + Reflect.deleteProperty(process.env, "BULLMQ_REDIS_URL"); } }); @@ -123,8 +123,8 @@ assert.equal(getBullMQConnectionOptions().host, "queue.example.com"); ])("keeps local development defaults when SELFHOST=%s and URLs are unset", (selfhost) => { process.env.NODE_ENV = "development"; process.env.SELFHOST = selfhost; - delete process.env.BULLMQ_REDIS_URL; - delete process.env.REDIS_URL; + Reflect.deleteProperty(process.env, "BULLMQ_REDIS_URL"); + Reflect.deleteProperty(process.env, "REDIS_URL"); expect(getBullMQConnectionOptions().host).toBe("localhost"); expect(getBullMQWorkerConnectionOptions().host).toBe("localhost");