diff --git a/.env.example b/.env.example index 13291ca290..cc94ece2c6 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,16 @@ # Local development. For Docker self-hosting, use selfhost.env.example. -CLICKHOUSE_URL="http://default:@localhost:8123/databuddy_analytics" +# DATABASE_URL, REDIS_URL, BULLMQ_REDIS_URL and CLICKHOUSE_URL default to the +# docker-compose services in development. Set them only to point elsewhere. + # Required by DQL. This must use the separately provisioned dql_user account; # never point it at the application/admin ClickHouse credentials. CLICKHOUSE_DQL_URL="http://dql_user:databuddy_dql_dev_password@localhost:8123/databuddy_analytics" -DATABASE_URL="postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy" -REDIS_URL="redis://localhost:6379" -BULLMQ_REDIS_URL="redis://localhost:6379" + +# Redpanda (Kafka) ingestion. Blank writes straight to ClickHouse. +REDPANDA_BROKER="" +REDPANDA_USER="" +REDPANDA_PASSWORD="" # Public URLs. Local development uses built-in localhost defaults when blank. DASHBOARD_URL="" @@ -65,6 +69,8 @@ SLACK_SIGNING_SECRET="" SLACK_CLIENT_ID="" SLACK_CLIENT_SECRET="" SLACK_REDIRECT_URI="" +# Internal signup, billing, and feedback notifications +SLACK_WEBHOOK_URL="" # Not Necessary unless using blog MARBLE_API_KEY= @@ -77,6 +83,11 @@ NOTRA_API_KEY= # Superlog OTLP logs export. When set, evlog drains in api/basket/insights/slack/uptime # also ship wide events to https://intake.superlog.sh/v1/logs. SUPERLOG_API_KEY="" +AXIOM_TOKEN="" +TCC_API_KEY="" + +# Databuddy self-analytics for product events +DATABUDDY_API_KEY="" # S3-compatible object storage for status page logo and favicon uploads. # Both credentials are required together; the API refuses to boot with only one. diff --git a/apps/api/src/lib/evlog-api.ts b/apps/api/src/lib/evlog-api.ts index 412ad1e326..ecea68ebf7 100644 --- a/apps/api/src/lib/evlog-api.ts +++ b/apps/api/src/lib/evlog-api.ts @@ -1,6 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog"; import type { DrainContext, EnrichContext } from "evlog"; import { createAxiomDrain } from "evlog/axiom"; @@ -15,7 +15,7 @@ import { createDrainPipeline } from "evlog/pipeline"; const batchedAxiomDrain = createDrainPipeline({ batch: { size: 50, intervalMs: 5000 }, maxBufferSize: 2000, -})(createAxiomDrain({ apiKey: process.env.AXIOM_TOKEN })); +})(createAxiomDrain({ apiKey: config.services.axiomToken })); const batchedSuperlogDrain = createBatchedSuperlogDrain(); @@ -30,8 +30,7 @@ const devFsLogsDir = join( const useLocalEvlogFiles = process.env.NODE_ENV === "development" || readBooleanEnv("API_EVLOG_FS"); -const drainToAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const drainToAxiom = Boolean(config.services.axiomToken); const devFsDrain = useLocalEvlogFiles ? createFsDrain({ dir: devFsLogsDir, pretty: false }) diff --git a/apps/api/src/lib/tcc-otel.ts b/apps/api/src/lib/tcc-otel.ts index 78bf99db4b..69abbc12b8 100644 --- a/apps/api/src/lib/tcc-otel.ts +++ b/apps/api/src/lib/tcc-otel.ts @@ -1,4 +1,5 @@ import { TCCSpanProcessor } from "@contextcompany/otel"; +import { config } from "@databuddy/env/app"; import { resourceFromAttributes } from "@opentelemetry/resources"; import { NodeSDK } from "@opentelemetry/sdk-node"; import { @@ -9,7 +10,7 @@ import pkg from "../../package.json"; let sdk: NodeSDK | null = null; export function initTccTracing(): void { - if (sdk || !process.env.TCC_API_KEY) { + if (sdk || !config.services.tccApiKey) { return; } diff --git a/apps/basket/src/lib/blocked-traffic-alerts.ts b/apps/basket/src/lib/blocked-traffic-alerts.ts index 88e2f1e808..fe36df54c1 100644 --- a/apps/basket/src/lib/blocked-traffic-alerts.ts +++ b/apps/basket/src/lib/blocked-traffic-alerts.ts @@ -5,7 +5,7 @@ import { normalizeEmailNotificationSettings, type EmailNotificationSettings, } from "@databuddy/db"; -import { config } from "@databuddy/env/app"; +import { config, dataUrl } from "@databuddy/env/app"; import { BlockedTrafficAlertEmail, render } from "@databuddy/email"; import { redis } from "@databuddy/redis"; import { @@ -282,7 +282,7 @@ async function sendAlertEmail(input: { previousBlocked: number; windowBlockedCount: number; }): Promise { - const apiKey = requireBlockedTrafficEmailApiKey(config.email.resendApiKey); + const apiKey = requireBlockedTrafficEmailApiKey(config.services.resendApiKey); const siteLabel = input.context.websiteName || @@ -346,7 +346,7 @@ async function maybeSendBlockedTrafficAlertAsync( if (!context.ownerId) { return; } - if (!process.env.REDIS_URL) { + if (!dataUrl("REDIS_URL")) { return; } @@ -379,7 +379,7 @@ async function maybeSendBlockedTrafficAlertAsync( return; } - if (!config.email.resendApiKey) { + if (!config.services.resendApiKey) { warnBlockedTrafficEmailUnconfigured(); return; } diff --git a/apps/basket/src/lib/evlog-basket.ts b/apps/basket/src/lib/evlog-basket.ts index f750b1123b..05271b4189 100644 --- a/apps/basket/src/lib/evlog-basket.ts +++ b/apps/basket/src/lib/evlog-basket.ts @@ -1,6 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createBatchedAxiomDrain, downgradeClientHttpError, @@ -12,7 +12,7 @@ import { CLIENT_ERROR_MESSAGES } from "@lib/structured-errors"; import type { DrainContext, EnrichContext } from "evlog"; import { createFsDrain } from "evlog/fs"; -const batchedAxiomDrain = createBatchedAxiomDrain(process.env.AXIOM_TOKEN); +const batchedAxiomDrain = createBatchedAxiomDrain(config.services.axiomToken); const batchedSuperlogDrain = createBatchedSuperlogDrain(); @@ -27,8 +27,7 @@ const devFsLogsDir = join( const useLocalEvlogFiles = process.env.NODE_ENV === "development" || readBooleanEnv("BASKET_EVLOG_FS"); -const drainToAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const drainToAxiom = Boolean(config.services.axiomToken); const devFsDrain = useLocalEvlogFiles ? createFsDrain({ dir: devFsLogsDir, pretty: false }) diff --git a/apps/dashboard/app/(auth)/layout.tsx b/apps/dashboard/app/(auth)/layout.tsx index 7928c75300..7db1ce9407 100644 --- a/apps/dashboard/app/(auth)/layout.tsx +++ b/apps/dashboard/app/(auth)/layout.tsx @@ -1,4 +1,4 @@ -import { readBooleanEnv } from "@databuddy/env/boolean"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { connection } from "next/server"; import type { ReactNode } from "react"; import AuthLayout from "./auth-layout"; @@ -12,7 +12,7 @@ export default async function Layout({ children }: { children: ReactNode }) { ({ maxBufferSize: 2000, }); -const hasAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const hasAxiom = Boolean(config.services.axiomToken); const batchedAxiomDrain = hasAxiom ? pipeline( createAxiomDrain({ - apiKey: process.env.AXIOM_TOKEN, + apiKey: config.services.axiomToken, dataset: "insights", }) ) diff --git a/apps/links/src/lib/logging.ts b/apps/links/src/lib/logging.ts index eeaa507013..5bbd0d8351 100644 --- a/apps/links/src/lib/logging.ts +++ b/apps/links/src/lib/logging.ts @@ -1,6 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createDatabuddyEvlogEnv } from "@databuddy/shared/evlog-redaction"; import type { DrainContext, EnrichContext, WideEvent } from "evlog"; import { log } from "evlog"; @@ -20,7 +20,7 @@ type LogFields = Record; const batchedAxiomDrain = createDrainPipeline({ batch: { size: 50, intervalMs: 5000 }, maxBufferSize: 2000, -})(createAxiomDrain({ apiKey: process.env.AXIOM_TOKEN })); +})(createAxiomDrain({ apiKey: config.services.axiomToken })); const fsDrain = process.env.NODE_ENV === "development" || readBooleanEnv("LINKS_EVLOG_FS") @@ -36,8 +36,7 @@ const fsDrain = }) : null; -const drainToAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const drainToAxiom = Boolean(config.services.axiomToken); const DURATION_RE = /^([\d.]+)(ms|s)$/; diff --git a/apps/slack/src/lib/evlog-slack.ts b/apps/slack/src/lib/evlog-slack.ts index 578b172cb7..a836d53f02 100644 --- a/apps/slack/src/lib/evlog-slack.ts +++ b/apps/slack/src/lib/evlog-slack.ts @@ -2,7 +2,7 @@ import { isRecord } from "@/lib/guards"; import { AsyncLocalStorage } from "node:async_hooks"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog"; import type { DrainContext, RequestLogger } from "evlog"; import { createLogger, log } from "evlog"; @@ -15,15 +15,14 @@ type SlackLogFields = Record; const activeSlackLog = new AsyncLocalStorage(); -const hasAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const hasAxiom = Boolean(config.services.axiomToken); const batchedAxiomDrain = hasAxiom ? createDrainPipeline({ batch: { size: 50, intervalMs: 5000 }, maxBufferSize: 2000, })( createAxiomDrain({ - apiKey: process.env.AXIOM_TOKEN, + apiKey: config.services.axiomToken, dataset: "slack", }) ) diff --git a/apps/uptime/src/lib/evlog-uptime.ts b/apps/uptime/src/lib/evlog-uptime.ts index 6482fda0fe..2bfd58ee69 100644 --- a/apps/uptime/src/lib/evlog-uptime.ts +++ b/apps/uptime/src/lib/evlog-uptime.ts @@ -1,6 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { readBooleanEnv } from "@databuddy/env/app"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { createBatchedAxiomDrain, enrichHttpWideEvent, @@ -10,7 +10,7 @@ import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog"; import type { DrainContext, EnrichContext } from "evlog"; import { createFsDrain } from "evlog/fs"; -const batchedAxiomDrain = createBatchedAxiomDrain(process.env.AXIOM_TOKEN); +const batchedAxiomDrain = createBatchedAxiomDrain(config.services.axiomToken); const batchedSuperlogDrain = createBatchedSuperlogDrain(); const devFsLogsDir = join( @@ -24,8 +24,7 @@ const devFsLogsDir = join( const useLocalEvlogFiles = process.env.NODE_ENV === "development" || readBooleanEnv("UPTIME_EVLOG_FS"); -const drainToAxiom = - process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN); +const drainToAxiom = Boolean(config.services.axiomToken); const devFsDrain = useLocalEvlogFiles ? createFsDrain({ dir: devFsLogsDir, pretty: false }) diff --git a/apps/uptime/src/uptime-transition-alerts.ts b/apps/uptime/src/uptime-transition-alerts.ts index d4421752a0..dd47bf6418 100644 --- a/apps/uptime/src/uptime-transition-alerts.ts +++ b/apps/uptime/src/uptime-transition-alerts.ts @@ -7,7 +7,7 @@ import { } from "@databuddy/db"; import { chQuery } from "@databuddy/db/clickhouse"; import { uptimeSchedules } from "@databuddy/db/schema"; -import { config } from "@databuddy/env/app"; +import { config, dataUrl } from "@databuddy/env/app"; import { NotificationClient, buildAlarmNotificationTargets, @@ -395,7 +395,7 @@ export async function writeMonitorState( } async function queryPreviousState(siteId: string): Promise { - if (!process.env.CLICKHOUSE_URL) { + if (!dataUrl("CLICKHOUSE_URL")) { return { kind: "missing" }; } try { diff --git a/bun.lock b/bun.lock index d6b8791a0b..af96f68afa 100644 --- a/bun.lock +++ b/bun.lock @@ -673,6 +673,7 @@ "name": "@databuddy/shared", "version": "0.0.1", "dependencies": { + "@databuddy/env": "workspace:*", "evlog": "catalog:", "ua-parser-js": "catalog:", "undici": "^8.2.0", diff --git a/package.json b/package.json index a3e2328c32..a761b7ca58 100644 --- a/package.json +++ b/package.json @@ -45,11 +45,11 @@ "format": "bunx ultracite fix", "check-types": "dotenv -- turbo run check-types", "generate-db": "turbo run generate --filter=@databuddy/db", - "db:studio": "dotenv -- turbo run db:studio", + "db:studio": "dotenv -- sh -c 'NODE_ENV=${NODE_ENV:-development} turbo run db:studio \"$@\"' --", "clickhouse:init": "dotenv -- turbo run clickhouse:init", "clickhouse:dql:init": "dotenv -- bun packages/db/src/clickhouse/dql.ts", "ch:verify": "dotenv -- bun packages/db/src/clickhouse/verify.ts", - "db:push": "dotenv -- turbo run db:push", + "db:push": "dotenv -- sh -c 'NODE_ENV=${NODE_ENV:-development} turbo run db:push \"$@\"' --", "db:migrate": "dotenv -- turbo run db:migrate", "db:deploy": "dotenv -- turbo run db:deploy", "db:seed": "dotenv -- sh -c 'cd packages/db && bun run db:seed \"$@\"' --", diff --git a/packages/ai/src/lib/databuddy.ts b/packages/ai/src/lib/databuddy.ts index 30af4270fa..3cf206cae7 100644 --- a/packages/ai/src/lib/databuddy.ts +++ b/packages/ai/src/lib/databuddy.ts @@ -1,7 +1,7 @@ -import { readBooleanEnv } from "@databuddy/env/boolean"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import { Databuddy } from "@databuddy/sdk/node"; -const apiKey = process.env.DATABUDDY_API_KEY; +const apiKey = config.services.databuddyApiKey; const websiteId = process.env.DATABUDDY_WEBSITE_ID; function createClient(source: string, namespace?: string) { diff --git a/packages/ai/src/lib/supermemory.ts b/packages/ai/src/lib/supermemory.ts index f565e11e99..9e5f75c2c1 100644 --- a/packages/ai/src/lib/supermemory.ts +++ b/packages/ai/src/lib/supermemory.ts @@ -1,8 +1,8 @@ +import { config } from "@databuddy/env/app"; import { getMemoryClient } from "@databuddy/services/business-memory"; export { getMemoryClient } from "@databuddy/services/business-memory"; import { stripHtmlTags } from "./sanitize"; -const apiKey = process.env.SUPERMEMORY_API_KEY; const MAX_MEMORY_LENGTH = 2000; const MEMORY_REQUEST_CLAUSE = /[^.!?;:,\n]+\??/g; const MEMORY_REQUEST_FILLER = @@ -46,7 +46,7 @@ const FORGET_ACKNOWLEDGEMENT = /^(?:(?:thanks|thank you|ok|okay|got it)[.!]\s*)+/i; export function isMemoryEnabled(): boolean { - return Boolean(apiKey); + return Boolean(config.services.supermemoryApiKey); } export function sanitizeMemoryContent( diff --git a/packages/auth/drizzle.config.ts b/packages/auth/drizzle.config.ts index e61d204043..5bb06bfd23 100644 --- a/packages/auth/drizzle.config.ts +++ b/packages/auth/drizzle.config.ts @@ -1,3 +1,4 @@ +import { dataUrl } from "@databuddy/env/app"; import { defineConfig } from "drizzle-kit"; export default defineConfig({ @@ -5,6 +6,6 @@ export default defineConfig({ schema: "./src/drizzle/schema.ts", dialect: "postgresql", dbCredentials: { - url: process.env.DATABASE_URL || "", + url: dataUrl("DATABASE_URL") || "", }, }); diff --git a/packages/db/drizzle.config.ts b/packages/db/drizzle.config.ts index dcb1caa703..b688a7798e 100644 --- a/packages/db/drizzle.config.ts +++ b/packages/db/drizzle.config.ts @@ -1,3 +1,4 @@ +import { dataUrl } from "@databuddy/env/app"; import { defineConfig } from "drizzle-kit"; export default defineConfig({ @@ -21,7 +22,7 @@ export default defineConfig({ ], dialect: "postgresql", dbCredentials: { - url: process.env.DATABASE_URL || "", + url: dataUrl("DATABASE_URL") || "", }, tablesFilter: ["!pg_stat_statements", "!pg_stat_statements_info"], }); diff --git a/packages/db/src/clickhouse/client.test.ts b/packages/db/src/clickhouse/client.test.ts index b2d331c55f..40675b5c41 100644 --- a/packages/db/src/clickhouse/client.test.ts +++ b/packages/db/src/clickhouse/client.test.ts @@ -177,27 +177,50 @@ describe("integration test loopback guard", () => { }); } - test("refuses a remote ClickHouse when integration tests are enabled", () => { + test.each([ + "clickhouse.example.test", + "staging", + "clickhouse", + "0.0.0.0", + ])("refuses non-loopback ClickHouse when integration tests are enabled: %s", (host) => { const result = importClient({ CLICKHOUSE_INTEGRATION_TESTS: "true", - CLICKHOUSE_URL: "http://default:@clickhouse.example.test:8123", + CLICKHOUSE_URL: `http://default:@${host}:8123`, }); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain( - 'only run against a loopback server; CLICKHOUSE_URL host is "clickhouse.example.test"' + `only run against a loopback server; CLICKHOUSE_URL host is "${host}"` ); }); - test("refuses an unset ClickHouse URL when integration tests are enabled", () => { - const result = importClient({ CLICKHOUSE_INTEGRATION_TESTS: "true" }); + test.each([ + "production", + "test", + ])("refuses an unset integration URL outside development: %s", (NODE_ENV) => { + const result = importClient({ + NODE_ENV, + CLICKHOUSE_INTEGRATION_TESTS: "true", + }); expect(result.exitCode).not.toBe(0); expect(result.stderr.toString()).toContain('host is "unset"'); }); - test("allows loopback when integration tests are enabled", () => { + test("allows the resolved development URL when CLICKHOUSE_URL is unset", () => { + const result = importClient({ + NODE_ENV: "development", + CLICKHOUSE_INTEGRATION_TESTS: "true", + }); + expect(result.exitCode, result.stderr.toString()).toBe(0); + }); + + test.each([ + "localhost", + "127.0.0.1", + "[::1]", + ])("allows loopback when integration tests are enabled: %s", (host) => { const result = importClient({ CLICKHOUSE_INTEGRATION_TESTS: "true", - CLICKHOUSE_URL: "http://default:@127.0.0.1:8123/databuddy_analytics", + CLICKHOUSE_URL: `http://default:@${host}:8123/databuddy_analytics`, }); expect(result.exitCode).toBe(0); }); diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts index 3bd83263cb..0da2b8e0e7 100644 --- a/packages/db/src/clickhouse/client.ts +++ b/packages/db/src/clickhouse/client.ts @@ -1,5 +1,6 @@ import { createClient, type ResponseJSON } from "@clickhouse/client"; import type { NodeClickHouseClientConfigOptions } from "@clickhouse/client/dist/config"; +import { dataUrl, isLoopbackHost } from "@databuddy/env/app"; import { finalizeDeliveryTables } from "./logical-reads"; export const TABLE_NAMES = { events: "analytics.events", @@ -82,18 +83,18 @@ function assertLoopbackForIntegrationTests(url: string | undefined): void { if (process.env.CLICKHOUSE_INTEGRATION_TESTS !== "true") { return; } - const hostname = url ? new URL(url).hostname : ""; - if (!["localhost", "127.0.0.1", "::1", "[::1]"].includes(hostname)) { + if (!(url && isLoopbackHost(url))) { throw new Error( - `ClickHouse integration tests only run against a loopback server; CLICKHOUSE_URL host is "${hostname || "unset"}"` + `ClickHouse integration tests only run against a loopback server; CLICKHOUSE_URL host is "${url ? new URL(url).hostname : "unset"}"` ); } } -assertLoopbackForIntegrationTests(process.env.CLICKHOUSE_URL); +const clickHouseUrl = dataUrl("CLICKHOUSE_URL"); +assertLoopbackForIntegrationTests(clickHouseUrl); const baseClient = createClient({ - url: process.env.CLICKHOUSE_URL, + url: clickHouseUrl, ...CLICKHOUSE_OPTIONS, }); diff --git a/packages/db/src/clickhouse/verify.ts b/packages/db/src/clickhouse/verify.ts index 13569de280..a31a036b9f 100644 --- a/packages/db/src/clickhouse/verify.ts +++ b/packages/db/src/clickhouse/verify.ts @@ -1,5 +1,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { dataUrl } from "@databuddy/env/app"; import { CLICKHOUSE_OPTIONS } from "./client"; import { type ParsedTable, @@ -27,7 +28,7 @@ function dbNameOf(sql: string): string { } async function fetchLive(): Promise> { - const raw = process.env.CLICKHOUSE_READONLY_URL ?? process.env.CLICKHOUSE_URL; + const raw = process.env.CLICKHOUSE_READONLY_URL ?? dataUrl("CLICKHOUSE_URL"); if (!raw) { throw new Error("CLICKHOUSE_READONLY_URL or CLICKHOUSE_URL must be set"); } diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index dae1a171bf..876d79a66a 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -1,5 +1,6 @@ /** biome-ignore-all lint/performance/noNamespaceImport: "Required" */ +import { dataUrl } from "@databuddy/env/app"; import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres"; import { Pool } from "pg"; import { relations } from "./drizzle/schema/relations"; @@ -92,7 +93,7 @@ let _pool: Pool | null = null; function getDb(): DB { if (!_db) { - const databaseUrl = process.env.DATABASE_URL; + const databaseUrl = dataUrl("DATABASE_URL"); if (!databaseUrl) { throw new Error("DATABASE_URL is not set"); } diff --git a/packages/db/src/db-command-env.test.ts b/packages/db/src/db-command-env.test.ts new file mode 100644 index 0000000000..cc8eb911f3 --- /dev/null +++ b/packages/db/src/db-command-env.test.ts @@ -0,0 +1,82 @@ +import { chmodSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "bun:test"; +import manifest from "../../../package.json"; + +const dotenv = Bun.resolveSync("dotenv-cli/cli.js", import.meta.dir); +const node = Bun.which("node"); +if (!node) { + throw new Error("Node is required to run the dotenv CLI"); +} + +it.each( + (["db:push", "db:studio"] as const).flatMap((command) => + ( + [ + ["", undefined, "development"], + ["NODE_ENV=\n", undefined, "development"], + ["NODE_ENV=production\n", undefined, "production"], + ["NODE_ENV=test\n", undefined, "test"], + ["NODE_ENV=development\n", "production", "production"], + ] as const + ).map(([file, inherited, expected]) => ({ + command, + file, + inherited, + expected, + })) + ) +)("runs $command with $expected mode after dotenv and preserves arguments", async ({ + command, + file, + inherited, + expected, +}) => { + const directory = mkdtempSync(join(tmpdir(), "databuddy-db-command-")); + try { + await Bun.write( + join(directory, "dotenv"), + '#!/bin/sh\nexec "$DB_COMMAND_NODE" "$DB_COMMAND_DOTENV" "$@"\n' + ); + await Bun.write( + join(directory, "turbo"), + '#!/bin/sh\nexec "$DB_COMMAND_BUN" --no-env-file "$DB_COMMAND_PROBE" "$@"\n' + ); + await Bun.write( + join(directory, "probe.ts"), + "console.log(JSON.stringify({ mode: process.env.NODE_ENV, args: process.argv.slice(2) }));\n" + ); + chmodSync(join(directory, "dotenv"), 0o755); + chmodSync(join(directory, "turbo"), 0o755); + + await Bun.write(join(directory, ".env"), file); + const child = Bun.spawn( + [ + "/bin/sh", + "-c", + `${manifest.scripts[command]} --dry-run "argument with spaces"`, + ], + { + cwd: directory, + env: { + PATH: `${directory}:/usr/bin:/bin`, + DB_COMMAND_BUN: process.execPath, + DB_COMMAND_NODE: node, + DB_COMMAND_DOTENV: dotenv, + DB_COMMAND_PROBE: join(directory, "probe.ts"), + ...(inherited === undefined ? {} : { NODE_ENV: inherited }), + }, + stdout: "pipe", + stderr: "pipe", + } + ); + expect(await child.exited).toBe(0); + expect(await new Response(child.stdout).json()).toEqual({ + mode: expected, + args: ["run", command, "--dry-run", "argument with spaces"], + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/packages/db/src/e2e-db-lifecycle.test.ts b/packages/db/src/e2e-db-lifecycle.test.ts index b4e7bcde74..3163955e83 100644 --- a/packages/db/src/e2e-db-lifecycle.test.ts +++ b/packages/db/src/e2e-db-lifecycle.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "bun:test"; import { deriveAdminDatabaseUrl, deriveDatabaseUrl, - isLocalDbHostname, parseLifecycleArgs, resolveE2EDatabaseName, resolveLifecycleConfig, @@ -68,18 +67,92 @@ describe("e2e db lifecycle helpers", () => { ); }); - it("refuses non-local database hosts by default", () => { - expect(isLocalDbHostname("localhost")).toBe(true); + it.each([ + "postgres://u:p@db:5432/databuddy", + "postgres://u:p@staging:5432/databuddy", + "postgres://u:p@0.0.0.0:5432/databuddy", + "postgres://u:p@db.example.com:5432/databuddy", + "postgres://[2001:db8::1]:5432/databuddy", + "postgres://[::ffff:8.8.8.8]:5432/databuddy", + ])("refuses non-local database hosts by default: %s", (baseDsn) => { expect(() => resolveLifecycleConfig({ allowNonLocal: false, - baseDsn: "postgres://u:p@db.example.com:5432/databuddy", + baseDsn, command: "create", dbPrefix: "databuddy_e2e", }) ).toThrow("Refusing to manage E2E DB on non-local host"); }); + it("refuses to drop a database on a single-label host without the override", () => { + const args = parseLifecycleArgs([ + "drop", + "--base-dsn", + "postgres://u:p@staging:5432/databuddy", + "--db-name", + "databuddy_e2e_run", + ]); + expect(() => resolveLifecycleConfig(args)).toThrow( + "Refusing to manage E2E DB on non-local host" + ); + expect( + resolveLifecycleConfig({ ...args, allowNonLocal: true }).adminDsn + ).toBe("postgres://u:p@staging:5432/postgres"); + }); + + it.each([ + "host=remote.example", + "%68ost=remote.example", + "host=localhost&host=remote.example", + "host=%2Ftmp%2Fpostgres", + ])("refuses PostgreSQL host overrides before create or drop: %s", (query) => { + for (const command of ["create", "drop"] as const) { + expect(() => + resolveLifecycleConfig({ + allowNonLocal: false, + baseDsn: `postgres://u:p@localhost:5432/databuddy?${query}`, + command, + dbName: "databuddy_e2e_run", + dbPrefix: "databuddy_e2e", + }) + ).toThrow("Refusing to manage E2E DB with a PostgreSQL host override"); + } + }); + + it("preserves an explicitly allowed PostgreSQL host override", () => { + const config = resolveLifecycleConfig({ + allowNonLocal: true, + baseDsn: "postgres://u:p@localhost:5432/databuddy?host=remote.example", + command: "create", + dbName: "databuddy_e2e_run", + dbPrefix: "databuddy_e2e", + }); + expect(config.adminDsn).toBe( + "postgres://u:p@localhost:5432/postgres?host=remote.example" + ); + expect(config.dbDsn).toBe( + "postgres://u:p@localhost:5432/databuddy_e2e_run?host=remote.example" + ); + }); + + it.each([ + "localhost", + "127.0.0.1", + "[::1]", + "[0:0:0:0:0:0:0:1]", + ])("allows an E2E database on loopback: %s", (host) => { + expect( + resolveLifecycleConfig({ + allowNonLocal: false, + baseDsn: `postgres://u:p@${host}:5432/databuddy`, + command: "create", + dbPrefix: "databuddy_e2e", + runId: "run", + }).dbName + ).toBe("databuddy_e2e_run"); + }); + it("prints shell-safe assignments", () => { expect( toShellAssignments({ DATABASE_URL: "postgres://u:p@localhost/db'quoted" }) diff --git a/packages/db/src/e2e-db-lifecycle.ts b/packages/db/src/e2e-db-lifecycle.ts index 3982e6aed1..aa4a5abfb8 100644 --- a/packages/db/src/e2e-db-lifecycle.ts +++ b/packages/db/src/e2e-db-lifecycle.ts @@ -1,11 +1,6 @@ +import { isLoopbackHost } from "@databuddy/env/app"; import { Client } from "pg"; -const LOCAL_DATABASE_HOSTS = new Set([ - "localhost", - "127.0.0.1", - "::1", - "[::1]", -]); const DEFAULT_E2E_DB_PREFIX = "databuddy_e2e"; const INVALID_DB_IDENTIFIER_PARTS = /[^A-Za-z0-9_]+/g; const REPEATED_UNDERSCORES = /_+/g; @@ -131,10 +126,6 @@ export function normalizeDatabaseUrl(databaseDsn: string): URL { return parsed; } -export function isLocalDbHostname(hostname: string): boolean { - return LOCAL_DATABASE_HOSTS.has(hostname.toLowerCase()); -} - export function deriveAdminDatabaseUrl(baseUrl: URL): URL { const url = new URL(baseUrl.href); url.pathname = "/postgres"; @@ -151,7 +142,12 @@ export function resolveLifecycleConfig( input: ParsedLifecycleArgs ): ResolvedLifecycleConfig { const baseUrl = normalizeDatabaseUrl(input.baseDsn); - if (!(input.allowNonLocal || isLocalDbHostname(baseUrl.hostname))) { + if (!input.allowNonLocal && baseUrl.searchParams.has("host")) { + throw new Error( + "Refusing to manage E2E DB with a PostgreSQL host override. Set --allow-non-local to override." + ); + } + if (!(input.allowNonLocal || isLoopbackHost(baseUrl.href))) { throw new Error( `Refusing to manage E2E DB on non-local host '${baseUrl.hostname}'. Set --allow-non-local to override.` ); diff --git a/packages/db/src/seed.ts b/packages/db/src/seed.ts index 96bd36bfca..6f1557e7bd 100644 --- a/packages/db/src/seed.ts +++ b/packages/db/src/seed.ts @@ -1,12 +1,12 @@ +import { dataUrl, isLoopbackHost } from "@databuddy/env/app"; import { faker } from "@faker-js/faker"; import { clickHouse, TABLE_NAMES } from "./clickhouse/client"; import { db } from "./client"; -import { isLocalDbHostname } from "./e2e-db-lifecycle"; for (const name of ["CLICKHOUSE_URL", "DATABASE_URL"] as const) { - const url = process.env[name]; + const url = dataUrl(name); const hostname = url && URL.canParse(url) ? new URL(url).hostname : ""; - if (!isLocalDbHostname(hostname)) { + if (!(url && hostname && isLoopbackHost(url))) { throw new Error( `db:seed only runs against local databases; ${name} host is "${hostname || "unset"}"` ); diff --git a/packages/db/src/test-env.test.ts b/packages/db/src/test-env.test.ts new file mode 100644 index 0000000000..7a79c660ba --- /dev/null +++ b/packages/db/src/test-env.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, test } from "bun:test"; + +describe("test environment service URLs", () => { + function importTestEnv(overrides: Record) { + return Bun.spawnSync({ + cmd: [process.execPath, "--no-env-file", "-e", 'import "./test-env.ts"'], + cwd: import.meta.dir, + env: { + ...process.env, + CI: "true", + DATABASE_URL: "postgres://u:p@localhost:5432/databuddy_test", + REDIS_URL: "redis://localhost:6379/1", + BULLMQ_REDIS_URL: "redis://localhost:6379/1", + CLICKHOUSE_URL: "http://localhost:8123", + ...overrides, + }, + stdout: "ignore", + stderr: "pipe", + }); + } + + test.each([ + ["DATABASE_URL", "postgres://u:p@staging:5432/databuddy_test"], + ["REDIS_URL", "redis://redis:6379/1"], + ["BULLMQ_REDIS_URL", "redis://queue:6379/1"], + ["CLICKHOUSE_URL", "http://clickhouse:8123"], + ])("refuses CI-supplied non-loopback %s", (name, url) => { + const result = importTestEnv({ [name]: url }); + expect(result.exitCode).not.toBe(0); + expect(result.stderr.toString()).toContain( + `${name} must point at a local test service` + ); + }); + + test.each([ + "localhost", + "127.0.0.1", + "[::1]", + ])("allows CI-supplied loopback services: %s", (host) => { + const result = importTestEnv({ + DATABASE_URL: `postgres://u:p@${host}:5432/databuddy_test`, + REDIS_URL: `redis://${host}:6379/1`, + BULLMQ_REDIS_URL: `redis://${host}:6379/1`, + CLICKHOUSE_URL: `http://${host}:8123`, + }); + expect(result.exitCode, result.stderr.toString()).toBe(0); + }); +}); diff --git a/packages/db/src/test-env.ts b/packages/db/src/test-env.ts index ae28c0396e..da77fedfba 100644 --- a/packages/db/src/test-env.ts +++ b/packages/db/src/test-env.ts @@ -1,17 +1,14 @@ -import { readBooleanEnv } from "@databuddy/env/app"; +import { isLoopbackHost, readBooleanEnv } from "@databuddy/env/app"; const useCiUrls = readBooleanEnv("CI"); const defaultDatabaseUrl = "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy_test"; const defaultRedisUrl = "redis://localhost:6379/1"; -const loopbackHosts = new Set(["localhost", "127.0.0.1", "::1", "[::1]"]); - function localTestUrl(name: string, value: string): string { - const { hostname } = new URL(value); - if (!loopbackHosts.has(hostname)) { + if (!isLoopbackHost(value)) { throw new Error( - `${name} must point at a local test service. Refusing to run tests against ${hostname}.` + `${name} must point at a local test service. Refusing to run tests against ${new URL(value).hostname}.` ); } return value; @@ -35,4 +32,5 @@ process.env.CLICKHOUSE_URL = : "http://default:@localhost:8123"; process.env.BETTER_AUTH_SECRET ??= "test-auth-secret-for-integration"; process.env.BETTER_AUTH_URL ??= "http://localhost:3001"; +process.env.AUTUMN_SECRET_KEY ||= "test-autumn-secret-key"; process.env.NODE_ENV = "test"; diff --git a/packages/env/src/app.test.ts b/packages/env/src/app.test.ts index 0734dd9ccc..42ce4d34e9 100644 --- a/packages/env/src/app.test.ts +++ b/packages/env/src/app.test.ts @@ -1,5 +1,13 @@ import { describe, expect, it } from "bun:test"; -import { assertConfigured, createConfig, readBooleanEnv } from "./app"; +import { + assertConfigured, + billingMode, + createConfig, + dataUrl, + isLocalHost, + isLoopbackHost, + readBooleanEnv, +} from "./app"; const HOSTED = { AUTUMN_SECRET_KEY: "am_sk_test", @@ -210,6 +218,158 @@ describe("assertConfigured", () => { }); }); +describe("isLocalHost", () => { + it("accepts loopback hosts and compose service names", () => { + for (const url of [ + "postgres://u:p@localhost:5432/databuddy", + "http://default:@127.0.0.1:8123", + "http://0.0.0.0:8123", + "redis://[::1]:6379", + "redis://[0:0:0:0:0:0:0:1]:6379", + "[::1]:6379", + "redis://redis:6379", + "redpanda:9092", + ]) { + expect(isLocalHost(url)).toBe(true); + } + }); + + it("rejects dotted remote hosts and non-loopback IPv6 literals", () => { + for (const url of [ + "postgres://u:p@db.example.com:5432/x", + "broker.example.com:9092", + "postgres://[2001:db8::1]:5432/x", + "http://[::ffff:8.8.8.8]:8123", + "[2001:db8::1]:9092", + ]) { + expect(isLocalHost(url)).toBe(false); + } + }); + + it("rejects malformed service URLs without exposing credentials", () => { + const invalid = "postgres://synthetic-user:synthetic-private@[broken"; + for (const checkHost of [isLocalHost, isLoopbackHost]) { + let failure: Error | undefined; + try { + checkHost(invalid); + } catch (error) { + if (!(error instanceof Error)) { + throw error; + } + failure = error; + } + expect(failure).toBeInstanceOf(Error); + expect(String(failure)).toContain("configured service URL"); + expect(JSON.stringify(failure)).not.toContain("synthetic-private"); + expect(Bun.inspect(failure)).not.toContain("synthetic-private"); + } + }); + + it("rejects unbracketed IPv6", () => { + // Unbracketed IPv6 is invalid URL input and must also fail closed. + expect(() => isLocalHost("2001:db8::1")).toThrow(); + }); +}); + +describe("isLoopbackHost", () => { + it.each([ + "postgres://u:p@localhost:5432/databuddy", + "http://127.0.0.1:8123", + "redis://[::1]:6379", + "redis://[0:0:0:0:0:0:0:1]:6379", + ])("accepts concrete loopback URLs: %s", (url) => { + expect(isLoopbackHost(url)).toBe(true); + }); + + it.each([ + "postgres://u:p@db:5432/databuddy", + "postgres://u:p@staging:5432/databuddy", + "redis://redis:6379", + "http://0.0.0.0:8123", + "http://clickhouse.example.com:8123", + "http://[2001:db8::1]:8123", + ])("refuses non-loopback URLs at test/destructive boundaries: %s", (url) => { + expect(isLoopbackHost(url)).toBe(false); + }); +}); + +describe("dataUrl", () => { + const REMOTE = "postgres://u:p@db.example.com:5432/databuddy"; + + it("defaults unset URLs to the local Docker services in development", () => { + const env = { NODE_ENV: "development" }; + expect(dataUrl("DATABASE_URL", env)).toBe( + "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy" + ); + expect(dataUrl("BULLMQ_REDIS_URL", env)).toBe("redis://localhost:6379"); + expect(dataUrl("CLICKHOUSE_URL", env)).toBe( + "http://default:@localhost:8123/databuddy_analytics" + ); + }); + + it("uses an explicit URL as is in every environment", () => { + for (const NODE_ENV of ["development", "production", "test"]) { + expect(dataUrl("DATABASE_URL", { DATABASE_URL: REMOTE, NODE_ENV })).toBe( + REMOTE + ); + } + }); + + it("leaves unset URLs unset outside development", () => { + for (const NODE_ENV of ["production", "test", undefined]) { + expect(dataUrl("REDIS_URL", { NODE_ENV })).toBeUndefined(); + } + }); +}); + +describe("services", () => { + it("reads trimmed keys live and treats blanks as unset", () => { + const env: Record = { + NODE_ENV: "development", + RESEND_API_KEY: " ", + SLACK_WEBHOOK_URL: " https://hooks.slack.com/services/x ", + }; + const config = createConfig(env); + expect(config.services.slackWebhookUrl).toBe( + "https://hooks.slack.com/services/x" + ); + expect(config.services.resendApiKey).toBeUndefined(); + env.RESEND_API_KEY = "re_live"; + expect(config.services.resendApiKey).toBe("re_live"); + }); + + it("keeps development logs out of Axiom even with a token set", () => { + const env: Record = { + NODE_ENV: "development", + AXIOM_TOKEN: "xaat-dev", + }; + const config = createConfig(env); + expect(config.services.axiomToken).toBeUndefined(); + env.NODE_ENV = "production"; + expect(config.services.axiomToken).toBe("xaat-dev"); + }); +}); + +describe("billingMode", () => { + it("keeps self-hosting unmetered even with a copied billing key", () => { + expect(billingMode({ ...HOSTED, SELFHOST: "true" })).toBe("selfhost"); + }); + + it("bills hosted production even when the key is missing", () => { + expect(billingMode(HOSTED)).toBe("live"); + expect(billingMode({ NODE_ENV: "production" })).toBe("live"); + }); + + it("bills elsewhere only when a billing key is set", () => { + for (const NODE_ENV of ["development", "test"]) { + expect(billingMode({ NODE_ENV })).toBe("disabled"); + expect(billingMode({ NODE_ENV, AUTUMN_SECRET_KEY: "am_sk" })).toBe( + "live" + ); + } + }); +}); + describe("readBooleanEnv", () => { it("only enables an explicit true value", () => { for (const value of [undefined, "", "false", "0", "1", "yes"]) { diff --git a/packages/env/src/app.ts b/packages/env/src/app.ts index 92b14e912f..27508a129f 100644 --- a/packages/env/src/app.ts +++ b/packages/env/src/app.ts @@ -1,6 +1,11 @@ -import { readBooleanEnv } from "./boolean"; +import { LOOPBACK_HOSTS, readBooleanEnv } from "./boolean"; -export { readBooleanEnv } from "./boolean"; +export { + dataUrl, + isLocalHost, + isLoopbackHost, + readBooleanEnv, +} from "./boolean"; // App-wide runtime config. // @@ -72,11 +77,11 @@ export interface Config { email: { alertsFrom: string; from: string; - resendApiKey?: string; }; integrations: { openAiAdsPixelId?: string; }; + services: ReturnType; storage?: StorageConfig; urls: { api: string; @@ -91,12 +96,22 @@ export interface Config { const REQUIRED_IN_PRODUCTION = ["BETTER_AUTH_SECRET"] as const; const REQUIRED_IN_HOSTED_CLOUD = ["AUTUMN_SECRET_KEY"] as const; -const LOOPBACK_HOSTS = new Set(["0.0.0.0", "127.0.0.1", "[::1]", "localhost"]); function isHostedCloud(env: Env): boolean { return env.NODE_ENV === "production" && !readBooleanEnv("SELFHOST", env); } +export function billingMode( + env: Env = process.env +): "selfhost" | "live" | "disabled" { + if (readBooleanEnv("SELFHOST", env)) { + return "selfhost"; + } + return isHostedCloud(env) || readOptional(env, "AUTUMN_SECRET_KEY") + ? "live" + : "disabled"; +} + function defaultUrl(env: Env, setting: UrlConfig): string { return isHostedCloud(env) ? setting.cloud : setting.local; } @@ -144,6 +159,23 @@ function readOrigins(values: Array): string[] { return [...new Set(values.flatMap(readList).map(normalizeOrigin))]; } +function readServices(env: Env) { + return { + autumnSecretKey: readOptional(env, "AUTUMN_SECRET_KEY"), + axiomToken: + env.NODE_ENV === "development" + ? undefined + : readOptional(env, "AXIOM_TOKEN"), + databuddyApiKey: readOptional(env, "DATABUDDY_API_KEY"), + dubApiKey: readOptional(env, "DUB_API_KEY"), + resendApiKey: readOptional(env, "RESEND_API_KEY"), + slackWebhookUrl: readOptional(env, "SLACK_WEBHOOK_URL"), + superlogApiKey: readOptional(env, "SUPERLOG_API_KEY"), + supermemoryApiKey: readOptional(env, "SUPERMEMORY_API_KEY"), + tccApiKey: readOptional(env, "TCC_API_KEY"), + }; +} + function readStorage(env: Env): StorageConfig | undefined { const accessKeyId = readOptional(env, "AWS_ACCESS_KEY_ID"); const secretAccessKey = readOptional(env, "AWS_SECRET_ACCESS_KEY"); @@ -168,7 +200,8 @@ function readStorage(env: Env): StorageConfig | undefined { }; } -export function createConfig(env: Env = process.env): Config { +export function createConfig(source?: Env): Config { + const env = source ?? process.env; const dashboardUrl = readUrl(env, URLS.dashboard); const apiUrl = readUrl(env, URLS.api); @@ -183,11 +216,13 @@ export function createConfig(env: Env = process.env): Config { email: { alertsFrom: readEmail(env, EMAIL.alertsFrom), from: readEmail(env, EMAIL.from), - resendApiKey: readOptional(env, "RESEND_API_KEY"), }, integrations: { openAiAdsPixelId: readOptional(env, "NEXT_PUBLIC_OPENAI_ADS_PIXEL_ID"), }, + get services() { + return readServices(source ?? process.env); + }, storage: readStorage(env), urls: { api: apiUrl, diff --git a/packages/env/src/boolean.ts b/packages/env/src/boolean.ts index 8fa1c655be..f607139674 100644 --- a/packages/env/src/boolean.ts +++ b/packages/env/src/boolean.ts @@ -1,6 +1,56 @@ +type Env = Record; + +export const LOOPBACK_HOSTS = new Set([ + "0.0.0.0", + "127.0.0.1", + "[::1]", + "localhost", +]); + +const LOCAL_DATA_URLS = { + BULLMQ_REDIS_URL: "redis://localhost:6379", + CLICKHOUSE_URL: "http://default:@localhost:8123/databuddy_analytics", + DATABASE_URL: + "postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy", + REDIS_URL: "redis://localhost:6379", +} as const; + export function readBooleanEnv( name: string, - environment: Record = process.env + environment: Env = process.env ): boolean { return environment[name]?.trim().toLowerCase() === "true"; } + +function hostnameOf(url: string): string { + try { + return new URL( + url.includes("://") ? url : `tcp://${url}` + ).hostname.toLowerCase(); + } catch { + throw new Error("Cannot parse host from the configured service URL"); + } +} + +export function isLocalHost(url: string): boolean { + const hostname = hostnameOf(url); + return ( + LOOPBACK_HOSTS.has(hostname) || + (hostname !== "" && !hostname.includes(".") && !hostname.includes(":")) + ); +} + +export function isLoopbackHost(url: string): boolean { + const hostname = hostnameOf(url); + return hostname !== "0.0.0.0" && LOOPBACK_HOSTS.has(hostname); +} + +export function dataUrl( + key: keyof typeof LOCAL_DATA_URLS, + env: Env = process.env +): string | undefined { + return ( + env[key] || + (env.NODE_ENV === "development" ? LOCAL_DATA_URLS[key] : undefined) + ); +} diff --git a/packages/notifications/src/alarm-config.ts b/packages/notifications/src/alarm-config.ts index 4d794e502e..6478ace983 100644 --- a/packages/notifications/src/alarm-config.ts +++ b/packages/notifications/src/alarm-config.ts @@ -1,4 +1,4 @@ -import { readBooleanEnv } from "@databuddy/env/boolean"; +import { config, readBooleanEnv } from "@databuddy/env/app"; import type { NotificationClientConfig } from "./client"; import type { NotificationChannel } from "./types"; @@ -122,7 +122,7 @@ export function buildAlarmNotificationTargets( }, }); } else if (dest.type === "email") { - if (!process.env.RESEND_API_KEY) { + if (!config.services.resendApiKey) { warnAlarmEmailUnconfigured(); continue; } @@ -139,7 +139,7 @@ export function buildAlarmNotificationTargets( text?: string; }) => { const { Resend } = await import("resend"); - const apiKey = process.env.RESEND_API_KEY; + const apiKey = config.services.resendApiKey; if (!apiKey) { throw new Error("Email delivery is not configured"); } diff --git a/packages/redis/bullmq.test.ts b/packages/redis/bullmq.test.ts index 5f2eae40e0..13371af98f 100644 --- a/packages/redis/bullmq.test.ts +++ b/packages/redis/bullmq.test.ts @@ -9,6 +9,7 @@ const originalEnv = process.env; beforeEach(() => { process.env = { ...originalEnv, + NODE_ENV: "production", SELFHOST: "false", REDIS_URL: "rediss://shared:secret@shared.test:6380/2", BULLMQ_REDIS_URL: "", @@ -57,7 +58,7 @@ assert.equal(getBullMQConnectionOptions().host, "queue.example.com"); }); it("requires BULLMQ_REDIS_URL", () => { - delete process.env.BULLMQ_REDIS_URL; + Reflect.deleteProperty(process.env, "BULLMQ_REDIS_URL"); expect(() => getBullMQConnectionOptions()).toThrow( "BULLMQ_REDIS_URL environment variable is required" @@ -95,6 +96,40 @@ assert.equal(getBullMQConnectionOptions().host, "queue.example.com"); expect(() => getBullMQConnectionOptions()).toThrow(); }); + it("prefers the self-hosted shared URL over development defaults", () => { + process.env.NODE_ENV = "development"; + process.env.SELFHOST = "true"; + Reflect.deleteProperty(process.env, "BULLMQ_REDIS_URL"); + + for (const resolve of [ + getBullMQConnectionOptions, + getBullMQWorkerConnectionOptions, + ]) { + expect(resolve({ envPrefix: "INSIGHTS" })).toMatchObject({ + host: "shared.test", + port: 6380, + db: 2, + tls: {}, + }); + process.env.BULLMQ_REDIS_URL = "redis://dedicated.test:6379/3"; + expect(resolve().host).toBe("dedicated.test"); + Reflect.deleteProperty(process.env, "BULLMQ_REDIS_URL"); + } + }); + + it.each([ + "false", + "true", + ])("keeps local development defaults when SELFHOST=%s and URLs are unset", (selfhost) => { + process.env.NODE_ENV = "development"; + process.env.SELFHOST = selfhost; + Reflect.deleteProperty(process.env, "BULLMQ_REDIS_URL"); + Reflect.deleteProperty(process.env, "REDIS_URL"); + + expect(getBullMQConnectionOptions().host).toBe("localhost"); + expect(getBullMQWorkerConnectionOptions().host).toBe("localhost"); + }); + it("parses redis URLs for queue producers", () => { process.env.BULLMQ_REDIS_URL = "redis://user:pass@example.test:6380/3"; diff --git a/packages/redis/bullmq.ts b/packages/redis/bullmq.ts index e1a500ce1c..8cc1ad00d2 100644 --- a/packages/redis/bullmq.ts +++ b/packages/redis/bullmq.ts @@ -1,4 +1,4 @@ -import { readBooleanEnv } from "@databuddy/env/boolean"; +import { dataUrl, readBooleanEnv } from "@databuddy/env/boolean"; import type { RedisOptions } from "ioredis"; export interface BullMQConnectionConfig { @@ -14,7 +14,8 @@ function resolveBullMQRedisUrl(config: BullMQConnectionConfig = {}): string { const redisUrl = prefixedUrl || fallbackUrl || - (readBooleanEnv("SELFHOST") ? process.env.REDIS_URL?.trim() : undefined); + (readBooleanEnv("SELFHOST") ? dataUrl("REDIS_URL")?.trim() : undefined) || + dataUrl("BULLMQ_REDIS_URL")?.trim(); if (!redisUrl) { throw new Error( `${prefixedName ? `${prefixedName} or ` : ""}BULLMQ_REDIS_URL environment variable is required` diff --git a/packages/redis/redis-options.ts b/packages/redis/redis-options.ts index 0f49234ab6..37c48c254b 100644 --- a/packages/redis/redis-options.ts +++ b/packages/redis/redis-options.ts @@ -1,3 +1,5 @@ +import { dataUrl } from "@databuddy/env/app"; + export interface RedisConnectionOptions { commandTimeout: number; connectTimeout: number; @@ -15,7 +17,7 @@ export interface LinkCacheRedisConnectionOptions export type RateLimitRedisConnectionOptions = LinkCacheRedisConnectionOptions; export function getRedisUrl(): string { - const url = process.env.REDIS_URL; + const url = dataUrl("REDIS_URL"); if (!url) { throw new Error("REDIS_URL environment variable is required"); } diff --git a/packages/services/src/business-memory.ts b/packages/services/src/business-memory.ts index 2eb9dbbe66..68336b1014 100644 --- a/packages/services/src/business-memory.ts +++ b/packages/services/src/business-memory.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import type { db } from "@databuddy/db"; +import { config } from "@databuddy/env/app"; import Supermemory from "supermemory"; import { z } from "zod"; @@ -13,7 +14,7 @@ export interface BusinessScope { let client: Supermemory | null = null; export function getMemoryClient(): Supermemory | null { - const apiKey = process.env.SUPERMEMORY_API_KEY; + const apiKey = config.services.supermemoryApiKey; if (!apiKey) { return null; } diff --git a/packages/services/src/feedback.ts b/packages/services/src/feedback.ts index ff067600db..deb99d1490 100644 --- a/packages/services/src/feedback.ts +++ b/packages/services/src/feedback.ts @@ -1,7 +1,8 @@ import { db, eq, type Feedback, feedback, user } from "@databuddy/db"; +import { config } from "@databuddy/env/app"; import { randomUUIDv7 } from "bun"; -const SLACK_WEBHOOK_URL = process.env.SLACK_WEBHOOK_URL ?? ""; +const SLACK_WEBHOOK_URL = config.services.slackWebhookUrl ?? ""; const SLACK_TIMEOUT_MS = 10_000; const DESCRIPTION_PREVIEW_LIMIT = 500; diff --git a/packages/services/src/website-cache.ts b/packages/services/src/website-cache.ts index 4c9d664fa3..05c99b0cb5 100644 --- a/packages/services/src/website-cache.ts +++ b/packages/services/src/website-cache.ts @@ -1,5 +1,6 @@ import { RedisClient } from "bun"; import type { Website } from "@databuddy/db/schema"; +import { dataUrl } from "@databuddy/env/app"; type WebsiteCacheRecord = Omit< Website, @@ -44,7 +45,7 @@ export class WebsiteCache { return this.redis; } - const url = process.env.REDIS_URL; + const url = dataUrl("REDIS_URL"); if (!url) { this.isHealthyCached = false; return null; diff --git a/packages/services/src/websites.ts b/packages/services/src/websites.ts index 577ec1c53e..a60480c70c 100644 --- a/packages/services/src/websites.ts +++ b/packages/services/src/websites.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { db, eq, isUniqueViolationFor } from "@databuddy/db"; +import { dataUrl } from "@databuddy/env/app"; import { type WebsiteInsert, type Website, @@ -88,7 +89,7 @@ export class WebsiteService { } private async invalidateReadCaches(id: string): Promise { - if (!process.env.REDIS_URL) { + if (!dataUrl("REDIS_URL")) { return; } diff --git a/packages/shared/package.json b/packages/shared/package.json index dd3d39d3c0..2c7ada3b20 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -46,6 +46,7 @@ "bots:sync": "curl -fsSL https://raw.githubusercontent.com/arcjet/well-known-bots/main/well-known-bots.json -o src/utils/bot-detection/well-known-bots.json && bun test src/utils/bot-detection" }, "dependencies": { + "@databuddy/env": "workspace:*", "evlog": "catalog:", "ua-parser-js": "catalog:", "undici": "^8.2.0", diff --git a/packages/shared/src/evlog-superlog.ts b/packages/shared/src/evlog-superlog.ts index 3cb13b18c0..09d10853dd 100644 --- a/packages/shared/src/evlog-superlog.ts +++ b/packages/shared/src/evlog-superlog.ts @@ -1,9 +1,10 @@ +import { config } from "@databuddy/env/app"; import type { DrainContext } from "evlog"; import { createOTLPDrain } from "evlog/otlp"; import { createDrainPipeline } from "evlog/pipeline"; export function createBatchedSuperlogDrain() { - const apiKey = process.env.SUPERLOG_API_KEY; + const apiKey = config.services.superlogApiKey; if (!apiKey) { return null; }