refactor(env): unify data URLs and outbound service keys

.env.example

@@ -1,12 +1,16 @@
 # Local development. For Docker self-hosting, use selfhost.env.example.
 
-CLICKHOUSE_URL="http://default:@localhost:8123/databuddy_analytics"
+# DATABASE_URL, REDIS_URL, BULLMQ_REDIS_URL and CLICKHOUSE_URL default to the
+# docker-compose services in development. Set them only to point elsewhere.
+
 # Required by DQL. This must use the separately provisioned dql_user account;
 # never point it at the application/admin ClickHouse credentials.
 CLICKHOUSE_DQL_URL="http://dql_user:databuddy_dql_dev_password@localhost:8123/databuddy_analytics"
-DATABASE_URL="postgres://databuddy:databuddy_dev_password@localhost:5432/databuddy"
-REDIS_URL="redis://localhost:6379"
-BULLMQ_REDIS_URL="redis://localhost:6379"
+
+# Redpanda (Kafka) ingestion. Blank writes straight to ClickHouse.
+REDPANDA_BROKER=""
+REDPANDA_USER=""
+REDPANDA_PASSWORD=""
 
 # Public URLs. Local development uses built-in localhost defaults when blank.
 DASHBOARD_URL=""
@@ -65,6 +69,8 @@ SLACK_SIGNING_SECRET=""
 SLACK_CLIENT_ID=""
 SLACK_CLIENT_SECRET=""
 SLACK_REDIRECT_URI=""
+# Internal signup, billing, and feedback notifications
+SLACK_WEBHOOK_URL=""
 
 # Not Necessary unless using blog
 MARBLE_API_KEY=
@@ -77,6 +83,11 @@ NOTRA_API_KEY=
 # Superlog OTLP logs export. When set, evlog drains in api/basket/insights/slack/uptime
 # also ship wide events to https://intake.superlog.sh/v1/logs.
 SUPERLOG_API_KEY=""
+AXIOM_TOKEN=""
+TCC_API_KEY=""
+
+# Databuddy self-analytics for product events
+DATABUDDY_API_KEY=""
 
 # S3-compatible object storage for status page logo and favicon uploads.
 # Both credentials are required together; the API refuses to boot with only one.

apps/api/src/lib/evlog-api.ts

@@ -1,6 +1,6 @@
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { readBooleanEnv } from "@databuddy/env/app";
+import { config, readBooleanEnv } from "@databuddy/env/app";
 import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog";
 import type { DrainContext, EnrichContext } from "evlog";
 import { createAxiomDrain } from "evlog/axiom";
@@ -15,7 +15,7 @@ import { createDrainPipeline } from "evlog/pipeline";
 const batchedAxiomDrain = createDrainPipeline<DrainContext>({
 	batch: { size: 50, intervalMs: 5000 },
 	maxBufferSize: 2000,
-})(createAxiomDrain({ apiKey: process.env.AXIOM_TOKEN }));
+})(createAxiomDrain({ apiKey: config.services.axiomToken }));
 
 const batchedSuperlogDrain = createBatchedSuperlogDrain();
 
@@ -30,8 +30,7 @@ const devFsLogsDir = join(
 const useLocalEvlogFiles =
 	process.env.NODE_ENV === "development" || readBooleanEnv("API_EVLOG_FS");
 
-const drainToAxiom =
-	process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN);
+const drainToAxiom = Boolean(config.services.axiomToken);
 
 const devFsDrain = useLocalEvlogFiles
 	? createFsDrain({ dir: devFsLogsDir, pretty: false })

apps/api/src/lib/tcc-otel.ts

@@ -1,4 +1,5 @@
 import { TCCSpanProcessor } from "@contextcompany/otel";
+import { config } from "@databuddy/env/app";
 import { resourceFromAttributes } from "@opentelemetry/resources";
 import { NodeSDK } from "@opentelemetry/sdk-node";
 import {
@@ -9,7 +10,7 @@ import pkg from "../../package.json";
 
 let sdk: NodeSDK | null = null;
 export function initTccTracing(): void {
-	if (sdk || !process.env.TCC_API_KEY) {
+	if (sdk || !config.services.tccApiKey) {
 		return;
 	}
 

apps/basket/src/lib/blocked-traffic-alerts.ts

@@ -5,7 +5,7 @@ import {
 	normalizeEmailNotificationSettings,
 	type EmailNotificationSettings,
 } from "@databuddy/db";
-import { config } from "@databuddy/env/app";
+import { config, dataUrl } from "@databuddy/env/app";
 import { BlockedTrafficAlertEmail, render } from "@databuddy/email";
 import { redis } from "@databuddy/redis";
 import {
@@ -282,7 +282,7 @@ async function sendAlertEmail(input: {
 	previousBlocked: number;
 	windowBlockedCount: number;
 }): Promise<void> {
-	const apiKey = requireBlockedTrafficEmailApiKey(config.email.resendApiKey);
+	const apiKey = requireBlockedTrafficEmailApiKey(config.services.resendApiKey);
 
 	const siteLabel =
 		input.context.websiteName ||
@@ -346,7 +346,7 @@ async function maybeSendBlockedTrafficAlertAsync(
 	if (!context.ownerId) {
 		return;
 	}
-	if (!process.env.REDIS_URL) {
+	if (!dataUrl("REDIS_URL")) {
 		return;
 	}
 
@@ -379,7 +379,7 @@ async function maybeSendBlockedTrafficAlertAsync(
 		return;
 	}
 
-	if (!config.email.resendApiKey) {
+	if (!config.services.resendApiKey) {
 		warnBlockedTrafficEmailUnconfigured();
 		return;
 	}

apps/basket/src/lib/evlog-basket.ts

@@ -1,6 +1,6 @@
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { readBooleanEnv } from "@databuddy/env/app";
+import { config, readBooleanEnv } from "@databuddy/env/app";
 import {
 	createBatchedAxiomDrain,
 	downgradeClientHttpError,
@@ -12,7 +12,7 @@ import { CLIENT_ERROR_MESSAGES } from "@lib/structured-errors";
 import type { DrainContext, EnrichContext } from "evlog";
 import { createFsDrain } from "evlog/fs";
 
-const batchedAxiomDrain = createBatchedAxiomDrain(process.env.AXIOM_TOKEN);
+const batchedAxiomDrain = createBatchedAxiomDrain(config.services.axiomToken);
 
 const batchedSuperlogDrain = createBatchedSuperlogDrain();
 
@@ -27,8 +27,7 @@ const devFsLogsDir = join(
 const useLocalEvlogFiles =
 	process.env.NODE_ENV === "development" || readBooleanEnv("BASKET_EVLOG_FS");
 
-const drainToAxiom =
-	process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN);
+const drainToAxiom = Boolean(config.services.axiomToken);
 
 const devFsDrain = useLocalEvlogFiles
 	? createFsDrain({ dir: devFsLogsDir, pretty: false })

apps/dashboard/app/(auth)/layout.tsx

@@ -1,4 +1,4 @@
-import { readBooleanEnv } from "@databuddy/env/boolean";
+import { config, readBooleanEnv } from "@databuddy/env/app";
 import { connection } from "next/server";
 import type { ReactNode } from "react";
 import AuthLayout from "./auth-layout";
@@ -12,7 +12,7 @@ export default async function Layout({ children }: { children: ReactNode }) {
 		<AuthLayout
 			capabilities={{
 				email: Boolean(
-					process.env.RESEND_API_KEY?.trim() && process.env.EMAIL_FROM?.trim()
+					config.services.resendApiKey && process.env.EMAIL_FROM?.trim()
 				),
 				github: Boolean(
 					process.env.GITHUB_CLIENT_ID && process.env.GITHUB_CLIENT_SECRET

apps/dashboard/app/(main)/billing/actions/cancel-feedback-action.ts

@@ -1,11 +1,12 @@
 "use server";
 
 import { auth } from "@databuddy/auth";
+import { config } from "@databuddy/env/app";
 import { Databuddy } from "@databuddy/sdk/node";
 import { headers } from "next/headers";
 import type { CancelFeedback } from "../components/cancel-subscription-dialog";
 
-const databuddyApiKey = process.env.DATABUDDY_API_KEY;
+const databuddyApiKey = config.services.databuddyApiKey;
 const client = databuddyApiKey
 	? new Databuddy({
 			apiKey: databuddyApiKey,

apps/dashboard/instrumentation.ts

@@ -1,3 +1,4 @@
+import { config } from "@databuddy/env/app";
 import type { DrainContext } from "evlog";
 
 export async function register() {
@@ -10,18 +11,18 @@ export async function register() {
 		import("@databuddy/shared/evlog-redaction"),
 		loadAxiomDrain(),
 	]);
-	const config = {
+	const loggerConfig = {
 		env: redaction.createDatabuddyEvlogEnv("dashboard"),
 		redact: redaction.databuddyEvlogRedaction,
 		drain,
 	};
-	authLogger.initLogger(config);
-	auditLogger.initLogger(config);
+	authLogger.initLogger(loggerConfig);
+	auditLogger.initLogger(loggerConfig);
 }
 
 async function loadAxiomDrain() {
-	const apiKey = process.env.AXIOM_TOKEN;
-	if (!apiKey || process.env.NODE_ENV === "development") {
+	const apiKey = config.services.axiomToken;
+	if (!apiKey) {
 		return;
 	}
 	const [{ createAxiomDrain }, { waitUntil }, axiom] = await Promise.all([

apps/insights/src/ai-digest.ts

@@ -171,7 +171,7 @@ async function sitesWhoseServerTrackingStopped(
 }
 
 export async function dispatchAiDigests(now = new Date()) {
-	if (!config.email.resendApiKey) {
+	if (!config.services.resendApiKey) {
 		return logOutcome({ reason: "email_not_configured", status: "skipped" });
 	}
 	const week = weekRange(previousWeekStart(now));
@@ -319,7 +319,7 @@ export async function sendAiDigest({
 	websiteId,
 	weekStart,
 }: AiDigestWebsiteJobData) {
-	const apiKey = config.email.resendApiKey;
+	const apiKey = config.services.resendApiKey;
 	if (!apiKey) {
 		return logOutcome({ reason: "email_not_configured", status: "skipped" });
 	}

apps/insights/src/lib/evlog-insights.ts

@@ -1,7 +1,7 @@
 import { AsyncLocalStorage } from "node:async_hooks";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { readBooleanEnv } from "@databuddy/env/app";
+import { config, readBooleanEnv } from "@databuddy/env/app";
 import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog";
 import type { DrainContext, RequestLogger } from "evlog";
 import { createLogger, log } from "evlog";
@@ -21,12 +21,11 @@ const pipeline = createDrainPipeline<DrainContext>({
 	maxBufferSize: 2000,
 });
 
-const hasAxiom =
-	process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN);
+const hasAxiom = Boolean(config.services.axiomToken);
 const batchedAxiomDrain = hasAxiom
 	? pipeline(
 			createAxiomDrain({
-				apiKey: process.env.AXIOM_TOKEN,
+				apiKey: config.services.axiomToken,
 				dataset: "insights",
 			})
 		)

apps/links/src/lib/logging.ts

@@ -1,6 +1,6 @@
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { readBooleanEnv } from "@databuddy/env/app";
+import { config, readBooleanEnv } from "@databuddy/env/app";
 import { createDatabuddyEvlogEnv } from "@databuddy/shared/evlog-redaction";
 import type { DrainContext, EnrichContext, WideEvent } from "evlog";
 import { log } from "evlog";
@@ -20,7 +20,7 @@ type LogFields = Record<string, LogField>;
 const batchedAxiomDrain = createDrainPipeline<DrainContext>({
 	batch: { size: 50, intervalMs: 5000 },
 	maxBufferSize: 2000,
-})(createAxiomDrain({ apiKey: process.env.AXIOM_TOKEN }));
+})(createAxiomDrain({ apiKey: config.services.axiomToken }));
 
 const fsDrain =
 	process.env.NODE_ENV === "development" || readBooleanEnv("LINKS_EVLOG_FS")
@@ -36,8 +36,7 @@ const fsDrain =
 			})
 		: null;
 
-const drainToAxiom =
-	process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN);
+const drainToAxiom = Boolean(config.services.axiomToken);
 
 const DURATION_RE = /^([\d.]+)(ms|s)$/;
 

apps/slack/src/lib/evlog-slack.ts

@@ -2,7 +2,7 @@ import { isRecord } from "@/lib/guards";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { readBooleanEnv } from "@databuddy/env/app";
+import { config, readBooleanEnv } from "@databuddy/env/app";
 import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog";
 import type { DrainContext, RequestLogger } from "evlog";
 import { createLogger, log } from "evlog";
@@ -15,15 +15,14 @@ type SlackLogFields = Record<string, SlackLogValue | null | undefined>;
 
 const activeSlackLog = new AsyncLocalStorage<RequestLogger>();
 
-const hasAxiom =
-	process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN);
+const hasAxiom = Boolean(config.services.axiomToken);
 const batchedAxiomDrain = hasAxiom
 	? createDrainPipeline<DrainContext>({
 			batch: { size: 50, intervalMs: 5000 },
 			maxBufferSize: 2000,
 		})(
 			createAxiomDrain({
-				apiKey: process.env.AXIOM_TOKEN,
+				apiKey: config.services.axiomToken,
 				dataset: "slack",
 			})
 		)

apps/uptime/src/lib/evlog-uptime.ts

@@ -1,6 +1,6 @@
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
-import { readBooleanEnv } from "@databuddy/env/app";
+import { config, readBooleanEnv } from "@databuddy/env/app";
 import {
 	createBatchedAxiomDrain,
 	enrichHttpWideEvent,
@@ -10,7 +10,7 @@ import { createBatchedSuperlogDrain } from "@databuddy/shared/evlog-superlog";
 import type { DrainContext, EnrichContext } from "evlog";
 import { createFsDrain } from "evlog/fs";
 
-const batchedAxiomDrain = createBatchedAxiomDrain(process.env.AXIOM_TOKEN);
+const batchedAxiomDrain = createBatchedAxiomDrain(config.services.axiomToken);
 const batchedSuperlogDrain = createBatchedSuperlogDrain();
 
 const devFsLogsDir = join(
@@ -24,8 +24,7 @@ const devFsLogsDir = join(
 const useLocalEvlogFiles =
 	process.env.NODE_ENV === "development" || readBooleanEnv("UPTIME_EVLOG_FS");
 
-const drainToAxiom =
-	process.env.NODE_ENV !== "development" && Boolean(process.env.AXIOM_TOKEN);
+const drainToAxiom = Boolean(config.services.axiomToken);
 
 const devFsDrain = useLocalEvlogFiles
 	? createFsDrain({ dir: devFsLogsDir, pretty: false })

apps/uptime/src/uptime-transition-alerts.ts

@@ -7,7 +7,7 @@ import {
 } from "@databuddy/db";
 import { chQuery } from "@databuddy/db/clickhouse";
 import { uptimeSchedules } from "@databuddy/db/schema";
-import { config } from "@databuddy/env/app";
+import { config, dataUrl } from "@databuddy/env/app";
 import {
 	NotificationClient,
 	buildAlarmNotificationTargets,
@@ -395,7 +395,7 @@ export async function writeMonitorState(
 }
 
 async function queryPreviousState(siteId: string): Promise<MonitorStateLookup> {
-	if (!process.env.CLICKHOUSE_URL) {
+	if (!dataUrl("CLICKHOUSE_URL")) {
 		return { kind: "missing" };
 	}
 	try {

package.json

@@ -45,11 +45,11 @@
     "format": "bunx ultracite fix",
     "check-types": "dotenv -- turbo run check-types",
     "generate-db": "turbo run generate --filter=@databuddy/db",
-    "db:studio": "dotenv -- turbo run db:studio",
+    "db:studio": "dotenv -- sh -c 'NODE_ENV=${NODE_ENV:-development} turbo run db:studio \"$@\"' --",
     "clickhouse:init": "dotenv -- turbo run clickhouse:init",
     "clickhouse:dql:init": "dotenv -- bun packages/db/src/clickhouse/dql.ts",
     "ch:verify": "dotenv -- bun packages/db/src/clickhouse/verify.ts",
-    "db:push": "dotenv -- turbo run db:push",
+    "db:push": "dotenv -- sh -c 'NODE_ENV=${NODE_ENV:-development} turbo run db:push \"$@\"' --",
     "db:migrate": "dotenv -- turbo run db:migrate",
     "db:deploy": "dotenv -- turbo run db:deploy",
     "db:seed": "dotenv -- sh -c 'cd packages/db && bun run db:seed \"$@\"' --",

packages/ai/src/lib/databuddy.ts

@@ -1,7 +1,7 @@
-import { readBooleanEnv } from "@databuddy/env/boolean";
+import { config, readBooleanEnv } from "@databuddy/env/app";
 import { Databuddy } from "@databuddy/sdk/node";
 
-const apiKey = process.env.DATABUDDY_API_KEY;
+const apiKey = config.services.databuddyApiKey;
 const websiteId = process.env.DATABUDDY_WEBSITE_ID;
 
 function createClient(source: string, namespace?: string) {

packages/ai/src/lib/supermemory.ts

@@ -1,8 +1,8 @@
+import { config } from "@databuddy/env/app";
 import { getMemoryClient } from "@databuddy/services/business-memory";
 export { getMemoryClient } from "@databuddy/services/business-memory";
 import { stripHtmlTags } from "./sanitize";
 
-const apiKey = process.env.SUPERMEMORY_API_KEY;
 const MAX_MEMORY_LENGTH = 2000;
 const MEMORY_REQUEST_CLAUSE = /[^.!?;:,\n]+\??/g;
 const MEMORY_REQUEST_FILLER =
@@ -46,7 +46,7 @@ const FORGET_ACKNOWLEDGEMENT =
 	/^(?:(?:thanks|thank you|ok|okay|got it)[.!]\s*)+/i;
 
 export function isMemoryEnabled(): boolean {
-	return Boolean(apiKey);
+	return Boolean(config.services.supermemoryApiKey);
 }
 
 export function sanitizeMemoryContent(