From de952bd48291872bc5f7273b3a8d3fdd5cd4c4e7 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Wed, 28 Oct 2026 15:52:06 -0700 Subject: [PATCH] agent: mail service group - Postfix/Dovecot/OpenDKIM (#67) Split 2/4 of #495. Adds the agent-side mail service: - Service groups via AGENT_SERVICES - Mail service group managing Postfix, Dovecot, and OpenDKIM - Config templates for mail stack (SNI, SQL maps, DKIM tables) - Only stops mail units this agent started Merge after the manager backend PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- agent/.fpm | 10 + agent/src/apply.ts | 76 +++- agent/src/config.ts | 22 ++ agent/src/index.ts | 57 ++- agent/src/mail.ts | 359 ++++++++++++++++++ agent/src/state.ts | 13 +- agent/src/system.ts | 64 +++- agent/src/types.ts | 33 ++ agent/templates/mail/dovecot.conf.ejs | 86 +++++ agent/templates/mail/opendkim-keytable.ejs | 3 + .../templates/mail/opendkim-signingtable.ejs | 3 + agent/templates/mail/opendkim.conf.ejs | 21 + agent/templates/mail/postfix-main.cf.ejs | 63 +++ agent/templates/mail/postfix-master.cf.ejs | 60 +++ agent/templates/mail/postfix-sni-map.ejs | 5 + .../mail/postfix-sql-mailboxes.cf.ejs | 9 + .../templates/mail/postfix-sql-senders.cf.ejs | 8 + agent/templates/mail/postfix-vdomains.ejs | 4 + agent/test/config.test.js | 32 ++ agent/test/mail-template.test.js | 131 +++++++ .../docs/developers/agent.md | 27 +- 21 files changed, 1065 insertions(+), 21 deletions(-) create mode 100644 agent/src/mail.ts create mode 100644 agent/templates/mail/dovecot.conf.ejs create mode 100644 agent/templates/mail/opendkim-keytable.ejs create mode 100644 agent/templates/mail/opendkim-signingtable.ejs create mode 100644 agent/templates/mail/opendkim.conf.ejs create mode 100644 agent/templates/mail/postfix-main.cf.ejs create mode 100644 agent/templates/mail/postfix-master.cf.ejs create mode 100644 agent/templates/mail/postfix-sni-map.ejs create mode 100644 agent/templates/mail/postfix-sql-mailboxes.cf.ejs create mode 100644 agent/templates/mail/postfix-sql-senders.cf.ejs create mode 100644 agent/templates/mail/postfix-vdomains.ejs create mode 100644 agent/test/config.test.js create mode 100644 agent/test/mail-template.test.js diff --git a/agent/.fpm b/agent/.fpm index 3c06e600..e9a5302f 100644 --- a/agent/.fpm +++ b/agent/.fpm @@ -14,5 +14,15 @@ --depends ca-certificates --depends ssl-cert --depends dnsmasq +--deb-suggests postfix +--deb-suggests postfix-pgsql +--deb-suggests postfix-mysql +--deb-suggests dovecot-core +--deb-suggests dovecot-imapd +--deb-suggests dovecot-lmtpd +--deb-suggests dovecot-pgsql +--deb-suggests dovecot-mysql +--deb-suggests opendkim +--deb-suggests opensource-mail-helper --deb-no-default-config-files --after-install contrib/postinstall.sh diff --git a/agent/src/apply.ts b/agent/src/apply.ts index 0efe413b..c4da8f7f 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -10,33 +10,52 @@ import fs from 'fs'; import path from 'path'; import { execFileSync } from 'child_process'; import ejs from 'ejs'; -import { reloadOrRestartService, restartService, sighupService } from './system'; +import { chownByName, reloadOrRestartService, restartService, sighupService } from './system'; import { log, commandOutput } from './log'; -import type { AgentConfig } from './config'; +import type { AgentConfig, ServiceGroup } from './config'; import type { ApplyResult, SiteConfig } from './types'; const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); +/** Binaries each service group needs; missing ones are reported at check-in + * (the manager refuses the mail-host claim until they're installed). */ +export const GROUP_BINARIES: Record = { + nginx: ['nginx'], + dnsmasq: ['dnsmasq'], + mail: ['postfix', 'dovecot', 'opendkim'], +}; + interface RenderedFile { dest: string; content: string; /** File mode for atomic writes (e.g. 0o600 for configs holding secrets). */ mode?: number; + /** `user:group` to chown to after writing (e.g. DKIM keys → opendkim). */ + owner?: string; } export interface ManagedService { /** systemd unit name (also the key reported at check-in). */ unit: string; + /** Service group this unit belongs to (AGENT_SERVICES opt-in). */ + group: ServiceGroup; + /** Unit whose ActiveState is reported at check-in when it differs from + * `unit` (Debian's postfix.service is a wrapper around postfix@-). */ + statusUnit?: string; /** Render all managed files. Returns null when there is nothing to manage * yet (e.g. dnsmasq before the site exists). */ render(config: SiteConfig, agent: AgentConfig): Promise; + /** Commands run after staging files, before `test` (e.g. `postmap -F` to + * compile the SNI hash map). A failure rolls the files back. */ + postWrite?: string[][]; /** Command that validates the staged config before it is kept. */ test?: string[]; /** Reload/restart after a successful apply. */ reload(changedFiles: string[]): Promise; } -function renderTemplate(template: string, data: object): Promise { +/** Render a template from the shared templates dir (used by mail.ts too). */ +export function renderTemplate(template: string, data: object): Promise { return ejs.renderFile(path.join(TEMPLATES_DIR, template), data); } @@ -51,6 +70,7 @@ function run(cmd: string[]): string { export const services: ManagedService[] = [ { unit: 'nginx', + group: 'nginx', async render(config, agent) { return [{ dest: '/etc/nginx/nginx.conf', @@ -73,6 +93,7 @@ export const services: ManagedService[] = [ }, { unit: 'dnsmasq', + group: 'dnsmasq', async render(config) { const site = config.site; // Skip dnsmasq management until the site's DHCP/DNS settings are fully @@ -112,12 +133,28 @@ function readIfExists(file: string): string | null { // Write via temp file + rename so a crash mid-write can never leave a // truncated config on disk. -function writeFileAtomic(dest: string, content: string, mode?: number): void { +function writeFileAtomic(dest: string, content: string, mode?: number, owner?: string): void { const tmp = `${dest}.tmp-${process.pid}`; fs.writeFileSync(tmp, content, mode !== undefined ? { mode } : {}); + if (owner) chownByName(tmp, owner); fs.renameSync(tmp, dest); } +/** Re-apply desired mode/owner to files whose content already matches. */ +function reconcileAttributes(svc: ManagedService, files: { dest: string; mode?: number; owner?: string }[]): void { + for (const f of files) { + try { + if (f.mode !== undefined && (fs.statSync(f.dest).mode & 0o7777) !== f.mode) { + fs.chmodSync(f.dest, f.mode); + log.info(`${svc.unit}: fixed mode on ${f.dest}`); + } + if (f.owner) chownByName(f.dest, f.owner); + } catch (err) { + log.warn(`${svc.unit}: could not reconcile attributes on ${f.dest}: ${(err as Error).message}`); + } + } +} + export async function applyService( svc: ManagedService, config: SiteConfig, @@ -133,6 +170,10 @@ export async function applyService( const current = new Map(files.map((f) => [f.dest, readIfExists(f.dest)])); const changed = files.filter((f) => current.get(f.dest) !== f.content).map((f) => f.dest); if (changed.length === 0) { + // Contents match, but desired mode/owner must still be reconciled — a + // DKIM key or helper config accidentally left world-readable would + // otherwise stay exposed forever. + reconcileAttributes(svc, files); log.debug(`${svc.unit}: config unchanged, nothing to do`); return 'success'; } @@ -140,21 +181,44 @@ export async function applyService( log.info(`${svc.unit}: ${changed.length} file(s) changed, applying: ${changed.join(', ')}`); const modeByDest = new Map(files.map((f) => [f.dest, f.mode])); + const ownerByDest = new Map(files.map((f) => [f.dest, f.owner])); // Stage the new files (previous contents kept in memory for rollback). for (const f of files) { fs.mkdirSync(path.dirname(f.dest), { recursive: true }); - writeFileAtomic(f.dest, f.content, f.mode); + writeFileAtomic(f.dest, f.content, f.mode, f.owner); log.debug(`${svc.unit}: wrote ${f.dest}`); } const rollback = () => { for (const [dest, prev] of current) { if (prev === null) fs.rmSync(dest, { force: true }); - else writeFileAtomic(dest, prev, modeByDest.get(dest)); + else writeFileAtomic(dest, prev, modeByDest.get(dest), ownerByDest.get(dest)); + } + // Post-write commands may have produced sidecar artifacts from the new + // sources (e.g. postmap compiling sni_map.db) — regenerate them from the + // restored files so no stale compiled state survives the rollback. + for (const cmd of svc.postWrite ?? []) { + try { + run(cmd); + } catch (err) { + log.error(`${svc.unit}: post-write command failed during rollback (\`${cmd.join(' ')}\`): ${(err as Error).message}`); + } } log.debug(`${svc.unit}: rolled back to previous config`); }; + for (const cmd of svc.postWrite ?? []) { + try { + run(cmd); + } catch (err) { + rollback(); + log.error(`${svc.unit}: post-write command failed (\`${cmd.join(' ')}\`), rolled back: ${(err as Error).message}`); + const output = commandOutput(err); + if (output) log.error(`${svc.unit}: post-write output:\n${output}`); + return 'failure'; + } + } + if (svc.test) { const testCmd = svc.test.join(' '); log.debug(`${svc.unit}: testing config with \`${testCmd}\``); diff --git a/agent/src/config.ts b/agent/src/config.ts index 27b93695..93c61b6f 100644 --- a/agent/src/config.ts +++ b/agent/src/config.ts @@ -3,12 +3,33 @@ import { setLogLevel, type LogLevel } from './log'; +/** Service groups the agent can run. `mail` is opt-in via AGENT_SERVICES. */ +export const KNOWN_SERVICE_GROUPS = ['nginx', 'dnsmasq', 'mail'] as const; +export type ServiceGroup = (typeof KNOWN_SERVICE_GROUPS)[number]; + +const DEFAULT_SERVICE_GROUPS: ServiceGroup[] = ['nginx', 'dnsmasq']; + export interface AgentConfig { siteId: number; managerUrl: string; apiKey?: string; stateDir: string; logLevel: LogLevel; + /** Enabled service groups (AGENT_SERVICES; unset means nginx,dnsmasq). */ + services: ServiceGroup[]; +} + +function parseServiceGroups(raw: string | undefined): ServiceGroup[] { + if (!raw || !raw.trim()) return [...DEFAULT_SERVICE_GROUPS]; + const groups: ServiceGroup[] = []; + for (const part of raw.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean)) { + if ((KNOWN_SERVICE_GROUPS as readonly string[]).includes(part)) { + if (!groups.includes(part as ServiceGroup)) groups.push(part as ServiceGroup); + } else { + throw new Error(`AGENT_SERVICES contains unknown service group "${part}" (known: ${KNOWN_SERVICE_GROUPS.join(', ')})`); + } + } + return groups; } export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { @@ -26,5 +47,6 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { // Set by systemd from StateDirectory=; fallback for manual runs. stateDir: env.STATE_DIRECTORY || '/var/lib/opensource-agent', logLevel, + services: parseServiceGroups(env.AGENT_SERVICES), }; } diff --git a/agent/src/index.ts b/agent/src/index.ts index cf37979a..dde4a33d 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -13,22 +13,29 @@ import os from 'os'; import { loadConfig, type AgentConfig } from './config'; import { State } from './state'; -import { getPrimaryIpv4, getServiceState, disconnectSystemBus } from './system'; +import { getPrimaryIpv4, getServiceState, findMissingBinaries, disconnectSystemBus } from './system'; import { checkin } from './api'; -import { services, applyService } from './apply'; +import { services, GROUP_BINARIES, applyService, type ManagedService } from './apply'; +import { mailServices, ensureMailProvisioned, syncMailUnits, stopMailUnitsIfRunning, gcMailboxes, missingMailHelper } from './mail'; import { reconcileVolumes } from './volumes'; import { log } from './log'; -import type { CheckinRequest, ServiceStatus } from './types'; +import type { CheckinRequest, ServiceStatus, SiteConfig } from './types'; // Safety cap: a flapping server-side config can't keep a single run alive // forever; the timer starts a fresh run 30s later anyway. const MAX_PASSES = 5; +const ALL_SERVICES: ManagedService[] = [...services, ...mailServices]; + +function enabledServices(cfg: AgentConfig): ManagedService[] { + return ALL_SERVICES.filter((svc) => cfg.services.includes(svc.group)); +} + async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { const serviceStatus: Record = {}; - for (const svc of services) { + for (const svc of enabledServices(cfg)) { serviceStatus[svc.unit] = { - state: await getServiceState(svc.unit), + state: await getServiceState(svc.statusUnit ?? svc.unit), lastApply: state.lastApply[svc.unit] ?? 'unknown', }; } @@ -38,6 +45,12 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise GROUP_BINARIES[g])), + // The helper is its own package under /opt, not a PATH binary. + ...(cfg.services.includes('mail') && missingMailHelper() ? ['opensource-mail-helper'] : []), + ], }; // Report any not-yet-delivered volume results (persisted across runs). Only // include the field when there is something to report so the manager isn't @@ -48,10 +61,35 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { + try { + if (cfg.services.includes('mail')) { + const mailManaged = await syncMailUnits(config, state.mailManaged); + if (mailManaged !== state.mailManaged) { + state.mailManaged = mailManaged; + state.save(); + } + if (config.mail) gcMailboxes(config.mail); + } else if (state.mailManaged) { + await stopMailUnitsIfRunning(); + state.mailManaged = false; + state.save(); + } + } catch (err) { + log.error(`mail: unit lifecycle sync failed: ${(err as Error).message}`); + } +} + async function main(): Promise { const cfg = loadConfig(); const state = State.load(cfg.stateDir); - log.info(`agent starting: siteId=${cfg.siteId}, manager=${cfg.managerUrl}`); + log.info(`agent starting: siteId=${cfg.siteId}, manager=${cfg.managerUrl}, services=${cfg.services.join(',')}`); log.debug(`state dir=${cfg.stateDir}, saved etag=${state.etag ?? '(none)'}`); if (Object.keys(state.pendingVolumeResults).length > 0) { log.debug(`carrying ${Object.keys(state.pendingVolumeResults).length} pending volume result(s) from a prior run`); @@ -77,9 +115,14 @@ async function main(): Promise { } log.info(`check-in: new config received (etag=${result.etag ?? '(none)'}), applying`); - for (const svc of services) { + // Mail dirs must exist before the mail configs reference them. + if (cfg.services.includes('mail') && result.config.mail) { + ensureMailProvisioned(cfg); + } + for (const svc of enabledServices(cfg)) { state.lastApply[svc.unit] = await applyService(svc, result.config, cfg); } + await syncMailLifecycle(cfg, result.config, state); // Ensure volume directories exist from the new snapshot; stash results in // persistent state so they are reported on the next check-in and survive a diff --git a/agent/src/mail.ts b/agent/src/mail.ts new file mode 100644 index 00000000..5905d878 --- /dev/null +++ b/agent/src/mail.ts @@ -0,0 +1,359 @@ +/** + * The `mail` service group (issue #67): Postfix + Dovecot + OpenDKIM + + * opensource-mail-helper (a milter, shipped as its own component under + * /opt/opensource-server/mail-helper), rendered from the holder-only `mail` + * section of the config snapshot. + * + * Lifecycle: while this agent holds the mail-host claim, the units are + * enabled and started after each apply; when the group is turned off or the + * claim is lost, they are stopped and disabled — configs and /var/vmail stay + * in place. System users (vmail, opensource-mail) come from the + * opensource-mail-helper package's maintainer scripts; the agent only + * manages directories and files. + * + * Agent-managed mail state (DKIM keys, OpenDKIM tables) lives under + * $STATE_DIRECTORY/mail, not /etc. Postfix/Dovecot/OpenDKIM daemon configs + * remain in /etc where their packages expect them. + */ + +import fs from 'fs'; +import path from 'path'; +import { execFileSync } from 'child_process'; +import { + chownByName, + enableAndStartServices, + stopAndDisableServices, + getServiceState, + reloadOrRestartService, + restartService, +} from './system'; +import { renderTemplate, type ManagedService } from './apply'; +import { log } from './log'; +import type { AgentConfig } from './config'; +import type { MailConfig, SiteConfig } from './types'; + +export const MAIL_UNITS = ['opendkim', 'dovecot', 'postfix', 'opensource-mail-helper']; + +// Paths are env-overridable for development stacks; defaults match the +// packages. Both milter sockets live inside the Postfix chroot so the +// chrooted smtpd can reach them without inet ports. +const HELPER_DIR = process.env.MAIL_HELPER_DIR || '/opt/opensource-server/mail-helper'; +const HELPER_SOCKET_DIR = process.env.MAIL_HELPER_SOCKET_DIR || '/var/spool/postfix/mail-helper'; +const OPENDKIM_SOCKET_DIR = process.env.OPENDKIM_SOCKET_DIR || '/var/spool/postfix/opendkim'; + +const VMAIL_DIR = process.env.VMAIL_DIR || '/var/vmail'; +const GC_STATE_FILE = process.env.GC_STATE_FILE || path.join(VMAIL_DIR, '.gc-state.json'); +const GC_RETENTION_MS = Number(process.env.GC_RETENTION_MS) > 0 + ? Number(process.env.GC_RETENTION_MS) + : 30 * 24 * 60 * 60 * 1000; + +function sh(cmd: string[]): void { + execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe', encoding: 'utf8' }); +} + +/** The helper is a separate package; report it like a missing binary. */ +export function missingMailHelper(): boolean { + return !fs.existsSync(path.join(HELPER_DIR, 'src', 'index.js')); +} + +// --- render context --------------------------------------------------------- + +interface CertPair { name: string; cert: string; key: string } + +/** Per-domain cert material shared with nginx. Domains without both files + * are skipped (they fall back to the fallback chain). */ +function domainCerts(mail: MailConfig): CertPair[] { + const names = new Set([ + ...mail.dkim.map((k) => k.domain), + ...mail.receiveDomains, + ]); + if (mail.hostname) names.add(mail.hostname); + const pairs: CertPair[] = []; + for (const name of [...names].sort()) { + const cert = `/etc/ssl/certs/${name}.crt`; + const key = `/etc/ssl/private/${name}.key`; + if (fs.existsSync(cert) && fs.existsSync(key)) pairs.push({ name, cert, key }); + } + return pairs; +} + +function fallbackChain(mail: MailConfig): { fallbackCert: string; fallbackKey: string } { + if (mail.hostname) { + const cert = `/etc/ssl/certs/${mail.hostname}.crt`; + const key = `/etc/ssl/private/${mail.hostname}.key`; + if (fs.existsSync(cert) && fs.existsSync(key)) return { fallbackCert: cert, fallbackKey: key }; + } + // ssl-cert package default (an agent package dependency). + return { + fallbackCert: '/etc/ssl/certs/ssl-cert-snakeoil.pem', + fallbackKey: '/etc/ssl/private/ssl-cert-snakeoil.key', + }; +} + +/** Agent-managed OpenDKIM material under the agent's state directory. */ +function mailStateDir(agent: AgentConfig): string { + return path.join(agent.stateDir, 'mail'); +} + +/** Everything the mail templates need, derived from the snapshot + local + * environment (cert presence, DB passwords from /etc/environment). */ +export function buildMailContext(mail: MailConfig, agent: AgentConfig, env: NodeJS.ProcessEnv = process.env) { + const mysql = mail.db.dialect === 'mysql' || mail.db.dialect === 'mariadb'; + const host = mail.db.host; + // Postfix map syntax: TCP "host:port", or a unix socket path for local + // socket auth (Postgres peer / MariaDB unix_socket — no password needed). + const pfHosts = host + ? (mail.db.port ? `${host}:${mail.db.port}` : host) + : (mysql ? 'unix:/var/run/mysqld/mysqld.sock' : 'unix:/var/run/postgresql'); + return { + mail, + stateDir: mailStateDir(agent), + // Milter endpoints as the chrooted smtpd resolves them (queue-dir relative). + helperMilter: `unix:${path.posix.relative('/var/spool/postfix', path.posix.join(HELPER_SOCKET_DIR, 'milter.sock'))}`, + opendkimMilter: `unix:${path.posix.relative('/var/spool/postfix', path.posix.join(OPENDKIM_SOCKET_DIR, 'opendkim.sock'))}`, + opendkimSocket: `local:${path.posix.join(OPENDKIM_SOCKET_DIR, 'opendkim.sock')}`, + certs: { ...fallbackChain(mail), domains: domainCerts(mail) }, + db: { + prefix: mysql ? 'mysql' : 'pgsql', + hosts: pfHosts, + user: mail.db.postfixUser, + password: env.MAIL_POSTFIX_DB_PASSWORD || null, + dbname: mail.db.database ?? '', + dovecotDriver: mysql ? 'mysql' : 'pgsql', + dovecotHost: host || (mysql ? '/var/run/mysqld/mysqld.sock' : '/var/run/postgresql'), + dovecotParams: { dbname: mail.db.database ?? '' }, + dovecotUser: mail.db.dovecotUser, + dovecotPassword: env.MAIL_DOVECOT_DB_PASSWORD || null, + }, + }; +} + +/** Overlay for the helper's committed config.json (see mail-helper/). */ +export function buildHelperLocalConfig(mail: MailConfig, agent: AgentConfig, env: NodeJS.ProcessEnv = process.env): string { + const mysql = mail.db.dialect === 'mysql' || mail.db.dialect === 'mariadb'; + return `${JSON.stringify({ + socket: path.posix.join(HELPER_SOCKET_DIR, 'milter.sock'), + db: { + dialect: mysql ? 'mysql' : 'postgres', + host: mail.db.host, + port: mail.db.port, + database: mail.db.database, + user: mail.db.postfixUser, + password: env.MAIL_POSTFIX_DB_PASSWORD || null, + }, + unsubscribe: { + // One-click links point at the manager itself (it serves /u/ and is + // never behind oauth2-proxy), so the base URL is this agent's + // MANAGER_URL rather than a separate setting. + baseUrl: agent.managerUrl, + keys: mail.unsubscribe.keys, + }, + }, null, 2)}\n`; +} + +// --- managed services -------------------------------------------------------- + +/** Mail is managed only while this agent holds the claim. */ +function mailOf(config: SiteConfig): MailConfig | null { + return config.mail ?? null; +} + +export const mailServices: ManagedService[] = [ + { + unit: 'opendkim', + group: 'mail', + async render(config, agent) { + const mail = mailOf(config); + if (!mail) return null; + const context = buildMailContext(mail, agent); + const files = [ + { dest: '/etc/opendkim.conf', content: await renderTemplate('mail/opendkim.conf.ejs', context) }, + { dest: path.join(context.stateDir, 'opendkim-keytable'), content: await renderTemplate('mail/opendkim-keytable.ejs', context) }, + { dest: path.join(context.stateDir, 'opendkim-signingtable'), content: await renderTemplate('mail/opendkim-signingtable.ejs', context) }, + ]; + // Private keys never serialized anywhere else on this host. + for (const k of mail.dkim) { + files.push({ + dest: path.join(context.stateDir, 'dkim', `${k.domain}.private`), + content: k.privateKey, + mode: 0o600, + owner: 'opendkim:opendkim', + } as (typeof files)[number]); + } + return files; + }, + test: ['opendkim', '-n'], + reload() { + return restartService('opendkim'); + }, + }, + { + unit: 'dovecot', + group: 'mail', + async render(config, agent) { + const mail = mailOf(config); + if (!mail) return null; + const context = buildMailContext(mail, agent); + return [{ + dest: '/etc/dovecot/dovecot.conf', + content: await renderTemplate('mail/dovecot.conf.ejs', context), + // Embeds the mail_dovecot DB password when TCP auth is used. + mode: 0o600, + }]; + }, + test: ['doveconf', '-n'], + async reload() { + await reloadOrRestartService('dovecot'); + // Flush cached credentials so disables/rotations apply immediately. + try { + sh(['doveadm', 'auth', 'cache', 'flush']); + } catch { + /* dovecot still starting — the cache is empty anyway */ + } + }, + }, + { + unit: 'postfix', + group: 'mail', + // Debian's postfix.service is a wrapper; the real daemon state lives on + // the postfix@- instance. + statusUnit: 'postfix@-', + async render(config, agent) { + const mail = mailOf(config); + if (!mail) return null; + const context = buildMailContext(mail, agent); + const files = [ + { dest: '/etc/postfix/main.cf', content: await renderTemplate('mail/postfix-main.cf.ejs', context) }, + { dest: '/etc/postfix/master.cf', content: await renderTemplate('mail/postfix-master.cf.ejs', context) }, + { dest: '/etc/postfix/vdomains', content: await renderTemplate('mail/postfix-vdomains.ejs', context) }, + { dest: '/etc/postfix/sni_map', content: await renderTemplate('mail/postfix-sni-map.ejs', context), mode: 0o600 }, + { dest: '/etc/postfix/sql/senders.cf', content: await renderTemplate('mail/postfix-sql-senders.cf.ejs', context), mode: 0o600 }, + { dest: '/etc/postfix/sql/mailboxes.cf', content: await renderTemplate('mail/postfix-sql-mailboxes.cf.ejs', context), mode: 0o600 }, + ]; + if (mail.relayhost?.username) { + files.push({ + dest: '/etc/postfix/sasl_passwd', + content: `${mail.relayhost.host} ${mail.relayhost.username}:${mail.relayhost.password ?? ''}\n`, + mode: 0o600, + } as (typeof files)[number]); + } + return files; + }, + // Compile the SNI key+chain hash before validating. + postWrite: [['postmap', '-F', 'hash:/etc/postfix/sni_map']], + test: ['postfix', 'check'], + reload() { + return reloadOrRestartService('postfix'); + }, + }, + { + unit: 'opensource-mail-helper', + group: 'mail', + async render(config, agent) { + const mail = mailOf(config); + if (!mail) return null; + // The committed config.json ships defaults; the agent only writes the + // environment-specific overlay (same split in dev and production). + return [{ + dest: path.join(HELPER_DIR, 'config.local.json'), + content: buildHelperLocalConfig(mail, agent), + mode: 0o640, + owner: 'root:opensource-mail', + }]; + }, + reload() { + return restartService('opensource-mail-helper'); + }, + }, +]; + +// --- lifecycle --------------------------------------------------------------- + +/** Create the directories the mail stack needs. Idempotent; runs only when + * this agent holds the claim. System users come from package maintainer + * scripts (opensource-mail-helper postinst creates vmail + opensource-mail). */ +export function ensureMailProvisioned(agent: AgentConfig): void { + fs.mkdirSync(VMAIL_DIR, { recursive: true }); + chownByName(VMAIL_DIR, 'vmail:vmail'); + fs.chmodSync(VMAIL_DIR, 0o750); + + // Agent-managed DKIM material under $STATE_DIRECTORY/mail. + const stateDir = mailStateDir(agent); + const dkimDir = path.join(stateDir, 'dkim'); + fs.mkdirSync(dkimDir, { recursive: true }); + for (const dir of [stateDir, dkimDir]) { + chownByName(dir, 'root:opendkim'); + fs.chmodSync(dir, 0o750); + } + + // Milter socket dirs inside the Postfix chroot. The chrooted smtpd + // connects as user postfix: setgid keeps the sockets group-owned by + // postfix (OpenDKIM runs as opendkim:postfix via UserID, so no group + // membership changes are needed anywhere). + fs.mkdirSync(HELPER_SOCKET_DIR, { recursive: true }); + chownByName(HELPER_SOCKET_DIR, 'opensource-mail:postfix'); + fs.chmodSync(HELPER_SOCKET_DIR, 0o2750); + fs.mkdirSync(OPENDKIM_SOCKET_DIR, { recursive: true }); + chownByName(OPENDKIM_SOCKET_DIR, 'opendkim:postfix'); + fs.chmodSync(OPENDKIM_SOCKET_DIR, 0o2750); +} + +/** Enable+start the mail units while holding the claim; stop+disable them + * otherwise (configs and /var/vmail are left in place). The shutdown path + * only runs when THIS agent started the units (state.mailManaged) — a + * host's own unrelated postfix/dovecot must never be touched. Returns the + * new mailManaged value for the caller to persist. */ +export async function syncMailUnits(config: SiteConfig, mailManaged: boolean): Promise { + if (config.mail) { + await enableAndStartServices(MAIL_UNITS); + return true; + } + if (!mailManaged) return false; + if (config.mailStatus) { + log.info(`mail: not the mail host (${config.mailStatus}); stopping the units this agent started`); + } + await stopAndDisableServices(MAIL_UNITS); + return false; +} + +/** Stop the stack when the `mail` group itself is disabled — again only + * when this agent started it, and only when something is actually running, + * so nothing is churned on agents that never ran mail. */ +export async function stopMailUnitsIfRunning(): Promise { + const states = await Promise.all(MAIL_UNITS.map((unit) => getServiceState(unit))); + if (states.some((state) => state === 'active' || state === 'activating')) { + log.info('mail: service group disabled but units are running; stopping them'); + await stopAndDisableServices(MAIL_UNITS); + } +} + +/** Remove Maildirs of deleted accounts after a retention window. A dir is + * eligible once its account id stops appearing in the snapshot's mailboxIds; + * first-missing timestamps persist in /var/vmail/.gc-state.json. */ +export function gcMailboxes(mail: MailConfig, now = Date.now()): void { + let entries: string[]; + try { + entries = fs.readdirSync(VMAIL_DIR).filter((e) => !e.startsWith('.')); + } catch { + return; + } + let gcState: Record = {}; + try { + gcState = JSON.parse(fs.readFileSync(GC_STATE_FILE, 'utf8')); + } catch { + /* first run or corrupt state — rebuild from scratch */ + } + const known = new Set(mail.mailboxIds); + const next: Record = {}; + for (const entry of entries) { + if (known.has(entry)) continue; + const firstMissing = gcState[entry] ?? now; + if (now - firstMissing >= GC_RETENTION_MS) { + log.info(`mail: removing mailbox of deleted account ${entry}`); + fs.rmSync(path.join(VMAIL_DIR, entry), { recursive: true, force: true }); + } else { + next[entry] = firstMissing; + } + } + fs.writeFileSync(GC_STATE_FILE, JSON.stringify(next, null, 2)); +} diff --git a/agent/src/state.ts b/agent/src/state.ts index bd691b2b..4cbe7df5 100644 --- a/agent/src/state.ts +++ b/agent/src/state.ts @@ -10,6 +10,10 @@ import type { ApplyResult, VolumeResult } from './types'; export class State { etag?: string; lastApply: Record = {}; + // True once THIS agent has enabled/started the mail units. Guards the + // shutdown paths: an agent must never stop a postfix/dovecot it did not + // start (e.g. a host's own unrelated mail setup). + mailManaged = false; // Volume provisioning results not yet confirmed delivered to the manager. // Persisted so they survive a process exit between reconcile and the next // check-in — otherwise a saved ETag would 304 the next run and the result @@ -32,10 +36,12 @@ export class State { const data = JSON.parse(raw) as { etag?: string; lastApply?: Record; + mailManaged?: boolean; pendingVolumeResults?: Record; }; state.etag = data.etag; state.lastApply = data.lastApply ?? {}; + state.mailManaged = data.mailManaged ?? false; state.pendingVolumeResults = data.pendingVolumeResults ?? {}; } catch (err) { if (!(err instanceof SyntaxError)) throw err; @@ -50,7 +56,12 @@ export class State { fs.writeFileSync( this.file, JSON.stringify( - { etag: this.etag, lastApply: this.lastApply, pendingVolumeResults: this.pendingVolumeResults }, + { + etag: this.etag, + lastApply: this.lastApply, + mailManaged: this.mailManaged, + pendingVolumeResults: this.pendingVolumeResults, + }, null, 2, ), diff --git a/agent/src/system.ts b/agent/src/system.ts index 5a3fda70..bc8cc5d2 100644 --- a/agent/src/system.ts +++ b/agent/src/system.ts @@ -1,6 +1,8 @@ /** Host system info and systemd unit control via the systemd D-Bus API. */ +import fs from 'fs'; import os from 'os'; +import path from 'path'; import dbus, { type MessageBus, type ProxyObject, type Variant } from '@particle/dbus-next'; /** First non-internal IPv4 address, or null when none is configured. */ @@ -13,6 +15,37 @@ export function getPrimaryIpv4(): string | null { return null; } +/** Which of `binaries` cannot be found on PATH. */ +export function findMissingBinaries(binaries: string[]): string[] { + const dirs = (process.env.PATH ?? '').split(':').filter(Boolean); + return binaries.filter((bin) => !dirs.some((dir) => { + try { + fs.accessSync(path.join(dir, bin), fs.constants.X_OK); + return true; + } catch { + return false; + } + })); +} + +// Node has no getpwnam/getgrnam binding, so resolve names from the system +// databases directly (local users only — exactly what the mail stack uses). +function lookupId(file: string, name: string): { id: number; gid: number } { + const entry = fs.readFileSync(file, 'utf8').split('\n') + .find((line) => line.startsWith(`${name}:`)); + if (!entry) throw new Error(`unknown ${file.includes('group') ? 'group' : 'user'} "${name}"`); + const fields = entry.split(':'); + return { id: Number(fields[2]), gid: Number(fields[3]) }; +} + +/** chown by `user` or `user:group` name via fs.chownSync (no shelling out). */ +export function chownByName(target: string, owner: string): void { + const [user, group] = owner.split(':'); + const pw = lookupId('/etc/passwd', user); + const gid = group ? lookupId('/etc/group', group).id : pw.gid; + fs.chownSync(target, pw.id, gid); +} + // --- systemd (org.freedesktop.systemd1 on the system bus) ------------------- const SYSTEMD = 'org.freedesktop.systemd1'; @@ -33,7 +66,12 @@ interface SystemdManager { LoadUnit(name: string): Promise; ReloadOrRestartUnit(name: string, mode: string): Promise; RestartUnit(name: string, mode: string): Promise; + StartUnit(name: string, mode: string): Promise; + StopUnit(name: string, mode: string): Promise; KillUnit(name: string, who: string, signal: number): Promise; + EnableUnitFiles(files: string[], runtime: boolean, force: boolean): Promise; + DisableUnitFiles(files: string[], runtime: boolean): Promise; + Reload(): Promise; Subscribe(): Promise; Unsubscribe(): Promise; on(event: 'JobRemoved', listener: (id: number, job: unknown, unit: string, result: string) => void): void; @@ -85,7 +123,7 @@ export async function getServiceState(unit: string): Promise { /** Enqueue a systemd job and wait for it to finish; throws unless the job * completes with result "done". */ -async function runJob(method: 'ReloadOrRestartUnit' | 'RestartUnit', unit: string): Promise { +async function runJob(method: 'ReloadOrRestartUnit' | 'RestartUnit' | 'StartUnit' | 'StopUnit', unit: string): Promise { const mgr = await getManager(); // JobRemoved is only broadcast to subscribed clients. await mgr.Subscribe(); @@ -140,3 +178,27 @@ export async function sighupService(unit: string): Promise { const mgr = await getManager(); await mgr.KillUnit(unitName(unit), 'main', SIGHUP); } + +/** `systemctl enable --now ` — boot persistence + immediate start. */ +export async function enableAndStartServices(units: string[]): Promise { + const mgr = await getManager(); + await mgr.EnableUnitFiles(units.map(unitName), false, true); + await mgr.Reload(); + for (const unit of units) await runJob('StartUnit', unit); +} + +/** `systemctl disable --now `; failures are swallowed — used when the + * mail group is turned off or the claim is lost, where units may not even be + * installed. Configs and data are left in place. */ +export async function stopAndDisableServices(units: string[]): Promise { + const mgr = await getManager(); + for (const unit of units) { + await runJob('StopUnit', unit).catch(() => undefined); + } + try { + await mgr.DisableUnitFiles(units.map(unitName), false); + await mgr.Reload(); + } catch { + /* not installed / already disabled */ + } +} diff --git a/agent/src/types.ts b/agent/src/types.ts index f9175779..8b455c2d 100644 --- a/agent/src/types.ts +++ b/agent/src/types.ts @@ -19,6 +19,10 @@ export interface CheckinRequest { currentTime: number; ipv4Address: string | null; services: Record; + /** Service groups this agent is configured to run (AGENT_SERVICES). */ + enabledServices: string[]; + /** Binaries missing for the enabled groups; blocks the mail-host claim. */ + missingBinaries: string[]; /** * Per-volume directory-provisioning results, keyed by the manager-assigned * Volume id: `{ : { applied, message? } }`. Present only when the @@ -106,4 +110,33 @@ export interface NginxConfig { export interface SiteConfig { site: SiteInfo | null; nginx: NginxConfig; + /** Mail-host claim outcome for agents reporting the `mail` group: + * holder | conflict | unsupported | missing_packages. Absent otherwise. */ + mailStatus?: string; + /** Mail snapshot — present ONLY when this agent holds the mail-host claim. */ + mail?: MailConfig; +} + +/** The `mail` snapshot section (manager utils/agent-config.js buildMailSnapshot). */ +export interface MailConfig { + hostname: string | null; + relayhost: { host: string; username: string | null; password: string | null } | null; + messageSizeLimitMb: number; + defaultQuotaMb: number; + dkim: { domain: string; selector: string; privateKey: string }[]; + receiveDomains: string[]; + /** Every account id (any state) — /var/vmail dirs not in this list belong + * to deleted accounts and are garbage-collected after a retention window. */ + mailboxIds: string[]; + db: { + dialect: string; + host: string | null; + port: number | null; + database: string | null; + dovecotUser: string; + postfixUser: string; + }; + unsubscribe: { + keys: { kid: string; secret: string; active: boolean }[]; + }; } diff --git a/agent/templates/mail/dovecot.conf.ejs b/agent/templates/mail/dovecot.conf.ejs new file mode 100644 index 00000000..9b1ad670 --- /dev/null +++ b/agent/templates/mail/dovecot.conf.ejs @@ -0,0 +1,86 @@ +<%_ /* Dovecot 2.4 (new config syntax; Debian 13). Validated with + `doveconf -n` at apply time — a rejected config is rolled back. */ -%> +# Managed by opensource-agent (mail service group) — do not edit. +dovecot_config_version = 2.4.0 +dovecot_storage_version = 2.4.0 + +protocols = imap lmtp + +# Virtual users: one Maildir per account under /var/vmail/, +# owned by the vmail system user (created by the agent). +mail_driver = maildir +mail_path = %{userdb:home}/Maildir +mail_uid = vmail +mail_gid = vmail + +# TLS: same per-domain certs as nginx; fallback below, per-domain overrides +# via local_name. Plaintext auth is only accepted over TLS. +ssl = yes +ssl_server_cert_file = <%= certs.fallbackCert %> +ssl_server_key_file = <%= certs.fallbackKey %> +auth_allow_cleartext = no +<%_ certs.domains.forEach((d) => { _%> +local_name <%= d.name %> { + ssl_server_cert_file = <%= d.cert %> + ssl_server_key_file = <%= d.key %> +} +<%_ }) _%> + +# Accounts live in the manager's database (mail_accounts_v view, read-only +# mail_dovecot user). Hash scheme is {ARGON2ID} (p=1). +sql_driver = <%= db.dovecotDriver %> +<%= db.dovecotDriver %> <%= db.dovecotHost %> { +<%_ if (db.dovecotParams.dbname) { _%> + dbname = <%= db.dovecotParams.dbname %> +<%_ } _%> + user = <%= db.dovecotUser %> +<%_ if (db.dovecotPassword) { _%> + password = <%= db.dovecotPassword %> +<%_ } _%> +} + +# Submission (protocol smtp, via Postfix SASL) requires can_send; IMAP stays +# readable for accounts on domains that can no longer send. +passdb sql { + query = SELECT address AS user, password FROM mail_accounts_v WHERE address = '%{user | lower}' AND (can_send OR '%{protocol}' != 'smtp') +} +userdb sql { + query = SELECT home, 'vmail' AS uid, 'vmail' AS gid, quota_bytes FROM mail_accounts_v WHERE address = '%{user | lower}' +} + +# Cache hits keep per-message SASL cheap; "instant disable" is bounded by +# this TTL. +auth_cache_size = 1M +auth_cache_ttl = 1 min +auth_cache_negative_ttl = 1 min + +mail_plugins { + quota = yes +} +quota_name = User quota +quota_storage_size = %{userdb:quota_bytes}B + +# Sockets for Postfix: SASL auth, LMTP delivery, and the quota-status +# policy service (rejects unknown / over-quota recipients at RCPT). +service auth { + unix_listener /var/spool/postfix/private/auth { + mode = 0660 + user = postfix + group = postfix + } +} +service lmtp { + unix_listener /var/spool/postfix/private/dovecot-lmtp { + mode = 0600 + user = postfix + group = postfix + } +} +service quota-status { + executable = quota-status -p postfix + unix_listener /var/spool/postfix/private/quota-status { + mode = 0660 + user = postfix + group = postfix + } +} diff --git a/agent/templates/mail/opendkim-keytable.ejs b/agent/templates/mail/opendkim-keytable.ejs new file mode 100644 index 00000000..a788e671 --- /dev/null +++ b/agent/templates/mail/opendkim-keytable.ejs @@ -0,0 +1,3 @@ +<%_ mail.dkim.forEach((k) => { _%> +<%= k.selector %>._domainkey.<%= k.domain %> <%= k.domain %>:<%= k.selector %>:<%= stateDir %>/dkim/<%= k.domain %>.private +<%_ }) _%> diff --git a/agent/templates/mail/opendkim-signingtable.ejs b/agent/templates/mail/opendkim-signingtable.ejs new file mode 100644 index 00000000..087621f6 --- /dev/null +++ b/agent/templates/mail/opendkim-signingtable.ejs @@ -0,0 +1,3 @@ +<%_ mail.dkim.forEach((k) => { _%> +*@<%= k.domain %> <%= k.selector %>._domainkey.<%= k.domain %> +<%_ }) _%> diff --git a/agent/templates/mail/opendkim.conf.ejs b/agent/templates/mail/opendkim.conf.ejs new file mode 100644 index 00000000..a368d4ef --- /dev/null +++ b/agent/templates/mail/opendkim.conf.ejs @@ -0,0 +1,21 @@ +# Managed by opensource-agent (mail service group) — do not edit. +Syslog yes +UMask 007 +Mode s +# Unix socket inside the Postfix chroot (no inet port). Running with the +# postfix group (UserID user:group form, opendkim.conf(5)) makes the socket +# 0660 opendkim:postfix under UMask 007, so the chrooted smtpd connects +# without postfix joining the opendkim group. +Socket <%= opendkimSocket %> +UserID opendkim:postfix +PidFile /run/opendkim/opendkim.pid + +Canonicalization relaxed/simple +# RFC 8058: the signature must cover List-Unsubscribe(-Post). Oversigning +# both signs them when present and prevents later injection. +OversignHeaders From,List-Unsubscribe,List-Unsubscribe-Post + +# Tables and keys are agent-managed state under $STATE_DIRECTORY/mail. +KeyTable <%= stateDir %>/opendkim-keytable +SigningTable refile:<%= stateDir %>/opendkim-signingtable +InternalHosts 127.0.0.0/8 diff --git a/agent/templates/mail/postfix-main.cf.ejs b/agent/templates/mail/postfix-main.cf.ejs new file mode 100644 index 00000000..9bc554af --- /dev/null +++ b/agent/templates/mail/postfix-main.cf.ejs @@ -0,0 +1,63 @@ +# Managed by opensource-agent (mail service group) — do not edit. +compatibility_level = 3.6 + +<%_ if (mail.hostname) { _%> +myhostname = <%= mail.hostname %> +<%_ } _%> +smtpd_banner = $myhostname ESMTP +biff = no +readme_directory = no +append_dot_mydomain = no + +# This host serves virtual mailboxes only — ALL delivery goes to Dovecot +# over LMTP (virtual_transport below). The empty local domain class merely +# disables Postfix's own local(8) delivery agent. +mydestination = +mynetworks = 127.0.0.0/8 [::1]/128 +alias_maps = +alias_database = +local_recipient_maps = +local_transport = error:5.1.1 local delivery disabled + +smtpd_helo_required = yes +disable_vrfy_command = yes +message_size_limit = <%= mail.messageSizeLimitMb * 1048576 %> + +# TLS: per-domain certs via SNI (compiled with `postmap -F`), shared with +# nginx under /etc/ssl. Fallback chain covers domains without a cert yet. +tls_server_sni_maps = hash:/etc/postfix/sni_map +smtpd_tls_chain_files = <%= certs.fallbackKey %>, <%= certs.fallbackCert %> +smtpd_tls_security_level = may +smtp_tls_security_level = may + +# Inbound (:25): only mailboxes that exist on can-receive domains, checked +# live from SQL so unknown recipients are rejected at RCPT (no backscatter). +# Dovecot's quota-status policy service additionally rejects over-quota. +virtual_mailbox_domains = texthash:/etc/postfix/vdomains +virtual_mailbox_maps = <%= db.prefix %>:/etc/postfix/sql/mailboxes.cf +virtual_transport = lmtp:unix:private/dovecot-lmtp +smtpd_relay_restrictions = permit_mynetworks, reject_unauth_destination +smtpd_recipient_restrictions = reject_unauth_destination, check_policy_service unix:private/quota-status + +# Sender ownership: submission enforces sender == SASL login (live from SQL). +smtpd_sender_login_maps = <%= db.prefix %>:/etc/postfix/sql/senders.cf + +# Submission policy, referenced from master.cf. Milters run pre-queue and in +# order: opensource-mail-helper rejects suppressed recipients at RCPT and +# injects per-recipient RFC 8058 List-Unsubscribe headers, then OpenDKIM +# signs — so the signature covers the added headers. Both listen on unix +# sockets inside the Postfix chroot. +submission_recipient_restrictions = permit_sasl_authenticated, reject +submission_milters = <%= helperMilter %>, <%= opendkimMilter %> +milter_default_action = tempfail +<%_ if (mail.relayhost) { _%> + +# Outbound smarthost — SPF for sending domains must include the relay +# (mail_spf_include setting). +relayhost = <%= mail.relayhost.host %> +<%_ if (mail.relayhost.username) { _%> +smtp_sasl_auth_enable = yes +smtp_sasl_password_maps = texthash:/etc/postfix/sasl_passwd +smtp_sasl_security_options = noanonymous +<%_ } _%> +<%_ } _%> diff --git a/agent/templates/mail/postfix-master.cf.ejs b/agent/templates/mail/postfix-master.cf.ejs new file mode 100644 index 00000000..89892cd3 --- /dev/null +++ b/agent/templates/mail/postfix-master.cf.ejs @@ -0,0 +1,60 @@ +# Managed by opensource-agent (mail service group) — do not edit. +# +# master.cf is Postfix's only mechanism for per-service parameter overrides +# (there are no per-service include files) — each -o below switches a +# setting on for the submission services only. The multi-value policy lists +# themselves (restrictions, milters) are defined once in main.cf as named +# parameters and referenced here as $name, keeping every -o single-valued. +# +# service type private unpriv chroot wakeup maxproc command + args +smtp inet n - y - - smtpd + +# Submission (STARTTLS on 587, implicit TLS on 465): AUTH only after TLS, +# sender must match the SASL login, and the pre-queue milters (mail-helper, +# then OpenDKIM) enforce suppressions at RCPT and sign with the injected +# List-Unsubscribe headers covered. +submission inet n - y - - smtpd + -o syslog_name=postfix/submission + -o smtpd_tls_security_level=encrypt + -o smtpd_tls_auth_only=yes + -o smtpd_sasl_auth_enable=yes + -o smtpd_sasl_type=dovecot + -o smtpd_sasl_path=private/auth + -o smtpd_sender_restrictions=reject_sender_login_mismatch + -o smtpd_relay_restrictions=$submission_recipient_restrictions + -o smtpd_recipient_restrictions=$submission_recipient_restrictions + -o smtpd_milters=$submission_milters +smtps inet n - y - - smtpd + -o syslog_name=postfix/smtps + -o smtpd_tls_wrappermode=yes + -o smtpd_sasl_auth_enable=yes + -o smtpd_sasl_type=dovecot + -o smtpd_sasl_path=private/auth + -o smtpd_sender_restrictions=reject_sender_login_mismatch + -o smtpd_relay_restrictions=$submission_recipient_restrictions + -o smtpd_recipient_restrictions=$submission_recipient_restrictions + -o smtpd_milters=$submission_milters + +pickup unix n - y 60 1 pickup +cleanup unix n - y - 0 cleanup +qmgr unix n - n 300 1 qmgr +tlsmgr unix - - y 1000? 1 tlsmgr +rewrite unix - - y - - trivial-rewrite +bounce unix - - y - 0 bounce +defer unix - - y - 0 bounce +trace unix - - y - 0 bounce +verify unix - - y - 1 verify +flush unix n - y 1000? 0 flush +proxymap unix - - n - - proxymap +proxywrite unix - - n - 1 proxymap +smtp unix - - y - - smtp +relay unix - - y - - smtp + -o syslog_name=postfix/$service_name +showq unix n - y - - showq +error unix - - y - - error +retry unix - - y - - error +discard unix - - y - - discard +lmtp unix - - y - - lmtp +anvil unix - - y - 1 anvil +scache unix - - y - 1 scache +postlog unix-dgram n - n - 1 postlogd diff --git a/agent/templates/mail/postfix-sni-map.ejs b/agent/templates/mail/postfix-sni-map.ejs new file mode 100644 index 00000000..e2f05bee --- /dev/null +++ b/agent/templates/mail/postfix-sni-map.ejs @@ -0,0 +1,5 @@ +<%_ /* SNI map: one line per domain with cert material on disk. Compiled with + `postmap -F` (postWrite), which stores the key+chain in the .db. */ -%> +<%_ certs.domains.forEach((d) => { _%> +<%= d.name %> <%= d.key %> <%= d.cert %> +<%_ }) _%> diff --git a/agent/templates/mail/postfix-sql-mailboxes.cf.ejs b/agent/templates/mail/postfix-sql-mailboxes.cf.ejs new file mode 100644 index 00000000..376714ea --- /dev/null +++ b/agent/templates/mail/postfix-sql-mailboxes.cf.ejs @@ -0,0 +1,9 @@ +# Managed by opensource-agent — Postfix virtual_mailbox_maps (live SQL). +# Existence check only: unknown recipients are rejected at RCPT. +hosts = <%= db.hosts %> +user = <%= db.user %> +<%_ if (db.password) { _%> +password = <%= db.password %> +<%_ } _%> +dbname = <%= db.dbname %> +query = SELECT 1 FROM mail_recipients_v WHERE recipient = '%s' diff --git a/agent/templates/mail/postfix-sql-senders.cf.ejs b/agent/templates/mail/postfix-sql-senders.cf.ejs new file mode 100644 index 00000000..d567dc0c --- /dev/null +++ b/agent/templates/mail/postfix-sql-senders.cf.ejs @@ -0,0 +1,8 @@ +# Managed by opensource-agent — Postfix smtpd_sender_login_maps (live SQL). +hosts = <%= db.hosts %> +user = <%= db.user %> +<%_ if (db.password) { _%> +password = <%= db.password %> +<%_ } _%> +dbname = <%= db.dbname %> +query = SELECT login FROM mail_senders_v WHERE sender = '%s' diff --git a/agent/templates/mail/postfix-vdomains.ejs b/agent/templates/mail/postfix-vdomains.ejs new file mode 100644 index 00000000..5ff741e1 --- /dev/null +++ b/agent/templates/mail/postfix-vdomains.ejs @@ -0,0 +1,4 @@ +<%_ /* Domains Postfix accepts inbound mail for (texthash: "domain ok"). */ -%> +<%_ mail.receiveDomains.forEach((domain) => { _%> +<%= domain %> ok +<%_ }) _%> diff --git a/agent/test/config.test.js b/agent/test/config.test.js new file mode 100644 index 00000000..11f4c091 --- /dev/null +++ b/agent/test/config.test.js @@ -0,0 +1,32 @@ +/** + * AGENT_SERVICES parsing (node --test): default groups, explicit lists, + * dedupe, and rejection of unknown groups. + */ + +const test = require('node:test'); +const assert = require('node:assert'); + +const { loadConfig } = require('../dist/config.js'); + +const BASE_ENV = { SITE_ID: '1', MANAGER_URL: 'http://localhost:3000' }; + +test('unset AGENT_SERVICES means nginx,dnsmasq', () => { + assert.deepEqual(loadConfig({ ...BASE_ENV }).services, ['nginx', 'dnsmasq']); + assert.deepEqual(loadConfig({ ...BASE_ENV, AGENT_SERVICES: ' ' }).services, ['nginx', 'dnsmasq']); +}); + +test('explicit list is parsed, trimmed, lowercased, and deduped', () => { + const cfg = loadConfig({ ...BASE_ENV, AGENT_SERVICES: ' Nginx, dnsmasq ,mail, mail ' }); + assert.deepEqual(cfg.services, ['nginx', 'dnsmasq', 'mail']); +}); + +test('mail-only host is allowed', () => { + assert.deepEqual(loadConfig({ ...BASE_ENV, AGENT_SERVICES: 'mail' }).services, ['mail']); +}); + +test('unknown groups are rejected', () => { + assert.throws( + () => loadConfig({ ...BASE_ENV, AGENT_SERVICES: 'nginx,launchpad' }), + /unknown service group "launchpad"/, + ); +}); diff --git a/agent/test/mail-template.test.js b/agent/test/mail-template.test.js new file mode 100644 index 00000000..2e5e4cdd --- /dev/null +++ b/agent/test/mail-template.test.js @@ -0,0 +1,131 @@ +/** + * Mail template rendering tests (node --test): assert the directives the + * #67 guardrails depend on — real validation happens on-host via + * `postfix check` / `doveconf -n` / `opendkim -n` at apply time. + */ + +const test = require('node:test'); +const assert = require('node:assert'); +const path = require('path'); +const ejs = require('ejs'); + +const TEMPLATES = path.join(__dirname, '..', 'templates'); + +const context = { + mail: { + hostname: 'mail.example.com', + relayhost: null, + messageSizeLimitMb: 25, + defaultQuotaMb: 1024, + dkim: [ + { domain: 'example.com', selector: 'os20261008', privateKey: 'PEM' }, + { domain: 'other.example', selector: 'os20261001', privateKey: 'PEM2' }, + ], + receiveDomains: ['example.com'], + mailboxIds: ['a-1', 'b-2'], + db: { + dialect: 'postgres', host: null, port: null, database: 'cc', + dovecotUser: 'mail_dovecot', postfixUser: 'mail_postfix', + }, + unsubscribe: { keys: [] }, + }, + stateDir: '/var/lib/opensource-agent/mail', + helperMilter: 'unix:mail-helper/milter.sock', + opendkimMilter: 'unix:opendkim/opendkim.sock', + opendkimSocket: 'local:/var/spool/postfix/opendkim/opendkim.sock', + certs: { + fallbackCert: '/etc/ssl/certs/mail.example.com.crt', + fallbackKey: '/etc/ssl/private/mail.example.com.key', + domains: [{ name: 'example.com', cert: '/etc/ssl/certs/example.com.crt', key: '/etc/ssl/private/example.com.key' }], + }, + db: { + prefix: 'pgsql', + hosts: 'unix:/var/run/postgresql', + user: 'mail_postfix', + password: null, + dbname: 'cc', + dovecotDriver: 'pgsql', + dovecotHost: '/var/run/postgresql', + dovecotParams: { dbname: 'cc' }, + dovecotUser: 'mail_dovecot', + dovecotPassword: null, + }, +}; + +function render(template, data = context) { + return ejs.renderFile(path.join(TEMPLATES, template), data); +} + +test('postfix main.cf: live recipient map, sender ownership, size limit, SNI, milter policy', async () => { + const out = await render('mail/postfix-main.cf.ejs'); + assert.match(out, /myhostname = mail\.example\.com/); + assert.match(out, /message_size_limit = 26214400/); + assert.match(out, /virtual_mailbox_domains = texthash:\/etc\/postfix\/vdomains/); + // Unknown recipients must be rejected at RCPT (live SQL), not via static:x. + assert.match(out, /virtual_mailbox_maps = pgsql:\/etc\/postfix\/sql\/mailboxes\.cf/); + assert.doesNotMatch(out, /static:x/); + assert.match(out, /check_policy_service unix:private\/quota-status/); + assert.match(out, /smtpd_sender_login_maps = pgsql:\/etc\/postfix\/sql\/senders\.cf/); + assert.match(out, /tls_server_sni_maps = hash:\/etc\/postfix\/sni_map/); + // Helper milter first, OpenDKIM second — the signature covers added headers. + assert.match(out, /submission_milters = unix:mail-helper\/milter\.sock, unix:opendkim\/opendkim\.sock/); + assert.match(out, /submission_recipient_restrictions = permit_sasl_authenticated, reject/); + assert.doesNotMatch(out, /relayhost =/); +}); + +test('postfix main.cf: relayhost with credentials', async () => { + const out = await render('mail/postfix-main.cf.ejs', { + ...context, + mail: { ...context.mail, relayhost: { host: '[relay.net]:587', username: 'u', password: 'p' } }, + }); + assert.match(out, /relayhost = \[relay\.net\]:587/); + assert.match(out, /smtp_sasl_password_maps = texthash:\/etc\/postfix\/sasl_passwd/); +}); + +test('postfix master.cf: TLS-only AUTH submission referencing main.cf policy, no inet filters', async () => { + const out = await render('mail/postfix-master.cf.ejs'); + assert.match(out, /submission[\s\S]*?smtpd_tls_security_level=encrypt/); + assert.match(out, /smtpd_sender_restrictions=reject_sender_login_mismatch/); + assert.match(out, /-o smtpd_milters=\$submission_milters/); + assert.match(out, /-o smtpd_recipient_restrictions=\$submission_recipient_restrictions/); + assert.match(out, /smtps[\s\S]*?smtpd_tls_wrappermode=yes/); + // The milter architecture removed the content filter, reinjection + // listener, and policy service — no localhost inet plumbing remains. + assert.doesNotMatch(out, /127\.0\.0\.1:\d+/); + assert.doesNotMatch(out, /content_filter/); +}); + +test('dovecot.conf: SQL auth with the can_send smtp gate, per-domain certs, postfix sockets', async () => { + const out = await render('mail/dovecot.conf.ejs'); + assert.match(out, /dovecot_config_version = 2\.4\.0/); + assert.match(out, /sql_driver = pgsql/); + assert.match(out, /can_send OR '%\{protocol\}' != 'smtp'/); + assert.match(out, /local_name example\.com \{/); + assert.match(out, /unix_listener \/var\/spool\/postfix\/private\/auth/); + assert.match(out, /unix_listener \/var\/spool\/postfix\/private\/dovecot-lmtp/); + assert.match(out, /quota-status -p postfix/); + assert.match(out, /auth_allow_cleartext = no/); +}); + +test('opendkim: unix socket in the postfix chroot, state-dir tables, oversigned List-Unsubscribe', async () => { + const conf = await render('mail/opendkim.conf.ejs'); + assert.match(conf, /Socket\s+local:\/var\/spool\/postfix\/opendkim\/opendkim\.sock/); + assert.match(conf, /OversignHeaders\s+From,List-Unsubscribe,List-Unsubscribe-Post/); + assert.match(conf, /KeyTable\s+\/var\/lib\/opensource-agent\/mail\/opendkim-keytable/); + + const keytable = await render('mail/opendkim-keytable.ejs'); + assert.match(keytable, /os20261008\._domainkey\.example\.com example\.com:os20261008:\/var\/lib\/opensource-agent\/mail\/dkim\/example\.com\.private/); + + const signing = await render('mail/opendkim-signingtable.ejs'); + assert.match(signing, /\*@example\.com os20261008\._domainkey\.example\.com/); +}); + +test('sql maps, vdomains, and sni_map list the right entries', async () => { + assert.match(await render('mail/postfix-vdomains.ejs'), /^example\.com ok$/m); + assert.match( + await render('mail/postfix-sni-map.ejs'), + /^example\.com \/etc\/ssl\/private\/example\.com\.key \/etc\/ssl\/certs\/example\.com\.crt$/m, + ); + assert.match(await render('mail/postfix-sql-mailboxes.cf.ejs'), /SELECT 1 FROM mail_recipients_v WHERE recipient = '%s'/); + assert.match(await render('mail/postfix-sql-senders.cf.ejs'), /SELECT login FROM mail_senders_v WHERE sender = '%s'/); +}); diff --git a/mie-opensource-landing/docs/developers/agent.md b/mie-opensource-landing/docs/developers/agent.md index 6b350a03..7739278e 100644 --- a/mie-opensource-landing/docs/developers/agent.md +++ b/mie-opensource-landing/docs/developers/agent.md @@ -89,20 +89,35 @@ Read from the process environment (systemd loads `/etc/environment`). Set via co | `SITE_ID` | Yes | Numeric site ID from the manager | | `MANAGER_URL` | Yes | Base URL of the manager (e.g., `http://192.168.1.10:3000`) | | `API_KEY` | No | Admin API key for remote agents. Not needed on the manager (localhost is trusted). | +| `AGENT_SERVICES` | No | Comma-separated service groups: `nginx`, `dnsmasq`, `mail`. Default `nginx,dnsmasq`. | +| `MAIL_POSTFIX_DB_PASSWORD` | No | DB password for the `mail_postfix` role (mail hosts without local socket auth) | +| `MAIL_DOVECOT_DB_PASSWORD` | No | DB password for the `mail_dovecot` role (mail hosts without local socket auth) | +| `MAIL_HELPER_DIR`, `MAIL_HELPER_SOCKET_DIR`, `OPENDKIM_SOCKET_DIR` | No | Mail helper install dir and milter socket dirs (defaults match the packages) | +| `VMAIL_DIR`, `GC_STATE_FILE`, `GC_RETENTION_MS` | No | Maildir root (default `/var/vmail`), mailbox GC state file, and GC retention (default 30 days) | | `STATE_DIRECTORY` | No | State directory, set by systemd via `StateDirectory=` (default `/var/lib/opensource-agent`) | +On mail hosts `MANAGER_URL` must be the manager's **public** URL — one-click unsubscribe links use it as their base. + The agent Dockerfile defaults to `SITE_ID=1` and `MANAGER_URL=http://localhost:3000` so the manager container works without configuration. ## Managed Services -| Service | Files | Test | Reload | -|---------|-------|------|--------| -| nginx | `/etc/nginx/nginx.conf` | `nginx -t` | `systemctl reload-or-restart nginx` | -| dnsmasq | `/etc/dnsmasq.conf`, `/var/lib/dnsmasq/{dhcp-hosts,hosts,dhcp-opts,servers}` | `dnsmasq --test` | restart when `/etc/dnsmasq.conf` changed, otherwise SIGHUP | +Services are organized in groups enabled via `AGENT_SERVICES`. The check-in reports the enabled groups plus any missing binaries; the manager refuses the mail-host claim while anything is missing. + +| Group | Service | Files | Test | Reload | +|-------|---------|-------|------|--------| +| nginx | nginx | `/etc/nginx/nginx.conf` | `nginx -t` | `systemctl reload-or-restart nginx` | +| dnsmasq | dnsmasq | `/etc/dnsmasq.conf`, `/var/lib/dnsmasq/{dhcp-hosts,hosts,dhcp-opts,servers}` | `dnsmasq --test` | restart when `/etc/dnsmasq.conf` changed, otherwise SIGHUP | +| mail | opendkim | `/etc/opendkim.conf`, `$STATE_DIRECTORY/mail/{opendkim-keytable,opendkim-signingtable,dkim/*.private}` | `opendkim -n` | restart | +| mail | dovecot | `/etc/dovecot/dovecot.conf` | `doveconf -n` | reload + `doveadm auth cache flush` | +| mail | postfix (status: `postfix@-`) | `/etc/postfix/{main.cf,master.cf,vdomains,sni_map,sql/senders.cf,sql/mailboxes.cf}` | `postfix check` (after `postmap -F` postWrite) | reload-or-restart | +| mail | opensource-mail-helper | `/opt/opensource-server/mail-helper/config.local.json` | — | restart | + +The mail group renders only on the agent holding the mail-host claim (the `mail` section of the snapshot). The helper is its own component/package ([mail-helper/](https://github.com/mieweb/opensource-server/tree/main/mail-helper)) — a Postfix milter on a unix socket inside the Postfix chroot, ordered before OpenDKIM so the DKIM signature covers the List-Unsubscribe headers it injects. Taking the claim provisions directories (system users come from the helper package) and enables + starts the units; losing it — or removing `mail` from `AGENT_SERVICES` — stops and disables them, leaving configs and `/var/vmail` in place. Maildirs of deleted accounts are garbage-collected after 30 days. -Apply flow per service: render templates, skip if nothing changed, stage new files, run the test command, roll back on failure, then reload. Failures are reported as `lastApply: "failure"` at the next check-in. +Apply flow per service: render templates, reconcile file mode/owner even when contents are unchanged, stage new files, run post-write commands, run the test command, roll back on failure (re-running post-write commands against the restored sources), then reload. Failures are reported as `lastApply: "failure"` at the next check-in. -To add a managed service, add a `ManagedService` entry in [`agent/src/apply.ts`](https://github.com/mieweb/opensource-server/blob/main/agent/src/apply.ts) plus its template(s) under `agent/templates/`, and extend the config snapshot in `create-a-container/utils/agent-config.js`. +To add a managed service, add a `ManagedService` entry in [`agent/src/apply.ts`](https://github.com/mieweb/opensource-server/blob/main/agent/src/apply.ts) (or `agent/src/mail.ts` for the mail group) plus its template(s) under `agent/templates/`, and extend the config snapshot in `create-a-container/utils/agent-config.js`. ## Running Manually