From 346864c9b42369f5e896932f69bad47e49c4ce24 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 09:32:48 -0700 Subject: [PATCH 01/19] mail: DB schema, SQL views, and roles script (#67) New tables DkimKeys, MailAccounts, MailSuppressions, MailUnsubscribeKeys; mail enablement/verification columns on ExternalDomains; agent service reporting + single-mail-host claim columns on Agents (partial unique index on Postgres/SQLite). mail_accounts_v / mail_senders_v / mail_suppressions_v views are the stable contract Dovecot and Postfix read live; bin/setup-mail-db-roles.sh creates the read-only mail_dovecot/mail_postfix DB users and grants. --- create-a-container/bin/setup-mail-db-roles.sh | 128 ++++++++++++++++++ ...000000-add-external-domain-mail-columns.js | 30 ++++ .../20261008000001-create-dkim-keys.js | 30 ++++ .../20261008000002-create-mail-accounts.js | 38 ++++++ .../20261008000003-add-agent-mail-fields.js | 56 ++++++++ ...20261008000004-create-mail-suppressions.js | 28 ++++ ...1008000005-create-mail-unsubscribe-keys.js | 21 +++ .../20261008000006-create-mail-views.js | 24 ++++ create-a-container/models/agent.js | 29 ++++ create-a-container/models/dkim-key.js | 57 ++++++++ create-a-container/models/external-domain.js | 35 +++++ create-a-container/models/mail-account.js | 94 +++++++++++++ create-a-container/models/mail-suppression.js | 50 +++++++ .../models/mail-unsubscribe-key.js | 41 ++++++ create-a-container/utils/mail-views.js | 68 ++++++++++ 15 files changed, 729 insertions(+) create mode 100644 create-a-container/bin/setup-mail-db-roles.sh create mode 100644 create-a-container/migrations/20261008000000-add-external-domain-mail-columns.js create mode 100644 create-a-container/migrations/20261008000001-create-dkim-keys.js create mode 100644 create-a-container/migrations/20261008000002-create-mail-accounts.js create mode 100644 create-a-container/migrations/20261008000003-add-agent-mail-fields.js create mode 100644 create-a-container/migrations/20261008000004-create-mail-suppressions.js create mode 100644 create-a-container/migrations/20261008000005-create-mail-unsubscribe-keys.js create mode 100644 create-a-container/migrations/20261008000006-create-mail-views.js create mode 100644 create-a-container/models/dkim-key.js create mode 100644 create-a-container/models/mail-account.js create mode 100644 create-a-container/models/mail-suppression.js create mode 100644 create-a-container/models/mail-unsubscribe-key.js create mode 100644 create-a-container/utils/mail-views.js diff --git a/create-a-container/bin/setup-mail-db-roles.sh b/create-a-container/bin/setup-mail-db-roles.sh new file mode 100644 index 00000000..75bfb0aa --- /dev/null +++ b/create-a-container/bin/setup-mail-db-roles.sh @@ -0,0 +1,128 @@ +#!/bin/bash +# setup-mail-db-roles.sh — create the read-only DB users the mail stack uses. +# +# mail_dovecot SELECT on mail_accounts_v (passdb/userdb) +# mail_postfix SELECT on mail_senders_v, mail_suppressions_v (sender/suppression maps) +# +# The views are created by migration 20261008000006-create-mail-views.js; this +# script only creates the users and grants. Passwords are generated and +# appended to the env file as MAIL_DOVECOT_DB_PASSWORD / MAIL_POSTFIX_DB_PASSWORD +# so the agent can template them into the Dovecot/Postfix SQL config. When the +# DB is local to the mail host, socket auth (Postgres peer / MariaDB +# unix_socket) also works and the passwords are unused. +# +# Runs idempotently from container-creator-init.service and the package +# postinst. Supports: +# --print-sql print the SQL for a DBA instead of executing it +# --rotate regenerate passwords even if already present +# +# Reads DATABASE_DIALECT and connection settings from ENV_FILE +# (default /etc/default/container-creator). +set -euo pipefail + +ENV_FILE="${ENV_FILE:-/etc/default/container-creator}" +PRINT_SQL=0 +ROTATE=0 +for arg in "$@"; do + case "$arg" in + --print-sql) PRINT_SQL=1 ;; + --rotate) ROTATE=1 ;; + *) echo "usage: $0 [--print-sql] [--rotate]" >&2; exit 2 ;; + esac +done + +if [ -f "$ENV_FILE" ]; then + # shellcheck disable=SC1090 + . "$ENV_FILE" +fi + +DIALECT="${DATABASE_DIALECT:-sqlite}" +case "$DIALECT" in + postgres|mysql|mariadb) ;; + *) + echo "setup-mail-db-roles: DATABASE_DIALECT=$DIALECT cannot host mail (needs postgres or mysql/mariadb); nothing to do." + exit 0 + ;; +esac + +# Read an existing password out of the env file, if any. +get_env_var() { grep -E "^$1=" "$ENV_FILE" 2>/dev/null | tail -n1 | cut -d= -f2-; } + +# Insert-or-replace VAR=value in the env file. +set_env_var() { + local var="$1" value="$2" + touch "$ENV_FILE" + chmod 600 "$ENV_FILE" + if grep -qE "^${var}=" "$ENV_FILE"; then + sed -i "s|^${var}=.*|${var}=${value}|" "$ENV_FILE" + else + echo "${var}=${value}" >> "$ENV_FILE" + fi +} + +gen_password() { openssl rand -base64 32 | tr -d '/+=' | head -c 32; } + +DOVECOT_PW="$(get_env_var MAIL_DOVECOT_DB_PASSWORD || true)" +POSTFIX_PW="$(get_env_var MAIL_POSTFIX_DB_PASSWORD || true)" +if [ "$ROTATE" = 1 ] || [ -z "$DOVECOT_PW" ]; then DOVECOT_PW="$(gen_password)"; fi +if [ "$ROTATE" = 1 ] || [ -z "$POSTFIX_PW" ]; then POSTFIX_PW="$(gen_password)"; fi + +sql_postgres() { + cat </dev/null 2>&1; then + echo "$SQL" | su -s /bin/sh postgres -c "psql -v ON_ERROR_STOP=1 -d '${POSTGRES_DATABASE:-container_creator}'" + else + echo "setup-mail-db-roles: psql not found — run with --print-sql and apply as a DBA." >&2 + exit 1 + fi +else + if command -v mysql >/dev/null 2>&1; then + echo "$SQL" | mysql + else + echo "setup-mail-db-roles: mysql client not found — run with --print-sql and apply as a DBA." >&2 + exit 1 + fi +fi + +set_env_var MAIL_DOVECOT_DB_PASSWORD "$DOVECOT_PW" +set_env_var MAIL_POSTFIX_DB_PASSWORD "$POSTFIX_PW" +echo "setup-mail-db-roles: mail_dovecot and mail_postfix are ready; passwords stored in $ENV_FILE." diff --git a/create-a-container/migrations/20261008000000-add-external-domain-mail-columns.js b/create-a-container/migrations/20261008000000-add-external-domain-mail-columns.js new file mode 100644 index 00000000..69bf349e --- /dev/null +++ b/create-a-container/migrations/20261008000000-add-external-domain-mail-columns.js @@ -0,0 +1,30 @@ +'use strict'; + +/** Mail enablement + DNS verification state on external domains (issue #67). */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('ExternalDomains', 'mailEnabled', { + type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false, + }); + await queryInterface.addColumn('ExternalDomains', 'mailDnsVerified', { + type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false, + }); + await queryInterface.addColumn('ExternalDomains', 'mailMxVerified', { + type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false, + }); + await queryInterface.addColumn('ExternalDomains', 'mailDnsCheckedAt', { + type: Sequelize.DATE, allowNull: true, + }); + await queryInterface.addColumn('ExternalDomains', 'mailDnsCheckResult', { + type: Sequelize.JSON, allowNull: true, + }); + }, + + async down(queryInterface) { + await queryInterface.removeColumn('ExternalDomains', 'mailDnsCheckResult'); + await queryInterface.removeColumn('ExternalDomains', 'mailDnsCheckedAt'); + await queryInterface.removeColumn('ExternalDomains', 'mailMxVerified'); + await queryInterface.removeColumn('ExternalDomains', 'mailDnsVerified'); + await queryInterface.removeColumn('ExternalDomains', 'mailEnabled'); + }, +}; diff --git a/create-a-container/migrations/20261008000001-create-dkim-keys.js b/create-a-container/migrations/20261008000001-create-dkim-keys.js new file mode 100644 index 00000000..92ed3740 --- /dev/null +++ b/create-a-container/migrations/20261008000001-create-dkim-keys.js @@ -0,0 +1,30 @@ +'use strict'; + +/** DKIM key pairs per external domain (issue #67). */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.createTable('DkimKeys', { + id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, allowNull: false }, + externalDomainId: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: 'ExternalDomains', key: 'id' }, + onDelete: 'CASCADE', + }, + selector: { type: Sequelize.STRING(63), allowNull: false }, + privateKey: { type: Sequelize.TEXT, allowNull: false }, + publicKey: { type: Sequelize.TEXT, allowNull: false }, + status: { type: Sequelize.ENUM('active', 'retired'), allowNull: false, defaultValue: 'active' }, + createdAt: { type: Sequelize.DATE, allowNull: false }, + updatedAt: { type: Sequelize.DATE, allowNull: false }, + }); + await queryInterface.addIndex('DkimKeys', ['externalDomainId', 'selector'], { unique: true }); + }, + + async down(queryInterface) { + await queryInterface.dropTable('DkimKeys'); + if (queryInterface.sequelize.getDialect() === 'postgres') { + await queryInterface.sequelize.query('DROP TYPE IF EXISTS "enum_DkimKeys_status";'); + } + }, +}; diff --git a/create-a-container/migrations/20261008000002-create-mail-accounts.js b/create-a-container/migrations/20261008000002-create-mail-accounts.js new file mode 100644 index 00000000..21e816ab --- /dev/null +++ b/create-a-container/migrations/20261008000002-create-mail-accounts.js @@ -0,0 +1,38 @@ +'use strict'; + +/** Per-user email service accounts (issue #67). */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.createTable('MailAccounts', { + id: { type: Sequelize.UUID, primaryKey: true, allowNull: false }, + uidNumber: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: 'Users', key: 'uidNumber' }, + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + }, + externalDomainId: { + type: Sequelize.INTEGER, + allowNull: false, + references: { model: 'ExternalDomains', key: 'id' }, + onDelete: 'CASCADE', + }, + localPart: { type: Sequelize.STRING(64), allowNull: false }, + description: { type: Sequelize.STRING(255), allowNull: true }, + passwordHash: { type: Sequelize.STRING(255), allowNull: false }, + enabled: { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: true }, + quotaBytes: { type: Sequelize.BIGINT, allowNull: false }, + unsubscribeHeaders: { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: true }, + lastRotatedAt: { type: Sequelize.DATE, allowNull: true }, + createdAt: { type: Sequelize.DATE, allowNull: false }, + updatedAt: { type: Sequelize.DATE, allowNull: false }, + }); + await queryInterface.addIndex('MailAccounts', ['externalDomainId', 'localPart'], { unique: true }); + await queryInterface.addIndex('MailAccounts', ['uidNumber']); + }, + + async down(queryInterface) { + await queryInterface.dropTable('MailAccounts'); + }, +}; diff --git a/create-a-container/migrations/20261008000003-add-agent-mail-fields.js b/create-a-container/migrations/20261008000003-add-agent-mail-fields.js new file mode 100644 index 00000000..737e52e0 --- /dev/null +++ b/create-a-container/migrations/20261008000003-add-agent-mail-fields.js @@ -0,0 +1,56 @@ +'use strict'; + +/** + * Agent service reporting + the mail-host claim (issue #67). + * + * The partial unique index over the constant (1) allows at most ONE row in + * the whole table with a non-null mailHostSince — the single-mail-host + * invariant enforced at the DB level on Postgres and SQLite. MySQL/MariaDB + * have no partial indexes; there the claim is serialized in application code + * (utils/mail-host.js runs it inside a transaction). + */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn('Agents', 'enabledServices', { + type: Sequelize.JSON, allowNull: true, + }); + await queryInterface.addColumn('Agents', 'missingBinaries', { + type: Sequelize.JSON, allowNull: true, + }); + await queryInterface.addColumn('Agents', 'mailHostSince', { + type: Sequelize.DATE, allowNull: true, + }); + await queryInterface.addColumn('Agents', 'isLocal', { + type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false, + }); + await queryInterface.addColumn('Agents', 'apiKeyId', { + type: Sequelize.UUID, + allowNull: true, + references: { model: 'ApiKeys', key: 'id' }, + onDelete: 'SET NULL', + }); + + const dialect = queryInterface.sequelize.getDialect(); + if (dialect === 'postgres') { + await queryInterface.sequelize.query( + 'CREATE UNIQUE INDEX "agents_single_mail_host" ON "Agents" ((1)) WHERE "mailHostSince" IS NOT NULL;', + ); + } else if (dialect === 'sqlite') { + await queryInterface.sequelize.query( + 'CREATE UNIQUE INDEX `agents_single_mail_host` ON `Agents` (1) WHERE `mailHostSince` IS NOT NULL;', + ); + } + }, + + async down(queryInterface) { + const dialect = queryInterface.sequelize.getDialect(); + if (dialect === 'postgres' || dialect === 'sqlite') { + await queryInterface.sequelize.query('DROP INDEX IF EXISTS "agents_single_mail_host";'); + } + await queryInterface.removeColumn('Agents', 'apiKeyId'); + await queryInterface.removeColumn('Agents', 'isLocal'); + await queryInterface.removeColumn('Agents', 'mailHostSince'); + await queryInterface.removeColumn('Agents', 'missingBinaries'); + await queryInterface.removeColumn('Agents', 'enabledServices'); + }, +}; diff --git a/create-a-container/migrations/20261008000004-create-mail-suppressions.js b/create-a-container/migrations/20261008000004-create-mail-suppressions.js new file mode 100644 index 00000000..f70ed6ae --- /dev/null +++ b/create-a-container/migrations/20261008000004-create-mail-suppressions.js @@ -0,0 +1,28 @@ +'use strict'; + +/** One-click unsubscribe suppressions per mail account (issue #67). */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.createTable('MailSuppressions', { + id: { type: Sequelize.INTEGER, primaryKey: true, autoIncrement: true, allowNull: false }, + mailAccountId: { + type: Sequelize.UUID, + allowNull: false, + references: { model: 'MailAccounts', key: 'id' }, + onDelete: 'CASCADE', + }, + recipient: { type: Sequelize.STRING(320), allowNull: false }, + source: { type: Sequelize.ENUM('one-click', 'admin'), allowNull: false, defaultValue: 'one-click' }, + createdAt: { type: Sequelize.DATE, allowNull: false }, + updatedAt: { type: Sequelize.DATE, allowNull: false }, + }); + await queryInterface.addIndex('MailSuppressions', ['mailAccountId', 'recipient'], { unique: true }); + }, + + async down(queryInterface) { + await queryInterface.dropTable('MailSuppressions'); + if (queryInterface.sequelize.getDialect() === 'postgres') { + await queryInterface.sequelize.query('DROP TYPE IF EXISTS "enum_MailSuppressions_source";'); + } + }, +}; diff --git a/create-a-container/migrations/20261008000005-create-mail-unsubscribe-keys.js b/create-a-container/migrations/20261008000005-create-mail-unsubscribe-keys.js new file mode 100644 index 00000000..a4d14c86 --- /dev/null +++ b/create-a-container/migrations/20261008000005-create-mail-unsubscribe-keys.js @@ -0,0 +1,21 @@ +'use strict'; + +/** AES-256-GCM keys for unsubscribe tokens (issue #67). */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.createTable('MailUnsubscribeKeys', { + kid: { type: Sequelize.STRING(16), primaryKey: true, allowNull: false }, + secret: { type: Sequelize.STRING(64), allowNull: false }, + status: { type: Sequelize.ENUM('active', 'retired'), allowNull: false, defaultValue: 'active' }, + createdAt: { type: Sequelize.DATE, allowNull: false }, + updatedAt: { type: Sequelize.DATE, allowNull: false }, + }); + }, + + async down(queryInterface) { + await queryInterface.dropTable('MailUnsubscribeKeys'); + if (queryInterface.sequelize.getDialect() === 'postgres') { + await queryInterface.sequelize.query('DROP TYPE IF EXISTS "enum_MailUnsubscribeKeys_status";'); + } + }, +}; diff --git a/create-a-container/migrations/20261008000006-create-mail-views.js b/create-a-container/migrations/20261008000006-create-mail-views.js new file mode 100644 index 00000000..515a48a2 --- /dev/null +++ b/create-a-container/migrations/20261008000006-create-mail-views.js @@ -0,0 +1,24 @@ +'use strict'; + +const { createViewsSql, dropViewsSql } = require('../utils/mail-views'); + +/** + * The mail SQL views — the stable contract Dovecot/Postfix (or a + * self-managed MTA) read live. See utils/mail-views.js for the shape. + * Grants are NOT done here: bin/setup-mail-db-roles.sh creates the + * mail_dovecot / mail_postfix users and grants SELECT on the views. + */ +module.exports = { + async up(queryInterface) { + const dialect = queryInterface.sequelize.getDialect(); + for (const sql of createViewsSql(dialect)) { + await queryInterface.sequelize.query(sql); + } + }, + + async down(queryInterface) { + for (const sql of dropViewsSql()) { + await queryInterface.sequelize.query(sql); + } + }, +}; diff --git a/create-a-container/models/agent.js b/create-a-container/models/agent.js index 16d36740..297402bd 100644 --- a/create-a-container/models/agent.js +++ b/create-a-container/models/agent.js @@ -28,6 +28,35 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.JSON, allowNull: true }, + enabledServices: { + // Service groups the agent is configured to run (AGENT_SERVICES), + // e.g. ['nginx', 'dnsmasq', 'mail']. + type: DataTypes.JSON, + allowNull: true + }, + missingBinaries: { + // Binaries/SQL drivers the agent reports missing for its enabled + // services, e.g. ['postfix', 'dovecot']. Blocks the mail-host claim. + type: DataTypes.JSON, + allowNull: true + }, + mailHostSince: { + // The mail-host claim: at most one agent holds it (partial unique + // index on (1) WHERE mailHostSince IS NOT NULL — Postgres/SQLite). + type: DataTypes.DATE, + allowNull: true + }, + isLocal: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false + }, + apiKeyId: { + // Pinned on the first remote check-in; later remote check-ins must + // present the same key. Cleared via DELETE /agents/:id/api-key-pin. + type: DataTypes.UUID, + allowNull: true + }, lastCheckinAt: { type: DataTypes.DATE, allowNull: true diff --git a/create-a-container/models/dkim-key.js b/create-a-container/models/dkim-key.js new file mode 100644 index 00000000..5bf4c6ab --- /dev/null +++ b/create-a-container/models/dkim-key.js @@ -0,0 +1,57 @@ +'use strict'; +const { Model } = require('sequelize'); + +module.exports = (sequelize, DataTypes) => { + class DkimKey extends Model { + static associate(models) { + DkimKey.belongsTo(models.ExternalDomain, { + foreignKey: 'externalDomainId', + as: 'domain', + }); + } + } + + DkimKey.init({ + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + externalDomainId: { + type: DataTypes.INTEGER, + allowNull: false, + references: { model: 'ExternalDomains', key: 'id' }, + onDelete: 'CASCADE', + }, + selector: { + type: DataTypes.STRING(63), + allowNull: false, + comment: 'DKIM selector — the DNS record lives at ._domainkey.', + }, + privateKey: { + type: DataTypes.TEXT, + allowNull: false, + comment: 'PEM RSA private key. Never serialized in API responses; sent only in the mail-host snapshot and the admin DKIM export.', + }, + publicKey: { + type: DataTypes.TEXT, + allowNull: false, + comment: 'Base64 DER (SPKI) public key — the p= value of the DNS TXT record', + }, + status: { + type: DataTypes.ENUM('active', 'retired'), + allowNull: false, + defaultValue: 'active', + }, + }, { + sequelize, + modelName: 'DkimKey', + tableName: 'DkimKeys', + timestamps: true, + indexes: [ + { unique: true, fields: ['externalDomainId', 'selector'] }, + ], + }); + + return DkimKey; +}; diff --git a/create-a-container/models/external-domain.js b/create-a-container/models/external-domain.js index 62d0ff8b..2127c4f7 100644 --- a/create-a-container/models/external-domain.js +++ b/create-a-container/models/external-domain.js @@ -75,6 +75,33 @@ module.exports = (sequelize) => { } }, comment: "Address of the oauth2-proxy process for nginx auth_request, e.g. http://127.0.0.1:4180. nginx proxies /oauth2/* straight to it in a single hop; do not point this at a path-prefixed URL." + }, + mailEnabled: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + comment: 'Admin switch — mail for this domain is off until enabled AND DNS-verified' + }, + mailDnsVerified: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + comment: 'SPF/DKIM/DMARC check passed. mailEnabled && mailDnsVerified => domain can send' + }, + mailMxVerified: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: false, + comment: 'MX resolves to the mail IP. mailEnabled && mailMxVerified => domain can receive' + }, + mailDnsCheckedAt: { + type: DataTypes.DATE, + allowNull: true + }, + mailDnsCheckResult: { + type: DataTypes.JSON, + allowNull: true, + comment: 'Per-record outcome of the last Check DNS run (see utils/mail-dns.js)' } }, { tableName: 'ExternalDomains', @@ -86,6 +113,14 @@ module.exports = (sequelize) => { foreignKey: 'siteId', as: 'site' }); + ExternalDomain.hasMany(models.DkimKey, { + foreignKey: 'externalDomainId', + as: 'dkimKeys' + }); + ExternalDomain.hasMany(models.MailAccount, { + foreignKey: 'externalDomainId', + as: 'mailAccounts' + }); }; return ExternalDomain; diff --git a/create-a-container/models/mail-account.js b/create-a-container/models/mail-account.js new file mode 100644 index 00000000..6c2824bc --- /dev/null +++ b/create-a-container/models/mail-account.js @@ -0,0 +1,94 @@ +'use strict'; +const { Model } = require('sequelize'); + +module.exports = (sequelize, DataTypes) => { + class MailAccount extends Model { + static associate(models) { + MailAccount.belongsTo(models.User, { + foreignKey: 'uidNumber', + as: 'owner', + }); + MailAccount.belongsTo(models.ExternalDomain, { + foreignKey: 'externalDomainId', + as: 'domain', + }); + MailAccount.hasMany(models.MailSuppression, { + foreignKey: 'mailAccountId', + as: 'suppressions', + }); + } + } + + MailAccount.init({ + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + allowNull: false, + }, + uidNumber: { + type: DataTypes.INTEGER, + allowNull: false, + references: { model: 'Users', key: 'uidNumber' }, + onUpdate: 'CASCADE', + onDelete: 'CASCADE', + comment: 'Owning user — admin-transferable, never shared', + }, + externalDomainId: { + type: DataTypes.INTEGER, + allowNull: false, + references: { model: 'ExternalDomains', key: 'id' }, + onDelete: 'CASCADE', + }, + localPart: { + type: DataTypes.STRING(64), + allowNull: false, + validate: { + // Conservative RFC 5321 dot-atom subset, stored lowercase. + is: /^[a-z0-9](?:[a-z0-9._+-]*[a-z0-9])?$/, + }, + comment: 'Lowercase local part — address is @', + }, + description: { + type: DataTypes.STRING(255), + allowNull: true, + }, + passwordHash: { + type: DataTypes.STRING(255), + allowNull: false, + comment: 'Argon2id PHC string (parallelism 1 for Dovecot compatibility). Never serialized.', + }, + enabled: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: true, + comment: 'Disabled accounts disappear from the SQL views — instant lockout', + }, + quotaBytes: { + type: DataTypes.BIGINT, + allowNull: false, + comment: 'Mailbox quota enforced by Dovecot (quota-status also rejects at RCPT when over)', + }, + unsubscribeHeaders: { + type: DataTypes.BOOLEAN, + allowNull: false, + defaultValue: true, + comment: 'Add RFC 8058 List-Unsubscribe headers to submitted mail', + }, + lastRotatedAt: { + type: DataTypes.DATE, + allowNull: true, + }, + }, { + sequelize, + modelName: 'MailAccount', + tableName: 'MailAccounts', + timestamps: true, + indexes: [ + { unique: true, fields: ['externalDomainId', 'localPart'] }, + { fields: ['uidNumber'] }, + ], + }); + + return MailAccount; +}; diff --git a/create-a-container/models/mail-suppression.js b/create-a-container/models/mail-suppression.js new file mode 100644 index 00000000..5ae437fb --- /dev/null +++ b/create-a-container/models/mail-suppression.js @@ -0,0 +1,50 @@ +'use strict'; +const { Model } = require('sequelize'); + +module.exports = (sequelize, DataTypes) => { + class MailSuppression extends Model { + static associate(models) { + MailSuppression.belongsTo(models.MailAccount, { + foreignKey: 'mailAccountId', + as: 'account', + }); + } + } + + MailSuppression.init({ + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + }, + mailAccountId: { + type: DataTypes.UUID, + allowNull: false, + references: { model: 'MailAccounts', key: 'id' }, + onDelete: 'CASCADE', + }, + recipient: { + type: DataTypes.STRING(320), + allowNull: false, + set(value) { + this.setDataValue('recipient', String(value || '').trim().toLowerCase()); + }, + comment: 'Lowercased recipient address that unsubscribed from this account', + }, + source: { + type: DataTypes.ENUM('one-click', 'admin'), + allowNull: false, + defaultValue: 'one-click', + }, + }, { + sequelize, + modelName: 'MailSuppression', + tableName: 'MailSuppressions', + timestamps: true, + indexes: [ + { unique: true, fields: ['mailAccountId', 'recipient'] }, + ], + }); + + return MailSuppression; +}; diff --git a/create-a-container/models/mail-unsubscribe-key.js b/create-a-container/models/mail-unsubscribe-key.js new file mode 100644 index 00000000..ceddeb4b --- /dev/null +++ b/create-a-container/models/mail-unsubscribe-key.js @@ -0,0 +1,41 @@ +'use strict'; +const { Model } = require('sequelize'); + +// Retired keys keep verifying previously issued unsubscribe links for this long. +const RETIRED_KEY_RETENTION_MS = 365 * 24 * 60 * 60 * 1000; + +module.exports = (sequelize, DataTypes) => { + class MailUnsubscribeKey extends Model { + /** True when this key may still verify tokens (active, or retired within retention). */ + canVerify(now = Date.now()) { + if (this.status === 'active') return true; + return now - new Date(this.updatedAt).getTime() < RETIRED_KEY_RETENTION_MS; + } + } + + MailUnsubscribeKey.init({ + kid: { + type: DataTypes.STRING(16), + primaryKey: true, + allowNull: false, + comment: 'Short key id embedded (unencrypted) in unsubscribe tokens', + }, + secret: { + type: DataTypes.STRING(64), + allowNull: false, + comment: 'Base64 of 32 random bytes (AES-256-GCM key). Never serialized; sent only in the mail-host snapshot.', + }, + status: { + type: DataTypes.ENUM('active', 'retired'), + allowNull: false, + defaultValue: 'active', + }, + }, { + sequelize, + modelName: 'MailUnsubscribeKey', + tableName: 'MailUnsubscribeKeys', + timestamps: true, + }); + + return MailUnsubscribeKey; +}; diff --git a/create-a-container/utils/mail-views.js b/create-a-container/utils/mail-views.js new file mode 100644 index 00000000..1e2eab2d --- /dev/null +++ b/create-a-container/utils/mail-views.js @@ -0,0 +1,68 @@ +/** + * Per-dialect SQL for the mail views — the stable contract Dovecot and + * Postfix read live (see docs: the views are consumed by `mail_dovecot` and + * `mail_postfix` DB users created by bin/setup-mail-db-roles.sh): + * + * mail_accounts_v address, password ({ARGON2ID}-prefixed PHC hash), + * home, quota_bytes, can_send, can_receive + * — enabled accounts only (Dovecot passdb/userdb) + * mail_senders_v sender, login for can-send accounts + * (Postfix smtpd_sender_login_maps) + * mail_suppressions_v sender, recipient + * (submission policy service — reject at RCPT) + * + * SQLite cannot host the mail stack, but the views are still created there so + * the schema stays identical across dialects for tests. + */ + +const VIEW_NAMES = ['mail_accounts_v', 'mail_senders_v', 'mail_suppressions_v']; + +/** + * Build the CREATE VIEW statements for a dialect. + * @param {string} dialect - 'postgres' | 'mysql' | 'mariadb' | 'sqlite' + * @returns {string[]} ordered CREATE VIEW statements + */ +function createViewsSql(dialect) { + const mysql = dialect === 'mysql' || dialect === 'mariadb'; + // Identifier quoting + const q = mysql ? (s) => `\`${s}\`` : (s) => `"${s}"`; + // String concatenation + const cat = mysql + ? (...parts) => `CONCAT(${parts.join(', ')})` + : (...parts) => parts.join(' || '); + + const address = `LOWER(${cat(`ma.${q('localPart')}`, "'@'", `ed.${q('name')}`)})`; + const canSend = `(ed.${q('mailEnabled')} AND ed.${q('mailDnsVerified')})`; + const canReceive = `(ed.${q('mailEnabled')} AND ed.${q('mailMxVerified')})`; + const joins = + `FROM ${q('MailAccounts')} ma ` + + `JOIN ${q('ExternalDomains')} ed ON ed.${q('id')} = ma.${q('externalDomainId')}`; + + return [ + `CREATE VIEW mail_accounts_v AS ` + + `SELECT ${address} AS address, ` + + `${cat("'{ARGON2ID}'", `ma.${q('passwordHash')}`)} AS password, ` + + `${cat("'/var/vmail/'", `ma.${q('id')}`)} AS home, ` + + `ma.${q('quotaBytes')} AS quota_bytes, ` + + `${canSend} AS can_send, ` + + `${canReceive} AS can_receive ` + + `${joins} WHERE ma.${q('enabled')}`, + + `CREATE VIEW mail_senders_v AS ` + + `SELECT ${address} AS sender, ${address} AS login ` + + `${joins} WHERE ma.${q('enabled')} AND ${canSend}`, + + `CREATE VIEW mail_suppressions_v AS ` + + `SELECT ${address} AS sender, ms.${q('recipient')} AS recipient ` + + `FROM ${q('MailSuppressions')} ms ` + + `JOIN ${q('MailAccounts')} ma ON ma.${q('id')} = ms.${q('mailAccountId')} ` + + `JOIN ${q('ExternalDomains')} ed ON ed.${q('id')} = ma.${q('externalDomainId')}`, + ]; +} + +/** DROP statements, reverse order. */ +function dropViewsSql() { + return [...VIEW_NAMES].reverse().map((v) => `DROP VIEW IF EXISTS ${v}`); +} + +module.exports = { VIEW_NAMES, createViewsSql, dropViewsSql }; From 3e668bcc49a7d2379704bde36180feb8fa30da62 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 09:51:41 -0700 Subject: [PATCH 02/19] mail: /api/v1/mail-accounts resource (#67) MVC-manifesto vertical slice: list/create/get/patch/rotate-password/delete plus GET /mail-accounts/domains (can-send domains) and the suppressions sub-resource (owner-readable, admin-removable). Passwords are 256-bit random, shown once, stored as Argon2id p=1 (Dovecot/libsodium compatible). Reserved role local parts are admin-only; quota and ownership transfer are admin-only; accounts can only be created on domains that can send. --- .../mailaccounts/__tests__/service.test.js | 153 ++++++++++++++++ .../resources/mailaccounts/controller.js | 80 +++++++++ .../resources/mailaccounts/repository.js | 91 ++++++++++ .../resources/mailaccounts/router.js | 33 ++++ .../resources/mailaccounts/serializer.js | 38 ++++ .../resources/mailaccounts/service.js | 169 ++++++++++++++++++ .../resources/mailaccounts/validator.js | 44 +++++ create-a-container/routers/api/v1/index.js | 1 + create-a-container/utils/mail-account.js | 41 +++++ 9 files changed, 650 insertions(+) create mode 100644 create-a-container/resources/mailaccounts/__tests__/service.test.js create mode 100644 create-a-container/resources/mailaccounts/controller.js create mode 100644 create-a-container/resources/mailaccounts/repository.js create mode 100644 create-a-container/resources/mailaccounts/router.js create mode 100644 create-a-container/resources/mailaccounts/serializer.js create mode 100644 create-a-container/resources/mailaccounts/service.js create mode 100644 create-a-container/resources/mailaccounts/validator.js create mode 100644 create-a-container/utils/mail-account.js diff --git a/create-a-container/resources/mailaccounts/__tests__/service.test.js b/create-a-container/resources/mailaccounts/__tests__/service.test.js new file mode 100644 index 00000000..94ea1e89 --- /dev/null +++ b/create-a-container/resources/mailaccounts/__tests__/service.test.js @@ -0,0 +1,153 @@ +/** + * Unit tests for the mail-accounts service — repository mocked per the MVC + * manifesto. Covers the #67 guardrails: one-time passwords, Argon2id p=1 + * hashes, reserved local parts, the domain can-send gate, and admin-only + * quota/transfer/suppression-removal. + */ + +jest.mock('../repository'); + +const repo = require('../repository'); +const svc = require('../service'); + +const user = { uidNumber: 3001, uid: 'alice', status: 'active' }; +const admin = { uidNumber: 3000, uid: 'root-admin', status: 'active' }; +const sendableDomain = { id: 1, name: 'example.com', mailEnabled: true, mailDnsVerified: true }; +const disabledDomain = { id: 2, name: 'off.example', mailEnabled: false, mailDnsVerified: false }; + +const session = { uid: 'alice', isAdmin: false }; +const adminSession = { uid: 'root-admin', isAdmin: true }; + +function accountRow(overrides = {}) { + return { + id: '7d9fb37e-0000-4000-8000-000000000001', + uidNumber: user.uidNumber, + localPart: 'app', + domain: sendableDomain, + owner: user, + quotaBytes: 1024 * 1024 * 1024, + ...overrides, + }; +} + +beforeEach(() => { + jest.resetAllMocks(); + repo.findUserByUid.mockImplementation(async (uid) => { + if (uid === 'alice') return user; + if (uid === 'root-admin') return admin; + return null; + }); + repo.getSetting.mockResolvedValue(null); + repo.findDomain.mockResolvedValue(sendableDomain); + repo.findByAddress.mockResolvedValue(null); + repo.create.mockImplementation(async (fields) => accountRow(fields)); + repo.update.mockImplementation(async (account, fields) => ({ ...account, ...fields })); +}); + +describe('createAccount', () => { + test('returns the password once and stores only an Argon2id p=1 hash', async () => { + const { account, password, connection } = await svc.createAccount(session, { + externalDomainId: 1, + localPart: 'app', + }); + expect(password).toHaveLength(43); // 32 bytes base64url + const stored = repo.create.mock.calls[0][0]; + expect(stored.passwordHash).toMatch(/^\$argon2id\$/); + expect(stored.passwordHash).toMatch(/p=1[,$]/); + expect(Object.values(stored)).not.toContain(password); + // Default quota applies when no setting and no quotaMb given. + expect(stored.quotaBytes).toBe(1024 * 1024 * 1024); + expect(account.uidNumber).toBe(user.uidNumber); + expect(connection.username).toBe('app@example.com'); + expect(connection.smtp.ports).toEqual([587, 465]); + }); + + test('rejects domains that cannot send', async () => { + repo.findDomain.mockResolvedValue(disabledDomain); + await expect(svc.createAccount(session, { externalDomainId: 2, localPart: 'app' })) + .rejects.toMatchObject({ status: 422, code: 'domain_cannot_send' }); + }); + + test('reserves role local parts for admins', async () => { + await expect(svc.createAccount(session, { externalDomainId: 1, localPart: 'postmaster' })) + .rejects.toMatchObject({ status: 403, code: 'reserved_local_part' }); + await expect(svc.createAccount(adminSession, { externalDomainId: 1, localPart: 'postmaster' })) + .resolves.toBeTruthy(); + }); + + test('rejects duplicate addresses with 409', async () => { + repo.findByAddress.mockResolvedValue(accountRow()); + await expect(svc.createAccount(session, { externalDomainId: 1, localPart: 'app' })) + .rejects.toMatchObject({ status: 409, code: 'address_taken' }); + }); + + test('only admins can set a quota at create', async () => { + await expect(svc.createAccount(session, { externalDomainId: 1, localPart: 'app', quotaMb: 10 })) + .rejects.toMatchObject({ status: 403, code: 'admin_required' }); + await svc.createAccount(adminSession, { externalDomainId: 1, localPart: 'app', quotaMb: 10 }); + expect(repo.create.mock.calls[0][0].quotaBytes).toBe(10 * 1024 * 1024); + }); +}); + +describe('updateAccount', () => { + test('owner can toggle enabled/unsubscribeHeaders but not quota or transfer', async () => { + repo.findById.mockResolvedValue(accountRow()); + await svc.updateAccount(session, 'x-id', { enabled: false, unsubscribeHeaders: false }); + expect(repo.update.mock.calls[0][1]).toEqual({ enabled: false, unsubscribeHeaders: false }); + + await expect(svc.updateAccount(session, 'x-id', { quotaMb: 5 })) + .rejects.toMatchObject({ status: 403, code: 'admin_required' }); + await expect(svc.updateAccount(session, 'x-id', { username: 'bob' })) + .rejects.toMatchObject({ status: 403, code: 'admin_required' }); + }); + + test('admin transfer resolves the target user and rejects inactive users', async () => { + repo.findById.mockResolvedValue(accountRow()); + repo.findUserByUid.mockImplementation(async (uid) => { + if (uid === 'root-admin') return admin; + if (uid === 'bob') return { uidNumber: 3002, uid: 'bob', status: 'active' }; + if (uid === 'carol') return { uidNumber: 3003, uid: 'carol', status: 'disabled' }; + return null; + }); + await svc.updateAccount(adminSession, 'x-id', { username: 'bob' }); + expect(repo.update.mock.calls[0][1]).toEqual({ uidNumber: 3002 }); + + await expect(svc.updateAccount(adminSession, 'x-id', { username: 'carol' })) + .rejects.toMatchObject({ status: 422, code: 'user_inactive' }); + await expect(svc.updateAccount(adminSession, 'x-id', { username: 'nobody' })) + .rejects.toMatchObject({ status: 404, code: 'user_not_found' }); + }); + + test('non-owner non-admin gets a 404 (existence hidden)', async () => { + repo.findById.mockResolvedValue(accountRow({ uidNumber: 9999 })); + await expect(svc.updateAccount(session, 'x-id', { enabled: false })) + .rejects.toMatchObject({ status: 404 }); + }); +}); + +describe('rotatePassword', () => { + test('returns a fresh password and stores a new hash + lastRotatedAt', async () => { + repo.findById.mockResolvedValue(accountRow()); + const { password } = await svc.rotatePassword(session, 'x-id'); + expect(password).toHaveLength(43); + const fields = repo.update.mock.calls[0][1]; + expect(fields.passwordHash).toMatch(/^\$argon2id\$/); + expect(fields.lastRotatedAt).toBeInstanceOf(Date); + }); +}); + +describe('suppressions', () => { + test('owner can list but only admins can remove', async () => { + repo.findById.mockResolvedValue(accountRow()); + repo.findSuppressions.mockResolvedValue([]); + await expect(svc.listSuppressions(session, 'x-id')).resolves.toEqual([]); + + await expect(svc.removeSuppression(session, 'x-id', 1)) + .rejects.toMatchObject({ status: 403, code: 'admin_required' }); + + repo.findSuppression.mockResolvedValue({ id: 1 }); + repo.destroySuppression.mockResolvedValue(); + await svc.removeSuppression(adminSession, 'x-id', 1); + expect(repo.destroySuppression).toHaveBeenCalled(); + }); +}); diff --git a/create-a-container/resources/mailaccounts/controller.js b/create-a-container/resources/mailaccounts/controller.js new file mode 100644 index 00000000..13e6418f --- /dev/null +++ b/create-a-container/resources/mailaccounts/controller.js @@ -0,0 +1,80 @@ +const svc = require('./service'); +const { serializeMailAccount, serializeSuppression } = require('./serializer'); +const { asyncHandler, ok, created, noContent } = require('../../middlewares/api'); + +const PASSWORD_WARNING = 'Save this password — it will not be shown again.'; + +function sessionOf(req) { + return { uid: req.session.user, isAdmin: !!req.session.isAdmin }; +} + +const list = asyncHandler(async (req, res) => { + const accounts = await svc.listAccounts(sessionOf(req), req.validated.query); + return ok(res, accounts.map(serializeMailAccount)); +}); + +const listDomains = asyncHandler(async (req, res) => { + const domains = await svc.listSendableDomains(sessionOf(req)); + return ok(res, domains.map((d) => ({ id: d.id, name: d.name }))); +}); + +const get = asyncHandler(async (req, res) => { + const account = await svc.getAccount(sessionOf(req), req.validated.params.id); + return ok(res, serializeMailAccount(account)); +}); + +const create = asyncHandler(async (req, res) => { + const { account, password, connection } = await svc.createAccount(sessionOf(req), req.validated.body); + return created(res, { + ...serializeMailAccount(account), + password, + connection, + warning: PASSWORD_WARNING, + }); +}); + +const update = asyncHandler(async (req, res) => { + const account = await svc.updateAccount(sessionOf(req), req.validated.params.id, req.validated.body); + return ok(res, serializeMailAccount(account)); +}); + +const rotatePassword = asyncHandler(async (req, res) => { + const { account, password, connection } = await svc.rotatePassword(sessionOf(req), req.validated.params.id); + return ok(res, { + ...serializeMailAccount(account), + password, + connection, + warning: PASSWORD_WARNING, + }); +}); + +const remove = asyncHandler(async (req, res) => { + await svc.deleteAccount(sessionOf(req), req.validated.params.id); + return noContent(res); +}); + +const listSuppressions = asyncHandler(async (req, res) => { + const suppressions = await svc.listSuppressions(sessionOf(req), req.validated.params.id); + return ok(res, suppressions.map(serializeSuppression)); +}); + +const removeSuppression = asyncHandler(async (req, res) => { + await svc.removeSuppression( + sessionOf(req), + req.validated.params.id, + req.validated.params.suppressionId, + ); + return noContent(res); +}); + +module.exports = { + list, + listDomains, + get, + create, + update, + rotatePassword, + remove, + listSuppressions, + removeSuppression, +}; diff --git a/create-a-container/resources/mailaccounts/repository.js b/create-a-container/resources/mailaccounts/repository.js new file mode 100644 index 00000000..ee52d56a --- /dev/null +++ b/create-a-container/resources/mailaccounts/repository.js @@ -0,0 +1,91 @@ +const { MailAccount, MailSuppression, ExternalDomain, User, Setting } = require('../../models'); + +// Always loaded with the owning user and domain (incl. mail gates) so the +// serializer can compose the address and send/receive state. +const INCLUDES = [ + { + model: ExternalDomain, + as: 'domain', + attributes: ['id', 'name', 'mailEnabled', 'mailDnsVerified', 'mailMxVerified'], + }, + { model: User, as: 'owner', attributes: ['uidNumber', 'uid'] }, +]; + +async function findAll(where = {}) { + return MailAccount.findAll({ where, include: INCLUDES, order: [['createdAt', 'DESC']] }); +} + +async function findById(id) { + return MailAccount.findByPk(id, { include: INCLUDES }); +} + +async function findByAddress(externalDomainId, localPart) { + return MailAccount.findOne({ where: { externalDomainId, localPart } }); +} + +async function create(fields) { + const account = await MailAccount.create(fields); + return findById(account.id); +} + +async function update(account, fields) { + await account.update(fields); + return findById(account.id); +} + +async function destroy(account) { + return account.destroy(); +} + +async function findDomain(id) { + return ExternalDomain.findByPk(id); +} + +async function findSendableDomains() { + return ExternalDomain.findAll({ + where: { mailEnabled: true, mailDnsVerified: true }, + attributes: ['id', 'name'], + order: [['name', 'ASC']], + }); +} + +async function findSuppressions(mailAccountId) { + return MailSuppression.findAll({ + where: { mailAccountId }, + order: [['createdAt', 'DESC']], + }); +} + +async function findSuppression(mailAccountId, suppressionId) { + return MailSuppression.findOne({ where: { id: suppressionId, mailAccountId } }); +} + +async function destroySuppression(suppression) { + return suppression.destroy(); +} + +// Owner lookup by session uid (manifesto: cross-resource access moves behind +// the users service once resources/users/ exists). +async function findUserByUid(uid) { + return User.findOne({ where: { uid } }); +} + +async function getSetting(key) { + return Setting.get(key); +} + +module.exports = { + findAll, + findById, + findByAddress, + create, + update, + destroy, + findDomain, + findSendableDomains, + findSuppressions, + findSuppression, + destroySuppression, + findUserByUid, + getSetting, +}; diff --git a/create-a-container/resources/mailaccounts/router.js b/create-a-container/resources/mailaccounts/router.js new file mode 100644 index 00000000..e1c81079 --- /dev/null +++ b/create-a-container/resources/mailaccounts/router.js @@ -0,0 +1,33 @@ +/** + * /api/v1/mail-accounts — email service accounts (issue #67). + * Wiring only; see controller/service/repository per the MVC manifesto. + */ + +const express = require('express'); +const { apiAuth } = require('../../middlewares/api'); +const { validate } = require('../../middlewares/validate'); +const { createMailAccount, updateMailAccount, idParam, suppressionParams, listQuery } = require('./validator'); +const ctrl = require('./controller'); + +const router = express.Router(); + +router.use(apiAuth); + +// `/domains` must precede `/:id`. +router.get('/domains', ctrl.listDomains); + +router.get('/', validate({ query: listQuery }), ctrl.list); +router.post('/', validate(createMailAccount), ctrl.create); +router.get('/:id', validate({ params: idParam }), ctrl.get); +router.patch('/:id', validate({ params: idParam, body: updateMailAccount }), ctrl.update); +router.post('/:id/rotate-password', validate({ params: idParam }), ctrl.rotatePassword); +router.delete('/:id', validate({ params: idParam }), ctrl.remove); + +router.get('/:id/suppressions', validate({ params: idParam }), ctrl.listSuppressions); +router.delete( + '/:id/suppressions/:suppressionId', + validate({ params: suppressionParams }), + ctrl.removeSuppression, +); + +module.exports = router; diff --git a/create-a-container/resources/mailaccounts/serializer.js b/create-a-container/resources/mailaccounts/serializer.js new file mode 100644 index 00000000..f095a0b5 --- /dev/null +++ b/create-a-container/resources/mailaccounts/serializer.js @@ -0,0 +1,38 @@ +const MB = 1024 * 1024; + +/** Never serialized: passwordHash. BIGINT comes back as a string on Postgres. */ +function serializeMailAccount(a) { + const domain = a.domain + ? { + id: a.domain.id, + name: a.domain.name, + canSend: !!(a.domain.mailEnabled && a.domain.mailDnsVerified), + canReceive: !!(a.domain.mailEnabled && a.domain.mailMxVerified), + } + : null; + return { + id: a.id, + address: domain ? `${a.localPart}@${domain.name}` : null, + localPart: a.localPart, + domain, + owner: a.owner ? a.owner.uid : null, + description: a.description, + enabled: a.enabled, + quotaMb: Math.round(Number(a.quotaBytes) / MB), + unsubscribeHeaders: a.unsubscribeHeaders, + lastRotatedAt: a.lastRotatedAt, + createdAt: a.createdAt, + updatedAt: a.updatedAt, + }; +} + +function serializeSuppression(s) { + return { + id: s.id, + recipient: s.recipient, + source: s.source, + createdAt: s.createdAt, + }; +} + +module.exports = { serializeMailAccount, serializeSuppression }; diff --git a/create-a-container/resources/mailaccounts/service.js b/create-a-container/resources/mailaccounts/service.js new file mode 100644 index 00000000..d1bcc1fc --- /dev/null +++ b/create-a-container/resources/mailaccounts/service.js @@ -0,0 +1,169 @@ +/** + * Mail accounts business logic (issue #67). + * + * Accounts exist only on domains that can send (mailEnabled + DNS verified). + * Passwords are generated server-side, returned exactly once, and stored as + * Argon2id (p=1) hashes that Dovecot verifies live through mail_accounts_v. + */ + +const repo = require('./repository'); +const { ApiError } = require('../../middlewares/api'); +const { + DEFAULT_QUOTA_MB, + generateMailPassword, + hashMailPassword, + isReservedLocalPart, +} = require('../../utils/mail-account'); + +const MB = 1024 * 1024; + +async function requireUser(session) { + const user = await repo.findUserByUid(session.uid); + if (!user) throw new ApiError(401, 'unauthorized', 'Session user not found'); + return user; +} + +function domainCanSend(domain) { + return !!(domain.mailEnabled && domain.mailDnsVerified); +} + +/** Owner or admin may see the account; others get a 404 (existence hidden). */ +async function loadAccountFor(session, id, user) { + const account = await repo.findById(id); + if (!account || (!session.isAdmin && account.uidNumber !== user.uidNumber)) { + throw new ApiError(404, 'not_found', 'Mail account not found'); + } + return account; +} + +/** SMTP/IMAP client settings returned alongside one-time passwords. */ +async function connectionInfo(account) { + const mailHostname = await repo.getSetting('mail_hostname'); + const address = `${account.localPart}@${account.domain.name}`; + return { + host: mailHostname || account.domain.name, + username: address, + smtp: { ports: [587, 465] }, + imap: { ports: [993, 143] }, + }; +} + +async function listAccounts(session, { all } = {}) { + const user = await requireUser(session); + if (all && !session.isAdmin) { + throw new ApiError(403, 'forbidden', 'Admin access required to list all accounts'); + } + return repo.findAll(all && session.isAdmin ? {} : { uidNumber: user.uidNumber }); +} + +async function getAccount(session, id) { + const user = await requireUser(session); + return loadAccountFor(session, id, user); +} + +async function listSendableDomains(session) { + await requireUser(session); + return repo.findSendableDomains(); +} + +/** Returns { account, password, connection } — password shown exactly once. */ +async function createAccount(session, { externalDomainId, localPart, description, quotaMb }) { + const user = await requireUser(session); + + const domain = await repo.findDomain(externalDomainId); + if (!domain) throw new ApiError(404, 'domain_not_found', 'External domain not found'); + if (!domainCanSend(domain)) { + throw new ApiError(422, 'domain_cannot_send', 'Mail is not enabled (or DNS is not verified) for this domain'); + } + if (isReservedLocalPart(localPart) && !session.isAdmin) { + throw new ApiError(403, 'reserved_local_part', `"${localPart}" is reserved for administrators`); + } + if (quotaMb !== undefined && !session.isAdmin) { + throw new ApiError(403, 'admin_required', 'Only admins can set a quota'); + } + if (await repo.findByAddress(externalDomainId, localPart)) { + throw new ApiError(409, 'address_taken', 'That address already exists'); + } + + const defaultQuotaMb = parseInt(await repo.getSetting('mail_default_quota_mb'), 10) || DEFAULT_QUOTA_MB; + const password = generateMailPassword(); + const account = await repo.create({ + uidNumber: user.uidNumber, + externalDomainId, + localPart, + description: description || null, + passwordHash: await hashMailPassword(password), + quotaBytes: (quotaMb ?? defaultQuotaMb) * MB, + }); + return { account, password, connection: await connectionInfo(account) }; +} + +async function updateAccount(session, id, body) { + const user = await requireUser(session); + const account = await loadAccountFor(session, id, user); + + if ((body.quotaMb !== undefined || body.username !== undefined) && !session.isAdmin) { + throw new ApiError(403, 'admin_required', 'Only admins can change quota or transfer ownership'); + } + + const fields = {}; + if (body.description !== undefined) fields.description = body.description; + if (body.enabled !== undefined) fields.enabled = body.enabled; + if (body.unsubscribeHeaders !== undefined) fields.unsubscribeHeaders = body.unsubscribeHeaders; + if (body.quotaMb !== undefined) fields.quotaBytes = body.quotaMb * MB; + if (body.username !== undefined) { + const target = await repo.findUserByUid(body.username); + if (!target) throw new ApiError(404, 'user_not_found', 'Target user not found'); + if (target.status !== 'active') throw new ApiError(422, 'user_inactive', 'Target user is not active'); + fields.uidNumber = target.uidNumber; + } + return repo.update(account, fields); +} + +/** Returns { account, password, connection } — fresh password shown exactly once. */ +async function rotatePassword(session, id) { + const user = await requireUser(session); + const account = await loadAccountFor(session, id, user); + const password = generateMailPassword(); + const updated = await repo.update(account, { + passwordHash: await hashMailPassword(password), + lastRotatedAt: new Date(), + }); + return { account: updated, password, connection: await connectionInfo(updated) }; +} + +async function deleteAccount(session, id) { + const user = await requireUser(session); + const account = await loadAccountFor(session, id, user); + await repo.destroy(account); +} + +async function listSuppressions(session, id) { + const user = await requireUser(session); + await loadAccountFor(session, id, user); + return repo.findSuppressions(id); +} + +/** Admin-only: an app owner must not be able to quietly re-subscribe people. */ +async function removeSuppression(session, id, suppressionId) { + const user = await requireUser(session); + await loadAccountFor(session, id, user); + if (!session.isAdmin) { + throw new ApiError(403, 'admin_required', 'Only admins can remove suppressions'); + } + const suppression = await repo.findSuppression(id, suppressionId); + if (!suppression) throw new ApiError(404, 'not_found', 'Suppression not found'); + await repo.destroySuppression(suppression); +} + +module.exports = { + listAccounts, + getAccount, + listSendableDomains, + createAccount, + updateAccount, + rotatePassword, + deleteAccount, + listSuppressions, + removeSuppression, +}; diff --git a/create-a-container/resources/mailaccounts/validator.js b/create-a-container/resources/mailaccounts/validator.js new file mode 100644 index 00000000..2afd2809 --- /dev/null +++ b/create-a-container/resources/mailaccounts/validator.js @@ -0,0 +1,44 @@ +const { z } = require('zod'); + +// Conservative RFC 5321 dot-atom subset; stored lowercase. +const localPart = z + .string() + .trim() + .toLowerCase() + .min(1) + .max(64) + .regex(/^[a-z0-9](?:[a-z0-9._+-]*[a-z0-9])?$/, 'invalid local part'); + +const createMailAccount = z.object({ + externalDomainId: z.coerce.number().int().positive(), + localPart, + description: z.string().max(255).nullish(), + // Admin-only (service enforces); non-admins get the default quota. + quotaMb: z.coerce.number().int().min(1).max(1048576).optional(), +}); + +const updateMailAccount = z.object({ + description: z.string().max(255).nullable().optional(), + enabled: z.boolean().optional(), + unsubscribeHeaders: z.boolean().optional(), + // Admin-only (service enforces). + quotaMb: z.coerce.number().int().min(1).max(1048576).optional(), + // Admin-only ownership transfer — target user's uid (username). + username: z.string().trim().min(1).max(255).optional(), +}); + +const idParam = z.object({ + id: z.uuid(), +}); + +const suppressionParams = z.object({ + id: z.uuid(), + suppressionId: z.coerce.number().int().positive(), +}); + +const listQuery = z.object({ + // `?all=true` — admins list every account instead of just their own. + all: z.literal('true').optional(), +}); + +module.exports = { createMailAccount, updateMailAccount, idParam, suppressionParams, listQuery }; diff --git a/create-a-container/routers/api/v1/index.js b/create-a-container/routers/api/v1/index.js index 3e45093a..93bd01d0 100644 --- a/create-a-container/routers/api/v1/index.js +++ b/create-a-container/routers/api/v1/index.js @@ -91,6 +91,7 @@ router.use('/external-domains', require('./external-domains')); router.use('/groups', require('./groups')); router.use('/users', require('./users')); router.use('/apikeys', require('../../../resources/apikeys/router')); +router.use('/mail-accounts', require('../../../resources/mailaccounts/router')); router.use('/settings', require('./settings')); router.use('/jobs', require('./jobs')); router.use('/resource-requests', require('./resource-requests')); diff --git a/create-a-container/utils/mail-account.js b/create-a-container/utils/mail-account.js new file mode 100644 index 00000000..c71c7c74 --- /dev/null +++ b/create-a-container/utils/mail-account.js @@ -0,0 +1,41 @@ +/** + * Mail account credential helpers (issue #67). + * + * Passwords are 256-bit random, shown to the user exactly once, and stored + * as Argon2id PHC strings. parallelism MUST stay 1: Dovecot verifies + * {ARGON2ID} hashes via libsodium, which only supports p=1. + */ + +const crypto = require('crypto'); +const argon2 = require('argon2'); + +// Admin-only local parts (RFC 2142 role addresses + operational names). +const RESERVED_LOCAL_PARTS = [ + 'postmaster', 'abuse', 'hostmaster', 'webmaster', 'root', + 'admin', 'security', 'noreply', 'dmarc', +]; + +const DEFAULT_QUOTA_MB = 1024; + +/** @returns {string} 43-char base64url string (32 random bytes). */ +function generateMailPassword() { + return crypto.randomBytes(32).toString('base64url'); +} + +/** @param {string} password @returns {Promise} Argon2id PHC string with p=1. */ +async function hashMailPassword(password) { + return argon2.hash(password, { type: argon2.argon2id, parallelism: 1 }); +} + +/** @param {string} localPart */ +function isReservedLocalPart(localPart) { + return RESERVED_LOCAL_PARTS.includes(String(localPart).toLowerCase()); +} + +module.exports = { + RESERVED_LOCAL_PARTS, + DEFAULT_QUOTA_MB, + generateMailPassword, + hashMailPassword, + isReservedLocalPart, +}; From d95d2a8c0d07a7dd80b74f4ef92be11afa055db4 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 09:56:18 -0700 Subject: [PATCH 03/19] mail: per-domain enablement, DKIM keys, and Check DNS (#67) Every external domain gets an RSA-2048 DKIM key (at create + backfill migration). Admin endpoints on /external-domains/{id}/mail: dns (records to publish + last check), check-dns, enable (422 dns_not_configured until SPF/DKIM/DMARC verify; MX only gates receiving), disable (keeps keys). Check DNS runs against configurable public resolvers; PTR and missing postmaster@/abuse@ are warnings. Mail settings keys added to /settings (relayhost requires mail_spf_include). --- .../20261008000007-backfill-dkim-keys.js | 36 +++ .../routers/api/v1/external-domains.js | 123 ++++++++++- create-a-container/routers/api/v1/settings.js | 57 ++++- .../utils/__tests__/mail-dns.test.js | 140 ++++++++++++ create-a-container/utils/dkim.js | 36 +++ create-a-container/utils/mail-dns.js | 207 ++++++++++++++++++ create-a-container/utils/mail-host.js | 90 ++++++++ 7 files changed, 686 insertions(+), 3 deletions(-) create mode 100644 create-a-container/migrations/20261008000007-backfill-dkim-keys.js create mode 100644 create-a-container/utils/__tests__/mail-dns.test.js create mode 100644 create-a-container/utils/dkim.js create mode 100644 create-a-container/utils/mail-dns.js create mode 100644 create-a-container/utils/mail-host.js diff --git a/create-a-container/migrations/20261008000007-backfill-dkim-keys.js b/create-a-container/migrations/20261008000007-backfill-dkim-keys.js new file mode 100644 index 00000000..065b525e --- /dev/null +++ b/create-a-container/migrations/20261008000007-backfill-dkim-keys.js @@ -0,0 +1,36 @@ +'use strict'; + +const { generateDkimKeyPair } = require('../utils/dkim'); + +/** + * Backfill DKIM keys for domains created before issue #67. New domains get + * a key at create time (routers/api/v1/external-domains.js). + */ +module.exports = { + async up(queryInterface) { + const mysql = ['mysql', 'mariadb'].includes(queryInterface.sequelize.getDialect()); + const q = mysql ? (s) => `\`${s}\`` : (s) => `"${s}"`; + const [domains] = await queryInterface.sequelize.query( + `SELECT id FROM ${q('ExternalDomains')} WHERE id NOT IN (SELECT ${q('externalDomainId')} FROM ${q('DkimKeys')})`, + ); + const now = new Date(); + for (const { id } of domains) { + const { selector, privateKey, publicKey } = generateDkimKeyPair(); + await queryInterface.bulkInsert('DkimKeys', [{ + externalDomainId: id, + selector, + privateKey, + publicKey, + status: 'active', + createdAt: now, + updatedAt: now, + }]); + } + }, + + async down(queryInterface) { + // Backfill only — keys for still-existing domains are kept on rollback + // of this migration; dropping the table removes them anyway. + void queryInterface; + }, +}; diff --git a/create-a-container/routers/api/v1/external-domains.js b/create-a-container/routers/api/v1/external-domains.js index bd1527b0..d8711f29 100644 --- a/create-a-container/routers/api/v1/external-domains.js +++ b/create-a-container/routers/api/v1/external-domains.js @@ -1,12 +1,16 @@ /** - * /api/v1/external-domains — admin-only CRUD. + * /api/v1/external-domains — admin-only CRUD + per-domain mail enablement. * The Cloudflare API key is write-only: never returned in any response. + * Mail flags are read-only here — they change only via the /mail endpoints. */ const express = require('express'); -const { ExternalDomain, Site } = require('../../../models'); +const { ExternalDomain, Site, DkimKey, MailAccount, Setting } = require('../../../models'); const { apiAuth, apiAdmin, asyncHandler, ok, created, noContent, ApiError } = require('../../../middlewares/api'); +const { generateDkimKeyPair } = require('../../../utils/dkim'); +const { buildMailDnsRecords, checkMailDns } = require('../../../utils/mail-dns'); +const { getMailIp } = require('../../../utils/mail-host'); const router = express.Router(); @@ -23,6 +27,11 @@ function serialize(d) { site: d.site ? { id: d.site.id, name: d.site.name } : null, authServer: d.authServer, hasCloudflareApiKey: !!d.cloudflareApiKey, + // Read-only mail state (issue #67) — mutate via the /mail endpoints. + mailEnabled: d.mailEnabled, + mailDnsVerified: d.mailDnsVerified, + mailMxVerified: d.mailMxVerified, + mailDnsCheckedAt: d.mailDnsCheckedAt, }; } @@ -62,6 +71,10 @@ router.post( siteId: siteId || null, authServer: authServer || null, }); + // Every domain gets a DKIM key at create so the DNS records can be + // published before mail is ever enabled (existing domains were + // backfilled by migration 20261008000007). + await DkimKey.create({ externalDomainId: d.id, ...generateDkimKeyPair() }); return created(res, serialize(d)); }), ); @@ -99,4 +112,110 @@ router.delete( }), ); +// --------------------------------------------------------------------------- +// Mail enablement (issue #67). Send gate = mailEnabled && mailDnsVerified; +// receive gate = mailEnabled && mailMxVerified. Re-verification is manual. +// --------------------------------------------------------------------------- + +async function loadDomain(id) { + const d = await ExternalDomain.findByPk(id); + if (!d) throw new ApiError(404, 'not_found', 'External domain not found'); + return d; +} + +/** Everything buildMailDnsRecords/checkMailDns need for one domain. */ +async function mailDnsContext(d) { + const settings = await Setting.getMultiple([ + 'mail_hostname', 'mail_spf_include', 'mail_dns_check_resolvers', + ]); + const dkimKey = await DkimKey.findOne({ + where: { externalDomainId: d.id, status: 'active' }, + order: [['createdAt', 'DESC']], + }); + return { + domain: d.name, + mailIp: await getMailIp(), + mailHostname: settings.mail_hostname || null, + spfInclude: settings.mail_spf_include || null, + dkimKey: dkimKey ? { selector: dkimKey.selector, publicKey: dkimKey.publicKey } : null, + resolvers: (settings.mail_dns_check_resolvers || '') + .split(',').map((s) => s.trim()).filter(Boolean), + }; +} + +/** Run Check DNS, persist the outcome, and flip the verification gates. */ +async function runDnsCheck(d) { + const context = await mailDnsContext(d); + const result = await checkMailDns(context); + const accounts = await MailAccount.findAll({ + where: { externalDomainId: d.id, localPart: ['postmaster', 'abuse'] }, + attributes: ['localPart'], + }); + for (const role of ['postmaster', 'abuse']) { + if (!accounts.some((a) => a.localPart === role)) { + result.warnings.push(`No ${role}@${d.name} mailbox exists yet (admins create role mailboxes).`); + } + } + await d.update({ + mailDnsCheckedAt: new Date(), + mailDnsCheckResult: result, + mailDnsVerified: result.spf.pass && result.dkim.pass && result.dmarc.pass, + mailMxVerified: result.mx.pass, + }); + return result; +} + +router.get( + '/:id/mail/dns', + asyncHandler(async (req, res) => { + const d = await loadDomain(req.params.id); + const context = await mailDnsContext(d); + return ok(res, { + records: buildMailDnsRecords(context), + mailEnabled: d.mailEnabled, + mailDnsVerified: d.mailDnsVerified, + mailMxVerified: d.mailMxVerified, + lastCheck: d.mailDnsCheckedAt + ? { checkedAt: d.mailDnsCheckedAt, result: d.mailDnsCheckResult } + : null, + }); + }), +); + +router.post( + '/:id/mail/check-dns', + asyncHandler(async (req, res) => { + const d = await loadDomain(req.params.id); + const result = await runDnsCheck(d); + return ok(res, { ...serialize(d), check: result }); + }), +); + +router.post( + '/:id/mail/enable', + asyncHandler(async (req, res) => { + const d = await loadDomain(req.params.id); + const result = await runDnsCheck(d); + // SPF/DKIM/DMARC are hard requirements to send; MX failure only means + // the domain can't receive yet, so it just warns. + if (!d.mailDnsVerified) { + throw new ApiError(422, 'dns_not_configured', + 'SPF, DKIM, or DMARC records are missing or wrong — publish the records from GET …/mail/dns and re-check'); + } + await d.update({ mailEnabled: true }); + return ok(res, { ...serialize(d), check: result }); + }), +); + +router.post( + '/:id/mail/disable', + asyncHandler(async (req, res) => { + const d = await loadDomain(req.params.id); + // Keys and config are kept; accounts stop authenticating immediately + // because the SQL views gate on mailEnabled. + await d.update({ mailEnabled: false }); + return ok(res, serialize(d)); + }), +); + module.exports = router; diff --git a/create-a-container/routers/api/v1/settings.js b/create-a-container/routers/api/v1/settings.js index bbe3e3dc..e359d36c 100644 --- a/create-a-container/routers/api/v1/settings.js +++ b/create-a-container/routers/api/v1/settings.js @@ -4,7 +4,7 @@ const express = require('express'); const { Setting } = require('../../../models'); -const { apiAuth, apiAdmin, asyncHandler, ok } = require('../../../middlewares/api'); +const { apiAuth, apiAdmin, asyncHandler, ok, fail } = require('../../../middlewares/api'); const router = express.Router(); @@ -18,6 +18,18 @@ const KEYS = [ 'netbox_token', 'banner_message', 'usage_psi_probe_limit', + // Mail service (issue #67) + 'mail_hostname', + 'mail_unsubscribe_base_url', + 'mail_relayhost', + 'mail_relayhost_username', + 'mail_relayhost_password', + 'mail_spf_include', + 'mail_dns_check_resolvers', + 'mail_db_host', + 'mail_self_managed_site_id', + 'mail_default_quota_mb', + 'mail_message_size_limit_mb', ]; router.get( @@ -38,6 +50,17 @@ router.get( netboxToken: settings.netbox_token || '', bannerMessage: settings.banner_message || '', usagePsiProbeLimit: settings.usage_psi_probe_limit || '', + mailHostname: settings.mail_hostname || '', + mailUnsubscribeBaseUrl: settings.mail_unsubscribe_base_url || '', + mailRelayhost: settings.mail_relayhost || '', + mailRelayhostUsername: settings.mail_relayhost_username || '', + mailRelayhostPassword: settings.mail_relayhost_password || '', + mailSpfInclude: settings.mail_spf_include || '', + mailDnsCheckResolvers: settings.mail_dns_check_resolvers || '', + mailDbHost: settings.mail_db_host || '', + mailSelfManagedSiteId: settings.mail_self_managed_site_id || '', + mailDefaultQuotaMb: settings.mail_default_quota_mb || '', + mailMessageSizeLimitMb: settings.mail_message_size_limit_mb || '', }); }), ); @@ -53,8 +76,24 @@ router.put( netboxToken, bannerMessage, usagePsiProbeLimit, + mailHostname, + mailUnsubscribeBaseUrl, + mailRelayhost, + mailRelayhostUsername, + mailRelayhostPassword, + mailSpfInclude, + mailDnsCheckResolvers, + mailDbHost, + mailSelfManagedSiteId, + mailDefaultQuotaMb, + mailMessageSizeLimitMb, } = req.body || {}; + // A relayhost sends from the relay's IPs, so SPF must delegate to it. + if ((mailRelayhost || '').trim() && !(mailSpfInclude || '').trim()) { + return fail(res, 422, 'validation_failed', 'mailSpfInclude is required when a relayhost is set'); + } + const envVars = []; if (Array.isArray(defaultContainerEnvVars)) { for (const e of defaultContainerEnvVars) { @@ -81,6 +120,22 @@ router.put( Number.isNaN(psiParsed) || psiParsed < 0 ? '' : String(psiParsed), ); + const positiveIntOrEmpty = (value) => { + const n = parseInt(String(value ?? '').trim(), 10); + return Number.isNaN(n) || n < 1 ? '' : String(n); + }; + await Setting.set('mail_hostname', (mailHostname || '').trim()); + await Setting.set('mail_unsubscribe_base_url', (mailUnsubscribeBaseUrl || '').trim().replace(/\/+$/, '')); + await Setting.set('mail_relayhost', (mailRelayhost || '').trim()); + await Setting.set('mail_relayhost_username', (mailRelayhostUsername || '').trim()); + await Setting.set('mail_relayhost_password', mailRelayhostPassword || ''); + await Setting.set('mail_spf_include', (mailSpfInclude || '').trim()); + await Setting.set('mail_dns_check_resolvers', (mailDnsCheckResolvers || '').trim()); + await Setting.set('mail_db_host', (mailDbHost || '').trim()); + await Setting.set('mail_self_managed_site_id', positiveIntOrEmpty(mailSelfManagedSiteId)); + await Setting.set('mail_default_quota_mb', positiveIntOrEmpty(mailDefaultQuotaMb)); + await Setting.set('mail_message_size_limit_mb', positiveIntOrEmpty(mailMessageSizeLimitMb)); + return ok(res, { saved: true }); }), ); diff --git a/create-a-container/utils/__tests__/mail-dns.test.js b/create-a-container/utils/__tests__/mail-dns.test.js new file mode 100644 index 00000000..eb2acfc0 --- /dev/null +++ b/create-a-container/utils/__tests__/mail-dns.test.js @@ -0,0 +1,140 @@ +/** + * Unit tests for utils/mail-dns.js — record building and the Check DNS + * pass/fail gates, with an injected fake resolver (no network). + */ + +const { + buildMailDnsRecords, + checkMailDns, + ipv4InCidr, + spfRecordPasses, + dkimRecordPublicKey, +} = require('../mail-dns'); + +const DKIM = { selector: 'os20261008', publicKey: 'MIIBIjANBgkq+TESTKEY' }; + +function fakeResolver({ txt = {}, mx = [], a = {}, ptr = {} } = {}) { + return { + resolveTxt: async (name) => { + if (!(name in txt)) { const e = new Error('ENOTFOUND'); e.code = 'ENOTFOUND'; throw e; } + return txt[name].map((record) => [record]); + }, + resolveMx: async () => { + if (!mx.length) { const e = new Error('ENODATA'); e.code = 'ENODATA'; throw e; } + return mx; + }, + resolve4: async (host) => { + if (!(host in a)) { const e = new Error('ENOTFOUND'); e.code = 'ENOTFOUND'; throw e; } + return a[host]; + }, + reverse: async (ip) => { + if (!(ip in ptr)) { const e = new Error('ENOTFOUND'); e.code = 'ENOTFOUND'; throw e; } + return ptr[ip]; + }, + }; +} + +describe('buildMailDnsRecords', () => { + test('produces SPF, DKIM, DMARC, and MX records', () => { + const records = buildMailDnsRecords({ + domain: 'example.com', + mailIp: '203.0.113.9', + mailHostname: 'mail.example.com', + spfInclude: null, + dkimKey: DKIM, + }); + expect(records).toEqual([ + { type: 'TXT', name: 'example.com', value: 'v=spf1 ip4:203.0.113.9 -all' }, + { type: 'TXT', name: 'os20261008._domainkey.example.com', value: `v=DKIM1; k=rsa; p=${DKIM.publicKey}` }, + { type: 'TXT', name: '_dmarc.example.com', value: 'v=DMARC1; p=quarantine; rua=mailto:dmarc@example.com' }, + { type: 'MX', name: 'example.com', value: '10 mail.example.com.' }, + ]); + }); + + test('adds include when a relayhost SPF include is configured', () => { + const [spf] = buildMailDnsRecords({ + domain: 'example.com', mailIp: '203.0.113.9', mailHostname: null, + spfInclude: 'relay.example.net', dkimKey: null, + }); + expect(spf.value).toBe('v=spf1 ip4:203.0.113.9 include:relay.example.net -all'); + }); +}); + +describe('spf matching', () => { + test('ipv4InCidr handles plain addresses and prefixes', () => { + expect(ipv4InCidr('203.0.113.9', '203.0.113.9')).toBe(true); + expect(ipv4InCidr('203.0.113.9', '203.0.113.0/24')).toBe(true); + expect(ipv4InCidr('203.0.114.9', '203.0.113.0/24')).toBe(false); + expect(ipv4InCidr('not-an-ip', '203.0.113.0/24')).toBe(false); + }); + + test('spfRecordPasses on ip4 or the configured include', () => { + expect(spfRecordPasses('v=spf1 ip4:203.0.113.0/24 -all', { mailIp: '203.0.113.9' })).toBe(true); + expect(spfRecordPasses('v=spf1 include:relay.net -all', { mailIp: '203.0.113.9', spfInclude: 'relay.net' })).toBe(true); + expect(spfRecordPasses('v=spf1 a mx -all', { mailIp: '203.0.113.9' })).toBe(false); + }); +}); + +test('dkimRecordPublicKey extracts and de-whitespaces p=', () => { + expect(dkimRecordPublicKey('v=DKIM1; k=rsa; p=AB CD')).toBe('ABCD'); + expect(dkimRecordPublicKey('v=DKIM1; k=rsa')).toBeNull(); +}); + +describe('checkMailDns', () => { + const happyZone = { + txt: { + 'example.com': ['v=spf1 ip4:203.0.113.9 -all'], + 'os20261008._domainkey.example.com': [`v=DKIM1; k=rsa; p=${DKIM.publicKey}`], + '_dmarc.example.com': ['v=DMARC1; p=quarantine'], + }, + mx: [{ exchange: 'mail.example.com', priority: 10 }], + a: { 'mail.example.com': ['203.0.113.9'] }, + ptr: { '203.0.113.9': ['mail.example.com'] }, + }; + const params = { + domain: 'example.com', + mailIp: '203.0.113.9', + mailHostname: 'mail.example.com', + spfInclude: null, + dkimKey: DKIM, + }; + + test('all green when every record is correct', async () => { + const result = await checkMailDns({ ...params, resolver: fakeResolver(happyZone) }); + expect(result.spf.pass).toBe(true); + expect(result.dkim.pass).toBe(true); + expect(result.dmarc.pass).toBe(true); + expect(result.mx.pass).toBe(true); + expect(result.ptr).toEqual({ name: 'mail.example.com', forwardConfirmed: true }); + expect(result.warnings).toEqual([]); + }); + + test('fails each gate independently on an empty zone', async () => { + const result = await checkMailDns({ ...params, resolver: fakeResolver() }); + expect(result.spf.pass).toBe(false); + expect(result.dkim.pass).toBe(false); + expect(result.dmarc.pass).toBe(false); + expect(result.mx.pass).toBe(false); + expect(result.warnings).toContain('No PTR record for 203.0.113.9.'); + }); + + test('wrong DKIM key fails even when a record is published', async () => { + const zone = { + ...happyZone, + txt: { ...happyZone.txt, 'os20261008._domainkey.example.com': ['v=DKIM1; k=rsa; p=SOMEOTHERKEY'] }, + }; + const result = await checkMailDns({ ...params, resolver: fakeResolver(zone) }); + expect(result.dkim.pass).toBe(false); + expect(result.dkim.published).toBe(true); + }); + + test('warns when PTR does not match mail_hostname', async () => { + const zone = { + ...happyZone, + ptr: { '203.0.113.9': ['other.example.net'] }, + a: { ...happyZone.a, 'other.example.net': ['203.0.113.9'] }, + }; + const result = await checkMailDns({ ...params, resolver: fakeResolver(zone) }); + expect(result.warnings).toContain('PTR other.example.net does not match mail_hostname mail.example.com.'); + }); +}); diff --git a/create-a-container/utils/dkim.js b/create-a-container/utils/dkim.js new file mode 100644 index 00000000..a9c21147 --- /dev/null +++ b/create-a-container/utils/dkim.js @@ -0,0 +1,36 @@ +/** + * DKIM key management (issue #67). RSA-2048 per the plan; rotation is a + * follow-up (DkimKeys.status already distinguishes active/retired). + */ + +const crypto = require('crypto'); + +/** Date-stamped selector, e.g. "os20261008" — unique enough per domain until rotation lands. */ +function defaultSelector(now = new Date()) { + const y = now.getUTCFullYear(); + const m = String(now.getUTCMonth() + 1).padStart(2, '0'); + const d = String(now.getUTCDate()).padStart(2, '0'); + return `os${y}${m}${d}`; +} + +/** + * Generate an RSA-2048 DKIM key pair. + * @returns {{selector: string, privateKey: string, publicKey: string}} + * privateKey: PEM (PKCS#1, what OpenDKIM reads); publicKey: base64 DER + * (SPKI) — the p= value of the DNS TXT record. + */ +function generateDkimKeyPair(selector = defaultSelector()) { + const { publicKey, privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 }); + return { + selector, + privateKey: privateKey.export({ type: 'pkcs1', format: 'pem' }).toString(), + publicKey: publicKey.export({ type: 'spki', format: 'der' }).toString('base64'), + }; +} + +/** The TXT record value to publish at ._domainkey.. */ +function dkimDnsRecordValue(publicKey) { + return `v=DKIM1; k=rsa; p=${publicKey}`; +} + +module.exports = { defaultSelector, generateDkimKeyPair, dkimDnsRecordValue }; diff --git a/create-a-container/utils/mail-dns.js b/create-a-container/utils/mail-dns.js new file mode 100644 index 00000000..c0d74eba --- /dev/null +++ b/create-a-container/utils/mail-dns.js @@ -0,0 +1,207 @@ +/** + * Mail DNS records + Check DNS (issue #67). + * + * buildMailDnsRecords: the records an admin must publish for a domain. + * checkMailDns: verifies them against public resolvers. Send gate = + * SPF + DKIM + DMARC; receive gate = MX. PTR problems are warnings only. + * + * SPF evaluation is deliberately record-based per the plan: pass when the + * record's ip4 mechanisms cover the mail IP, or when it contains the + * configured mail_spf_include. (Full RFC 7208 evaluation — a/mx/redirect + * chasing — is a follow-up.) + */ + +const dns = require('dns'); +const { dkimDnsRecordValue } = require('./dkim'); + +const DEFAULT_RESOLVERS = ['1.1.1.1', '8.8.8.8']; + +/** + * @param {object} p + * @param {string} p.domain + * @param {string|null} p.mailIp - externalIp of the mail host's site + * @param {string|null} p.mailHostname - mail_hostname setting (MX target / HELO) + * @param {string|null} p.spfInclude - mail_spf_include setting (required with a relayhost) + * @param {{selector: string, publicKey: string}|null} p.dkimKey + * @returns {Array<{type: string, name: string, value: string}>} + */ +function buildMailDnsRecords({ domain, mailIp, mailHostname, spfInclude, dkimKey }) { + const spfParts = ['v=spf1']; + if (mailIp) spfParts.push(`ip4:${mailIp}`); + if (spfInclude) spfParts.push(`include:${spfInclude}`); + spfParts.push('-all'); + + const records = [ + { type: 'TXT', name: domain, value: spfParts.join(' ') }, + ]; + if (dkimKey) { + records.push({ + type: 'TXT', + name: `${dkimKey.selector}._domainkey.${domain}`, + value: dkimDnsRecordValue(dkimKey.publicKey), + }); + } + records.push({ + type: 'TXT', + name: `_dmarc.${domain}`, + value: `v=DMARC1; p=quarantine; rua=mailto:dmarc@${domain}`, + }); + if (mailHostname) { + records.push({ type: 'MX', name: domain, value: `10 ${mailHostname}.` }); + } + return records; +} + +/** ip4:[/] mechanism match for an IPv4 address. */ +function ipv4InCidr(ip, cidr) { + const [base, prefixStr] = cidr.split('/'); + const prefix = prefixStr === undefined ? 32 : parseInt(prefixStr, 10); + if (!Number.isInteger(prefix) || prefix < 0 || prefix > 32) return false; + const toInt = (addr) => { + const parts = addr.split('.').map(Number); + if (parts.length !== 4 || parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)) return null; + return ((parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]) >>> 0; + }; + const ipInt = toInt(ip); + const baseInt = toInt(base); + if (ipInt === null || baseInt === null) return false; + if (prefix === 0) return true; + const mask = (0xffffffff << (32 - prefix)) >>> 0; + return (ipInt & mask) === (baseInt & mask); +} + +/** + * Record-based SPF pass: any ip4 mechanism covers mailIp, or the record + * contains include:. + */ +function spfRecordPasses(record, { mailIp, spfInclude }) { + const mechanisms = record.trim().split(/\s+/).slice(1); + for (const mech of mechanisms) { + const m = mech.replace(/^\+/, ''); + if (mailIp && m.toLowerCase().startsWith('ip4:') && ipv4InCidr(mailIp, m.slice(4))) return true; + if (spfInclude && m.toLowerCase() === `include:${spfInclude.toLowerCase()}`) return true; + } + return false; +} + +/** Strip quotes/whitespace variations; TXT comes back as chunk arrays. */ +function joinTxt(chunks) { + return chunks.join(''); +} + +/** Normalize a DKIM TXT record's p= value for comparison. */ +function dkimRecordPublicKey(record) { + const match = /(?:^|;)\s*p=([^;]*)/.exec(record); + return match ? match[1].replace(/\s+/g, '') : null; +} + +function makeResolver(resolvers) { + const resolver = new dns.promises.Resolver({ timeout: 5000, tries: 2 }); + resolver.setServers(resolvers && resolvers.length ? resolvers : DEFAULT_RESOLVERS); + return resolver; +} + +async function resolveTxtSafe(resolver, name) { + try { + return (await resolver.resolveTxt(name)).map(joinTxt); + } catch { + return []; + } +} + +/** + * Run the Check DNS pass for a domain. + * + * @param {object} p + * @param {string} p.domain + * @param {string|null} p.mailIp + * @param {string|null} p.mailHostname + * @param {string|null} p.spfInclude + * @param {{selector: string, publicKey: string}|null} p.dkimKey + * @param {string[]} [p.resolvers] - mail_dns_check_resolvers setting + * @param {object} [p.resolver] - injectable for tests + * @returns {Promise} stored as ExternalDomains.mailDnsCheckResult: + * { checkedAt, spf|dkim|dmarc|mx: { pass, ... }, ptr: {...}, warnings: [] } + */ +async function checkMailDns({ domain, mailIp, mailHostname, spfInclude, dkimKey, resolvers, resolver }) { + const r = resolver || makeResolver(resolvers); + const result = { checkedAt: new Date().toISOString(), warnings: [] }; + + // SPF + const txt = await resolveTxtSafe(r, domain); + const spfRecord = txt.find((t) => /^v=spf1(\s|$)/i.test(t)) || null; + result.spf = { + pass: !!(spfRecord && spfRecordPasses(spfRecord, { mailIp, spfInclude })), + record: spfRecord, + }; + if (!mailIp) result.warnings.push('No mail IP known yet — no agent holds the mail-host claim.'); + + // DKIM + if (dkimKey) { + const dkimTxt = await resolveTxtSafe(r, `${dkimKey.selector}._domainkey.${domain}`); + const published = dkimTxt.map(dkimRecordPublicKey).find((p) => p) || null; + result.dkim = { + pass: !!published && published === dkimKey.publicKey.replace(/\s+/g, ''), + selector: dkimKey.selector, + published: !!published, + }; + } else { + result.dkim = { pass: false, selector: null, published: false }; + result.warnings.push('No active DKIM key for this domain.'); + } + + // DMARC + const dmarcTxt = await resolveTxtSafe(r, `_dmarc.${domain}`); + const dmarcRecord = dmarcTxt.find((t) => /^v=DMARC1(\s*;|$)/i.test(t)) || null; + result.dmarc = { pass: !!dmarcRecord, record: dmarcRecord }; + + // MX → must resolve to the mail IP (receive gate) + let mxPass = false; + let mxHosts = []; + try { + const mx = await r.resolveMx(domain); + mxHosts = mx.sort((a, b) => a.priority - b.priority).map((m) => m.exchange); + for (const host of mxHosts) { + try { + const addrs = await r.resolve4(host); + if (mailIp && addrs.includes(mailIp)) { mxPass = true; break; } + } catch { /* try next exchange */ } + } + } catch { /* no MX */ } + result.mx = { pass: mxPass, hosts: mxHosts }; + + // PTR (warnings only) + if (mailIp) { + try { + const names = await r.reverse(mailIp); + const ptr = names[0] || null; + let forwardConfirmed = false; + if (ptr) { + try { + forwardConfirmed = (await r.resolve4(ptr)).includes(mailIp); + } catch { /* not forward-confirmed */ } + } + result.ptr = { name: ptr, forwardConfirmed }; + if (!ptr) result.warnings.push(`No PTR record for ${mailIp}.`); + else if (!forwardConfirmed) result.warnings.push(`PTR ${ptr} is not forward-confirmed.`); + else if (mailHostname && ptr.toLowerCase() !== mailHostname.toLowerCase()) { + result.warnings.push(`PTR ${ptr} does not match mail_hostname ${mailHostname}.`); + } + } catch { + result.ptr = { name: null, forwardConfirmed: false }; + result.warnings.push(`No PTR record for ${mailIp}.`); + } + } + + return result; +} + +module.exports = { + DEFAULT_RESOLVERS, + buildMailDnsRecords, + checkMailDns, + // exported for tests + ipv4InCidr, + spfRecordPasses, + dkimRecordPublicKey, +}; diff --git a/create-a-container/utils/mail-host.js b/create-a-container/utils/mail-host.js new file mode 100644 index 00000000..8d8ba6be --- /dev/null +++ b/create-a-container/utils/mail-host.js @@ -0,0 +1,90 @@ +/** + * The mail-host claim (issue #67): at most one agent runs the mail stack. + * + * The claim is granted to the first agent that checks in with the `mail` + * service group enabled and everything installed. Postgres/SQLite enforce + * single-holder with a partial unique index; the transaction here serializes + * the race on MySQL/MariaDB too. Mailboxes are local Maildir (Dovecot 2.4 CE + * has no director/replicator), hence one host at a time. + */ + +const { Op } = require('sequelize'); +const { sequelize, Agent, Site, Setting } = require('../models'); + +const MAIL_SERVICE = 'mail'; + +/** The agent currently holding the mail-host claim (with its site), or null. */ +async function getMailHostAgent() { + return Agent.findOne({ + where: { mailHostSince: { [Op.ne]: null } }, + include: [{ model: Site, as: 'site', attributes: ['id', 'name', 'externalIp'] }], + }); +} + +/** + * The IP mail is sent from: externalIp of the mail host's site, or of the + * mail_self_managed_site_id site when no agent holds the claim. + */ +async function getMailIp() { + const holder = await getMailHostAgent(); + if (holder?.site?.externalIp) return holder.site.externalIp; + const selfManagedSiteId = parseInt(await Setting.get('mail_self_managed_site_id'), 10); + if (Number.isInteger(selfManagedSiteId)) { + const site = await Site.findByPk(selfManagedSiteId, { attributes: ['externalIp'] }); + return site?.externalIp || null; + } + return null; +} + +/** + * Process an agent's mail claim at check-in. + * + * @param {object} agent - the checking-in Agent row + * @param {object} p + * @param {boolean} p.wantsMail - agent reports `mail` in enabledServices + * @param {string[]} p.missingBinaries - as reported by the agent + * @returns {Promise<{status: string, missing?: string[]}>} + * status: holder | conflict | unsupported | missing_packages | disabled + */ +async function processMailClaim(agent, { wantsMail, missingBinaries }) { + if (!wantsMail) { + // Turning `mail` off releases the claim; the agent stops/disables the + // units and leaves configs and /var/vmail in place. + if (agent.mailHostSince) await agent.update({ mailHostSince: null }); + return { status: 'disabled' }; + } + if (sequelize.getDialect() === 'sqlite') { + return { status: 'unsupported' }; + } + if (Array.isArray(missingBinaries) && missingBinaries.length > 0) { + return { status: 'missing_packages', missing: missingBinaries }; + } + return sequelize.transaction(async (transaction) => { + const holder = await Agent.findOne({ + where: { mailHostSince: { [Op.ne]: null } }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (!holder) { + await agent.update({ mailHostSince: new Date() }, { transaction }); + return { status: 'holder' }; + } + if (holder.id === agent.id) return { status: 'holder' }; + return { status: 'conflict' }; + }); +} + +/** Admin release — the next qualifying check-in takes the claim. */ +async function releaseMailHost() { + const holder = await getMailHostAgent(); + if (holder) await holder.update({ mailHostSince: null }); + return holder; +} + +module.exports = { + MAIL_SERVICE, + getMailHostAgent, + getMailIp, + processMailClaim, + releaseMailHost, +}; From 733d303d8e3ad6ee894463a1d4dc29101ddac1d8 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:07:21 -0700 Subject: [PATCH 04/19] mail: mail-host claim and agent snapshot section (#67) Agents report enabledServices + missingBinaries at check-in; the first agent reporting 'mail' with everything installed takes the single mail-host claim (conflict for others, unsupported on SQLite, missing_packages when binaries are absent; turning mail off releases it). Only the holder receives config.mail: DKIM private keys for can-send domains, receiving domains, mail_hostname, relayhost, limits, and DB connection (no DB passwords - those stay in /etc/default/container-creator). Remote agents are pinned to their first API key (DELETE /agents/:id/api-key-pin to clear). New admin router /api/v1/mail: GET /host, POST /host/release, GET /ptr. --- create-a-container/routers/api/v1/agents.js | 49 ++++++++++++++- create-a-container/routers/api/v1/index.js | 1 + create-a-container/routers/api/v1/mail.js | 70 +++++++++++++++++++++ create-a-container/utils/agent-config.js | 64 ++++++++++++++++++- create-a-container/utils/mail-dns.js | 47 ++++++++------ 5 files changed, 208 insertions(+), 23 deletions(-) create mode 100644 create-a-container/routers/api/v1/mail.js diff --git a/create-a-container/routers/api/v1/agents.js b/create-a-container/routers/api/v1/agents.js index 3accd43e..64f9bf9a 100644 --- a/create-a-container/routers/api/v1/agents.js +++ b/create-a-container/routers/api/v1/agents.js @@ -9,8 +9,12 @@ const express = require('express'); const { Agent, Site, Container, Volume } = require('../../../models'); -const { apiAuth, apiAdmin, localhostOrAdmin, asyncHandler, ok, fail } = require('../../../middlewares/api'); -const { buildAgentConfig, computeConfigEtag } = require('../../../utils/agent-config'); +const { apiAuth, apiAdmin, localhostOrAdmin, isLocalhostRequest, asyncHandler, ok, noContent, fail } = require('../../../middlewares/api'); +const { buildAgentConfig, buildMailSnapshot, computeConfigEtag } = require('../../../utils/agent-config'); +const { MAIL_SERVICE, processMailClaim } = require('../../../utils/mail-host'); + +const env = process.env.NODE_ENV || 'development'; +const dbConfig = require('../../../config/config.js')[env]; const router = express.Router(); @@ -53,7 +57,7 @@ async function applyVolumeResults(siteId, volumesResult) { } router.post('/', localhostOrAdmin, asyncHandler(async (req, res) => { - const { siteId, hostname, ipv4Address, services, volumes } = req.body || {}; + const { siteId, hostname, ipv4Address, services, volumes, enabledServices, missingBinaries } = req.body || {}; const parsedSiteId = typeof siteId === 'number' ? siteId : Number(siteId); if (!Number.isInteger(parsedSiteId) || !hostname || typeof hostname !== 'string') { return fail(res, 422, 'validation_failed', 'siteId and hostname are required'); @@ -61,22 +65,47 @@ router.post('/', localhostOrAdmin, asyncHandler(async (req, res) => { // Record the check-in. Skipped during bootstrap (the site row doesn't exist // yet) since the foreign key has nothing to point at. + let mailClaim = null; const site = await Site.findByPk(parsedSiteId); if (site) { const [agent] = await Agent.findOrCreate({ where: { siteId: parsedSiteId, hostname }, }); + const isLocal = isLocalhostRequest(req); + // Remote agents are pinned to the API key of their first check-in so a + // leaked lesser key can't impersonate an established agent (issue #67). + // Admins clear a pin via DELETE /agents/:id/api-key-pin. + if (!isLocal && agent.apiKeyId && req.apiKey && agent.apiKeyId !== req.apiKey.id) { + return fail(res, 403, 'agent_key_mismatch', 'This agent is pinned to a different API key'); + } await agent.update({ ipv4Address: ipv4Address || null, services: services || null, + enabledServices: Array.isArray(enabledServices) ? enabledServices : null, + missingBinaries: Array.isArray(missingBinaries) ? missingBinaries : null, + isLocal, + apiKeyId: agent.apiKeyId || (!isLocal && req.apiKey ? req.apiKey.id : null), lastCheckinAt: new Date(), }); // Transition Volume.status from the agent's per-volume directory results, // scoped to this site so a check-in can't touch another site's volumes. await applyVolumeResults(parsedSiteId, volumes); + // Mail-host claim: first agent reporting `mail` with everything installed + // wins; the rest see conflict (or unsupported on SQLite). + mailClaim = await processMailClaim(agent, { + wantsMail: Array.isArray(enabledServices) && enabledServices.includes(MAIL_SERVICE), + missingBinaries, + }); } const config = await buildAgentConfig(parsedSiteId); + if (mailClaim && mailClaim.status !== 'disabled') { + config.mailStatus = mailClaim.status; + // Only the claim holder receives the mail section (DKIM private keys etc.). + if (mailClaim.status === 'holder') { + config.mail = await buildMailSnapshot(dbConfig); + } + } // Manual conditional-request handling: Express's built-in ETag/fresh logic // (res.send + req.fresh) only produces 304s for GET/HEAD, and the check-in // is a POST. @@ -101,6 +130,11 @@ router.get('/', apiAuth, apiAdmin, asyncHandler(async (req, res) => { hostname: a.hostname, ipv4Address: a.ipv4Address, services: a.services, + enabledServices: a.enabledServices, + missingBinaries: a.missingBinaries, + mailHostSince: a.mailHostSince, + isLocal: a.isLocal, + hasApiKeyPin: !!a.apiKeyId, lastCheckinAt: a.lastCheckinAt, // Computed server-side so UI staleness judgments don't depend on the // client's clock. @@ -110,4 +144,13 @@ router.get('/', apiAuth, apiAdmin, asyncHandler(async (req, res) => { }))); })); +// Clear a remote agent's API-key pin so it can re-pin on its next check-in +// (e.g. after rotating the agent's key). +router.delete('/:id/api-key-pin', apiAuth, apiAdmin, asyncHandler(async (req, res) => { + const agent = await Agent.findByPk(req.params.id); + if (!agent) return fail(res, 404, 'not_found', 'Agent not found'); + await agent.update({ apiKeyId: null }); + return noContent(res); +})); + module.exports = router; diff --git a/create-a-container/routers/api/v1/index.js b/create-a-container/routers/api/v1/index.js index 93bd01d0..c27f2308 100644 --- a/create-a-container/routers/api/v1/index.js +++ b/create-a-container/routers/api/v1/index.js @@ -92,6 +92,7 @@ router.use('/groups', require('./groups')); router.use('/users', require('./users')); router.use('/apikeys', require('../../../resources/apikeys/router')); router.use('/mail-accounts', require('../../../resources/mailaccounts/router')); +router.use('/mail', require('./mail')); router.use('/settings', require('./settings')); router.use('/jobs', require('./jobs')); router.use('/resource-requests', require('./resource-requests')); diff --git a/create-a-container/routers/api/v1/mail.js b/create-a-container/routers/api/v1/mail.js new file mode 100644 index 00000000..5185e45b --- /dev/null +++ b/create-a-container/routers/api/v1/mail.js @@ -0,0 +1,70 @@ +/** + * /api/v1/mail — admin mail-host management (issue #67). + * + * GET /host who holds the mail-host claim + the mail IP + * POST /host/release release the claim (next qualifying check-in takes it) + * GET /ptr PTR suggestion for the mail_hostname setting + */ + +const express = require('express'); +const { Setting } = require('../../../models'); +const { apiAuth, apiAdmin, asyncHandler, ok } = require('../../../middlewares/api'); +const { getMailHostAgent, getMailIp, releaseMailHost } = require('../../../utils/mail-host'); +const { lookupPtr } = require('../../../utils/mail-dns'); + +const router = express.Router(); + +router.use(apiAuth, apiAdmin); + +router.get( + '/host', + asyncHandler(async (_req, res) => { + const holder = await getMailHostAgent(); + const settings = await Setting.getMultiple(['mail_hostname', 'mail_self_managed_site_id']); + return ok(res, { + agent: holder + ? { + id: holder.id, + siteId: holder.siteId, + siteName: holder.site?.name || null, + hostname: holder.hostname, + mailHostSince: holder.mailHostSince, + enabledServices: holder.enabledServices, + missingBinaries: holder.missingBinaries, + } + : null, + mailIp: await getMailIp(), + mailHostname: settings.mail_hostname || null, + selfManagedSiteId: parseInt(settings.mail_self_managed_site_id, 10) || null, + }); + }), +); + +router.post( + '/host/release', + asyncHandler(async (_req, res) => { + const released = await releaseMailHost(); + return ok(res, { released: !!released }); + }), +); + +router.get( + '/ptr', + asyncHandler(async (_req, res) => { + const mailIp = await getMailIp(); + const settings = await Setting.getMultiple(['mail_hostname', 'mail_dns_check_resolvers']); + const resolvers = (settings.mail_dns_check_resolvers || '') + .split(',').map((s) => s.trim()).filter(Boolean); + const ptr = mailIp ? await lookupPtr({ ip: mailIp, resolvers }) : { name: null, forwardConfirmed: false }; + return ok(res, { + mailIp, + ptr: ptr.name, + forwardConfirmed: ptr.forwardConfirmed, + mailHostname: settings.mail_hostname || null, + // Suggest the forward-confirmed PTR name as mail_hostname. + suggestion: ptr.forwardConfirmed ? ptr.name : null, + }); + }), +); + +module.exports = router; diff --git a/create-a-container/utils/agent-config.js b/create-a-container/utils/agent-config.js index 86a39f39..da416bfb 100644 --- a/create-a-container/utils/agent-config.js +++ b/create-a-container/utils/agent-config.js @@ -9,8 +9,9 @@ const crypto = require('crypto'); const { Op } = require('sequelize'); -const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain, Volume } = require('../models'); +const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain, Volume, DkimKey, Setting } = require('../models'); const { isAgentlessNodeType } = require('./volumes'); +const { DEFAULT_QUOTA_MB } = require('./mail-account'); // Owning host UID/GID for volume directories: Proxmox maps an unprivileged CT's // UID/GID 0 to host 100000, so RW volumes must be owned by 100000 to be writable @@ -225,4 +226,63 @@ function computeConfigEtag(config) { return `"${hash}"`; } -module.exports = { buildAgentConfig, computeConfigEtag }; +/** + * The `mail` snapshot section, sent ONLY to the agent holding the mail-host + * claim (issue #67). Carries everything the agent templates into + * Postfix/Dovecot/OpenDKIM config that is NOT read live from SQL: DKIM + * private keys for can-send domains, receiving domains, mail_hostname, + * relayhost, limits, and the DB connection. The mail_dovecot/mail_postfix + * DB passwords are NOT included — they live in /etc/default/container-creator + * on the mail host (bin/setup-mail-db-roles.sh) or socket auth is used. + * + * @param {object} dbConfig - resolved config/config.js entry for this env + * @returns {Promise} + */ +async function buildMailSnapshot(dbConfig) { + const settings = await Setting.getMultiple([ + 'mail_hostname', 'mail_relayhost', 'mail_relayhost_username', 'mail_relayhost_password', + 'mail_db_host', 'mail_default_quota_mb', 'mail_message_size_limit_mb', + ]); + + const sendDomains = await ExternalDomain.findAll({ + where: { mailEnabled: true, mailDnsVerified: true }, + include: [{ model: DkimKey, as: 'dkimKeys', where: { status: 'active' }, required: true }], + order: [['name', 'ASC'], [{ model: DkimKey, as: 'dkimKeys' }, 'createdAt', 'DESC']], + }); + const receiveDomains = await ExternalDomain.findAll({ + where: { mailEnabled: true, mailMxVerified: true }, + attributes: ['name'], + order: [['name', 'ASC']], + }); + + return { + hostname: settings.mail_hostname || null, + relayhost: settings.mail_relayhost + ? { + host: settings.mail_relayhost, + username: settings.mail_relayhost_username || null, + password: settings.mail_relayhost_password || null, + } + : null, + messageSizeLimitMb: parseInt(settings.mail_message_size_limit_mb, 10) || 25, + defaultQuotaMb: parseInt(settings.mail_default_quota_mb, 10) || DEFAULT_QUOTA_MB, + // DKIM signing material for domains that can send (newest active key). + dkim: sendDomains.map((d) => ({ + domain: d.name, + selector: d.dkimKeys[0].selector, + privateKey: d.dkimKeys[0].privateKey, + })), + // Domains Postfix accepts inbound mail for on :25. + receiveDomains: receiveDomains.map((d) => d.name), + db: { + dialect: dbConfig.dialect, + host: settings.mail_db_host || dbConfig.host || null, + port: dbConfig.port ? Number(dbConfig.port) : null, + database: dbConfig.database || null, + dovecotUser: 'mail_dovecot', + postfixUser: 'mail_postfix', + }, + }; +} + +module.exports = { buildAgentConfig, buildMailSnapshot, computeConfigEtag }; diff --git a/create-a-container/utils/mail-dns.js b/create-a-container/utils/mail-dns.js index c0d74eba..9a0bd1f7 100644 --- a/create-a-container/utils/mail-dns.js +++ b/create-a-container/utils/mail-dns.js @@ -109,6 +109,28 @@ async function resolveTxtSafe(resolver, name) { } } +/** + * Reverse-lookup an IP and forward-confirm the result. Used by Check DNS + * warnings and by GET /mail/ptr to suggest a mail_hostname. + * @returns {Promise<{name: string|null, forwardConfirmed: boolean}>} + */ +async function lookupPtr({ ip, resolvers, resolver }) { + const r = resolver || makeResolver(resolvers); + try { + const names = await r.reverse(ip); + const name = names[0] || null; + let forwardConfirmed = false; + if (name) { + try { + forwardConfirmed = (await r.resolve4(name)).includes(ip); + } catch { /* not forward-confirmed */ } + } + return { name, forwardConfirmed }; + } catch { + return { name: null, forwardConfirmed: false }; + } +} + /** * Run the Check DNS pass for a domain. * @@ -172,24 +194,12 @@ async function checkMailDns({ domain, mailIp, mailHostname, spfInclude, dkimKey, // PTR (warnings only) if (mailIp) { - try { - const names = await r.reverse(mailIp); - const ptr = names[0] || null; - let forwardConfirmed = false; - if (ptr) { - try { - forwardConfirmed = (await r.resolve4(ptr)).includes(mailIp); - } catch { /* not forward-confirmed */ } - } - result.ptr = { name: ptr, forwardConfirmed }; - if (!ptr) result.warnings.push(`No PTR record for ${mailIp}.`); - else if (!forwardConfirmed) result.warnings.push(`PTR ${ptr} is not forward-confirmed.`); - else if (mailHostname && ptr.toLowerCase() !== mailHostname.toLowerCase()) { - result.warnings.push(`PTR ${ptr} does not match mail_hostname ${mailHostname}.`); - } - } catch { - result.ptr = { name: null, forwardConfirmed: false }; - result.warnings.push(`No PTR record for ${mailIp}.`); + const ptr = await lookupPtr({ ip: mailIp, resolver: r }); + result.ptr = ptr; + if (!ptr.name) result.warnings.push(`No PTR record for ${mailIp}.`); + else if (!ptr.forwardConfirmed) result.warnings.push(`PTR ${ptr.name} is not forward-confirmed.`); + else if (mailHostname && ptr.name.toLowerCase() !== mailHostname.toLowerCase()) { + result.warnings.push(`PTR ${ptr.name} does not match mail_hostname ${mailHostname}.`); } } @@ -200,6 +210,7 @@ module.exports = { DEFAULT_RESOLVERS, buildMailDnsRecords, checkMailDns, + lookupPtr, // exported for tests ipv4InCidr, spfRecordPasses, From 6156cd1e1a9f1b61a3b9ca0a967b9ab693f952bc Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:10:28 -0700 Subject: [PATCH 05/19] mail: one-click unsubscribe - keys, tokens, suppressions, public /u (#67) AES-256-GCM tokens over {v, kid, accountId, recipient, iat} (kid bound as AAD, base64url): unforgeable, reveal nothing, and any failure mode answers the same generic 400. Public router at /u (outside /api/v1, before the CSRF guard, per-IP rate limited): GET renders a confirm page and never unsubscribes (prefetch-safe); POST requires the RFC 8058 List-Unsubscribe=One-Click body and idempotently upserts the suppression. MailUnsubscribeKey.ensureActive()/verifiable() manage the single active key; retired keys verify for one more year. The mail snapshot now carries the keys + mail_unsubscribe_base_url to the claim holder so opensource-mail-helper can mint the same tokens. --- create-a-container/app.js | 7 ++ create-a-container/middlewares/rate-limit.js | 37 +++++++ .../models/mail-unsubscribe-key.js | 20 ++++ create-a-container/routers/unsubscribe.js | 96 +++++++++++++++++++ .../utils/__tests__/unsubscribe-token.test.js | 50 ++++++++++ create-a-container/utils/agent-config.js | 16 +++- create-a-container/utils/unsubscribe-token.js | 78 +++++++++++++++ 7 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 create-a-container/middlewares/rate-limit.js create mode 100644 create-a-container/routers/unsubscribe.js create mode 100644 create-a-container/utils/__tests__/unsubscribe-token.test.js create mode 100644 create-a-container/utils/unsubscribe-token.js diff --git a/create-a-container/app.js b/create-a-container/app.js index 482c8d85..5e11c306 100644 --- a/create-a-container/app.js +++ b/create-a-container/app.js @@ -94,6 +94,13 @@ function buildApp({ app.use(express.static('public')); + // Public one-click unsubscribe (RFC 8058) — mounted before the CSRF guard + // because requests come from recipients' mail clients with no session or + // token (the router rate-limits per IP and authenticates via the encrypted + // unsubscribe token itself). Must also bypass oauth2-proxy when the manager + // sits behind one — handled in the nginx template, not here. + app.use('/u', require('./routers/unsubscribe')); + // CSRF guard for every handler that can see the session cookie (CodeQL // js/missing-token-validation). Behavior-preserving: csrfGuard skips // GET/HEAD/OPTIONS and Bearer-only requests, and every state-changing diff --git a/create-a-container/middlewares/rate-limit.js b/create-a-container/middlewares/rate-limit.js new file mode 100644 index 00000000..af509a3b --- /dev/null +++ b/create-a-container/middlewares/rate-limit.js @@ -0,0 +1,37 @@ +/** + * Minimal in-memory fixed-window per-IP rate limiter. Good enough for the + * public unsubscribe endpoint (single-process deployment); swap for a + * store-backed limiter if the manager ever runs multi-process. + */ + +/** + * @param {object} options + * @param {number} options.windowMs - window length + * @param {number} options.max - allowed requests per window per IP + */ +function rateLimit({ windowMs, max }) { + const hits = new Map(); + return function rateLimiter(req, res, next) { + const now = Date.now(); + // Opportunistic pruning keeps the map bounded without a timer. + if (hits.size > 10000) { + for (const [k, v] of hits) { + if (v.resetAt <= now) hits.delete(k); + } + } + const key = req.ip || 'unknown'; + let entry = hits.get(key); + if (!entry || entry.resetAt <= now) { + entry = { count: 0, resetAt: now + windowMs }; + hits.set(key, entry); + } + entry.count += 1; + if (entry.count > max) { + res.set('Retry-After', String(Math.ceil((entry.resetAt - now) / 1000))); + return res.status(429).json({ error: { code: 'rate_limited', message: 'Too many requests' } }); + } + return next(); + }; +} + +module.exports = { rateLimit }; diff --git a/create-a-container/models/mail-unsubscribe-key.js b/create-a-container/models/mail-unsubscribe-key.js index ceddeb4b..3049f30b 100644 --- a/create-a-container/models/mail-unsubscribe-key.js +++ b/create-a-container/models/mail-unsubscribe-key.js @@ -1,4 +1,5 @@ 'use strict'; +const crypto = require('crypto'); const { Model } = require('sequelize'); // Retired keys keep verifying previously issued unsubscribe links for this long. @@ -11,6 +12,25 @@ module.exports = (sequelize, DataTypes) => { if (this.status === 'active') return true; return now - new Date(this.updatedAt).getTime() < RETIRED_KEY_RETENTION_MS; } + + /** The single active key, created on first use. */ + static async ensureActive() { + const existing = await MailUnsubscribeKey.findOne({ + where: { status: 'active' }, + order: [['createdAt', 'DESC']], + }); + if (existing) return existing; + return MailUnsubscribeKey.create({ + kid: crypto.randomBytes(4).toString('hex'), + secret: crypto.randomBytes(32).toString('base64'), + }); + } + + /** Active + recently retired keys, newest first — the set that may verify tokens. */ + static async verifiable() { + const keys = await MailUnsubscribeKey.findAll({ order: [['createdAt', 'DESC']] }); + return keys.filter((k) => k.canVerify()); + } } MailUnsubscribeKey.init({ diff --git a/create-a-container/routers/unsubscribe.js b/create-a-container/routers/unsubscribe.js new file mode 100644 index 00000000..2afd8347 --- /dev/null +++ b/create-a-container/routers/unsubscribe.js @@ -0,0 +1,96 @@ +/** + * Public one-click unsubscribe endpoint (RFC 8058, issue #67). Mounted at + * /u OUTSIDE /api/v1 and before the CSRF guard: requests come from mail + * clients and recipients with no session, token, or credentials. + * + * GET /u/:token confirmation page — a GET NEVER unsubscribes, so link + * scanners that prefetch URLs can't trigger it. + * POST /u/:token body `List-Unsubscribe=One-Click` — verify, upsert the + * suppression, 200. Idempotent. + * + * Every failure mode (bad token, unknown kid, deleted account) returns the + * same generic 400 so the endpoint can't be used to probe for accounts or + * addresses. + */ + +const express = require('express'); +const { MailAccount, MailSuppression, MailUnsubscribeKey } = require('../models'); +const { asyncHandler } = require('../middlewares/api'); +const { rateLimit } = require('../middlewares/rate-limit'); +const { verifyUnsubscribeToken } = require('../utils/unsubscribe-token'); + +const router = express.Router(); + +router.use(rateLimit({ windowMs: 10 * 60 * 1000, max: 60 })); +router.use(express.urlencoded({ extended: false })); + +function escapeHtml(s) { + return String(s) + .replace(/&/g, '&').replace(//g, '>') + .replace(/"/g, '"').replace(/'/g, '''); +} + +function page(title, body) { + return ` + + + + + +${escapeHtml(title)} + + + +
${body}
+ +`; +} + +const GENERIC_400 = page( + 'Invalid link', + '

Invalid link

This unsubscribe link is invalid or has expired.

', +); + +/** Verify the token and load its account; null means "answer generic 400". */ +async function resolveToken(token) { + const keys = await MailUnsubscribeKey.verifiable(); + const payload = verifyUnsubscribeToken(token, keys); + if (!payload) return null; + const account = await MailAccount.findByPk(payload.accountId); + if (!account) return null; + return { account, recipient: payload.recipient }; +} + +router.get('/:token', asyncHandler(async (req, res) => { + const resolved = await resolveToken(req.params.token); + if (!resolved) return res.status(400).type('html').send(GENERIC_400); + const recipient = escapeHtml(resolved.recipient); + return res.type('html').send(page('Unsubscribe', ` +

Unsubscribe

+

Stop receiving email to ${recipient} from this sender?

+
+ + +
`)); +})); + +router.post('/:token', asyncHandler(async (req, res) => { + // RFC 8058 §3.2: the POST body must be List-Unsubscribe=One-Click. + if ((req.body || {})['List-Unsubscribe'] !== 'One-Click') { + return res.status(400).type('html').send(GENERIC_400); + } + const resolved = await resolveToken(req.params.token); + if (!resolved) return res.status(400).type('html').send(GENERIC_400); + await MailSuppression.findOrCreate({ + where: { mailAccountId: resolved.account.id, recipient: resolved.recipient }, + defaults: { source: 'one-click' }, + }); + return res.type('html').send(page('Unsubscribed', ` +

Unsubscribed

+

${escapeHtml(resolved.recipient)} will no longer receive email from this sender.

`)); +})); + +module.exports = router; diff --git a/create-a-container/utils/__tests__/unsubscribe-token.test.js b/create-a-container/utils/__tests__/unsubscribe-token.test.js new file mode 100644 index 00000000..6a95b7c7 --- /dev/null +++ b/create-a-container/utils/__tests__/unsubscribe-token.test.js @@ -0,0 +1,50 @@ +/** + * Unit tests for utils/unsubscribe-token.js — roundtrip, tamper resistance, + * key selection, and the all-failures-look-identical contract. + */ + +const crypto = require('crypto'); +const { createUnsubscribeToken, verifyUnsubscribeToken } = require('../unsubscribe-token'); + +const key = { kid: 'a1b2c3d4', secret: crypto.randomBytes(32).toString('base64') }; +const otherKey = { kid: 'ffffffff', secret: crypto.randomBytes(32).toString('base64') }; +const payload = { accountId: '7d9fb37e-0000-4000-8000-000000000001', recipient: 'Someone@Example.COM' }; + +test('roundtrip returns accountId and lowercased recipient', () => { + const token = createUnsubscribeToken(payload, key); + expect(token).not.toMatch(/[^A-Za-z0-9_-]/); // base64url — URL-safe + const verified = verifyUnsubscribeToken(token, [key]); + expect(verified).toMatchObject({ + accountId: payload.accountId, + recipient: 'someone@example.com', + }); + expect(typeof verified.issuedAt).toBe('number'); +}); + +test('any tampering fails verification', () => { + const token = createUnsubscribeToken(payload, key); + const buf = Buffer.from(token, 'base64url'); + for (const index of [2, buf.length - 1, 20]) { + const tampered = Buffer.from(buf); + tampered[index] ^= 0x01; + expect(verifyUnsubscribeToken(tampered.toString('base64url'), [key])).toBeNull(); + } +}); + +test('wrong key and unknown kid both fail', () => { + const token = createUnsubscribeToken(payload, key); + expect(verifyUnsubscribeToken(token, [{ kid: key.kid, secret: otherKey.secret }])).toBeNull(); + expect(verifyUnsubscribeToken(token, [otherKey])).toBeNull(); + expect(verifyUnsubscribeToken(token, [])).toBeNull(); +}); + +test('verifies against the matching key in a multi-key set (rotation)', () => { + const token = createUnsubscribeToken(payload, key); + expect(verifyUnsubscribeToken(token, [otherKey, key])).not.toBeNull(); +}); + +test('garbage input returns null, never throws', () => { + for (const junk of ['', 'x', '!!!!', 'AAAA', 'A'.repeat(500), null, undefined]) { + expect(verifyUnsubscribeToken(junk, [key])).toBeNull(); + } +}); diff --git a/create-a-container/utils/agent-config.js b/create-a-container/utils/agent-config.js index da416bfb..eae82d4e 100644 --- a/create-a-container/utils/agent-config.js +++ b/create-a-container/utils/agent-config.js @@ -9,7 +9,7 @@ const crypto = require('crypto'); const { Op } = require('sequelize'); -const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain, Volume, DkimKey, Setting } = require('../models'); +const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain, Volume, DkimKey, Setting, MailUnsubscribeKey } = require('../models'); const { isAgentlessNodeType } = require('./volumes'); const { DEFAULT_QUOTA_MB } = require('./mail-account'); @@ -242,6 +242,7 @@ async function buildMailSnapshot(dbConfig) { const settings = await Setting.getMultiple([ 'mail_hostname', 'mail_relayhost', 'mail_relayhost_username', 'mail_relayhost_password', 'mail_db_host', 'mail_default_quota_mb', 'mail_message_size_limit_mb', + 'mail_unsubscribe_base_url', ]); const sendDomains = await ExternalDomain.findAll({ @@ -282,6 +283,19 @@ async function buildMailSnapshot(dbConfig) { dovecotUser: 'mail_dovecot', postfixUser: 'mail_postfix', }, + // opensource-mail-helper mints List-Unsubscribe tokens with the active + // key; retired-but-verifiable keys ride along so old links keep working + // after a rotation. The manager verifies with the same set. + unsubscribe: await buildUnsubscribeSection(settings.mail_unsubscribe_base_url), + }; +} + +async function buildUnsubscribeSection(baseUrl) { + await MailUnsubscribeKey.ensureActive(); + const keys = await MailUnsubscribeKey.verifiable(); + return { + baseUrl: baseUrl || null, + keys: keys.map((k) => ({ kid: k.kid, secret: k.secret, active: k.status === 'active' })), }; } diff --git a/create-a-container/utils/unsubscribe-token.js b/create-a-container/utils/unsubscribe-token.js new file mode 100644 index 00000000..4398a6da --- /dev/null +++ b/create-a-container/utils/unsubscribe-token.js @@ -0,0 +1,78 @@ +/** + * RFC 8058 one-click unsubscribe tokens (issue #67). + * + * AES-256-GCM over { v, accountId, recipient, iat } with the kid bound as + * AAD. The URL reveals nothing about sender or recipient; tampering fails + * authentication; tokens cannot be generated without the key, so knowing + * the URL format doesn't let anyone unsubscribe other addresses. + * + * Layout (base64url): [version:1][kidLen:1][kid][iv:12][tag:16][ciphertext] + * + * Pure crypto — key rows live in the MailUnsubscribeKeys model. + */ + +const crypto = require('crypto'); + +const VERSION = 1; +const IV_LEN = 12; +const TAG_LEN = 16; + +function secretBuffer(secret) { + return Buffer.isBuffer(secret) ? secret : Buffer.from(secret, 'base64'); +} + +/** + * @param {{accountId: string, recipient: string}} payload + * @param {{kid: string, secret: string|Buffer}} key - the active key + * @returns {string} base64url token + */ +function createUnsubscribeToken({ accountId, recipient }, key) { + const kidBuf = Buffer.from(key.kid, 'utf8'); + const iv = crypto.randomBytes(IV_LEN); + const cipher = crypto.createCipheriv('aes-256-gcm', secretBuffer(key.secret), iv); + cipher.setAAD(kidBuf); + const payload = Buffer.from(JSON.stringify({ + v: VERSION, + a: accountId, + r: String(recipient).trim().toLowerCase(), + t: Math.floor(Date.now() / 1000), + })); + const ciphertext = Buffer.concat([cipher.update(payload), cipher.final()]); + const tag = cipher.getAuthTag(); + return Buffer.concat([Buffer.from([VERSION, kidBuf.length]), kidBuf, iv, tag, ciphertext]) + .toString('base64url'); +} + +/** + * @param {string} token + * @param {Array<{kid: string, secret: string|Buffer}>} keys - verifiable keys + * @returns {{accountId: string, recipient: string, issuedAt: number}|null} + * null for ANY failure — callers must answer with the same generic 400. + */ +function verifyUnsubscribeToken(token, keys) { + const buf = Buffer.from(String(token), 'base64url'); + if (buf.length < 2 || buf[0] !== VERSION) return null; + const kidLen = buf[1]; + const headerLen = 2 + kidLen + IV_LEN + TAG_LEN; + if (buf.length <= headerLen) return null; + const kid = buf.subarray(2, 2 + kidLen).toString('utf8'); + const key = keys.find((k) => k.kid === kid); + if (!key) return null; + const iv = buf.subarray(2 + kidLen, 2 + kidLen + IV_LEN); + const tag = buf.subarray(2 + kidLen + IV_LEN, headerLen); + const ciphertext = buf.subarray(headerLen); + try { + const decipher = crypto.createDecipheriv('aes-256-gcm', secretBuffer(key.secret), iv); + decipher.setAAD(Buffer.from(kid, 'utf8')); + decipher.setAuthTag(tag); + const payload = JSON.parse( + Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString('utf8'), + ); + if (payload.v !== VERSION || !payload.a || !payload.r) return null; + return { accountId: payload.a, recipient: payload.r, issuedAt: payload.t }; + } catch { + return null; + } +} + +module.exports = { createUnsubscribeToken, verifyUnsubscribeToken }; From ce21fc081fde9f2fa1c49475380ed64c0b777ddf Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:12:15 -0700 Subject: [PATCH 06/19] mail: OpenAPI spec for mail endpoints (#67) Mail tag + MailAccount/MailAccountSecret/MailDnsRecord/MailSuppression schemas; paths for /mail-accounts (+domains, rotate-password, suppressions), /external-domains/{id}/mail/*, /mail/host|ptr, and /agents/{id}/api-key-pin. Agent check-in documents enabledServices, missingBinaries, mailStatus, and the holder-only mail snapshot. The MCP server picks these up automatically. Public /u/{token} is outside /api/v1 and intentionally not in this spec. --- create-a-container/openapi.v1.yaml | 315 ++++++++++++++++++++++++++++- 1 file changed, 314 insertions(+), 1 deletion(-) diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index ad95793e..fb1d4535 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -49,6 +49,7 @@ tags: - name: Notifications - name: Jobs - name: External Domains + - name: Mail - name: Users - name: Groups - name: API Keys @@ -463,6 +464,10 @@ components: name: { type: string } authServer: { type: string, nullable: true } hasCloudflareApiKey: { type: boolean } + mailEnabled: { type: boolean, description: 'Read-only here — toggled via the …/mail/enable|disable endpoints' } + mailDnsVerified: { type: boolean, description: 'SPF+DKIM+DMARC verified. With mailEnabled => domain can send' } + mailMxVerified: { type: boolean, description: 'MX resolves to the mail IP. With mailEnabled => domain can receive' } + mailDnsCheckedAt: { type: string, format: date-time, nullable: true } ExternalDomainInput: type: object properties: @@ -493,6 +498,66 @@ components: netboxToken: { type: string } bannerMessage: { type: string, description: 'Announcement banner shown at the top of the app (supports [text](url) links); blank disables it. Also surfaced unauthenticated via GET /health as `banner`.' } usagePsiProbeLimit: { type: string, description: 'Max PSI (pressure stall) probes per usage report, spent on the highest-utilization running containers. Non-negative integer as string; "0" disables PSI, blank uses the default (16).' } + mailHostname: { type: string, description: 'HELO / MX hostname — the forward-confirmed PTR name of the mail IP (see GET /mail/ptr)' } + mailUnsubscribeBaseUrl: { type: string, description: 'Public base URL of the manager for one-click unsubscribe links (RFC 8058)' } + mailRelayhost: { type: string, description: 'Optional smarthost; requires mailSpfInclude' } + mailRelayhostUsername: { type: string } + mailRelayhostPassword: { type: string } + mailSpfInclude: { type: string, description: 'include: domain added to suggested SPF; required with a relayhost' } + mailDnsCheckResolvers: { type: string, description: 'Comma-separated resolver IPs for Check DNS (default 1.1.1.1, 8.8.8.8)' } + mailDbHost: { type: string, description: 'DB host the mail stack connects to when the DB is not local to the mail host' } + mailSelfManagedSiteId: { type: string, description: 'Site whose externalIp is the mail IP when no agent runs mail (self-managed MTA)' } + mailDefaultQuotaMb: { type: string, description: 'Mailbox quota for new accounts (default 1024)' } + mailMessageSizeLimitMb: { type: string, description: 'Postfix message size limit (default 25)' } + MailAccount: + type: object + properties: + id: { type: string, format: uuid } + address: { type: string, description: '@' } + localPart: { type: string } + domain: + type: object + nullable: true + properties: + id: { type: integer } + name: { type: string } + canSend: { type: boolean } + canReceive: { type: boolean } + owner: { type: string, nullable: true, description: Username } + description: { type: string, nullable: true } + enabled: { type: boolean } + quotaMb: { type: integer } + unsubscribeHeaders: { type: boolean, description: 'Add RFC 8058 List-Unsubscribe headers to submitted mail' } + lastRotatedAt: { type: string, format: date-time, nullable: true } + createdAt: { type: string, format: date-time } + updatedAt: { type: string, format: date-time } + MailAccountSecret: + allOf: + - $ref: '#/components/schemas/MailAccount' + - type: object + properties: + password: { type: string, description: 'Generated password — this is the only time it is returned' } + connection: + type: object + properties: + host: { type: string } + username: { type: string } + smtp: { type: object, properties: { ports: { type: array, items: { type: integer } } } } + imap: { type: object, properties: { ports: { type: array, items: { type: integer } } } } + warning: { type: string } + MailDnsRecord: + type: object + properties: + type: { type: string, enum: [TXT, MX] } + name: { type: string } + value: { type: string } + MailSuppression: + type: object + properties: + id: { type: integer } + recipient: { type: string } + source: { type: string, enum: [one-click, admin] } + createdAt: { type: string, format: date-time } ResourceRequest: type: object properties: @@ -1309,10 +1374,24 @@ paths: properties: applied: { type: boolean } message: { type: string, description: 'Failure detail (mkdir/chown error)' } + enabledServices: + type: array + items: { type: string } + description: 'Service groups the agent runs (AGENT_SERVICES), e.g. [nginx, dnsmasq, mail]' + missingBinaries: + type: array + items: { type: string } + description: Binaries/SQL drivers missing for the enabled services; blocks the mail-host claim responses: '200': - description: 'Config snapshot (`{ data: { site, nginx } }`) with `ETag` header' + description: >- + Config snapshot (`{ data: { site, nginx } }`) with `ETag` header. + When the agent reports `mail`, `data.mailStatus` is one of + holder|conflict|unsupported|missing_packages, and only the claim + holder additionally receives `data.mail` (DKIM keys, receiving + domains, relayhost, limits, DB connection, unsubscribe keys). '304': { description: Config unchanged since If-None-Match } + '403': { description: 'Agent pinned to a different API key (code: agent_key_mismatch)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } '422': { description: 'Missing siteId/hostname (code: validation_failed)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } get: operationId: list_agents @@ -1337,6 +1416,11 @@ paths: hostname: { type: string } ipv4Address: { type: string, nullable: true } services: { type: object, nullable: true } + enabledServices: { type: array, items: { type: string }, nullable: true } + missingBinaries: { type: array, items: { type: string }, nullable: true } + mailHostSince: { type: string, format: date-time, nullable: true } + isLocal: { type: boolean } + hasApiKeyPin: { type: boolean } lastCheckinAt: { type: string, format: date-time, nullable: true } secondsSinceCheckin: { type: integer, nullable: true, description: Computed server-side at response time } '403': { $ref: '#/components/responses/Forbidden' } @@ -1539,6 +1623,235 @@ paths: '403': { $ref: '#/components/responses/Forbidden' } '404': { $ref: '#/components/responses/NotFound' } + /external-domains/{id}/mail/dns: + parameters: [{ in: path, name: id, required: true, schema: { type: integer } }] + get: + operationId: get_domain_mail_dns + tags: [Mail] + summary: DNS records to publish for mail + last check result (admin) + responses: + '200': + description: Records and verification state + content: + application/json: + schema: + type: object + properties: + data: + type: object + properties: + records: { type: array, items: { $ref: '#/components/schemas/MailDnsRecord' } } + mailEnabled: { type: boolean } + mailDnsVerified: { type: boolean } + mailMxVerified: { type: boolean } + lastCheck: { type: object, nullable: true } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + /external-domains/{id}/mail/check-dns: + parameters: [{ in: path, name: id, required: true, schema: { type: integer } }] + post: + operationId: check_domain_mail_dns + tags: [Mail] + summary: Re-verify SPF/DKIM/DMARC/MX against public resolvers (admin) + responses: + '200': + description: Domain with updated gates plus the full check result under `check` + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + /external-domains/{id}/mail/enable: + parameters: [{ in: path, name: id, required: true, schema: { type: integer } }] + post: + operationId: enable_domain_mail + tags: [Mail] + summary: Enable mail for a domain (admin) — runs Check DNS first + responses: + '200': { description: Enabled } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + '422': { description: 'SPF/DKIM/DMARC not verified (code: dns_not_configured); MX failure only warns', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + /external-domains/{id}/mail/disable: + parameters: [{ in: path, name: id, required: true, schema: { type: integer } }] + post: + operationId: disable_domain_mail + tags: [Mail] + summary: Disable mail for a domain (admin) — DKIM keys and config are kept + responses: + '200': { description: Disabled } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + + /mail-accounts: + get: + operationId: list_mail_accounts + tags: [Mail] + summary: List the current user's mail accounts (`?all=true` for admins) + parameters: + - { in: query, name: all, required: false, schema: { type: string, enum: ['true'] } } + responses: + '200': + description: List + content: + application/json: + schema: { type: object, properties: { data: { type: array, items: { $ref: '#/components/schemas/MailAccount' } } } } + '403': { $ref: '#/components/responses/Forbidden' } + post: + operationId: create_mail_account + tags: [Mail] + summary: Create a mail account (password returned ONCE) + description: >- + Only on domains that can send. Role local parts + (postmaster, abuse, hostmaster, webmaster, root, admin, security, + noreply, dmarc) and `quotaMb` are admin-only. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [externalDomainId, localPart] + properties: + externalDomainId: { type: integer } + localPart: { type: string, maxLength: 64 } + description: { type: string, maxLength: 255, nullable: true } + quotaMb: { type: integer, description: Admin-only } + responses: + '201': + description: Created with one-time `password` and `connection` info + content: + application/json: + schema: { type: object, properties: { data: { $ref: '#/components/schemas/MailAccountSecret' } } } + '400': { $ref: '#/components/responses/BadRequest' } + '403': { description: 'Reserved local part or admin-only field (codes: reserved_local_part, admin_required)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + '404': { description: 'Domain not found (code: domain_not_found)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + '409': { description: 'Address already exists (code: address_taken)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + '422': { description: 'Domain cannot send (code: domain_cannot_send)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + /mail-accounts/domains: + get: + operationId: list_mail_domains + tags: [Mail] + summary: Domains mail accounts can be created on (enabled + DNS verified) + responses: + '200': + description: List of `{ id, name }` + /mail-accounts/{id}: + parameters: [{ in: path, name: id, required: true, schema: { type: string, format: uuid } }] + get: + operationId: get_mail_account + tags: [Mail] + summary: Get a mail account (owner or admin) + responses: + '200': + description: Item + content: + application/json: + schema: { type: object, properties: { data: { $ref: '#/components/schemas/MailAccount' } } } + '404': { $ref: '#/components/responses/NotFound' } + patch: + operationId: update_mail_account + tags: [Mail] + summary: Update a mail account (quota/transfer admin-only) + requestBody: + content: + application/json: + schema: + type: object + properties: + description: { type: string, maxLength: 255, nullable: true } + enabled: { type: boolean, description: Disabling locks the account out instantly } + unsubscribeHeaders: { type: boolean } + quotaMb: { type: integer, description: Admin-only } + username: { type: string, description: 'Admin-only ownership transfer — target username' } + responses: + '200': + description: Updated + content: + application/json: + schema: { type: object, properties: { data: { $ref: '#/components/schemas/MailAccount' } } } + '403': { description: 'Admin-only field (code: admin_required)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + '404': { $ref: '#/components/responses/NotFound' } + '422': { description: 'Transfer target inactive (code: user_inactive)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + delete: + operationId: delete_mail_account + tags: [Mail] + summary: Delete a mail account (owner or admin) + responses: + '204': { description: Deleted } + '404': { $ref: '#/components/responses/NotFound' } + /mail-accounts/{id}/rotate-password: + parameters: [{ in: path, name: id, required: true, schema: { type: string, format: uuid } }] + post: + operationId: rotate_mail_account_password + tags: [Mail] + summary: Rotate the password (owner or admin; returned ONCE) + responses: + '200': + description: Account with one-time `password` and `connection` info + content: + application/json: + schema: { type: object, properties: { data: { $ref: '#/components/schemas/MailAccountSecret' } } } + '404': { $ref: '#/components/responses/NotFound' } + /mail-accounts/{id}/suppressions: + parameters: [{ in: path, name: id, required: true, schema: { type: string, format: uuid } }] + get: + operationId: list_mail_account_suppressions + tags: [Mail] + summary: Recipients who unsubscribed from this account (owner or admin) + responses: + '200': + description: List + content: + application/json: + schema: { type: object, properties: { data: { type: array, items: { $ref: '#/components/schemas/MailSuppression' } } } } + '404': { $ref: '#/components/responses/NotFound' } + /mail-accounts/{id}/suppressions/{suppressionId}: + parameters: + - { in: path, name: id, required: true, schema: { type: string, format: uuid } } + - { in: path, name: suppressionId, required: true, schema: { type: integer } } + delete: + operationId: delete_mail_account_suppression + tags: [Mail] + summary: Remove a suppression (admin only — owners cannot re-subscribe people) + responses: + '204': { description: Removed } + '403': { description: 'Not an admin (code: admin_required)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } + '404': { $ref: '#/components/responses/NotFound' } + + /mail/host: + get: + operationId: get_mail_host + tags: [Mail] + summary: Current mail-host claim holder, mail IP, and hostname (admin) + responses: + '200': { description: 'Holder agent (or null), mailIp, mailHostname, selfManagedSiteId' } + '403': { $ref: '#/components/responses/Forbidden' } + /mail/host/release: + post: + operationId: release_mail_host + tags: [Mail] + summary: Release the mail-host claim (admin) — next qualifying check-in takes it + responses: + '200': { description: '`{ released: boolean }`' } + '403': { $ref: '#/components/responses/Forbidden' } + /mail/ptr: + get: + operationId: get_mail_ptr + tags: [Mail] + summary: PTR suggestion for the mail_hostname setting (admin) + responses: + '200': { description: '`{ mailIp, ptr, forwardConfirmed, mailHostname, suggestion }`' } + '403': { $ref: '#/components/responses/Forbidden' } + + /agents/{id}/api-key-pin: + parameters: [{ in: path, name: id, required: true, schema: { type: integer } }] + delete: + operationId: delete_agent_api_key_pin + tags: [Agents] + summary: Clear a remote agent's API-key pin (admin) — it re-pins on next check-in + responses: + '204': { description: Cleared } + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } + /users: get: operationId: list_users From e5baba43531992b018e9f89f2313e4474051e9a3 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:34:31 -0700 Subject: [PATCH 07/19] agent: service groups via AGENT_SERVICES (#67) Service groups nginx, dnsmasq, mail - unset AGENT_SERVICES means nginx,dnsmasq, unknown names are rejected at startup. The agent only renders/applies and reports unit status for its enabled groups, and every check-in now carries enabledServices plus missingBinaries (PATH + sbin scan of each group's required binaries; the manager refuses the mail-host claim while anything is missing). --- agent/src/apply.ts | 19 ++++++++++++++++++- agent/src/config.ts | 22 ++++++++++++++++++++++ agent/src/index.ts | 12 +++++++----- agent/src/system.ts | 19 +++++++++++++++++++ agent/src/types.ts | 31 +++++++++++++++++++++++++++++++ agent/test/config.test.js | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 agent/test/config.test.js diff --git a/agent/src/apply.ts b/agent/src/apply.ts index 0efe413b..39ff82c2 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -12,11 +12,19 @@ import { execFileSync } from 'child_process'; import ejs from 'ejs'; import { reloadOrRestartService, restartService, sighupService } from './system'; import { log, commandOutput } from './log'; -import type { AgentConfig } from './config'; +import type { AgentConfig, ServiceGroup } from './config'; import type { ApplyResult, SiteConfig } from './types'; const TEMPLATES_DIR = path.join(__dirname, '..', 'templates'); +/** Binaries each service group needs; missing ones are reported at check-in + * (the manager refuses the mail-host claim until they're installed). */ +export const GROUP_BINARIES: Record = { + nginx: ['nginx'], + dnsmasq: ['dnsmasq'], + mail: ['postfix', 'dovecot', 'opendkim'], +}; + interface RenderedFile { dest: string; content: string; @@ -27,6 +35,8 @@ interface RenderedFile { export interface ManagedService { /** systemd unit name (also the key reported at check-in). */ unit: string; + /** Service group this unit belongs to (AGENT_SERVICES opt-in). */ + group: ServiceGroup; /** Render all managed files. Returns null when there is nothing to manage * yet (e.g. dnsmasq before the site exists). */ render(config: SiteConfig, agent: AgentConfig): Promise; @@ -51,6 +61,7 @@ function run(cmd: string[]): string { export const services: ManagedService[] = [ { unit: 'nginx', + group: 'nginx', async render(config, agent) { return [{ dest: '/etc/nginx/nginx.conf', @@ -73,6 +84,7 @@ export const services: ManagedService[] = [ }, { unit: 'dnsmasq', + group: 'dnsmasq', async render(config) { const site = config.site; // Skip dnsmasq management until the site's DHCP/DNS settings are fully @@ -102,6 +114,11 @@ export const services: ManagedService[] = [ }, ]; +/** The managed services for the agent's enabled service groups. */ +export function enabledServices(agent: AgentConfig): ManagedService[] { + return services.filter((svc) => agent.services.includes(svc.group)); +} + function readIfExists(file: string): string | null { try { return fs.readFileSync(file, 'utf8'); diff --git a/agent/src/config.ts b/agent/src/config.ts index 27b93695..93c61b6f 100644 --- a/agent/src/config.ts +++ b/agent/src/config.ts @@ -3,12 +3,33 @@ import { setLogLevel, type LogLevel } from './log'; +/** Service groups the agent can run. `mail` is opt-in via AGENT_SERVICES. */ +export const KNOWN_SERVICE_GROUPS = ['nginx', 'dnsmasq', 'mail'] as const; +export type ServiceGroup = (typeof KNOWN_SERVICE_GROUPS)[number]; + +const DEFAULT_SERVICE_GROUPS: ServiceGroup[] = ['nginx', 'dnsmasq']; + export interface AgentConfig { siteId: number; managerUrl: string; apiKey?: string; stateDir: string; logLevel: LogLevel; + /** Enabled service groups (AGENT_SERVICES; unset means nginx,dnsmasq). */ + services: ServiceGroup[]; +} + +function parseServiceGroups(raw: string | undefined): ServiceGroup[] { + if (!raw || !raw.trim()) return [...DEFAULT_SERVICE_GROUPS]; + const groups: ServiceGroup[] = []; + for (const part of raw.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean)) { + if ((KNOWN_SERVICE_GROUPS as readonly string[]).includes(part)) { + if (!groups.includes(part as ServiceGroup)) groups.push(part as ServiceGroup); + } else { + throw new Error(`AGENT_SERVICES contains unknown service group "${part}" (known: ${KNOWN_SERVICE_GROUPS.join(', ')})`); + } + } + return groups; } export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { @@ -26,5 +47,6 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): AgentConfig { // Set by systemd from StateDirectory=; fallback for manual runs. stateDir: env.STATE_DIRECTORY || '/var/lib/opensource-agent', logLevel, + services: parseServiceGroups(env.AGENT_SERVICES), }; } diff --git a/agent/src/index.ts b/agent/src/index.ts index cf37979a..058ab146 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -13,9 +13,9 @@ import os from 'os'; import { loadConfig, type AgentConfig } from './config'; import { State } from './state'; -import { getPrimaryIpv4, getServiceState, disconnectSystemBus } from './system'; +import { getPrimaryIpv4, getServiceState, findMissingBinaries, disconnectSystemBus } from './system'; import { checkin } from './api'; -import { services, applyService } from './apply'; +import { enabledServices, GROUP_BINARIES, applyService } from './apply'; import { reconcileVolumes } from './volumes'; import { log } from './log'; import type { CheckinRequest, ServiceStatus } from './types'; @@ -26,7 +26,7 @@ const MAX_PASSES = 5; async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { const serviceStatus: Record = {}; - for (const svc of services) { + for (const svc of enabledServices(cfg)) { serviceStatus[svc.unit] = { state: await getServiceState(svc.unit), lastApply: state.lastApply[svc.unit] ?? 'unknown', @@ -38,6 +38,8 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise GROUP_BINARIES[g])), }; // Report any not-yet-delivered volume results (persisted across runs). Only // include the field when there is something to report so the manager isn't @@ -51,7 +53,7 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { const cfg = loadConfig(); const state = State.load(cfg.stateDir); - log.info(`agent starting: siteId=${cfg.siteId}, manager=${cfg.managerUrl}`); + log.info(`agent starting: siteId=${cfg.siteId}, manager=${cfg.managerUrl}, services=${cfg.services.join(',')}`); log.debug(`state dir=${cfg.stateDir}, saved etag=${state.etag ?? '(none)'}`); if (Object.keys(state.pendingVolumeResults).length > 0) { log.debug(`carrying ${Object.keys(state.pendingVolumeResults).length} pending volume result(s) from a prior run`); @@ -77,7 +79,7 @@ async function main(): Promise { } log.info(`check-in: new config received (etag=${result.etag ?? '(none)'}), applying`); - for (const svc of services) { + for (const svc of enabledServices(cfg)) { state.lastApply[svc.unit] = await applyService(svc, result.config, cfg); } diff --git a/agent/src/system.ts b/agent/src/system.ts index 5a3fda70..167abb04 100644 --- a/agent/src/system.ts +++ b/agent/src/system.ts @@ -1,6 +1,8 @@ /** Host system info and systemd unit control via the systemd D-Bus API. */ +import fs from 'fs'; import os from 'os'; +import path from 'path'; import dbus, { type MessageBus, type ProxyObject, type Variant } from '@particle/dbus-next'; /** First non-internal IPv4 address, or null when none is configured. */ @@ -13,6 +15,23 @@ export function getPrimaryIpv4(): string | null { return null; } +// sbin dirs are not on the agent's PATH under systemd, but that's where +// postfix/dovecot/dnsmasq binaries live. +const EXTRA_BIN_DIRS = ['/usr/sbin', '/usr/bin', '/sbin', '/bin', '/usr/local/sbin', '/usr/local/bin']; + +/** Which of `binaries` cannot be found on PATH or the usual sbin dirs. */ +export function findMissingBinaries(binaries: string[]): string[] { + const dirs = [...new Set([...(process.env.PATH ?? '').split(':').filter(Boolean), ...EXTRA_BIN_DIRS])]; + return binaries.filter((bin) => !dirs.some((dir) => { + try { + fs.accessSync(path.join(dir, bin), fs.constants.X_OK); + return true; + } catch { + return false; + } + })); +} + // --- systemd (org.freedesktop.systemd1 on the system bus) ------------------- const SYSTEMD = 'org.freedesktop.systemd1'; diff --git a/agent/src/types.ts b/agent/src/types.ts index f9175779..fe1ddcb9 100644 --- a/agent/src/types.ts +++ b/agent/src/types.ts @@ -19,6 +19,10 @@ export interface CheckinRequest { currentTime: number; ipv4Address: string | null; services: Record; + /** Service groups this agent is configured to run (AGENT_SERVICES). */ + enabledServices: string[]; + /** Binaries missing for the enabled groups; blocks the mail-host claim. */ + missingBinaries: string[]; /** * Per-volume directory-provisioning results, keyed by the manager-assigned * Volume id: `{ : { applied, message? } }`. Present only when the @@ -106,4 +110,31 @@ export interface NginxConfig { export interface SiteConfig { site: SiteInfo | null; nginx: NginxConfig; + /** Mail-host claim outcome for agents reporting the `mail` group: + * holder | conflict | unsupported | missing_packages. Absent otherwise. */ + mailStatus?: string; + /** Mail snapshot — present ONLY when this agent holds the mail-host claim. */ + mail?: MailConfig; +} + +/** The `mail` snapshot section (manager utils/agent-config.js buildMailSnapshot). */ +export interface MailConfig { + hostname: string | null; + relayhost: { host: string; username: string | null; password: string | null } | null; + messageSizeLimitMb: number; + defaultQuotaMb: number; + dkim: { domain: string; selector: string; privateKey: string }[]; + receiveDomains: string[]; + db: { + dialect: string; + host: string | null; + port: number | null; + database: string | null; + dovecotUser: string; + postfixUser: string; + }; + unsubscribe: { + baseUrl: string | null; + keys: { kid: string; secret: string; active: boolean }[]; + }; } diff --git a/agent/test/config.test.js b/agent/test/config.test.js new file mode 100644 index 00000000..11f4c091 --- /dev/null +++ b/agent/test/config.test.js @@ -0,0 +1,32 @@ +/** + * AGENT_SERVICES parsing (node --test): default groups, explicit lists, + * dedupe, and rejection of unknown groups. + */ + +const test = require('node:test'); +const assert = require('node:assert'); + +const { loadConfig } = require('../dist/config.js'); + +const BASE_ENV = { SITE_ID: '1', MANAGER_URL: 'http://localhost:3000' }; + +test('unset AGENT_SERVICES means nginx,dnsmasq', () => { + assert.deepEqual(loadConfig({ ...BASE_ENV }).services, ['nginx', 'dnsmasq']); + assert.deepEqual(loadConfig({ ...BASE_ENV, AGENT_SERVICES: ' ' }).services, ['nginx', 'dnsmasq']); +}); + +test('explicit list is parsed, trimmed, lowercased, and deduped', () => { + const cfg = loadConfig({ ...BASE_ENV, AGENT_SERVICES: ' Nginx, dnsmasq ,mail, mail ' }); + assert.deepEqual(cfg.services, ['nginx', 'dnsmasq', 'mail']); +}); + +test('mail-only host is allowed', () => { + assert.deepEqual(loadConfig({ ...BASE_ENV, AGENT_SERVICES: 'mail' }).services, ['mail']); +}); + +test('unknown groups are rejected', () => { + assert.throws( + () => loadConfig({ ...BASE_ENV, AGENT_SERVICES: 'nginx,launchpad' }), + /unknown service group "launchpad"/, + ); +}); From fed174f7aca917ea33d2c65e28aed12c3486568a Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:46:52 -0700 Subject: [PATCH 08/19] agent: mail service group - Postfix/Dovecot/OpenDKIM + mail helper (#67) The claim holder renders the full mail stack from the snapshot: Postfix (submission 587/465 with TLS-only AUTH, sender==login, suppression policy at RCPT, after-queue content filter; :25 only accepts can-receive domains gated by Dovecot quota-status; SNI certs shared with nginx, compiled via postmap -F postWrite), Dovecot 2.4 (live SQL passdb/userdb against mail_accounts_v - submission requires can_send, IMAP stays readable; LMTP/auth/quota-status sockets; doveadm auth cache flush on reload), OpenDKIM (keys under /etc/dkimkeys, milter on the reinjection listener only, oversigns List-Unsubscribe per RFC 8058), and opensource-mail-helper - a small Node daemon (content filter that strips forged List-Unsubscribe headers and injects tokenized one-click headers per recipient, plus the suppression policy service; fail-open on DB errors). apply.ts gains file owner, postWrite, and statusUnit (postfix@-); system.ts gains enable/disable+start/stop. Turning mail off or losing the claim stops and disables the units, keeping configs and /var/vmail. Mailbox GC removes Maildirs of deleted accounts after 30 days using mailboxIds now carried in the snapshot; mail_senders_v gains account_id + unsubscribe_headers for the helper. --- .../systemd/opensource-mail-helper.service | 20 + agent/package-lock.json | 267 ++++++++++++- agent/package.json | 4 +- agent/src/apply.ts | 36 +- agent/src/index.ts | 31 +- agent/src/mail-helper.ts | 357 ++++++++++++++++++ agent/src/mail.ts | 309 +++++++++++++++ agent/src/system.ts | 31 +- agent/src/types.ts | 3 + agent/src/unsubscribe-token.ts | 37 ++ agent/templates/mail/dovecot.conf.ejs | 86 +++++ agent/templates/mail/opendkim-keytable.ejs | 3 + .../templates/mail/opendkim-signingtable.ejs | 3 + agent/templates/mail/opendkim.conf.ejs | 16 + agent/templates/mail/postfix-main.cf.ejs | 56 +++ agent/templates/mail/postfix-master.cf.ejs | 73 ++++ agent/templates/mail/postfix-sni-map.ejs | 5 + .../templates/mail/postfix-sql-senders.cf.ejs | 8 + agent/templates/mail/postfix-vdomains.ejs | 4 + agent/test/mail-helper.test.js | 72 ++++ agent/test/mail-template.test.js | 120 ++++++ create-a-container/utils/agent-config.js | 8 +- create-a-container/utils/mail-views.js | 11 +- 23 files changed, 1540 insertions(+), 20 deletions(-) create mode 100644 agent/contrib/systemd/opensource-mail-helper.service create mode 100644 agent/src/mail-helper.ts create mode 100644 agent/src/mail.ts create mode 100644 agent/src/unsubscribe-token.ts create mode 100644 agent/templates/mail/dovecot.conf.ejs create mode 100644 agent/templates/mail/opendkim-keytable.ejs create mode 100644 agent/templates/mail/opendkim-signingtable.ejs create mode 100644 agent/templates/mail/opendkim.conf.ejs create mode 100644 agent/templates/mail/postfix-main.cf.ejs create mode 100644 agent/templates/mail/postfix-master.cf.ejs create mode 100644 agent/templates/mail/postfix-sni-map.ejs create mode 100644 agent/templates/mail/postfix-sql-senders.cf.ejs create mode 100644 agent/templates/mail/postfix-vdomains.ejs create mode 100644 agent/test/mail-helper.test.js create mode 100644 agent/test/mail-template.test.js diff --git a/agent/contrib/systemd/opensource-mail-helper.service b/agent/contrib/systemd/opensource-mail-helper.service new file mode 100644 index 00000000..0491e965 --- /dev/null +++ b/agent/contrib/systemd/opensource-mail-helper.service @@ -0,0 +1,20 @@ +[Unit] +Description=MIE opensource-server mail helper (List-Unsubscribe filter + suppression policy) +Documentation=https://github.com/mieweb/opensource-server/issues/67 +After=network.target +PartOf=postfix.service + +[Service] +Type=simple +User=opensource-mail +Group=opensource-mail +ExecStart=/usr/bin/node /opt/opensource-server/agent/dist/mail-helper.js +Restart=on-failure +RestartSec=5 +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes + +[Install] +WantedBy=multi-user.target diff --git a/agent/package-lock.json b/agent/package-lock.json index 0e8963a7..425910f9 100644 --- a/agent/package-lock.json +++ b/agent/package-lock.json @@ -10,7 +10,9 @@ "license": "Apache-2.0", "dependencies": { "@particle/dbus-next": "^0.11.4", - "ejs": "^6.0.1" + "ejs": "^6.0.1", + "mysql2": "^3.15.3", + "pg": "^8.16.3" }, "devDependencies": { "@types/ejs": "^3.1.5", @@ -52,12 +54,20 @@ "version": "24.13.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" } }, + "node_modules/aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -97,6 +107,37 @@ "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", "license": "MIT" }, + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "license": "MIT", + "dependencies": { + "is-property": "^1.0.2" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", + "license": "MIT" + }, "node_modules/jsbi": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.2.tgz", @@ -109,12 +150,66 @@ "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", "license": "Apache-2.0" }, + "node_modules/lru.min": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.5.tgz", + "integrity": "sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==", + "license": "MIT", + "engines": { + "bun": ">=1.0.0", + "deno": ">=1.30.0", + "node": ">=8.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wellwelwel" + } + }, "node_modules/map-stream": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", "integrity": "sha512-C0X0KQmGm3N2ftbTGBhSyuydQ+vV1LC3f3zPvT3RXHXNZrvfPZcoXp/N5DOa8vedX/rTMm2CjTtivFg2STJMRQ==", "license": "MIT" }, + "node_modules/mysql2": { + "version": "3.24.5", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.24.5.tgz", + "integrity": "sha512-X6Ujsr2QSkkLpkQGjxzpKRAPn9nu4axpR63ntBzquFVEvPOArgbUQ1sJjFKI7hnaYtiVZxa17Z7q18KSubW0IQ==", + "license": "MIT", + "dependencies": { + "aws-ssl-profiles": "^1.1.2", + "generate-function": "^2.3.1", + "iconv-lite": "^0.7.3", + "long": "^5.3.2", + "lru.min": "^1.1.4", + "named-placeholders": "^1.1.6", + "sql-escaper": "^1.5.1" + }, + "engines": { + "node": ">= 8.0" + }, + "peerDependencies": { + "@types/node": ">= 8" + } + }, + "node_modules/mysql2/node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/named-placeholders": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.6.tgz", + "integrity": "sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==", + "license": "MIT", + "dependencies": { + "lru.min": "^1.1.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/pause-stream": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", @@ -127,6 +222,134 @@ "through": "~2.3" } }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -147,6 +370,12 @@ ], "license": "MIT" }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, "node_modules/sax": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", @@ -168,6 +397,30 @@ "node": "*" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/sql-escaper": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.5.2.tgz", + "integrity": "sha512-6CKD38c31SENivxOADeMNLdukOnUxUcflKtzVWzace7Riv1v7cAEym5Cx9Q7gYZ3ezIJI7ZpqecQq8cqNeSSFg==", + "license": "MIT", + "engines": { + "bun": ">=1.0.0", + "deno": ">=2.0.0", + "node": ">=12.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/mysqljs/sql-escaper?sponsor=1" + } + }, "node_modules/stream-combiner": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", @@ -202,7 +455,6 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", - "dev": true, "license": "MIT" }, "node_modules/xml2js": { @@ -226,6 +478,15 @@ "engines": { "node": ">=4.0" } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } } } } diff --git a/agent/package.json b/agent/package.json index dbfc21b2..f4244c0e 100644 --- a/agent/package.json +++ b/agent/package.json @@ -11,7 +11,9 @@ }, "dependencies": { "@particle/dbus-next": "^0.11.4", - "ejs": "^6.0.1" + "ejs": "^6.0.1", + "mysql2": "^3.15.3", + "pg": "^8.16.3" }, "devDependencies": { "@types/ejs": "^3.1.5", diff --git a/agent/src/apply.ts b/agent/src/apply.ts index 39ff82c2..5c445e08 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -30,6 +30,8 @@ interface RenderedFile { content: string; /** File mode for atomic writes (e.g. 0o600 for configs holding secrets). */ mode?: number; + /** `user:group` to chown to after writing (e.g. DKIM keys → opendkim). */ + owner?: string; } export interface ManagedService { @@ -37,9 +39,15 @@ export interface ManagedService { unit: string; /** Service group this unit belongs to (AGENT_SERVICES opt-in). */ group: ServiceGroup; + /** Unit whose ActiveState is reported at check-in when it differs from + * `unit` (Debian's postfix.service is a wrapper around postfix@-). */ + statusUnit?: string; /** Render all managed files. Returns null when there is nothing to manage * yet (e.g. dnsmasq before the site exists). */ render(config: SiteConfig, agent: AgentConfig): Promise; + /** Commands run after staging files, before `test` (e.g. `postmap -F` to + * compile the SNI hash map). A failure rolls the files back. */ + postWrite?: string[][]; /** Command that validates the staged config before it is kept. */ test?: string[]; /** Reload/restart after a successful apply. */ @@ -50,6 +58,9 @@ function renderTemplate(template: string, data: object): Promise { return ejs.renderFile(path.join(TEMPLATES_DIR, template), data); } +/** Render a template from the shared templates dir (used by mail.ts too). */ +export { renderTemplate }; + // Run a command, capturing its output. execFileSync with stdio 'pipe' attaches // the captured stdout/stderr to the thrown error on failure (see commandOutput // in log.ts), so callers can surface e.g. exactly why `nginx -t` rejected a @@ -114,11 +125,6 @@ export const services: ManagedService[] = [ }, ]; -/** The managed services for the agent's enabled service groups. */ -export function enabledServices(agent: AgentConfig): ManagedService[] { - return services.filter((svc) => agent.services.includes(svc.group)); -} - function readIfExists(file: string): string | null { try { return fs.readFileSync(file, 'utf8'); @@ -129,9 +135,10 @@ function readIfExists(file: string): string | null { // Write via temp file + rename so a crash mid-write can never leave a // truncated config on disk. -function writeFileAtomic(dest: string, content: string, mode?: number): void { +function writeFileAtomic(dest: string, content: string, mode?: number, owner?: string): void { const tmp = `${dest}.tmp-${process.pid}`; fs.writeFileSync(tmp, content, mode !== undefined ? { mode } : {}); + if (owner) run(['chown', owner, tmp]); fs.renameSync(tmp, dest); } @@ -157,21 +164,34 @@ export async function applyService( log.info(`${svc.unit}: ${changed.length} file(s) changed, applying: ${changed.join(', ')}`); const modeByDest = new Map(files.map((f) => [f.dest, f.mode])); + const ownerByDest = new Map(files.map((f) => [f.dest, f.owner])); // Stage the new files (previous contents kept in memory for rollback). for (const f of files) { fs.mkdirSync(path.dirname(f.dest), { recursive: true }); - writeFileAtomic(f.dest, f.content, f.mode); + writeFileAtomic(f.dest, f.content, f.mode, f.owner); log.debug(`${svc.unit}: wrote ${f.dest}`); } const rollback = () => { for (const [dest, prev] of current) { if (prev === null) fs.rmSync(dest, { force: true }); - else writeFileAtomic(dest, prev, modeByDest.get(dest)); + else writeFileAtomic(dest, prev, modeByDest.get(dest), ownerByDest.get(dest)); } log.debug(`${svc.unit}: rolled back to previous config`); }; + for (const cmd of svc.postWrite ?? []) { + try { + run(cmd); + } catch (err) { + rollback(); + log.error(`${svc.unit}: post-write command failed (\`${cmd.join(' ')}\`), rolled back: ${(err as Error).message}`); + const output = commandOutput(err); + if (output) log.error(`${svc.unit}: post-write output:\n${output}`); + return 'failure'; + } + } + if (svc.test) { const testCmd = svc.test.join(' '); log.debug(`${svc.unit}: testing config with \`${testCmd}\``); diff --git a/agent/src/index.ts b/agent/src/index.ts index 058ab146..5ce8e87b 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -15,20 +15,27 @@ import { loadConfig, type AgentConfig } from './config'; import { State } from './state'; import { getPrimaryIpv4, getServiceState, findMissingBinaries, disconnectSystemBus } from './system'; import { checkin } from './api'; -import { enabledServices, GROUP_BINARIES, applyService } from './apply'; +import { services, GROUP_BINARIES, applyService, type ManagedService } from './apply'; +import { mailServices, ensureMailProvisioned, syncMailUnits, gcMailboxes } from './mail'; import { reconcileVolumes } from './volumes'; import { log } from './log'; -import type { CheckinRequest, ServiceStatus } from './types'; +import type { CheckinRequest, ServiceStatus, SiteConfig } from './types'; // Safety cap: a flapping server-side config can't keep a single run alive // forever; the timer starts a fresh run 30s later anyway. const MAX_PASSES = 5; +const ALL_SERVICES: ManagedService[] = [...services, ...mailServices]; + +function enabledServices(cfg: AgentConfig): ManagedService[] { + return ALL_SERVICES.filter((svc) => cfg.services.includes(svc.group)); +} + async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { const serviceStatus: Record = {}; for (const svc of enabledServices(cfg)) { serviceStatus[svc.unit] = { - state: await getServiceState(svc.unit), + state: await getServiceState(svc.statusUnit ?? svc.unit), lastApply: state.lastApply[svc.unit] ?? 'unknown', }; } @@ -50,6 +57,19 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { + if (!cfg.services.includes('mail')) return; + try { + await syncMailUnits(config); + if (config.mail) gcMailboxes(config.mail); + } catch (err) { + log.error(`mail: unit lifecycle sync failed: ${(err as Error).message}`); + } +} + async function main(): Promise { const cfg = loadConfig(); const state = State.load(cfg.stateDir); @@ -79,9 +99,14 @@ async function main(): Promise { } log.info(`check-in: new config received (etag=${result.etag ?? '(none)'}), applying`); + // Mail users/dirs must exist before the mail configs reference them. + if (cfg.services.includes('mail') && result.config.mail) { + ensureMailProvisioned(); + } for (const svc of enabledServices(cfg)) { state.lastApply[svc.unit] = await applyService(svc, result.config, cfg); } + await syncMailLifecycle(cfg, result.config); // Ensure volume directories exist from the new snapshot; stash results in // persistent state so they are reported on the next check-in and survive a diff --git a/agent/src/mail-helper.ts b/agent/src/mail-helper.ts new file mode 100644 index 00000000..69f053e2 --- /dev/null +++ b/agent/src/mail-helper.ts @@ -0,0 +1,357 @@ +#!/usr/bin/env node +/** + * opensource-mail-helper: the mail host's after-queue content filter and + * submission policy service (issue #67). Shipped in the agent package and + * managed as part of the `mail` service group. + * + * - Content filter (127.0.0.1:10025): Postfix pipes submitted mail here one + * recipient per copy; the helper strips any pre-existing List-Unsubscribe + * headers, adds RFC 8058 one-click headers with an encrypted per-recipient + * token (unless the account disabled them), and reinjects on + * 127.0.0.1:10026 — where the OpenDKIM milter signs, so the signature + * covers the new headers. + * - Policy service (127.0.0.1:10027): at RCPT TO on the submission ports, + * rejects recipients who unsubscribed from the authenticated sender + * (mail_suppressions_v) with a synchronous 5.7.1. + * + * Config: /etc/opensource-server/mail-helper.json, rendered by the agent from + * the mail-host snapshot. A self-managed MTA can run this daemon too — it + * only needs the DB views and the unsubscribe keys. + */ + +import fs from 'fs'; +import net from 'net'; +import { createUnsubscribeToken, type UnsubscribeKey } from './unsubscribe-token'; +import { log } from './log'; + +const CONFIG_FILE = process.env.MAIL_HELPER_CONFIG || '/etc/opensource-server/mail-helper.json'; + +interface HelperConfig { + listen: { filterPort: number; reinjectPort: number; policyPort: number }; + db: { + dialect: 'postgres' | 'mysql'; + host: string | null; + port: number | null; + database: string | null; + user: string; + password: string | null; + }; + unsubscribe: { + baseUrl: string | null; + keys: (UnsubscribeKey & { active: boolean })[]; + }; +} + +interface SenderInfo { + accountId: string; + unsubscribeHeaders: boolean; +} + +/** Thin dialect-agnostic wrapper over the two mail views. */ +class Db { + private pool: { + query(sql: string, params: unknown[]): Promise<{ rows: Record[] }>; + }; + + constructor(cfg: HelperConfig['db']) { + if (cfg.dialect === 'mysql') { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const mysql = require('mysql2/promise'); + const pool = mysql.createPool({ + ...(cfg.host ? { host: cfg.host, port: cfg.port ?? 3306 } : { socketPath: '/var/run/mysqld/mysqld.sock' }), + user: cfg.user, + ...(cfg.password ? { password: cfg.password } : {}), + database: cfg.database, + connectionLimit: 5, + }); + this.pool = { + async query(sql, params) { + const [rows] = await pool.query(sql.replace(/\$\d+/g, '?'), params); + return { rows: rows as Record[] }; + }, + }; + } else { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const { Pool } = require('pg'); + const pool = new Pool({ + // No host means the local unix socket (peer auth, no password). + ...(cfg.host ? { host: cfg.host, port: cfg.port ?? 5432 } : { host: '/var/run/postgresql' }), + user: cfg.user, + ...(cfg.password ? { password: cfg.password } : {}), + database: cfg.database ?? undefined, + max: 5, + }); + this.pool = pool; + } + } + + async getSender(address: string): Promise { + const { rows } = await this.pool.query( + 'SELECT account_id, unsubscribe_headers FROM mail_senders_v WHERE sender = $1', + [address.toLowerCase()], + ); + if (!rows[0]) return null; + const raw = rows[0].unsubscribe_headers; // boolean on Postgres, 0/1 on MySQL + return { + accountId: String(rows[0].account_id), + unsubscribeHeaders: raw === true || raw === 1 || raw === '1', + }; + } + + async isSuppressed(sender: string, recipient: string): Promise { + const { rows } = await this.pool.query( + 'SELECT 1 AS hit FROM mail_suppressions_v WHERE sender = $1 AND recipient = $2', + [sender.toLowerCase(), recipient.toLowerCase()], + ); + return rows.length > 0; + } +} + +// --- header manipulation ----------------------------------------------------- + +const CRLF = '\r\n'; + +/** Strip client-supplied List-Unsubscribe(-Post) headers (including folded + * continuation lines) so apps can't forge unsubscribe targets. */ +export function stripUnsubscribeHeaders(headerBlock: string): string { + const lines = headerBlock.split(CRLF); + const kept: string[] = []; + let skipping = false; + for (const line of lines) { + if (/^list-unsubscribe(-post)?\s*:/i.test(line)) { + skipping = true; + continue; + } + if (skipping && /^[ \t]/.test(line)) continue; // folded continuation + skipping = false; + kept.push(line); + } + return kept.join(CRLF); +} + +/** Add the RFC 8058 headers for one recipient. `message` is the raw DATA + * payload (dot-stuffed lines are untouched — headers never start with '.'). */ +export function addUnsubscribeHeaders( + message: string, + { baseUrl, token }: { baseUrl: string; token: string }, +): string { + const splitAt = message.indexOf(CRLF + CRLF); + const headers = splitAt === -1 ? message : message.slice(0, splitAt); + const rest = splitAt === -1 ? '' : message.slice(splitAt); + const cleaned = stripUnsubscribeHeaders(headers); + const added = + `List-Unsubscribe: <${baseUrl}/u/${token}>${CRLF}` + + `List-Unsubscribe-Post: List-Unsubscribe=One-Click`; + return `${added}${CRLF}${cleaned}${rest}`; +} + +// --- minimal SMTP client (reinjection) ---------------------------------------- + +function reinject( + port: number, + envelope: { sender: string; recipient: string }, + message: string, +): Promise { + return new Promise((resolve, reject) => { + const socket = net.connect(port, '127.0.0.1'); + socket.setTimeout(30_000, () => socket.destroy(new Error('reinjection timed out'))); + let buffer = ''; + const steps: { expect: RegExp; send: string | null }[] = [ + { expect: /^220 /, send: `HELO mail-helper${CRLF}` }, + { expect: /^250/, send: `MAIL FROM:<${envelope.sender}>${CRLF}` }, + { expect: /^250/, send: `RCPT TO:<${envelope.recipient}>${CRLF}` }, + { expect: /^250/, send: `DATA${CRLF}` }, + { expect: /^354/, send: `${message}${CRLF}.${CRLF}` }, + { expect: /^250/, send: `QUIT${CRLF}` }, + { expect: /^221/, send: null }, + ]; + let step = 0; + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + // Process complete (possibly multi-line) SMTP replies. + let eol; + while ((eol = buffer.indexOf(CRLF)) !== -1) { + const line = buffer.slice(0, eol); + buffer = buffer.slice(eol + 2); + if (/^\d{3}-/.test(line)) continue; // multi-line reply continues + const current = steps[step]; + if (!current.expect.test(line)) { + socket.destroy(); + return reject(new Error(`reinjection step ${step} got "${line}"`)); + } + step += 1; + if (current.send !== null) socket.write(current.send); + if (step === steps.length) { + socket.end(); + return resolve(); + } + } + }); + socket.on('error', reject); + }); +} + +// --- content filter SMTP server ------------------------------------------------ + +interface FilterDeps { + db: Db; + unsubscribe: HelperConfig['unsubscribe']; + reinjectPort: number; +} + +/** Build the final message for one (sender, recipient) copy. Exported for tests. */ +export async function processMessage( + deps: Pick, + envelope: { sender: string; recipient: string }, + message: string, +): Promise { + const activeKey = deps.unsubscribe.keys.find((k) => k.active); + const { baseUrl } = deps.unsubscribe; + if (!activeKey || !baseUrl) return message; + const sender = await deps.db.getSender(envelope.sender); + if (!sender || !sender.unsubscribeHeaders) return message; + const token = createUnsubscribeToken( + { accountId: sender.accountId, recipient: envelope.recipient }, + activeKey, + ); + return addUnsubscribeHeaders(message, { baseUrl, token }); +} + +function startContentFilter(deps: FilterDeps, port: number): net.Server { + const server = net.createServer((socket) => { + socket.setTimeout(5 * 60_000, () => socket.destroy()); + let buffer = ''; + let inData = false; + let sender = ''; + let recipient = ''; + let data = ''; + + const reply = (line: string) => socket.write(`${line}${CRLF}`); + reply('220 mail-helper ESMTP'); + + const reset = () => { + sender = ''; + recipient = ''; + data = ''; + }; + + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + void (async () => { + let eol; + while ((eol = buffer.indexOf(CRLF)) !== -1) { + const line = buffer.slice(0, eol); + buffer = buffer.slice(eol + 2); + + if (inData) { + if (line === '.') { + inData = false; + try { + const message = await processMessage(deps, { sender, recipient }, data); + await reinject(deps.reinjectPort, { sender, recipient }, message); + reply('250 2.0.0 Ok'); + } catch (err) { + // Tempfail: Postfix keeps the message queued and retries. + log.error(`mail-helper: filter failed: ${(err as Error).message}`); + reply('451 4.3.0 content filter failure'); + } + reset(); + } else { + data += (data ? CRLF : '') + line; + } + continue; + } + + const upper = line.toUpperCase(); + if (upper.startsWith('EHLO') || upper.startsWith('HELO')) { + reply('250-mail-helper'); + reply('250-8BITMIME'); + reply('250 XFORWARD NAME ADDR PORT PROTO HELO IDENT SOURCE'); + } else if (upper.startsWith('XFORWARD')) { + reply('250 2.0.0 Ok'); + } else if (upper.startsWith('MAIL FROM:')) { + sender = (line.slice(10).match(/<([^>]*)>/)?.[1] ?? line.slice(10)).trim(); + reply('250 2.1.0 Ok'); + } else if (upper.startsWith('RCPT TO:')) { + // mailhelper_destination_recipient_limit = 1 — one recipient per copy. + recipient = (line.slice(8).match(/<([^>]*)>/)?.[1] ?? line.slice(8)).trim(); + reply('250 2.1.5 Ok'); + } else if (upper === 'DATA') { + inData = true; + reply('354 End data with .'); + } else if (upper === 'RSET' || upper === 'NOOP') { + if (upper === 'RSET') reset(); + reply('250 2.0.0 Ok'); + } else if (upper === 'QUIT') { + reply('221 2.0.0 Bye'); + socket.end(); + } else { + reply('502 5.5.2 Command not recognized'); + } + } + })(); + }); + socket.on('error', () => socket.destroy()); + }); + server.listen(port, '127.0.0.1', () => log.info(`mail-helper: content filter on 127.0.0.1:${port}`)); + return server; +} + +// --- Postfix policy service ------------------------------------------------------ + +function startPolicyService(db: Db, port: number): net.Server { + const server = net.createServer((socket) => { + socket.setTimeout(60_000, () => socket.destroy()); + let buffer = ''; + const attrs: Record = {}; + socket.on('data', (chunk) => { + buffer += chunk.toString('utf8'); + void (async () => { + let eol; + while ((eol = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, eol).replace(/\r$/, ''); + buffer = buffer.slice(eol + 1); + if (line !== '') { + const eq = line.indexOf('='); + if (eq > 0) attrs[line.slice(0, eq)] = line.slice(eq + 1); + continue; + } + // Blank line = end of request. Sender == SASL login on these ports + // (reject_sender_login_mismatch), but prefer the login when present. + const sender = attrs.sasl_username || attrs.sender || ''; + const recipient = attrs.recipient || ''; + let action = 'DUNNO'; + try { + if (sender && recipient && (await db.isSuppressed(sender, recipient))) { + action = `REJECT 5.7.1 Recipient has unsubscribed from ${sender}`; + } + } catch (err) { + // Fail open: a DB hiccup must not block all submission. + log.error(`mail-helper: policy lookup failed: ${(err as Error).message}`); + } + socket.write(`action=${action}\n\n`); + for (const k of Object.keys(attrs)) delete attrs[k]; + } + })(); + }); + socket.on('error', () => socket.destroy()); + }); + server.listen(port, '127.0.0.1', () => log.info(`mail-helper: policy service on 127.0.0.1:${port}`)); + return server; +} + +// --- main ------------------------------------------------------------------------ + +function main(): void { + const config = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')) as HelperConfig; + const db = new Db(config.db); + startContentFilter( + { db, unsubscribe: config.unsubscribe, reinjectPort: config.listen.reinjectPort }, + config.listen.filterPort, + ); + startPolicyService(db, config.listen.policyPort); +} + +if (require.main === module) { + main(); +} diff --git a/agent/src/mail.ts b/agent/src/mail.ts new file mode 100644 index 00000000..f43687b2 --- /dev/null +++ b/agent/src/mail.ts @@ -0,0 +1,309 @@ +/** + * The `mail` service group (issue #67): Postfix + Dovecot + OpenDKIM + + * opensource-mail-helper, rendered from the holder-only `mail` section of + * the config snapshot. + * + * Lifecycle: while this agent holds the mail-host claim, the units are + * enabled and started after each apply; when the group is turned off or the + * claim is lost, they are stopped and disabled — configs and /var/vmail stay + * in place. + */ + +import fs from 'fs'; +import path from 'path'; +import { execFileSync } from 'child_process'; +import { + enableAndStartServices, + stopAndDisableServices, + reloadOrRestartService, + restartService, +} from './system'; +import { renderTemplate, type ManagedService } from './apply'; +import { log } from './log'; +import type { MailConfig, SiteConfig } from './types'; + +export const MAIL_UNITS = ['opendkim', 'dovecot', 'postfix', 'opensource-mail-helper']; + +/** Localhost plumbing between Postfix, the helper, and OpenDKIM. */ +export const HELPER_PORTS = { filterPort: 10025, reinjectPort: 10026, policyPort: 10027 }; +export const OPENDKIM_PORT = 8891; + +export const HELPER_CONFIG_FILE = '/etc/opensource-server/mail-helper.json'; + +const VMAIL_DIR = '/var/vmail'; +const GC_STATE_FILE = path.join(VMAIL_DIR, '.gc-state.json'); +const GC_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; + +function sh(cmd: string[]): void { + execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe', encoding: 'utf8' }); +} + +function fileExists(file: string): boolean { + try { + fs.accessSync(file); + return true; + } catch { + return false; + } +} + +// --- render context --------------------------------------------------------- + +interface CertPair { name: string; cert: string; key: string } + +/** Per-domain cert material shared with nginx. Domains without both files + * are skipped (they fall back to the fallback chain). */ +function domainCerts(mail: MailConfig): CertPair[] { + const names = new Set([ + ...mail.dkim.map((k) => k.domain), + ...mail.receiveDomains, + ]); + if (mail.hostname) names.add(mail.hostname); + const pairs: CertPair[] = []; + for (const name of [...names].sort()) { + const cert = `/etc/ssl/certs/${name}.crt`; + const key = `/etc/ssl/private/${name}.key`; + if (fileExists(cert) && fileExists(key)) pairs.push({ name, cert, key }); + } + return pairs; +} + +function fallbackChain(mail: MailConfig): { fallbackCert: string; fallbackKey: string } { + if (mail.hostname) { + const cert = `/etc/ssl/certs/${mail.hostname}.crt`; + const key = `/etc/ssl/private/${mail.hostname}.key`; + if (fileExists(cert) && fileExists(key)) return { fallbackCert: cert, fallbackKey: key }; + } + // ssl-cert package default (an agent package dependency). + return { + fallbackCert: '/etc/ssl/certs/ssl-cert-snakeoil.pem', + fallbackKey: '/etc/ssl/private/ssl-cert-snakeoil.key', + }; +} + +/** Everything the mail templates need, derived from the snapshot + local + * environment (cert presence, DB passwords from /etc/environment). */ +export function buildMailContext(mail: MailConfig, env: NodeJS.ProcessEnv = process.env) { + const mysql = mail.db.dialect === 'mysql' || mail.db.dialect === 'mariadb'; + const host = mail.db.host; + // Postfix map syntax: TCP "host:port", or a unix socket path for local + // socket auth (Postgres peer / MariaDB unix_socket — no password needed). + const pfHosts = host + ? (mail.db.port ? `${host}:${mail.db.port}` : host) + : (mysql ? 'unix:/var/run/mysqld/mysqld.sock' : 'unix:/var/run/postgresql'); + return { + mail, + helper: HELPER_PORTS, + opendkimPort: OPENDKIM_PORT, + certs: { ...fallbackChain(mail), domains: domainCerts(mail) }, + db: { + prefix: mysql ? 'mysql' : 'pgsql', + hosts: pfHosts, + user: mail.db.postfixUser, + password: env.MAIL_POSTFIX_DB_PASSWORD || null, + dbname: mail.db.database ?? '', + dovecotDriver: mysql ? 'mysql' : 'pgsql', + dovecotHost: host || (mysql ? '/var/run/mysqld/mysqld.sock' : '/var/run/postgresql'), + dovecotParams: { dbname: mail.db.database ?? '' }, + dovecotUser: mail.db.dovecotUser, + dovecotPassword: env.MAIL_DOVECOT_DB_PASSWORD || null, + }, + }; +} + +/** opensource-mail-helper daemon config (JSON, root:opensource-mail 0640). */ +export function buildHelperConfig(mail: MailConfig, env: NodeJS.ProcessEnv = process.env): string { + const context = buildMailContext(mail, env); + return `${JSON.stringify({ + listen: HELPER_PORTS, + db: { + dialect: context.db.prefix === 'mysql' ? 'mysql' : 'postgres', + host: mail.db.host, + port: mail.db.port, + database: mail.db.database, + user: mail.db.postfixUser, + password: env.MAIL_POSTFIX_DB_PASSWORD || null, + }, + unsubscribe: mail.unsubscribe, + }, null, 2)}\n`; +} + +// --- managed services -------------------------------------------------------- + +/** Mail is managed only while this agent holds the claim. */ +function mailOf(config: SiteConfig): MailConfig | null { + return config.mail ?? null; +} + +export const mailServices: ManagedService[] = [ + { + unit: 'opendkim', + group: 'mail', + async render(config) { + const mail = mailOf(config); + if (!mail) return null; + const context = buildMailContext(mail); + const files = [ + { dest: '/etc/opendkim.conf', content: await renderTemplate('mail/opendkim.conf.ejs', context) }, + { dest: '/etc/opendkim/keytable', content: await renderTemplate('mail/opendkim-keytable.ejs', context) }, + { dest: '/etc/opendkim/signingtable', content: await renderTemplate('mail/opendkim-signingtable.ejs', context) }, + ]; + // Private keys never serialized anywhere else on this host. + for (const k of mail.dkim) { + files.push({ + dest: `/etc/dkimkeys/${k.domain}.private`, + content: k.privateKey, + mode: 0o600, + owner: 'opendkim:opendkim', + } as (typeof files)[number]); + } + return files; + }, + test: ['opendkim', '-n'], + reload() { + return restartService('opendkim'); + }, + }, + { + unit: 'dovecot', + group: 'mail', + async render(config) { + const mail = mailOf(config); + if (!mail) return null; + const context = buildMailContext(mail); + return [{ + dest: '/etc/dovecot/dovecot.conf', + content: await renderTemplate('mail/dovecot.conf.ejs', context), + // Embeds the mail_dovecot DB password when TCP auth is used. + mode: 0o600, + }]; + }, + test: ['doveconf', '-n'], + async reload() { + await reloadOrRestartService('dovecot'); + // Flush cached credentials so disables/rotations apply immediately. + try { + sh(['doveadm', 'auth', 'cache', 'flush']); + } catch { + /* dovecot still starting — the cache is empty anyway */ + } + }, + }, + { + unit: 'postfix', + group: 'mail', + // Debian's postfix.service is a wrapper; the real daemon state lives on + // the postfix@- instance. + statusUnit: 'postfix@-', + async render(config) { + const mail = mailOf(config); + if (!mail) return null; + const context = buildMailContext(mail); + const files = [ + { dest: '/etc/postfix/main.cf', content: await renderTemplate('mail/postfix-main.cf.ejs', context) }, + { dest: '/etc/postfix/master.cf', content: await renderTemplate('mail/postfix-master.cf.ejs', context) }, + { dest: '/etc/postfix/vdomains', content: await renderTemplate('mail/postfix-vdomains.ejs', context) }, + { dest: '/etc/postfix/sni_map', content: await renderTemplate('mail/postfix-sni-map.ejs', context), mode: 0o600 }, + { dest: '/etc/postfix/sql/senders.cf', content: await renderTemplate('mail/postfix-sql-senders.cf.ejs', context), mode: 0o600 }, + ]; + if (mail.relayhost?.username) { + files.push({ + dest: '/etc/postfix/sasl_passwd', + content: `${mail.relayhost.host} ${mail.relayhost.username}:${mail.relayhost.password ?? ''}\n`, + mode: 0o600, + } as (typeof files)[number]); + } + return files; + }, + // Compile the SNI key+chain hash before validating. + postWrite: [['postmap', '-F', 'hash:/etc/postfix/sni_map']], + test: ['postfix', 'check'], + reload() { + return reloadOrRestartService('postfix'); + }, + }, + { + unit: 'opensource-mail-helper', + group: 'mail', + async render(config) { + const mail = mailOf(config); + if (!mail) return null; + return [{ + dest: HELPER_CONFIG_FILE, + content: buildHelperConfig(mail), + mode: 0o640, + owner: 'root:opensource-mail', + }]; + }, + reload() { + return restartService('opensource-mail-helper'); + }, + }, +]; + +// --- lifecycle --------------------------------------------------------------- + +/** Create the system users/dirs the mail stack needs. Idempotent; runs only + * when this agent holds the claim. */ +export function ensureMailProvisioned(): void { + const ensureUser = (name: string, opts: string[]) => { + try { + sh(['id', '-u', name]); + } catch { + sh(['useradd', '--system', ...opts, name]); + } + }; + // Mailbox owner (Dovecot mail_uid/mail_gid). + ensureUser('vmail', ['--home-dir', VMAIL_DIR, '--create-home', '--shell', '/usr/sbin/nologin']); + // The helper daemon's unprivileged user (reads its config via group). + ensureUser('opensource-mail', ['--no-create-home', '--shell', '/usr/sbin/nologin']); + fs.mkdirSync(VMAIL_DIR, { recursive: true }); + sh(['chown', 'vmail:vmail', VMAIL_DIR]); + sh(['chmod', '0750', VMAIL_DIR]); + fs.mkdirSync('/etc/dkimkeys', { recursive: true }); + fs.mkdirSync('/etc/opensource-server', { recursive: true }); +} + +/** Enable+start the mail units while holding the claim; stop+disable them + * otherwise (configs and /var/vmail are left in place). */ +export async function syncMailUnits(config: SiteConfig): Promise { + if (config.mail) { + await enableAndStartServices(MAIL_UNITS); + return; + } + if (config.mailStatus) { + log.info(`mail: not the mail host (${config.mailStatus}); ensuring units are stopped`); + } + await stopAndDisableServices(MAIL_UNITS); +} + +/** Remove Maildirs of deleted accounts after a retention window. A dir is + * eligible once its account id stops appearing in the snapshot's mailboxIds; + * first-missing timestamps persist in /var/vmail/.gc-state.json. */ +export function gcMailboxes(mail: MailConfig, now = Date.now()): void { + let entries: string[]; + try { + entries = fs.readdirSync(VMAIL_DIR).filter((e) => !e.startsWith('.')); + } catch { + return; + } + let gcState: Record = {}; + try { + gcState = JSON.parse(fs.readFileSync(GC_STATE_FILE, 'utf8')); + } catch { + /* first run or corrupt state — rebuild from scratch */ + } + const known = new Set(mail.mailboxIds); + const next: Record = {}; + for (const entry of entries) { + if (known.has(entry)) continue; + const firstMissing = gcState[entry] ?? now; + if (now - firstMissing >= GC_RETENTION_MS) { + log.info(`mail: removing mailbox of deleted account ${entry}`); + fs.rmSync(path.join(VMAIL_DIR, entry), { recursive: true, force: true }); + } else { + next[entry] = firstMissing; + } + } + fs.writeFileSync(GC_STATE_FILE, JSON.stringify(next, null, 2)); +} diff --git a/agent/src/system.ts b/agent/src/system.ts index 167abb04..80015b65 100644 --- a/agent/src/system.ts +++ b/agent/src/system.ts @@ -52,7 +52,12 @@ interface SystemdManager { LoadUnit(name: string): Promise; ReloadOrRestartUnit(name: string, mode: string): Promise; RestartUnit(name: string, mode: string): Promise; + StartUnit(name: string, mode: string): Promise; + StopUnit(name: string, mode: string): Promise; KillUnit(name: string, who: string, signal: number): Promise; + EnableUnitFiles(files: string[], runtime: boolean, force: boolean): Promise; + DisableUnitFiles(files: string[], runtime: boolean): Promise; + Reload(): Promise; Subscribe(): Promise; Unsubscribe(): Promise; on(event: 'JobRemoved', listener: (id: number, job: unknown, unit: string, result: string) => void): void; @@ -104,7 +109,7 @@ export async function getServiceState(unit: string): Promise { /** Enqueue a systemd job and wait for it to finish; throws unless the job * completes with result "done". */ -async function runJob(method: 'ReloadOrRestartUnit' | 'RestartUnit', unit: string): Promise { +async function runJob(method: 'ReloadOrRestartUnit' | 'RestartUnit' | 'StartUnit' | 'StopUnit', unit: string): Promise { const mgr = await getManager(); // JobRemoved is only broadcast to subscribed clients. await mgr.Subscribe(); @@ -159,3 +164,27 @@ export async function sighupService(unit: string): Promise { const mgr = await getManager(); await mgr.KillUnit(unitName(unit), 'main', SIGHUP); } + +/** `systemctl enable --now ` — boot persistence + immediate start. */ +export async function enableAndStartServices(units: string[]): Promise { + const mgr = await getManager(); + await mgr.EnableUnitFiles(units.map(unitName), false, true); + await mgr.Reload(); + for (const unit of units) await runJob('StartUnit', unit); +} + +/** `systemctl disable --now `; failures are swallowed — used when the + * mail group is turned off or the claim is lost, where units may not even be + * installed. Configs and data are left in place. */ +export async function stopAndDisableServices(units: string[]): Promise { + const mgr = await getManager(); + for (const unit of units) { + await runJob('StopUnit', unit).catch(() => undefined); + } + try { + await mgr.DisableUnitFiles(units.map(unitName), false); + await mgr.Reload(); + } catch { + /* not installed / already disabled */ + } +} diff --git a/agent/src/types.ts b/agent/src/types.ts index fe1ddcb9..e64be280 100644 --- a/agent/src/types.ts +++ b/agent/src/types.ts @@ -125,6 +125,9 @@ export interface MailConfig { defaultQuotaMb: number; dkim: { domain: string; selector: string; privateKey: string }[]; receiveDomains: string[]; + /** Every account id (any state) — /var/vmail dirs not in this list belong + * to deleted accounts and are garbage-collected after a retention window. */ + mailboxIds: string[]; db: { dialect: string; host: string | null; diff --git a/agent/src/unsubscribe-token.ts b/agent/src/unsubscribe-token.ts new file mode 100644 index 00000000..8a2e7965 --- /dev/null +++ b/agent/src/unsubscribe-token.ts @@ -0,0 +1,37 @@ +/** + * Unsubscribe token minting — MUST stay byte-compatible with the manager's + * create-a-container/utils/unsubscribe-token.js (same layout, same AAD): + * [version:1][kidLen:1][kid][iv:12][tag:16][ciphertext], base64url, where the + * ciphertext is AES-256-GCM over JSON {v, a: accountId, r: recipient, t: iat}. + */ + +import crypto from 'crypto'; + +const VERSION = 1; +const IV_LEN = 12; + +export interface UnsubscribeKey { + kid: string; + /** Base64 of 32 random bytes. */ + secret: string; +} + +export function createUnsubscribeToken( + { accountId, recipient }: { accountId: string; recipient: string }, + key: UnsubscribeKey, +): string { + const kidBuf = Buffer.from(key.kid, 'utf8'); + const iv = crypto.randomBytes(IV_LEN); + const cipher = crypto.createCipheriv('aes-256-gcm', Buffer.from(key.secret, 'base64'), iv); + cipher.setAAD(kidBuf); + const payload = Buffer.from(JSON.stringify({ + v: VERSION, + a: accountId, + r: recipient.trim().toLowerCase(), + t: Math.floor(Date.now() / 1000), + })); + const ciphertext = Buffer.concat([cipher.update(payload), cipher.final()]); + const tag = cipher.getAuthTag(); + return Buffer.concat([Buffer.from([VERSION, kidBuf.length]), kidBuf, iv, tag, ciphertext]) + .toString('base64url'); +} diff --git a/agent/templates/mail/dovecot.conf.ejs b/agent/templates/mail/dovecot.conf.ejs new file mode 100644 index 00000000..9b1ad670 --- /dev/null +++ b/agent/templates/mail/dovecot.conf.ejs @@ -0,0 +1,86 @@ +<%_ /* Dovecot 2.4 (new config syntax; Debian 13). Validated with + `doveconf -n` at apply time — a rejected config is rolled back. */ -%> +# Managed by opensource-agent (mail service group) — do not edit. +dovecot_config_version = 2.4.0 +dovecot_storage_version = 2.4.0 + +protocols = imap lmtp + +# Virtual users: one Maildir per account under /var/vmail/, +# owned by the vmail system user (created by the agent). +mail_driver = maildir +mail_path = %{userdb:home}/Maildir +mail_uid = vmail +mail_gid = vmail + +# TLS: same per-domain certs as nginx; fallback below, per-domain overrides +# via local_name. Plaintext auth is only accepted over TLS. +ssl = yes +ssl_server_cert_file = <%= certs.fallbackCert %> +ssl_server_key_file = <%= certs.fallbackKey %> +auth_allow_cleartext = no +<%_ certs.domains.forEach((d) => { _%> +local_name <%= d.name %> { + ssl_server_cert_file = <%= d.cert %> + ssl_server_key_file = <%= d.key %> +} +<%_ }) _%> + +# Accounts live in the manager's database (mail_accounts_v view, read-only +# mail_dovecot user). Hash scheme is {ARGON2ID} (p=1). +sql_driver = <%= db.dovecotDriver %> +<%= db.dovecotDriver %> <%= db.dovecotHost %> { +<%_ if (db.dovecotParams.dbname) { _%> + dbname = <%= db.dovecotParams.dbname %> +<%_ } _%> + user = <%= db.dovecotUser %> +<%_ if (db.dovecotPassword) { _%> + password = <%= db.dovecotPassword %> +<%_ } _%> +} + +# Submission (protocol smtp, via Postfix SASL) requires can_send; IMAP stays +# readable for accounts on domains that can no longer send. +passdb sql { + query = SELECT address AS user, password FROM mail_accounts_v WHERE address = '%{user | lower}' AND (can_send OR '%{protocol}' != 'smtp') +} +userdb sql { + query = SELECT home, 'vmail' AS uid, 'vmail' AS gid, quota_bytes FROM mail_accounts_v WHERE address = '%{user | lower}' +} + +# Cache hits keep per-message SASL cheap; "instant disable" is bounded by +# this TTL. +auth_cache_size = 1M +auth_cache_ttl = 1 min +auth_cache_negative_ttl = 1 min + +mail_plugins { + quota = yes +} +quota_name = User quota +quota_storage_size = %{userdb:quota_bytes}B + +# Sockets for Postfix: SASL auth, LMTP delivery, and the quota-status +# policy service (rejects unknown / over-quota recipients at RCPT). +service auth { + unix_listener /var/spool/postfix/private/auth { + mode = 0660 + user = postfix + group = postfix + } +} +service lmtp { + unix_listener /var/spool/postfix/private/dovecot-lmtp { + mode = 0600 + user = postfix + group = postfix + } +} +service quota-status { + executable = quota-status -p postfix + unix_listener /var/spool/postfix/private/quota-status { + mode = 0660 + user = postfix + group = postfix + } +} diff --git a/agent/templates/mail/opendkim-keytable.ejs b/agent/templates/mail/opendkim-keytable.ejs new file mode 100644 index 00000000..2e087f1b --- /dev/null +++ b/agent/templates/mail/opendkim-keytable.ejs @@ -0,0 +1,3 @@ +<%_ mail.dkim.forEach((k) => { _%> +<%= k.selector %>._domainkey.<%= k.domain %> <%= k.domain %>:<%= k.selector %>:/etc/dkimkeys/<%= k.domain %>.private +<%_ }) _%> diff --git a/agent/templates/mail/opendkim-signingtable.ejs b/agent/templates/mail/opendkim-signingtable.ejs new file mode 100644 index 00000000..087621f6 --- /dev/null +++ b/agent/templates/mail/opendkim-signingtable.ejs @@ -0,0 +1,3 @@ +<%_ mail.dkim.forEach((k) => { _%> +*@<%= k.domain %> <%= k.selector %>._domainkey.<%= k.domain %> +<%_ }) _%> diff --git a/agent/templates/mail/opendkim.conf.ejs b/agent/templates/mail/opendkim.conf.ejs new file mode 100644 index 00000000..059ca4bf --- /dev/null +++ b/agent/templates/mail/opendkim.conf.ejs @@ -0,0 +1,16 @@ +# Managed by opensource-agent (mail service group) — do not edit. +Syslog yes +UMask 007 +Mode s +Socket inet:<%= opendkimPort %>@127.0.0.1 +UserID opendkim +PidFile /run/opendkim/opendkim.pid + +Canonicalization relaxed/simple +# RFC 8058: the signature must cover List-Unsubscribe(-Post). Oversigning +# both signs them when present and prevents later injection. +OversignHeaders From,List-Unsubscribe,List-Unsubscribe-Post + +KeyTable /etc/opendkim/keytable +SigningTable refile:/etc/opendkim/signingtable +InternalHosts 127.0.0.0/8 diff --git a/agent/templates/mail/postfix-main.cf.ejs b/agent/templates/mail/postfix-main.cf.ejs new file mode 100644 index 00000000..ff0782ac --- /dev/null +++ b/agent/templates/mail/postfix-main.cf.ejs @@ -0,0 +1,56 @@ +# Managed by opensource-agent (mail service group) — do not edit. +compatibility_level = 3.6 + +<%_ if (mail.hostname) { _%> +myhostname = <%= mail.hostname %> +<%_ } _%> +smtpd_banner = $myhostname ESMTP +biff = no +readme_directory = no +append_dot_mydomain = no + +# This host serves virtual mailboxes only. +mydestination = +mynetworks = 127.0.0.0/8 [::1]/128 +alias_maps = +alias_database = +local_recipient_maps = +local_transport = error:5.1.1 local delivery disabled + +smtpd_helo_required = yes +disable_vrfy_command = yes +message_size_limit = <%= mail.messageSizeLimitMb * 1048576 %> + +# TLS: per-domain certs via SNI (compiled with `postmap -F`), shared with +# nginx under /etc/ssl. Fallback chain covers domains without a cert yet. +tls_server_sni_maps = hash:/etc/postfix/sni_map +smtpd_tls_chain_files = <%= certs.fallbackKey %>, <%= certs.fallbackCert %> +smtpd_tls_security_level = may +smtp_tls_security_level = may + +# Inbound (:25): only local recipients on can-receive domains. Dovecot's +# quota-status policy service rejects unknown and over-quota users at RCPT. +virtual_mailbox_domains = texthash:/etc/postfix/vdomains +virtual_mailbox_maps = static:x +virtual_transport = lmtp:unix:private/dovecot-lmtp +smtpd_relay_restrictions = permit_mynetworks, reject_unauth_destination +smtpd_recipient_restrictions = reject_unauth_destination, check_policy_service unix:private/quota-status + +# Sender ownership: submission enforces sender == SASL login (live from SQL). +smtpd_sender_login_maps = <%= db.prefix %>:/etc/postfix/sql/senders.cf + +# After-queue content filter copies are delivered one recipient at a time so +# opensource-mail-helper can add a per-recipient List-Unsubscribe token +# (RFC 8058) before OpenDKIM signs on reinjection. +mailhelper_destination_recipient_limit = 1 +<%_ if (mail.relayhost) { _%> + +# Outbound smarthost — SPF for sending domains must include the relay +# (mail_spf_include setting). +relayhost = <%= mail.relayhost.host %> +<%_ if (mail.relayhost.username) { _%> +smtp_sasl_auth_enable = yes +smtp_sasl_password_maps = texthash:/etc/postfix/sasl_passwd +smtp_sasl_security_options = noanonymous +<%_ } _%> +<%_ } _%> diff --git a/agent/templates/mail/postfix-master.cf.ejs b/agent/templates/mail/postfix-master.cf.ejs new file mode 100644 index 00000000..539fa06b --- /dev/null +++ b/agent/templates/mail/postfix-master.cf.ejs @@ -0,0 +1,73 @@ +# Managed by opensource-agent (mail service group) — do not edit. +# +# service type private unpriv chroot wakeup maxproc command + args +smtp inet n - y - - smtpd + +# Submission (STARTTLS on 587, implicit TLS on 465): AUTH only after TLS, +# sender must match the SASL login, suppressions are enforced at RCPT by the +# opensource-mail-helper policy service, and everything accepted goes through +# the mailhelper content filter for List-Unsubscribe headers. +submission inet n - y - - smtpd + -o syslog_name=postfix/submission + -o smtpd_tls_security_level=encrypt + -o smtpd_tls_auth_only=yes + -o smtpd_sasl_auth_enable=yes + -o smtpd_sasl_type=dovecot + -o smtpd_sasl_path=private/auth + -o smtpd_sender_restrictions=reject_sender_login_mismatch + -o { smtpd_relay_restrictions = permit_sasl_authenticated, reject } + -o { smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:<%= helper.policyPort %>, permit_sasl_authenticated, reject } + -o content_filter=mailhelper:[127.0.0.1]:<%= helper.filterPort %> +smtps inet n - y - - smtpd + -o syslog_name=postfix/smtps + -o smtpd_tls_wrappermode=yes + -o smtpd_sasl_auth_enable=yes + -o smtpd_sasl_type=dovecot + -o smtpd_sasl_path=private/auth + -o smtpd_sender_restrictions=reject_sender_login_mismatch + -o { smtpd_relay_restrictions = permit_sasl_authenticated, reject } + -o { smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:<%= helper.policyPort %>, permit_sasl_authenticated, reject } + -o content_filter=mailhelper:[127.0.0.1]:<%= helper.filterPort %> + +# Transport to the opensource-mail-helper content filter. +mailhelper unix - - y - - smtp + -o syslog_name=postfix/mailhelper + -o smtp_send_xforward_command=yes + -o disable_dns_lookups=yes + -o smtp_tls_security_level=none + +# Reinjection listener: the OpenDKIM milter runs HERE (not on 587/465) so the +# signature covers the headers the helper added. +127.0.0.1:<%= helper.reinjectPort %> inet n - y - - smtpd + -o syslog_name=postfix/reinject + -o smtpd_milters=inet:127.0.0.1:<%= opendkimPort %> + -o content_filter= + -o mynetworks=127.0.0.0/8 + -o { smtpd_relay_restrictions = permit_mynetworks, reject } + -o { smtpd_recipient_restrictions = permit_mynetworks, reject } + -o smtpd_tls_security_level=none + -o smtpd_authorized_xforward_hosts=127.0.0.0/8 + +pickup unix n - y 60 1 pickup +cleanup unix n - y - 0 cleanup +qmgr unix n - n 300 1 qmgr +tlsmgr unix - - y 1000? 1 tlsmgr +rewrite unix - - y - - trivial-rewrite +bounce unix - - y - 0 bounce +defer unix - - y - 0 bounce +trace unix - - y - 0 bounce +verify unix - - y - 1 verify +flush unix n - y 1000? 0 flush +proxymap unix - - n - - proxymap +proxywrite unix - - n - 1 proxymap +smtp unix - - y - - smtp +relay unix - - y - - smtp + -o syslog_name=postfix/$service_name +showq unix n - y - - showq +error unix - - y - - error +retry unix - - y - - error +discard unix - - y - - discard +lmtp unix - - y - - lmtp +anvil unix - - y - 1 anvil +scache unix - - y - 1 scache +postlog unix-dgram n - n - 1 postlogd diff --git a/agent/templates/mail/postfix-sni-map.ejs b/agent/templates/mail/postfix-sni-map.ejs new file mode 100644 index 00000000..e2f05bee --- /dev/null +++ b/agent/templates/mail/postfix-sni-map.ejs @@ -0,0 +1,5 @@ +<%_ /* SNI map: one line per domain with cert material on disk. Compiled with + `postmap -F` (postWrite), which stores the key+chain in the .db. */ -%> +<%_ certs.domains.forEach((d) => { _%> +<%= d.name %> <%= d.key %> <%= d.cert %> +<%_ }) _%> diff --git a/agent/templates/mail/postfix-sql-senders.cf.ejs b/agent/templates/mail/postfix-sql-senders.cf.ejs new file mode 100644 index 00000000..d567dc0c --- /dev/null +++ b/agent/templates/mail/postfix-sql-senders.cf.ejs @@ -0,0 +1,8 @@ +# Managed by opensource-agent — Postfix smtpd_sender_login_maps (live SQL). +hosts = <%= db.hosts %> +user = <%= db.user %> +<%_ if (db.password) { _%> +password = <%= db.password %> +<%_ } _%> +dbname = <%= db.dbname %> +query = SELECT login FROM mail_senders_v WHERE sender = '%s' diff --git a/agent/templates/mail/postfix-vdomains.ejs b/agent/templates/mail/postfix-vdomains.ejs new file mode 100644 index 00000000..5ff741e1 --- /dev/null +++ b/agent/templates/mail/postfix-vdomains.ejs @@ -0,0 +1,4 @@ +<%_ /* Domains Postfix accepts inbound mail for (texthash: "domain ok"). */ -%> +<%_ mail.receiveDomains.forEach((domain) => { _%> +<%= domain %> ok +<%_ }) _%> diff --git a/agent/test/mail-helper.test.js b/agent/test/mail-helper.test.js new file mode 100644 index 00000000..ed8ee6c2 --- /dev/null +++ b/agent/test/mail-helper.test.js @@ -0,0 +1,72 @@ +/** + * mail-helper header-injection tests (node --test): stripping forged + * List-Unsubscribe headers, adding tokenized RFC 8058 headers, and the + * per-account / missing-key opt-outs. + */ + +const test = require('node:test'); +const assert = require('node:assert'); +const crypto = require('crypto'); + +const { stripUnsubscribeHeaders, addUnsubscribeHeaders, processMessage } = require('../dist/mail-helper.js'); + +const CRLF = '\r\n'; +const MESSAGE = [ + 'From: app@example.com', + 'To: someone@example.net', + 'Subject: hi', + '', + 'body line', + '..leading dot stays stuffed', +].join(CRLF); + +const key = { kid: 'a1b2c3d4', secret: crypto.randomBytes(32).toString('base64'), active: true }; +const unsubscribe = { baseUrl: 'https://manager.example.com', keys: [key] }; +const db = { + async getSender(address) { + if (address === 'app@example.com') return { accountId: 'acc-1', unsubscribeHeaders: true }; + if (address === 'quiet@example.com') return { accountId: 'acc-2', unsubscribeHeaders: false }; + return null; + }, +}; + +test('stripUnsubscribeHeaders removes forged headers including folded lines', () => { + const headers = [ + 'From: a@b.c', + 'List-Unsubscribe: ,', + ' ', + 'List-Unsubscribe-Post: List-Unsubscribe=One-Click', + 'Subject: x', + ].join(CRLF); + assert.equal(stripUnsubscribeHeaders(headers), `From: a@b.c${CRLF}Subject: x`); +}); + +test('addUnsubscribeHeaders prepends both headers before the body', () => { + const out = addUnsubscribeHeaders(MESSAGE, { baseUrl: 'https://m.example', token: 'TOK' }); + assert.ok(out.startsWith(`List-Unsubscribe: ${CRLF}List-Unsubscribe-Post: List-Unsubscribe=One-Click${CRLF}From:`)); + // Body (including dot-stuffed lines) is untouched. + assert.ok(out.endsWith(`body line${CRLF}..leading dot stays stuffed`)); +}); + +test('processMessage injects a valid tokenized header for opted-in accounts', async () => { + const out = await processMessage( + { db, unsubscribe }, + { sender: 'app@example.com', recipient: 'Someone@Example.NET' }, + MESSAGE, + ); + const match = out.match(/^List-Unsubscribe: /); + assert.ok(match, 'List-Unsubscribe header present'); + assert.match(out, /^List-Unsubscribe-Post: List-Unsubscribe=One-Click$/m); +}); + +test('processMessage leaves mail untouched when opted out, unknown, or unconfigured', async () => { + const cases = [ + [{ db, unsubscribe }, 'quiet@example.com'], // account opted out + [{ db, unsubscribe }, 'ghost@example.com'], // not a managed sender + [{ db, unsubscribe: { baseUrl: null, keys: [key] } }, 'app@example.com'], // no base URL + [{ db, unsubscribe: { baseUrl: 'https://m', keys: [] } }, 'app@example.com'], // no active key + ]; + for (const [deps, sender] of cases) { + assert.equal(await processMessage(deps, { sender, recipient: 'r@x.y' }, MESSAGE), MESSAGE); + } +}); diff --git a/agent/test/mail-template.test.js b/agent/test/mail-template.test.js new file mode 100644 index 00000000..efb5f90d --- /dev/null +++ b/agent/test/mail-template.test.js @@ -0,0 +1,120 @@ +/** + * Mail template rendering tests (node --test): assert the directives the + * #67 guardrails depend on — real validation happens on-host via + * `postfix check` / `doveconf -n` / `opendkim -n` at apply time. + */ + +const test = require('node:test'); +const assert = require('node:assert'); +const path = require('path'); +const ejs = require('ejs'); + +const TEMPLATES = path.join(__dirname, '..', 'templates'); + +const context = { + mail: { + hostname: 'mail.example.com', + relayhost: null, + messageSizeLimitMb: 25, + defaultQuotaMb: 1024, + dkim: [ + { domain: 'example.com', selector: 'os20261008', privateKey: 'PEM' }, + { domain: 'other.example', selector: 'os20261001', privateKey: 'PEM2' }, + ], + receiveDomains: ['example.com'], + mailboxIds: ['a-1', 'b-2'], + db: { + dialect: 'postgres', host: null, port: null, database: 'cc', + dovecotUser: 'mail_dovecot', postfixUser: 'mail_postfix', + }, + unsubscribe: { baseUrl: 'https://manager.example.com', keys: [] }, + }, + helper: { filterPort: 10025, reinjectPort: 10026, policyPort: 10027 }, + opendkimPort: 8891, + certs: { + fallbackCert: '/etc/ssl/certs/mail.example.com.crt', + fallbackKey: '/etc/ssl/private/mail.example.com.key', + domains: [{ name: 'example.com', cert: '/etc/ssl/certs/example.com.crt', key: '/etc/ssl/private/example.com.key' }], + }, + db: { + prefix: 'pgsql', + hosts: 'unix:/var/run/postgresql', + user: 'mail_postfix', + password: null, + dbname: 'cc', + dovecotDriver: 'pgsql', + dovecotHost: '/var/run/postgresql', + dovecotParams: { dbname: 'cc' }, + dovecotUser: 'mail_dovecot', + dovecotPassword: null, + }, +}; + +function render(template, data = context) { + return ejs.renderFile(path.join(TEMPLATES, template), data); +} + +test('postfix main.cf: recipient gates, sender ownership, size limit, SNI', async () => { + const out = await render('mail/postfix-main.cf.ejs'); + assert.match(out, /myhostname = mail\.example\.com/); + assert.match(out, /message_size_limit = 26214400/); + assert.match(out, /virtual_mailbox_domains = texthash:\/etc\/postfix\/vdomains/); + assert.match(out, /check_policy_service unix:private\/quota-status/); + assert.match(out, /smtpd_sender_login_maps = pgsql:\/etc\/postfix\/sql\/senders\.cf/); + assert.match(out, /tls_server_sni_maps = hash:\/etc\/postfix\/sni_map/); + assert.match(out, /mailhelper_destination_recipient_limit = 1/); + assert.doesNotMatch(out, /relayhost =/); +}); + +test('postfix main.cf: relayhost with credentials', async () => { + const out = await render('mail/postfix-main.cf.ejs', { + ...context, + mail: { ...context.mail, relayhost: { host: '[relay.net]:587', username: 'u', password: 'p' } }, + }); + assert.match(out, /relayhost = \[relay\.net\]:587/); + assert.match(out, /smtp_sasl_password_maps = texthash:\/etc\/postfix\/sasl_passwd/); +}); + +test('postfix master.cf: TLS-only AUTH submission, policy + filter, DKIM on reinject only', async () => { + const out = await render('mail/postfix-master.cf.ejs'); + assert.match(out, /submission[\s\S]*?smtpd_tls_security_level=encrypt/); + assert.match(out, /smtpd_sender_restrictions=reject_sender_login_mismatch/); + assert.match(out, /check_policy_service inet:127\.0\.0\.1:10027/); + assert.match(out, /content_filter=mailhelper:\[127\.0\.0\.1\]:10025/); + assert.match(out, /smtps[\s\S]*?smtpd_tls_wrappermode=yes/); + assert.match(out, /127\.0\.0\.1:10026 inet[\s\S]*?smtpd_milters=inet:127\.0\.0\.1:8891/); + // The milter must NOT run on the submission listeners. + assert.doesNotMatch(out, /submission[\s\S]{0,600}smtpd_milters/); +}); + +test('dovecot.conf: SQL auth with the can_send smtp gate, per-domain certs, postfix sockets', async () => { + const out = await render('mail/dovecot.conf.ejs'); + assert.match(out, /dovecot_config_version = 2\.4\.0/); + assert.match(out, /sql_driver = pgsql/); + assert.match(out, /can_send OR '%\{protocol\}' != 'smtp'/); + assert.match(out, /local_name example\.com \{/); + assert.match(out, /unix_listener \/var\/spool\/postfix\/private\/auth/); + assert.match(out, /unix_listener \/var\/spool\/postfix\/private\/dovecot-lmtp/); + assert.match(out, /quota-status -p postfix/); + assert.match(out, /auth_allow_cleartext = no/); +}); + +test('opendkim: signs on localhost socket and oversigns List-Unsubscribe', async () => { + const conf = await render('mail/opendkim.conf.ejs'); + assert.match(conf, /Socket\s+inet:8891@127\.0\.0\.1/); + assert.match(conf, /OversignHeaders\s+From,List-Unsubscribe,List-Unsubscribe-Post/); + + const keytable = await render('mail/opendkim-keytable.ejs'); + assert.match(keytable, /os20261008\._domainkey\.example\.com example\.com:os20261008:\/etc\/dkimkeys\/example\.com\.private/); + + const signing = await render('mail/opendkim-signingtable.ejs'); + assert.match(signing, /\*@example\.com os20261008\._domainkey\.example\.com/); +}); + +test('vdomains and sni_map list the right domains', async () => { + assert.match(await render('mail/postfix-vdomains.ejs'), /^example\.com ok$/m); + assert.match( + await render('mail/postfix-sni-map.ejs'), + /^example\.com \/etc\/ssl\/private\/example\.com\.key \/etc\/ssl\/certs\/example\.com\.crt$/m, + ); +}); diff --git a/create-a-container/utils/agent-config.js b/create-a-container/utils/agent-config.js index eae82d4e..b2449687 100644 --- a/create-a-container/utils/agent-config.js +++ b/create-a-container/utils/agent-config.js @@ -9,7 +9,7 @@ const crypto = require('crypto'); const { Op } = require('sequelize'); -const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain, Volume, DkimKey, Setting, MailUnsubscribeKey } = require('../models'); +const { Site, Node, Container, Service, HTTPService, TransportService, ExternalDomain, Volume, DkimKey, Setting, MailUnsubscribeKey, MailAccount } = require('../models'); const { isAgentlessNodeType } = require('./volumes'); const { DEFAULT_QUOTA_MB } = require('./mail-account'); @@ -255,6 +255,11 @@ async function buildMailSnapshot(dbConfig) { attributes: ['name'], order: [['name', 'ASC']], }); + // Every account id (enabled or not) — the agent garbage-collects + // /var/vmail dirs that are no longer in this list, after a retention + // window (deleted accounts). + const mailboxIds = (await MailAccount.findAll({ attributes: ['id'], order: [['id', 'ASC']] })) + .map((a) => a.id); return { hostname: settings.mail_hostname || null, @@ -275,6 +280,7 @@ async function buildMailSnapshot(dbConfig) { })), // Domains Postfix accepts inbound mail for on :25. receiveDomains: receiveDomains.map((d) => d.name), + mailboxIds, db: { dialect: dbConfig.dialect, host: settings.mail_db_host || dbConfig.host || null, diff --git a/create-a-container/utils/mail-views.js b/create-a-container/utils/mail-views.js index 1e2eab2d..1de7080a 100644 --- a/create-a-container/utils/mail-views.js +++ b/create-a-container/utils/mail-views.js @@ -6,8 +6,11 @@ * mail_accounts_v address, password ({ARGON2ID}-prefixed PHC hash), * home, quota_bytes, can_send, can_receive * — enabled accounts only (Dovecot passdb/userdb) - * mail_senders_v sender, login for can-send accounts - * (Postfix smtpd_sender_login_maps) + * mail_senders_v sender, login, account_id, unsubscribe_headers + * for can-send accounts (Postfix + * smtpd_sender_login_maps; opensource-mail-helper uses + * account_id + unsubscribe_headers to mint + * List-Unsubscribe tokens) * mail_suppressions_v sender, recipient * (submission policy service — reject at RCPT) * @@ -49,7 +52,9 @@ function createViewsSql(dialect) { `${joins} WHERE ma.${q('enabled')}`, `CREATE VIEW mail_senders_v AS ` + - `SELECT ${address} AS sender, ${address} AS login ` + + `SELECT ${address} AS sender, ${address} AS login, ` + + `ma.${q('id')} AS account_id, ` + + `ma.${q('unsubscribeHeaders')} AS unsubscribe_headers ` + `${joins} WHERE ma.${q('enabled')} AND ${canSend}`, `CREATE VIEW mail_suppressions_v AS ` + From c2f3b1052130f561131977aa7d74f007b4b0b894 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:48:52 -0700 Subject: [PATCH 09/19] mail: packaging and manager image (#67) opensource-agent.deb suggests the mail packages (postfix(+sql), dovecot core/imapd/lmtpd(+sql), opendkim) and ships opensource-mail-helper.service. The manager image preseeds Postfix 'No configuration', installs the pgsql variants, keeps the units disabled until the agent claims, sets AGENT_SERVICES=nginx,dnsmasq,mail, maps postfix/dovecot/opensource-mail OS users onto the read-only mail_* DB roles via pg_hba/pg_ident peer auth, and EXPOSEs 25/143/465/587/993. setup-mail-db-roles.sh runs from container-creator-init.service and the server package postinst; roles script and views migration each grant best-effort so either ordering converges. --- agent/.fpm | 9 ++++++ agent/Makefile | 1 + create-a-container/bin/setup-mail-db-roles.sh | 8 ++++-- create-a-container/contrib/postinstall.sh | 7 +++++ .../20261008000006-create-mail-views.js | 24 ++++++++++++++++ images/manager/Dockerfile | 28 ++++++++++++++++++- images/manager/container-creator-init.service | 3 +- 7 files changed, 76 insertions(+), 4 deletions(-) diff --git a/agent/.fpm b/agent/.fpm index 3c06e600..d34a7d95 100644 --- a/agent/.fpm +++ b/agent/.fpm @@ -14,5 +14,14 @@ --depends ca-certificates --depends ssl-cert --depends dnsmasq +--deb-suggests postfix +--deb-suggests postfix-pgsql +--deb-suggests postfix-mysql +--deb-suggests dovecot-core +--deb-suggests dovecot-imapd +--deb-suggests dovecot-lmtpd +--deb-suggests dovecot-pgsql +--deb-suggests dovecot-mysql +--deb-suggests opendkim --deb-no-default-config-files --after-install contrib/postinstall.sh diff --git a/agent/Makefile b/agent/Makefile index fb051803..91bde761 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -52,6 +52,7 @@ install: build $(INSTALL) -d $(UNIT_DIR) $(INSTALL_DATA) contrib/systemd/opensource-agent.service $(UNIT_DIR)/ $(INSTALL_DATA) contrib/systemd/opensource-agent.timer $(UNIT_DIR)/ + $(INSTALL_DATA) contrib/systemd/opensource-mail-helper.service $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)$(PREFIX)/error-pages $(INSTALL_DATA) $(ERROR_PAGES)/* $(DESTDIR)$(PREFIX)/error-pages/ $(INSTALL) -d -m 0755 $(DESTDIR)/var/cache/nginx/auth_cache diff --git a/create-a-container/bin/setup-mail-db-roles.sh b/create-a-container/bin/setup-mail-db-roles.sh index 75bfb0aa..94e15d7d 100644 --- a/create-a-container/bin/setup-mail-db-roles.sh +++ b/create-a-container/bin/setup-mail-db-roles.sh @@ -80,9 +80,13 @@ DO \$\$ BEGIN ELSE ALTER ROLE mail_postfix WITH LOGIN PASSWORD '${POSTFIX_PW}'; END IF; + -- Views appear with the first migration run; that migration also grants, + -- so skipping here just means the other side finishes the handshake. + IF EXISTS (SELECT 1 FROM pg_views WHERE viewname = 'mail_accounts_v') THEN + GRANT SELECT ON mail_accounts_v TO mail_dovecot; + GRANT SELECT ON mail_senders_v, mail_suppressions_v TO mail_postfix; + END IF; END \$\$; -GRANT SELECT ON mail_accounts_v TO mail_dovecot; -GRANT SELECT ON mail_senders_v, mail_suppressions_v TO mail_postfix; SQL } diff --git a/create-a-container/contrib/postinstall.sh b/create-a-container/contrib/postinstall.sh index e92c7acf..1974e220 100755 --- a/create-a-container/contrib/postinstall.sh +++ b/create-a-container/contrib/postinstall.sh @@ -16,4 +16,11 @@ if [ -d /run/systemd/system ]; then systemctl restart $UNITS fi +# Mail DB roles (issue #67): (re)create the read-only mail_dovecot / +# mail_postfix users + view grants. Skipped before first-boot DB init +# (container-creator-init.service runs it then) and on SQLite. +if [ -f /etc/default/container-creator ]; then + bash /opt/opensource-server/create-a-container/bin/setup-mail-db-roles.sh || true +fi + exit 0 diff --git a/create-a-container/migrations/20261008000006-create-mail-views.js b/create-a-container/migrations/20261008000006-create-mail-views.js index 515a48a2..e6e1bc9c 100644 --- a/create-a-container/migrations/20261008000006-create-mail-views.js +++ b/create-a-container/migrations/20261008000006-create-mail-views.js @@ -14,6 +14,30 @@ module.exports = { for (const sql of createViewsSql(dialect)) { await queryInterface.sequelize.query(sql); } + // Grant SELECT when the roles already exist (setup-mail-db-roles.sh may + // run before or after this migration; each side grants best-effort so + // either ordering converges). + const grants = []; + if (dialect === 'postgres') { + grants.push( + 'GRANT SELECT ON mail_accounts_v TO mail_dovecot', + 'GRANT SELECT ON mail_senders_v, mail_suppressions_v TO mail_postfix', + ); + } else if (dialect === 'mysql' || dialect === 'mariadb') { + const db = queryInterface.sequelize.config.database; + grants.push( + `GRANT SELECT ON \`${db}\`.mail_accounts_v TO 'mail_dovecot'@'%'`, + `GRANT SELECT ON \`${db}\`.mail_senders_v TO 'mail_postfix'@'%'`, + `GRANT SELECT ON \`${db}\`.mail_suppressions_v TO 'mail_postfix'@'%'`, + ); + } + for (const sql of grants) { + try { + await queryInterface.sequelize.query(sql); + } catch { + /* roles not created yet — the script grants when it creates them */ + } + } }, async down(queryInterface) { diff --git a/images/manager/Dockerfile b/images/manager/Dockerfile index d8d7a38a..15f1416b 100644 --- a/images/manager/Dockerfile +++ b/images/manager/Dockerfile @@ -3,9 +3,11 @@ FROM agent # Default environment for the site agent. The manager runs the app on # localhost so it doesn't need an API key. Remote agents must set these -# explicitly. +# explicitly. The manager is the default mail host (issue #67): its agent +# reports the `mail` service group and takes the mail-host claim. ENV SITE_ID=1 ENV MANAGER_URL=http://localhost:3000 +ENV AGENT_SERVICES=nginx,dnsmasq,mail # Install Postgres 18 from the PGDG repository and enable remote access only # for TLS encrypted connections. Debian creates a self-signed cert for this so @@ -23,6 +25,23 @@ RUN apt-get update && apt-get -y install postgresql-common \ # override prevents a race condition at boot. COPY images/manager/wait-proxmox-regenerate-snakeoil.conf /etc/systemd/system/postgresql@.service.d/ +# Mail stack (issue #67): Postfix + Dovecot + OpenDKIM, configured entirely by +# the agent once it takes the mail-host claim — preseed "No configuration" and +# keep the units disabled until then. Dovecot/Postfix read accounts live from +# Postgres over the local socket: pg_ident maps their OS users onto the +# read-only mail_* roles created by setup-mail-db-roles.sh (peer auth, no +# passwords). The pg_hba rules must precede the catch-all "local all all peer". +RUN echo 'postfix postfix/main_mailer_type select No configuration' | debconf-set-selections && \ + apt-get update && \ + apt-get -y install postfix postfix-pgsql dovecot-core dovecot-imapd dovecot-lmtpd dovecot-pgsql opendkim && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + systemctl disable postfix dovecot opendkim && \ + sed -i '1i # Mail stack peer-auth maps (opensource-server issue #67)\nlocal all mail_postfix peer map=osmail\nlocal all mail_dovecot peer map=osmail' /etc/postgresql/18/main/pg_hba.conf && \ + { echo 'osmail postfix mail_postfix'; \ + echo 'osmail opensource-mail mail_postfix'; \ + echo 'osmail dovecot mail_dovecot'; } >>/etc/postgresql/18/main/pg_ident.conf + # First-boot database initialization. COPY images/manager/container-creator-init.service /etc/systemd/system/container-creator-init.service @@ -36,3 +55,10 @@ RUN --mount=from=builder,source=/dist,target=/dist \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ systemctl enable container-creator-init.service + +# Mail ports: SMTP, IMAP(S), submission (STARTTLS + implicit TLS). +EXPOSE 25 +EXPOSE 143 +EXPOSE 465 +EXPOSE 587 +EXPOSE 993 diff --git a/images/manager/container-creator-init.service b/images/manager/container-creator-init.service index 32082007..62081362 100644 --- a/images/manager/container-creator-init.service +++ b/images/manager/container-creator-init.service @@ -23,7 +23,8 @@ ExecStart=/bin/bash -e -c '\ echo "POSTGRES_DATABASE=$${POSTGRES_DATABASE}" >> /etc/default/container-creator; \ echo "POSTGRES_USER=$${POSTGRES_USER}" >> /etc/default/container-creator; \ echo "POSTGRES_PASSWORD=$${POSTGRES_PASSWORD}" >> /etc/default/container-creator; \ - echo "MCP_SERVER_URL=http://127.0.0.1:8100" >> /etc/default/container-creator;' + echo "MCP_SERVER_URL=http://127.0.0.1:8100" >> /etc/default/container-creator; \ + bash /opt/opensource-server/create-a-container/bin/setup-mail-db-roles.sh || true;' [Install] WantedBy=multi-user.target From ec8cd7ce9997df7990127ba2ebfb334e4b7adf05 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:50:43 -0700 Subject: [PATCH 10/19] mail: dev stack (#67) Publish and DNAT the mail ports (25/143/465/587/993) to the manager CT like 80/443; bootstrap-manager points the relayhost at MailDev (with the required mailSpfInclude placeholder) so submitted mail lands in http://localhost:1080; Check DNS gets a non-production-only bypass for localhost / *.localhost domains so the whole flow is testable in the dev stack. --- compose.yml | 9 ++++++++- create-a-container/utils/mail-dns.js | 21 +++++++++++++++++++++ images/proxmox-ve/interfaces | 4 ++-- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/compose.yml b/compose.yml index 66cd77b2..d25c45de 100644 --- a/compose.yml +++ b/compose.yml @@ -136,6 +136,12 @@ services: - 127.0.0.1:443:443 - 127.0.0.1:8006:8006 - 127.0.0.1:1080:1080 # maildev web UI (maildev shares this network namespace) + # Mail (issue #67): DNATed to the manager CT like 80/443. + - 127.0.0.1:25:25 + - 127.0.0.1:143:143 + - 127.0.0.1:465:465 + - 127.0.0.1:587:587 + - 127.0.0.1:993:993 hostname: proxmox domainname: cluster.internal dns: @@ -197,7 +203,8 @@ services: curl -H "X-CSRF-Token: $${CSRF}" --json '{"apiUrl":"https://10.254.0.1:8006","username":"root@pam","password":"root","tlsVerify":false}' https://proxmox/api/v1/sites/1/nodes/import curl -H "X-CSRF-Token: $${CSRF}" --json '{"name":"localhost","siteId":1,"acmeEmail":"","acmeDirectoryUrl":"","cloudflareApiEmail":"","cloudflareApiKey":"","authServer":""}' https://proxmox/api/v1/external-domains # Point SMTP at maildev only when unset, so a developer's own SMTP setting survives. PUT replaces every setting, hence the GET round-trip. - curl https://proxmox/api/v1/settings | jq '.data | select(.smtpUrl == "") | .smtpUrl = "smtp://10.254.0.1:1025" | .smtpNoreplyAddress |= if . == "" then "noreply@cluster.internal" else . end' > settings.json + # Mail service (issue #67): maildev doubles as the relayhost so outbound mail lands in http://localhost:1080; mailSpfInclude is required alongside a relayhost (dev placeholder). + curl https://proxmox/api/v1/settings | jq '.data | select(.smtpUrl == "" or .mailRelayhost == "") | .smtpUrl |= if . == "" then "smtp://10.254.0.1:1025" else . end | .smtpNoreplyAddress |= if . == "" then "noreply@cluster.internal" else . end | .mailRelayhost |= if . == "" then "[10.254.0.1]:1025" else . end | .mailSpfInclude |= if . == "" then "maildev.localhost" else . end | .mailHostname |= if . == "" then "localhost" else . end | .mailUnsubscribeBaseUrl |= if . == "" then "https://manager.localhost" else . end' > settings.json [ -s settings.json ] && curl -H "X-CSRF-Token: $${CSRF}" -X PUT --json @settings.json https://proxmox/api/v1/settings curl -H "X-CSRF-Token: $${CSRF}" -X PUT --json '{"hostname":"manager","template":"","entrypoint":"","nvidiaRequested":false,"services":{"s0":{"type":"http","internalPort":3000,"externalHostname":"manager","externalDomainId":1,"dnsName":"","authRequired":false}},"environmentVars":[],"restart":false}' https://proxmox/api/v1/sites/1/containers/1 depends_on: diff --git a/create-a-container/utils/mail-dns.js b/create-a-container/utils/mail-dns.js index 9a0bd1f7..32f74938 100644 --- a/create-a-container/utils/mail-dns.js +++ b/create-a-container/utils/mail-dns.js @@ -131,6 +131,26 @@ async function lookupPtr({ ip, resolvers, resolver }) { } } +/** Dev-stack bypass: `localhost` (and *.localhost) can never pass a real + * check. Outside production, Check DNS returns an all-pass synthetic result + * so the mail flow is testable end-to-end against MailDev. */ +function isDevBypassDomain(domain) { + return process.env.NODE_ENV !== 'production' && /(^|\.)localhost$/i.test(domain); +} + +function devBypassCheckResult(domain) { + return { + checkedAt: new Date().toISOString(), + devBypass: true, + spf: { pass: true, record: 'dev bypass' }, + dkim: { pass: true, selector: null, published: false }, + dmarc: { pass: true, record: 'dev bypass' }, + mx: { pass: true, hosts: [] }, + ptr: { name: null, forwardConfirmed: false }, + warnings: [`${domain} is a development domain — DNS checks were bypassed (non-production only).`], + }; +} + /** * Run the Check DNS pass for a domain. * @@ -146,6 +166,7 @@ async function lookupPtr({ ip, resolvers, resolver }) { * { checkedAt, spf|dkim|dmarc|mx: { pass, ... }, ptr: {...}, warnings: [] } */ async function checkMailDns({ domain, mailIp, mailHostname, spfInclude, dkimKey, resolvers, resolver }) { + if (isDevBypassDomain(domain)) return devBypassCheckResult(domain); const r = resolver || makeResolver(resolvers); const result = { checkedAt: new Date().toISOString(), warnings: [] }; diff --git a/images/proxmox-ve/interfaces b/images/proxmox-ve/interfaces index bc7f6296..fa23d106 100644 --- a/images/proxmox-ve/interfaces +++ b/images/proxmox-ve/interfaces @@ -21,8 +21,8 @@ iface vmbr0 inet static post-up echo 1 > /proc/sys/net/ipv4/ip_forward post-up iptables -t nat -A POSTROUTING -s 10.254.0.0/16 -o eth0 -j MASQUERADE - post-up iptables -t nat -A PREROUTING -i eth0 -p tcp -m multiport --dports 80,443,2000:2999 -j DNAT --to-destination 10.254.0.2 + post-up iptables -t nat -A PREROUTING -i eth0 -p tcp -m multiport --dports 80,443,2000:2999,25,143,465,587,993 -j DNAT --to-destination 10.254.0.2 post-up iptables -t nat -A PREROUTING -i eth0 -p udp -m multiport --dports 80,443,2000:2999 -j DNAT --to-destination 10.254.0.2 post-down iptables -t nat -D POSTROUTING -s 10.254.0.0/16 -o eth0 -j MASQUERADE - post-down iptables -t nat -D PREROUTING -i eth0 -p tcp -m multiport --dports 80,443,2000:2999 -j DNAT --to-destination 10.254.0.2 + post-down iptables -t nat -D PREROUTING -i eth0 -p tcp -m multiport --dports 80,443,2000:2999,25,143,465,587,993 -j DNAT --to-destination 10.254.0.2 post-down iptables -t nat -D PREROUTING -i eth0 -p udp -m multiport --dports 80,443,2000:2999 -j DNAT --to-destination 10.254.0.2 From 897cca2687c37dc934ef0bf47a418611345c63c9 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:54:19 -0700 Subject: [PATCH 11/19] mail UI: email accounts page with suppressions (#67) /mail-accounts page for all users, modeled on API keys: inline create form (can-send domain picker; helpful empty state when no domain can send), one-time password + connection-settings alert with copy button, enable/ disable, quota display, per-account RFC 8058 unsubscribe-headers toggle, rotate-password (re-shows credentials once), delete, and a suppressions modal (admins can remove entries). The public unsubscribe page itself is the server-rendered /u/{token} confirm page from the backend commit - deliberately outside the authenticated SPA. --- create-a-container/client/src/app/Sidebar.tsx | 2 + create-a-container/client/src/app/router.tsx | 3 + create-a-container/client/src/lib/queries.ts | 12 + create-a-container/client/src/lib/types.ts | 40 ++++ .../mail-accounts/MailAccountsListPage.tsx | 210 ++++++++++++++++++ .../mail-accounts/MailCredentialsAlert.tsx | 57 +++++ .../mail-accounts/NewMailAccountForm.tsx | 100 +++++++++ .../pages/mail-accounts/SuppressionsModal.tsx | 105 +++++++++ 8 files changed, 529 insertions(+) create mode 100644 create-a-container/client/src/pages/mail-accounts/MailAccountsListPage.tsx create mode 100644 create-a-container/client/src/pages/mail-accounts/MailCredentialsAlert.tsx create mode 100644 create-a-container/client/src/pages/mail-accounts/NewMailAccountForm.tsx create mode 100644 create-a-container/client/src/pages/mail-accounts/SuppressionsModal.tsx diff --git a/create-a-container/client/src/app/Sidebar.tsx b/create-a-container/client/src/app/Sidebar.tsx index bf6e4b17..d888878e 100644 --- a/create-a-container/client/src/app/Sidebar.tsx +++ b/create-a-container/client/src/app/Sidebar.tsx @@ -20,6 +20,7 @@ import { Container as ContainerIcon, Globe, KeyRound, + Mail, Radio, Server, Settings, @@ -61,6 +62,7 @@ const ADMIN: NavLink[] = [ }, { to: '/agents', label: 'Agents', icon: , adminOnly: true }, { to: '/apikeys', label: 'API Keys', icon: }, + { to: '/mail-accounts', label: 'Email', icon: }, { to: '/settings', label: 'Settings', icon: , adminOnly: true }, ]; diff --git a/create-a-container/client/src/app/router.tsx b/create-a-container/client/src/app/router.tsx index a0450d92..46282262 100644 --- a/create-a-container/client/src/app/router.tsx +++ b/create-a-container/client/src/app/router.tsx @@ -25,6 +25,7 @@ import { InviteUserPage } from '@/pages/users/InviteUserPage'; import { GroupsListPage } from '@/pages/groups/GroupsListPage'; import { GroupFormPage } from '@/pages/groups/GroupFormPage'; import { ApiKeysListPage } from '@/pages/apikeys/ApiKeysListPage'; +import { MailAccountsListPage } from '@/pages/mail-accounts/MailAccountsListPage'; import { SettingsPage } from '@/pages/settings/SettingsPage'; import { ResourceRequestsPage } from '@/pages/resource-requests/ResourceRequestsPage'; import { MyRequestsPage } from '@/pages/resource-requests/MyRequestsPage'; @@ -84,6 +85,8 @@ export const router = createBrowserRouter([ { path: '/apikeys', element: }, + { path: '/mail-accounts', element: }, + { path: '/settings', element: }, { path: '/resource-requests', element: }, diff --git a/create-a-container/client/src/lib/queries.ts b/create-a-container/client/src/lib/queries.ts index da4d006e..41040112 100644 --- a/create-a-container/client/src/lib/queries.ts +++ b/create-a-container/client/src/lib/queries.ts @@ -15,6 +15,9 @@ import type { Group, Job, JobStatusRow, + MailAccount, + MailDomain, + MailSuppression, Node, NodeStats, AppSettings, @@ -49,6 +52,9 @@ export const keys = { groups: () => ['groups'] as const, group: (id: number | string) => ['groups', String(id)] as const, apikeys: () => ['apikeys'] as const, + mailAccounts: () => ['mail-accounts'] as const, + mailDomains: () => ['mail-accounts', 'domains'] as const, + mailSuppressions: (id: string) => ['mail-accounts', id, 'suppressions'] as const, settings: () => ['settings'] as const, job: (id: number | string) => ['jobs', String(id)] as const, jobStatuses: (id: number | string) => ['jobs', String(id), 'statuses'] as const, @@ -142,6 +148,12 @@ export const queries = { // API keys listApiKeys: () => api.get('/api/v1/apikeys'), + // Mail accounts (issue #67) + listMailAccounts: () => api.get('/api/v1/mail-accounts'), + listMailDomains: () => api.get('/api/v1/mail-accounts/domains'), + listMailSuppressions: (id: string) => + api.get(`/api/v1/mail-accounts/${id}/suppressions`), + // Settings getSettings: () => api.get('/api/v1/settings'), diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index 3e391095..30bcf9f2 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -353,6 +353,46 @@ export interface ApiKeyCreated extends ApiKey { warning: string; } +/** A domain mail accounts can be created on (mail enabled + DNS verified). */ +export interface MailDomain { + id: number; + name: string; +} + +export interface MailAccount { + id: string; + address: string | null; + localPart: string; + domain: { id: number; name: string; canSend: boolean; canReceive: boolean } | null; + owner: string | null; + description: string | null; + enabled: boolean; + quotaMb: number; + unsubscribeHeaders: boolean; + lastRotatedAt: string | null; + createdAt: string; + updatedAt: string; +} + +/** Create/rotate response — the password is returned exactly once. */ +export interface MailAccountSecret extends MailAccount { + password: string; + connection: { + host: string; + username: string; + smtp: { ports: number[] }; + imap: { ports: number[] }; + }; + warning: string; +} + +export interface MailSuppression { + id: number; + recipient: string; + source: 'one-click' | 'admin'; + createdAt: string; +} + export interface AppSettings { smtpUrl: string; smtpNoreplyAddress: string; diff --git a/create-a-container/client/src/pages/mail-accounts/MailAccountsListPage.tsx b/create-a-container/client/src/pages/mail-accounts/MailAccountsListPage.tsx new file mode 100644 index 00000000..76046068 --- /dev/null +++ b/create-a-container/client/src/pages/mail-accounts/MailAccountsListPage.tsx @@ -0,0 +1,210 @@ +import { useState } from 'react'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { + Alert, AlertDescription, Badge, Button, PageHeader, Spinner, Switch, + Table, TableBody, TableCell, TableHead, TableHeader, TableRow, useToast, +} from '@mieweb/ui'; +import { Mail, MailX, Plus, RefreshCw, Trash2 } from 'lucide-react'; +import { api, ApiError } from '@/lib/api'; +import { keys, queries } from '@/lib/queries'; +import type { MailAccount, MailAccountSecret } from '@/lib/types'; +import { useDocumentTitle } from '@/lib/useDocumentTitle'; +import { MailCredentialsAlert } from './MailCredentialsAlert'; +import { NewMailAccountForm } from './NewMailAccountForm'; +import { SuppressionsModal } from './SuppressionsModal'; + +export function MailAccountsListPage() { + useDocumentTitle('Email accounts'); + const { data, isLoading, error } = useQuery({ + queryKey: keys.mailAccounts(), + queryFn: queries.listMailAccounts, + }); + const qc = useQueryClient(); + const toast = useToast(); + const [creating, setCreating] = useState(false); + const [created, setCreated] = useState(null); + const [suppressionsFor, setSuppressionsFor] = useState(null); + + const invalidate = () => qc.invalidateQueries({ queryKey: keys.mailAccounts() }); + + const patch = useMutation({ + mutationFn: ({ id, body }: { id: string; body: Record }) => + api.patch(`/api/v1/mail-accounts/${id}`, body), + onSuccess: invalidate, + onError: (err: ApiError) => toast.error(err.message), + }); + + const rotate = useMutation({ + mutationFn: (id: string) => + api.post(`/api/v1/mail-accounts/${id}/rotate-password`), + onSuccess: (secret) => { + setCreated(secret); + invalidate(); + }, + onError: (err: ApiError) => toast.error(err.message), + }); + + const del = useMutation({ + mutationFn: (id: string) => api.delete(`/api/v1/mail-accounts/${id}`), + onSuccess: () => { + toast.success('Email account deleted'); + invalidate(); + }, + onError: (err: ApiError) => toast.error(err.message), + }); + + return ( +
+ } + actions={ + !creating && ( + + ) + } + /> + + {created && setCreated(null)} />} + + {creating && ( + { + setCreated(secret); + setCreating(false); + }} + onCancel={() => setCreating(false)} + /> + )} + + {error && ( + + {(error as ApiError).message} + + )} + {isLoading && ( +
+ +
+ )} + + {data && data.length === 0 && !creating && ( + + + No email accounts yet. Create one to send via SMTP (ports 587/465) and read replies + via IMAP (ports 993/143) on an enabled domain. + + + )} + + {data && data.length > 0 && ( + + + + Address + Description + Status + Quota + Unsubscribe headers + Actions + + + + {data.map((a: MailAccount) => ( + + {a.address} + {a.description || '—'} + +
+ {a.enabled ? ( + Enabled + ) : ( + Disabled + )} + {a.domain && !a.domain.canSend && ( + + Domain can't send + + )} +
+ + {a.quotaMb} MB + + + patch.mutate({ id: a.id, body: { unsubscribeHeaders: checked } }) + } + aria-label={`Toggle RFC 8058 unsubscribe headers for ${a.address}`} + /> + + +
+ + + + +
+ + + ))} + +
+ )} + + {suppressionsFor && ( + setSuppressionsFor(null)} /> + )} +
+ ); +} diff --git a/create-a-container/client/src/pages/mail-accounts/MailCredentialsAlert.tsx b/create-a-container/client/src/pages/mail-accounts/MailCredentialsAlert.tsx new file mode 100644 index 00000000..cf84d8d1 --- /dev/null +++ b/create-a-container/client/src/pages/mail-accounts/MailCredentialsAlert.tsx @@ -0,0 +1,57 @@ +import { Alert, AlertDescription, Button, useToast } from '@mieweb/ui'; +import type { MailAccountSecret } from '@/lib/types'; + +/** One-time password + connection settings shown after create/rotate. */ +export function MailCredentialsAlert({ + created, + onDismiss, +}: { + created: MailAccountSecret; + onDismiss: () => void; +}) { + const toast = useToast(); + const copy = (value: string) => { + navigator.clipboard.writeText(value).catch(() => undefined); + toast.success('Copied to clipboard'); + }; + return ( + + +
+ {created.warning} +
+ + {created.password} + + +
+
+
Username
+
{created.connection.username}
+
Server
+
{created.connection.host}
+
SMTP
+
+ ports {created.connection.smtp.ports.join(', ')} (STARTTLS on 587, implicit TLS on 465) +
+
IMAP
+
ports {created.connection.imap.ports.join(', ')}
+
+
+ +
+
+ + + ); +} diff --git a/create-a-container/client/src/pages/mail-accounts/NewMailAccountForm.tsx b/create-a-container/client/src/pages/mail-accounts/NewMailAccountForm.tsx new file mode 100644 index 00000000..bb932f63 --- /dev/null +++ b/create-a-container/client/src/pages/mail-accounts/NewMailAccountForm.tsx @@ -0,0 +1,100 @@ +import { useState } from 'react'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { Alert, AlertDescription, Button, Input, Select } from '@mieweb/ui'; +import { api, ApiError } from '@/lib/api'; +import { keys, queries } from '@/lib/queries'; +import type { MailAccountSecret } from '@/lib/types'; + +/** Inline create form — accounts can only be created on can-send domains. */ +export function NewMailAccountForm({ + onCreated, + onCancel, +}: { + onCreated: (created: MailAccountSecret) => void; + onCancel: () => void; +}) { + const qc = useQueryClient(); + const { data: domains } = useQuery({ queryKey: keys.mailDomains(), queryFn: queries.listMailDomains }); + const [externalDomainId, setExternalDomainId] = useState(''); + const [localPart, setLocalPart] = useState(''); + const [description, setDescription] = useState(''); + + const create = useMutation({ + mutationFn: () => + api.post('/api/v1/mail-accounts', { + externalDomainId: parseInt(externalDomainId, 10), + localPart, + description: description || null, + }), + onSuccess: (created) => { + qc.invalidateQueries({ queryKey: keys.mailAccounts() }); + onCreated(created); + }, + }); + + if (domains && domains.length === 0) { + return ( + + + No domain can send mail yet. An administrator must enable mail on an external domain + (publish its DNS records and pass Check DNS) before accounts can be created. + + + ); + } + + const domainName = domains?.find((d) => String(d.id) === externalDomainId)?.name; + + return ( +
{ + e.preventDefault(); + create.mutate(); + }} + className="flex flex-col gap-3 rounded-lg border border-(--color-border,#e5e7eb) p-4" + > +
+ setLocalPart(e.target.value)} + helperText={domainName ? `Will become ${localPart || '
'}@${domainName}` : undefined} + required + /> +
+ setDescription(e.target.value)} + /> + {create.error && ( + + {(create.error as ApiError).message} + + )} +
+ + +
+
+ ); +} diff --git a/create-a-container/client/src/pages/mail-accounts/SuppressionsModal.tsx b/create-a-container/client/src/pages/mail-accounts/SuppressionsModal.tsx new file mode 100644 index 00000000..7477a702 --- /dev/null +++ b/create-a-container/client/src/pages/mail-accounts/SuppressionsModal.tsx @@ -0,0 +1,105 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { + Alert, AlertDescription, Button, Modal, ModalBody, ModalHeader, ModalTitle, Spinner, + Table, TableBody, TableCell, TableHead, TableHeader, TableRow, useToast, +} from '@mieweb/ui'; +import { Trash2 } from 'lucide-react'; +import { api, ApiError } from '@/lib/api'; +import { useSession } from '@/lib/auth'; +import { keys, queries } from '@/lib/queries'; +import type { MailAccount, MailSuppression } from '@/lib/types'; + +/** Recipients who one-click-unsubscribed from this account. Only admins can + * remove entries — owners must not quietly re-subscribe people. */ +export function SuppressionsModal({ + account, + onClose, +}: { + account: MailAccount; + onClose: () => void; +}) { + const { data: session } = useSession(); + const isAdmin = !!session?.isAdmin; + const qc = useQueryClient(); + const toast = useToast(); + const { data, isLoading, error } = useQuery({ + queryKey: keys.mailSuppressions(account.id), + queryFn: () => queries.listMailSuppressions(account.id), + }); + + const remove = useMutation({ + mutationFn: (suppressionId: number) => + api.delete(`/api/v1/mail-accounts/${account.id}/suppressions/${suppressionId}`), + onSuccess: () => { + toast.success('Suppression removed'); + qc.invalidateQueries({ queryKey: keys.mailSuppressions(account.id) }); + }, + onError: (err: ApiError) => toast.error(err.message), + }); + + return ( + !open && onClose()}> + + Unsubscribed recipients — {account.address} + + + {error && ( + + {(error as ApiError).message} + + )} + {isLoading && ( +
+ +
+ )} + {data && data.length === 0 && ( +

+ Nobody has unsubscribed from this account. +

+ )} + {data && data.length > 0 && ( + + + + Recipient + Source + Since + {isAdmin && ( + + Actions + + )} + + + + {data.map((s: MailSuppression) => ( + + {s.recipient} + {s.source} + {new Date(s.createdAt).toLocaleDateString()} + {isAdmin && ( + + + + )} + + ))} + +
+ )} + + + ); +} From 21e61d915bb1fea571ed1e3bd60277ee78bb9a5c Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 10:57:58 -0700 Subject: [PATCH 12/19] mail UI: domain enablement, agents mail-host, settings (#67) External-domain edit page gains a Mail section: DNS records to publish with copy buttons, Check DNS, enable/disable, send/receive badges with reasons, per-gate last-check badges + warnings, and a notice when nothing runs the mail service. Agents page shows a Mail column (Mail host / Standby / Missing packages badges) plus release-claim and clear-key-pin actions. Settings gains a Mail service section for all mail_* settings with a 'Suggest from PTR' helper and relayhost=>SPF-include validation. --- create-a-container/client/src/lib/queries.ts | 9 + create-a-container/client/src/lib/types.ts | 78 +++++++ .../src/pages/agents/AgentRowActions.tsx | 76 +++++++ .../src/pages/agents/AgentsListPage.tsx | 10 + .../client/src/pages/agents/MailHostBadge.tsx | 28 +++ .../external-domains/DomainMailSection.tsx | 203 ++++++++++++++++++ .../ExternalDomainFormPage.tsx | 3 + .../src/pages/settings/SettingsPage.tsx | 131 ++++++++++- 8 files changed, 537 insertions(+), 1 deletion(-) create mode 100644 create-a-container/client/src/pages/agents/AgentRowActions.tsx create mode 100644 create-a-container/client/src/pages/agents/MailHostBadge.tsx create mode 100644 create-a-container/client/src/pages/external-domains/DomainMailSection.tsx diff --git a/create-a-container/client/src/lib/queries.ts b/create-a-container/client/src/lib/queries.ts index 41040112..e6e4e7ea 100644 --- a/create-a-container/client/src/lib/queries.ts +++ b/create-a-container/client/src/lib/queries.ts @@ -17,6 +17,9 @@ import type { JobStatusRow, MailAccount, MailDomain, + MailDnsInfo, + MailHostInfo, + MailPtrInfo, MailSuppression, Node, NodeStats, @@ -55,6 +58,8 @@ export const keys = { mailAccounts: () => ['mail-accounts'] as const, mailDomains: () => ['mail-accounts', 'domains'] as const, mailSuppressions: (id: string) => ['mail-accounts', id, 'suppressions'] as const, + mailDns: (domainId: number | string) => ['external-domains', String(domainId), 'mail-dns'] as const, + mailHost: () => ['mail', 'host'] as const, settings: () => ['settings'] as const, job: (id: number | string) => ['jobs', String(id)] as const, jobStatuses: (id: number | string) => ['jobs', String(id), 'statuses'] as const, @@ -153,6 +158,10 @@ export const queries = { listMailDomains: () => api.get('/api/v1/mail-accounts/domains'), listMailSuppressions: (id: string) => api.get(`/api/v1/mail-accounts/${id}/suppressions`), + getDomainMailDns: (domainId: number | string) => + api.get(`/api/v1/external-domains/${domainId}/mail/dns`), + getMailHost: () => api.get('/api/v1/mail/host'), + getMailPtr: () => api.get('/api/v1/mail/ptr'), // Settings getSettings: () => api.get('/api/v1/settings'), diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index 30bcf9f2..21435225 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -156,6 +156,14 @@ export interface Agent { hostname: string; ipv4Address: string | null; services: Record | null; + /** Service groups the agent runs (AGENT_SERVICES), e.g. ['nginx','dnsmasq','mail']. */ + enabledServices: string[] | null; + /** Binaries missing for the enabled groups; blocks the mail-host claim. */ + missingBinaries: string[] | null; + /** Non-null on the single agent holding the mail-host claim. */ + mailHostSince: string | null; + isLocal: boolean; + hasApiKeyPin: boolean; lastCheckinAt: string | null; /** Server-computed, so it is immune to client clock drift. */ secondsSinceCheckin: number | null; @@ -194,6 +202,11 @@ export interface ExternalDomain { site: { id: number; name: string } | null; authServer: string | null; hasCloudflareApiKey: boolean; + /** Read-only mail state (issue #67) — changed via the …/mail endpoints. */ + mailEnabled: boolean; + mailDnsVerified: boolean; + mailMxVerified: boolean; + mailDnsCheckedAt: string | null; } export interface ServiceHttp { @@ -393,6 +406,59 @@ export interface MailSuppression { createdAt: string; } +export interface MailDnsRecord { + type: 'TXT' | 'MX'; + name: string; + value: string; +} + +/** One gate of the stored Check DNS outcome. */ +export interface MailDnsGate { + pass: boolean; + [key: string]: unknown; +} + +export interface MailDnsCheckResult { + checkedAt: string; + spf: MailDnsGate; + dkim: MailDnsGate; + dmarc: MailDnsGate; + mx: MailDnsGate; + ptr?: { name: string | null; forwardConfirmed: boolean }; + warnings: string[]; +} + +export interface MailDnsInfo { + records: MailDnsRecord[]; + mailEnabled: boolean; + mailDnsVerified: boolean; + mailMxVerified: boolean; + lastCheck: { checkedAt: string; result: MailDnsCheckResult | null } | null; +} + +export interface MailHostInfo { + agent: { + id: number; + siteId: number; + siteName: string | null; + hostname: string; + mailHostSince: string; + enabledServices: string[] | null; + missingBinaries: string[] | null; + } | null; + mailIp: string | null; + mailHostname: string | null; + selfManagedSiteId: number | null; +} + +export interface MailPtrInfo { + mailIp: string | null; + ptr: string | null; + forwardConfirmed: boolean; + mailHostname: string | null; + suggestion: string | null; +} + export interface AppSettings { smtpUrl: string; smtpNoreplyAddress: string; @@ -403,6 +469,18 @@ export interface AppSettings { bannerMessage: string; /** Max PSI probes per usage report; '' uses the server default (16), '0' disables. */ usagePsiProbeLimit: string; + /** Mail service (issue #67). All stored as strings; '' means unset. */ + mailHostname: string; + mailUnsubscribeBaseUrl: string; + mailRelayhost: string; + mailRelayhostUsername: string; + mailRelayhostPassword: string; + mailSpfInclude: string; + mailDnsCheckResolvers: string; + mailDbHost: string; + mailSelfManagedSiteId: string; + mailDefaultQuotaMb: string; + mailMessageSizeLimitMb: string; } export type ResourceType = 'memory' | 'swap' | 'cpus' | 'rootfs'; diff --git a/create-a-container/client/src/pages/agents/AgentRowActions.tsx b/create-a-container/client/src/pages/agents/AgentRowActions.tsx new file mode 100644 index 00000000..ac5488e0 --- /dev/null +++ b/create-a-container/client/src/pages/agents/AgentRowActions.tsx @@ -0,0 +1,76 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import { Button, useToast } from '@mieweb/ui'; +import { KeyRound, MailMinus } from 'lucide-react'; +import { api, ApiError } from '@/lib/api'; +import { keys } from '@/lib/queries'; +import type { Agent } from '@/lib/types'; + +/** Admin actions on one agent row: release the mail-host claim (holder only) + * and clear the API-key pin (remote agents re-pin on their next check-in). */ +export function AgentRowActions({ agent }: { agent: Agent }) { + const qc = useQueryClient(); + const toast = useToast(); + const invalidate = () => { + qc.invalidateQueries({ queryKey: keys.agents() }); + qc.invalidateQueries({ queryKey: keys.mailHost() }); + }; + + const release = useMutation({ + mutationFn: () => api.post('/api/v1/mail/host/release'), + onSuccess: () => { + toast.success('Mail-host claim released — the next qualifying check-in takes it'); + invalidate(); + }, + onError: (err: ApiError) => toast.error(err.message), + }); + + const clearPin = useMutation({ + mutationFn: () => api.delete(`/api/v1/agents/${agent.id}/api-key-pin`), + onSuccess: () => { + toast.success('API-key pin cleared'); + invalidate(); + }, + onError: (err: ApiError) => toast.error(err.message), + }); + + if (!agent.mailHostSince && !agent.hasApiKeyPin) { + return —; + } + + return ( +
+ {agent.mailHostSince && ( + + )} + {agent.hasApiKeyPin && ( + + )} +
+ ); +} diff --git a/create-a-container/client/src/pages/agents/AgentsListPage.tsx b/create-a-container/client/src/pages/agents/AgentsListPage.tsx index b7a04c01..f1505958 100644 --- a/create-a-container/client/src/pages/agents/AgentsListPage.tsx +++ b/create-a-container/client/src/pages/agents/AgentsListPage.tsx @@ -18,6 +18,8 @@ import { keys, queries } from '@/lib/queries'; import type { Agent } from '@/lib/types'; import { useDocumentTitle } from '@/lib/useDocumentTitle'; import { AgentServiceBadges } from './AgentServiceBadges'; +import { AgentRowActions } from './AgentRowActions'; +import { MailHostBadge } from './MailHostBadge'; import { OnlineBadge } from './OnlineBadge'; export function AgentsListPage() { @@ -65,7 +67,9 @@ export function AgentsListPage() { IPv4 Status Services + Mail Last check-in + Actions @@ -80,7 +84,13 @@ export function AgentsListPage() { + + + {formatRelativeTime(agent.lastCheckinAt, agent.secondsSinceCheckin)} + + + ))} diff --git a/create-a-container/client/src/pages/agents/MailHostBadge.tsx b/create-a-container/client/src/pages/agents/MailHostBadge.tsx new file mode 100644 index 00000000..3aa4f6fc --- /dev/null +++ b/create-a-container/client/src/pages/agents/MailHostBadge.tsx @@ -0,0 +1,28 @@ +import { Badge } from '@mieweb/ui'; +import type { Agent } from '@/lib/types'; + +/** Mail-host claim state for one agent row. */ +export function MailHostBadge({ agent }: { agent: Agent }) { + if (agent.mailHostSince) { + return ( + + Mail host + + ); + } + if (!agent.enabledServices?.includes('mail')) { + return —; + } + if (agent.missingBinaries && agent.missingBinaries.length > 0) { + return ( + + Missing packages + + ); + } + return ( + + Standby + + ); +} diff --git a/create-a-container/client/src/pages/external-domains/DomainMailSection.tsx b/create-a-container/client/src/pages/external-domains/DomainMailSection.tsx new file mode 100644 index 00000000..394294d0 --- /dev/null +++ b/create-a-container/client/src/pages/external-domains/DomainMailSection.tsx @@ -0,0 +1,203 @@ +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { + Alert, AlertDescription, Badge, Button, Spinner, + Table, TableBody, TableCell, TableHead, TableHeader, TableRow, useToast, +} from '@mieweb/ui'; +import { Copy, MailCheck, RefreshCw } from 'lucide-react'; +import { api, ApiError } from '@/lib/api'; +import { keys, queries } from '@/lib/queries'; +import type { ExternalDomain, MailDnsCheckResult } from '@/lib/types'; + +const GATES: { key: keyof Pick; label: string }[] = [ + { key: 'spf', label: 'SPF' }, + { key: 'dkim', label: 'DKIM' }, + { key: 'dmarc', label: 'DMARC' }, + { key: 'mx', label: 'MX' }, +]; + +/** Mail enablement for one external domain: records to publish (with copy + * buttons), Check DNS, enable/disable, and send/receive badges with reasons. */ +export function DomainMailSection({ domain }: { domain: ExternalDomain }) { + const qc = useQueryClient(); + const toast = useToast(); + const { data, isLoading } = useQuery({ + queryKey: keys.mailDns(domain.id), + queryFn: () => queries.getDomainMailDns(domain.id), + }); + const { data: host } = useQuery({ queryKey: keys.mailHost(), queryFn: queries.getMailHost }); + + const invalidate = () => { + qc.invalidateQueries({ queryKey: keys.mailDns(domain.id) }); + qc.invalidateQueries({ queryKey: keys.externalDomain(domain.id) }); + qc.invalidateQueries({ queryKey: keys.externalDomains() }); + }; + + const action = useMutation({ + mutationFn: (verb: 'check-dns' | 'enable' | 'disable') => + api.post(`/api/v1/external-domains/${domain.id}/mail/${verb}`), + onSuccess: (_data, verb) => { + toast.success( + verb === 'check-dns' ? 'DNS re-checked' : verb === 'enable' ? 'Mail enabled' : 'Mail disabled', + ); + invalidate(); + }, + onError: (err: ApiError) => { + toast.error(err.message); + invalidate(); // check-dns ran even when enable was refused + }, + }); + + const copy = (value: string) => { + navigator.clipboard.writeText(value).catch(() => undefined); + toast.success('Copied to clipboard'); + }; + + if (isLoading || !data) { + return ( +
+ +
+ ); + } + + const canSend = data.mailEnabled && data.mailDnsVerified; + const canReceive = data.mailEnabled && data.mailMxVerified; + const check = data.lastCheck?.result ?? null; + const noMailHost = host && !host.agent && !host.selfManagedSiteId; + + return ( +
+
+

+ Mail +

+
+ + {canSend ? 'Can send' : "Can't send"} + + + {canReceive ? 'Can receive' : "Can't receive"} + +
+
+ + {noMailHost && ( + + + No agent currently runs the mail service and no self-managed site is configured — mail + can be prepared here but nothing will deliver it. See the Agents page. + + + )} + +

+ Publish these DNS records, then run Check DNS. Sending needs SPF, DKIM, and DMARC; + receiving needs the MX record. +

+ + + + Type + Name + Value + + Copy + + + + + {data.records.map((r) => ( + + {r.type} + {r.name} + + {r.value} + + + + + + ))} + +
+ + {check && ( +
+ + Last check {new Date(check.checkedAt).toLocaleString()}: + + {GATES.map(({ key, label }) => ( + + {label} + + ))} +
+ )} + {check && check.warnings.length > 0 && ( + + +
    + {check.warnings.map((w) => ( +
  • {w}
    • + ))} +
+ + + )} + +
+ + {data.mailEnabled ? ( + + ) : ( + + )} +
+
+ ); +} diff --git a/create-a-container/client/src/pages/external-domains/ExternalDomainFormPage.tsx b/create-a-container/client/src/pages/external-domains/ExternalDomainFormPage.tsx index a3f1cf14..8a135f34 100644 --- a/create-a-container/client/src/pages/external-domains/ExternalDomainFormPage.tsx +++ b/create-a-container/client/src/pages/external-domains/ExternalDomainFormPage.tsx @@ -17,6 +17,7 @@ import { Globe } from 'lucide-react'; import { api, ApiError } from '@/lib/api'; import { keys, queries } from '@/lib/queries'; import { FormPageLayout } from '@/components/FormPageLayout'; +import { DomainMailSection } from './DomainMailSection'; import type { ExternalDomain } from '@/lib/types'; const schema = z.object({ @@ -176,6 +177,8 @@ export function ExternalDomainFormPage() { {(mutation.error as ApiError).message} )} + {/* Mail enablement mutates via its own endpoints — not part of the PUT form. */} + {isEdit && domain && } ); diff --git a/create-a-container/client/src/pages/settings/SettingsPage.tsx b/create-a-container/client/src/pages/settings/SettingsPage.tsx index d01628d8..8acd70f8 100644 --- a/create-a-container/client/src/pages/settings/SettingsPage.tsx +++ b/create-a-container/client/src/pages/settings/SettingsPage.tsx @@ -38,6 +38,20 @@ const schema = z.object({ netboxToken: z.string(), bannerMessage: z.string(), usagePsiProbeLimit: z.string().regex(/^\d*$/, 'Must be a non-negative whole number'), + mailHostname: z.string(), + mailUnsubscribeBaseUrl: z.string(), + mailRelayhost: z.string(), + mailRelayhostUsername: z.string(), + mailRelayhostPassword: z.string(), + mailSpfInclude: z.string(), + mailDnsCheckResolvers: z.string(), + mailDbHost: z.string(), + mailSelfManagedSiteId: z.string().regex(/^\d*$/, 'Must be a site id'), + mailDefaultQuotaMb: z.string().regex(/^\d*$/, 'Must be a whole number of MB'), + mailMessageSizeLimitMb: z.string().regex(/^\d*$/, 'Must be a whole number of MB'), +}).refine((v) => !v.mailRelayhost.trim() || !!v.mailSpfInclude.trim(), { + path: ['mailSpfInclude'], + message: 'Required when a relayhost is set — SPF must delegate to the relay', }); type FormData = z.infer; @@ -46,7 +60,7 @@ export function SettingsPage() { const toast = useToast(); const { data, isLoading, error } = useQuery({ queryKey: keys.settings(), queryFn: queries.getSettings }); - const { register, handleSubmit, reset, control, formState } = useForm({ + const { register, handleSubmit, reset, control, formState, setValue } = useForm({ resolver: zodResolver(schema), defaultValues: { smtpUrl: '', @@ -56,6 +70,17 @@ export function SettingsPage() { netboxToken: '', bannerMessage: '', usagePsiProbeLimit: '', + mailHostname: '', + mailUnsubscribeBaseUrl: '', + mailRelayhost: '', + mailRelayhostUsername: '', + mailRelayhostPassword: '', + mailSpfInclude: '', + mailDnsCheckResolvers: '', + mailDbHost: '', + mailSelfManagedSiteId: '', + mailDefaultQuotaMb: '', + mailMessageSizeLimitMb: '', }, }); const { fields, append, remove } = useFieldArray({ control, name: 'defaultContainerEnvVars' }); @@ -76,6 +101,22 @@ export function SettingsPage() { onError: (err: ApiError) => toast.error(err.message), }); + // Suggest the forward-confirmed PTR of the mail IP as mail_hostname. + const suggestPtr = useMutation({ + mutationFn: queries.getMailPtr, + onSuccess: (ptr) => { + if (ptr.suggestion) { + setValue('mailHostname', ptr.suggestion, { shouldDirty: true }); + toast.success(`Suggested ${ptr.suggestion} from the PTR of ${ptr.mailIp}`); + } else if (!ptr.mailIp) { + toast.error('No mail IP yet — no agent holds the mail-host claim'); + } else { + toast.error(`No forward-confirmed PTR for ${ptr.mailIp}${ptr.ptr ? ` (found ${ptr.ptr})` : ''}`); + } + }, + onError: (err: ApiError) => toast.error(err.message), + }); + if (isLoading) return
; if (error) return {(error as ApiError).message}; @@ -94,6 +135,94 @@ export function SettingsPage() { +
+

Mail service

+
+
+ +
+ +
+ + +
+ + +
+ + + +
+ + + +
+
+

Default container environment variables

From 616a583d7d8a463bc9d19646865a8ab2b59098e8 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Thu, 8 Oct 2026 11:01:14 -0700 Subject: [PATCH 13/19] mail: docs (#67) New admins/email.md (mail host claim, settings, where mail runs A/B/C, DB roles, backups) and users/email-accounts.md (client settings, unsubscribe headers + suppressions behavior, limits). External-domains doc gains the Mail enablement section; deploying-agents and the agent developer doc cover AGENT_SERVICES, the mail service group, and the MAIL_*_DB_PASSWORD env vars; installation.md documents the mail ports and the PTR requirement; database-schema.md adds the mail tables to the ER diagram plus the mail-views contract table. Nav updated. --- .../admins/core-concepts/external-domains.md | 12 +++ .../docs/admins/deploying-agents.md | 1 + mie-opensource-landing/docs/admins/email.md | 51 +++++++++++++ .../docs/admins/installation.md | 4 + .../docs/developers/agent.md | 23 ++++-- .../docs/developers/database-schema.md | 76 ++++++++++++++++++- .../docs/users/email-accounts.md | 41 ++++++++++ mie-opensource-landing/zensical.toml | 2 + 8 files changed, 203 insertions(+), 7 deletions(-) create mode 100644 mie-opensource-landing/docs/admins/email.md create mode 100644 mie-opensource-landing/docs/users/email-accounts.md diff --git a/mie-opensource-landing/docs/admins/core-concepts/external-domains.md b/mie-opensource-landing/docs/admins/core-concepts/external-domains.md index 967723fb..8d6d5a76 100644 --- a/mie-opensource-landing/docs/admins/core-concepts/external-domains.md +++ b/mie-opensource-landing/docs/admins/core-concepts/external-domains.md @@ -103,6 +103,18 @@ On container or service deletion, cross-site A records are cleaned up automatica When creating a container service, users select an external domain and specify a subdomain (e.g., `app` for `app.example.com`). All external domains are available regardless of which site the container is on. See the [Web GUI guide](../../users/creating-containers/web-gui.md) for details. +## Mail + +Every domain gets an RSA-2048 DKIM key at creation. Mail stays off until an admin enables it on the domain's edit page: + +1. Publish the DNS records shown in the **Mail** section (SPF, DKIM, DMARC; MX to receive). Copy buttons provided. +2. **Check DNS** — verified against public resolvers (configurable in [Settings](../email.md#settings)). +3. **Enable mail** — refused (`dns_not_configured`) until SPF + DKIM + DMARC verify. A failing MX only blocks *receiving*. + +Gates: `enabled + DNS verified = can send` (accounts can be created and SMTP login works); `enabled + MX verified = can receive`. Re-verification is manual — re-run Check DNS after changing records. Disabling keeps keys and config; accounts stop authenticating immediately, IMAP stays readable. + +PTR problems and missing `postmaster@`/`abuse@` mailboxes are warnings, not blockers. See [Email Service](../email.md) for the mail host and settings. + ## Security - Issue Cloudflare User API Tokens with minimum scope (`Zone:DNS:Edit` for the target zone only) diff --git a/mie-opensource-landing/docs/admins/deploying-agents.md b/mie-opensource-landing/docs/admins/deploying-agents.md index 133c93f4..0a806bae 100644 --- a/mie-opensource-landing/docs/admins/deploying-agents.md +++ b/mie-opensource-landing/docs/admins/deploying-agents.md @@ -70,6 +70,7 @@ lxc.environment = API_KEY= | `SITE_ID` | Numeric site ID from the manager (visible in the URL when viewing the site) | | `MANAGER_URL` | Base URL of the manager container (e.g., `http://192.168.1.10:3000`) | | `API_KEY` | API key from an admin account. Used to authenticate check-ins. | +| `AGENT_SERVICES` | Optional. Comma-separated service groups to run: `nginx`, `dnsmasq`, `mail`. Default `nginx,dnsmasq`. Add `mail` to make this host the [mail host](email.md) (install the mail packages first). | ## Volume storage for persistent volumes diff --git a/mie-opensource-landing/docs/admins/email.md b/mie-opensource-landing/docs/admins/email.md new file mode 100644 index 00000000..11d78b8c --- /dev/null +++ b/mie-opensource-landing/docs/admins/email.md @@ -0,0 +1,51 @@ +# Email Service + +Authenticated SMTP submission + IMAP hosted by an agent (Postfix + Dovecot + OpenDKIM), with DKIM signing, per-domain enablement, and RFC 8058 one-click unsubscribe. User-facing usage is covered in [Email Accounts](../users/email-accounts.md); per-domain DNS setup in [External Domains](core-concepts/external-domains.md#mail). + +## The Mail Host + +Exactly one agent runs the mail stack at a time (mailboxes are local Maildir). The claim goes to the first agent that checks in with the `mail` service group enabled and all packages installed; other agents show **Standby** on the Agents page. Release the claim from the Agents page (**Release mail host**) — the next qualifying check-in takes it. + +The mail IP is the `externalIp` of the mail host's site. Dovecot and Postfix read accounts **live** from the manager's database through three read-only views (`mail_accounts_v`, `mail_senders_v`, `mail_suppressions_v`), so disabling an account or domain takes effect immediately (bounded by Dovecot's 1-minute auth cache). + +## Settings + +Settings → Mail service: + +| Setting | Purpose | +|---------|---------| +| Mail hostname | HELO name and MX target. Set to the forward-confirmed PTR of the mail IP — use **Suggest from PTR**. | +| Unsubscribe base URL | Public manager URL; one-click unsubscribe links point at `/u/` | +| Relayhost (+ credentials) | Optional smarthost. Requires **SPF include**, since mail then leaves from the relay's IPs | +| SPF include | `include:` added to the suggested SPF records | +| DNS check resolvers | Resolvers used by Check DNS (default `1.1.1.1`, `8.8.8.8`) | +| Mail DB host | Only when the mail host cannot reach the DB over the local socket | +| Self-managed site id | Mail IP source when no agent runs mail (option C below) | +| Default quota / message size limit | `1024` MB and `25` MB when empty | + +## Where Mail Runs + +**A. Manager (default).** The manager image ships the packages and sets `AGENT_SERVICES=nginx,dnsmasq,mail`. DB access uses Postgres peer auth (`pg_ident` maps the `postfix`/`dovecot`/`opensource-mail` OS users to the `mail_postfix`/`mail_dovecot` roles) — no passwords. + +**B. Another agent host.** + +1. Install the suggested packages: `apt install postfix postfix-pgsql dovecot-core dovecot-imapd dovecot-lmtpd dovecot-pgsql opendkim` (use the `-mysql` variants on MySQL/MariaDB). +2. Install the per-domain certs under `/etc/ssl` (same layout as nginx). +3. Set the **Mail DB host** setting, and put `MAIL_POSTFIX_DB_PASSWORD` / `MAIL_DOVECOT_DB_PASSWORD` (from `/etc/default/container-creator` on the manager, created by `bin/setup-mail-db-roles.sh`) in the agent's `/etc/environment`. +4. Remove `mail` from the manager's `AGENT_SERVICES`, add it on the new host (see [Deploying Agents](deploying-agents.md)). It takes the claim. +5. `rsync /var/vmail` from the old host. +6. Repoint the port forwards (25, 143, 465, 587, 993) and re-run Check DNS on each domain — the mail IP follows the host's site automatically. + +**C. Self-managed MTA (no agent runs `mail`).** + +1. Set **Self-managed site id** to the site whose `externalIp` sends mail. +2. Point Dovecot at `mail_accounts_v` and Postfix at `mail_senders_v`, with Dovecot's `quota-status` guarding RCPT. The views are a documented, stable contract — see [Database Schema](../developers/database-schema.md#mail-views). +3. Export DKIM keys per domain (`GET /api/v1/external-domains/{id}/mail/dkim` is logged) and use the agent templates under `agent/templates/mail/` as a reference. Run `opensource-mail-helper` yourself for unsubscribe headers, or skip them. + +## DB Roles + +`create-a-container/bin/setup-mail-db-roles.sh` creates the read-only `mail_dovecot` and `mail_postfix` users and grants SELECT on the views. It runs automatically on the manager (first boot + package upgrades); run it with `--print-sql` for a DBA, `--rotate` to regenerate the passwords in `/etc/default/container-creator`. + +## Backups + +Everything except mailbox content lives in the manager database (accounts, DKIM keys, suppressions, unsubscribe keys) — covered by your normal DB backup. Mailboxes are Maildirs under `/var/vmail/` on the mail host; back that directory up separately. Deleted accounts' Maildirs are garbage-collected 30 days after deletion. diff --git a/mie-opensource-landing/docs/admins/installation.md b/mie-opensource-landing/docs/admins/installation.md index 28287920..2f960c87 100644 --- a/mie-opensource-landing/docs/admins/installation.md +++ b/mie-opensource-landing/docs/admins/installation.md @@ -16,6 +16,10 @@ The following instructions make several assumptions for clarity. You may have to 4. The domains `example.org` and `*.example.org` have public DNS entries pointing to the firewall in front of this Proxmox cluster. 5. At least one Proxmox node is accessible at `https://example.org:8006` with a valid HTTPS certificate. +## Mail Ports and PTR + +To use the built-in [email service](email.md), additionally forward these TCP ports to the Manager container: `25` (SMTP), `587`/`465` (submission), `993`/`143` (IMAP). Deliverability requires a **PTR record** on the public mail IP that forward-confirms to the `Mail hostname` setting — arrange this with the IP's owner (ISP/hosting provider) before enabling mail on a domain. + ## Air-Gapped Installation The standard installation flow assumes that the Proxmox host can access GitHub Container Registry (GHCR). For an air-gapped deployment, container images must instead be downloaded on an Internet-connected machine and transferred into the isolated Proxmox network. diff --git a/mie-opensource-landing/docs/developers/agent.md b/mie-opensource-landing/docs/developers/agent.md index 6b350a03..116a4b24 100644 --- a/mie-opensource-landing/docs/developers/agent.md +++ b/mie-opensource-landing/docs/developers/agent.md @@ -89,20 +89,31 @@ Read from the process environment (systemd loads `/etc/environment`). Set via co | `SITE_ID` | Yes | Numeric site ID from the manager | | `MANAGER_URL` | Yes | Base URL of the manager (e.g., `http://192.168.1.10:3000`) | | `API_KEY` | No | Admin API key for remote agents. Not needed on the manager (localhost is trusted). | +| `AGENT_SERVICES` | No | Comma-separated service groups: `nginx`, `dnsmasq`, `mail`. Default `nginx,dnsmasq`. | +| `MAIL_POSTFIX_DB_PASSWORD` | No | DB password for the `mail_postfix` role (mail hosts without local socket auth) | +| `MAIL_DOVECOT_DB_PASSWORD` | No | DB password for the `mail_dovecot` role (mail hosts without local socket auth) | | `STATE_DIRECTORY` | No | State directory, set by systemd via `StateDirectory=` (default `/var/lib/opensource-agent`) | The agent Dockerfile defaults to `SITE_ID=1` and `MANAGER_URL=http://localhost:3000` so the manager container works without configuration. ## Managed Services -| Service | Files | Test | Reload | -|---------|-------|------|--------| -| nginx | `/etc/nginx/nginx.conf` | `nginx -t` | `systemctl reload-or-restart nginx` | -| dnsmasq | `/etc/dnsmasq.conf`, `/var/lib/dnsmasq/{dhcp-hosts,hosts,dhcp-opts,servers}` | `dnsmasq --test` | restart when `/etc/dnsmasq.conf` changed, otherwise SIGHUP | +Services are organized in groups enabled via `AGENT_SERVICES`. The check-in reports the enabled groups plus any missing binaries; the manager refuses the mail-host claim while anything is missing. -Apply flow per service: render templates, skip if nothing changed, stage new files, run the test command, roll back on failure, then reload. Failures are reported as `lastApply: "failure"` at the next check-in. +| Group | Service | Files | Test | Reload | +|-------|---------|-------|------|--------| +| nginx | nginx | `/etc/nginx/nginx.conf` | `nginx -t` | `systemctl reload-or-restart nginx` | +| dnsmasq | dnsmasq | `/etc/dnsmasq.conf`, `/var/lib/dnsmasq/{dhcp-hosts,hosts,dhcp-opts,servers}` | `dnsmasq --test` | restart when `/etc/dnsmasq.conf` changed, otherwise SIGHUP | +| mail | opendkim | `/etc/opendkim.conf`, `/etc/opendkim/{keytable,signingtable}`, `/etc/dkimkeys/*.private` | `opendkim -n` | restart | +| mail | dovecot | `/etc/dovecot/dovecot.conf` | `doveconf -n` | reload + `doveadm auth cache flush` | +| mail | postfix (status: `postfix@-`) | `/etc/postfix/{main.cf,master.cf,vdomains,sni_map,sql/senders.cf}` | `postfix check` (after `postmap -F` postWrite) | reload-or-restart | +| mail | opensource-mail-helper | `/etc/opensource-server/mail-helper.json` | — | restart | -To add a managed service, add a `ManagedService` entry in [`agent/src/apply.ts`](https://github.com/mieweb/opensource-server/blob/main/agent/src/apply.ts) plus its template(s) under `agent/templates/`, and extend the config snapshot in `create-a-container/utils/agent-config.js`. +The mail group renders only on the agent holding the mail-host claim (the `mail` section of the snapshot). Taking the claim creates the `vmail`/`opensource-mail` users and enables + starts the units; losing it (or removing `mail` from `AGENT_SERVICES`) stops and disables them, leaving configs and `/var/vmail` in place. Maildirs of deleted accounts are garbage-collected after 30 days. + +Apply flow per service: render templates, skip if nothing changed, stage new files, run post-write commands, run the test command, roll back on failure, then reload. Failures are reported as `lastApply: "failure"` at the next check-in. + +To add a managed service, add a `ManagedService` entry in [`agent/src/apply.ts`](https://github.com/mieweb/opensource-server/blob/main/agent/src/apply.ts) (or `agent/src/mail.ts` for the mail group) plus its template(s) under `agent/templates/`, and extend the config snapshot in `create-a-container/utils/agent-config.js`. ## Running Manually diff --git a/mie-opensource-landing/docs/developers/database-schema.md b/mie-opensource-landing/docs/developers/database-schema.md index a498ae50..a00d5691 100644 --- a/mie-opensource-landing/docs/developers/database-schema.md +++ b/mie-opensource-landing/docs/developers/database-schema.md @@ -20,6 +20,10 @@ erDiagram Services ||--|| TransportServices : "type: transport" Services ||--|| DnsServices : "type: dns" ExternalDomains ||--o{ HTTPServices : "used by" + ExternalDomains ||--o{ DkimKeys : signs + ExternalDomains ||--o{ MailAccounts : hosts + MailAccounts ||--o{ MailSuppressions : "unsubscribed by" + Users ||--o{ MailAccounts : owns Jobs ||--o{ JobStatuses : tracks Users }o--o{ Groups : "member of" UserGroups }|--|| Users : joins @@ -60,6 +64,11 @@ erDiagram string hostname "unique per site" string ipv4Address json services "per-service state + lastApply" + json enabledServices "AGENT_SERVICES groups" + json missingBinaries "blocks the mail-host claim" + date mailHostSince "single holder (partial unique index)" + boolean isLocal + uuid apiKeyId FK "pinned on first remote check-in" date lastCheckinAt } @@ -132,6 +141,46 @@ erDiagram string cloudflareApiKey int siteId FK "nullable, default site" string authServer "nullable, oauth2-proxy process address" + boolean mailEnabled "admin switch, default false" + boolean mailDnsVerified "SPF+DKIM+DMARC; with mailEnabled => can send" + boolean mailMxVerified "MX -> mail IP; with mailEnabled => can receive" + date mailDnsCheckedAt + json mailDnsCheckResult "last Check DNS outcome" + } + + DkimKeys { + int id PK + int externalDomainId FK + string selector "unique per domain" + text privateKey "never serialized in API responses" + text publicKey "DNS p= value" + enum status "active | retired" + } + + MailAccounts { + uuid id PK + int uidNumber FK "owner; admin-transferable" + int externalDomainId FK + string localPart "lowercase; unique per domain" + string description + string passwordHash "Argon2id p=1; never serialized" + boolean enabled "default true; gates the SQL views" + bigint quotaBytes + boolean unsubscribeHeaders "default true (RFC 8058)" + date lastRotatedAt + } + + MailSuppressions { + int id PK + uuid mailAccountId FK + string recipient "lowercase; unique per account" + enum source "one-click | admin" + } + + MailUnsubscribeKeys { + string kid PK + string secret "AES-256-GCM key; never serialized" + enum status "active | retired (verifies 1 more year)" } Jobs { @@ -222,7 +271,32 @@ Base model with `type` discriminator (`http`, `transport`, `dns`). Belongs to Co Persistent bind-mount attached to a container (issue #421). Unique composite indexes on `(containerId, name)` and `(containerId, mountPath)`. `hostPath` is derived server-side from the node volume storage's actual configured `path` (`/volumes/site-///`; scoped by site so containers with the same hostname across sites don't collide on shared storage, and by owner so retained data is only reattached for the same owner) and is null until the create job derives it. `mode` is `ro`/`rw`; each row renders to a Proxmox `mpN` bind mount (`Volume.buildMountConfig`). `status` (`pending` → `ready` | `failed`) is the readiness barrier the create/reconfigure jobs block on — the site agent creates the host directory and reports the result at check-in, which the manager writes to `status`/`statusMessage`/`appliedAt`. `builtin` marks the retired `quick_and_dirty` mount backfilled onto pre-#421 containers for reference (never re-applied; its live `mp0` slot is reserved by `buildMountConfig`). Belongs to Container. See [Volumes](../admins/core-concepts/volumes.md). ### ExternalDomain -Manages public domains for HTTP service exposure. `siteId` is nullable — when set, indicates the "default site" whose DNS is assumed pre-configured (e.g., wildcard A record). Global resource available to all sites. Has many HTTPServices. Cloudflare credentials used for both ACME DNS-01 challenges and cross-site A record management. `authServer` is an optional address of an [oauth2-proxy](https://oauth2-proxy.github.io/oauth2-proxy/) process (e.g. `http://127.0.0.1:4180`) that NGINX proxies `/oauth2/*` to for `auth_request` (see [External Domains](../admins/core-concepts/external-domains.md#authentication)). +Manages public domains for HTTP service exposure. `siteId` is nullable — when set, indicates the "default site" whose DNS is assumed pre-configured (e.g., wildcard A record). Global resource available to all sites. Has many HTTPServices. Cloudflare credentials used for both ACME DNS-01 challenges and cross-site A record management. `authServer` is an optional address of an [oauth2-proxy](https://oauth2-proxy.github.io/oauth2-proxy/) process (e.g. `http://127.0.0.1:4180`) that NGINX proxies `/oauth2/*` to for `auth_request` (see [External Domains](../admins/core-concepts/external-domains.md#authentication)). Mail gates (`mailEnabled`, `mailDnsVerified`, `mailMxVerified`) are changed only via the `…/mail/*` endpoints; has many DkimKeys and MailAccounts. + +## Mail Models (issue #67) + +### DkimKey +RSA-2048 signing key per external domain, generated at domain creation (selector `osYYYYMMDD`). `status` supports future rotation. The private key is never serialized in API responses — it reaches only the mail-host agent snapshot and the admin DKIM export. + +### MailAccount +A user-owned SMTP/IMAP account on a can-send domain. Password stored as an Argon2id PHC string with `p=1` (Dovecot verifies via libsodium, which only supports parallelism 1); the plaintext is returned exactly once at create/rotate. `enabled` gates the SQL views, so disabling locks the account out instantly. + +### MailSuppression +A recipient who one-click-unsubscribed (RFC 8058) from a MailAccount, or was suppressed by an admin. Enforced synchronously at RCPT on the submission ports. + +### MailUnsubscribeKey +AES-256-GCM key for unsubscribe tokens. One `active` key; `retired` keys keep verifying previously issued links for one year. + +### Mail Views +The dialect-aware views created by migration `20261008000006` (SQL built in `create-a-container/utils/mail-views.js`) are the **stable contract** Dovecot/Postfix — or a self-managed MTA — read live: + +| View | Columns | Consumer | +|------|---------|----------| +| `mail_accounts_v` | `address`, `password` (`{ARGON2ID}`-prefixed), `home`, `quota_bytes`, `can_send`, `can_receive` — enabled accounts only | Dovecot passdb/userdb (`mail_dovecot` role) | +| `mail_senders_v` | `sender`, `login`, `account_id`, `unsubscribe_headers` — can-send accounts | Postfix `smtpd_sender_login_maps` + opensource-mail-helper (`mail_postfix` role) | +| `mail_suppressions_v` | `sender`, `recipient` | Submission policy service (`mail_postfix` role) | + +`bin/setup-mail-db-roles.sh` creates the two read-only roles and grants; the views migration also grants best-effort so either ordering converges. ## User Management Models diff --git a/mie-opensource-landing/docs/users/email-accounts.md b/mie-opensource-landing/docs/users/email-accounts.md new file mode 100644 index 00000000..a0aa3fd7 --- /dev/null +++ b/mie-opensource-landing/docs/users/email-accounts.md @@ -0,0 +1,41 @@ +# Email Accounts + +Send mail over authenticated SMTP and read replies over IMAP from any app in the environment — no external mail provider needed. Manage accounts under **Email** in the sidebar. + +## Creating an Account + +1. **Email → New email account**, pick a domain (only domains an admin has [enabled for mail](../admins/core-concepts/external-domains.md#mail) appear) and a local part, e.g. `my-app@example.com`. +2. Copy the generated password — it is shown **once**. Use **Rotate password** to get a new one at any time (the old one stops working immediately). + +Role addresses (`postmaster`, `abuse`, `noreply`, `admin`, …) are reserved for administrators. + +## Client Settings + +| | | +|---|---| +| Username | the full address (e.g. `my-app@example.com`) | +| SMTP | port `587` (STARTTLS) or `465` (implicit TLS) — authentication required, TLS always | +| IMAP | port `993` (TLS) or `143` | + +```bash +# Quick smoke test +curl --ssl-reqd --url smtp://example.com:587 \ + --user 'my-app@example.com:' \ + --mail-from my-app@example.com --mail-rcpt you@example.net \ + --upload-file message.eml +``` + +Mail is DKIM-signed automatically. You can only send from your own address — the envelope sender must match your login. + +## Unsubscribe Headers and Suppressions + +Outbound mail gets RFC 8058 `List-Unsubscribe` headers by default, so recipients can one-click unsubscribe (Gmail and others surface this as an "Unsubscribe" button). Per account: + +- **Unsubscribe headers** toggle — turn the headers off for purely transactional accounts. +- **Suppressions** — recipients who unsubscribed. Further mail from this account to them is rejected synchronously at submission (`5.7.1`), so your app sees the error instead of silently dropped mail. Only admins can remove a suppression. + +## Limits + +- Sending works only while the domain can send (admin-controlled, DNS-verified). If the domain is disabled, SMTP login is refused but IMAP stays readable. +- Mailboxes have a quota (shown in the list); over-quota delivery is rejected at RCPT. +- Disabled or deleted accounts stop authenticating immediately. A deleted account's mailbox is kept for 30 days. diff --git a/mie-opensource-landing/zensical.toml b/mie-opensource-landing/zensical.toml index 6461d144..e7f034f3 100644 --- a/mie-opensource-landing/zensical.toml +++ b/mie-opensource-landing/zensical.toml @@ -21,6 +21,7 @@ nav = [ { "API Keys" = "users/creating-containers/api-keys.md" }, ] }, { "Monitoring Containers" = "users/monitoring-container.md" }, + { "Email Accounts" = "users/email-accounts.md" }, { "Adding Authentication" = "users/consuming-auth.md" }, { "Launchpad" = [ { "What is Launchpad" = "users/proxmox-launchpad/what-is-proxmox-launchpad.md" }, @@ -42,6 +43,7 @@ nav = [ { "Volumes" = "admins/core-concepts/volumes.md" }, ] }, { "Settings" = "admins/settings.md" }, + { "Email Service" = "admins/email.md" }, { "OIDC Single Sign-On" = "admins/oidc.md" }, { "LDAP Authentication" = "admins/ldap-servers.md" }, { "Deploying Agents" = "admins/deploying-agents.md" }, From ea39acea7f827f2125c53c0a85feafd5897b2352 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Mon, 12 Oct 2026 14:53:28 -0700 Subject: [PATCH 14/19] mail: manager-side review fixes (PR #495) - settings: strip trailing slashes without regex (CodeQL polynomial ReDoS) - Check DNS: DMARC requires exactly one record with a valid p= tag; multiple records warn (permerror) - mail IP: self-managed fallback applies only when NO agent holds the claim, so checks can't verify against an unrelated IP - claim: a holder reporting missing binaries releases the claim instead of wedging every other agent in conflict - local part: model + API validators reject consecutive dots (a..b) - domain create: ExternalDomain + DkimKey in one transaction - new admin GET /external-domains/{id}/mail/dkim export (audited) so the documented self-managed workflow exists - agents: the secret-bearing mail snapshot is only sent to local or TLS check-ins; plaintext remote holders get mailStatus=insecure_transport - nginx: the snapshot carries the unsubscribe host and the template adds an auth_request-exempt /u/ location on that server, so oauth2-proxy can't block RFC 8058 one-click POSTs - in-app rate limiting removed per review (proxy concern, out of scope) - setup-mail-db-roles.sh replaced by migration 20261012000000 using Sequelize's dialect handling; init service grants CREATEROLE; remote hosts get passwords via ALTER ROLE (docs updated) - new mail_recipients_v view backs a live Postfix recipient map (no more static:x backscatter; granted to mail_postfix) --- agent/src/types.ts | 3 + agent/templates/nginx.conf.ejs | 16 +++ create-a-container/app.js | 6 +- create-a-container/bin/setup-mail-db-roles.sh | 132 ------------------ create-a-container/contrib/postinstall.sh | 7 - create-a-container/middlewares/rate-limit.js | 37 ----- .../20261008000006-create-mail-views.js | 11 +- .../20261012000000-create-mail-db-roles.js | 79 +++++++++++ create-a-container/models/mail-account.js | 5 +- create-a-container/openapi.v1.yaml | 19 ++- .../resources/mailaccounts/validator.js | 5 +- create-a-container/routers/api/v1/agents.js | 11 +- .../routers/api/v1/external-domains.js | 49 +++++-- create-a-container/routers/api/v1/settings.js | 5 +- create-a-container/routers/unsubscribe.js | 5 +- create-a-container/utils/agent-config.js | 12 ++ create-a-container/utils/mail-dns.js | 16 ++- create-a-container/utils/mail-host.js | 11 +- create-a-container/utils/mail-views.js | 15 +- images/manager/container-creator-init.service | 4 +- 20 files changed, 227 insertions(+), 221 deletions(-) delete mode 100644 create-a-container/bin/setup-mail-db-roles.sh delete mode 100644 create-a-container/middlewares/rate-limit.js create mode 100644 create-a-container/migrations/20261012000000-create-mail-db-roles.js diff --git a/agent/src/types.ts b/agent/src/types.ts index e64be280..8c9ef1f9 100644 --- a/agent/src/types.ts +++ b/agent/src/types.ts @@ -103,6 +103,9 @@ export interface NginxConfig { httpServices: HttpService[]; streamServices: StreamService[]; externalDomains: { name: string }[]; + /** Host serving the public /u unsubscribe endpoint — its server block gets + * an auth-exempt /u/ location. Absent on older managers. */ + unsubscribeHost?: string | null; } /** Config snapshot for the whole site. `site` is null before the first site diff --git a/agent/templates/nginx.conf.ejs b/agent/templates/nginx.conf.ejs index 8b5c2392..5d7f5991 100644 --- a/agent/templates/nginx.conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -223,6 +223,8 @@ http { <%_ httpServices.forEach((service, index) => { _%> <%_ const authRequired = service.authRequired; _%> <%_ const authServer = service.externalDomain.authServer; _%> + <%_ const serverName = `${service.externalHostname}.${service.externalDomain.name}`; _%> + <%_ const servesUnsubscribe = (typeof unsubscribeHost !== 'undefined' && unsubscribeHost) === serverName; _%> server { listen 443 ssl; listen [::]:443 ssl; @@ -234,6 +236,20 @@ http { ssl_certificate /etc/ssl/certs/<%= service.externalDomain.name %>.crt; ssl_certificate_key /etc/ssl/private/<%= service.externalDomain.name %>.key; + <%_ if (servesUnsubscribe && authRequired && authServer) { _%> + <%_ /* RFC 8058 one-click unsubscribe: POSTs come from recipients' mail + clients with no session, so this location must stay outside + auth_request. The endpoint authenticates via its encrypted token. */ -%> + location ^~ /u/ { + proxy_pass <%= service.backendProtocol %>://<%= service.container.ipv4Address %>:<%= service.internalPort %>; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + <%_ } _%> + error_page 403 @403; location @403 { rewrite ^ /403.html break; diff --git a/create-a-container/app.js b/create-a-container/app.js index 5e11c306..f21a4442 100644 --- a/create-a-container/app.js +++ b/create-a-container/app.js @@ -96,9 +96,9 @@ function buildApp({ // Public one-click unsubscribe (RFC 8058) — mounted before the CSRF guard // because requests come from recipients' mail clients with no session or - // token (the router rate-limits per IP and authenticates via the encrypted - // unsubscribe token itself). Must also bypass oauth2-proxy when the manager - // sits behind one — handled in the nginx template, not here. + // token (authentication is the encrypted unsubscribe token itself). Served + // without oauth2-proxy interference via the dedicated /u/ location in the + // agent's nginx template. app.use('/u', require('./routers/unsubscribe')); // CSRF guard for every handler that can see the session cookie (CodeQL diff --git a/create-a-container/bin/setup-mail-db-roles.sh b/create-a-container/bin/setup-mail-db-roles.sh deleted file mode 100644 index 94e15d7d..00000000 --- a/create-a-container/bin/setup-mail-db-roles.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -# setup-mail-db-roles.sh — create the read-only DB users the mail stack uses. -# -# mail_dovecot SELECT on mail_accounts_v (passdb/userdb) -# mail_postfix SELECT on mail_senders_v, mail_suppressions_v (sender/suppression maps) -# -# The views are created by migration 20261008000006-create-mail-views.js; this -# script only creates the users and grants. Passwords are generated and -# appended to the env file as MAIL_DOVECOT_DB_PASSWORD / MAIL_POSTFIX_DB_PASSWORD -# so the agent can template them into the Dovecot/Postfix SQL config. When the -# DB is local to the mail host, socket auth (Postgres peer / MariaDB -# unix_socket) also works and the passwords are unused. -# -# Runs idempotently from container-creator-init.service and the package -# postinst. Supports: -# --print-sql print the SQL for a DBA instead of executing it -# --rotate regenerate passwords even if already present -# -# Reads DATABASE_DIALECT and connection settings from ENV_FILE -# (default /etc/default/container-creator). -set -euo pipefail - -ENV_FILE="${ENV_FILE:-/etc/default/container-creator}" -PRINT_SQL=0 -ROTATE=0 -for arg in "$@"; do - case "$arg" in - --print-sql) PRINT_SQL=1 ;; - --rotate) ROTATE=1 ;; - *) echo "usage: $0 [--print-sql] [--rotate]" >&2; exit 2 ;; - esac -done - -if [ -f "$ENV_FILE" ]; then - # shellcheck disable=SC1090 - . "$ENV_FILE" -fi - -DIALECT="${DATABASE_DIALECT:-sqlite}" -case "$DIALECT" in - postgres|mysql|mariadb) ;; - *) - echo "setup-mail-db-roles: DATABASE_DIALECT=$DIALECT cannot host mail (needs postgres or mysql/mariadb); nothing to do." - exit 0 - ;; -esac - -# Read an existing password out of the env file, if any. -get_env_var() { grep -E "^$1=" "$ENV_FILE" 2>/dev/null | tail -n1 | cut -d= -f2-; } - -# Insert-or-replace VAR=value in the env file. -set_env_var() { - local var="$1" value="$2" - touch "$ENV_FILE" - chmod 600 "$ENV_FILE" - if grep -qE "^${var}=" "$ENV_FILE"; then - sed -i "s|^${var}=.*|${var}=${value}|" "$ENV_FILE" - else - echo "${var}=${value}" >> "$ENV_FILE" - fi -} - -gen_password() { openssl rand -base64 32 | tr -d '/+=' | head -c 32; } - -DOVECOT_PW="$(get_env_var MAIL_DOVECOT_DB_PASSWORD || true)" -POSTFIX_PW="$(get_env_var MAIL_POSTFIX_DB_PASSWORD || true)" -if [ "$ROTATE" = 1 ] || [ -z "$DOVECOT_PW" ]; then DOVECOT_PW="$(gen_password)"; fi -if [ "$ROTATE" = 1 ] || [ -z "$POSTFIX_PW" ]; then POSTFIX_PW="$(gen_password)"; fi - -sql_postgres() { - cat </dev/null 2>&1; then - echo "$SQL" | su -s /bin/sh postgres -c "psql -v ON_ERROR_STOP=1 -d '${POSTGRES_DATABASE:-container_creator}'" - else - echo "setup-mail-db-roles: psql not found — run with --print-sql and apply as a DBA." >&2 - exit 1 - fi -else - if command -v mysql >/dev/null 2>&1; then - echo "$SQL" | mysql - else - echo "setup-mail-db-roles: mysql client not found — run with --print-sql and apply as a DBA." >&2 - exit 1 - fi -fi - -set_env_var MAIL_DOVECOT_DB_PASSWORD "$DOVECOT_PW" -set_env_var MAIL_POSTFIX_DB_PASSWORD "$POSTFIX_PW" -echo "setup-mail-db-roles: mail_dovecot and mail_postfix are ready; passwords stored in $ENV_FILE." diff --git a/create-a-container/contrib/postinstall.sh b/create-a-container/contrib/postinstall.sh index 1974e220..e92c7acf 100755 --- a/create-a-container/contrib/postinstall.sh +++ b/create-a-container/contrib/postinstall.sh @@ -16,11 +16,4 @@ if [ -d /run/systemd/system ]; then systemctl restart $UNITS fi -# Mail DB roles (issue #67): (re)create the read-only mail_dovecot / -# mail_postfix users + view grants. Skipped before first-boot DB init -# (container-creator-init.service runs it then) and on SQLite. -if [ -f /etc/default/container-creator ]; then - bash /opt/opensource-server/create-a-container/bin/setup-mail-db-roles.sh || true -fi - exit 0 diff --git a/create-a-container/middlewares/rate-limit.js b/create-a-container/middlewares/rate-limit.js deleted file mode 100644 index af509a3b..00000000 --- a/create-a-container/middlewares/rate-limit.js +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Minimal in-memory fixed-window per-IP rate limiter. Good enough for the - * public unsubscribe endpoint (single-process deployment); swap for a - * store-backed limiter if the manager ever runs multi-process. - */ - -/** - * @param {object} options - * @param {number} options.windowMs - window length - * @param {number} options.max - allowed requests per window per IP - */ -function rateLimit({ windowMs, max }) { - const hits = new Map(); - return function rateLimiter(req, res, next) { - const now = Date.now(); - // Opportunistic pruning keeps the map bounded without a timer. - if (hits.size > 10000) { - for (const [k, v] of hits) { - if (v.resetAt <= now) hits.delete(k); - } - } - const key = req.ip || 'unknown'; - let entry = hits.get(key); - if (!entry || entry.resetAt <= now) { - entry = { count: 0, resetAt: now + windowMs }; - hits.set(key, entry); - } - entry.count += 1; - if (entry.count > max) { - res.set('Retry-After', String(Math.ceil((entry.resetAt - now) / 1000))); - return res.status(429).json({ error: { code: 'rate_limited', message: 'Too many requests' } }); - } - return next(); - }; -} - -module.exports = { rateLimit }; diff --git a/create-a-container/migrations/20261008000006-create-mail-views.js b/create-a-container/migrations/20261008000006-create-mail-views.js index e6e1bc9c..a583ecdb 100644 --- a/create-a-container/migrations/20261008000006-create-mail-views.js +++ b/create-a-container/migrations/20261008000006-create-mail-views.js @@ -14,20 +14,21 @@ module.exports = { for (const sql of createViewsSql(dialect)) { await queryInterface.sequelize.query(sql); } - // Grant SELECT when the roles already exist (setup-mail-db-roles.sh may - // run before or after this migration; each side grants best-effort so - // either ordering converges). + // Grant SELECT when the roles already exist (they are created by + // migration 20261012000000; each side grants best-effort so either + // ordering converges). const grants = []; if (dialect === 'postgres') { grants.push( 'GRANT SELECT ON mail_accounts_v TO mail_dovecot', - 'GRANT SELECT ON mail_senders_v, mail_suppressions_v TO mail_postfix', + 'GRANT SELECT ON mail_senders_v, mail_recipients_v, mail_suppressions_v TO mail_postfix', ); } else if (dialect === 'mysql' || dialect === 'mariadb') { const db = queryInterface.sequelize.config.database; grants.push( `GRANT SELECT ON \`${db}\`.mail_accounts_v TO 'mail_dovecot'@'%'`, `GRANT SELECT ON \`${db}\`.mail_senders_v TO 'mail_postfix'@'%'`, + `GRANT SELECT ON \`${db}\`.mail_recipients_v TO 'mail_postfix'@'%'`, `GRANT SELECT ON \`${db}\`.mail_suppressions_v TO 'mail_postfix'@'%'`, ); } @@ -35,7 +36,7 @@ module.exports = { try { await queryInterface.sequelize.query(sql); } catch { - /* roles not created yet — the script grants when it creates them */ + /* roles not created yet — migration 20261012000000 grants then */ } } }, diff --git a/create-a-container/migrations/20261012000000-create-mail-db-roles.js b/create-a-container/migrations/20261012000000-create-mail-db-roles.js new file mode 100644 index 00000000..2b2dec8b --- /dev/null +++ b/create-a-container/migrations/20261012000000-create-mail-db-roles.js @@ -0,0 +1,79 @@ +'use strict'; + +const crypto = require('crypto'); + +/** + * Read-only DB roles for the mail stack (issue #67, replaces + * bin/setup-mail-db-roles.sh per PR #495 review): mail_dovecot (SELECT on + * mail_accounts_v) and mail_postfix (SELECT on mail_senders_v + + * mail_suppressions_v). + * + * Local mail hosts use socket auth (Postgres peer via the manager image's + * pg_ident map), so the roles are created without a usable password — on + * MySQL a random throwaway password blocks TCP logins. Remote mail hosts + * need the operator to set one (ALTER ROLE ... PASSWORD / ALTER USER). + * + * Role creation needs CREATEROLE (granted to the app user by + * container-creator-init.service). Where the app user lacks the privilege, + * the statements are logged for a DBA and the migration continues — mail + * simply stays unavailable until the roles exist. + */ +module.exports = { + async up(queryInterface) { + const dialect = queryInterface.sequelize.getDialect(); + let statements = []; + if (dialect === 'postgres') { + statements = [ + `DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mail_dovecot') THEN + CREATE ROLE mail_dovecot LOGIN; + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mail_postfix') THEN + CREATE ROLE mail_postfix LOGIN; + END IF; + END $$;`, + 'GRANT SELECT ON mail_accounts_v TO mail_dovecot', + 'GRANT SELECT ON mail_senders_v, mail_recipients_v, mail_suppressions_v TO mail_postfix', + ]; + } else if (dialect === 'mysql' || dialect === 'mariadb') { + const db = queryInterface.sequelize.config.database; + const throwaway = () => crypto.randomBytes(24).toString('base64url'); + statements = [ + `CREATE USER IF NOT EXISTS 'mail_dovecot'@'%' IDENTIFIED BY '${throwaway()}'`, + `CREATE USER IF NOT EXISTS 'mail_postfix'@'%' IDENTIFIED BY '${throwaway()}'`, + `GRANT SELECT ON \`${db}\`.mail_accounts_v TO 'mail_dovecot'@'%'`, + `GRANT SELECT ON \`${db}\`.mail_senders_v TO 'mail_postfix'@'%'`, + `GRANT SELECT ON \`${db}\`.mail_recipients_v TO 'mail_postfix'@'%'`, + `GRANT SELECT ON \`${db}\`.mail_suppressions_v TO 'mail_postfix'@'%'`, + ]; + } else { + return; // SQLite cannot host mail; the views exist for tests only. + } + try { + for (const sql of statements) { + await queryInterface.sequelize.query(sql); + } + } catch (err) { + console.warn( + 'create-mail-db-roles: could not create the mail DB roles ' + + `(${err.message}). Run the following as a privileged DB user:\n` + + statements.join(';\n'), + ); + } + }, + + async down(queryInterface) { + const dialect = queryInterface.sequelize.getDialect(); + try { + if (dialect === 'postgres') { + await queryInterface.sequelize.query('DROP ROLE IF EXISTS mail_dovecot'); + await queryInterface.sequelize.query('DROP ROLE IF EXISTS mail_postfix'); + } else if (dialect === 'mysql' || dialect === 'mariadb') { + await queryInterface.sequelize.query("DROP USER IF EXISTS 'mail_dovecot'@'%'"); + await queryInterface.sequelize.query("DROP USER IF EXISTS 'mail_postfix'@'%'"); + } + } catch { + /* lacking privilege — leave the roles for a DBA */ + } + }, +}; diff --git a/create-a-container/models/mail-account.js b/create-a-container/models/mail-account.js index 6c2824bc..882746ab 100644 --- a/create-a-container/models/mail-account.js +++ b/create-a-container/models/mail-account.js @@ -44,8 +44,9 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.STRING(64), allowNull: false, validate: { - // Conservative RFC 5321 dot-atom subset, stored lowercase. - is: /^[a-z0-9](?:[a-z0-9._+-]*[a-z0-9])?$/, + // Conservative RFC 5321 dot-atom subset, stored lowercase — single + // separators only (no consecutive dots), matching the API validator. + is: /^[a-z0-9]+(?:[._+-][a-z0-9]+)*$/, }, comment: 'Lowercase local part — address is @', }, diff --git a/create-a-container/openapi.v1.yaml b/create-a-container/openapi.v1.yaml index fb1d4535..f51f6ed8 100644 --- a/create-a-container/openapi.v1.yaml +++ b/create-a-container/openapi.v1.yaml @@ -1387,9 +1387,11 @@ paths: description: >- Config snapshot (`{ data: { site, nginx } }`) with `ETag` header. When the agent reports `mail`, `data.mailStatus` is one of - holder|conflict|unsupported|missing_packages, and only the claim - holder additionally receives `data.mail` (DKIM keys, receiving - domains, relayhost, limits, DB connection, unsubscribe keys). + holder|conflict|unsupported|missing_packages|insecure_transport + (remote holders only receive the secret-bearing `data.mail` + section over TLS), and only the claim holder additionally + receives `data.mail` (DKIM keys, receiving domains, relayhost, + limits, DB connection, unsubscribe keys). '304': { description: Config unchanged since If-None-Match } '403': { description: 'Agent pinned to a different API key (code: agent_key_mismatch)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } '422': { description: 'Missing siteId/hostname (code: validation_failed)', content: { application/json: { schema: { $ref: '#/components/schemas/Error' } } } } @@ -1679,6 +1681,17 @@ paths: '200': { description: Disabled } '403': { $ref: '#/components/responses/Forbidden' } '404': { $ref: '#/components/responses/NotFound' } + /external-domains/{id}/mail/dkim: + parameters: [{ in: path, name: id, required: true, schema: { type: integer } }] + get: + operationId: export_domain_dkim_key + tags: [Mail] + summary: Export the active DKIM key pair for a self-managed MTA (admin; audited in the logs) + responses: + '200': + description: '`{ domain, selector, privateKey, publicKey }`' + '403': { $ref: '#/components/responses/Forbidden' } + '404': { $ref: '#/components/responses/NotFound' } /mail-accounts: get: diff --git a/create-a-container/resources/mailaccounts/validator.js b/create-a-container/resources/mailaccounts/validator.js index 2afd2809..94919496 100644 --- a/create-a-container/resources/mailaccounts/validator.js +++ b/create-a-container/resources/mailaccounts/validator.js @@ -1,13 +1,14 @@ const { z } = require('zod'); -// Conservative RFC 5321 dot-atom subset; stored lowercase. +// Conservative RFC 5321 dot-atom subset; stored lowercase. Alphanumeric +// runs joined by single separators — rejects consecutive dots like "a..b". const localPart = z .string() .trim() .toLowerCase() .min(1) .max(64) - .regex(/^[a-z0-9](?:[a-z0-9._+-]*[a-z0-9])?$/, 'invalid local part'); + .regex(/^[a-z0-9]+(?:[._+-][a-z0-9]+)*$/, 'invalid local part'); const createMailAccount = z.object({ externalDomainId: z.coerce.number().int().positive(), diff --git a/create-a-container/routers/api/v1/agents.js b/create-a-container/routers/api/v1/agents.js index 64f9bf9a..6e8ba276 100644 --- a/create-a-container/routers/api/v1/agents.js +++ b/create-a-container/routers/api/v1/agents.js @@ -101,9 +101,16 @@ router.post('/', localhostOrAdmin, asyncHandler(async (req, res) => { const config = await buildAgentConfig(parsedSiteId); if (mailClaim && mailClaim.status !== 'disabled') { config.mailStatus = mailClaim.status; - // Only the claim holder receives the mail section (DKIM private keys etc.). + // Only the claim holder receives the mail section (DKIM private keys, + // unsubscribe AES keys, relay credentials) — and never over plaintext: + // remote check-ins must arrive on TLS (req.secure honors X-Forwarded-Proto + // via trust proxy) or the secrets would cross the network readable. if (mailClaim.status === 'holder') { - config.mail = await buildMailSnapshot(dbConfig); + if (isLocalhostRequest(req) || req.secure) { + config.mail = await buildMailSnapshot(dbConfig); + } else { + config.mailStatus = 'insecure_transport'; + } } } // Manual conditional-request handling: Express's built-in ETag/fresh logic diff --git a/create-a-container/routers/api/v1/external-domains.js b/create-a-container/routers/api/v1/external-domains.js index d8711f29..467fa4f0 100644 --- a/create-a-container/routers/api/v1/external-domains.js +++ b/create-a-container/routers/api/v1/external-domains.js @@ -5,7 +5,7 @@ */ const express = require('express'); -const { ExternalDomain, Site, DkimKey, MailAccount, Setting } = require('../../../models'); +const { ExternalDomain, Site, DkimKey, MailAccount, Setting, sequelize } = require('../../../models'); const { apiAuth, apiAdmin, asyncHandler, ok, created, noContent, ApiError } = require('../../../middlewares/api'); const { generateDkimKeyPair } = require('../../../utils/dkim'); @@ -62,19 +62,23 @@ router.post( asyncHandler(async (req, res) => { const { name, acmeEmail, acmeDirectoryUrl, cloudflareApiEmail, cloudflareApiKey, siteId, authServer } = req.body || {}; - const d = await ExternalDomain.create({ - name, - acmeEmail: acmeEmail || null, - acmeDirectoryUrl: acmeDirectoryUrl || null, - cloudflareApiEmail: cloudflareApiEmail || null, - cloudflareApiKey: cloudflareApiKey || null, - siteId: siteId || null, - authServer: authServer || null, - }); // Every domain gets a DKIM key at create so the DNS records can be // published before mail is ever enabled (existing domains were - // backfilled by migration 20261008000007). - await DkimKey.create({ externalDomainId: d.id, ...generateDkimKeyPair() }); + // backfilled by migration 20261008000007). One transaction — a failed + // key insert must not leave a keyless domain behind. + const d = await sequelize.transaction(async (transaction) => { + const domain = await ExternalDomain.create({ + name, + acmeEmail: acmeEmail || null, + acmeDirectoryUrl: acmeDirectoryUrl || null, + cloudflareApiEmail: cloudflareApiEmail || null, + cloudflareApiKey: cloudflareApiKey || null, + siteId: siteId || null, + authServer: authServer || null, + }, { transaction }); + await DkimKey.create({ externalDomainId: domain.id, ...generateDkimKeyPair() }, { transaction }); + return domain; + }); return created(res, serialize(d)); }), ); @@ -218,4 +222,25 @@ router.post( }), ); +// Self-managed MTA export: the active DKIM private key. Deliberately noisy +// in the logs — private-key reads should be rare, intentional events. +router.get( + '/:id/mail/dkim', + asyncHandler(async (req, res) => { + const d = await loadDomain(req.params.id); + const key = await DkimKey.findOne({ + where: { externalDomainId: d.id, status: 'active' }, + order: [['createdAt', 'DESC']], + }); + if (!key) throw new ApiError(404, 'not_found', 'No active DKIM key for this domain'); + console.warn(`AUDIT: DKIM private key for ${d.name} exported by admin ${req.session.user}`); + return ok(res, { + domain: d.name, + selector: key.selector, + privateKey: key.privateKey, + publicKey: key.publicKey, + }); + }), +); + module.exports = router; diff --git a/create-a-container/routers/api/v1/settings.js b/create-a-container/routers/api/v1/settings.js index e359d36c..a98a4c12 100644 --- a/create-a-container/routers/api/v1/settings.js +++ b/create-a-container/routers/api/v1/settings.js @@ -125,7 +125,10 @@ router.put( return Number.isNaN(n) || n < 1 ? '' : String(n); }; await Setting.set('mail_hostname', (mailHostname || '').trim()); - await Setting.set('mail_unsubscribe_base_url', (mailUnsubscribeBaseUrl || '').trim().replace(/\/+$/, '')); + // Strip trailing slashes without regex (CodeQL js/polynomial-redos). + let unsubscribeBase = (mailUnsubscribeBaseUrl || '').trim(); + while (unsubscribeBase.endsWith('/')) unsubscribeBase = unsubscribeBase.slice(0, -1); + await Setting.set('mail_unsubscribe_base_url', unsubscribeBase); await Setting.set('mail_relayhost', (mailRelayhost || '').trim()); await Setting.set('mail_relayhost_username', (mailRelayhostUsername || '').trim()); await Setting.set('mail_relayhost_password', mailRelayhostPassword || ''); diff --git a/create-a-container/routers/unsubscribe.js b/create-a-container/routers/unsubscribe.js index 2afd8347..beeaea31 100644 --- a/create-a-container/routers/unsubscribe.js +++ b/create-a-container/routers/unsubscribe.js @@ -1,7 +1,8 @@ /** * Public one-click unsubscribe endpoint (RFC 8058, issue #67). Mounted at * /u OUTSIDE /api/v1 and before the CSRF guard: requests come from mail - * clients and recipients with no session, token, or credentials. + * clients and recipients with no session, token, or credentials. Abuse + * control (rate limiting) belongs in the proxy, not in-app. * * GET /u/:token confirmation page — a GET NEVER unsubscribes, so link * scanners that prefetch URLs can't trigger it. @@ -16,12 +17,10 @@ const express = require('express'); const { MailAccount, MailSuppression, MailUnsubscribeKey } = require('../models'); const { asyncHandler } = require('../middlewares/api'); -const { rateLimit } = require('../middlewares/rate-limit'); const { verifyUnsubscribeToken } = require('../utils/unsubscribe-token'); const router = express.Router(); -router.use(rateLimit({ windowMs: 10 * 60 * 1000, max: 60 })); router.use(express.urlencoded({ extended: false })); function escapeHtml(s) { diff --git a/create-a-container/utils/agent-config.js b/create-a-container/utils/agent-config.js index b2449687..aeb9a3e6 100644 --- a/create-a-container/utils/agent-config.js +++ b/create-a-container/utils/agent-config.js @@ -143,6 +143,17 @@ async function buildAgentConfig(siteId) { // the agent only owns user volume directories. const volumes = await buildSiteVolumes(site); + // Host serving the public one-click unsubscribe endpoint (/u/). + // The nginx template keeps /u/ outside auth_request on this host so + // oauth2-proxy can never block RFC 8058 POSTs from mail clients. + let unsubscribeHost = null; + try { + const baseUrl = await Setting.get('mail_unsubscribe_base_url'); + if (baseUrl) unsubscribeHost = new URL(baseUrl).hostname; + } catch { + /* unset or malformed — no bypass location rendered */ + } + return { site: { id: site.id, @@ -168,6 +179,7 @@ async function buildAgentConfig(siteId) { httpServices, streamServices, externalDomains: externalDomains.map((d) => ({ name: d.name })), + unsubscribeHost, }, }; } diff --git a/create-a-container/utils/mail-dns.js b/create-a-container/utils/mail-dns.js index 32f74938..d529376a 100644 --- a/create-a-container/utils/mail-dns.js +++ b/create-a-container/utils/mail-dns.js @@ -193,10 +193,20 @@ async function checkMailDns({ domain, mailIp, mailHostname, spfInclude, dkimKey, result.warnings.push('No active DKIM key for this domain.'); } - // DMARC + // DMARC — receivers treat multiple records as a permerror and ignore + // records without a valid p= tag, so both fail the gate. const dmarcTxt = await resolveTxtSafe(r, `_dmarc.${domain}`); - const dmarcRecord = dmarcTxt.find((t) => /^v=DMARC1(\s*;|$)/i.test(t)) || null; - result.dmarc = { pass: !!dmarcRecord, record: dmarcRecord }; + const dmarcRecords = dmarcTxt.filter((t) => /^v=DMARC1\s*;/i.test(t.trim()) || /^v=DMARC1$/i.test(t.trim())); + const dmarcRecord = dmarcRecords.length === 1 ? dmarcRecords[0] : null; + const dmarcPolicyOk = !!dmarcRecord && /(?:^|;)\s*p\s*=\s*(none|quarantine|reject)\s*(?:;|$)/i.test(dmarcRecord); + result.dmarc = { + pass: dmarcPolicyOk, + record: dmarcRecord ?? dmarcRecords[0] ?? null, + recordCount: dmarcRecords.length, + }; + if (dmarcRecords.length > 1) { + result.warnings.push(`Multiple DMARC records published for _dmarc.${domain} — receivers treat this as an error.`); + } // MX → must resolve to the mail IP (receive gate) let mxPass = false; diff --git a/create-a-container/utils/mail-host.js b/create-a-container/utils/mail-host.js index 8d8ba6be..fdf541bc 100644 --- a/create-a-container/utils/mail-host.js +++ b/create-a-container/utils/mail-host.js @@ -22,12 +22,14 @@ async function getMailHostAgent() { } /** - * The IP mail is sent from: externalIp of the mail host's site, or of the - * mail_self_managed_site_id site when no agent holds the claim. + * The IP mail is sent from: externalIp of the mail host's site. The + * mail_self_managed_site_id fallback applies ONLY when no agent holds the + * claim — otherwise DNS checks could verify against an IP unrelated to + * where mail actually runs. */ async function getMailIp() { const holder = await getMailHostAgent(); - if (holder?.site?.externalIp) return holder.site.externalIp; + if (holder) return holder.site?.externalIp || null; const selfManagedSiteId = parseInt(await Setting.get('mail_self_managed_site_id'), 10); if (Number.isInteger(selfManagedSiteId)) { const site = await Site.findByPk(selfManagedSiteId, { attributes: ['externalIp'] }); @@ -57,6 +59,9 @@ async function processMailClaim(agent, { wantsMail, missingBinaries }) { return { status: 'unsupported' }; } if (Array.isArray(missingBinaries) && missingBinaries.length > 0) { + // A holder that lost its packages must also lose the claim — otherwise + // every other agent stays in `conflict` while nothing serves mail. + if (agent.mailHostSince) await agent.update({ mailHostSince: null }); return { status: 'missing_packages', missing: missingBinaries }; } return sequelize.transaction(async (transaction) => { diff --git a/create-a-container/utils/mail-views.js b/create-a-container/utils/mail-views.js index 1de7080a..ee7df713 100644 --- a/create-a-container/utils/mail-views.js +++ b/create-a-container/utils/mail-views.js @@ -1,7 +1,7 @@ /** * Per-dialect SQL for the mail views — the stable contract Dovecot and - * Postfix read live (see docs: the views are consumed by `mail_dovecot` and - * `mail_postfix` DB users created by bin/setup-mail-db-roles.sh): + * Postfix read live (consumed by the read-only `mail_dovecot` and + * `mail_postfix` DB roles created by migration 20261012000000): * * mail_accounts_v address, password ({ARGON2ID}-prefixed PHC hash), * home, quota_bytes, can_send, can_receive @@ -11,14 +11,17 @@ * smtpd_sender_login_maps; opensource-mail-helper uses * account_id + unsubscribe_headers to mint * List-Unsubscribe tokens) + * mail_recipients_v recipient — deliverable addresses on can-receive + * domains (Postfix virtual_mailbox_maps: unknown + * recipients rejected at RCPT, no backscatter) * mail_suppressions_v sender, recipient - * (submission policy service — reject at RCPT) + * (submission milter — reject at RCPT) * * SQLite cannot host the mail stack, but the views are still created there so * the schema stays identical across dialects for tests. */ -const VIEW_NAMES = ['mail_accounts_v', 'mail_senders_v', 'mail_suppressions_v']; +const VIEW_NAMES = ['mail_accounts_v', 'mail_senders_v', 'mail_recipients_v', 'mail_suppressions_v']; /** * Build the CREATE VIEW statements for a dialect. @@ -57,6 +60,10 @@ function createViewsSql(dialect) { `ma.${q('unsubscribeHeaders')} AS unsubscribe_headers ` + `${joins} WHERE ma.${q('enabled')} AND ${canSend}`, + `CREATE VIEW mail_recipients_v AS ` + + `SELECT ${address} AS recipient ` + + `${joins} WHERE ma.${q('enabled')} AND ${canReceive}`, + `CREATE VIEW mail_suppressions_v AS ` + `SELECT ${address} AS sender, ms.${q('recipient')} AS recipient ` + `FROM ${q('MailSuppressions')} ms ` + diff --git a/images/manager/container-creator-init.service b/images/manager/container-creator-init.service index 62081362..fd459934 100644 --- a/images/manager/container-creator-init.service +++ b/images/manager/container-creator-init.service @@ -18,13 +18,13 @@ ExecStart=/bin/bash -e -c '\ export POSTGRES_HOST="localhost"; \ sudo -u postgres psql -c "CREATE USER $${POSTGRES_USER} WITH PASSWORD '"'"'$${POSTGRES_PASSWORD}'"'"';"; \ sudo -u postgres psql -c "CREATE DATABASE $${POSTGRES_DATABASE} OWNER $${POSTGRES_USER};"; \ + sudo -u postgres psql -c "ALTER ROLE $${POSTGRES_USER} CREATEROLE;"; \ echo "DATABASE_DIALECT=$${DATABASE_DIALECT}" > /etc/default/container-creator; \ echo "POSTGRES_HOST=$${POSTGRES_HOST}" >> /etc/default/container-creator; \ echo "POSTGRES_DATABASE=$${POSTGRES_DATABASE}" >> /etc/default/container-creator; \ echo "POSTGRES_USER=$${POSTGRES_USER}" >> /etc/default/container-creator; \ echo "POSTGRES_PASSWORD=$${POSTGRES_PASSWORD}" >> /etc/default/container-creator; \ - echo "MCP_SERVER_URL=http://127.0.0.1:8100" >> /etc/default/container-creator; \ - bash /opt/opensource-server/create-a-container/bin/setup-mail-db-roles.sh || true;' + echo "MCP_SERVER_URL=http://127.0.0.1:8100" >> /etc/default/container-creator;' [Install] WantedBy=multi-user.target From ff9da0648ceeac11f2a28b5895a1fafe00fc7dcc Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Mon, 12 Oct 2026 14:53:42 -0700 Subject: [PATCH 15/19] mail: opensource-mail-helper is its own component, as a milter (PR #495) New top-level mail-helper/ component (own Makefile, .fpm, package name, built by the root Makefile, installed to /opt/opensource-server/mail-helper, suggested by opensource-agent) replacing the hand-rolled SMTP content filter + policy service hidden in the agent package. It is now a Postfix milter built on the well-tested 'milter' npm library, listening on a unix socket inside the Postfix chroot - no localhost ports, no reinjection listener, no master.cf '-o { }' overrides (submission policy lives in main.cf as named parameters): - RCPT: suppressed recipients rejected 5.7.1; DB outages DEFER (fail closed - no more delivering to opted-out recipients) - EOM: forged List-Unsubscribe headers stripped, tokenized RFC 8058 headers added (single-recipient submissions; tokens are per-recipient), then OpenDKIM - also moved to a chroot unix socket - signs them - unsubscribe-token.js is copied from the manager at build time (gitignored) so the two implementations stay in sync by construction - config: committed config.json + agent-written config.local.json overlay (same split in dev and production); paths env-overridable - vmail/opensource-mail users come from the package postinst; the agent only manages directories, with DKIM keys and OpenDKIM tables under \/mail instead of /etc Agent: apply.ts reconciles file mode/owner even when contents are unchanged and re-runs postWrite commands after rollback (no stale sni_map.db); removing 'mail' from AGENT_SERVICES now stops and disables a running mail stack. --- Makefile | 2 +- agent/.fpm | 1 + agent/Makefile | 1 - .../systemd/opensource-mail-helper.service | 20 - agent/package-lock.json | 267 +------ agent/package.json | 4 +- agent/src/apply.ts | 30 + agent/src/index.ts | 27 +- agent/src/mail-helper.ts | 357 --------- agent/src/mail.ts | 133 ++-- agent/src/unsubscribe-token.ts | 37 - agent/templates/mail/opendkim-keytable.ejs | 2 +- agent/templates/mail/opendkim.conf.ejs | 9 +- agent/templates/mail/postfix-main.cf.ejs | 23 +- agent/templates/mail/postfix-master.cf.ejs | 40 +- .../mail/postfix-sql-mailboxes.cf.ejs | 9 + agent/test/mail-helper.test.js | 72 -- agent/test/mail-template.test.js | 39 +- compose.yml | 1 + images/manager/Dockerfile | 2 +- mail-helper/.fpm | 11 + mail-helper/.gitignore | 7 + mail-helper/Makefile | 73 ++ mail-helper/README.md | 28 + mail-helper/config.json | 15 + mail-helper/contrib/postinstall.sh | 19 + .../systemd/opensource-mail-helper.service | 23 + mail-helper/package-lock.json | 720 ++++++++++++++++++ mail-helper/package.json | 20 + mail-helper/src/config.js | 41 + mail-helper/src/db.js | 61 ++ mail-helper/src/filter.js | 62 ++ mail-helper/src/index.js | 122 +++ mail-helper/test/filter.test.js | 74 ++ mie-opensource-landing/docs/admins/email.md | 17 +- .../docs/developers/agent.md | 10 +- .../docs/developers/database-schema.md | 5 +- 37 files changed, 1507 insertions(+), 877 deletions(-) delete mode 100644 agent/contrib/systemd/opensource-mail-helper.service delete mode 100644 agent/src/mail-helper.ts delete mode 100644 agent/src/unsubscribe-token.ts create mode 100644 agent/templates/mail/postfix-sql-mailboxes.cf.ejs delete mode 100644 agent/test/mail-helper.test.js create mode 100644 mail-helper/.fpm create mode 100644 mail-helper/.gitignore create mode 100644 mail-helper/Makefile create mode 100644 mail-helper/README.md create mode 100644 mail-helper/config.json create mode 100644 mail-helper/contrib/postinstall.sh create mode 100644 mail-helper/contrib/systemd/opensource-mail-helper.service create mode 100644 mail-helper/package-lock.json create mode 100644 mail-helper/package.json create mode 100644 mail-helper/src/config.js create mode 100644 mail-helper/src/db.js create mode 100644 mail-helper/src/filter.js create mode 100644 mail-helper/src/index.js create mode 100644 mail-helper/test/filter.test.js diff --git a/Makefile b/Makefile index b1da4341..5b8d63be 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -COMPONENTS := agent mie-opensource-landing manager-control-program create-a-container +COMPONENTS := agent mail-helper mie-opensource-landing manager-control-program create-a-container PACKAGER ?= deb # Forwarded to every component Makefile. diff --git a/agent/.fpm b/agent/.fpm index d34a7d95..e9a5302f 100644 --- a/agent/.fpm +++ b/agent/.fpm @@ -23,5 +23,6 @@ --deb-suggests dovecot-pgsql --deb-suggests dovecot-mysql --deb-suggests opendkim +--deb-suggests opensource-mail-helper --deb-no-default-config-files --after-install contrib/postinstall.sh diff --git a/agent/Makefile b/agent/Makefile index 91bde761..fb051803 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -52,7 +52,6 @@ install: build $(INSTALL) -d $(UNIT_DIR) $(INSTALL_DATA) contrib/systemd/opensource-agent.service $(UNIT_DIR)/ $(INSTALL_DATA) contrib/systemd/opensource-agent.timer $(UNIT_DIR)/ - $(INSTALL_DATA) contrib/systemd/opensource-mail-helper.service $(UNIT_DIR)/ $(INSTALL) -d $(DESTDIR)$(PREFIX)/error-pages $(INSTALL_DATA) $(ERROR_PAGES)/* $(DESTDIR)$(PREFIX)/error-pages/ $(INSTALL) -d -m 0755 $(DESTDIR)/var/cache/nginx/auth_cache diff --git a/agent/contrib/systemd/opensource-mail-helper.service b/agent/contrib/systemd/opensource-mail-helper.service deleted file mode 100644 index 0491e965..00000000 --- a/agent/contrib/systemd/opensource-mail-helper.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=MIE opensource-server mail helper (List-Unsubscribe filter + suppression policy) -Documentation=https://github.com/mieweb/opensource-server/issues/67 -After=network.target -PartOf=postfix.service - -[Service] -Type=simple -User=opensource-mail -Group=opensource-mail -ExecStart=/usr/bin/node /opt/opensource-server/agent/dist/mail-helper.js -Restart=on-failure -RestartSec=5 -NoNewPrivileges=yes -ProtectSystem=strict -ProtectHome=yes -PrivateTmp=yes - -[Install] -WantedBy=multi-user.target diff --git a/agent/package-lock.json b/agent/package-lock.json index 425910f9..0e8963a7 100644 --- a/agent/package-lock.json +++ b/agent/package-lock.json @@ -10,9 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@particle/dbus-next": "^0.11.4", - "ejs": "^6.0.1", - "mysql2": "^3.15.3", - "pg": "^8.16.3" + "ejs": "^6.0.1" }, "devDependencies": { "@types/ejs": "^3.1.5", @@ -54,20 +52,12 @@ "version": "24.13.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.3.tgz", "integrity": "sha512-Dh8vAsV36ig5wa9OX4pXvMc9D3Veibfw2wix0CUwYODLD8nkj9UsLjASr49nPg+2eKzxhBV+v7L8pXvT4e639Q==", + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.18.0" } }, - "node_modules/aws-ssl-profiles": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", - "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", - "license": "MIT", - "engines": { - "node": ">= 6.0.0" - } - }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -107,37 +97,6 @@ "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", "license": "MIT" }, - "node_modules/generate-function": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", - "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", - "license": "MIT", - "dependencies": { - "is-property": "^1.0.2" - } - }, - "node_modules/iconv-lite": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", - "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/is-property": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", - "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", - "license": "MIT" - }, "node_modules/jsbi": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/jsbi/-/jsbi-4.3.2.tgz", @@ -150,66 +109,12 @@ "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", "license": "Apache-2.0" }, - "node_modules/lru.min": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.5.tgz", - "integrity": "sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==", - "license": "MIT", - "engines": { - "bun": ">=1.0.0", - "deno": ">=1.30.0", - "node": ">=8.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wellwelwel" - } - }, "node_modules/map-stream": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", "integrity": "sha512-C0X0KQmGm3N2ftbTGBhSyuydQ+vV1LC3f3zPvT3RXHXNZrvfPZcoXp/N5DOa8vedX/rTMm2CjTtivFg2STJMRQ==", "license": "MIT" }, - "node_modules/mysql2": { - "version": "3.24.5", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.24.5.tgz", - "integrity": "sha512-X6Ujsr2QSkkLpkQGjxzpKRAPn9nu4axpR63ntBzquFVEvPOArgbUQ1sJjFKI7hnaYtiVZxa17Z7q18KSubW0IQ==", - "license": "MIT", - "dependencies": { - "aws-ssl-profiles": "^1.1.2", - "generate-function": "^2.3.1", - "iconv-lite": "^0.7.3", - "long": "^5.3.2", - "lru.min": "^1.1.4", - "named-placeholders": "^1.1.6", - "sql-escaper": "^1.5.1" - }, - "engines": { - "node": ">= 8.0" - }, - "peerDependencies": { - "@types/node": ">= 8" - } - }, - "node_modules/mysql2/node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0" - }, - "node_modules/named-placeholders": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.6.tgz", - "integrity": "sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==", - "license": "MIT", - "dependencies": { - "lru.min": "^1.1.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/pause-stream": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", @@ -222,134 +127,6 @@ "through": "~2.3" } }, - "node_modules/pg": { - "version": "8.23.1", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", - "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", - "license": "MIT", - "dependencies": { - "pg-connection-string": "^2.14.1", - "pg-pool": "^3.14.0", - "pg-protocol": "^1.16.1", - "pg-types": "2.2.0", - "pgpass": "1.0.5" - }, - "engines": { - "node": ">= 16.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.4.1" - }, - "peerDependencies": { - "pg-native": ">=3.0.1" - }, - "peerDependenciesMeta": { - "pg-native": { - "optional": true - } - } - }, - "node_modules/pg-cloudflare": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", - "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", - "license": "MIT", - "optional": true - }, - "node_modules/pg-connection-string": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", - "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", - "license": "MIT" - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", - "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-pool": { - "version": "3.14.0", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", - "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", - "license": "MIT", - "peerDependencies": { - "pg": ">=8.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.16.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", - "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", - "license": "MIT" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", - "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pgpass": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", - "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", - "license": "MIT", - "dependencies": { - "split2": "^4.1.0" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", - "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", - "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", - "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -370,12 +147,6 @@ ], "license": "MIT" }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, "node_modules/sax": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", @@ -397,30 +168,6 @@ "node": "*" } }, - "node_modules/split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "license": "ISC", - "engines": { - "node": ">= 10.x" - } - }, - "node_modules/sql-escaper": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.5.2.tgz", - "integrity": "sha512-6CKD38c31SENivxOADeMNLdukOnUxUcflKtzVWzace7Riv1v7cAEym5Cx9Q7gYZ3ezIJI7ZpqecQq8cqNeSSFg==", - "license": "MIT", - "engines": { - "bun": ">=1.0.0", - "deno": ">=2.0.0", - "node": ">=12.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/mysqljs/sql-escaper?sponsor=1" - } - }, "node_modules/stream-combiner": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", @@ -455,6 +202,7 @@ "version": "7.18.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "dev": true, "license": "MIT" }, "node_modules/xml2js": { @@ -478,15 +226,6 @@ "engines": { "node": ">=4.0" } - }, - "node_modules/xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "license": "MIT", - "engines": { - "node": ">=0.4" - } } } } diff --git a/agent/package.json b/agent/package.json index f4244c0e..dbfc21b2 100644 --- a/agent/package.json +++ b/agent/package.json @@ -11,9 +11,7 @@ }, "dependencies": { "@particle/dbus-next": "^0.11.4", - "ejs": "^6.0.1", - "mysql2": "^3.15.3", - "pg": "^8.16.3" + "ejs": "^6.0.1" }, "devDependencies": { "@types/ejs": "^3.1.5", diff --git a/agent/src/apply.ts b/agent/src/apply.ts index 5c445e08..b404b70a 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -142,6 +142,22 @@ function writeFileAtomic(dest: string, content: string, mode?: number, owner?: s fs.renameSync(tmp, dest); } +/** Re-apply desired mode/owner to files whose content already matches. */ +function reconcileAttributes(svc: ManagedService, files: { dest: string; mode?: number; owner?: string }[]): void { + for (const f of files) { + try { + if (f.mode !== undefined && (fs.statSync(f.dest).mode & 0o7777) !== f.mode) { + fs.chmodSync(f.dest, f.mode); + log.info(`${svc.unit}: fixed mode on ${f.dest}`); + } + // chown is idempotent and cheap; names are resolved by chown itself. + if (f.owner) run(['chown', f.owner, f.dest]); + } catch (err) { + log.warn(`${svc.unit}: could not reconcile attributes on ${f.dest}: ${(err as Error).message}`); + } + } +} + export async function applyService( svc: ManagedService, config: SiteConfig, @@ -157,6 +173,10 @@ export async function applyService( const current = new Map(files.map((f) => [f.dest, readIfExists(f.dest)])); const changed = files.filter((f) => current.get(f.dest) !== f.content).map((f) => f.dest); if (changed.length === 0) { + // Contents match, but desired mode/owner must still be reconciled — a + // DKIM key or helper config accidentally left world-readable would + // otherwise stay exposed forever. + reconcileAttributes(svc, files); log.debug(`${svc.unit}: config unchanged, nothing to do`); return 'success'; } @@ -177,6 +197,16 @@ export async function applyService( if (prev === null) fs.rmSync(dest, { force: true }); else writeFileAtomic(dest, prev, modeByDest.get(dest), ownerByDest.get(dest)); } + // Post-write commands may have produced sidecar artifacts from the new + // sources (e.g. postmap compiling sni_map.db) — regenerate them from the + // restored files so no stale compiled state survives the rollback. + for (const cmd of svc.postWrite ?? []) { + try { + run(cmd); + } catch (err) { + log.error(`${svc.unit}: post-write command failed during rollback (\`${cmd.join(' ')}\`): ${(err as Error).message}`); + } + } log.debug(`${svc.unit}: rolled back to previous config`); }; diff --git a/agent/src/index.ts b/agent/src/index.ts index 5ce8e87b..8b0f7bf0 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -16,7 +16,7 @@ import { State } from './state'; import { getPrimaryIpv4, getServiceState, findMissingBinaries, disconnectSystemBus } from './system'; import { checkin } from './api'; import { services, GROUP_BINARIES, applyService, type ManagedService } from './apply'; -import { mailServices, ensureMailProvisioned, syncMailUnits, gcMailboxes } from './mail'; +import { mailServices, ensureMailProvisioned, syncMailUnits, stopMailUnitsIfRunning, gcMailboxes, missingMailHelper } from './mail'; import { reconcileVolumes } from './volumes'; import { log } from './log'; import type { CheckinRequest, ServiceStatus, SiteConfig } from './types'; @@ -46,7 +46,11 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise GROUP_BINARIES[g])), + missingBinaries: [ + ...findMissingBinaries(cfg.services.flatMap((g) => GROUP_BINARIES[g])), + // The helper is its own package under /opt, not a PATH binary. + ...(cfg.services.includes('mail') && missingMailHelper() ? ['opensource-mail-helper'] : []), + ], }; // Report any not-yet-delivered volume results (persisted across runs). Only // include the field when there is something to report so the manager isn't @@ -58,13 +62,18 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { - if (!cfg.services.includes('mail')) return; try { - await syncMailUnits(config); - if (config.mail) gcMailboxes(config.mail); + if (cfg.services.includes('mail')) { + await syncMailUnits(config); + if (config.mail) gcMailboxes(config.mail); + } else { + await stopMailUnitsIfRunning(); + } } catch (err) { log.error(`mail: unit lifecycle sync failed: ${(err as Error).message}`); } @@ -99,9 +108,9 @@ async function main(): Promise { } log.info(`check-in: new config received (etag=${result.etag ?? '(none)'}), applying`); - // Mail users/dirs must exist before the mail configs reference them. + // Mail dirs must exist before the mail configs reference them. if (cfg.services.includes('mail') && result.config.mail) { - ensureMailProvisioned(); + ensureMailProvisioned(cfg); } for (const svc of enabledServices(cfg)) { state.lastApply[svc.unit] = await applyService(svc, result.config, cfg); diff --git a/agent/src/mail-helper.ts b/agent/src/mail-helper.ts deleted file mode 100644 index 69f053e2..00000000 --- a/agent/src/mail-helper.ts +++ /dev/null @@ -1,357 +0,0 @@ -#!/usr/bin/env node -/** - * opensource-mail-helper: the mail host's after-queue content filter and - * submission policy service (issue #67). Shipped in the agent package and - * managed as part of the `mail` service group. - * - * - Content filter (127.0.0.1:10025): Postfix pipes submitted mail here one - * recipient per copy; the helper strips any pre-existing List-Unsubscribe - * headers, adds RFC 8058 one-click headers with an encrypted per-recipient - * token (unless the account disabled them), and reinjects on - * 127.0.0.1:10026 — where the OpenDKIM milter signs, so the signature - * covers the new headers. - * - Policy service (127.0.0.1:10027): at RCPT TO on the submission ports, - * rejects recipients who unsubscribed from the authenticated sender - * (mail_suppressions_v) with a synchronous 5.7.1. - * - * Config: /etc/opensource-server/mail-helper.json, rendered by the agent from - * the mail-host snapshot. A self-managed MTA can run this daemon too — it - * only needs the DB views and the unsubscribe keys. - */ - -import fs from 'fs'; -import net from 'net'; -import { createUnsubscribeToken, type UnsubscribeKey } from './unsubscribe-token'; -import { log } from './log'; - -const CONFIG_FILE = process.env.MAIL_HELPER_CONFIG || '/etc/opensource-server/mail-helper.json'; - -interface HelperConfig { - listen: { filterPort: number; reinjectPort: number; policyPort: number }; - db: { - dialect: 'postgres' | 'mysql'; - host: string | null; - port: number | null; - database: string | null; - user: string; - password: string | null; - }; - unsubscribe: { - baseUrl: string | null; - keys: (UnsubscribeKey & { active: boolean })[]; - }; -} - -interface SenderInfo { - accountId: string; - unsubscribeHeaders: boolean; -} - -/** Thin dialect-agnostic wrapper over the two mail views. */ -class Db { - private pool: { - query(sql: string, params: unknown[]): Promise<{ rows: Record[] }>; - }; - - constructor(cfg: HelperConfig['db']) { - if (cfg.dialect === 'mysql') { - // eslint-disable-next-line @typescript-eslint/no-require-imports - const mysql = require('mysql2/promise'); - const pool = mysql.createPool({ - ...(cfg.host ? { host: cfg.host, port: cfg.port ?? 3306 } : { socketPath: '/var/run/mysqld/mysqld.sock' }), - user: cfg.user, - ...(cfg.password ? { password: cfg.password } : {}), - database: cfg.database, - connectionLimit: 5, - }); - this.pool = { - async query(sql, params) { - const [rows] = await pool.query(sql.replace(/\$\d+/g, '?'), params); - return { rows: rows as Record[] }; - }, - }; - } else { - // eslint-disable-next-line @typescript-eslint/no-require-imports - const { Pool } = require('pg'); - const pool = new Pool({ - // No host means the local unix socket (peer auth, no password). - ...(cfg.host ? { host: cfg.host, port: cfg.port ?? 5432 } : { host: '/var/run/postgresql' }), - user: cfg.user, - ...(cfg.password ? { password: cfg.password } : {}), - database: cfg.database ?? undefined, - max: 5, - }); - this.pool = pool; - } - } - - async getSender(address: string): Promise { - const { rows } = await this.pool.query( - 'SELECT account_id, unsubscribe_headers FROM mail_senders_v WHERE sender = $1', - [address.toLowerCase()], - ); - if (!rows[0]) return null; - const raw = rows[0].unsubscribe_headers; // boolean on Postgres, 0/1 on MySQL - return { - accountId: String(rows[0].account_id), - unsubscribeHeaders: raw === true || raw === 1 || raw === '1', - }; - } - - async isSuppressed(sender: string, recipient: string): Promise { - const { rows } = await this.pool.query( - 'SELECT 1 AS hit FROM mail_suppressions_v WHERE sender = $1 AND recipient = $2', - [sender.toLowerCase(), recipient.toLowerCase()], - ); - return rows.length > 0; - } -} - -// --- header manipulation ----------------------------------------------------- - -const CRLF = '\r\n'; - -/** Strip client-supplied List-Unsubscribe(-Post) headers (including folded - * continuation lines) so apps can't forge unsubscribe targets. */ -export function stripUnsubscribeHeaders(headerBlock: string): string { - const lines = headerBlock.split(CRLF); - const kept: string[] = []; - let skipping = false; - for (const line of lines) { - if (/^list-unsubscribe(-post)?\s*:/i.test(line)) { - skipping = true; - continue; - } - if (skipping && /^[ \t]/.test(line)) continue; // folded continuation - skipping = false; - kept.push(line); - } - return kept.join(CRLF); -} - -/** Add the RFC 8058 headers for one recipient. `message` is the raw DATA - * payload (dot-stuffed lines are untouched — headers never start with '.'). */ -export function addUnsubscribeHeaders( - message: string, - { baseUrl, token }: { baseUrl: string; token: string }, -): string { - const splitAt = message.indexOf(CRLF + CRLF); - const headers = splitAt === -1 ? message : message.slice(0, splitAt); - const rest = splitAt === -1 ? '' : message.slice(splitAt); - const cleaned = stripUnsubscribeHeaders(headers); - const added = - `List-Unsubscribe: <${baseUrl}/u/${token}>${CRLF}` + - `List-Unsubscribe-Post: List-Unsubscribe=One-Click`; - return `${added}${CRLF}${cleaned}${rest}`; -} - -// --- minimal SMTP client (reinjection) ---------------------------------------- - -function reinject( - port: number, - envelope: { sender: string; recipient: string }, - message: string, -): Promise { - return new Promise((resolve, reject) => { - const socket = net.connect(port, '127.0.0.1'); - socket.setTimeout(30_000, () => socket.destroy(new Error('reinjection timed out'))); - let buffer = ''; - const steps: { expect: RegExp; send: string | null }[] = [ - { expect: /^220 /, send: `HELO mail-helper${CRLF}` }, - { expect: /^250/, send: `MAIL FROM:<${envelope.sender}>${CRLF}` }, - { expect: /^250/, send: `RCPT TO:<${envelope.recipient}>${CRLF}` }, - { expect: /^250/, send: `DATA${CRLF}` }, - { expect: /^354/, send: `${message}${CRLF}.${CRLF}` }, - { expect: /^250/, send: `QUIT${CRLF}` }, - { expect: /^221/, send: null }, - ]; - let step = 0; - socket.on('data', (chunk) => { - buffer += chunk.toString('utf8'); - // Process complete (possibly multi-line) SMTP replies. - let eol; - while ((eol = buffer.indexOf(CRLF)) !== -1) { - const line = buffer.slice(0, eol); - buffer = buffer.slice(eol + 2); - if (/^\d{3}-/.test(line)) continue; // multi-line reply continues - const current = steps[step]; - if (!current.expect.test(line)) { - socket.destroy(); - return reject(new Error(`reinjection step ${step} got "${line}"`)); - } - step += 1; - if (current.send !== null) socket.write(current.send); - if (step === steps.length) { - socket.end(); - return resolve(); - } - } - }); - socket.on('error', reject); - }); -} - -// --- content filter SMTP server ------------------------------------------------ - -interface FilterDeps { - db: Db; - unsubscribe: HelperConfig['unsubscribe']; - reinjectPort: number; -} - -/** Build the final message for one (sender, recipient) copy. Exported for tests. */ -export async function processMessage( - deps: Pick, - envelope: { sender: string; recipient: string }, - message: string, -): Promise { - const activeKey = deps.unsubscribe.keys.find((k) => k.active); - const { baseUrl } = deps.unsubscribe; - if (!activeKey || !baseUrl) return message; - const sender = await deps.db.getSender(envelope.sender); - if (!sender || !sender.unsubscribeHeaders) return message; - const token = createUnsubscribeToken( - { accountId: sender.accountId, recipient: envelope.recipient }, - activeKey, - ); - return addUnsubscribeHeaders(message, { baseUrl, token }); -} - -function startContentFilter(deps: FilterDeps, port: number): net.Server { - const server = net.createServer((socket) => { - socket.setTimeout(5 * 60_000, () => socket.destroy()); - let buffer = ''; - let inData = false; - let sender = ''; - let recipient = ''; - let data = ''; - - const reply = (line: string) => socket.write(`${line}${CRLF}`); - reply('220 mail-helper ESMTP'); - - const reset = () => { - sender = ''; - recipient = ''; - data = ''; - }; - - socket.on('data', (chunk) => { - buffer += chunk.toString('utf8'); - void (async () => { - let eol; - while ((eol = buffer.indexOf(CRLF)) !== -1) { - const line = buffer.slice(0, eol); - buffer = buffer.slice(eol + 2); - - if (inData) { - if (line === '.') { - inData = false; - try { - const message = await processMessage(deps, { sender, recipient }, data); - await reinject(deps.reinjectPort, { sender, recipient }, message); - reply('250 2.0.0 Ok'); - } catch (err) { - // Tempfail: Postfix keeps the message queued and retries. - log.error(`mail-helper: filter failed: ${(err as Error).message}`); - reply('451 4.3.0 content filter failure'); - } - reset(); - } else { - data += (data ? CRLF : '') + line; - } - continue; - } - - const upper = line.toUpperCase(); - if (upper.startsWith('EHLO') || upper.startsWith('HELO')) { - reply('250-mail-helper'); - reply('250-8BITMIME'); - reply('250 XFORWARD NAME ADDR PORT PROTO HELO IDENT SOURCE'); - } else if (upper.startsWith('XFORWARD')) { - reply('250 2.0.0 Ok'); - } else if (upper.startsWith('MAIL FROM:')) { - sender = (line.slice(10).match(/<([^>]*)>/)?.[1] ?? line.slice(10)).trim(); - reply('250 2.1.0 Ok'); - } else if (upper.startsWith('RCPT TO:')) { - // mailhelper_destination_recipient_limit = 1 — one recipient per copy. - recipient = (line.slice(8).match(/<([^>]*)>/)?.[1] ?? line.slice(8)).trim(); - reply('250 2.1.5 Ok'); - } else if (upper === 'DATA') { - inData = true; - reply('354 End data with .'); - } else if (upper === 'RSET' || upper === 'NOOP') { - if (upper === 'RSET') reset(); - reply('250 2.0.0 Ok'); - } else if (upper === 'QUIT') { - reply('221 2.0.0 Bye'); - socket.end(); - } else { - reply('502 5.5.2 Command not recognized'); - } - } - })(); - }); - socket.on('error', () => socket.destroy()); - }); - server.listen(port, '127.0.0.1', () => log.info(`mail-helper: content filter on 127.0.0.1:${port}`)); - return server; -} - -// --- Postfix policy service ------------------------------------------------------ - -function startPolicyService(db: Db, port: number): net.Server { - const server = net.createServer((socket) => { - socket.setTimeout(60_000, () => socket.destroy()); - let buffer = ''; - const attrs: Record = {}; - socket.on('data', (chunk) => { - buffer += chunk.toString('utf8'); - void (async () => { - let eol; - while ((eol = buffer.indexOf('\n')) !== -1) { - const line = buffer.slice(0, eol).replace(/\r$/, ''); - buffer = buffer.slice(eol + 1); - if (line !== '') { - const eq = line.indexOf('='); - if (eq > 0) attrs[line.slice(0, eq)] = line.slice(eq + 1); - continue; - } - // Blank line = end of request. Sender == SASL login on these ports - // (reject_sender_login_mismatch), but prefer the login when present. - const sender = attrs.sasl_username || attrs.sender || ''; - const recipient = attrs.recipient || ''; - let action = 'DUNNO'; - try { - if (sender && recipient && (await db.isSuppressed(sender, recipient))) { - action = `REJECT 5.7.1 Recipient has unsubscribed from ${sender}`; - } - } catch (err) { - // Fail open: a DB hiccup must not block all submission. - log.error(`mail-helper: policy lookup failed: ${(err as Error).message}`); - } - socket.write(`action=${action}\n\n`); - for (const k of Object.keys(attrs)) delete attrs[k]; - } - })(); - }); - socket.on('error', () => socket.destroy()); - }); - server.listen(port, '127.0.0.1', () => log.info(`mail-helper: policy service on 127.0.0.1:${port}`)); - return server; -} - -// --- main ------------------------------------------------------------------------ - -function main(): void { - const config = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8')) as HelperConfig; - const db = new Db(config.db); - startContentFilter( - { db, unsubscribe: config.unsubscribe, reinjectPort: config.listen.reinjectPort }, - config.listen.filterPort, - ); - startPolicyService(db, config.listen.policyPort); -} - -if (require.main === module) { - main(); -} diff --git a/agent/src/mail.ts b/agent/src/mail.ts index f43687b2..aab43961 100644 --- a/agent/src/mail.ts +++ b/agent/src/mail.ts @@ -1,12 +1,19 @@ /** * The `mail` service group (issue #67): Postfix + Dovecot + OpenDKIM + - * opensource-mail-helper, rendered from the holder-only `mail` section of - * the config snapshot. + * opensource-mail-helper (a milter, shipped as its own component under + * /opt/opensource-server/mail-helper), rendered from the holder-only `mail` + * section of the config snapshot. * * Lifecycle: while this agent holds the mail-host claim, the units are * enabled and started after each apply; when the group is turned off or the * claim is lost, they are stopped and disabled — configs and /var/vmail stay - * in place. + * in place. System users (vmail, opensource-mail) come from the + * opensource-mail-helper package's maintainer scripts; the agent only + * manages directories and files. + * + * Agent-managed mail state (DKIM keys, OpenDKIM tables) lives under + * $STATE_DIRECTORY/mail, not /etc. Postfix/Dovecot/OpenDKIM daemon configs + * remain in /etc where their packages expect them. */ import fs from 'fs'; @@ -15,20 +22,23 @@ import { execFileSync } from 'child_process'; import { enableAndStartServices, stopAndDisableServices, + getServiceState, reloadOrRestartService, restartService, } from './system'; import { renderTemplate, type ManagedService } from './apply'; import { log } from './log'; +import type { AgentConfig } from './config'; import type { MailConfig, SiteConfig } from './types'; export const MAIL_UNITS = ['opendkim', 'dovecot', 'postfix', 'opensource-mail-helper']; -/** Localhost plumbing between Postfix, the helper, and OpenDKIM. */ -export const HELPER_PORTS = { filterPort: 10025, reinjectPort: 10026, policyPort: 10027 }; -export const OPENDKIM_PORT = 8891; - -export const HELPER_CONFIG_FILE = '/etc/opensource-server/mail-helper.json'; +// Paths are env-overridable for development stacks; defaults match the +// packages. Both milter sockets live inside the Postfix chroot so the +// chrooted smtpd can reach them without inet ports. +const HELPER_DIR = process.env.MAIL_HELPER_DIR || '/opt/opensource-server/mail-helper'; +const HELPER_SOCKET_DIR = process.env.MAIL_HELPER_SOCKET_DIR || '/var/spool/postfix/mail-helper'; +const OPENDKIM_SOCKET_DIR = process.env.OPENDKIM_SOCKET_DIR || '/var/spool/postfix/opendkim'; const VMAIL_DIR = '/var/vmail'; const GC_STATE_FILE = path.join(VMAIL_DIR, '.gc-state.json'); @@ -47,6 +57,11 @@ function fileExists(file: string): boolean { } } +/** The helper is a separate package; report it like a missing binary. */ +export function missingMailHelper(): boolean { + return !fileExists(path.join(HELPER_DIR, 'src', 'index.js')); +} + // --- render context --------------------------------------------------------- interface CertPair { name: string; cert: string; key: string } @@ -81,9 +96,14 @@ function fallbackChain(mail: MailConfig): { fallbackCert: string; fallbackKey: s }; } +/** Agent-managed OpenDKIM material under the agent's state directory. */ +function mailStateDir(agent: AgentConfig): string { + return path.join(agent.stateDir, 'mail'); +} + /** Everything the mail templates need, derived from the snapshot + local * environment (cert presence, DB passwords from /etc/environment). */ -export function buildMailContext(mail: MailConfig, env: NodeJS.ProcessEnv = process.env) { +export function buildMailContext(mail: MailConfig, agent: AgentConfig, env: NodeJS.ProcessEnv = process.env) { const mysql = mail.db.dialect === 'mysql' || mail.db.dialect === 'mariadb'; const host = mail.db.host; // Postfix map syntax: TCP "host:port", or a unix socket path for local @@ -93,8 +113,11 @@ export function buildMailContext(mail: MailConfig, env: NodeJS.ProcessEnv = proc : (mysql ? 'unix:/var/run/mysqld/mysqld.sock' : 'unix:/var/run/postgresql'); return { mail, - helper: HELPER_PORTS, - opendkimPort: OPENDKIM_PORT, + stateDir: mailStateDir(agent), + // Milter endpoints as the chrooted smtpd resolves them (queue-dir relative). + helperMilter: `unix:${path.posix.relative('/var/spool/postfix', path.posix.join(HELPER_SOCKET_DIR, 'milter.sock'))}`, + opendkimMilter: `unix:${path.posix.relative('/var/spool/postfix', path.posix.join(OPENDKIM_SOCKET_DIR, 'opendkim.sock'))}`, + opendkimSocket: `local:${path.posix.join(OPENDKIM_SOCKET_DIR, 'opendkim.sock')}`, certs: { ...fallbackChain(mail), domains: domainCerts(mail) }, db: { prefix: mysql ? 'mysql' : 'pgsql', @@ -111,13 +134,13 @@ export function buildMailContext(mail: MailConfig, env: NodeJS.ProcessEnv = proc }; } -/** opensource-mail-helper daemon config (JSON, root:opensource-mail 0640). */ -export function buildHelperConfig(mail: MailConfig, env: NodeJS.ProcessEnv = process.env): string { - const context = buildMailContext(mail, env); +/** Overlay for the helper's committed config.json (see mail-helper/). */ +export function buildHelperLocalConfig(mail: MailConfig, env: NodeJS.ProcessEnv = process.env): string { + const mysql = mail.db.dialect === 'mysql' || mail.db.dialect === 'mariadb'; return `${JSON.stringify({ - listen: HELPER_PORTS, + socket: path.posix.join(HELPER_SOCKET_DIR, 'milter.sock'), db: { - dialect: context.db.prefix === 'mysql' ? 'mysql' : 'postgres', + dialect: mysql ? 'mysql' : 'postgres', host: mail.db.host, port: mail.db.port, database: mail.db.database, @@ -139,19 +162,19 @@ export const mailServices: ManagedService[] = [ { unit: 'opendkim', group: 'mail', - async render(config) { + async render(config, agent) { const mail = mailOf(config); if (!mail) return null; - const context = buildMailContext(mail); + const context = buildMailContext(mail, agent); const files = [ { dest: '/etc/opendkim.conf', content: await renderTemplate('mail/opendkim.conf.ejs', context) }, - { dest: '/etc/opendkim/keytable', content: await renderTemplate('mail/opendkim-keytable.ejs', context) }, - { dest: '/etc/opendkim/signingtable', content: await renderTemplate('mail/opendkim-signingtable.ejs', context) }, + { dest: path.join(context.stateDir, 'opendkim-keytable'), content: await renderTemplate('mail/opendkim-keytable.ejs', context) }, + { dest: path.join(context.stateDir, 'opendkim-signingtable'), content: await renderTemplate('mail/opendkim-signingtable.ejs', context) }, ]; // Private keys never serialized anywhere else on this host. for (const k of mail.dkim) { files.push({ - dest: `/etc/dkimkeys/${k.domain}.private`, + dest: path.join(context.stateDir, 'dkim', `${k.domain}.private`), content: k.privateKey, mode: 0o600, owner: 'opendkim:opendkim', @@ -167,10 +190,10 @@ export const mailServices: ManagedService[] = [ { unit: 'dovecot', group: 'mail', - async render(config) { + async render(config, agent) { const mail = mailOf(config); if (!mail) return null; - const context = buildMailContext(mail); + const context = buildMailContext(mail, agent); return [{ dest: '/etc/dovecot/dovecot.conf', content: await renderTemplate('mail/dovecot.conf.ejs', context), @@ -195,16 +218,17 @@ export const mailServices: ManagedService[] = [ // Debian's postfix.service is a wrapper; the real daemon state lives on // the postfix@- instance. statusUnit: 'postfix@-', - async render(config) { + async render(config, agent) { const mail = mailOf(config); if (!mail) return null; - const context = buildMailContext(mail); + const context = buildMailContext(mail, agent); const files = [ { dest: '/etc/postfix/main.cf', content: await renderTemplate('mail/postfix-main.cf.ejs', context) }, { dest: '/etc/postfix/master.cf', content: await renderTemplate('mail/postfix-master.cf.ejs', context) }, { dest: '/etc/postfix/vdomains', content: await renderTemplate('mail/postfix-vdomains.ejs', context) }, { dest: '/etc/postfix/sni_map', content: await renderTemplate('mail/postfix-sni-map.ejs', context), mode: 0o600 }, { dest: '/etc/postfix/sql/senders.cf', content: await renderTemplate('mail/postfix-sql-senders.cf.ejs', context), mode: 0o600 }, + { dest: '/etc/postfix/sql/mailboxes.cf', content: await renderTemplate('mail/postfix-sql-mailboxes.cf.ejs', context), mode: 0o600 }, ]; if (mail.relayhost?.username) { files.push({ @@ -228,9 +252,11 @@ export const mailServices: ManagedService[] = [ async render(config) { const mail = mailOf(config); if (!mail) return null; + // The committed config.json ships defaults; the agent only writes the + // environment-specific overlay (same split in dev and production). return [{ - dest: HELPER_CONFIG_FILE, - content: buildHelperConfig(mail), + dest: path.join(HELPER_DIR, 'config.local.json'), + content: buildHelperLocalConfig(mail), mode: 0o640, owner: 'root:opensource-mail', }]; @@ -243,25 +269,35 @@ export const mailServices: ManagedService[] = [ // --- lifecycle --------------------------------------------------------------- -/** Create the system users/dirs the mail stack needs. Idempotent; runs only - * when this agent holds the claim. */ -export function ensureMailProvisioned(): void { - const ensureUser = (name: string, opts: string[]) => { - try { - sh(['id', '-u', name]); - } catch { - sh(['useradd', '--system', ...opts, name]); - } - }; - // Mailbox owner (Dovecot mail_uid/mail_gid). - ensureUser('vmail', ['--home-dir', VMAIL_DIR, '--create-home', '--shell', '/usr/sbin/nologin']); - // The helper daemon's unprivileged user (reads its config via group). - ensureUser('opensource-mail', ['--no-create-home', '--shell', '/usr/sbin/nologin']); +/** Create the directories the mail stack needs. Idempotent; runs only when + * this agent holds the claim. System users come from package maintainer + * scripts (opensource-mail-helper postinst creates vmail + opensource-mail). */ +export function ensureMailProvisioned(agent: AgentConfig): void { fs.mkdirSync(VMAIL_DIR, { recursive: true }); sh(['chown', 'vmail:vmail', VMAIL_DIR]); sh(['chmod', '0750', VMAIL_DIR]); - fs.mkdirSync('/etc/dkimkeys', { recursive: true }); - fs.mkdirSync('/etc/opensource-server', { recursive: true }); + + // Agent-managed DKIM material under $STATE_DIRECTORY/mail. + const stateDir = mailStateDir(agent); + fs.mkdirSync(path.join(stateDir, 'dkim'), { recursive: true }); + sh(['chown', 'root:opendkim', stateDir, path.join(stateDir, 'dkim')]); + sh(['chmod', '0750', stateDir, path.join(stateDir, 'dkim')]); + + // Milter socket dirs inside the Postfix chroot. The chrooted smtpd + // connects as user postfix: setgid keeps sockets group-owned, and postfix + // joins the opendkim group for OpenDKIM's socket (the standard Debian + // recipe). + fs.mkdirSync(HELPER_SOCKET_DIR, { recursive: true }); + sh(['chown', 'opensource-mail:postfix', HELPER_SOCKET_DIR]); + sh(['chmod', '2750', HELPER_SOCKET_DIR]); + fs.mkdirSync(OPENDKIM_SOCKET_DIR, { recursive: true }); + sh(['chown', 'opendkim:postfix', OPENDKIM_SOCKET_DIR]); + sh(['chmod', '2750', OPENDKIM_SOCKET_DIR]); + try { + sh(['usermod', '-a', '-G', 'opendkim', 'postfix']); + } catch { + /* already a member, or opendkim not installed yet */ + } } /** Enable+start the mail units while holding the claim; stop+disable them @@ -277,6 +313,17 @@ export async function syncMailUnits(config: SiteConfig): Promise { await stopAndDisableServices(MAIL_UNITS); } +/** Stop the stack when the `mail` group itself is disabled — but only when + * something is actually running, so plain nginx/dnsmasq agents don't issue + * no-op unit operations every 30 seconds. */ +export async function stopMailUnitsIfRunning(): Promise { + const states = await Promise.all(MAIL_UNITS.map((unit) => getServiceState(unit))); + if (states.some((state) => state === 'active' || state === 'activating')) { + log.info('mail: service group disabled but units are running; stopping them'); + await stopAndDisableServices(MAIL_UNITS); + } +} + /** Remove Maildirs of deleted accounts after a retention window. A dir is * eligible once its account id stops appearing in the snapshot's mailboxIds; * first-missing timestamps persist in /var/vmail/.gc-state.json. */ diff --git a/agent/src/unsubscribe-token.ts b/agent/src/unsubscribe-token.ts deleted file mode 100644 index 8a2e7965..00000000 --- a/agent/src/unsubscribe-token.ts +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Unsubscribe token minting — MUST stay byte-compatible with the manager's - * create-a-container/utils/unsubscribe-token.js (same layout, same AAD): - * [version:1][kidLen:1][kid][iv:12][tag:16][ciphertext], base64url, where the - * ciphertext is AES-256-GCM over JSON {v, a: accountId, r: recipient, t: iat}. - */ - -import crypto from 'crypto'; - -const VERSION = 1; -const IV_LEN = 12; - -export interface UnsubscribeKey { - kid: string; - /** Base64 of 32 random bytes. */ - secret: string; -} - -export function createUnsubscribeToken( - { accountId, recipient }: { accountId: string; recipient: string }, - key: UnsubscribeKey, -): string { - const kidBuf = Buffer.from(key.kid, 'utf8'); - const iv = crypto.randomBytes(IV_LEN); - const cipher = crypto.createCipheriv('aes-256-gcm', Buffer.from(key.secret, 'base64'), iv); - cipher.setAAD(kidBuf); - const payload = Buffer.from(JSON.stringify({ - v: VERSION, - a: accountId, - r: recipient.trim().toLowerCase(), - t: Math.floor(Date.now() / 1000), - })); - const ciphertext = Buffer.concat([cipher.update(payload), cipher.final()]); - const tag = cipher.getAuthTag(); - return Buffer.concat([Buffer.from([VERSION, kidBuf.length]), kidBuf, iv, tag, ciphertext]) - .toString('base64url'); -} diff --git a/agent/templates/mail/opendkim-keytable.ejs b/agent/templates/mail/opendkim-keytable.ejs index 2e087f1b..a788e671 100644 --- a/agent/templates/mail/opendkim-keytable.ejs +++ b/agent/templates/mail/opendkim-keytable.ejs @@ -1,3 +1,3 @@ <%_ mail.dkim.forEach((k) => { _%> -<%= k.selector %>._domainkey.<%= k.domain %> <%= k.domain %>:<%= k.selector %>:/etc/dkimkeys/<%= k.domain %>.private +<%= k.selector %>._domainkey.<%= k.domain %> <%= k.domain %>:<%= k.selector %>:<%= stateDir %>/dkim/<%= k.domain %>.private <%_ }) _%> diff --git a/agent/templates/mail/opendkim.conf.ejs b/agent/templates/mail/opendkim.conf.ejs index 059ca4bf..3b7bc28f 100644 --- a/agent/templates/mail/opendkim.conf.ejs +++ b/agent/templates/mail/opendkim.conf.ejs @@ -2,7 +2,9 @@ Syslog yes UMask 007 Mode s -Socket inet:<%= opendkimPort %>@127.0.0.1 +# Unix socket inside the Postfix chroot (no inet port); postfix is a member +# of the opendkim group to connect. +Socket <%= opendkimSocket %> UserID opendkim PidFile /run/opendkim/opendkim.pid @@ -11,6 +13,7 @@ Canonicalization relaxed/simple # both signs them when present and prevents later injection. OversignHeaders From,List-Unsubscribe,List-Unsubscribe-Post -KeyTable /etc/opendkim/keytable -SigningTable refile:/etc/opendkim/signingtable +# Tables and keys are agent-managed state under $STATE_DIRECTORY/mail. +KeyTable <%= stateDir %>/opendkim-keytable +SigningTable refile:<%= stateDir %>/opendkim-signingtable InternalHosts 127.0.0.0/8 diff --git a/agent/templates/mail/postfix-main.cf.ejs b/agent/templates/mail/postfix-main.cf.ejs index ff0782ac..9bc554af 100644 --- a/agent/templates/mail/postfix-main.cf.ejs +++ b/agent/templates/mail/postfix-main.cf.ejs @@ -9,7 +9,9 @@ biff = no readme_directory = no append_dot_mydomain = no -# This host serves virtual mailboxes only. +# This host serves virtual mailboxes only — ALL delivery goes to Dovecot +# over LMTP (virtual_transport below). The empty local domain class merely +# disables Postfix's own local(8) delivery agent. mydestination = mynetworks = 127.0.0.0/8 [::1]/128 alias_maps = @@ -28,10 +30,11 @@ smtpd_tls_chain_files = <%= certs.fallbackKey %>, <%= certs.fallbackCert %> smtpd_tls_security_level = may smtp_tls_security_level = may -# Inbound (:25): only local recipients on can-receive domains. Dovecot's -# quota-status policy service rejects unknown and over-quota users at RCPT. +# Inbound (:25): only mailboxes that exist on can-receive domains, checked +# live from SQL so unknown recipients are rejected at RCPT (no backscatter). +# Dovecot's quota-status policy service additionally rejects over-quota. virtual_mailbox_domains = texthash:/etc/postfix/vdomains -virtual_mailbox_maps = static:x +virtual_mailbox_maps = <%= db.prefix %>:/etc/postfix/sql/mailboxes.cf virtual_transport = lmtp:unix:private/dovecot-lmtp smtpd_relay_restrictions = permit_mynetworks, reject_unauth_destination smtpd_recipient_restrictions = reject_unauth_destination, check_policy_service unix:private/quota-status @@ -39,10 +42,14 @@ smtpd_recipient_restrictions = reject_unauth_destination, check_policy_service u # Sender ownership: submission enforces sender == SASL login (live from SQL). smtpd_sender_login_maps = <%= db.prefix %>:/etc/postfix/sql/senders.cf -# After-queue content filter copies are delivered one recipient at a time so -# opensource-mail-helper can add a per-recipient List-Unsubscribe token -# (RFC 8058) before OpenDKIM signs on reinjection. -mailhelper_destination_recipient_limit = 1 +# Submission policy, referenced from master.cf. Milters run pre-queue and in +# order: opensource-mail-helper rejects suppressed recipients at RCPT and +# injects per-recipient RFC 8058 List-Unsubscribe headers, then OpenDKIM +# signs — so the signature covers the added headers. Both listen on unix +# sockets inside the Postfix chroot. +submission_recipient_restrictions = permit_sasl_authenticated, reject +submission_milters = <%= helperMilter %>, <%= opendkimMilter %> +milter_default_action = tempfail <%_ if (mail.relayhost) { _%> # Outbound smarthost — SPF for sending domains must include the relay diff --git a/agent/templates/mail/postfix-master.cf.ejs b/agent/templates/mail/postfix-master.cf.ejs index 539fa06b..ec0bbabe 100644 --- a/agent/templates/mail/postfix-master.cf.ejs +++ b/agent/templates/mail/postfix-master.cf.ejs @@ -1,12 +1,15 @@ # Managed by opensource-agent (mail service group) — do not edit. # +# Submission policy (restrictions, milters) is defined as named parameters +# in main.cf and referenced here, keeping master.cf overrides single-valued. +# # service type private unpriv chroot wakeup maxproc command + args smtp inet n - y - - smtpd # Submission (STARTTLS on 587, implicit TLS on 465): AUTH only after TLS, -# sender must match the SASL login, suppressions are enforced at RCPT by the -# opensource-mail-helper policy service, and everything accepted goes through -# the mailhelper content filter for List-Unsubscribe headers. +# sender must match the SASL login, and the pre-queue milters (mail-helper, +# then OpenDKIM) enforce suppressions at RCPT and sign with the injected +# List-Unsubscribe headers covered. submission inet n - y - - smtpd -o syslog_name=postfix/submission -o smtpd_tls_security_level=encrypt @@ -15,9 +18,9 @@ submission inet n - y - - smtpd -o smtpd_sasl_type=dovecot -o smtpd_sasl_path=private/auth -o smtpd_sender_restrictions=reject_sender_login_mismatch - -o { smtpd_relay_restrictions = permit_sasl_authenticated, reject } - -o { smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:<%= helper.policyPort %>, permit_sasl_authenticated, reject } - -o content_filter=mailhelper:[127.0.0.1]:<%= helper.filterPort %> + -o smtpd_relay_restrictions=$submission_recipient_restrictions + -o smtpd_recipient_restrictions=$submission_recipient_restrictions + -o smtpd_milters=$submission_milters smtps inet n - y - - smtpd -o syslog_name=postfix/smtps -o smtpd_tls_wrappermode=yes @@ -25,28 +28,9 @@ smtps inet n - y - - smtpd -o smtpd_sasl_type=dovecot -o smtpd_sasl_path=private/auth -o smtpd_sender_restrictions=reject_sender_login_mismatch - -o { smtpd_relay_restrictions = permit_sasl_authenticated, reject } - -o { smtpd_recipient_restrictions = check_policy_service inet:127.0.0.1:<%= helper.policyPort %>, permit_sasl_authenticated, reject } - -o content_filter=mailhelper:[127.0.0.1]:<%= helper.filterPort %> - -# Transport to the opensource-mail-helper content filter. -mailhelper unix - - y - - smtp - -o syslog_name=postfix/mailhelper - -o smtp_send_xforward_command=yes - -o disable_dns_lookups=yes - -o smtp_tls_security_level=none - -# Reinjection listener: the OpenDKIM milter runs HERE (not on 587/465) so the -# signature covers the headers the helper added. -127.0.0.1:<%= helper.reinjectPort %> inet n - y - - smtpd - -o syslog_name=postfix/reinject - -o smtpd_milters=inet:127.0.0.1:<%= opendkimPort %> - -o content_filter= - -o mynetworks=127.0.0.0/8 - -o { smtpd_relay_restrictions = permit_mynetworks, reject } - -o { smtpd_recipient_restrictions = permit_mynetworks, reject } - -o smtpd_tls_security_level=none - -o smtpd_authorized_xforward_hosts=127.0.0.0/8 + -o smtpd_relay_restrictions=$submission_recipient_restrictions + -o smtpd_recipient_restrictions=$submission_recipient_restrictions + -o smtpd_milters=$submission_milters pickup unix n - y 60 1 pickup cleanup unix n - y - 0 cleanup diff --git a/agent/templates/mail/postfix-sql-mailboxes.cf.ejs b/agent/templates/mail/postfix-sql-mailboxes.cf.ejs new file mode 100644 index 00000000..376714ea --- /dev/null +++ b/agent/templates/mail/postfix-sql-mailboxes.cf.ejs @@ -0,0 +1,9 @@ +# Managed by opensource-agent — Postfix virtual_mailbox_maps (live SQL). +# Existence check only: unknown recipients are rejected at RCPT. +hosts = <%= db.hosts %> +user = <%= db.user %> +<%_ if (db.password) { _%> +password = <%= db.password %> +<%_ } _%> +dbname = <%= db.dbname %> +query = SELECT 1 FROM mail_recipients_v WHERE recipient = '%s' diff --git a/agent/test/mail-helper.test.js b/agent/test/mail-helper.test.js deleted file mode 100644 index ed8ee6c2..00000000 --- a/agent/test/mail-helper.test.js +++ /dev/null @@ -1,72 +0,0 @@ -/** - * mail-helper header-injection tests (node --test): stripping forged - * List-Unsubscribe headers, adding tokenized RFC 8058 headers, and the - * per-account / missing-key opt-outs. - */ - -const test = require('node:test'); -const assert = require('node:assert'); -const crypto = require('crypto'); - -const { stripUnsubscribeHeaders, addUnsubscribeHeaders, processMessage } = require('../dist/mail-helper.js'); - -const CRLF = '\r\n'; -const MESSAGE = [ - 'From: app@example.com', - 'To: someone@example.net', - 'Subject: hi', - '', - 'body line', - '..leading dot stays stuffed', -].join(CRLF); - -const key = { kid: 'a1b2c3d4', secret: crypto.randomBytes(32).toString('base64'), active: true }; -const unsubscribe = { baseUrl: 'https://manager.example.com', keys: [key] }; -const db = { - async getSender(address) { - if (address === 'app@example.com') return { accountId: 'acc-1', unsubscribeHeaders: true }; - if (address === 'quiet@example.com') return { accountId: 'acc-2', unsubscribeHeaders: false }; - return null; - }, -}; - -test('stripUnsubscribeHeaders removes forged headers including folded lines', () => { - const headers = [ - 'From: a@b.c', - 'List-Unsubscribe: ,', - ' ', - 'List-Unsubscribe-Post: List-Unsubscribe=One-Click', - 'Subject: x', - ].join(CRLF); - assert.equal(stripUnsubscribeHeaders(headers), `From: a@b.c${CRLF}Subject: x`); -}); - -test('addUnsubscribeHeaders prepends both headers before the body', () => { - const out = addUnsubscribeHeaders(MESSAGE, { baseUrl: 'https://m.example', token: 'TOK' }); - assert.ok(out.startsWith(`List-Unsubscribe: ${CRLF}List-Unsubscribe-Post: List-Unsubscribe=One-Click${CRLF}From:`)); - // Body (including dot-stuffed lines) is untouched. - assert.ok(out.endsWith(`body line${CRLF}..leading dot stays stuffed`)); -}); - -test('processMessage injects a valid tokenized header for opted-in accounts', async () => { - const out = await processMessage( - { db, unsubscribe }, - { sender: 'app@example.com', recipient: 'Someone@Example.NET' }, - MESSAGE, - ); - const match = out.match(/^List-Unsubscribe: /); - assert.ok(match, 'List-Unsubscribe header present'); - assert.match(out, /^List-Unsubscribe-Post: List-Unsubscribe=One-Click$/m); -}); - -test('processMessage leaves mail untouched when opted out, unknown, or unconfigured', async () => { - const cases = [ - [{ db, unsubscribe }, 'quiet@example.com'], // account opted out - [{ db, unsubscribe }, 'ghost@example.com'], // not a managed sender - [{ db, unsubscribe: { baseUrl: null, keys: [key] } }, 'app@example.com'], // no base URL - [{ db, unsubscribe: { baseUrl: 'https://m', keys: [] } }, 'app@example.com'], // no active key - ]; - for (const [deps, sender] of cases) { - assert.equal(await processMessage(deps, { sender, recipient: 'r@x.y' }, MESSAGE), MESSAGE); - } -}); diff --git a/agent/test/mail-template.test.js b/agent/test/mail-template.test.js index efb5f90d..5877acc4 100644 --- a/agent/test/mail-template.test.js +++ b/agent/test/mail-template.test.js @@ -29,8 +29,10 @@ const context = { }, unsubscribe: { baseUrl: 'https://manager.example.com', keys: [] }, }, - helper: { filterPort: 10025, reinjectPort: 10026, policyPort: 10027 }, - opendkimPort: 8891, + stateDir: '/var/lib/opensource-agent/mail', + helperMilter: 'unix:mail-helper/milter.sock', + opendkimMilter: 'unix:opendkim/opendkim.sock', + opendkimSocket: 'local:/var/spool/postfix/opendkim/opendkim.sock', certs: { fallbackCert: '/etc/ssl/certs/mail.example.com.crt', fallbackKey: '/etc/ssl/private/mail.example.com.key', @@ -54,15 +56,20 @@ function render(template, data = context) { return ejs.renderFile(path.join(TEMPLATES, template), data); } -test('postfix main.cf: recipient gates, sender ownership, size limit, SNI', async () => { +test('postfix main.cf: live recipient map, sender ownership, size limit, SNI, milter policy', async () => { const out = await render('mail/postfix-main.cf.ejs'); assert.match(out, /myhostname = mail\.example\.com/); assert.match(out, /message_size_limit = 26214400/); assert.match(out, /virtual_mailbox_domains = texthash:\/etc\/postfix\/vdomains/); + // Unknown recipients must be rejected at RCPT (live SQL), not via static:x. + assert.match(out, /virtual_mailbox_maps = pgsql:\/etc\/postfix\/sql\/mailboxes\.cf/); + assert.doesNotMatch(out, /static:x/); assert.match(out, /check_policy_service unix:private\/quota-status/); assert.match(out, /smtpd_sender_login_maps = pgsql:\/etc\/postfix\/sql\/senders\.cf/); assert.match(out, /tls_server_sni_maps = hash:\/etc\/postfix\/sni_map/); - assert.match(out, /mailhelper_destination_recipient_limit = 1/); + // Helper milter first, OpenDKIM second — the signature covers added headers. + assert.match(out, /submission_milters = unix:mail-helper\/milter\.sock, unix:opendkim\/opendkim\.sock/); + assert.match(out, /submission_recipient_restrictions = permit_sasl_authenticated, reject/); assert.doesNotMatch(out, /relayhost =/); }); @@ -75,16 +82,17 @@ test('postfix main.cf: relayhost with credentials', async () => { assert.match(out, /smtp_sasl_password_maps = texthash:\/etc\/postfix\/sasl_passwd/); }); -test('postfix master.cf: TLS-only AUTH submission, policy + filter, DKIM on reinject only', async () => { +test('postfix master.cf: TLS-only AUTH submission referencing main.cf policy, no inet filters', async () => { const out = await render('mail/postfix-master.cf.ejs'); assert.match(out, /submission[\s\S]*?smtpd_tls_security_level=encrypt/); assert.match(out, /smtpd_sender_restrictions=reject_sender_login_mismatch/); - assert.match(out, /check_policy_service inet:127\.0\.0\.1:10027/); - assert.match(out, /content_filter=mailhelper:\[127\.0\.0\.1\]:10025/); + assert.match(out, /-o smtpd_milters=\$submission_milters/); + assert.match(out, /-o smtpd_recipient_restrictions=\$submission_recipient_restrictions/); assert.match(out, /smtps[\s\S]*?smtpd_tls_wrappermode=yes/); - assert.match(out, /127\.0\.0\.1:10026 inet[\s\S]*?smtpd_milters=inet:127\.0\.0\.1:8891/); - // The milter must NOT run on the submission listeners. - assert.doesNotMatch(out, /submission[\s\S]{0,600}smtpd_milters/); + // The milter architecture removed the content filter, reinjection + // listener, and policy service — no localhost inet plumbing remains. + assert.doesNotMatch(out, /127\.0\.0\.1:\d+/); + assert.doesNotMatch(out, /content_filter/); }); test('dovecot.conf: SQL auth with the can_send smtp gate, per-domain certs, postfix sockets', async () => { @@ -99,22 +107,25 @@ test('dovecot.conf: SQL auth with the can_send smtp gate, per-domain certs, post assert.match(out, /auth_allow_cleartext = no/); }); -test('opendkim: signs on localhost socket and oversigns List-Unsubscribe', async () => { +test('opendkim: unix socket in the postfix chroot, state-dir tables, oversigned List-Unsubscribe', async () => { const conf = await render('mail/opendkim.conf.ejs'); - assert.match(conf, /Socket\s+inet:8891@127\.0\.0\.1/); + assert.match(conf, /Socket\s+local:\/var\/spool\/postfix\/opendkim\/opendkim\.sock/); assert.match(conf, /OversignHeaders\s+From,List-Unsubscribe,List-Unsubscribe-Post/); + assert.match(conf, /KeyTable\s+\/var\/lib\/opensource-agent\/mail\/opendkim-keytable/); const keytable = await render('mail/opendkim-keytable.ejs'); - assert.match(keytable, /os20261008\._domainkey\.example\.com example\.com:os20261008:\/etc\/dkimkeys\/example\.com\.private/); + assert.match(keytable, /os20261008\._domainkey\.example\.com example\.com:os20261008:\/var\/lib\/opensource-agent\/mail\/dkim\/example\.com\.private/); const signing = await render('mail/opendkim-signingtable.ejs'); assert.match(signing, /\*@example\.com os20261008\._domainkey\.example\.com/); }); -test('vdomains and sni_map list the right domains', async () => { +test('sql maps, vdomains, and sni_map list the right entries', async () => { assert.match(await render('mail/postfix-vdomains.ejs'), /^example\.com ok$/m); assert.match( await render('mail/postfix-sni-map.ejs'), /^example\.com \/etc\/ssl\/private\/example\.com\.key \/etc\/ssl\/certs\/example\.com\.crt$/m, ); + assert.match(await render('mail/postfix-sql-mailboxes.cf.ejs'), /SELECT 1 FROM mail_recipients_v WHERE recipient = '%s'/); + assert.match(await render('mail/postfix-sql-senders.cf.ejs'), /SELECT login FROM mail_senders_v WHERE sender = '%s'/); }); diff --git a/compose.yml b/compose.yml index d25c45de..84e06a1e 100644 --- a/compose.yml +++ b/compose.yml @@ -42,6 +42,7 @@ services: - | npm -C create-a-container ci --no-audit --no-fund make -C agent build + make -C mail-helper build # Watches the React client and rebuilds the production bundle on change. The # Manager LXC inside Proxmox serves create-a-container/client/dist statically diff --git a/images/manager/Dockerfile b/images/manager/Dockerfile index 15f1416b..199823a1 100644 --- a/images/manager/Dockerfile +++ b/images/manager/Dockerfile @@ -51,7 +51,7 @@ COPY images/manager/container-creator-init.service /etc/systemd/system/container # enable the init service. RUN --mount=from=builder,source=/dist,target=/dist \ apt-get update && \ - apt-get install -y /dist/opensource-docs_*.deb /dist/opensource-mcp_*.deb /dist/opensource-server_*.deb && \ + apt-get install -y /dist/opensource-docs_*.deb /dist/opensource-mcp_*.deb /dist/opensource-server_*.deb /dist/opensource-mail-helper_*.deb && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ systemctl enable container-creator-init.service diff --git a/mail-helper/.fpm b/mail-helper/.fpm new file mode 100644 index 00000000..23737e35 --- /dev/null +++ b/mail-helper/.fpm @@ -0,0 +1,11 @@ +--name opensource-mail-helper +--architecture all +--license Apache-2.0 +--maintainer "Medical Informatics Engineering " +--vendor "Medical Informatics Engineering" +--url "https://github.com/mieweb/opensource-server" +--category mail +--description "MIE opensource-server mail helper: a Postfix milter that enforces one-click unsubscribe suppressions at RCPT and injects RFC 8058 List-Unsubscribe headers before DKIM signing. Managed by the opensource-agent mail service group." +--depends nodejs +--deb-no-default-config-files +--after-install contrib/postinstall.sh diff --git a/mail-helper/.gitignore b/mail-helper/.gitignore new file mode 100644 index 00000000..0e59bd6c --- /dev/null +++ b/mail-helper/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +config.local.json +src/unsubscribe-token.js +.pkg/ +*.deb +*.rpm +*.apk diff --git a/mail-helper/Makefile b/mail-helper/Makefile new file mode 100644 index 00000000..d9e34565 --- /dev/null +++ b/mail-helper/Makefile @@ -0,0 +1,73 @@ +.DEFAULT_GOAL := help + +PREFIX ?= /opt/opensource-server +DESTDIR ?= / +DESTBIN := $(DESTDIR)$(PREFIX)/mail-helper +UNIT_DIR := $(DESTDIR)/usr/lib/systemd/system + +# Staging directory for packaging +STAGE := $(CURDIR)/.pkg/buildroot + +INSTALL := install +INSTALL_DATA := $(INSTALL) -m 0644 + +# Canonical unsubscribe-token implementation lives in the manager; it is +# copied here at build time (gitignored) so the two stay in sync by +# construction instead of by hand. +TOKEN_SRC := ../create-a-container/utils/unsubscribe-token.js + +.PHONY: help deps build dev test install deb rpm apk package clean + +help: + @echo "mail-helper — builds the opensource-mail-helper package." + @echo "" + @echo "Targets:" + @echo " deps install dependencies (npm ci) and sync shared sources" + @echo " build prepare the runtime tree" + @echo " test run tests" + @echo " install stage files into DESTDIR (default /)" + @echo " deb build the .deb package" + @echo " rpm build the .rpm package" + @echo " apk build the .apk package" + @echo " clean remove build output, staging dir and built packages" + @echo " help show this message" + @echo "" + @echo "Variables: PREFIX (default /opt/opensource-server), DESTDIR (default /)." + +deps: + npm ci + cp $(TOKEN_SRC) src/unsubscribe-token.js + +build: deps + npm prune --omit=dev + +dev: build + +test: deps + npm test + +install: build + $(INSTALL) -d $(DESTBIN)/src + $(INSTALL_DATA) package.json config.json $(DESTBIN)/ + $(INSTALL_DATA) src/*.js $(DESTBIN)/src/ + cp -a node_modules $(DESTBIN)/ + $(INSTALL) -d $(UNIT_DIR) + $(INSTALL_DATA) contrib/systemd/opensource-mail-helper.service $(UNIT_DIR)/ + +PACKAGER ?= deb +package: + rm -rf $(STAGE) + $(MAKE) install DESTDIR=$(STAGE) PREFIX=$(PREFIX) + fpm -s dir -t $(PACKAGER) -v "$$(../package-version $(PACKAGER))" \ + -C $(STAGE) -p $(CURDIR) -f + +deb: + $(MAKE) package PACKAGER=deb +rpm: + $(MAKE) package PACKAGER=rpm +apk: + $(MAKE) package PACKAGER=apk + +clean: + rm -rf $(STAGE) node_modules + rm -f src/unsubscribe-token.js *.deb *.rpm *.apk diff --git a/mail-helper/README.md b/mail-helper/README.md new file mode 100644 index 00000000..c513e83b --- /dev/null +++ b/mail-helper/README.md @@ -0,0 +1,28 @@ +# opensource-mail-helper + +A Postfix [milter](https://www.postfix.org/MILTER_README/) for the mail service (issue #67). It runs pre-queue on the submission ports, listening on a unix socket inside the Postfix chroot, ordered before OpenDKIM so the DKIM signature covers the headers it adds: + +- **RCPT**: rejects recipients who one-click-unsubscribed from the authenticated sender (`mail_suppressions_v`) with a synchronous `5.7.1`; database outages defer (fail closed). +- **End of message**: strips client-forged `List-Unsubscribe` headers and, for single-recipient submissions, injects RFC 8058 one-click headers with an encrypted per-recipient token. + +## Layout + +| Path | Purpose | +|------|---------| +| `src/index.js` | Daemon entry — milter server wiring | +| `src/filter.js` | Decision logic (unit-tested, no milter plumbing) | +| `src/db.js` | Read-only access to the mail views as `mail_postfix` | +| `src/config.js` | `config.json` + `config.local.json` overlay loading | +| `src/unsubscribe-token.js` | Copied from `create-a-container/utils/unsubscribe-token.js` by `make deps` (gitignored) — single source of truth for the token format | +| `config.json` | Committed defaults (not a dpkg conffile) | +| `config.local.json` | Environment overlay, written by the agent's mail service group (gitignored) | + +## Build and test + +```bash +make deps # npm ci + sync shared sources +make test # node --test +make deb # package (installs to /opt/opensource-server/mail-helper) +``` + +The package is suggested by `opensource-agent`; the agent enables/starts the unit only on the host holding the mail-host claim. A [self-managed MTA](../mie-opensource-landing/docs/admins/email.md) can run the daemon standalone — it only needs the DB views and the unsubscribe keys in `config.local.json`. diff --git a/mail-helper/config.json b/mail-helper/config.json new file mode 100644 index 00000000..61880911 --- /dev/null +++ b/mail-helper/config.json @@ -0,0 +1,15 @@ +{ + "socket": "/var/spool/postfix/mail-helper/milter.sock", + "db": { + "dialect": "postgres", + "host": null, + "port": null, + "database": null, + "user": "mail_postfix", + "password": null + }, + "unsubscribe": { + "baseUrl": null, + "keys": [] + } +} diff --git a/mail-helper/contrib/postinstall.sh b/mail-helper/contrib/postinstall.sh new file mode 100644 index 00000000..46aeac66 --- /dev/null +++ b/mail-helper/contrib/postinstall.sh @@ -0,0 +1,19 @@ +#!/bin/sh +set -e + +# System users for the mail stack. vmail owns the Maildirs (Dovecot +# mail_uid/mail_gid); opensource-mail runs the milter daemon. The agent's +# mail service group manages directories, config, and the unit itself. +if ! id -u vmail >/dev/null 2>&1; then + useradd --system --home-dir /var/vmail --create-home --shell /usr/sbin/nologin vmail +fi +if ! id -u opensource-mail >/dev/null 2>&1; then + useradd --system --no-create-home --shell /usr/sbin/nologin opensource-mail +fi + +command -v systemctl >/dev/null 2>&1 || exit 0 +if [ -d /run/systemd/system ]; then + systemctl daemon-reload +fi + +exit 0 diff --git a/mail-helper/contrib/systemd/opensource-mail-helper.service b/mail-helper/contrib/systemd/opensource-mail-helper.service new file mode 100644 index 00000000..8a6c03f0 --- /dev/null +++ b/mail-helper/contrib/systemd/opensource-mail-helper.service @@ -0,0 +1,23 @@ +[Unit] +Description=MIE opensource-server mail helper (unsubscribe milter) +Documentation=https://github.com/mieweb/opensource-server/issues/67 +After=network.target +PartOf=postfix.service + +[Service] +Type=simple +User=opensource-mail +# postfix group so the chrooted smtpd can connect to the milter socket. +Group=postfix +WorkingDirectory=/opt/opensource-server/mail-helper +ExecStart=/usr/bin/node /opt/opensource-server/mail-helper/src/index.js +Restart=on-failure +RestartSec=5 +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +ReadWritePaths=/var/spool/postfix/mail-helper + +[Install] +WantedBy=multi-user.target diff --git a/mail-helper/package-lock.json b/mail-helper/package-lock.json new file mode 100644 index 00000000..9de070a0 --- /dev/null +++ b/mail-helper/package-lock.json @@ -0,0 +1,720 @@ +{ + "name": "opensource-mail-helper", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "opensource-mail-helper", + "version": "0.0.0", + "license": "Apache-2.0", + "dependencies": { + "milter": "^3.0.0", + "mysql2": "^3.15.3", + "pg": "^8.16.3" + }, + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/@hapi/address": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@hapi/address/-/address-5.1.1.tgz", + "integrity": "sha512-A+po2d/dVoY7cYajycYI43ZbYMXukuopIsqCjh5QzsBCipDtdofHntljDlpccMjIfTy6UOkg+5KPriwYch2bXA==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "^11.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@hapi/formula": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@hapi/formula/-/formula-3.0.2.tgz", + "integrity": "sha512-hY5YPNXzw1He7s0iqkRQi+uMGh383CGdyyIGYtB+W5N3KHPXoqychklvHhKCC9M3Xtv0OCs/IHw+r4dcHtBYWw==", + "license": "BSD-3-Clause" + }, + "node_modules/@hapi/hoek": { + "version": "11.0.7", + "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-11.0.7.tgz", + "integrity": "sha512-HV5undWkKzcB4RZUusqOpcgxOaq6VOAH7zhhIr2g3G8NF/MlFO75SjOr2NfuSx0Mh40+1FqCkagKLJRykUWoFQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@hapi/pinpoint": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@hapi/pinpoint/-/pinpoint-2.0.1.tgz", + "integrity": "sha512-EKQmr16tM8s16vTT3cA5L0kZZcTMU5DUOZTuvpnY738m+jyP3JIUj+Mm1xc1rsLkGBQ/gVnfKYPwOmPg1tUR4Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@hapi/tlds": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@hapi/tlds/-/tlds-1.1.7.tgz", + "integrity": "sha512-MgNjRwy9Ti92yVAixLmDc8dd1bJIKwO9qlWCfFQRwRmUEDPQHYn4G6hwPFvFGUTzAa0FsS+inMjLin7GnyBRhA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@hapi/topo": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@hapi/topo/-/topo-6.0.2.tgz", + "integrity": "sha512-KR3rD5inZbGMrHmgPxsJ9dbi6zEK+C3ZwUwTa+eMwWLz7oijWUTWD2pMSNNYJAU6Qq+65NkxXjqHr/7LM2Xkqg==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/hoek": "^11.0.2" + } + }, + "node_modules/@nodable/entities": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.1.0.tgz", + "integrity": "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@peculiar/asn1-schema": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.8.0.tgz", + "integrity": "sha512-7YT0U/ze0tF2QOBbE15gKZwy5tvgGyLRiRHLzhlbOpf7BT032oBSd0haZqXn5W6l26WLlu3dyxzjM+2638/z2Q==", + "license": "MIT", + "dependencies": { + "@peculiar/utils": "^2.0.2", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-x509": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509/-/asn1-x509-2.8.0.tgz", + "integrity": "sha512-N0CMuhWUzsWEVq6F1q9X6+VKUnWzSW+cSVg+aPaGGwDdbFoFWTYgin5MHwXgpWd6y9COMBxnfy/Qc+Xc7F0Zwg==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.8.0", + "@peculiar/utils": "^2.0.2", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-x509-logotype": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509-logotype/-/asn1-x509-logotype-2.8.0.tgz", + "integrity": "sha512-79px79a3YB8OnzVh+hbUlJKNaWymsf+im26AsgHnOhZ1859qVR4P0C4JBq/52dF8LXzNfvioqIOew1Xg7l2nDA==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.8.0", + "@peculiar/asn1-x509": "^2.8.0", + "asn1js": "^3.0.10", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@peculiar/utils/-/utils-2.0.3.tgz", + "integrity": "sha512-+oL3HPFRIZ1St2K50lWCXiioIgSoxzz7R1J3uF6neO2yl1sgmpgY6XXJH4BdpoDkMWznQTeYF6oWNDZLCdQ4eQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/@postalsys/vmc": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@postalsys/vmc/-/vmc-1.1.5.tgz", + "integrity": "sha512-WwleGWC51o/VWQFTId6xMiO9aqkIVCM7ODy6UWfzFoQ6UzXewsNrm3ANa4HbBcyYaTK6YPvp4Gt0U+1cXLTbzw==", + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "2.8.0", + "@peculiar/asn1-x509": "2.8.0", + "@peculiar/asn1-x509-logotype": "2.8.0" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "26.6.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.5.tgz", + "integrity": "sha512-uMzsv9stDdofIZBx4UGh4Z9ZgXnGC9seRDscilBcTHAeOFk67MZg/k9bWVfO+bk4qZmG+l52hRnLwUiH2LnJTg==", + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/asn1js": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.10.tgz", + "integrity": "sha512-S2s3aOytiKdFRdulw2qPE51MzjzVOisppcVv7jVFR+Kw0kxwvFrDcYA0h7Ndqbmj0HkMIXYWaoj7fli8kgx1eg==", + "license": "BSD-3-Clause", + "dependencies": { + "pvtsutils": "^1.3.6", + "pvutils": "^1.1.5", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/aws-ssl-profiles": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/aws-ssl-profiles/-/aws-ssl-profiles-1.1.2.tgz", + "integrity": "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/commander": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-15.0.0.tgz", + "integrity": "sha512-z67u4ZhzCL/Tydu1lJARtEZYWbWaN7oYLHbsuzocr6y4N6WZAagG3RQ4FW61V1/0+jImpj293XfrcYnd1qxtPg==", + "license": "MIT", + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/encoding-japanese": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/encoding-japanese/-/encoding-japanese-2.2.0.tgz", + "integrity": "sha512-EuJWwlHPZ1LbADuKTClvHtwbaFn4rOD+dRAbWysqEOXRc2Uui0hJInNJrsdH0c+OhJA4nrCBdSkW4DD5YxAo6A==", + "license": "MIT", + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", + "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.1", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/generate-function": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.3.1.tgz", + "integrity": "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==", + "license": "MIT", + "dependencies": { + "is-property": "^1.0.2" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ipaddr.js": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.5.0.tgz", + "integrity": "sha512-aq+t5NAc+cS6rZQQVWC2x98CPqGtKKTMDd4Gaodv0wShnItdKg/51djkGJ1hqH+Oy0ivDftCbSLCQob8zso01w==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/is-property": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", + "integrity": "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==", + "license": "MIT" + }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/joi": { + "version": "18.2.3", + "resolved": "https://registry.npmjs.org/joi/-/joi-18.2.3.tgz", + "integrity": "sha512-N5A3KTWQpPWT4ExxxPlUx7WmykGXRzhNidWhV41d6Abu9YfI2NyWCJuxdPnslJCPWtbRpSVOWSnSS6GakLM/Rg==", + "license": "BSD-3-Clause", + "dependencies": { + "@hapi/address": "^5.1.1", + "@hapi/formula": "^3.0.2", + "@hapi/hoek": "^11.0.7", + "@hapi/pinpoint": "^2.0.1", + "@hapi/tlds": "^1.1.1", + "@hapi/topo": "^6.0.2", + "@standard-schema/spec": "^1.1.0" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/libbase64": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-1.3.0.tgz", + "integrity": "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg==", + "license": "MIT" + }, + "node_modules/libmime": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.4.1.tgz", + "integrity": "sha512-0wHGhsofo9IdQPenr3BBHXuxcwMq4atFUTsZ9Ogc1OvI5h4rUdDIrBQEN9JHjCXfDMrE59LUMJWsTD82wTYk8A==", + "license": "MIT", + "dependencies": { + "encoding-japanese": "2.2.0", + "iconv-lite": "0.7.3", + "libbase64": "1.3.0", + "libqp": "2.1.1" + } + }, + "node_modules/libqp": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/libqp/-/libqp-2.1.1.tgz", + "integrity": "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow==", + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, + "node_modules/lru.min": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/lru.min/-/lru.min-1.1.5.tgz", + "integrity": "sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==", + "license": "MIT", + "engines": { + "bun": ">=1.0.0", + "deno": ">=1.30.0", + "node": ">=8.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wellwelwel" + } + }, + "node_modules/mailauth": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/mailauth/-/mailauth-5.0.3.tgz", + "integrity": "sha512-eAOpTwU2zKDCOusM6AIxl9ZZqkUWUoyoE+Ax3HzqdPs6R8xL/DqqbjpSFB14DUuBywKfrBLCa7qOE9++1V3h/Q==", + "license": "MIT", + "dependencies": { + "@postalsys/vmc": "1.1.5", + "commander": "15.0.0", + "fast-xml-parser": "5.10.1", + "ipaddr.js": "2.5.0", + "joi": "18.2.3", + "libmime": "5.4.1", + "nodemailer": "9.0.4", + "punycode.js": "2.3.1", + "tldts": "7.4.10", + "undici": "8.10.0" + }, + "bin": { + "mailauth": "bin/mailauth.js" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/milter": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/milter/-/milter-3.0.0.tgz", + "integrity": "sha512-MqwG/W2UFS6wpRSoB44ItjlRIA8vUrcJe+j04WPSUvmmBwx+vivffBW3OCDgpC1/tKgRPCarW7JqPJEcBUD6tg==", + "license": "MIT", + "dependencies": { + "mailauth": "^5.0.3" + }, + "engines": { + "node": ">=22.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, + "node_modules/mysql2": { + "version": "3.24.5", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.24.5.tgz", + "integrity": "sha512-X6Ujsr2QSkkLpkQGjxzpKRAPn9nu4axpR63ntBzquFVEvPOArgbUQ1sJjFKI7hnaYtiVZxa17Z7q18KSubW0IQ==", + "license": "MIT", + "dependencies": { + "aws-ssl-profiles": "^1.1.2", + "generate-function": "^2.3.1", + "iconv-lite": "^0.7.3", + "long": "^5.3.2", + "lru.min": "^1.1.4", + "named-placeholders": "^1.1.6", + "sql-escaper": "^1.5.1" + }, + "engines": { + "node": ">= 8.0" + }, + "peerDependencies": { + "@types/node": ">= 8" + } + }, + "node_modules/named-placeholders": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/named-placeholders/-/named-placeholders-1.1.6.tgz", + "integrity": "sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==", + "license": "MIT", + "dependencies": { + "lru.min": "^1.1.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/nodemailer": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-9.0.4.tgz", + "integrity": "sha512-LmJNRVRtfSCULxcZpy0Cpg4WWenlUZ9+zbmTO+S7v9wD6XreYLjXRFtDjtV/4F0HT5p1GyZfA0Ux/myxHb18CQ==", + "license": "MIT-0", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/pg": { + "version": "8.23.1", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.1.tgz", + "integrity": "sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==", + "license": "MIT", + "dependencies": { + "pg-connection-string": "^2.14.1", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.16.1", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.4.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz", + "integrity": "sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.1.tgz", + "integrity": "sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.16.1.tgz", + "integrity": "sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/punycode.js": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode.js/-/punycode.js-2.3.1.tgz", + "integrity": "sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/pvtsutils": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/pvtsutils/-/pvtsutils-1.3.6.tgz", + "integrity": "sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/pvutils": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pvutils/-/pvutils-1.2.0.tgz", + "integrity": "sha512-BbubeCEyTuQjVMakvJQ/Sxbc93F2pwmbsxONT/ZRrwU7Ua38d8unYTwXpTVLAKJ4BDuH9IGztCjQcd/N/39Dvg==", + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/sql-escaper": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/sql-escaper/-/sql-escaper-1.5.2.tgz", + "integrity": "sha512-6CKD38c31SENivxOADeMNLdukOnUxUcflKtzVWzace7Riv1v7cAEym5Cx9Q7gYZ3ezIJI7ZpqecQq8cqNeSSFg==", + "license": "MIT", + "engines": { + "bun": ">=1.0.0", + "deno": ">=2.0.0", + "node": ">=12.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/mysqljs/sql-escaper?sponsor=1" + } + }, + "node_modules/strnum": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.5.1.tgz", + "integrity": "sha512-UM4nTLWzQiDBseVxUGxN0wJzIk0UU89Qi7pKXz9qISo+lhviJikTuikBsK/oFRlpvYHvSdz0TVoLyQ+sEpYwxA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, + "node_modules/tldts": { + "version": "7.4.10", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.10.tgz", + "integrity": "sha512-GgouD1B+sWwvkaEq8vXC15DjQitxbvs12oIXELpconwm+Tg3zfcEv4jgzq3vtKverDXsg3VI8aRgNL2Nra0Iog==", + "license": "MIT", + "dependencies": { + "tldts-core": "^7.4.10" + }, + "bin": { + "tldts": "bin/cli.js" + } + }, + "node_modules/tldts-core": { + "version": "7.4.18", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.18.tgz", + "integrity": "sha512-KI2PKvCmvQI/xD6ypPqkU0i7oxNXR/O58RLRaHR4svsgVUQ2JE/XC8BtMbG6PSehD27I8sZJRcC1ASnCy6lp3g==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/undici": { + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.0.tgz", + "integrity": "sha512-HvltHd7avK13QIw/oLe4qoOLyoVSoafqJ2jYOrtMRBkbYT31eiBQ8O0ehRKZiEZCMEyLFQNIADpgCWC5fALvYQ==", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "license": "MIT", + "peer": true + }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } + } + } +} diff --git a/mail-helper/package.json b/mail-helper/package.json new file mode 100644 index 00000000..e0f43519 --- /dev/null +++ b/mail-helper/package.json @@ -0,0 +1,20 @@ +{ + "name": "opensource-mail-helper", + "version": "0.0.0", + "private": true, + "description": "MIE opensource-server mail helper: a Postfix milter that enforces one-click unsubscribe suppressions at RCPT and injects RFC 8058 List-Unsubscribe headers before DKIM signing.", + "license": "Apache-2.0", + "main": "src/index.js", + "scripts": { + "start": "node src/index.js", + "test": "node --test \"test/**/*.test.js\"" + }, + "dependencies": { + "milter": "^3.0.0", + "mysql2": "^3.15.3", + "pg": "^8.16.3" + }, + "engines": { + "node": ">=20.18.1" + } +} diff --git a/mail-helper/src/config.js b/mail-helper/src/config.js new file mode 100644 index 00000000..400f30b0 --- /dev/null +++ b/mail-helper/src/config.js @@ -0,0 +1,41 @@ +/** + * Config loading: the committed ./config.json ships defaults; the agent (or + * a developer) overlays environment-specific values in ./config.local.json — + * the same split in development and production. MAIL_HELPER_DIR overrides + * the directory for tests and dev stacks. + */ + +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +function readJson(file) { + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (err) { + if (err.code === 'ENOENT') return null; + throw new Error(`mail-helper: cannot read ${file}: ${err.message}`); + } +} + +/** Plain-object deep merge; local values win, arrays replace. */ +function merge(base, overlay) { + if (overlay === null || overlay === undefined) return base; + if (Array.isArray(base) || Array.isArray(overlay) || typeof base !== 'object' || typeof overlay !== 'object') { + return overlay; + } + const out = { ...base }; + for (const [key, value] of Object.entries(overlay)) { + out[key] = merge(base[key], value); + } + return out; +} + +function loadConfig(dir = process.env.MAIL_HELPER_DIR || path.join(__dirname, '..')) { + const defaults = readJson(path.join(dir, 'config.json')); + if (!defaults) throw new Error(`mail-helper: missing config.json in ${dir}`); + return merge(defaults, readJson(path.join(dir, 'config.local.json'))); +} + +module.exports = { loadConfig, merge }; diff --git a/mail-helper/src/db.js b/mail-helper/src/db.js new file mode 100644 index 00000000..08fdf193 --- /dev/null +++ b/mail-helper/src/db.js @@ -0,0 +1,61 @@ +/** + * Thin dialect-agnostic access to the read-only mail views (mail_senders_v, + * mail_suppressions_v) as the mail_postfix role. No host means the local + * unix socket (Postgres peer / MariaDB socket auth — no password). + */ + +'use strict'; + +class Db { + constructor(cfg) { + if (cfg.dialect === 'mysql') { + const mysql = require('mysql2/promise'); + const pool = mysql.createPool({ + ...(cfg.host ? { host: cfg.host, port: cfg.port ?? 3306 } : { socketPath: '/var/run/mysqld/mysqld.sock' }), + user: cfg.user, + ...(cfg.password ? { password: cfg.password } : {}), + database: cfg.database, + connectionLimit: 5, + }); + this.pool = { + async query(sql, params) { + const [rows] = await pool.query(sql.replace(/\$\d+/g, '?'), params); + return { rows }; + }, + }; + } else { + const { Pool } = require('pg'); + this.pool = new Pool({ + ...(cfg.host ? { host: cfg.host, port: cfg.port ?? 5432 } : { host: '/var/run/postgresql' }), + user: cfg.user, + ...(cfg.password ? { password: cfg.password } : {}), + database: cfg.database ?? undefined, + max: 5, + }); + } + } + + /** Sender account info, or null when the address is not a managed sender. */ + async getSender(address) { + const { rows } = await this.pool.query( + 'SELECT account_id, unsubscribe_headers FROM mail_senders_v WHERE sender = $1', + [String(address).toLowerCase()], + ); + if (!rows[0]) return null; + const raw = rows[0].unsubscribe_headers; // boolean on Postgres, 0/1 on MySQL + return { + accountId: String(rows[0].account_id), + unsubscribeHeaders: raw === true || raw === 1 || raw === '1', + }; + } + + async isSuppressed(sender, recipient) { + const { rows } = await this.pool.query( + 'SELECT 1 AS hit FROM mail_suppressions_v WHERE sender = $1 AND recipient = $2', + [String(sender).toLowerCase(), String(recipient).toLowerCase()], + ); + return rows.length > 0; + } +} + +module.exports = { Db }; diff --git a/mail-helper/src/filter.js b/mail-helper/src/filter.js new file mode 100644 index 00000000..88607394 --- /dev/null +++ b/mail-helper/src/filter.js @@ -0,0 +1,62 @@ +/** + * Filter decisions, kept free of milter/server plumbing so they are unit + * testable. The milter runs pre-queue on the submission ports only, where + * reject_sender_login_mismatch already guarantees envelope sender == SASL + * login. + */ + +'use strict'; + +const { createUnsubscribeToken } = require('./unsubscribe-token'); + +/** Envelope address normalization: strip <>, ESMTP params, case. */ +function parseAddress(raw) { + const first = String(raw ?? '').trim().split(/\s+/)[0] ?? ''; + const match = /<([^>]*)>/.exec(first); + return (match ? match[1] : first).trim().toLowerCase(); +} + +/** + * RCPT decision: reject recipients who unsubscribed from this sender. A DB + * failure DEFERS (fail closed) — delivering to someone who opted out would + * break the synchronous suppression guarantee, so Postfix must retry. + * + * @returns {Promise<{action: 'continue'|'reject'|'tempfail', message?: string}>} + */ +async function rcptDecision(db, sender, recipient) { + if (!sender || !recipient) return { action: 'continue' }; + try { + if (await db.isSuppressed(sender, recipient)) { + return { action: 'reject', message: `5.7.1 Recipient has unsubscribed from ${sender}` }; + } + return { action: 'continue' }; + } catch (err) { + return { action: 'tempfail', message: `4.3.0 Suppression lookup unavailable (${err.code ?? 'db error'})` }; + } +} + +/** + * Headers to add at end-of-message. Tokens identify (account, recipient), so + * headers are only added when the transaction has exactly one recipient — + * multi-recipient submissions (rare for app mail) get none. + * + * @returns {Promise<{name: string, value: string}[]>} empty when not applicable + */ +async function unsubscribeHeaders({ db, unsubscribe }, sender, recipients) { + const activeKey = (unsubscribe.keys ?? []).find((k) => k.active); + if (!activeKey || !unsubscribe.baseUrl || recipients.length !== 1) return []; + const account = await db.getSender(sender); + if (!account || !account.unsubscribeHeaders) return []; + const token = createUnsubscribeToken({ accountId: account.accountId, recipient: recipients[0] }, activeKey); + return [ + { name: 'List-Unsubscribe', value: `<${unsubscribe.baseUrl}/u/${token}>` }, + { name: 'List-Unsubscribe-Post', value: 'List-Unsubscribe=One-Click' }, + ]; +} + +/** Header names the helper owns — client-supplied ones are forged and removed. */ +function isUnsubscribeHeader(name) { + return /^list-unsubscribe(-post)?$/i.test(String(name).trim()); +} + +module.exports = { parseAddress, rcptDecision, unsubscribeHeaders, isUnsubscribeHeader }; diff --git a/mail-helper/src/index.js b/mail-helper/src/index.js new file mode 100644 index 00000000..c5227a91 --- /dev/null +++ b/mail-helper/src/index.js @@ -0,0 +1,122 @@ +#!/usr/bin/env node +/** + * opensource-mail-helper: a Postfix milter (issue #67, PR #495 review). + * + * Runs pre-queue on the submission ports (587/465), listening on a unix + * socket inside the Postfix chroot. Ordered BEFORE OpenDKIM in + * submission_milters so the DKIM signature covers the headers added here. + * + * - RCPT: rejects recipients who one-click-unsubscribed from the + * authenticated sender (mail_suppressions_v) with a synchronous 5.7.1; + * DB outages DEFER (fail closed). + * - End of message: strips client-forged List-Unsubscribe headers and, for + * single-recipient submissions, injects RFC 8058 one-click headers with + * an encrypted per-recipient token. + * + * Config: ./config.json (committed defaults) + ./config.local.json overlay + * (written by the agent from the mail-host snapshot; gitignored). A + * self-managed MTA can run this daemon too — it only needs the DB views and + * the unsubscribe keys. + */ + +'use strict'; + +const { Decision, MilterServer, SMFIF } = require('milter'); +const { loadConfig } = require('./config'); +const { Db } = require('./db'); +const { parseAddress, rcptDecision, unsubscribeHeaders, isUnsubscribeHeader } = require('./filter'); + +function main() { + const config = loadConfig(); + const db = new Db(config.db); + + const server = new MilterServer({ + socketPath: config.socket, + // Group postfix (unit Group=) + 0660 lets the chrooted smtpd connect. + chmod: 0o660, + // We never need the body — decisions use only envelope and headers. + collectBody: false, + actions: SMFIF.ADDHDRS | SMFIF.CHGHDRS, + }); + + // Per-transaction state keyed on the milter context. + const transactions = new WeakMap(); + const stateOf = (ctx) => { + let state = transactions.get(ctx); + if (!state) { + state = { sender: '', recipients: [], forged: {} }; + transactions.set(ctx, state); + } + return state; + }; + + server.on('mail', (from, ctx) => { + transactions.delete(ctx); // new transaction on the same connection + stateOf(ctx).sender = parseAddress(from[0]); + return Decision.continue(); + }); + + server.on('rcpt', async (to, ctx) => { + const state = stateOf(ctx); + const address = parseAddress(to[0]); + const decision = await rcptDecision(db, state.sender, address); + if (decision.action === 'reject') { + ctx.replyCode('550', decision.message); + return Decision.reject(); + } + if (decision.action === 'tempfail') { + ctx.replyCode('451', decision.message); + return Decision.tempfail(); + } + state.recipients.push(address); + return Decision.continue(); + }); + + server.on('headerLine', (name, _value, ctx) => { + // Track client-supplied List-Unsubscribe headers (forgeries) by their + // per-name occurrence index so they can be deleted at end of message. + if (isUnsubscribeHeader(name)) { + const state = stateOf(ctx); + state.forged[name] = (state.forged[name] ?? 0) + 1; + } + return Decision.continue(); + }); + + server.on('bodyEnd', async (_body, ctx) => { + const state = stateOf(ctx); + try { + for (const [name, count] of Object.entries(state.forged)) { + for (let index = count; index >= 1; index--) { + ctx.changeHeader(name, index, ''); // empty value deletes + } + } + for (const header of await unsubscribeHeaders({ db, unsubscribe: config.unsubscribe }, state.sender, state.recipients)) { + ctx.addHeader(header.name, header.value); + } + } catch (err) { + // Header injection is best-effort; suppression enforcement already + // happened at RCPT. Never block delivery here. + console.error(`mail-helper: header injection failed: ${err.message}`); + } finally { + transactions.delete(ctx); + } + return Decision.accept(); + }); + + server.on('abort', (ctx) => transactions.delete(ctx)); + server.on('close', (ctx) => transactions.delete(ctx)); + server.on('error', (error, ctx) => { + console.error('mail-helper: milter error', ctx?.id, error); + }); + + return server.listen().then(() => { + console.log(`mail-helper: milter listening on ${config.socket}`); + }); +} + +if (require.main === module) { + main().catch((err) => { + console.error(err.stack ?? String(err)); + process.exit(1); + }); +} diff --git a/mail-helper/test/filter.test.js b/mail-helper/test/filter.test.js new file mode 100644 index 00000000..2d7165b2 --- /dev/null +++ b/mail-helper/test/filter.test.js @@ -0,0 +1,74 @@ +/** + * Filter decision tests (node --test). Run `make deps` first — it copies the + * canonical unsubscribe-token module from create-a-container. + */ + +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert'); +const crypto = require('crypto'); + +const { parseAddress, rcptDecision, unsubscribeHeaders, isUnsubscribeHeader } = require('../src/filter'); + +const key = { kid: 'a1b2c3d4', secret: crypto.randomBytes(32).toString('base64'), active: true }; +const unsubscribe = { baseUrl: 'https://manager.example.com', keys: [key] }; + +const db = { + async getSender(address) { + if (address === 'app@example.com') return { accountId: 'acc-1', unsubscribeHeaders: true }; + if (address === 'quiet@example.com') return { accountId: 'acc-2', unsubscribeHeaders: false }; + return null; + }, + async isSuppressed(sender, recipient) { + return sender === 'app@example.com' && recipient === 'optedout@example.net'; + }, +}; + +test('parseAddress normalizes envelope forms', () => { + assert.equal(parseAddress(''), 'app@example.com'); + assert.equal(parseAddress('app@example.com SIZE=1024'), 'app@example.com'); + assert.equal(parseAddress('<>'), ''); +}); + +test('rcptDecision rejects suppressed recipients with 5.7.1', async () => { + const hit = await rcptDecision(db, 'app@example.com', 'optedout@example.net'); + assert.equal(hit.action, 'reject'); + assert.match(hit.message, /^5\.7\.1 /); + + const miss = await rcptDecision(db, 'app@example.com', 'someone@example.net'); + assert.equal(miss.action, 'continue'); +}); + +test('rcptDecision DEFERS on DB failure (fail closed)', async () => { + const broken = { async isSuppressed() { throw Object.assign(new Error('down'), { code: 'ECONNREFUSED' }); } }; + const decision = await rcptDecision(broken, 'app@example.com', 'x@y.z'); + assert.equal(decision.action, 'tempfail'); + assert.match(decision.message, /^4\.3\.0 /); +}); + +test('unsubscribeHeaders adds tokenized headers for one opted-in recipient', async () => { + const headers = await unsubscribeHeaders({ db, unsubscribe }, 'app@example.com', ['someone@example.net']); + assert.equal(headers.length, 2); + assert.match(headers[0].value, /^$/); + assert.equal(headers[1].value, 'List-Unsubscribe=One-Click'); +}); + +test('unsubscribeHeaders is skipped when not applicable', async () => { + const cases = [ + [{ db, unsubscribe }, 'quiet@example.com', ['r@x.y']], // account opted out + [{ db, unsubscribe }, 'ghost@example.com', ['r@x.y']], // not a managed sender + [{ db, unsubscribe }, 'app@example.com', ['a@x.y', 'b@x.y']], // tokens are per-recipient + [{ db, unsubscribe: { baseUrl: null, keys: [key] } }, 'app@example.com', ['r@x.y']], + [{ db, unsubscribe: { baseUrl: 'https://m', keys: [] } }, 'app@example.com', ['r@x.y']], + ]; + for (const [deps, sender, recipients] of cases) { + assert.deepEqual(await unsubscribeHeaders(deps, sender, recipients), []); + } +}); + +test('isUnsubscribeHeader matches both owned header names, case-insensitive', () => { + assert.ok(isUnsubscribeHeader('List-Unsubscribe')); + assert.ok(isUnsubscribeHeader('list-unsubscribe-post')); + assert.ok(!isUnsubscribeHeader('List-Id')); +}); diff --git a/mie-opensource-landing/docs/admins/email.md b/mie-opensource-landing/docs/admins/email.md index 11d78b8c..2a48b362 100644 --- a/mie-opensource-landing/docs/admins/email.md +++ b/mie-opensource-landing/docs/admins/email.md @@ -29,22 +29,23 @@ Settings → Mail service: **B. Another agent host.** -1. Install the suggested packages: `apt install postfix postfix-pgsql dovecot-core dovecot-imapd dovecot-lmtpd dovecot-pgsql opendkim` (use the `-mysql` variants on MySQL/MariaDB). +1. Install the suggested packages: `apt install opensource-mail-helper postfix postfix-pgsql dovecot-core dovecot-imapd dovecot-lmtpd dovecot-pgsql opendkim` (use the `-mysql` variants on MySQL/MariaDB). 2. Install the per-domain certs under `/etc/ssl` (same layout as nginx). -3. Set the **Mail DB host** setting, and put `MAIL_POSTFIX_DB_PASSWORD` / `MAIL_DOVECOT_DB_PASSWORD` (from `/etc/default/container-creator` on the manager, created by `bin/setup-mail-db-roles.sh`) in the agent's `/etc/environment`. -4. Remove `mail` from the manager's `AGENT_SERVICES`, add it on the new host (see [Deploying Agents](deploying-agents.md)). It takes the claim. -5. `rsync /var/vmail` from the old host. -6. Repoint the port forwards (25, 143, 465, 587, 993) and re-run Check DNS on each domain — the mail IP follows the host's site automatically. +3. Set the **Mail DB host** setting, give the `mail_postfix`/`mail_dovecot` roles passwords (`ALTER ROLE mail_postfix WITH PASSWORD '...'`), and put `MAIL_POSTFIX_DB_PASSWORD` / `MAIL_DOVECOT_DB_PASSWORD` in the agent's `/etc/environment`. +4. Make sure the agent checks in over HTTPS — the manager refuses to send the mail snapshot (DKIM keys, relay credentials) over plaintext (`insecure_transport`). +5. Remove `mail` from the manager's `AGENT_SERVICES`, add it on the new host (see [Deploying Agents](deploying-agents.md)). It takes the claim. +6. `rsync /var/vmail` from the old host. +7. Repoint the port forwards (25, 143, 465, 587, 993) and re-run Check DNS on each domain — the mail IP follows the host's site automatically. **C. Self-managed MTA (no agent runs `mail`).** 1. Set **Self-managed site id** to the site whose `externalIp` sends mail. -2. Point Dovecot at `mail_accounts_v` and Postfix at `mail_senders_v`, with Dovecot's `quota-status` guarding RCPT. The views are a documented, stable contract — see [Database Schema](../developers/database-schema.md#mail-views). -3. Export DKIM keys per domain (`GET /api/v1/external-domains/{id}/mail/dkim` is logged) and use the agent templates under `agent/templates/mail/` as a reference. Run `opensource-mail-helper` yourself for unsubscribe headers, or skip them. +2. Point Dovecot at `mail_accounts_v` and Postfix at `mail_senders_v` + `mail_recipients_v`, with Dovecot's `quota-status` guarding RCPT. The views are a documented, stable contract — see [Database Schema](../developers/database-schema.md#mail-views). +3. Export DKIM keys per domain (`GET /api/v1/external-domains/{id}/mail/dkim` — admin-only, audited in the manager log) and use the agent templates under `agent/templates/mail/` as a reference. Run the `opensource-mail-helper` milter yourself for suppressions + unsubscribe headers (it only needs the DB views and the unsubscribe keys in its `config.local.json`), or skip them. ## DB Roles -`create-a-container/bin/setup-mail-db-roles.sh` creates the read-only `mail_dovecot` and `mail_postfix` users and grants SELECT on the views. It runs automatically on the manager (first boot + package upgrades); run it with `--print-sql` for a DBA, `--rotate` to regenerate the passwords in `/etc/default/container-creator`. +Migration `20261012000000` creates the read-only `mail_dovecot` and `mail_postfix` roles and grants SELECT on the views (the manager grants its app user `CREATEROLE` at first boot; elsewhere the migration logs the SQL for a DBA). Local mail hosts authenticate over the socket (Postgres peer via `pg_ident`); remote hosts need operator-set passwords (`ALTER ROLE … WITH PASSWORD`). ## Backups diff --git a/mie-opensource-landing/docs/developers/agent.md b/mie-opensource-landing/docs/developers/agent.md index 116a4b24..8dd6276e 100644 --- a/mie-opensource-landing/docs/developers/agent.md +++ b/mie-opensource-landing/docs/developers/agent.md @@ -104,14 +104,14 @@ Services are organized in groups enabled via `AGENT_SERVICES`. The check-in repo |-------|---------|-------|------|--------| | nginx | nginx | `/etc/nginx/nginx.conf` | `nginx -t` | `systemctl reload-or-restart nginx` | | dnsmasq | dnsmasq | `/etc/dnsmasq.conf`, `/var/lib/dnsmasq/{dhcp-hosts,hosts,dhcp-opts,servers}` | `dnsmasq --test` | restart when `/etc/dnsmasq.conf` changed, otherwise SIGHUP | -| mail | opendkim | `/etc/opendkim.conf`, `/etc/opendkim/{keytable,signingtable}`, `/etc/dkimkeys/*.private` | `opendkim -n` | restart | +| mail | opendkim | `/etc/opendkim.conf`, `$STATE_DIRECTORY/mail/{opendkim-keytable,opendkim-signingtable,dkim/*.private}` | `opendkim -n` | restart | | mail | dovecot | `/etc/dovecot/dovecot.conf` | `doveconf -n` | reload + `doveadm auth cache flush` | -| mail | postfix (status: `postfix@-`) | `/etc/postfix/{main.cf,master.cf,vdomains,sni_map,sql/senders.cf}` | `postfix check` (after `postmap -F` postWrite) | reload-or-restart | -| mail | opensource-mail-helper | `/etc/opensource-server/mail-helper.json` | — | restart | +| mail | postfix (status: `postfix@-`) | `/etc/postfix/{main.cf,master.cf,vdomains,sni_map,sql/senders.cf,sql/mailboxes.cf}` | `postfix check` (after `postmap -F` postWrite) | reload-or-restart | +| mail | opensource-mail-helper | `/opt/opensource-server/mail-helper/config.local.json` | — | restart | -The mail group renders only on the agent holding the mail-host claim (the `mail` section of the snapshot). Taking the claim creates the `vmail`/`opensource-mail` users and enables + starts the units; losing it (or removing `mail` from `AGENT_SERVICES`) stops and disables them, leaving configs and `/var/vmail` in place. Maildirs of deleted accounts are garbage-collected after 30 days. +The mail group renders only on the agent holding the mail-host claim (the `mail` section of the snapshot). The helper is its own component/package ([mail-helper/](https://github.com/mieweb/opensource-server/tree/main/mail-helper)) — a Postfix milter on a unix socket inside the Postfix chroot, ordered before OpenDKIM so the DKIM signature covers the List-Unsubscribe headers it injects. Taking the claim provisions directories (system users come from the helper package) and enables + starts the units; losing it — or removing `mail` from `AGENT_SERVICES` — stops and disables them, leaving configs and `/var/vmail` in place. Maildirs of deleted accounts are garbage-collected after 30 days. -Apply flow per service: render templates, skip if nothing changed, stage new files, run post-write commands, run the test command, roll back on failure, then reload. Failures are reported as `lastApply: "failure"` at the next check-in. +Apply flow per service: render templates, reconcile file mode/owner even when contents are unchanged, stage new files, run post-write commands, run the test command, roll back on failure (re-running post-write commands against the restored sources), then reload. Failures are reported as `lastApply: "failure"` at the next check-in. To add a managed service, add a `ManagedService` entry in [`agent/src/apply.ts`](https://github.com/mieweb/opensource-server/blob/main/agent/src/apply.ts) (or `agent/src/mail.ts` for the mail group) plus its template(s) under `agent/templates/`, and extend the config snapshot in `create-a-container/utils/agent-config.js`. diff --git a/mie-opensource-landing/docs/developers/database-schema.md b/mie-opensource-landing/docs/developers/database-schema.md index a00d5691..fe40c158 100644 --- a/mie-opensource-landing/docs/developers/database-schema.md +++ b/mie-opensource-landing/docs/developers/database-schema.md @@ -294,9 +294,10 @@ The dialect-aware views created by migration `20261008000006` (SQL built in `cre |------|---------|----------| | `mail_accounts_v` | `address`, `password` (`{ARGON2ID}`-prefixed), `home`, `quota_bytes`, `can_send`, `can_receive` — enabled accounts only | Dovecot passdb/userdb (`mail_dovecot` role) | | `mail_senders_v` | `sender`, `login`, `account_id`, `unsubscribe_headers` — can-send accounts | Postfix `smtpd_sender_login_maps` + opensource-mail-helper (`mail_postfix` role) | -| `mail_suppressions_v` | `sender`, `recipient` | Submission policy service (`mail_postfix` role) | +| `mail_recipients_v` | `recipient` — deliverable addresses on can-receive domains | Postfix `virtual_mailbox_maps` (unknown recipients rejected at RCPT) | +| `mail_suppressions_v` | `sender`, `recipient` | opensource-mail-helper milter — reject at RCPT (`mail_postfix` role) | -`bin/setup-mail-db-roles.sh` creates the two read-only roles and grants; the views migration also grants best-effort so either ordering converges. +Migration `20261012000000` creates the two read-only roles and grants; the views migration also grants best-effort so either ordering converges. ## User Management Models From 2100142edfd247f819e6756526834d4f3dd50384 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Tue, 13 Oct 2026 13:12:38 -0700 Subject: [PATCH 16/19] agent: only stop mail units this agent started (PR #495) Dev-stack testing caught stopMailUnitsIfRunning disabling a host's stock postfix that the agent never managed. state.json now tracks mailManaged (set when the holder enables the units, cleared on shutdown), and both shutdown paths - losing the claim and removing 'mail' from AGENT_SERVICES - are gated on it, so an unrelated postfix/dovecot on the host is never touched. --- agent/src/index.ts | 21 ++++++++++++++------- agent/src/mail.ts | 19 ++++++++++++------- agent/src/state.ts | 13 ++++++++++++- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/agent/src/index.ts b/agent/src/index.ts index 8b0f7bf0..dde4a33d 100644 --- a/agent/src/index.ts +++ b/agent/src/index.ts @@ -63,16 +63,23 @@ async function buildCheckinBody(cfg: AgentConfig, state: State): Promise { + * is NOT enabled — removing `mail` from AGENT_SERVICES must still stop a + * mail stack this agent started (and ONLY one it started: state.mailManaged + * guards against touching a host's own unrelated postfix/dovecot). + * Failures are logged, not fatal — the next timer run retries. */ +async function syncMailLifecycle(cfg: AgentConfig, config: SiteConfig, state: State): Promise { try { if (cfg.services.includes('mail')) { - await syncMailUnits(config); + const mailManaged = await syncMailUnits(config, state.mailManaged); + if (mailManaged !== state.mailManaged) { + state.mailManaged = mailManaged; + state.save(); + } if (config.mail) gcMailboxes(config.mail); - } else { + } else if (state.mailManaged) { await stopMailUnitsIfRunning(); + state.mailManaged = false; + state.save(); } } catch (err) { log.error(`mail: unit lifecycle sync failed: ${(err as Error).message}`); @@ -115,7 +122,7 @@ async function main(): Promise { for (const svc of enabledServices(cfg)) { state.lastApply[svc.unit] = await applyService(svc, result.config, cfg); } - await syncMailLifecycle(cfg, result.config); + await syncMailLifecycle(cfg, result.config, state); // Ensure volume directories exist from the new snapshot; stash results in // persistent state so they are reported on the next check-in and survive a diff --git a/agent/src/mail.ts b/agent/src/mail.ts index aab43961..b9da8ced 100644 --- a/agent/src/mail.ts +++ b/agent/src/mail.ts @@ -301,21 +301,26 @@ export function ensureMailProvisioned(agent: AgentConfig): void { } /** Enable+start the mail units while holding the claim; stop+disable them - * otherwise (configs and /var/vmail are left in place). */ -export async function syncMailUnits(config: SiteConfig): Promise { + * otherwise (configs and /var/vmail are left in place). The shutdown path + * only runs when THIS agent started the units (state.mailManaged) — a + * host's own unrelated postfix/dovecot must never be touched. Returns the + * new mailManaged value for the caller to persist. */ +export async function syncMailUnits(config: SiteConfig, mailManaged: boolean): Promise { if (config.mail) { await enableAndStartServices(MAIL_UNITS); - return; + return true; } + if (!mailManaged) return false; if (config.mailStatus) { - log.info(`mail: not the mail host (${config.mailStatus}); ensuring units are stopped`); + log.info(`mail: not the mail host (${config.mailStatus}); stopping the units this agent started`); } await stopAndDisableServices(MAIL_UNITS); + return false; } -/** Stop the stack when the `mail` group itself is disabled — but only when - * something is actually running, so plain nginx/dnsmasq agents don't issue - * no-op unit operations every 30 seconds. */ +/** Stop the stack when the `mail` group itself is disabled — again only + * when this agent started it, and only when something is actually running, + * so nothing is churned on agents that never ran mail. */ export async function stopMailUnitsIfRunning(): Promise { const states = await Promise.all(MAIL_UNITS.map((unit) => getServiceState(unit))); if (states.some((state) => state === 'active' || state === 'activating')) { diff --git a/agent/src/state.ts b/agent/src/state.ts index bd691b2b..4cbe7df5 100644 --- a/agent/src/state.ts +++ b/agent/src/state.ts @@ -10,6 +10,10 @@ import type { ApplyResult, VolumeResult } from './types'; export class State { etag?: string; lastApply: Record = {}; + // True once THIS agent has enabled/started the mail units. Guards the + // shutdown paths: an agent must never stop a postfix/dovecot it did not + // start (e.g. a host's own unrelated mail setup). + mailManaged = false; // Volume provisioning results not yet confirmed delivered to the manager. // Persisted so they survive a process exit between reconcile and the next // check-in — otherwise a saved ETag would 304 the next run and the result @@ -32,10 +36,12 @@ export class State { const data = JSON.parse(raw) as { etag?: string; lastApply?: Record; + mailManaged?: boolean; pendingVolumeResults?: Record; }; state.etag = data.etag; state.lastApply = data.lastApply ?? {}; + state.mailManaged = data.mailManaged ?? false; state.pendingVolumeResults = data.pendingVolumeResults ?? {}; } catch (err) { if (!(err instanceof SyntaxError)) throw err; @@ -50,7 +56,12 @@ export class State { fs.writeFileSync( this.file, JSON.stringify( - { etag: this.etag, lastApply: this.lastApply, pendingVolumeResults: this.pendingVolumeResults }, + { + etag: this.etag, + lastApply: this.lastApply, + mailManaged: this.mailManaged, + pendingVolumeResults: this.pendingVolumeResults, + }, null, 2, ), From f9b77662dfdafd9ded5a6060717bc3e2b7ebb520 Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Mon, 26 Oct 2026 13:02:01 -0700 Subject: [PATCH 17/19] Use manager URL for unsubscribe links Derive one-click unsubscribe URLs from the agent's MANAGER_URL and remove the separate setting and nginx auth bypass. Also simplify file ownership handling, configure OpenDKIM to use the postfix group, and make mail paths and GC settings configurable through environment variables. --- agent/src/apply.ts | 13 ++-- agent/src/mail.ts | 70 +++++++++---------- agent/src/system.ts | 26 +++++-- agent/src/types.ts | 4 -- agent/templates/mail/opendkim.conf.ejs | 8 ++- agent/templates/mail/postfix-master.cf.ejs | 7 +- agent/templates/nginx.conf.ejs | 16 ----- agent/test/mail-template.test.js | 2 +- compose.yml | 2 +- create-a-container/client/src/lib/types.ts | 1 - .../src/pages/settings/SettingsPage.tsx | 8 --- create-a-container/openapi.v1.yaml | 1 - create-a-container/routers/api/v1/settings.js | 7 -- create-a-container/utils/agent-config.js | 22 ++---- mail-helper/README.md | 2 +- mie-opensource-landing/docs/admins/email.md | 3 +- .../docs/developers/agent.md | 4 ++ 17 files changed, 83 insertions(+), 113 deletions(-) diff --git a/agent/src/apply.ts b/agent/src/apply.ts index b404b70a..c4da8f7f 100644 --- a/agent/src/apply.ts +++ b/agent/src/apply.ts @@ -10,7 +10,7 @@ import fs from 'fs'; import path from 'path'; import { execFileSync } from 'child_process'; import ejs from 'ejs'; -import { reloadOrRestartService, restartService, sighupService } from './system'; +import { chownByName, reloadOrRestartService, restartService, sighupService } from './system'; import { log, commandOutput } from './log'; import type { AgentConfig, ServiceGroup } from './config'; import type { ApplyResult, SiteConfig } from './types'; @@ -54,13 +54,11 @@ export interface ManagedService { reload(changedFiles: string[]): Promise; } -function renderTemplate(template: string, data: object): Promise { +/** Render a template from the shared templates dir (used by mail.ts too). */ +export function renderTemplate(template: string, data: object): Promise { return ejs.renderFile(path.join(TEMPLATES_DIR, template), data); } -/** Render a template from the shared templates dir (used by mail.ts too). */ -export { renderTemplate }; - // Run a command, capturing its output. execFileSync with stdio 'pipe' attaches // the captured stdout/stderr to the thrown error on failure (see commandOutput // in log.ts), so callers can surface e.g. exactly why `nginx -t` rejected a @@ -138,7 +136,7 @@ function readIfExists(file: string): string | null { function writeFileAtomic(dest: string, content: string, mode?: number, owner?: string): void { const tmp = `${dest}.tmp-${process.pid}`; fs.writeFileSync(tmp, content, mode !== undefined ? { mode } : {}); - if (owner) run(['chown', owner, tmp]); + if (owner) chownByName(tmp, owner); fs.renameSync(tmp, dest); } @@ -150,8 +148,7 @@ function reconcileAttributes(svc: ManagedService, files: { dest: string; mode?: fs.chmodSync(f.dest, f.mode); log.info(`${svc.unit}: fixed mode on ${f.dest}`); } - // chown is idempotent and cheap; names are resolved by chown itself. - if (f.owner) run(['chown', f.owner, f.dest]); + if (f.owner) chownByName(f.dest, f.owner); } catch (err) { log.warn(`${svc.unit}: could not reconcile attributes on ${f.dest}: ${(err as Error).message}`); } diff --git a/agent/src/mail.ts b/agent/src/mail.ts index b9da8ced..5905d878 100644 --- a/agent/src/mail.ts +++ b/agent/src/mail.ts @@ -20,6 +20,7 @@ import fs from 'fs'; import path from 'path'; import { execFileSync } from 'child_process'; import { + chownByName, enableAndStartServices, stopAndDisableServices, getServiceState, @@ -40,26 +41,19 @@ const HELPER_DIR = process.env.MAIL_HELPER_DIR || '/opt/opensource-server/mail-h const HELPER_SOCKET_DIR = process.env.MAIL_HELPER_SOCKET_DIR || '/var/spool/postfix/mail-helper'; const OPENDKIM_SOCKET_DIR = process.env.OPENDKIM_SOCKET_DIR || '/var/spool/postfix/opendkim'; -const VMAIL_DIR = '/var/vmail'; -const GC_STATE_FILE = path.join(VMAIL_DIR, '.gc-state.json'); -const GC_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; +const VMAIL_DIR = process.env.VMAIL_DIR || '/var/vmail'; +const GC_STATE_FILE = process.env.GC_STATE_FILE || path.join(VMAIL_DIR, '.gc-state.json'); +const GC_RETENTION_MS = Number(process.env.GC_RETENTION_MS) > 0 + ? Number(process.env.GC_RETENTION_MS) + : 30 * 24 * 60 * 60 * 1000; function sh(cmd: string[]): void { execFileSync(cmd[0], cmd.slice(1), { stdio: 'pipe', encoding: 'utf8' }); } -function fileExists(file: string): boolean { - try { - fs.accessSync(file); - return true; - } catch { - return false; - } -} - /** The helper is a separate package; report it like a missing binary. */ export function missingMailHelper(): boolean { - return !fileExists(path.join(HELPER_DIR, 'src', 'index.js')); + return !fs.existsSync(path.join(HELPER_DIR, 'src', 'index.js')); } // --- render context --------------------------------------------------------- @@ -78,7 +72,7 @@ function domainCerts(mail: MailConfig): CertPair[] { for (const name of [...names].sort()) { const cert = `/etc/ssl/certs/${name}.crt`; const key = `/etc/ssl/private/${name}.key`; - if (fileExists(cert) && fileExists(key)) pairs.push({ name, cert, key }); + if (fs.existsSync(cert) && fs.existsSync(key)) pairs.push({ name, cert, key }); } return pairs; } @@ -87,7 +81,7 @@ function fallbackChain(mail: MailConfig): { fallbackCert: string; fallbackKey: s if (mail.hostname) { const cert = `/etc/ssl/certs/${mail.hostname}.crt`; const key = `/etc/ssl/private/${mail.hostname}.key`; - if (fileExists(cert) && fileExists(key)) return { fallbackCert: cert, fallbackKey: key }; + if (fs.existsSync(cert) && fs.existsSync(key)) return { fallbackCert: cert, fallbackKey: key }; } // ssl-cert package default (an agent package dependency). return { @@ -135,7 +129,7 @@ export function buildMailContext(mail: MailConfig, agent: AgentConfig, env: Node } /** Overlay for the helper's committed config.json (see mail-helper/). */ -export function buildHelperLocalConfig(mail: MailConfig, env: NodeJS.ProcessEnv = process.env): string { +export function buildHelperLocalConfig(mail: MailConfig, agent: AgentConfig, env: NodeJS.ProcessEnv = process.env): string { const mysql = mail.db.dialect === 'mysql' || mail.db.dialect === 'mariadb'; return `${JSON.stringify({ socket: path.posix.join(HELPER_SOCKET_DIR, 'milter.sock'), @@ -147,7 +141,13 @@ export function buildHelperLocalConfig(mail: MailConfig, env: NodeJS.ProcessEnv user: mail.db.postfixUser, password: env.MAIL_POSTFIX_DB_PASSWORD || null, }, - unsubscribe: mail.unsubscribe, + unsubscribe: { + // One-click links point at the manager itself (it serves /u/ and is + // never behind oauth2-proxy), so the base URL is this agent's + // MANAGER_URL rather than a separate setting. + baseUrl: agent.managerUrl, + keys: mail.unsubscribe.keys, + }, }, null, 2)}\n`; } @@ -249,14 +249,14 @@ export const mailServices: ManagedService[] = [ { unit: 'opensource-mail-helper', group: 'mail', - async render(config) { + async render(config, agent) { const mail = mailOf(config); if (!mail) return null; // The committed config.json ships defaults; the agent only writes the // environment-specific overlay (same split in dev and production). return [{ dest: path.join(HELPER_DIR, 'config.local.json'), - content: buildHelperLocalConfig(mail), + content: buildHelperLocalConfig(mail, agent), mode: 0o640, owner: 'root:opensource-mail', }]; @@ -274,30 +274,28 @@ export const mailServices: ManagedService[] = [ * scripts (opensource-mail-helper postinst creates vmail + opensource-mail). */ export function ensureMailProvisioned(agent: AgentConfig): void { fs.mkdirSync(VMAIL_DIR, { recursive: true }); - sh(['chown', 'vmail:vmail', VMAIL_DIR]); - sh(['chmod', '0750', VMAIL_DIR]); + chownByName(VMAIL_DIR, 'vmail:vmail'); + fs.chmodSync(VMAIL_DIR, 0o750); // Agent-managed DKIM material under $STATE_DIRECTORY/mail. const stateDir = mailStateDir(agent); - fs.mkdirSync(path.join(stateDir, 'dkim'), { recursive: true }); - sh(['chown', 'root:opendkim', stateDir, path.join(stateDir, 'dkim')]); - sh(['chmod', '0750', stateDir, path.join(stateDir, 'dkim')]); + const dkimDir = path.join(stateDir, 'dkim'); + fs.mkdirSync(dkimDir, { recursive: true }); + for (const dir of [stateDir, dkimDir]) { + chownByName(dir, 'root:opendkim'); + fs.chmodSync(dir, 0o750); + } // Milter socket dirs inside the Postfix chroot. The chrooted smtpd - // connects as user postfix: setgid keeps sockets group-owned, and postfix - // joins the opendkim group for OpenDKIM's socket (the standard Debian - // recipe). + // connects as user postfix: setgid keeps the sockets group-owned by + // postfix (OpenDKIM runs as opendkim:postfix via UserID, so no group + // membership changes are needed anywhere). fs.mkdirSync(HELPER_SOCKET_DIR, { recursive: true }); - sh(['chown', 'opensource-mail:postfix', HELPER_SOCKET_DIR]); - sh(['chmod', '2750', HELPER_SOCKET_DIR]); + chownByName(HELPER_SOCKET_DIR, 'opensource-mail:postfix'); + fs.chmodSync(HELPER_SOCKET_DIR, 0o2750); fs.mkdirSync(OPENDKIM_SOCKET_DIR, { recursive: true }); - sh(['chown', 'opendkim:postfix', OPENDKIM_SOCKET_DIR]); - sh(['chmod', '2750', OPENDKIM_SOCKET_DIR]); - try { - sh(['usermod', '-a', '-G', 'opendkim', 'postfix']); - } catch { - /* already a member, or opendkim not installed yet */ - } + chownByName(OPENDKIM_SOCKET_DIR, 'opendkim:postfix'); + fs.chmodSync(OPENDKIM_SOCKET_DIR, 0o2750); } /** Enable+start the mail units while holding the claim; stop+disable them diff --git a/agent/src/system.ts b/agent/src/system.ts index 80015b65..bc8cc5d2 100644 --- a/agent/src/system.ts +++ b/agent/src/system.ts @@ -15,13 +15,9 @@ export function getPrimaryIpv4(): string | null { return null; } -// sbin dirs are not on the agent's PATH under systemd, but that's where -// postfix/dovecot/dnsmasq binaries live. -const EXTRA_BIN_DIRS = ['/usr/sbin', '/usr/bin', '/sbin', '/bin', '/usr/local/sbin', '/usr/local/bin']; - -/** Which of `binaries` cannot be found on PATH or the usual sbin dirs. */ +/** Which of `binaries` cannot be found on PATH. */ export function findMissingBinaries(binaries: string[]): string[] { - const dirs = [...new Set([...(process.env.PATH ?? '').split(':').filter(Boolean), ...EXTRA_BIN_DIRS])]; + const dirs = (process.env.PATH ?? '').split(':').filter(Boolean); return binaries.filter((bin) => !dirs.some((dir) => { try { fs.accessSync(path.join(dir, bin), fs.constants.X_OK); @@ -32,6 +28,24 @@ export function findMissingBinaries(binaries: string[]): string[] { })); } +// Node has no getpwnam/getgrnam binding, so resolve names from the system +// databases directly (local users only — exactly what the mail stack uses). +function lookupId(file: string, name: string): { id: number; gid: number } { + const entry = fs.readFileSync(file, 'utf8').split('\n') + .find((line) => line.startsWith(`${name}:`)); + if (!entry) throw new Error(`unknown ${file.includes('group') ? 'group' : 'user'} "${name}"`); + const fields = entry.split(':'); + return { id: Number(fields[2]), gid: Number(fields[3]) }; +} + +/** chown by `user` or `user:group` name via fs.chownSync (no shelling out). */ +export function chownByName(target: string, owner: string): void { + const [user, group] = owner.split(':'); + const pw = lookupId('/etc/passwd', user); + const gid = group ? lookupId('/etc/group', group).id : pw.gid; + fs.chownSync(target, pw.id, gid); +} + // --- systemd (org.freedesktop.systemd1 on the system bus) ------------------- const SYSTEMD = 'org.freedesktop.systemd1'; diff --git a/agent/src/types.ts b/agent/src/types.ts index 8c9ef1f9..8b455c2d 100644 --- a/agent/src/types.ts +++ b/agent/src/types.ts @@ -103,9 +103,6 @@ export interface NginxConfig { httpServices: HttpService[]; streamServices: StreamService[]; externalDomains: { name: string }[]; - /** Host serving the public /u unsubscribe endpoint — its server block gets - * an auth-exempt /u/ location. Absent on older managers. */ - unsubscribeHost?: string | null; } /** Config snapshot for the whole site. `site` is null before the first site @@ -140,7 +137,6 @@ export interface MailConfig { postfixUser: string; }; unsubscribe: { - baseUrl: string | null; keys: { kid: string; secret: string; active: boolean }[]; }; } diff --git a/agent/templates/mail/opendkim.conf.ejs b/agent/templates/mail/opendkim.conf.ejs index 3b7bc28f..a368d4ef 100644 --- a/agent/templates/mail/opendkim.conf.ejs +++ b/agent/templates/mail/opendkim.conf.ejs @@ -2,10 +2,12 @@ Syslog yes UMask 007 Mode s -# Unix socket inside the Postfix chroot (no inet port); postfix is a member -# of the opendkim group to connect. +# Unix socket inside the Postfix chroot (no inet port). Running with the +# postfix group (UserID user:group form, opendkim.conf(5)) makes the socket +# 0660 opendkim:postfix under UMask 007, so the chrooted smtpd connects +# without postfix joining the opendkim group. Socket <%= opendkimSocket %> -UserID opendkim +UserID opendkim:postfix PidFile /run/opendkim/opendkim.pid Canonicalization relaxed/simple diff --git a/agent/templates/mail/postfix-master.cf.ejs b/agent/templates/mail/postfix-master.cf.ejs index ec0bbabe..89892cd3 100644 --- a/agent/templates/mail/postfix-master.cf.ejs +++ b/agent/templates/mail/postfix-master.cf.ejs @@ -1,7 +1,10 @@ # Managed by opensource-agent (mail service group) — do not edit. # -# Submission policy (restrictions, milters) is defined as named parameters -# in main.cf and referenced here, keeping master.cf overrides single-valued. +# master.cf is Postfix's only mechanism for per-service parameter overrides +# (there are no per-service include files) — each -o below switches a +# setting on for the submission services only. The multi-value policy lists +# themselves (restrictions, milters) are defined once in main.cf as named +# parameters and referenced here as $name, keeping every -o single-valued. # # service type private unpriv chroot wakeup maxproc command + args smtp inet n - y - - smtpd diff --git a/agent/templates/nginx.conf.ejs b/agent/templates/nginx.conf.ejs index 5d7f5991..8b5c2392 100644 --- a/agent/templates/nginx.conf.ejs +++ b/agent/templates/nginx.conf.ejs @@ -223,8 +223,6 @@ http { <%_ httpServices.forEach((service, index) => { _%> <%_ const authRequired = service.authRequired; _%> <%_ const authServer = service.externalDomain.authServer; _%> - <%_ const serverName = `${service.externalHostname}.${service.externalDomain.name}`; _%> - <%_ const servesUnsubscribe = (typeof unsubscribeHost !== 'undefined' && unsubscribeHost) === serverName; _%> server { listen 443 ssl; listen [::]:443 ssl; @@ -236,20 +234,6 @@ http { ssl_certificate /etc/ssl/certs/<%= service.externalDomain.name %>.crt; ssl_certificate_key /etc/ssl/private/<%= service.externalDomain.name %>.key; - <%_ if (servesUnsubscribe && authRequired && authServer) { _%> - <%_ /* RFC 8058 one-click unsubscribe: POSTs come from recipients' mail - clients with no session, so this location must stay outside - auth_request. The endpoint authenticates via its encrypted token. */ -%> - location ^~ /u/ { - proxy_pass <%= service.backendProtocol %>://<%= service.container.ipv4Address %>:<%= service.internalPort %>; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } - <%_ } _%> - error_page 403 @403; location @403 { rewrite ^ /403.html break; diff --git a/agent/test/mail-template.test.js b/agent/test/mail-template.test.js index 5877acc4..2e5e4cdd 100644 --- a/agent/test/mail-template.test.js +++ b/agent/test/mail-template.test.js @@ -27,7 +27,7 @@ const context = { dialect: 'postgres', host: null, port: null, database: 'cc', dovecotUser: 'mail_dovecot', postfixUser: 'mail_postfix', }, - unsubscribe: { baseUrl: 'https://manager.example.com', keys: [] }, + unsubscribe: { keys: [] }, }, stateDir: '/var/lib/opensource-agent/mail', helperMilter: 'unix:mail-helper/milter.sock', diff --git a/compose.yml b/compose.yml index 84e06a1e..8e0c07d1 100644 --- a/compose.yml +++ b/compose.yml @@ -205,7 +205,7 @@ services: curl -H "X-CSRF-Token: $${CSRF}" --json '{"name":"localhost","siteId":1,"acmeEmail":"","acmeDirectoryUrl":"","cloudflareApiEmail":"","cloudflareApiKey":"","authServer":""}' https://proxmox/api/v1/external-domains # Point SMTP at maildev only when unset, so a developer's own SMTP setting survives. PUT replaces every setting, hence the GET round-trip. # Mail service (issue #67): maildev doubles as the relayhost so outbound mail lands in http://localhost:1080; mailSpfInclude is required alongside a relayhost (dev placeholder). - curl https://proxmox/api/v1/settings | jq '.data | select(.smtpUrl == "" or .mailRelayhost == "") | .smtpUrl |= if . == "" then "smtp://10.254.0.1:1025" else . end | .smtpNoreplyAddress |= if . == "" then "noreply@cluster.internal" else . end | .mailRelayhost |= if . == "" then "[10.254.0.1]:1025" else . end | .mailSpfInclude |= if . == "" then "maildev.localhost" else . end | .mailHostname |= if . == "" then "localhost" else . end | .mailUnsubscribeBaseUrl |= if . == "" then "https://manager.localhost" else . end' > settings.json + curl https://proxmox/api/v1/settings | jq '.data | select(.smtpUrl == "" or .mailRelayhost == "") | .smtpUrl |= if . == "" then "smtp://10.254.0.1:1025" else . end | .smtpNoreplyAddress |= if . == "" then "noreply@cluster.internal" else . end | .mailRelayhost |= if . == "" then "[10.254.0.1]:1025" else . end | .mailSpfInclude |= if . == "" then "maildev.localhost" else . end | .mailHostname |= if . == "" then "localhost" else . end' > settings.json [ -s settings.json ] && curl -H "X-CSRF-Token: $${CSRF}" -X PUT --json @settings.json https://proxmox/api/v1/settings curl -H "X-CSRF-Token: $${CSRF}" -X PUT --json '{"hostname":"manager","template":"","entrypoint":"","nvidiaRequested":false,"services":{"s0":{"type":"http","internalPort":3000,"externalHostname":"manager","externalDomainId":1,"dnsName":"","authRequired":false}},"environmentVars":[],"restart":false}' https://proxmox/api/v1/sites/1/containers/1 depends_on: diff --git a/create-a-container/client/src/lib/types.ts b/create-a-container/client/src/lib/types.ts index 21435225..ffe6b36a 100644 --- a/create-a-container/client/src/lib/types.ts +++ b/create-a-container/client/src/lib/types.ts @@ -471,7 +471,6 @@ export interface AppSettings { usagePsiProbeLimit: string; /** Mail service (issue #67). All stored as strings; '' means unset. */ mailHostname: string; - mailUnsubscribeBaseUrl: string; mailRelayhost: string; mailRelayhostUsername: string; mailRelayhostPassword: string; diff --git a/create-a-container/client/src/pages/settings/SettingsPage.tsx b/create-a-container/client/src/pages/settings/SettingsPage.tsx index 8acd70f8..aa32783a 100644 --- a/create-a-container/client/src/pages/settings/SettingsPage.tsx +++ b/create-a-container/client/src/pages/settings/SettingsPage.tsx @@ -39,7 +39,6 @@ const schema = z.object({ bannerMessage: z.string(), usagePsiProbeLimit: z.string().regex(/^\d*$/, 'Must be a non-negative whole number'), mailHostname: z.string(), - mailUnsubscribeBaseUrl: z.string(), mailRelayhost: z.string(), mailRelayhostUsername: z.string(), mailRelayhostPassword: z.string(), @@ -71,7 +70,6 @@ export function SettingsPage() { bannerMessage: '', usagePsiProbeLimit: '', mailHostname: '', - mailUnsubscribeBaseUrl: '', mailRelayhost: '', mailRelayhostUsername: '', mailRelayhostPassword: '', @@ -159,12 +157,6 @@ export function SettingsPage() { Suggest from PTR
- ). - // The nginx template keeps /u/ outside auth_request on this host so - // oauth2-proxy can never block RFC 8058 POSTs from mail clients. - let unsubscribeHost = null; - try { - const baseUrl = await Setting.get('mail_unsubscribe_base_url'); - if (baseUrl) unsubscribeHost = new URL(baseUrl).hostname; - } catch { - /* unset or malformed — no bypass location rendered */ - } - return { site: { id: site.id, @@ -179,7 +168,6 @@ async function buildAgentConfig(siteId) { httpServices, streamServices, externalDomains: externalDomains.map((d) => ({ name: d.name })), - unsubscribeHost, }, }; } @@ -254,7 +242,6 @@ async function buildMailSnapshot(dbConfig) { const settings = await Setting.getMultiple([ 'mail_hostname', 'mail_relayhost', 'mail_relayhost_username', 'mail_relayhost_password', 'mail_db_host', 'mail_default_quota_mb', 'mail_message_size_limit_mb', - 'mail_unsubscribe_base_url', ]); const sendDomains = await ExternalDomain.findAll({ @@ -303,16 +290,17 @@ async function buildMailSnapshot(dbConfig) { }, // opensource-mail-helper mints List-Unsubscribe tokens with the active // key; retired-but-verifiable keys ride along so old links keep working - // after a rotation. The manager verifies with the same set. - unsubscribe: await buildUnsubscribeSection(settings.mail_unsubscribe_base_url), + // after a rotation. The manager verifies with the same set. The links' + // base URL is NOT sent — the agent derives it from its MANAGER_URL (the + // manager serves /u/ itself and is never behind oauth2-proxy). + unsubscribe: await buildUnsubscribeSection(), }; } -async function buildUnsubscribeSection(baseUrl) { +async function buildUnsubscribeSection() { await MailUnsubscribeKey.ensureActive(); const keys = await MailUnsubscribeKey.verifiable(); return { - baseUrl: baseUrl || null, keys: keys.map((k) => ({ kid: k.kid, secret: k.secret, active: k.status === 'active' })), }; } diff --git a/mail-helper/README.md b/mail-helper/README.md index c513e83b..52f5371e 100644 --- a/mail-helper/README.md +++ b/mail-helper/README.md @@ -25,4 +25,4 @@ make test # node --test make deb # package (installs to /opt/opensource-server/mail-helper) ``` -The package is suggested by `opensource-agent`; the agent enables/starts the unit only on the host holding the mail-host claim. A [self-managed MTA](../mie-opensource-landing/docs/admins/email.md) can run the daemon standalone — it only needs the DB views and the unsubscribe keys in `config.local.json`. +The package is suggested by `opensource-agent`; the agent enables/starts the unit only on the host holding the mail-host claim. A [self-managed MTA](../mie-opensource-landing/docs/admins/email.md) can run the daemon standalone — it only needs the DB views and the unsubscribe base URL + keys in `config.local.json`. diff --git a/mie-opensource-landing/docs/admins/email.md b/mie-opensource-landing/docs/admins/email.md index 2a48b362..bb5b926d 100644 --- a/mie-opensource-landing/docs/admins/email.md +++ b/mie-opensource-landing/docs/admins/email.md @@ -15,7 +15,6 @@ Settings → Mail service: | Setting | Purpose | |---------|---------| | Mail hostname | HELO name and MX target. Set to the forward-confirmed PTR of the mail IP — use **Suggest from PTR**. | -| Unsubscribe base URL | Public manager URL; one-click unsubscribe links point at `/u/` | | Relayhost (+ credentials) | Optional smarthost. Requires **SPF include**, since mail then leaves from the relay's IPs | | SPF include | `include:` added to the suggested SPF records | | DNS check resolvers | Resolvers used by Check DNS (default `1.1.1.1`, `8.8.8.8`) | @@ -23,6 +22,8 @@ Settings → Mail service: | Self-managed site id | Mail IP source when no agent runs mail (option C below) | | Default quota / message size limit | `1024` MB and `25` MB when empty | +One-click unsubscribe links (`/u/`) use the mail host agent's `MANAGER_URL` as the base — set it to the manager's public URL. + ## Where Mail Runs **A. Manager (default).** The manager image ships the packages and sets `AGENT_SERVICES=nginx,dnsmasq,mail`. DB access uses Postgres peer auth (`pg_ident` maps the `postfix`/`dovecot`/`opensource-mail` OS users to the `mail_postfix`/`mail_dovecot` roles) — no passwords. diff --git a/mie-opensource-landing/docs/developers/agent.md b/mie-opensource-landing/docs/developers/agent.md index 8dd6276e..7739278e 100644 --- a/mie-opensource-landing/docs/developers/agent.md +++ b/mie-opensource-landing/docs/developers/agent.md @@ -92,8 +92,12 @@ Read from the process environment (systemd loads `/etc/environment`). Set via co | `AGENT_SERVICES` | No | Comma-separated service groups: `nginx`, `dnsmasq`, `mail`. Default `nginx,dnsmasq`. | | `MAIL_POSTFIX_DB_PASSWORD` | No | DB password for the `mail_postfix` role (mail hosts without local socket auth) | | `MAIL_DOVECOT_DB_PASSWORD` | No | DB password for the `mail_dovecot` role (mail hosts without local socket auth) | +| `MAIL_HELPER_DIR`, `MAIL_HELPER_SOCKET_DIR`, `OPENDKIM_SOCKET_DIR` | No | Mail helper install dir and milter socket dirs (defaults match the packages) | +| `VMAIL_DIR`, `GC_STATE_FILE`, `GC_RETENTION_MS` | No | Maildir root (default `/var/vmail`), mailbox GC state file, and GC retention (default 30 days) | | `STATE_DIRECTORY` | No | State directory, set by systemd via `StateDirectory=` (default `/var/lib/opensource-agent`) | +On mail hosts `MANAGER_URL` must be the manager's **public** URL — one-click unsubscribe links use it as their base. + The agent Dockerfile defaults to `SITE_ID=1` and `MANAGER_URL=http://localhost:3000` so the manager container works without configuration. ## Managed Services From 28fbc0e2eb98ac1e6dc5468a972a79dcedefde3f Mon Sep 17 00:00:00 2001 From: cmyers-mieweb Date: Mon, 26 Oct 2026 13:56:46 -0700 Subject: [PATCH 18/19] Update database host configuration logic Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- create-a-container/utils/agent-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create-a-container/utils/agent-config.js b/create-a-container/utils/agent-config.js index 3c8ecac2..94668018 100644 --- a/create-a-container/utils/agent-config.js +++ b/create-a-container/utils/agent-config.js @@ -282,7 +282,7 @@ async function buildMailSnapshot(dbConfig) { mailboxIds, db: { dialect: dbConfig.dialect, - host: settings.mail_db_host || dbConfig.host || null, + host: settings.mail_db_host || null, port: dbConfig.port ? Number(dbConfig.port) : null, database: dbConfig.database || null, dovecotUser: 'mail_dovecot', From d95389cdf49b3d1af5ce6a730b284831cdfaa2d0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:57:34 +0000 Subject: [PATCH 19/19] Reject null sender on submission Co-authored-by: cmyers-mieweb <216552724+cmyers-mieweb@users.noreply.github.com> --- mail-helper/src/filter.js | 7 ++++--- mail-helper/test/filter.test.js | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/mail-helper/src/filter.js b/mail-helper/src/filter.js index 88607394..7a5c4a4b 100644 --- a/mail-helper/src/filter.js +++ b/mail-helper/src/filter.js @@ -1,8 +1,8 @@ /** * Filter decisions, kept free of milter/server plumbing so they are unit * testable. The milter runs pre-queue on the submission ports only, where - * reject_sender_login_mismatch already guarantees envelope sender == SASL - * login. + * reject_sender_login_mismatch enforces envelope sender == SASL login, except + * for the null reverse path, which is rejected explicitly below. */ 'use strict'; @@ -24,7 +24,8 @@ function parseAddress(raw) { * @returns {Promise<{action: 'continue'|'reject'|'tempfail', message?: string}>} */ async function rcptDecision(db, sender, recipient) { - if (!sender || !recipient) return { action: 'continue' }; + if (!sender) return { action: 'reject', message: '5.7.1 Envelope sender required' }; + if (!recipient) return { action: 'continue' }; try { if (await db.isSuppressed(sender, recipient)) { return { action: 'reject', message: `5.7.1 Recipient has unsubscribed from ${sender}` }; diff --git a/mail-helper/test/filter.test.js b/mail-helper/test/filter.test.js index 2d7165b2..c2036d20 100644 --- a/mail-helper/test/filter.test.js +++ b/mail-helper/test/filter.test.js @@ -40,6 +40,12 @@ test('rcptDecision rejects suppressed recipients with 5.7.1', async () => { assert.equal(miss.action, 'continue'); }); +test('rcptDecision rejects a null envelope sender', async () => { + const decision = await rcptDecision(db, '', 'optedout@example.net'); + assert.equal(decision.action, 'reject'); + assert.match(decision.message, /^5\.7\.1 /); +}); + test('rcptDecision DEFERS on DB failure (fail closed)', async () => { const broken = { async isSuppressed() { throw Object.assign(new Error('down'), { code: 'ECONNREFUSED' }); } }; const decision = await rcptDecision(broken, 'app@example.com', 'x@y.z');