feat: retry and redrive dead effects and broadcasts

CHANGELOG.md

@@ -2,6 +2,29 @@
 
 ## Unreleased
 
+- Retry a dead effect or broadcast. `runtime.deadLetters` keeps its message
+  meaning and answers `effects` and `broadcasts`, so the kind rides on the
+  receiver. `retry` returns a dead row to pending with a zero attempt count and
+  no claim, keeps its id so a deduplicating handler sees the same key, and acts
+  only on a dead row, so a second press cannot double-enqueue.
+- Redrive a whole scope. `redrive` opens a durable task, returns at once, and is
+  idempotent over its scope and filters, which a dashboard button needs. A
+  unique index on the active scope enforces that in the database.
+  `runtime.run()` advances one bounded batch per pass, so a redrive never holds
+  a transaction longer than one batch. `runtime.redrives` reads tasks back, and
+  `task.cancel()` stops one and leaves the rows it already moved. A redrive
+  moves what was dead when it started, so a still-broken handler cannot make it
+  run forever.
+- Record who pressed what. Every retry and every redrive transition writes one
+  row to `solid_objects_administration_events`. The identity comes from the
+  authorization context through a new `administrationIdentity` option.
+- Add `redriveBatchSize`, which defaults to 100, and
+  `redriveBatchPauseMilliseconds`, which defaults to 50.
+- Add schema version 11: two tables, `solid_objects_administration_events` and
+  `solid_objects_redrives`, and a `failed_at_ms` stamp on effects and
+  broadcasts, which a redrive filters on. Existing dead rows take their
+  availability stamp.
+
 - Select a wake-up adapter automatically. `wakeUp` now takes a name or an
   adapter, as `config.cache_store` does in Rails, and defaults to
   `"automatic"`. Selection prefers `SOLID_OBJECTS_REDIS_URL`, then PostgreSQL

docs/api.md

@@ -551,6 +551,18 @@ Every manager below is available as a property on `SolidObjectsRuntime`; the
 class and result types are also exported for integration typing.
 
 - `runtime.deadLetters` / `DeadLetterManager`: `all()` and idempotent `retry()`.
+- `runtime.deadLetters.effects` and `runtime.deadLetters.broadcasts`: a
+  `DeadLetterScope` for one `DeadLetterKind`. `all()` lists its dead rows as
+  `DeadRow` values, `retry(id)` returns one to pending, and `redrive(options)`
+  moves a whole scope. `RedriveOptions` names `actorType`, `failedAfter`, and
+  `limit`, which become the `RedriveFilters` the task records. `UnknownDeadRow`
+  reports an id that does not exist.
+- `runtime.redrives` / `RedriveManager`: `find(id)`, `all({ status })`,
+  `cancel(id)`, and `advance()`, which moves one bounded batch. A `RedriveTask`
+  carries its id, kind, filters, `RedriveStatus`, `moved`, `remaining`,
+  `startedAt`, `finishedAt`, and its own `cancel()`. `RedriveScheduler` is the
+  component that advances tasks inside `runtime.run()`. `UnknownRedrive` and
+  `RedriveNotStarted` report a missing task and a task that could not open.
 - `runtime.reminders` / `ReminderManager`: cursor-paginated `all()` and
   idempotent paused-alarm `resume()`.
 - `runtime.processes` / `ProcessManager`: immutable role `all()` and stale-owner

docs/configuration.md

@@ -78,15 +78,18 @@ polling remains the correctness path.
 
 ## Retention and cleanup
 
-| Option                                   |     Default | Contract                                                         |
-| ---------------------------------------- | ----------: | ---------------------------------------------------------------- |
-| `retentionIntervalMilliseconds`          | `3_600_000` | Non-negative automatic history-pruning cadence; `0` disables it. |
-| `deadProcessCleanupIntervalMilliseconds` |    `60_000` | Non-negative stale-owner recovery cadence; `0` disables it.      |
-| `messageRetentionMilliseconds`           |     30 days | Positive default completed-message retention.                    |
-| `messageRetentionByActorType`            |        `{}` | Positive per-type message-retention overrides.                   |
-| `instanceRetentionByActorType`           |        `{}` | Positive per-type instance-expiration opt-ins.                   |
-| `processRetentionMilliseconds`           |      7 days | Positive stopped-process retention.                              |
-| `pruneBatchSize`                         |     `1_000` | Positive integer maximum rows rechecked per pruning transaction. |
+| Option                                   |           Default | Contract                                                         |
+| ---------------------------------------- | ----------------: | ---------------------------------------------------------------- |
+| `retentionIntervalMilliseconds`          |       `3_600_000` | Non-negative automatic history-pruning cadence; `0` disables it. |
+| `deadProcessCleanupIntervalMilliseconds` |          `60_000` | Non-negative stale-owner recovery cadence; `0` disables it.      |
+| `messageRetentionMilliseconds`           |           30 days | Positive default completed-message retention.                    |
+| `messageRetentionByActorType`            |              `{}` | Positive per-type message-retention overrides.                   |
+| `instanceRetentionByActorType`           |              `{}` | Positive per-type instance-expiration opt-ins.                   |
+| `processRetentionMilliseconds`           |            7 days | Positive stopped-process retention.                              |
+| `pruneBatchSize`                         |           `1_000` | Positive integer maximum rows rechecked per pruning transaction. |
+| `redriveBatchSize`                       |             `100` | Positive integer rows a redrive moves per transaction.           |
+| `redriveBatchPauseMilliseconds`          |              `50` | Non-negative pause between redrive batches.                      |
+| `administrationIdentity`                 | `String(context)` | Names the operator recorded in an administration event.          |
 
 Automatic retention prunes messages and stopped process records. Actor
 instance expiration remains an explicit administration action even when a type

docs/dashboard.md

@@ -141,6 +141,15 @@ Dead-letter retry calls `runtime.deadLetters.retry()`. It keeps the durable
 idempotency and the actor-operation validation of that method. If the runtime
 refuses a retry, the detail page shows it with status 422.
 
+Dead effects and broadcasts have the same API, which the dashboard does not yet
+surface. `runtime.deadLetters.effects` and `runtime.deadLetters.broadcasts` read
+and retry their own kind, and `redrive` moves a whole scope as a durable task.
+See [Operations](operations.md) for both.
+
+Every retry and every redrive transition writes one row to
+`solid_objects_administration_events`, holding the action, the kind, the
+subject, and the identity that asked for it.
+
 `HEAD /` performs only a schema reachability query and creates no CSRF session
 state. Use it for liveness checks instead of polling the full dashboard.
 

docs/operations.md

@@ -227,3 +227,91 @@ be resumed.
 Scheduling an existing actor operation for a different time emits
 `solid_objects.reminder.replaced` after the fenced actor commit. Scheduling it
 for the same time emits nothing.
+
+## Dead letters, retry, and redrive
+
+A message that exhausts its attempts becomes a dead letter. An effect or a
+broadcast that exhausts its attempts stays in its own table with
+`status = 'dead'`. All three are read and retried through one receiver, which
+carries the kind:
+
+```ts
+await runtime.deadLetters.all({ authorizationContext })
+await runtime.deadLetters.retry(deadLetterId, { authorizationContext })
+
+await runtime.deadLetters.effects.all({ authorizationContext })
+await runtime.deadLetters.effects.retry(effectId, { authorizationContext })
+await runtime.deadLetters.broadcasts.retry(broadcastId, { authorizationContext })
+```
+
+An effect or broadcast retry returns the row to pending with a zero attempt
+count, no claim, and immediate availability, and keeps its id, so a handler that
+deduplicates on the effect id still sees the same key. An effect is
+at-least-once by contract, so a retried effect can run twice.
+
+Retry acts only on a dead row. A row that is pending, processing, or completed
+comes back unchanged, so pressing a button twice cannot double-enqueue and
+cannot take a row away from a worker that holds it.
+
+An incident produces dead rows in the hundreds, so a scope also answers
+`redrive`:
+
+```ts
+const task = await runtime.deadLetters.effects.redrive({
+  actorType: "payments",
+  failedAfter: new Date(Date.now() - 6 * 60 * 60 * 1000),
+  limit: 5_000,
+  authorizationContext,
+})
+
+await task.cancel({ authorizationContext })
+```
+
+`redrive` returns at once. The task is durable, and `runtime.run()` advances one
+bounded batch per pass, so a redrive of thousands of rows never holds a
+transaction longer than one batch. `redriveBatchSize` defaults to 100 and
+`redriveBatchPauseMilliseconds` to 50.
+
+A redrive moves the rows that were already dead when it started. A row that
+fails again lands back in the same scope, and without that bound a task whose
+handler is still broken would move it forever.
+
+A redrive is idempotent over its scope and its filters. Starting the same one
+while it runs returns the running task rather than a second one, which a
+dashboard button an operator can press twice needs. A different scope or a
+different filter starts its own task, and the same scope can be redriven again
+once the first task finishes.
+
+Read tasks back with `runtime.redrives`:
+
+```ts
+await runtime.redrives.find(task.id, { authorizationContext })
+await runtime.redrives.all({ status: "running", authorizationContext })
+```
+
+A running task reports what is left to move rather than a stored estimate,
+because rows die and are retried while it runs.
+
+Retry, redrive, and cancel each go through `authorizeAdministration` under their
+own resource name: `dead_letters`, `effect_dead_letters`,
+`broadcast_dead_letters`, and `redrives`. Every retry and every task transition
+writes one row to `solid_objects_administration_events`, holding the action, the
+kind, the subject, the identity, and when it happened. The identity comes from
+`administrationIdentity`, which receives the authorization context the caller
+passed and defaults to its `String` form.
+
+An event records an authorized press, not a state transition. Pressing retry
+twice writes two rows, because an operator did two things and a log that shows
+one cannot answer who pressed what. The row the event names carries the outcome.
+A refused caller writes nothing, and a retry that raises after the lookup writes
+nothing, because the event shares the transaction with the work. The redrive
+transitions are different: `redrive.start`, `redrive.finish`, and
+`redrive.cancel` are written only when the task actually changes.
+
+The Durable Objects engine keeps its own message and outbox tables inside each
+object, so these scopes and redrive cover the SQL backends. `deadLetters` there
+reports its own dead rows as it did before.
+
+Automatic redrive on a schedule is deliberately absent. A dead row means a
+person decided something, and these APIs give that person an alternative to an
+`UPDATE` against a runtime table.

docs/parity.md

@@ -129,6 +129,21 @@ edit of the settings makes impossible is reported once here and raised in Ruby;
 both runtimes refuse an unknown name when the configuration is built, so this
 only reaches code that changes the setting afterwards.
 
+## Dead letters, retry, and redrive
+
+Both runtimes scope dead letters by kind through one receiver, retry a dead
+message, effect, or broadcast, redrive a whole scope as a durable and idempotent
+task, bound that task to the rows that were dead when it started, advance it in
+bounded batches, and write one administration event per retry and per task
+transition, in the transaction that causes it. The filters, the resource names,
+the audit actions, and the configuration defaults match.
+
+Two details differ. This runtime filters on a `failed_at_ms` stamp that schema
+version 11 adds, while Ruby filters on the `updated_at` column Active Record
+already maintains. The Durable Objects engine keeps its own message and outbox
+tables inside each object, so these scopes cover the SQL backends here; its
+`deadLetters` call is unchanged.
+
 ## Realtime and browser behavior
 
 | Capability                                                   | Status         | TypeScript shape or remaining work                                                                                                                                                                                                                                                                                 |

src/configuration.ts

@@ -1,6 +1,13 @@
 import { InvalidActor } from "./errors.js"
 import type { Database } from "./database/types.js"
-import type { DeepReadonly, JsonObject, JsonValue, Logger, LongRunningComponent } from "./types.js"
+import type {
+  AdministrationOptions,
+  DeepReadonly,
+  JsonObject,
+  JsonValue,
+  Logger,
+  LongRunningComponent,
+} from "./types.js"
 import { WAKE_UP_NAMES, type WakeUpAdapter, type WakeUpSetting } from "./wake-up.js"
 
 const wakeUpNames: readonly string[] = WAKE_UP_NAMES
@@ -82,6 +89,11 @@ export interface SolidObjectsConfiguration {
   instrumentation?: (event: InstrumentationEvent) => void
   broadcast?: (event: BroadcastEvent) => Promise<void>
   wakeUp?: WakeUpSetting
+  redriveBatchSize?: number
+  redriveBatchPauseMilliseconds?: number
+  administrationIdentity?: (
+    authorizationContext: AdministrationOptions["authorizationContext"],
+  ) => string | null | Promise<string | null>
 }
 
 export interface BroadcastEvent {
@@ -156,6 +168,14 @@ export function buildSettings(configuration: SolidObjectsConfiguration): Runtime
     }),
     processRetentionMilliseconds: configuration.processRetentionMilliseconds ?? 7 * 86_400_000,
     pruneBatchSize: configuration.pruneBatchSize ?? 1_000,
+    redriveBatchSize: configuration.redriveBatchSize ?? 100,
+    redriveBatchPauseMilliseconds: configuration.redriveBatchPauseMilliseconds ?? 50,
+    administrationIdentity:
+      configuration.administrationIdentity ??
+      ((authorizationContext) =>
+        authorizationContext === undefined || authorizationContext === null
+          ? null
+          : String(authorizationContext)),
     logger: configuration.logger ?? consoleLogger,
     wakeUp: configuration.wakeUp ?? "automatic",
     authorizeMessage: configuration.authorizeMessage ?? (() => false),
@@ -280,6 +300,18 @@ function validateSettings(settings: RuntimeSettings): void {
       throw new TypeError(`${name} must be a non-negative integer`)
   }
 
+  if (!Number.isSafeInteger(settings.redriveBatchSize) || settings.redriveBatchSize < 1) {
+    throw new TypeError("redriveBatchSize must be a positive safe integer")
+  }
+  if (
+    !Number.isFinite(settings.redriveBatchPauseMilliseconds) ||
+    settings.redriveBatchPauseMilliseconds < 0
+  ) {
+    throw new TypeError("redriveBatchPauseMilliseconds must not be negative")
+  }
+  if (typeof settings.administrationIdentity !== "function") {
+    throw new TypeError("administrationIdentity must be a function")
+  }
   if (!Number.isSafeInteger(settings.pruneBatchSize) || settings.pruneBatchSize < 1) {
     throw new TypeError("pruneBatchSize must be a positive safe integer")
   }