fix(tables): write the cell state when a resumed run throws

runResumeAndCellTerminal only wrote the cell terminal after a resume returned, so a resume that threw left the cell on its last partial running state, where it could not even be cancelled. The resume manager now records, on the error it rethrows, when the attempt kept its pause resumable (admission refused, run buffer unavailable). The resume job mirrors that onto the cell: back to paused when the pause was kept, failed otherwise. A failed cell write is logged without masking the resume error, which is still rethrown.

Author: waleedlatif1

apps/sim/background/resume-execution.ts

@@ -1,4 +1,5 @@
 import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { task, timeout } from '@trigger.dev/sdk'
 import {
@@ -22,6 +23,7 @@ import type { CellResumeContext } from '@/lib/table/workflow-columns'
 import {
   createResumeAttemptTimeoutController,
   PauseResumeManager,
+  wasPausedExecutionRetained,
 } from '@/lib/workflows/executor/human-in-the-loop-manager'
 import { RESUME_EXECUTION_CONCURRENCY_LIMIT } from '@/background/concurrency-limits'
 import { ExecutionSnapshot } from '@/executor/execution/snapshot'
@@ -359,6 +361,27 @@ async function buildResumeCellWriters(
   return { cellOnBlockComplete, writeCellTerminal }
 }
 
+/**
+ * A resume that throws never reaches the terminal write in
+ * {@link runResumeAndCellTerminal}, which would leave the cell showing its last
+ * partial `running` state. Mirror the execution instead: a pause the resume kept
+ * resumable goes back to paused, and any other failure ended the run.
+ */
+async function writeFailedResumeCellTerminal(writers: CellWriters, error: unknown): Promise<void> {
+  try {
+    if (wasPausedExecutionRetained(error)) {
+      await writers.writeCellTerminal('paused', null)
+    } else {
+      await writers.writeCellTerminal('error', getErrorMessage(error, 'Resume execution failed'))
+    }
+  } catch (writeError) {
+    logger.error(
+      'Failed to write the cell state after a failed resume',
+      projectResolvedSecretDiagnosticError(writeError, undefined)
+    )
+  }
+}
+
 async function runResumeAndCellTerminal(
   payload: ResumeExecutionPayload,
   pausedExecution: Awaited<ReturnType<typeof PauseResumeManager.getPausedExecutionById>>,
@@ -367,16 +390,22 @@ async function runResumeAndCellTerminal(
   timeoutController: ReturnType<typeof createTimeoutAbortController>
 ): Promise<Awaited<ReturnType<typeof PauseResumeManager.startResumeExecution>>> {
   if (!pausedExecution) throw new Error('Paused execution missing — already nulled by caller')
-  const result = await PauseResumeManager.startResumeExecution({
-    resumeEntryId: payload.resumeEntryId,
-    resumeExecutionId: payload.resumeExecutionId,
-    pausedExecution,
-    contextId: payload.contextId,
-    resumeInput: payload.resumeInput,
-    userId: payload.userId,
-    onBlockComplete: writers.cellOnBlockComplete,
-    abortSignal: signal,
-  })
+  let result: Awaited<ReturnType<typeof PauseResumeManager.startResumeExecution>>
+  try {
+    result = await PauseResumeManager.startResumeExecution({
+      resumeEntryId: payload.resumeEntryId,
+      resumeExecutionId: payload.resumeExecutionId,
+      pausedExecution,
+      contextId: payload.contextId,
+      resumeInput: payload.resumeInput,
+      userId: payload.userId,
+      onBlockComplete: writers.cellOnBlockComplete,
+      abortSignal: signal,
+    })
+  } catch (error) {
+    await writeFailedResumeCellTerminal(writers, error)
+    throw error
+  }
 
   if (result.status === 'paused') {
     await writers.writeCellTerminal('paused', null)

apps/sim/background/resume-governed-subject.test.ts

@@ -57,6 +57,7 @@ const mocks = {
   ...hoisted,
   getPausedExecutionById: humanInTheLoopManagerMockFns.mockGetPausedExecutionById,
   startResumeExecution: humanInTheLoopManagerMockFns.mockStartResumeExecution,
+  wasPausedExecutionRetained: humanInTheLoopManagerMockFns.mockWasPausedExecutionRetained,
   createResumeAttemptTimeoutController:
     humanInTheLoopManagerMockFns.mockCreateResumeAttemptTimeoutController,
   findCellContextByExecutionId: tableWorkflowColumnsMockFns.mockFindCellContextByExecutionId,
@@ -217,4 +218,47 @@ describe('resuming a paused table cell', () => {
     const [cascadePayload] = mocks.runRowCascadeLoop.mock.calls[0]
     expect(cascadePayload.capabilityGovernedUserId).toBe('requesting-member')
   }, 20_000)
+
+  describe('when the resume throws', () => {
+    /** The execution state the last cell write persisted. */
+    function lastCellExecutionState() {
+      const [, payload] = mocks.writeWorkflowGroupState.mock.calls.at(-1) ?? []
+      return payload?.executionState
+    }
+
+    it('marks the cell failed when the resumed run itself failed', async () => {
+      const runFailure = new Error('writeLedger: Unique constraint violation')
+      mocks.startResumeExecution.mockRejectedValueOnce(runFailure)
+
+      await expect(executeResumeJob(PAYLOAD)).rejects.toBe(runFailure)
+
+      expect(lastCellExecutionState()).toMatchObject({
+        status: 'error',
+        executionId: 'parent-execution-1',
+        error: 'writeLedger: Unique constraint violation',
+      })
+    }, 20_000)
+
+    it('puts the cell back to paused when the pause stayed resumable', async () => {
+      const admissionRefusal = new Error('Execution can no longer be resumed')
+      mocks.startResumeExecution.mockRejectedValueOnce(admissionRefusal)
+      mocks.wasPausedExecutionRetained.mockReturnValueOnce(true)
+
+      await expect(executeResumeJob(PAYLOAD)).rejects.toBe(admissionRefusal)
+
+      expect(lastCellExecutionState()).toMatchObject({
+        status: 'pending',
+        executionId: 'parent-execution-1',
+        jobId: 'paused-parent-execution-1',
+      })
+    }, 20_000)
+
+    it('still reports the resume failure when the cell write also fails', async () => {
+      const runFailure = new Error('Block failed')
+      mocks.startResumeExecution.mockRejectedValueOnce(runFailure)
+      mocks.writeWorkflowGroupState.mockRejectedValueOnce(new Error('Database unavailable'))
+
+      await expect(executeResumeJob(PAYLOAD)).rejects.toBe(runFailure)
+    }, 20_000)
+  })
 })

apps/sim/lib/workflows/executor/human-in-the-loop-manager.test.ts

@@ -64,6 +64,7 @@ import {
   PauseResumeManager,
   requireResumeDeploymentVersion,
   updateResumeOutputInAggregationBuffers,
+  wasPausedExecutionRetained,
 } from '@/lib/workflows/executor/human-in-the-loop-manager'
 import { getAutomaticResumeWaitingMetadata } from '@/lib/workflows/executor/paused-execution-metadata'
 import { AUTOMATIC_RESUME_WAITING_REASON_MAX_LENGTH } from '@/lib/workflows/executor/resume-policy'
@@ -210,6 +211,72 @@ describe('queued resume attempt deadlines', () => {
   })
 })
 
+describe('which failed resumes keep the pause resumable', () => {
+  type StartResumeArgs = Parameters<typeof PauseResumeManager.startResumeExecution>[0]
+  const pausedExecution = {
+    id: 'paused-execution-1',
+    workflowId: 'workflow-1',
+    executionId: 'parent-execution-1',
+    pausePoints: {
+      'context-1': { contextId: 'context-1', blockId: 'hitl-1' },
+    },
+    executionSnapshot: createSnapshotSeed(),
+    metadata: {},
+  } as StartResumeArgs['pausedExecution']
+  const resumeArgs: StartResumeArgs = {
+    resumeEntryId: 'resume-entry-1',
+    resumeExecutionId: 'resume-execution-1',
+    pausedExecution,
+    contextId: 'context-1',
+    resumeInput: { approved: true },
+    userId: 'user-1',
+  }
+
+  beforeEach(() => {
+    resetDbChainMock()
+  })
+
+  it('keeps the pause resumable when the paused log can no longer be claimed', async () => {
+    const markResumeAttemptFailedSpy = vi
+      .spyOn(PauseResumeManager, 'markResumeAttemptFailed')
+      .mockResolvedValueOnce()
+
+    try {
+      const thrown = await PauseResumeManager.startResumeExecution(resumeArgs).catch(
+        (error: unknown) => error
+      )
+
+      expect(thrown).toMatchObject({ name: 'ResumeAdmissionError' })
+      expect(wasPausedExecutionRetained(thrown)).toBe(true)
+    } finally {
+      markResumeAttemptFailedSpy.mockRestore()
+    }
+  })
+
+  it('does not keep the pause when the resumed run itself failed', async () => {
+    const rawError = new Error('Block failed')
+    const managerInternals = PauseResumeManager as unknown as PauseResumeManagerInternals
+    const runResumeExecutionSpy = vi
+      .spyOn(managerInternals, 'runResumeExecution')
+      .mockRejectedValueOnce(rawError)
+    const markResumeFailedSpy = vi
+      .spyOn(managerInternals, 'markResumeFailed')
+      .mockResolvedValueOnce()
+    const processQueuedResumesSpy = vi
+      .spyOn(PauseResumeManager, 'processQueuedResumes')
+      .mockResolvedValueOnce()
+
+    try {
+      await expect(PauseResumeManager.startResumeExecution(resumeArgs)).rejects.toBe(rawError)
+      expect(wasPausedExecutionRetained(rawError)).toBe(false)
+    } finally {
+      runResumeExecutionSpy.mockRestore()
+      markResumeFailedSpy.mockRestore()
+      processQueuedResumesSpy.mockRestore()
+    }
+  })
+})
+
 describe('resume failure diagnostic projection', () => {
   beforeEach(() => {
     resetDbChainMock()

apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts

@@ -134,6 +134,26 @@ class ResumeAdmissionError extends Error {
   }
 }
 
+/**
+ * Errors from resume attempts that failed without consuming their pause: the
+ * attempt was refused or could not start, so the paused execution stays
+ * resumable. Every other failed resume leaves the execution terminal.
+ */
+const pauseRetainingFailures = new WeakSet<object>()
+
+/**
+ * Whether `error` was thrown by a resume attempt that left its paused execution
+ * resumable, so a caller mirroring the run's state (a table cell) keeps it paused
+ * rather than failed.
+ */
+export function wasPausedExecutionRetained(error: unknown): boolean {
+  return typeof error === 'object' && error !== null && pauseRetainingFailures.has(error)
+}
+
+function retainPausedExecution(error: unknown): void {
+  if (typeof error === 'object' && error !== null) pauseRetainingFailures.add(error)
+}
+
 /** Matches the paused execution mode to the deployment recorded on its durable root log. */
 export function requireResumeDeploymentVersion(
   useDraftState: unknown,
@@ -987,6 +1007,7 @@ export class PauseResumeManager {
       const message = toError(error).message
       await releaseExecutionSlot(resumeEntryId)
       if (error instanceof ResumeAdmissionError) {
+        retainPausedExecution(error)
         await PauseResumeManager.markResumeAttemptFailed({
           resumeEntryId,
           pausedExecutionId: pausedExecution.id,
@@ -997,6 +1018,7 @@ export class PauseResumeManager {
           retryable: error.retryable,
         })
       } else if (message === RUN_BUFFER_UNAVAILABLE_ERROR) {
+        retainPausedExecution(error)
         await PauseResumeManager.markResumeAttemptFailed({
           resumeEntryId,
           pausedExecutionId: pausedExecution.id,

packages/testing/src/mocks/human-in-the-loop-manager.mock.ts

@@ -162,6 +162,7 @@ export const humanInTheLoopManagerMockFns = {
     }
   ),
   mockCreateResumeAttemptTimeoutController: vi.fn(),
+  mockWasPausedExecutionRetained: vi.fn((_error: unknown): boolean => false),
   mockExtractResumeBillingAttributionFromSnapshot: vi.fn(),
   mockComputeEarliestResumeAt: vi.fn(
     (points: Iterable<MockPausePoint>, options: { after?: Date } = {}): Date | null => {
@@ -195,6 +196,7 @@ export const humanInTheLoopManagerMock = {
     humanInTheLoopManagerMockFns.mockUpdateResumeOutputInAggregationBuffers,
   createResumeAttemptTimeoutController:
     humanInTheLoopManagerMockFns.mockCreateResumeAttemptTimeoutController,
+  wasPausedExecutionRetained: humanInTheLoopManagerMockFns.mockWasPausedExecutionRetained,
   extractResumeBillingAttributionFromSnapshot:
     humanInTheLoopManagerMockFns.mockExtractResumeBillingAttributionFromSnapshot,
   computeEarliestResumeAt: humanInTheLoopManagerMockFns.mockComputeEarliestResumeAt,