fix(cli)!: exit 2 for invalid flag and argument values [RED-1277] [ship]

AGENTS.md

@@ -57,6 +57,9 @@ exit-code table, lives in `docs/output-contract.md`.
   answer from stdin and stderr, not stdout.
 - Never print anything to stdout around a `--output json` document. A failure
   oclif raises leaves stdout empty; see the contract's exit codes.
+- Invalid input exits 2, other failures 1. Validation helpers throw
+  `InvalidInputError`; a catch block that reports the error itself sets
+  `process.exitCode = exitCodeFor(err)` rather than a literal 1.
 - Terminal users see both streams, so moving chatter to stderr does not change
   the interactive experience — that is the default for new output.
 

docs/output-contract.md

@@ -99,13 +99,18 @@ change in the PR.
 | Code | Meaning |
 |---|---|
 | 0 | Success. Also a declined confirmation, `--dry-run` previews, and runs whose checks passed or only degraded (a degraded check does not fail the run). A run stopped with Ctrl-C has no code of its own: it exits 1 if a result that arrived failed, and 0 otherwise. |
-| 1 | Failure: a failed check or run error, no matching checks under `--fail-on-no-matching` (the default), API errors, invalid configuration, or a failure reported only on stderr. |
-| 2 | Usage error: unknown flags or arguments, oclif parse errors, or invalid flag or argument values a command rejects with `this.error`. The message is on stderr and stdout is empty. |
+| 1 | Failure: a failed check or run error, no matching checks under `--fail-on-no-matching` (the default), API errors, invalid configuration, well-formed input that names something that does not exist (an unknown location, plan or entitlement), or a failure reported only on stderr. |
+| 2 | Usage error: unknown flags or arguments, oclif parse errors, invalid flag or argument values, or flags that cannot be combined. The message is on stderr; stdout is empty, except that some commands print their JSON error document under `--output json`. |
 | 3 | `confirmation_required` (agent and CI mode): a write command stopped for approval and printed the envelope on stdout. |
 | 127 | Unknown command. |
 
-oclif's `this.error` exits 2 by default, so commands report any other failure
-through it with `{ exit: 1 }`; a bare `this.error` is for invalid input.
+Commands reject invalid input with a bare `this.error` (oclif's default exit
+is 2), by throwing `InvalidInputError` (`helpers/invalid-input-error.ts`), or
+with `style.longError`/`style.fatal` followed by `this.exit(2)`. Validation
+helpers throw `InvalidInputError`, since they cannot call `this.error`; a catch
+block that reports an error itself takes the code from `exitCodeFor(err)`, so
+invalid input caught there still exits 2. Any other failure exits 1, through
+`this.error` with `{ exit: 1 }` when it uses oclif's.
 
 ## Notes
 

packages/cli/e2e/__tests__/cli-parse-error.spec.ts

@@ -43,4 +43,25 @@ describe('cli parse error', () => {
       }
     }
   })
+
+  // Input a command rejects itself, before any API call, exits with the
+  // usage-error code like oclif's own parse errors.
+  it.each([
+    ['a malformed api field', ['api', '/v1/checks', '-F', 'broken'], 'Invalid field format'],
+    ['a custom flag parse failure', ['deploy', '--schedule-on-deploy-threshold', 'abc'],
+      'Expected a non-negative whole number or "auto"'],
+    ['an unsupported import resource type', ['import', 'plan', 'bogus:1'], 'Unsupported resource type'],
+  ])('exits 2 for %s', async (_, args, message) => {
+    try {
+      await runCheckly(fixt, args)
+      expect.unreachable('Expected command to fail')
+    } catch (err) {
+      if (err instanceof ExecaError) {
+        expect(err.exitCode).toBe(2)
+        expect(err.stderr).toContain(message)
+      } else {
+        throw err
+      }
+    }
+  })
 })

packages/cli/src/commands/__tests__/assets.spec.ts

@@ -127,7 +127,7 @@ describe('assets commands', () => {
     })
     await expect(AssetsList.prototype.run.call(missing as any))
       .rejects
-      .toThrow('Use exactly one of --check-id or --test-session-id.')
+      .toMatchObject({ message: 'Use exactly one of --check-id or --test-session-id.', oclif: { exit: 2 } })
 
     const both = createCommandContext({
       flags: {
@@ -139,7 +139,7 @@ describe('assets commands', () => {
     })
     await expect(AssetsList.prototype.run.call(both as any))
       .rejects
-      .toThrow('Use exactly one of --check-id or --test-session-id, not both.')
+      .toMatchObject({ message: 'Use exactly one of --check-id or --test-session-id, not both.', oclif: { exit: 2 } })
   })
 
   it('lists check result assets as a table with copyable Asset values and truncation warning', async () => {
@@ -743,7 +743,7 @@ describe('assets commands', () => {
 
     await expect(AssetsDownload.prototype.run.call(ctx as any))
       .rejects
-      .toThrow('--force and --skip-existing are mutually exclusive.')
+      .toMatchObject({ message: '--force and --skip-existing are mutually exclusive.', oclif: { exit: 2 } })
   })
 
   it('does not expose archive extraction before it is implemented', () => {

packages/cli/src/commands/__tests__/confirm-flow-deploy.spec.ts

@@ -1187,7 +1187,7 @@ describe('deploy with --no-plan', () => {
       vi.mocked(parseProject).mockClear()
       const ctx = createCommandContext({ plan: false, force: true, ...flags })
 
-      await expect(Deploy.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_1')
+      await expect(Deploy.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_2')
 
       expect(ctx.style.longError).toHaveBeenCalledWith(
         `${named} applies to a planned deploy only.`,
@@ -1999,7 +1999,7 @@ describe('deploy of a plan with nothing to apply', () => {
         vi.mocked(parseProject).mockClear()
         const ctx = createCommandContext({ force: true, ...flags })
 
-        await expect(Deploy.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_1')
+        await expect(Deploy.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_2')
 
         expect(ctx.style.longError).toHaveBeenCalledWith(
           `${named} applies only when checks are scheduled after the deploy.`,

packages/cli/src/commands/account/usage/__tests__/usage.spec.ts

@@ -110,6 +110,7 @@ describe('usage terms commands', () => {
     expect(api.usage.getTerm).not.toHaveBeenCalled()
     expect(bad.style.longError).toHaveBeenCalledWith('Failed to load usage term.',
       expect.objectContaining({ message: 'Usage term ID must be a valid UUID.' }))
+    expect(process.exitCode).toBe(2)
     vi.mocked(api.usage.getTerm).mockRejectedValue(new NotFoundError({
       statusCode: 404, error: 'Not Found', message: 'No usage terms found', code: 'NO_USAGE_TERMS',
     }))
@@ -331,7 +332,7 @@ describe('report failure paths', () => {
     expect(api.usage.getSeries).not.toHaveBeenCalled()
     expect(ctx.style.longError).toHaveBeenCalledWith(expect.any(String),
       expect.objectContaining({ message: expect.stringContaining('does not support --check-type') }))
-    expect(process.exitCode).toBe(1)
+    expect(process.exitCode).toBe(2)
   })
 
   it('reports usage:read permission failures', async () => {

packages/cli/src/commands/account/usage/projections.ts

@@ -2,6 +2,7 @@ import { Args, Flags } from '@oclif/core'
 import { AuthCommand } from '../../authCommand.js'
 import { outputFlag } from '../../../helpers/flags.js'
 import { describeUsageError, projectionsParams, validateUsageTermId } from '../../../helpers/usage.js'
+import { exitCodeFor } from '../../../helpers/invalid-input-error.js'
 import * as api from '../../../rest/api.js'
 import { formatUsageProjections } from '../../../formatters/usage.js'
 
@@ -33,7 +34,7 @@ export default class UsageProjectionsCommand extends AuthCommand {
       this.log(formatUsageProjections(projections, flags.output === 'md' ? 'md' : 'terminal', term))
     } catch (err: unknown) {
       this.style.longError('Failed to load usage projections.', describeUsageError(err, { termId: args.id }))
-      process.exitCode = 1
+      process.exitCode = exitCodeFor(err)
     }
   }
 }

packages/cli/src/commands/account/usage/series.ts

@@ -1,6 +1,7 @@
 import { AuthCommand } from '../../authCommand.js'
 import { outputFlag } from '../../../helpers/flags.js'
 import { describeUsageError, loadUsageNames, usageSeriesFlags, usageSeriesParams } from '../../../helpers/usage.js'
+import { exitCodeFor } from '../../../helpers/invalid-input-error.js'
 import * as api from '../../../rest/api.js'
 import { heading, type OutputFormat } from '../../../formatters/render.js'
 import {
@@ -50,7 +51,7 @@ export default class UsageSeriesCommand extends AuthCommand {
       ].filter(Boolean).join('\n\n'))
     } catch (err: unknown) {
       this.style.longError('Failed to load usage series.', describeUsageError(err))
-      process.exitCode = 1
+      process.exitCode = exitCodeFor(err)
     }
   }
 }

packages/cli/src/commands/account/usage/summary.ts

@@ -1,6 +1,7 @@
 import { AuthCommand } from '../../authCommand.js'
 import { outputFlag } from '../../../helpers/flags.js'
 import { describeUsageError, usageRangeFlags, usageRangeParams } from '../../../helpers/usage.js'
+import { exitCodeFor } from '../../../helpers/invalid-input-error.js'
 import * as api from '../../../rest/api.js'
 import { type OutputFormat, renderCommandHints } from '../../../formatters/render.js'
 import { formatUsageSummary, usageListCommand } from '../../../formatters/usage.js'
@@ -36,7 +37,7 @@ export default class UsageSummaryCommand extends AuthCommand {
       this.log(output.join('\n\n'))
     } catch (err: unknown) {
       this.style.longError('Failed to load usage summary.', describeUsageError(err))
-      process.exitCode = 1
+      process.exitCode = exitCodeFor(err)
     }
   }
 }

packages/cli/src/commands/account/usage/term.ts

@@ -2,6 +2,7 @@ import { Args } from '@oclif/core'
 import { AuthCommand } from '../../authCommand.js'
 import { outputFlag } from '../../../helpers/flags.js'
 import { describeUsageError, validateUsageTermId } from '../../../helpers/usage.js'
+import { exitCodeFor } from '../../../helpers/invalid-input-error.js'
 import * as api from '../../../rest/api.js'
 import { formatUsageTermDetail } from '../../../formatters/usage.js'
 
@@ -25,7 +26,7 @@ export default class UsageTermCommand extends AuthCommand {
         : formatUsageTermDetail(term, flags.output === 'md' ? 'md' : 'terminal'))
     } catch (err: unknown) {
       this.style.longError('Failed to load usage term.', describeUsageError(err, { termId: args.id }))
-      process.exitCode = 1
+      process.exitCode = exitCodeFor(err)
     }
   }
 }

packages/cli/src/commands/account/usage/terms.ts

@@ -1,6 +1,7 @@
 import { AuthCommand } from '../../authCommand.js'
 import { outputFlag } from '../../../helpers/flags.js'
 import { describeUsageError, usageTermsListFlags, usageTermsParams } from '../../../helpers/usage.js'
+import { exitCodeFor } from '../../../helpers/invalid-input-error.js'
 import * as api from '../../../rest/api.js'
 import type { OutputFormat } from '../../../formatters/render.js'
 import {
@@ -41,7 +42,7 @@ export default class UsageTermsCommand extends AuthCommand {
       ].filter(Boolean).join('\n\n'))
     } catch (err: unknown) {
       this.style.longError('Failed to load usage terms.', describeUsageError(err))
-      process.exitCode = 1
+      process.exitCode = exitCodeFor(err)
     }
   }
 }

packages/cli/src/commands/alert-channels/__tests__/alert-channels.spec.ts

@@ -89,6 +89,32 @@ describe('alert-channels commands', () => {
     expect(JSON.parse(ctx.logged[0])).toEqual(alertChannelFixture)
   })
 
+  it('get <id> rejects an invalid ID with the usage-error exit code', async () => {
+    const ctx = createCommandContext({
+      args: { id: 'abc' },
+      flags: { output: 'json' },
+    })
+
+    await AlertChannelsGet.prototype.run.call(ctx as any)
+
+    expect(api.alertChannels.get).not.toHaveBeenCalled()
+    expect(ctx.style.longError).toHaveBeenCalledWith('Failed to get alert channel details.',
+      expect.objectContaining({ message: 'Alert channel ID must be a positive integer.' }))
+    expect(process.exitCode).toBe(2)
+  })
+
+  it('get <id> keeps exit 1 for an API failure', async () => {
+    vi.mocked(api.alertChannels.get).mockRejectedValue(new Error('boom'))
+    const ctx = createCommandContext({
+      args: { id: '123' },
+      flags: { output: 'json' },
+    })
+
+    await AlertChannelsGet.prototype.run.call(ctx as any)
+
+    expect(process.exitCode).toBe(1)
+  })
+
   it('get <id> includes navigation hints in terminal output', async () => {
     const ctx = createCommandContext({
       args: { id: '123' },

packages/cli/src/commands/alert-channels/get.ts

@@ -3,6 +3,7 @@ import chalk from 'chalk'
 import { AuthCommand } from '../authCommand.js'
 import { outputFlag } from '../../helpers/flags.js'
 import { parsePositiveInteger } from '../../helpers/number.js'
+import { exitCodeFor } from '../../helpers/invalid-input-error.js'
 import * as api from '../../rest/api.js'
 import type { OutputFormat } from '../../formatters/render.js'
 import { formatAlertChannelDetail } from '../../formatters/alert-channels.js'
@@ -53,7 +54,7 @@ export default class AlertChannelsGet extends AuthCommand {
       this.log(output.join('\n'))
     } catch (err: any) {
       this.style.longError('Failed to get alert channel details.', err)
-      process.exitCode = 1
+      process.exitCode = exitCodeFor(err)
     }
   }
 }

packages/cli/src/commands/alert-channels/list.ts

@@ -2,6 +2,7 @@ import { Flags } from '@oclif/core'
 import { AuthCommand } from '../authCommand.js'
 import { outputFlag } from '../../helpers/flags.js'
 import { validateIntegerRange } from '../../helpers/number.js'
+import { exitCodeFor } from '../../helpers/invalid-input-error.js'
 import * as api from '../../rest/api.js'
 import type { OutputFormat } from '../../formatters/render.js'
 import {
@@ -71,7 +72,7 @@ export default class AlertChannelsList extends AuthCommand {
       this.log(output.join('\n'))
     } catch (err: any) {
       this.style.longError('Failed to list alert channels.', err)
-      process.exitCode = 1
+      process.exitCode = exitCodeFor(err)
     }
   }
 }

packages/cli/src/commands/alert-channels/logs.ts

@@ -8,6 +8,7 @@ import {
   validateIntegerRange,
 } from '../../helpers/number.js'
 import * as api from '../../rest/api.js'
+import { exitCodeFor } from '../../helpers/invalid-input-error.js'
 import type { AlertNotificationListParams } from '../../rest/alert-notifications.js'
 import type { OutputFormat } from '../../formatters/render.js'
 import { formatAlertNotificationLogs } from '../../formatters/alert-channels.js'
@@ -98,7 +99,7 @@ export default class AlertChannelsLogs extends AuthCommand {
       this.log(output.join('\n'))
     } catch (err: any) {
       this.style.longError('Failed to list alert channel logs.', err)
-      process.exitCode = 1
+      process.exitCode = exitCodeFor(err)
     }
   }
 }

packages/cli/src/commands/assets/download.ts

@@ -1,6 +1,7 @@
 import { Flags } from '@oclif/core'
 import path from 'node:path'
 import { AuthCommand } from '../authCommand.js'
+import { InvalidInputError } from '../../helpers/invalid-input-error.js'
 import { outputFlag } from '../../helpers/flags.js'
 import { assetSelectorValue, formatDownloadedAssets, type DownloadedAssetRow } from '../../formatters/assets.js'
 import {
@@ -87,7 +88,7 @@ export default class AssetsDownload extends AuthCommand {
     }
 
     if (flags.force && flags['skip-existing']) {
-      throw new Error('--force and --skip-existing are mutually exclusive.')
+      throw new InvalidInputError('--force and --skip-existing are mutually exclusive.')
     }
 
     try {

packages/cli/src/commands/deploy.ts

@@ -290,7 +290,7 @@ export default class Deploy extends AuthCommand {
           `${needsPlan} applies to a planned deploy only.`,
           `Re-run without --no-plan, or without ${needsPlan}.`,
         )
-        this.exit(1)
+        this.exit(2)
       }
     }
 
@@ -308,7 +308,7 @@ export default class Deploy extends AuthCommand {
           `--${refiningFlag} applies only when checks are scheduled after the deploy.`,
           `Add --schedule-on-deploy to schedule them, or drop --${refiningFlag}.`,
         )
-        this.exit(1)
+        this.exit(2)
       }
     }
 

packages/cli/src/commands/import/__tests__/cancel.spec.ts

@@ -131,14 +131,14 @@ describe('import cancel command (flag validation)', () => {
     vi.mocked(api.projects.cancelImportPlan).mockResolvedValue({} as any)
   })
 
-  it('rejects --all together with --plan-id (exit 1) instead of silently ignoring one', async () => {
+  it('rejects --all together with --plan-id (exit 2) instead of silently ignoring one', async () => {
     vi.mocked(detectCliMode).mockReturnValue('agent')
     vi.mocked(api.projects.findImportPlans).mockResolvedValue({
       data: [plan('a'), plan('b')],
     } as any)
     const ctx = createCommandContext(cancelFlags({ 'all': true, 'plan-id': 'a' }))
 
-    await expect(ImportCancelCommand.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_1')
+    await expect(ImportCancelCommand.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_2')
     expect(ctx.style.fatal).toHaveBeenCalledWith(expect.stringContaining('cannot be used together'))
     expect(api.projects.cancelImportPlan).not.toHaveBeenCalled()
   })
@@ -148,7 +148,7 @@ describe('import cancel command (flag validation)', () => {
     vi.mocked(api.projects.findImportPlans).mockResolvedValue({ data: [] } as any)
     const ctx = createCommandContext(cancelFlags({ 'all': true, 'plan-id': 'a' }))
 
-    await expect(ImportCancelCommand.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_1')
+    await expect(ImportCancelCommand.prototype.run.call(ctx as any)).rejects.toThrow('EXIT_2')
     expect(ctx.style.fatal).toHaveBeenCalledWith(expect.stringContaining('cannot be used together'))
     expect(api.projects.findImportPlans).not.toHaveBeenCalled()
   })

packages/cli/src/commands/import/plan.ts

@@ -12,6 +12,7 @@ import { execa } from 'execa'
 import { LOGICAL_ID_PATTERN } from '../../constants.js'
 import * as api from '../../rest/api.js'
 import { AuthCommand } from '../authCommand.js'
+import { InvalidInputError } from '../../helpers/invalid-input-error.js'
 import commonMessages from '../../messages/common-messages.js'
 import { getRepoUrlFromGit, splitConfigFilePath } from '../../services/util.js'
 import { ChecklyConfig, ConfigNotFoundError, loadChecklyConfig } from '../../services/checkly-config-loader.js'
@@ -1536,7 +1537,7 @@ function parseFilter (spec: string): ImportPlanFilter {
 
   const parseId = importables[type as keyof typeof importables]
   if (parseId === undefined) {
-    throw new Error(`Invalid resource specifier '${spec}': Unsupported resource type '${type}'`)
+    throw new InvalidInputError(`Invalid resource specifier '${spec}': Unsupported resource type '${type}'`)
   }
 
   try {
@@ -1546,7 +1547,7 @@ function parseFilter (spec: string): ImportPlanFilter {
     }
   } catch (err) {
     if (err instanceof InvalidResourceIdentifierError) {
-      throw new Error(`Invalid resource specifier '${spec}': ${err.message}`, { cause: err })
+      throw new InvalidInputError(`Invalid resource specifier '${spec}': ${err.message}`, { cause: err })
     }
 
     throw err

packages/cli/src/helpers/__tests__/api-fields.spec.ts

@@ -1,5 +1,6 @@
 import { describe, it, expect } from 'vitest'
 import { parseFields } from '../api-fields.js'
+import { InvalidInputError } from '../invalid-input-error.js'
 
 describe('parseFields', () => {
   it('parses string fields with =', () => {
@@ -33,6 +34,7 @@ describe('parseFields', () => {
 
   it('throws on field without = or :=', () => {
     expect(() => parseFields(['broken'])).toThrow('Invalid field format: "broken"')
+    expect(() => parseFields(['broken'])).toThrow(InvalidInputError)
   })
 
   it('throws on invalid JSON in := field', () => {