From 91730bcd071fe11df3dea5f587ea6406357d25a9 Mon Sep 17 00:00:00 2001 From: Alexandre Dubois Date: Thu, 29 Oct 2026 23:58:12 +0100 Subject: [PATCH] Improve tests --- README.md | 2 + benchmarks/result.md | 10 +- skills/coderuler-new-rule/SKILL.md | 8 +- src/adapter/filesystem/rule-tests.ts | 90 +++++++++--- tests/rule-resolution.test.ts | 25 +++- tests/rule-testing.test.ts | 200 +++++++++++++++++++++++++++ website/src/content/docs/testing.md | 6 +- 7 files changed, 310 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index ae714b1..8168aba 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Install the CLI globally: ```sh npm install -g @alex-dbs/coderuler coderuler --help +coderuler skill install ``` The npm package is scoped. The command is `coderuler`. A global installation can load rules without a project-local dependency. @@ -24,6 +25,7 @@ For a project-specific CLI version, install it locally: ```sh npm install --save-dev @alex-dbs/coderuler npx coderuler --help +npx coderuler skill install ``` The global installation is optional: with only the project-local dependency, run commands through `npx coderuler`. diff --git a/benchmarks/result.md b/benchmarks/result.md index 4cb522f..00b694e 100644 --- a/benchmarks/result.md +++ b/benchmarks/result.md @@ -1,8 +1,8 @@ -| Method | Accuracy | Precision | Recall | F1 | FP / FN | Tokens used (input / output) | -| --------------------------- | --------: | ---------: | ------: | -----: | -------: | ----------------------------: | -| Generic reviewer, first run | 47.5% | 68.4% | 34.2% | 45.6% | 6 / 25 | 17,403,725 / 231,053 | -| OpenAI | 52.5% | 72.7% | 42.1% | 53.3% | 6 / 22 | 863,703 / 0 | -| Jev | 54.2% | 78.9% | 39.5% | 52.6% | 4 / 23 | 863,703 / 0 | +| Method | Accuracy | Precision | Recall | F1 | FP / FN | Tokens used (input / output) | +| ---------------- | --------: | ---------: | ------: | -----: | -------: | -------------------------------: | +| Generic reviewer | 47.5% | 68.4% | 34.2% | 45.6% | 6 / 25 | 17,403,725 / 231,053 - ($74.24) | +| OpenAI | 52.5% | 72.7% | 42.1% | 53.3% | 6 / 22 | 863,703 / 0 - ($0.06) | +| Jev | 54.2% | 78.9% | 39.5% | 52.6% | 4 / 23 | 863,703 / 0 - ($0.04) | diff --git a/skills/coderuler-new-rule/SKILL.md b/skills/coderuler-new-rule/SKILL.md index 4f01f0b..3980ba8 100644 --- a/skills/coderuler-new-rule/SKILL.md +++ b/skills/coderuler-new-rule/SKILL.md @@ -1,6 +1,6 @@ --- name: coderuler-new-rule -description: Create CodeRuler rules, including custom selection, filters and checks, inline tests, and inspection. +description: Create CodeRuler rules, including custom selection, filters and checks, inline or companion tests, and inspection. --- # Create a CodeRuler rule @@ -29,7 +29,9 @@ Start the name with its language or technology prefix, such as `go-`, `ts-`, or Keep rules short, with only a few lines of policy code. Reuse built-in selectors, evaluators, and standard libraries before writing custom logic. Add helpers only for real reuse. -For a deterministic rule, add a named `tests` export with `.passes()` and `.violates()` cases. For an AI rule, put every test case in `aiTests` and do not add a separate `tests` export. Omit `.skips()` assertions when creating a rule: skipped inputs never reach the check and provide no evidence of policy behavior. +Test cases can live in the rule module or in a sibling `.test` module. For example, `sample.rule.ts` pairs with `sample.test.ts`; keep the same extension for `.mts`, `.js`, and `.mjs`. The `test` command combines exports from both modules. It runs ordinary `tests` by default; include `--ai` to run `aiTests`. + +For a deterministic rule, export named `tests` with `.passes()` and `.violates()` cases. For an AI rule, put every test case in `aiTests` and do not add a separate `tests` export. Omit `.skips()` assertions when creating a rule: skipped inputs never reach the check and provide no evidence of policy behavior. For an AI rule, use `instruction` to state what compliant code does, beginning with a direct requirement. Add only applicability boundaries or real exceptions needed to define the policy; keep one policy per rule. Keep reporting language in `violationMessage`, examples in `examples`, and test cases in `aiTests`—not in the instruction. See [AI instruction style](reference.md#ai-instruction-style). @@ -94,7 +96,7 @@ coderuler validate coderuler test ``` -`validate` checks the module contract, not policy behavior or TypeScript semantics. `check` and `validate` do not execute inline tests. `test` with no selected cases is an error, not success. +`validate` checks the module contract, not policy behavior or TypeScript semantics. `check` and `validate` do not run test cases or load sibling `.test` modules. `test` with no selected cases is an error, not success. Useful test options: diff --git a/src/adapter/filesystem/rule-tests.ts b/src/adapter/filesystem/rule-tests.ts index c08c226..74725dd 100644 --- a/src/adapter/filesystem/rule-tests.ts +++ b/src/adapter/filesystem/rule-tests.ts @@ -1,4 +1,4 @@ -import { mkdtemp, realpath, rm } from 'node:fs/promises'; +import { lstat, mkdtemp, realpath, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -8,7 +8,7 @@ import type { LoadedRule } from '../../app/ports.js'; import type { RuleTestCase, RuleTestResult } from '../../app/rule-tests.js'; import { isRecord, validateJsonValue } from '../../app/validate.js'; import type { RuleTestWorkerMessage, RuleTestWorkerOutcome, RuleTestWorkerRequest } from './rule-test-worker.js'; -import { getRuleDependencies } from './rule-resolution.js'; +import { getRuleDependencies, registerRuleDependencies } from './rule-resolution.js'; // A JavaScript entry lets inherited tsx hooks resolve the source worker's .js import. const WORKER_MODULE = new URL('./rule-test-worker.js', import.meta.url); @@ -29,6 +29,50 @@ function validateCase(value: unknown, label: string): RuleTestCase { return value as unknown as RuleTestCase; } +interface RuleTestModule { + readonly tests?: unknown; + readonly aiTests?: unknown; +} + +async function loadExternalTests(ruleFile: string, testFile: string): Promise { + try { + await lstat(testFile); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } + + try { + const dependencies = getRuleDependencies(ruleFile); + if (dependencies !== undefined) registerRuleDependencies(testFile, dependencies); + // The companion path comes from the selected rule filename. + return await import(pathToFileURL(testFile).href) as RuleTestModule; + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + throw new Error(`Rule test module ${testFile} failed to import: ${detail}`, { cause: error }); + } +} + +interface RuleTestGroup { + readonly values: readonly unknown[]; + readonly label: string; + readonly ai: boolean; +} + +function addTestGroup( + groups: RuleTestGroup[], + values: unknown, + sourceLabel: string, + exportName: 'tests' | 'aiTests', + ai: boolean, +): void { + if (values === undefined) return; + if (!Array.isArray(values)) { + throw new Error(`${sourceLabel} ${exportName}: expected an array of test cases`); + } + groups.push({ values, label: `${sourceLabel} ${exportName}`, ai }); +} + type TestActual = RuleTestResult['actual']; class RuleTestWorkerSession { @@ -222,22 +266,24 @@ export async function runRuleTests( if (failureObserved) throw failure; if (!includeDisabled && !rule.rule.enabled) continue; // Discovery selects the rule module path at runtime. - const { tests, aiTests } = await import(pathToFileURL(rule.ruleFile).href) as { - readonly tests?: unknown; - readonly aiTests?: unknown; - }; + const ruleModule = await import(pathToFileURL(rule.ruleFile).href) as RuleTestModule; + const testFile = rule.ruleFile.replace(/\.rule\.(ts|mts|js|mjs)$/, '.test.$1'); + const externalModule = await loadExternalTests(rule.ruleFile, testFile); + const { tests, aiTests } = ruleModule; if (failureObserved) throw failure; - if (!includeAI && aiTests !== undefined) hasExcludedAITests = true; - const selectedAI = includeAI ? aiTests : undefined; - const label = `Rule "${rule.name}"`; - if (tests !== undefined && !Array.isArray(tests)) { - throw new Error(`${label} tests: expected an array of test cases`); + if (!includeAI && (aiTests !== undefined || externalModule?.aiTests !== undefined)) { + hasExcludedAITests = true; } - if (selectedAI !== undefined && !Array.isArray(selectedAI)) { - throw new Error(`${label} aiTests: expected an array of test cases`); + const label = `Rule "${rule.name}"`; + const externalLabel = `${label} test module ${testFile}`; + const groups: RuleTestGroup[] = []; + addTestGroup(groups, tests, label, 'tests', false); + addTestGroup(groups, externalModule?.tests, externalLabel, 'tests', false); + if (includeAI) { + addTestGroup(groups, aiTests, label, 'aiTests', true); + addTestGroup(groups, externalModule?.aiTests, externalLabel, 'aiTests', true); } - const regularCount = tests?.length ?? 0; - const count = regularCount + (selectedAI?.length ?? 0); + const count = groups.reduce((total, group) => total + group.values.length, 0); const completed = new Array(count); let nextIndex = 0; let nextResult = 0; @@ -246,11 +292,15 @@ export async function runRuleTests( const index = nextIndex++; if (index >= count) return; try { - const ai = index >= regularCount; - const caseIndex = ai ? index - regularCount : index; - const value = ai ? selectedAI![caseIndex] : tests![caseIndex]; - const example = validateCase(value, `${label} ${ai ? 'aiTests' : 'tests'}[${caseIndex}]`); - completed[index] = await runCase(rule, example, ai, timeoutMs, session, fail, includeDisabled); + let groupIndex = 0; + let caseIndex = index; + while (caseIndex >= groups[groupIndex]!.values.length) { + caseIndex -= groups[groupIndex]!.values.length; + groupIndex += 1; + } + const group = groups[groupIndex]!; + const example = validateCase(group.values[caseIndex], `${group.label}[${caseIndex}]`); + completed[index] = await runCase(rule, example, group.ai, timeoutMs, session, fail, includeDisabled); if (failureObserved) return; while (completed[nextResult] !== undefined) { const result = completed[nextResult]!; diff --git a/tests/rule-resolution.test.ts b/tests/rule-resolution.test.ts index af77b6e..4d86d06 100644 --- a/tests/rule-resolution.test.ts +++ b/tests/rule-resolution.test.ts @@ -201,6 +201,24 @@ for (const [locked, commonjs] of [[false, false], [false, true], [true, false], }); } +test('a rule-test companion cannot borrow an undeclared application package', async () => { + const directory = await project('ambient-test-companion'); + await put(join(directory, 'node_modules', 'ambient-only', 'package.json'), JSON.stringify({ + name: 'ambient-only', type: 'module', exports: './index.js', + })); + const marker = join(directory, 'ambient-test-executed'); + await put(join(directory, 'node_modules', 'ambient-only', 'index.js'), `import { writeFileSync } from 'node:fs'; writeFileSync(${JSON.stringify(marker)}, 'executed'); export const value = true;\n`); + await put(join(directory, '.coderuler', 'ambient.rule.mjs'), rule('', 'null')); + await put(join(directory, '.coderuler', 'ambient.test.mjs'), `import { test } from '@alex-dbs/coderuler'; +import { value } from 'ambient-only'; +export const tests = [test('ambient package').withFiles({ 'input.txt': String(value) }).passes()]; +`); + const dependencies = await installedDependencies(directory); + const loaded = await loadPrepared(directory, dependencies); + await rejects(() => runRuleTests(loaded, undefined, 10_000, 1), /ambient-only/, /undeclared|declar|dependency/i); + assert.equal(existsSync(marker), false, 'undeclared application package code must not execute'); +}); + test('a transitive dependency is not a direct rule import even when hoisted in local node_modules', async () => { const directory = await project('transitive-boundary', 1); await put(join(directory, '.coderuler', 'transitive.rule.mjs'), rule("import { message } from 'fixture-shared';", 'message')); @@ -536,7 +554,12 @@ export default class Versioned extends Rule { } export const tests = [ test('${name} first').withFiles({ 'inputs/a.txt': ${JSON.stringify(message)} }).passes(), - test('${name} second').withFiles({ 'inputs/a.txt': ${JSON.stringify(message)} }).passes(), +]; +`); + await put(join(owner, '.coderuler', `${name}.test.mjs`), `import { test } from '@alex-dbs/coderuler'; +import { message } from 'fixture-policy'; +export const tests = [ + test('${name} second').withFiles({ 'inputs/a.txt': message }).passes(), ]; `); } diff --git a/tests/rule-testing.test.ts b/tests/rule-testing.test.ts index 112f85c..266ac97 100644 --- a/tests/rule-testing.test.ts +++ b/tests/rule-testing.test.ts @@ -22,6 +22,206 @@ function writeRule(target: Sandbox, name: string, cases: string): string { return writeFile(join(target.cwd, '.coderuler', `${name}.rule.ts`), `${TEXT_RULE}\nexport const tests = [${cases}];\n`); } +test('external-only datasets load for each supported rule extension', async t => { + for (const extension of ['ts', 'mts', 'js', 'mjs']) { + await t.test(extension, async () => { + const target = sandbox.isolated(`external-only-${extension}`); + const directory = join(target.cwd, '.coderuler/nested'); + writeFile(join(directory, `text.rule.${extension}`), TEXT_RULE); + writeFile(join(directory, `text.test.${extension}`), `import { test } from '@alex-dbs/coderuler'; +export const tests = [ + test('external good ${extension}').withFiles({ 'inputs/good.txt': 'GOOD' }).passes(), + test('external bad ${extension}').withFiles({ 'inputs/bad.txt': 'BAD' }).violates(), +];`); + writeFile(join(directory, `unrelated.test.${extension}`), "throw new Error('unrelated test file was imported');\n"); + + const result = await runCli(['test'], target); + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.match(result.stdout, new RegExp(`PASS external good ${extension}`)); + assert.match(result.stdout, new RegExp(`PASS external bad ${extension}`)); + assert.equal(result.stdout.includes('unrelated test file'), false, result.stdout); + }); + } +}); + +test('external cases follow inline cases and rule selection stays on rule modules', async () => { + const target = sandbox.isolated('external-selection'); + const localPath = writeFile(join(target.cwd, '.coderuler/nested/local.rule.ts'), `${TEXT_RULE} +export const tests = [test('local inline').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()];`); + writeFile(join(target.cwd, '.coderuler/nested/local.test.ts'), `import { test } from '@alex-dbs/coderuler'; +export const tests = [test('local external').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()];`); + writeFile(join(target.home, '.coderuler/home.rule.ts'), `${TEXT_RULE} +export const tests = [test('home inline').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()];`); + writeFile(join(target.home, '.coderuler/home.test.ts'), `import { test } from '@alex-dbs/coderuler'; +export const tests = [test('home external').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()];`); + + const discovered = await runCli(['test'], target); + assert.equal(discovered.status, 0, discovered.stderr || discovered.stdout); + for (const name of ['local inline', 'local external', 'home inline', 'home external']) { + assert.equal(discovered.stdout.match(new RegExp(`PASS ${name}`, 'g'))?.length, 1, discovered.stdout); + } + assert.ok(discovered.stdout.indexOf('local inline') < discovered.stdout.indexOf('local external'), discovered.stdout); + assert.ok(discovered.stdout.indexOf('home inline') < discovered.stdout.indexOf('home external'), discovered.stdout); + assert.match(discovered.stdout, /^local$/m); + assert.match(discovered.stdout, /^home$/m); + + for (const [selector, options] of [ + ['local', []], + [localPath, ['--concurrency', '2']], + ['.coderuler/nested', []], + ] as const) { + const selected = await runCli(['test', ...options, selector], target); + assert.equal(selected.status, 0, selected.stderr || selected.stdout); + assert.match(selected.stdout, /^local$/m); + assert.doesNotMatch(selected.stdout, /^home$/m); + for (const name of ['local inline', 'local external']) { + assert.equal(selected.stdout.match(new RegExp(`PASS ${name}`, 'g'))?.length, 1, selected.stdout); + } + assert.ok(selected.stdout.indexOf('local inline') < selected.stdout.indexOf('local external'), selected.stdout); + } +}); + +test('check and validate ignore test companions, and test imports only selected enabled companions', async () => { + const target = sandbox.isolated('companion-selection'); + const throwingTest = (name: string): void => { + writeFile(join(target.cwd, `.coderuler/${name}.test.ts`), "throw new Error('companion exploded');\n"); + }; + writeFile(join(target.cwd, '.coderuler/throwing.rule.ts'), TEXT_RULE); + const throwingPath = join(target.cwd, '.coderuler/throwing.test.ts'); + throwingTest('throwing'); + writeFile(join(target.cwd, '.coderuler/unselected.rule.ts'), TEXT_RULE); + throwingTest('unselected'); + writeFile(join(target.cwd, '.coderuler/disabled.rule.ts'), `import { Rule } from '@alex-dbs/coderuler'; +export default class Disabled extends Rule { + enabled = false; + check = () => { throw new Error('disabled rule ran'); }; +}`); + throwingTest('disabled'); + writeRule(target, 'selected', `test('selected case').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()`); + writeFile(join(target.cwd, '.coderuler/selected.test.ts'), `import { test } from '@alex-dbs/coderuler'; +export const tests = [test('selected external').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()];`); + + for (const command of ['check', 'validate']) { + const result = await runCli([command], target); + assert.equal(result.status, 0, result.stderr || result.stdout); + assert.equal(result.stderr.includes('companion exploded'), false, result.stderr); + } + const selected = await runCli(['test', 'selected'], target); + assert.equal(selected.status, 0, selected.stderr || selected.stdout); + assert.match(selected.stdout, /PASS selected case/); + assert.match(selected.stdout, /PASS selected external/); + assert.doesNotMatch(selected.stderr, /companion exploded/); + + const broken = await runCli(['test', 'throwing'], target); + assert.equal(broken.status, 2, broken.stdout || broken.stderr); + assert.ok(broken.stderr.includes(throwingPath), broken.stderr); + assert.match(broken.stderr, /companion exploded/); +}); + +test('external test declarations report malformed data, import errors, and empty datasets', async () => { + const cases = [ + { + name: 'malformed', + source: 'export const tests = {};', + expected: /tests: expected an array of test cases/, + }, + { + name: 'unfinished', + source: `import { test } from '@alex-dbs/coderuler'; +export const tests = [test('unfinished case').withFiles({ 'inputs/a.txt': 'GOOD' })];`, + expected: /tests\[0\].*name must be/, + }, + { + name: 'missing-import', + source: `import './missing-helper.js'; +export const tests = [];`, + expected: /missing-helper/, + }, + ]; + for (const scenario of cases) { + const target = sandbox.isolated(`external-${scenario.name}`); + const rulePath = writeFile(join(target.cwd, `.coderuler/${scenario.name}.rule.ts`), TEXT_RULE); + const testPath = writeFile(join(target.cwd, `.coderuler/${scenario.name}.test.ts`), scenario.source); + const result = await runCli(['test', rulePath], target); + assert.equal(result.status, 2, result.stdout || result.stderr); + assert.ok(result.stderr.includes(testPath), result.stderr); + assert.match(result.stderr, scenario.expected); + } + + const empty = sandbox.isolated('external-empty'); + const emptyPath = writeFile(join(empty.cwd, '.coderuler/empty.rule.ts'), TEXT_RULE); + writeFile(join(empty.cwd, '.coderuler/empty.test.ts'), 'export const tests = [];\n'); + const noCases = await runCli(['test', emptyPath], empty); + assert.equal(noCases.status, 2, noCases.stdout || noCases.stderr); + assert.match(noCases.stderr, /No rule test cases found in the selected rules/); + + const inline = sandbox.isolated('external-empty-inline'); + const inlinePath = writeRule(inline, 'inline', `test('inline still runs').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()`); + writeFile(join(inline.cwd, '.coderuler/inline.test.ts'), 'export const tests = [];\n'); + const withInline = await runCli(['test', inlinePath], inline); + assert.equal(withInline.status, 0, withInline.stderr || withInline.stdout); + assert.match(withInline.stdout, /PASS inline still runs/); +}); + +test('external AI cases run after ordinary cases only with --ai', async t => { + const target = sandbox.isolated('external-ai'); + const fixture = await startDecisionFixture(); + t.after(() => fixture.close()); + fixture.answer = (_request, index) => ({ probability: index === 0 ? 0.9 : 0.1 }); + const rulePath = writeFile(join(target.cwd, '.coderuler/mixed.rule.ts'), `import { Rule, pass, test } from '@alex-dbs/coderuler'; +export default class Mixed extends Rule { + files = ['inputs/*.txt']; + check = (subject, context) => subject.source === 'LOCAL' + ? pass() + : context.ai({ rule: 'The input must be good.' }); +} +export const tests = [test('inline ordinary').withFiles({ 'inputs/inline.txt': 'LOCAL' }).passes()]; +export const aiTests = [test('inline AI pass').withFiles({ 'inputs/ai-good.txt': 'GOOD' }).passes()];`); + writeFile(join(target.cwd, '.coderuler/mixed.test.ts'), `import { test } from '@alex-dbs/coderuler'; +export const tests = [test('external ordinary').withFiles({ 'inputs/external.txt': 'LOCAL' }).passes()]; +export const aiTests = [test('external AI violation').withFiles({ 'inputs/ai-bad.txt': 'BAD' }).violates()];`); + + const ordinary = await runCli(['test', rulePath], target, fixture.env); + assert.equal(ordinary.status, 0, ordinary.stderr || ordinary.stdout); + assert.match(ordinary.stdout, /PASS inline ordinary/); + assert.match(ordinary.stdout, /PASS external ordinary/); + assert.doesNotMatch(ordinary.stdout, /inline AI pass|external AI violation/); + assert.equal(fixture.requests.length, 0); + + const selected = await runCli(['test', '--ai', '--concurrency', '1', rulePath], target, fixture.env); + assert.equal(selected.status, 0, selected.stderr || selected.stdout); + assert.equal(fixture.requests.length, 2); + const names = ['inline ordinary', 'external ordinary', 'inline AI pass', 'external AI violation']; + for (const name of names) { + assert.equal(selected.stdout.match(new RegExp(`PASS ${name}`, 'g'))?.length, 1, selected.stdout); + } + const positions = names.map(name => selected.stdout.indexOf(`PASS ${name}`)); + assert.ok(positions.every((position, index) => index === 0 || positions[index - 1]! < position), selected.stdout); +}); + +test('external AI-only and malformed AI exports keep the --ai selection rules', async () => { + const onlyAI = sandbox.isolated('external-ai-only'); + const onlyPath = writeFile(join(onlyAI.cwd, '.coderuler/live.rule.ts'), TEXT_RULE); + writeFile(join(onlyAI.cwd, '.coderuler/live.test.ts'), `import { test } from '@alex-dbs/coderuler'; +export const aiTests = [test('external live').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()];`); + const hint = await runCli(['test', onlyPath], onlyAI); + assert.equal(hint.status, 2, hint.stdout || hint.stderr); + assert.match(hint.stderr, /--ai/); + assert.match(hint.stderr, /No ordinary rule test cases found/); + + const malformed = sandbox.isolated('external-invalid-ai'); + const malformedPath = writeRule(malformed, 'ordinary', `test('ordinary case').withFiles({ 'inputs/a.txt': 'GOOD' }).passes()`); + const malformedTest = writeFile(join(malformed.cwd, '.coderuler/ordinary.test.ts'), 'export const aiTests = {};\n'); + const ordinary = await runCli(['test', malformedPath], malformed); + assert.equal(ordinary.status, 0, ordinary.stderr || ordinary.stdout); + assert.match(ordinary.stdout, /PASS ordinary case/); + const selected = await runCli(['test', '--ai', malformedPath], malformed); + assert.equal(selected.status, 2, selected.stdout || selected.stderr); + assert.ok(selected.stderr.includes(malformedTest), selected.stderr); + assert.match(selected.stderr, /aiTests: expected an array of test cases/); +}); + + test('test runs fluent cases through selection, filtering and grouping with isolated state and files', async () => { const target = sandbox.isolated('pipeline'); writeFile(join(target.cwd, '.coderuler/budget.rule.ts'), `import { Rule, scopes, pass, test, violation } from '@alex-dbs/coderuler'; diff --git a/website/src/content/docs/testing.md b/website/src/content/docs/testing.md index e01d0cc..dd3d84d 100644 --- a/website/src/content/docs/testing.md +++ b/website/src/content/docs/testing.md @@ -26,6 +26,8 @@ export const tests = [ ]; ``` +You can put these exports in `.coderuler/internal-url.test.ts`, beside `.coderuler/internal-url.rule.ts`. Keep the same extension for each pair, such as `.rule.mts` and `.test.mts`. Inline and companion cases run together. `coderuler test` selects rules, not test files. `check` and `validate` do not load companions. + Use this export in the [quickstart rule](/docs/quickstart/). `test(name).withFiles(files)` accepts a nonblank name and a map of root-relative fixture paths to JSON-compatible values. Strings become literal file content. Objects, arrays, numbers, booleans, and `null` become pretty-printed JSON. For YAML-specific syntax or several YAML documents, supply a string rather than an object. @@ -75,11 +77,11 @@ export TYPESAFE_API_KEY='your-api-key' coderuler test .coderuler/actionable-instructions.rule.ts --ai --ai-scores --timeout 45s ``` -`--ai` adds `aiTests` to the test cases and authorizes live provider calls. The requests send selected fixture content to the configured service and can incur charges. Results can vary. Provider errors are errors, not passes. `--ai-scores` displays each Jev probability; it does not authorize calls without `--ai`. +`--ai` adds inline and companion `aiTests` to the run and authorizes live provider calls. The requests send selected fixture content to the configured service and can incur charges. Results can vary. Provider errors are errors, not passes. `--ai-scores` displays each Jev probability; it does not authorize calls without `--ai`. ## What validation does not test -`validate` checks rule structure, not TypeScript semantics or policy behavior. Neither `validate` nor `check` executes inline tests. +`validate` checks rule structure, not TypeScript semantics or policy behavior. Neither `validate` nor `check` runs test cases or loads companion files. A selected rule set with no test cases from enabled rules is an error. If only `aiTests` exist, an ordinary `test` run tells you to include them with `--ai`. Do not use an empty dataset as a successful test run.