External rule tests

README.md

@@ -15,6 +15,7 @@ Install the CLI globally:
 ```sh
 npm install -g @alex-dbs/coderuler
 coderuler --help
+coderuler skill install
 ```
 
 The npm package is scoped. The command is `coderuler`. A global installation can load rules without a project-local dependency.
@@ -24,6 +25,7 @@ For a project-specific CLI version, install it locally:
 ```sh
 npm install --save-dev @alex-dbs/coderuler
 npx coderuler --help
+npx coderuler skill install
 ```
 
 The global installation is optional: with only the project-local dependency, run commands through `npx coderuler`.

benchmarks/result.md

@@ -1,8 +1,8 @@
 
-| Method                      | Accuracy | Precision | Recall | F1    | FP / FN | Tokens used (input / output) |
-| --------------------------- | --------: | ---------: | ------: | -----: | -------: | ----------------------------: |
-| Generic reviewer, first run | 47.5%    | 68.4%     | 34.2%  | 45.6% | 6 / 25  | 17,403,725 / 231,053         |
-| OpenAI                      | 52.5%    | 72.7%     | 42.1%  | 53.3% | 6 / 22  | 863,703 / 0                  |
-| Jev                         | 54.2%    | 78.9%     | 39.5%  | 52.6% | 4 / 23  | 863,703 / 0                  |
+| Method           | Accuracy | Precision | Recall | F1    | FP / FN | Tokens used (input / output)    |
+| ---------------- | --------: | ---------: | ------: | -----: | -------: | -------------------------------: |
+| Generic reviewer | 47.5%    | 68.4%     | 34.2%  | 45.6% | 6 / 25  | 17,403,725 / 231,053 - ($74.24) |
+| OpenAI           | 52.5%    | 72.7%     | 42.1%  | 53.3% | 6 / 22  | 863,703 / 0 - ($0.06)           |
+| Jev              | 54.2%    | 78.9%     | 39.5%  | 52.6% | 4 / 23  | 863,703 / 0 - ($0.04)           |
 
 

skills/coderuler-new-rule/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: coderuler-new-rule
-description: Create CodeRuler rules, including custom selection, filters and checks, inline tests, and inspection.
+description: Create CodeRuler rules, including custom selection, filters and checks, inline or companion tests, and inspection.
 ---
 
 # Create a CodeRuler rule
@@ -29,7 +29,9 @@ Start the name with its language or technology prefix, such as `go-`, `ts-`, or
 
 Keep rules short, with only a few lines of policy code. Reuse built-in selectors, evaluators, and standard libraries before writing custom logic. Add helpers only for real reuse.
 
-For a deterministic rule, add a named `tests` export with `.passes()` and `.violates()` cases. For an AI rule, put every test case in `aiTests` and do not add a separate `tests` export. Omit `.skips()` assertions when creating a rule: skipped inputs never reach the check and provide no evidence of policy behavior.
+Test cases can live in the rule module or in a sibling `.test` module. For example, `sample.rule.ts` pairs with `sample.test.ts`; keep the same extension for `.mts`, `.js`, and `.mjs`. The `test` command combines exports from both modules. It runs ordinary `tests` by default; include `--ai` to run `aiTests`.
+
+For a deterministic rule, export named `tests` with `.passes()` and `.violates()` cases. For an AI rule, put every test case in `aiTests` and do not add a separate `tests` export. Omit `.skips()` assertions when creating a rule: skipped inputs never reach the check and provide no evidence of policy behavior.
 
 For an AI rule, use `instruction` to state what compliant code does, beginning with a direct requirement. Add only applicability boundaries or real exceptions needed to define the policy; keep one policy per rule. Keep reporting language in `violationMessage`, examples in `examples`, and test cases in `aiTests`—not in the instruction. See [AI instruction style](reference.md#ai-instruction-style).
 
@@ -94,7 +96,7 @@ coderuler validate
 coderuler test
 ```
 
-`validate` checks the module contract, not policy behavior or TypeScript semantics. `check` and `validate` do not execute inline tests. `test` with no selected cases is an error, not success.
+`validate` checks the module contract, not policy behavior or TypeScript semantics. `check` and `validate` do not run test cases or load sibling `.test` modules. `test` with no selected cases is an error, not success.
 
 Useful test options:
 

src/adapter/filesystem/rule-tests.ts

@@ -1,4 +1,4 @@
-import { mkdtemp, realpath, rm } from 'node:fs/promises';
+import { lstat, mkdtemp, realpath, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { pathToFileURL } from 'node:url';
@@ -8,7 +8,7 @@ import type { LoadedRule } from '../../app/ports.js';
 import type { RuleTestCase, RuleTestResult } from '../../app/rule-tests.js';
 import { isRecord, validateJsonValue } from '../../app/validate.js';
 import type { RuleTestWorkerMessage, RuleTestWorkerOutcome, RuleTestWorkerRequest } from './rule-test-worker.js';
-import { getRuleDependencies } from './rule-resolution.js';
+import { getRuleDependencies, registerRuleDependencies } from './rule-resolution.js';
 
 // A JavaScript entry lets inherited tsx hooks resolve the source worker's .js import.
 const WORKER_MODULE = new URL('./rule-test-worker.js', import.meta.url);
@@ -29,6 +29,50 @@ function validateCase(value: unknown, label: string): RuleTestCase {
   return value as unknown as RuleTestCase;
 }
 
+interface RuleTestModule {
+  readonly tests?: unknown;
+  readonly aiTests?: unknown;
+}
+
+async function loadExternalTests(ruleFile: string, testFile: string): Promise<RuleTestModule | undefined> {
+  try {
+    await lstat(testFile);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
+    throw error;
+  }
+
+  try {
+    const dependencies = getRuleDependencies(ruleFile);
+    if (dependencies !== undefined) registerRuleDependencies(testFile, dependencies);
+    // The companion path comes from the selected rule filename.
+    return await import(pathToFileURL(testFile).href) as RuleTestModule;
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new Error(`Rule test module ${testFile} failed to import: ${detail}`, { cause: error });
+  }
+}
+
+interface RuleTestGroup {
+  readonly values: readonly unknown[];
+  readonly label: string;
+  readonly ai: boolean;
+}
+
+function addTestGroup(
+  groups: RuleTestGroup[],
+  values: unknown,
+  sourceLabel: string,
+  exportName: 'tests' | 'aiTests',
+  ai: boolean,
+): void {
+  if (values === undefined) return;
+  if (!Array.isArray(values)) {
+    throw new Error(`${sourceLabel} ${exportName}: expected an array of test cases`);
+  }
+  groups.push({ values, label: `${sourceLabel} ${exportName}`, ai });
+}
+
 type TestActual = RuleTestResult['actual'];
 
 class RuleTestWorkerSession {
@@ -222,22 +266,24 @@ export async function runRuleTests(
       if (failureObserved) throw failure;
       if (!includeDisabled && !rule.rule.enabled) continue;
       // Discovery selects the rule module path at runtime.
-      const { tests, aiTests } = await import(pathToFileURL(rule.ruleFile).href) as {
-        readonly tests?: unknown;
-        readonly aiTests?: unknown;
-      };
+      const ruleModule = await import(pathToFileURL(rule.ruleFile).href) as RuleTestModule;
+      const testFile = rule.ruleFile.replace(/\.rule\.(ts|mts|js|mjs)$/, '.test.$1');
+      const externalModule = await loadExternalTests(rule.ruleFile, testFile);
+      const { tests, aiTests } = ruleModule;
       if (failureObserved) throw failure;
-      if (!includeAI && aiTests !== undefined) hasExcludedAITests = true;
-      const selectedAI = includeAI ? aiTests : undefined;
-      const label = `Rule "${rule.name}"`;
-      if (tests !== undefined && !Array.isArray(tests)) {
-        throw new Error(`${label} tests: expected an array of test cases`);
+      if (!includeAI && (aiTests !== undefined || externalModule?.aiTests !== undefined)) {
+        hasExcludedAITests = true;
       }
-      if (selectedAI !== undefined && !Array.isArray(selectedAI)) {
-        throw new Error(`${label} aiTests: expected an array of test cases`);
+      const label = `Rule "${rule.name}"`;
+      const externalLabel = `${label} test module ${testFile}`;
+      const groups: RuleTestGroup[] = [];
+      addTestGroup(groups, tests, label, 'tests', false);
+      addTestGroup(groups, externalModule?.tests, externalLabel, 'tests', false);
+      if (includeAI) {
+        addTestGroup(groups, aiTests, label, 'aiTests', true);
+        addTestGroup(groups, externalModule?.aiTests, externalLabel, 'aiTests', true);
       }
-      const regularCount = tests?.length ?? 0;
-      const count = regularCount + (selectedAI?.length ?? 0);
+      const count = groups.reduce((total, group) => total + group.values.length, 0);
       const completed = new Array<RuleTestResult | undefined>(count);
       let nextIndex = 0;
       let nextResult = 0;
@@ -246,11 +292,15 @@ export async function runRuleTests(
           const index = nextIndex++;
           if (index >= count) return;
           try {
-            const ai = index >= regularCount;
-            const caseIndex = ai ? index - regularCount : index;
-            const value = ai ? selectedAI![caseIndex] : tests![caseIndex];
-            const example = validateCase(value, `${label} ${ai ? 'aiTests' : 'tests'}[${caseIndex}]`);
-            completed[index] = await runCase(rule, example, ai, timeoutMs, session, fail, includeDisabled);
+            let groupIndex = 0;
+            let caseIndex = index;
+            while (caseIndex >= groups[groupIndex]!.values.length) {
+              caseIndex -= groups[groupIndex]!.values.length;
+              groupIndex += 1;
+            }
+            const group = groups[groupIndex]!;
+            const example = validateCase(group.values[caseIndex], `${group.label}[${caseIndex}]`);
+            completed[index] = await runCase(rule, example, group.ai, timeoutMs, session, fail, includeDisabled);
             if (failureObserved) return;
             while (completed[nextResult] !== undefined) {
               const result = completed[nextResult]!;

tests/rule-resolution.test.ts

@@ -201,6 +201,24 @@ for (const [locked, commonjs] of [[false, false], [false, true], [true, false],
   });
 }
 
+test('a rule-test companion cannot borrow an undeclared application package', async () => {
+  const directory = await project('ambient-test-companion');
+  await put(join(directory, 'node_modules', 'ambient-only', 'package.json'), JSON.stringify({
+    name: 'ambient-only', type: 'module', exports: './index.js',
+  }));
+  const marker = join(directory, 'ambient-test-executed');
+  await put(join(directory, 'node_modules', 'ambient-only', 'index.js'), `import { writeFileSync } from 'node:fs'; writeFileSync(${JSON.stringify(marker)}, 'executed'); export const value = true;\n`);
+  await put(join(directory, '.coderuler', 'ambient.rule.mjs'), rule('', 'null'));
+  await put(join(directory, '.coderuler', 'ambient.test.mjs'), `import { test } from '@alex-dbs/coderuler';
+import { value } from 'ambient-only';
+export const tests = [test('ambient package').withFiles({ 'input.txt': String(value) }).passes()];
+`);
+  const dependencies = await installedDependencies(directory);
+  const loaded = await loadPrepared(directory, dependencies);
+  await rejects(() => runRuleTests(loaded, undefined, 10_000, 1), /ambient-only/, /undeclared|declar|dependency/i);
+  assert.equal(existsSync(marker), false, 'undeclared application package code must not execute');
+});
+
 test('a transitive dependency is not a direct rule import even when hoisted in local node_modules', async () => {
   const directory = await project('transitive-boundary', 1);
   await put(join(directory, '.coderuler', 'transitive.rule.mjs'), rule("import { message } from 'fixture-shared';", 'message'));
@@ -536,7 +554,12 @@ export default class Versioned extends Rule {
 }
 export const tests = [
   test('${name} first').withFiles({ 'inputs/a.txt': ${JSON.stringify(message)} }).passes(),
-  test('${name} second').withFiles({ 'inputs/a.txt': ${JSON.stringify(message)} }).passes(),
+];
+`);
+    await put(join(owner, '.coderuler', `${name}.test.mjs`), `import { test } from '@alex-dbs/coderuler';
+import { message } from 'fixture-policy';
+export const tests = [
+  test('${name} second').withFiles({ 'inputs/a.txt': message }).passes(),
 ];
 `);
   }