Add validated PR Review definitions

.github/workflows/pr-review-evaluation.yml

@@ -18,6 +18,14 @@ on:
           - "gpt-5.6-luna"
           - "gpt-5.3-codex"
           - "mai-code-1.1-flash"
+      definition-id:
+        description: "Registered PR Review definition (unclassified runs are never published)"
+        required: false
+        default: "pr-review-production-sol-luna-serial-v1"
+        type: choice
+        options:
+          - "pr-review-production-sol-luna-serial-v1"
+          - "unclassified"
       leaf-model:
         description: "Model requested for domain leaf agents"
         required: false
@@ -168,6 +176,7 @@ jobs:
 
           uv run bcbench evaluate pr-review "${{ matrix.entry }}" `
             --model "${{ inputs.model }}" `
+            --definition-id "${{ inputs.definition-id }}" `
             --engine-path "${{ steps.install-harnesses.outputs.bc-alagents-path }}" `
             --repo-path "${{ steps.setup-env.outputs.repo_path }}" `
             --cli-version "${{ steps.install-harnesses.outputs.copilot-version }}" `
@@ -195,7 +204,7 @@ jobs:
       agent: "BC PR Review"
       # Scores only the modified entries, so publishing it to Braintrust/Kusto or the
       # leaderboard would record a partial corpus as a benchmark result.
-      mock: ${{ inputs.test-run || inputs.modified-only || inputs.engine-sha != '' || inputs.entries != '' }}
+      mock: ${{ inputs.test-run || inputs.modified-only || inputs.engine-sha != '' || inputs.definition-id != 'pr-review-production-sol-luna-serial-v1' || inputs.entries != '' }}
       category: code-review
       git-ref: ${{ inputs.git-ref || github.ref_name }}
       benchmark-commit: ${{ github.sha }}
@@ -219,4 +228,4 @@ jobs:
       repeat: ${{ inputs.repeat }}
       existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "leaf-model": "${{ inputs.leaf-model }}", "leaf-execution": "${{ inputs.leaf-execution }}", "max-leaf-concurrency": "${{ inputs.max-leaf-concurrency }}", "engine-sha": ${{ toJSON(inputs.engine-sha) }}, "test-run": "${{ inputs.test-run }}", "entries": ${{ toJSON(inputs.entries) }}, "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "definition-id": "${{ inputs.definition-id }}", "leaf-model": "${{ inputs.leaf-model }}", "leaf-execution": "${{ inputs.leaf-execution }}", "max-leaf-concurrency": "${{ inputs.max-leaf-concurrency }}", "engine-sha": ${{ toJSON(inputs.engine-sha) }}, "test-run": "${{ inputs.test-run }}", "entries": ${{ toJSON(inputs.entries) }}, "git-ref": "${{ inputs.git-ref || github.ref_name }}"}

CATEGORIES.md

@@ -23,6 +23,12 @@ Start with `EvaluationCategory` in [src/bcbench/types.py](src/bcbench/types.py).
 
 Keep dataset entry classes and result classes focused on typed data. Put category-specific behavior in the pipeline.
 
+### Named definitions
+
+Standard single-model agents are identified by the shared `model` field. A composed runner may additionally expose a category-owned named definition when its category can validate the complete configuration from runtime evidence. Keep that optional `definition_id` and display name in the category's result, summary, and aggregate models; do not add a universal profile or definition object to shared base result, summary, aggregate, or metric schemas.
+
+The `code-review` category demonstrates this boundary for BC PR Review. Its release-owned registry accepts a definition only when root and leaf models, scheduling, engine and CLI pins, BCQuality provenance, and recorded review policy all match. The normal [Code Review dashboard](docs/code-review.md) displays the definition when present and otherwise falls back to `model`; the [Advanced Metrics view](docs/code-review-details.md) is the detailed provenance surface. Other categories, including BCal scenarios and checkpointed bug-fix lifecycles, may opt in later through their own category-owned metadata without changing shared schemas.
+
 ## Checklist
 
 Use the existing implementations as examples: `bug-fix` and `test-generation` for execution-based categories, `code-review` and `nl2al` for judge-based ones.

docs/code-review-details.md

@@ -49,7 +49,9 @@ Diagnostic averages use only tasks that reported the metric. Cross-run averages
   <thead>
     <tr>
       <th>Agent</th>
-      <th>Model</th>
+      <th>Root model</th>
+      <th>Definition</th>
+      <th>Definition ID</th>
       <th>Configuration</th>
       <th>Models (leaf → root)</th>
       <th>Leaf Model</th>
@@ -99,6 +101,10 @@ Diagnostic averages use only tasks that reported the metric. Cross-run averages
       <th>Copilot CLI</th>
       <th>BC-ALAgents</th>
       <th>BCQuality</th>
+      <th>Leaf model</th>
+      <th>Leaf scheduling</th>
+      <th>Review policy</th>
+      <th>BCQuality source snapshot</th>
     </tr>
   </thead>
   <tbody>
@@ -107,6 +113,8 @@ Diagnostic averages use only tasks that reported the metric. Cross-run averages
     <tr>
       <td>{{ agg.agent_name }}</td>
       <td>{{ agg.model }}</td>
+      <td>{{ agg.definition_name | default: agg.model }}</td>
+      <td>{% if agg.definition_id %}<code>{{ agg.definition_id }}</code>{% else %}—{% endif %}</td>
       <td>{% if agg.experiment %}<code>{{ agg.experiment | jsonify }}</code>{% else %}Baseline{% endif %}</td>
       <td>{% if agg.models and agg.models.size > 0 %}<code>{{ agg.models | join: " → " }}</code>{% else %}—{% endif %}</td>
       <td>{% if agg.leaf_model %}{{ agg.leaf_model }}{% else %}—{% endif %}</td>
@@ -156,6 +164,10 @@ Diagnostic averages use only tasks that reported the metric. Cross-run averages
       <td>{% if agg.copilot_cli_version %}{{ agg.copilot_cli_version }}{% else %}—{% endif %}</td>
       <td>{% if agg.bc_alagents_commit %}<a href="https://github.com/{{ agg.bc_alagents_repository }}/commit/{{ agg.bc_alagents_commit }}" target="_blank">{{ agg.bc_alagents_commit | slice: 0, 8 }}</a>{% else %}—{% endif %}</td>
       <td>{% if agg.bcquality_commit %}<a href="https://github.com/{{ agg.bcquality_repository }}/commit/{{ agg.bcquality_commit }}" target="_blank">{{ agg.bcquality_commit | slice: 0, 8 }}</a>{% if agg.bcquality_version %} ({{ agg.bcquality_version }}){% endif %}{% else %}—{% endif %}</td>
+      <td>{% if agg.leaf_model %}{{ agg.leaf_model }}{% else %}—{% endif %}</td>
+      <td>{% if agg.leaf_execution %}{{ agg.leaf_execution }} ({{ agg.max_leaf_concurrency }}){% else %}—{% endif %}</td>
+      <td>{% if agg.minimum_severity %}knowledge ≥ {{ agg.minimum_severity }}; agent ≥ {{ agg.agent_minimum_severity }}; {{ agg.cli_timeout_minutes }}m{% else %}—{% endif %}</td>
+      <td>{% if agg.bcquality_source_snapshot %}<code>{{ agg.bcquality_source_snapshot }}</code>{% else %}—{% endif %}</td>
     </tr>
     {% endfor %}
   </tbody>
@@ -174,7 +186,9 @@ Diagnostic averages use only tasks that reported the metric. Cross-run averages
     <tr>
       <th>Run</th>
       <th>Agent</th>
-      <th>Model</th>
+      <th>Root model</th>
+      <th>Definition</th>
+      <th>Definition ID</th>
       <th>Configuration</th>
       <th>Models (leaf → root)</th>
       <th>Leaf Model</th>
@@ -230,6 +244,10 @@ Diagnostic averages use only tasks that reported the metric. Cross-run averages
       <th>Copilot CLI</th>
       <th>BC-ALAgents</th>
       <th>BCQuality</th>
+      <th>Leaf model</th>
+      <th>Leaf scheduling</th>
+      <th>Review policy</th>
+      <th>BCQuality source snapshot</th>
     </tr>
   </thead>
   <tbody>
@@ -239,6 +257,8 @@ Diagnostic averages use only tasks that reported the metric. Cross-run averages
       <td>{% if run.github_run_id %}<a href="https://github.com/microsoft/BC-Bench/actions/runs/{{ run.github_run_id }}" target="_blank">{{ run.github_run_id }}</a>{% else %}—{% endif %}</td>
       <td>{{ run.agent_name }}</td>
       <td>{{ run.model }}</td>
+      <td>{{ run.definition_name | default: run.model }}</td>
+      <td>{% if run.definition_id %}<code>{{ run.definition_id }}</code>{% else %}—{% endif %}</td>
       <td>{% if run.experiment %}<code>{{ run.experiment | jsonify }}</code>{% else %}Baseline{% endif %}</td>
       <td>{% if run.models and run.models.size > 0 %}<code>{{ run.models | join: " → " }}</code>{% else %}—{% endif %}</td>
       <td>{% if run.leaf_model %}{{ run.leaf_model }}{% else %}—{% endif %}</td>
@@ -294,6 +314,10 @@ Diagnostic averages use only tasks that reported the metric. Cross-run averages
       <td>{% if run.copilot_cli_version %}{{ run.copilot_cli_version }}{% else %}—{% endif %}</td>
       <td>{% if run.bc_alagents_commit %}<a href="https://github.com/{{ run.bc_alagents_repository }}/commit/{{ run.bc_alagents_commit }}" target="_blank">{{ run.bc_alagents_commit | slice: 0, 8 }}</a>{% else %}—{% endif %}</td>
       <td>{% if run.bcquality_commit %}<a href="https://github.com/{{ run.bcquality_repository }}/commit/{{ run.bcquality_commit }}" target="_blank">{{ run.bcquality_commit | slice: 0, 8 }}</a>{% if run.bcquality_version %} ({{ run.bcquality_version }}){% endif %}{% else %}—{% endif %}</td>
+      <td>{% if run.leaf_model %}{{ run.leaf_model }}{% else %}—{% endif %}</td>
+      <td>{% if run.leaf_execution %}{{ run.leaf_execution }} ({{ run.max_leaf_concurrency }}){% else %}—{% endif %}</td>
+      <td>{% if run.minimum_severity %}knowledge ≥ {{ run.minimum_severity }}; agent ≥ {{ run.agent_minimum_severity }}; {{ run.cli_timeout_minutes }}m{% else %}—{% endif %}</td>
+      <td>{% if run.bcquality_source_snapshot %}<code>{{ run.bcquality_source_snapshot }}</code>{% else %}—{% endif %}</td>
     </tr>
     {% endfor %}
   </tbody>

docs/code-review.md

@@ -63,6 +63,8 @@ Aggregate telemetry must identify the expected CLI version and root/leaf model s
 
 Version `0.16.0` separates these PR Review routing and usage-eligibility changes from earlier benchmark results. Its default harness pins are Copilot CLI `1.0.88` (PR Review uses the telemetry-compatible `1.0.83`), Claude Code `2.1.281`, and BC-ALAgents `03239afe611a3eff490002eba0e4098b48099ba3`. The `run pr-review` and `evaluate pr-review` commands default to a Sol root and Luna leaves, matching the workflow.
 
+When a run exactly matches a release-owned PR Review definition, the default dashboard displays that **Definition** name. Standard single-model agents, historical rows, and PR Review rows with no recognized definition keep their existing model display instead. Historical data is never rewritten or guessed; only validated runtime evidence can produce a definition. The v1 registry's BCQuality commit and source snapshot are taken from the successful [evaluation-identity run 36001653480](https://github.com/microsoft/BC-Bench/actions/runs/36001653480) manifest artifact. The [Advanced Metrics view](code-review-details.html) shows the definition ID alongside the underlying root/leaf, scheduling, policy, engine, CLI, and BCQuality provenance.
+
 Unavailable AI credits remain `null` in bceval exports; observed zero remains zero. The pinned bc-eval 0.6.1 consumer requires numeric prompt/completion tokens, so its existing zero fallbacks for missing tokens remain unchanged. Use the original per-entry result metrics, not bceval token fields, to distinguish unknown usage from measured zero.
 
 ## Production BC PR Review Baseline
@@ -72,7 +74,7 @@ Unavailable AI credits remain `null` in bceval exports; observed zero remains ze
   <thead>
     <tr>
       <th>Agent</th>
-      <th>Model</th>
+      <th>Definition</th>
       <th>Micro F1 (95% CI)</th>
       <th>Precision</th>
       <th>Recall</th>
@@ -88,7 +90,7 @@ Unavailable AI credits remain `null` in bceval exports; observed zero remains ze
       {% if agg.experiment == null or agg.experiment.is_experiment == false %}
     <tr>
       <td>{{ agg.agent_name }}</td>
-      <td>{{ agg.model }}</td>
+      <td>{{ agg.definition_name | default: agg.model }}</td>
       <td>{{ agg.f1 | times: 100.0 | round: 1 }}%{% if agg.f1_ci_low %} ({{ agg.f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}</td>
       <td>{{ agg.precision | times: 100.0 | round: 1 }}%</td>
       <td>{{ agg.recall | times: 100.0 | round: 1 }}%</td>
@@ -111,7 +113,7 @@ Unavailable AI credits remain `null` in bceval exports; observed zero remains ze
   <thead>
     <tr>
       <th>Agent</th>
-      <th>Model</th>
+      <th>Definition</th>
       <th>Avg Time</th>
       <th>Avg Prompt Tokens</th>
       <th>Avg Completion Tokens</th>
@@ -127,7 +129,7 @@ Unavailable AI credits remain `null` in bceval exports; observed zero remains ze
       {% if agg.experiment == null or agg.experiment.is_experiment == false %}
     <tr>
       <td>{{ agg.agent_name }}</td>
-      <td>{{ agg.model }}</td>
+      <td>{{ agg.definition_name | default: agg.model }}</td>
       <td>{{ agg.average_duration | round: 1 }}s</td>
       <td>{% if agg.average_prompt_tokens != null %}{{ agg.average_prompt_tokens | round: 0 }}{% else %}—{% endif %}</td>
       <td>{% if agg.average_completion_tokens != null %}{{ agg.average_completion_tokens | round: 0 }}{% else %}—{% endif %}</td>
@@ -153,7 +155,7 @@ These historical rows evaluate the generic Copilot or Claude runners against the
   <thead>
     <tr>
       <th>Agent</th>
-      <th>Model</th>
+      <th>Definition</th>
       <th>Micro F1</th>
       <th>Precision</th>
       <th>Recall</th>
@@ -166,7 +168,7 @@ These historical rows evaluate the generic Copilot or Claude runners against the
     {% for agg in legacy_results %}
     <tr>
       <td>{{ agg.agent_name }}</td>
-      <td>{{ agg.model }}</td>
+      <td>{{ agg.definition_name | default: agg.model }}</td>
       <td>{{ agg.f1 | times: 100.0 | round: 1 }}%</td>
       <td>{{ agg.precision | times: 100.0 | round: 1 }}%</td>
       <td>{{ agg.recall | times: 100.0 | round: 1 }}%</td>
@@ -194,7 +196,7 @@ For detailed aggregate and per-run quality, performance, configuration, and usag
     <tr>
       <th>Variant</th>
       <th>Agent</th>
-      <th>Model</th>
+      <th>Definition</th>
       <th>Micro F1 (95% CI)</th>
       <th>Macro F1 (95% CI)</th>
       <th>Precision</th>
@@ -212,7 +214,7 @@ For detailed aggregate and per-run quality, performance, configuration, and usag
         {%- if agg.experiment.custom_instructions -%}Inline knowledge (pre-#8700){%- else -%}Other{%- endif -%}
       </td>
       <td>{{ agg.agent_name }}</td>
-      <td>{{ agg.model }}</td>
+      <td>{{ agg.definition_name | default: agg.model }}</td>
       <td>{{ agg.f1 | times: 100.0 | round: 1 }}%{% if agg.f1_ci_low %} ({{ agg.f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}</td>
       <td>{{ agg.macro_f1 | times: 100.0 | round: 1 }}%{% if agg.macro_f1_ci_low %} ({{ agg.macro_f1_ci_low | times: 100.0 | round: 1 }}-{{ agg.macro_f1_ci_high | times: 100.0 | round: 1 }}%){% endif %}</td>
       <td>{{ agg.precision | times: 100.0 | round: 1 }}%</td>

src/bcbench/agent/pr_review/agent.py

@@ -24,6 +24,7 @@
 from bcbench_core.git import commit_changes, has_changes, init_repo
 
 from bcbench.agent.copilot.cli import get_copilot_version
+from bcbench.agent.pr_review.definitions import validate_requested_pr_review_definition
 from bcbench.agent.pr_review.metrics import build_pr_review_metrics
 from bcbench.agent.pr_review.review_output import engine_report_to_review_comments, load_engine_report
 from bcbench.agent.pr_review.run_manifest import RUN_MANIFEST_FILE_NAME, load_run_manifest, validate_run_manifest
@@ -183,6 +184,7 @@ def run_pr_review_agent(
     leaf_execution: Literal["serial", "parallel"] = "serial",
     max_leaf_concurrency: int = 4,
     cli_timeout_minutes: int = 30,
+    definition_id: str | None = None,
 ) -> tuple[PRReviewMetrics, ExperimentConfiguration]:
     """Run the engine's complete local review pipeline and write review.json.
 
@@ -221,6 +223,15 @@ def run_pr_review_agent(
         raise AgentError("PR Review max leaf concurrency must be a positive integer.")
     if cli_timeout_minutes < 0:
         raise AgentError("PR Review CLI timeout minutes must be a non-negative integer.")
+    requested_definition = validate_requested_pr_review_definition(
+        definition_id,
+        engine_commit=agent_version,
+        copilot_cli_version=cli_version,
+        root_model=model,
+        leaf_model=leaf_model,
+        leaf_execution=leaf_execution,
+        max_leaf_concurrency=max_leaf_concurrency,
+    )
 
     engine = engine_root / "agents" / "ALReviewAgent" / "scripts" / "Invoke-CopilotPRReview.ps1"
     env = {
@@ -239,6 +250,7 @@ def run_pr_review_agent(
         "COPILOT_REVIEW_LEAF_EXECUTION": leaf_execution,
         "COPILOT_REVIEW_MAX_LEAF_CONCURRENCY": str(max_leaf_concurrency),
         "COPILOT_REVIEW_CLI_TIMEOUT_MINUTES": str(cli_timeout_minutes),
+        "MINIMUM_SEVERITY": settings["min_severity"],
         "AGENT_MINIMUM_SEVERITY": severity,
     }
 
@@ -282,13 +294,13 @@ def run_pr_review_agent(
         logger.exception("Unexpected error running engine review")
         raise
     else:
-        return (
-            build_pr_review_metrics(
-                output_dir,
-                bcquality_root,
-                time.monotonic() - start,
-                manifest=manifest,
-                engine_root=engine_root,
-            ),
-            config,
+        metrics = build_pr_review_metrics(
+            output_dir,
+            bcquality_root,
+            time.monotonic() - start,
+            manifest=manifest,
+            engine_root=engine_root,
         )
+        if requested_definition is not None and metrics.definition_id != requested_definition.id:
+            raise AgentError(f"Validated engine evidence did not resolve to requested BC PR Review definition {requested_definition.id!r}.")
+        return metrics, config