Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions evaluation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ clean control deliberately writes UTF-8 text and downloads the existing bytes
through a default-encoding InStream: transport alone does not transcode text
and must not trigger the encoding rule.

An article whose rule is about comments themselves cannot be evaluated once its samples lose their full-line comments. List it under `keepComments` in `review-fixtures.json` (an `article` path and a non-empty `reason`) and preparation keeps that article's sample comments; `Good`/`Bad` tokens are still neutralized. Use it only when the rule is about the comments, and prefer `coverageWaivers` when keeping them would reveal the expected outcome.

The SCM `articles` override deliberately selects every rule in the initial
functional domain, producing nine positive cases and nine clean controls.
Business context is executable: document/status `TestField` guards,
Expand Down
10 changes: 10 additions & 0 deletions evaluation/review-fixtures.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,16 @@
"reason": "The defect is the expectation that the layout becomes the default; the code alone is identical to a legitimate optional layout once comments are removed."
}
],
"keepComments": [
{
"article": "microsoft/knowledge/testing/test-feature-scenario-tags.md",
"reason": "The rule is about the [FEATURE]/[SCENARIO]/[GIVEN]/[WHEN]/[THEN] comment tags themselves; stripping full-line comments removes the tags from the good sample."
},
{
"article": "microsoft/knowledge/style/al-comments-must-not-restate-what-code-already-shows.md",
"reason": "The rule is about comments themselves; stripping full-line comments leaves its bad sample with no comments, so the rule can never fire."
}
],
"overrides": {
"copilot": {
"articles": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ codeunit 50102 "Item Price Testing"
SalesHeader: Record "Sales Header";
SalesLine: Record "Sales Line";
begin
// setup mixed with assertions, no clear layers, no FEATURE/SCENARIO/GIVEN/WHEN/THEN tags
LibrarySales.CreateCustomer(Customer);
LibraryInventory.CreateItem(Item);
LibraryPriceCalculation.CreatePriceHeader(
Expand Down
33 changes: 32 additions & 1 deletion tools/Test-ReviewFixtures.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,22 @@ foreach ($overrideDomain in $overrides.Keys) {
}
}

$keepCommentsArticles = @{}
$keepCommentsSeen = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
if ($manifest.PSObject.Properties.Name -contains 'keepComments') {
foreach ($entry in @($manifest.keepComments)) {
if (-not $entry.article -or -not $entry.reason) {
$problems.Add('Each keepComments entry requires non-empty article and reason values.') | Out-Null
continue
}
if ($keepCommentsArticles.ContainsKey([string]$entry.article)) {
$problems.Add("Duplicate keepComments entry: $($entry.article)") | Out-Null
continue
}
$keepCommentsArticles[[string]$entry.article] = [string]$entry.reason
}
}

$coverageWaivers = @{}
if ($manifest.PSObject.Properties.Name -contains 'coverageWaivers') {
foreach ($waiver in @($manifest.coverageWaivers)) {
Expand Down Expand Up @@ -300,12 +316,21 @@ foreach ($domain in $leafDomains) {
if ($context) {
$case | Add-Member -NotePropertyName context -NotePropertyValue $context
}
if ($keepCommentsArticles.ContainsKey($articlePath)) {
$case | Add-Member -NotePropertyName keepComments -NotePropertyValue $true
$keepCommentsSeen.Add($articlePath) | Out-Null
}
$caseList.Add($case) | Out-Null
}
}

}
$cases = @($caseList)
foreach ($keepCommentsArticle in $keepCommentsArticles.Keys) {
if (-not $keepCommentsSeen.Contains($keepCommentsArticle)) {
$problems.Add("keepComments article is not a selected evaluation article: $keepCommentsArticle") | Out-Null
}
}

$seenIds = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::Ordinal)
foreach ($case in $cases) {
Expand Down Expand Up @@ -464,7 +489,13 @@ if ($PrepareDirectory) {
# Good, Bad, or Anti-pattern. Strip full-line comments and neutralize those
# object-name tokens so model-facing fixtures do not reveal the expected
# outcome while preserving executable AL structure and references.
$neutralText = [regex]::Replace($sourceText, '(?m)^\s*//.*(?:\r?\n|$)', '')
# An article whose rule is about the comments themselves is listed under
# keepComments in review-fixtures.json and keeps them; its samples carry no
# answer-revealing labels (Good/Bad is still neutralized below).
$neutralText = $sourceText
if (-not ($case.PSObject.Properties.Name -contains 'keepComments' -and $case.keepComments)) {
$neutralText = [regex]::Replace($neutralText, '(?m)^\s*//.*(?:\r?\n|$)', '')
}
$neutralText = [regex]::Replace($neutralText, '\b(?:Good|Bad)\b', 'Eval')
Set-Content -LiteralPath (Join-Path $PrepareDirectory $neutralName) -Value $neutralText -Encoding UTF8
$requestCase = [pscustomobject]@{ id = $modelId; file = $neutralName }
Expand Down
Loading