diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c6741836..6e18361c 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ { "name": "bcquality", "source": "./", - "description": "Business Central AL quality knowledge base and review skills, packaged as an installable plugin. Exposes AL code review and preview plan guidance skills while preserving BCQuality's internal Entry and action-skill protocols.", + "description": "Business Central AL quality knowledge base and review skills, packaged as an installable plugin. Exposes AL code review, cited knowledge consultation and preview plan guidance skills while preserving BCQuality's internal Entry and action-skill protocols.", "version": "1.9.0", "skills": [ "./skills/" diff --git a/.github/scripts/Test-SkillIndex.ps1 b/.github/scripts/Test-SkillIndex.ps1 index 704a8d8e..ab560067 100644 --- a/.github/scripts/Test-SkillIndex.ps1 +++ b/.github/scripts/Test-SkillIndex.ps1 @@ -100,6 +100,30 @@ try { 'microsoft/skills/review/al-finance-review.md' ) $review = @($skills | Where-Object id -eq 'al-code-review') + $knowledge = @($skills | Where-Object id -eq 'al-knowledge') + if ($knowledge.Count -ne 1 -or (@($knowledge[0].inputs) -join ',') -ne 'knowledge-query' -or + (@($knowledge[0].outputs) -join ',') -ne 'knowledge-response') { + throw 'Expected one knowledge-only al-knowledge action.' + } + $knowledgeCandidates = @($skills | Where-Object { 'knowledge-query' -in $_.inputs }) + if ($knowledgeCandidates.Count -ne 1 -or $knowledgeCandidates[0].id -ne 'al-knowledge') { + throw 'A knowledge-only request must exclude review actions.' + } + # Entry applies READ's non-empty technology intersection to indexed actions. + $alKnowledge = @($knowledgeCandidates | Where-Object { 'al' -in $_.filters.technologies }) + if ($alKnowledge.Count -ne 1 -or $alKnowledge[0].id -ne 'al-knowledge') { + throw 'Entry must admit knowledge-query with technologies: [al].' + } + foreach ($dimension in 'bc-version', 'countries', 'application-area') { + $expected = if ($dimension -eq 'countries') { 'w1' } else { 'all' } + if ($expected -notin $knowledge[0].filters.$dimension) { + throw "Missing knowledge action applicability: $dimension" + } + } + $reviewCandidates = @($skills | Where-Object { 'file-path' -in $_.inputs -and 'al' -in $_.filters.technologies }) + if ('al-code-review' -notin $reviewCandidates.id -or 'al-knowledge' -in $reviewCandidates.id) { + throw 'AL file input must admit code review and exclude knowledge consultation.' + } if ($review.Count -ne 1) { throw "Expected exactly one al-code-review record, found $($review.Count)." } diff --git a/.github/scripts/test_knowledge_response.py b/.github/scripts/test_knowledge_response.py new file mode 100644 index 00000000..cbe7d80b --- /dev/null +++ b/.github/scripts/test_knowledge_response.py @@ -0,0 +1,57 @@ +"""Validate knowledge-response examples and reject unsafe citation shapes.""" +import copy +import json +from pathlib import Path + +from jsonschema import Draft7Validator + +ROOT = Path(__file__).resolve().parents[2] +schema = json.loads((ROOT / 'schemas/knowledge-response.schema.json').read_text()) +Draft7Validator.check_schema(schema) +validator = Draft7Validator(schema) +base = { + 'skill': {'id': 'al-knowledge', 'version': 1}, + 'outcome': 'completed', 'question': 'When is SetLoadFields useful?', + 'answer': 'Use it for partial records under the cited conditions.', + 'references': [{ + 'path': 'microsoft/knowledge/performance/use-setloadfields-for-partial-records.md', + 'applicability': 'conditional', 'unknown': ['bc-version'], + }], 'suppressed': [], +} +assert (ROOT / base['references'][0]['path']).is_file() +cases = 0 + + +def check(value, valid): + global cases + errors = list(validator.iter_errors(value)) + assert bool(errors) != valid, [error.message for error in errors] + cases += 1 + + +check(base, True) +value = copy.deepcopy(base) +value['references'][0]['unknown'] = [] +check(value, False) +value['references'][0]['applicability'] = 'applicable' +check(value, True) +value['references'][0]['unknown'] = ['bc-version'] +check(value, False) +for outcome in ('failed', 'no-knowledge'): + value = copy.deepcopy(base) + value.update(outcome=outcome, **{'outcome-reason': 'No usable knowledge'}) + check(value, False) + value['references'] = [] + check(value, True) +value = copy.deepcopy(base) +value['references'] = [] +check(value, False) +value = copy.deepcopy(base) +value['outcome'] = 'partial' +check(value, False) +value['outcome-reason'] = 'Target version is unknown' +check(value, True) +value = copy.deepcopy(base) +value['references'][0]['path'] = 'invented/article.md' +check(value, False) +print(f'Knowledge-response schema: {cases} positive/negative cases passed.') diff --git a/.github/scripts/test_knowledge_response_semantics.py b/.github/scripts/test_knowledge_response_semantics.py new file mode 100644 index 00000000..777e9b42 --- /dev/null +++ b/.github/scripts/test_knowledge_response_semantics.py @@ -0,0 +1,187 @@ +"""Regress input fidelity and complete citation reads using consumer evidence.""" +import copy +import hashlib +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +REPO = Path(__file__).resolve().parents[2] +TOOL = REPO / "tools/validate_knowledge_response.py" +spec = importlib.util.spec_from_file_location("knowledge_validator", TOOL) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class KnowledgeSemantics(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / "schemas").mkdir() + (self.root / "schemas/knowledge-response.schema.json").write_bytes((REPO / "schemas/knowledge-response.schema.json").read_bytes()) + self.path = "microsoft/knowledge/performance/example.md" + self.article = self.root / self.path + self.article.parent.mkdir(parents=True) + self.article.write_bytes(b"# Example\n\nComplete article content.\n") + self.question = " Question with accents: caf\u00e9?\r\n " + self.response = {"skill": {"id": "al-knowledge", "version": 1}, + "outcome": "completed", "question": self.question, "answer": "Supported answer.", + "references": [{"path": self.path, "applicability": "applicable", "unknown": []}], "suppressed": []} + content = self.article.read_bytes() + self.evidence = {"runId": "test-run", "questionSha256": hashlib.sha256(self.question.encode()).hexdigest(), + "reads": [{"path": self.path, "complete": True, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()}]} + + def errors(self, evidence=True): + return module.validate_response(self.response, question=self.question, root=self.root, + evidence=self.evidence if evidence else None, run_id="test-run") + + def test_disabled_citation_layer(self): + errors = module.validate_response(self.response, question=self.question, root=self.root, + evidence=self.evidence, run_id="test-run", enabled_layers=["community"]) + self.assertTrue(any("disabled layer" in error for error in errors)) + + def test_configuration_suppression_disabled_layer(self): + self.response.update(outcome="no-knowledge", references=[], suppressed=[{"path": self.path, "reason": "configuration"}]) + self.assertEqual(module.validate_response(self.response, question=self.question, root=self.root, + run_id="test-run", enabled_layers=[]), []) + + def test_precedence_suppression_disabled_layer(self): + self.response.update(outcome="no-knowledge", references=[], suppressed=[{"path": self.path, "reason": "layer-precedence"}]) + self.assertTrue(module.validate_response(self.response, question=self.question, root=self.root, + run_id="test-run", enabled_layers=[])) + + def test_suppressed_path_traversal(self): + self.response["suppressed"] = [{"path": "microsoft/knowledge/../skills/x.md", "reason": "configuration"}] + self.assertTrue(self.errors()) + + def test_missing_suppressed_file(self): + self.response["suppressed"] = [{"path": "custom/knowledge/missing.md", "reason": "configuration"}] + self.assertTrue(self.errors()) + + def test_enabled_layers_cli(self): + response_file = self.root / "response.json" + question_file = self.root / "question.txt" + evidence_file = self.root / "reads.json" + response_file.write_text(json.dumps(self.response), encoding="utf-8") + question_file.write_bytes(self.question.encode("utf-8")) + evidence_file.write_text(json.dumps(self.evidence), encoding="utf-8") + result = subprocess.run([sys.executable, str(TOOL), str(response_file), "--root", str(self.root), + "--question-file", str(question_file), "--read-evidence", str(evidence_file), + "--run-id", "test-run", "--enabled-layers", "community"], capture_output=True, text=True) + self.assertEqual(result.returncode, 1) + self.assertIn("disabled layer", result.stdout) + + def test_unknown_enabled_layer(self): + self.assertTrue(module.validate_response(self.response, question=self.question, root=self.root, + evidence=self.evidence, run_id="test-run", enabled_layers=["unknown"])) + + def test_valid_and_unchanged(self): + original = copy.deepcopy(self.response) + self.assertEqual(self.errors(), []) + self.assertEqual(self.response, original) + + def test_rewritten_question(self): + self.response["question"] = "A paraphrase" + self.assertTrue(self.errors()) + + def test_question_whitespace(self): + self.response["question"] = self.question.strip() + self.assertTrue(self.errors()) + + def test_missing_article(self): + self.response["references"][0]["path"] = "microsoft/knowledge/performance/missing.md" + self.assertTrue(self.errors()) + + def test_invalid_corpus_path_is_reported_once(self): + for collection in ("references", "suppressed"): + for path in ("microsoft/knowledge/performance/missing.md", + "microsoft/knowledge/performance/../performance/example.md"): + with self.subTest(collection=collection, path=path): + response = copy.deepcopy(self.response) + if collection == "references": + response[collection][0]["path"] = path + else: + response[collection] = [{"path": path, "reason": "configuration"}] + errors = module.validate_response(response, question=self.question, root=self.root, + evidence=self.evidence, run_id="test-run") + self.assertEqual(len(errors), 1, errors) + self.assertIn(path, errors[0]) + + def test_missing_read_evidence(self): + self.assertTrue(self.errors(evidence=False)) + + def test_truncated_read(self): + self.evidence["reads"][0]["complete"] = False + self.assertTrue(self.errors()) + + def test_byte_count(self): + self.evidence["reads"][0]["bytes"] -= 1 + self.assertTrue(self.errors()) + + def test_changed_body(self): + self.article.write_bytes(b"Changed body\n") + self.assertTrue(self.errors()) + + def test_wrong_hash(self): + self.evidence["reads"][0]["sha256"] = "0" * 64 + self.assertTrue(self.errors()) + + def test_same_basename_other_layer(self): + self.evidence["reads"][0]["path"] = self.path.replace("microsoft/", "custom/") + self.assertTrue(self.errors()) + + def test_wrong_run(self): + self.evidence["runId"] = "previous-run" + self.assertTrue(self.errors()) + + def test_wrong_question_evidence(self): + self.evidence["questionSha256"] = "0" * 64 + self.assertTrue(self.errors()) + + def test_path_traversal(self): + self.response["references"][0]["path"] = "microsoft/knowledge/../skills/example.md" + self.assertTrue(self.errors()) + + def test_empty_reference_outcomes(self): + for outcome in ("failed", "no-knowledge"): + self.response.update(outcome=outcome, references=[], **{"outcome-reason": "No usable knowledge"}) + self.assertEqual(self.errors(evidence=False), []) + + def test_schema_violation(self): + self.response["references"][0].update(applicability="conditional", unknown=[]) + self.assertTrue(self.errors()) + + def test_duplicate_read_evidence(self): + self.evidence["reads"].append(copy.deepcopy(self.evidence["reads"][0])) + self.assertTrue(self.errors()) + + def test_cli_preserves_question_and_rejects_changes(self): + question = self.root / "question.txt" + question.write_bytes(self.question.encode()) + response = self.root / "response.json" + evidence = self.root / "evidence.json" + response.write_text(json.dumps(self.response), encoding="utf-8") + evidence.write_text(json.dumps(self.evidence), encoding="utf-8") + command = [sys.executable, str(TOOL), str(response), "--root", str(self.root), + "--question-file", str(question), "--read-evidence", str(evidence), "--run-id", "test-run"] + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertTrue(json.loads(result.stdout)["valid"]) + self.response["question"] = self.question.strip() + response.write_text(json.dumps(self.response), encoding="utf-8") + result = subprocess.run(command, capture_output=True, text=True) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + + def test_strict_json_duplicate_key(self): + file = self.root / "duplicate.json" + file.write_text('{"question":"one","question":"two"}') + with self.assertRaises(ValueError): + module.strict_json(file) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/validate_frontmatter.py b/.github/scripts/validate_frontmatter.py index c16b3dc7..449e96f0 100644 --- a/.github/scripts/validate_frontmatter.py +++ b/.github/scripts/validate_frontmatter.py @@ -46,9 +46,9 @@ STANDARD_INPUTS = { "pr-diff", "object-list", "file-path", "folder-path", "repository", "telemetry-query", - "development-plan", + "development-plan", "knowledge-query", } -ALLOWED_OUTPUTS = {"findings-report", "development-guidance-report"} +ALLOWED_OUTPUTS = {"findings-report", "development-guidance-report", "knowledge-response"} VALID_SAMPLE_KINDS = {"good", "bad"} ACTION_SKILL_SECTIONS = ["Source", "Relevance", "Worklist", "Action", "Output"] diff --git a/.github/workflows/validate-frontmatter.yml b/.github/workflows/validate-frontmatter.yml index dd7922d4..45ffde56 100644 --- a/.github/workflows/validate-frontmatter.yml +++ b/.github/workflows/validate-frontmatter.yml @@ -19,7 +19,13 @@ jobs: python-version: "3.12" - name: Install dependencies - run: pip install pyyaml + run: pip install pyyaml jsonschema - name: Run validator run: python .github/scripts/validate_frontmatter.py --root . + + - name: Validate knowledge response contract + run: python .github/scripts/test_knowledge_response.py + + - name: Validate knowledge response semantics + run: python .github/scripts/test_knowledge_response_semantics.py diff --git a/README.md b/README.md index a7846471..c4392f40 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ Copilot Chat panel. ``` The list should include `bcquality`. In a Copilot CLI session, `/skills - list` shows its two skills: `al-code-review` and the preview - `al-plan-guidance`. + list` shows `al-code-review`, [`al-knowledge`](skills/al-knowledge/SKILL.md) + for cited knowledge consultation, and the preview `al-plan-guidance`. 2. Start a **new** Copilot CLI session in your app folder, replacing the example path: @@ -87,6 +87,7 @@ the review can still discover knowledge by reading the folders. | --- | --- | | Choose direct reading, a supplied skill, or my own agent | [Ways to use BCQuality](docs/using-bcquality.md#choose-how-to-use-bcquality) | | Review a file, changes, a branch, or a particular concern | [Using BCQuality](docs/using-bcquality.md) | +| Support design and specification with cited knowledge | [Knowledge consultation](docs/knowledge-consultation.md) | | Resolve setup problems, incomplete reviews, or incorrect findings | [Troubleshooting and support](docs/troubleshooting.md) | | Browse the available guidance | [Knowledge by domain](docs/using-bcquality.md#knowledge-by-domain) | | Configure the plugin or use my organization's rules | [Customizing BCQuality](docs/customizing-bcquality.md) | diff --git a/docs/README.md b/docs/README.md index adddaafd..fc465eef 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,7 @@ of BCQuality's internal protocol is needed. | --- | --- | | Choose direct reading, a supplied skill, or my own agent | [Ways to use BCQuality](using-bcquality.md#choose-how-to-use-bcquality) | | Review an app, file, changes, or branch | [Using BCQuality](using-bcquality.md) | +| Support design and specification with cited knowledge | [Knowledge consultation](knowledge-consultation.md) | | Understand a report and its limitations | [Reading your results](using-bcquality.md#reading-your-results) | | Find a particular rule or example | [Knowledge by domain](using-bcquality.md#knowledge-by-domain) | | Fix setup problems or report an incorrect finding | [Troubleshooting and support](troubleshooting.md) | @@ -29,6 +30,7 @@ prerequisites for using the plugin. | [Standalone runner](standalone-runner.md) | Optional model selection, scheduling, retries, and telemetry. | | [SARIF export](standalone-runner.md#publish-findings-as-sarif) | Show accepted findings as code scanning annotations in GitHub or Azure DevOps. | | [Global skills](../skills/README.md) | Host adapters versus internal protocol files. | +| [Knowledge-response validation](knowledge-response-validation.md) | Optional consumer checks for question fidelity and complete citation reads. | | [Entry](../skills/entry.md) | Task context and skill dispatch. | | [READ](../skills/read.md) | Knowledge schema, applicability, and precedence. | | [DO](../skills/do.md) | Action-skill format and structured output contract. | diff --git a/docs/agent-consumption.md b/docs/agent-consumption.md index 31c33185..f3e2cc2a 100644 --- a/docs/agent-consumption.md +++ b/docs/agent-consumption.md @@ -78,10 +78,10 @@ Add scheduling, retries, and rendering only when needed, using the - **Layer content** in `/microsoft/`, `/community/`, and `/custom/` — knowledge files and action skills grouped by authority. When BCQuality is installed as a standalone plugin, it additionally exposes -one host-format adapter per capability: `skills/al-code-review/SKILL.md` and -the preview `skills/al-plan-guidance/SKILL.md`. Adapters are not additional -action skills: each creates the task context for its capability and enters the -same flow at Entry. +one host-format adapter per capability: `skills/al-code-review/SKILL.md`, +`skills/al-knowledge/SKILL.md`, and the preview +`skills/al-plan-guidance/SKILL.md`. Adapters are not additional action skills: +each creates the task context for its capability and enters the same flow at Entry. ## Repository structure @@ -123,10 +123,12 @@ The orchestrator has a URL setting that points at BCQuality (default: `github.co ### 2. Agent invokes Entry The agent reads `/skills/entry.md` and runs it against the task context. Entry applies its Source → Relevance → Worklist → Action steps over the action skills under `*/skills/**/*.md` and returns a **dispatch record**: the set of action skills to invoke, plus a list of candidates it skipped (with reasons). Routing is a skill, not orchestrator logic. -For a standalone plugin installation, the host activates the review adapter -first. The adapter preserves the caller's actual goal, constructs the task -context, and invokes Entry. It does not select the internal review action skill -itself or duplicate Entry's preparation, routing, and failure semantics. +For a standalone plugin installation, the host activates the +selected public skill entry point first. It preserves the caller's actual goal, +constructs the task context, and invokes Entry. It does not select an internal +action skill itself or duplicate Entry's preparation, routing, and failure +semantics. For knowledge consultation, it binds the exact question as +`knowledge-query`; see [knowledge consultation](knowledge-consultation.md). ### 3. Agent consumes the dispatch record The dispatch record names one or more action skills, the subset of inputs each @@ -177,7 +179,14 @@ The output contracts are defined in the DO meta-skill: - A **findings report** carries review findings, domain labels, references, confidence, and suppressions. - A **development guidance report** carries read-only knowledge constraints and validation considerations for an existing plan. -The orchestrator parses this **without skill-specific logic**. This is the point of the contract: orchestrators and action skills evolve independently. +Knowledge-consultation actions produce `knowledge-response`, with the exact +question, supported answer and article references. See the +[DO knowledge-response contract](../skills/do.md#knowledge-response-contract) +and [consumer validation](knowledge-response-validation.md). + +The orchestrator validates the declared output kind without interpreting +private skill-specific fields. Action skills and consumers share the same +output contracts. For plan enrichment, the skill reads the existing plan and target repository, selects applicable knowledge, and returns constraints without changing the @@ -300,6 +309,9 @@ final reviews, including failures and unresolved results. Credential-free fixture preparation and scorer regressions do not demonstrate improved repair quality, compilation, runtime success, or production integration. +For knowledge consultation, it displays validated guidance with citations. +It does not convert that guidance into findings or use it as an approval gate. + ## Knowledge-backed and agent findings BCQuality is an **additive** knowledge layer. The agent surfaces two kinds of findings, both shaped to the same DO output contract: diff --git a/docs/assets/al-knowledge-agent-diagram-en.png b/docs/assets/al-knowledge-agent-diagram-en.png new file mode 100644 index 00000000..fc9702d2 Binary files /dev/null and b/docs/assets/al-knowledge-agent-diagram-en.png differ diff --git a/docs/assets/al-knowledge-diagram-en.mmd b/docs/assets/al-knowledge-diagram-en.mmd new file mode 100644 index 00000000..88ac4116 --- /dev/null +++ b/docs/assets/al-knowledge-diagram-en.mmd @@ -0,0 +1,14 @@ +flowchart TD + A["You or the Agent
Ask a question"] + B["1 · Skill entry point
Keeps your exact question"] + C["2 · Entry
Selects the knowledge action"] + D["3 · Find and filter
Topic, BC version, technology and country"] + E["4 · Read full articles
The index helps find articles; it is not evidence"] + F["5 · Answer with sources
knowledge-response"] + G["No useful knowledge
no-knowledge
No invented sources"] + A --> B --> C --> D --> E + E -->|Evidence found| F + E -->|No evidence| G + classDef node fill:#fce7f3,stroke:#ddd4da,color:#9e3868,rx:20,ry:20; + class A,B,C,D,E,F,G node; + linkStyle default stroke:#999,stroke-width:1.5px; diff --git a/docs/assets/al-knowledge-technical-en.mmd b/docs/assets/al-knowledge-technical-en.mmd new file mode 100644 index 00000000..d0639ad4 --- /dev/null +++ b/docs/assets/al-knowledge-technical-en.mmd @@ -0,0 +1,26 @@ +flowchart TD + Q["User or agent
Exact question + known target context"] + A["Public skill entry point
The agent reads SKILL.md
Preserve the question and resolve BCQuality root"] + E["Entry
Prepare the index when possible
Dispatch by goal and compatible inputs"] + D["Internal al-knowledge action"] + C[("Enabled knowledge corpus
custom / community / microsoft")] + R["READ rules
Find candidates, check applicability
and apply layer precedence"] + B["Read complete article bodies
Bounded helpers or native reads through EOF
Follow continuation pages"] + O["knowledge-response JSON
completed / no-knowledge / partial / failed"] + W["Consumer-collected read evidence
Run ID, exact paths, full-read status,
content hashes and byte counts"] + V{"Consumer validation
Schema + exact question
+ same-run full reads against live corpus"} + Y["Accept the unchanged response"] + N["Reject acceptance
Preserve raw response
Record validation errors separately"] + X["Entry no-match or failed
Return the dispatch record unchanged"] + Q --> A --> E + E -->|routed| D --> R --> B --> O --> V + E -->|no-match or failed| X + C --> R + B -. "successful tool results" .-> W + W --> V + Q -. "original bound question" .-> V + V -->|valid| Y + V -->|invalid| N + classDef process fill:#fce7f3,stroke:#ddd4da,color:#9e3868; + class Q,A,E,D,C,R,B,O,W,V,Y,N,X process; + linkStyle default stroke:#999,stroke-width:1.5px; diff --git a/docs/knowledge-consultation.md b/docs/knowledge-consultation.md new file mode 100644 index 00000000..6850e30d --- /dev/null +++ b/docs/knowledge-consultation.md @@ -0,0 +1,99 @@ +# Knowledge consultation for design and specification + +[Documentation](README.md) | [Public skill entry point](../skills/al-knowledge/SKILL.md) | [Response validation](knowledge-response-validation.md) + +`al-knowledge` helps AL coding agents design and specify Business Central +solutions using cited guidance from BCQuality. A user or agent can ask a focused +development question before source code exists. The skill follows BCQuality's +contracts and is independent of any particular consumer framework or agent role. +Available guidance is limited to the installed corpus. + +## Ask a question + +> Use al-knowledge to explain when SetLoadFields is useful in Business Central AL. +> Cite the BCQuality articles you read. Do not review my app. Return the +> knowledge-response JSON unchanged. + +Supply the BC version, technologies, countries and application area when known. +Do not invent target details to make an article applicable. Unknown dimensions +remain explicit on conditional references under [READ](../skills/read.md). + +## Request path + +The public skill entry point is the file the host agent reads to start. It +preserves the exact question as `knowledge-query`, resolves the installed +BCQuality root, and follows [Entry](../skills/entry.md). Entry prepares the +knowledge index when possible and selects an action with compatible input types. +A question-only request cannot dispatch review actions. + +The [internal knowledge action](../microsoft/skills/knowledge/al-knowledge.md) +finds relevant articles, checks applicability and layer precedence, reads their +full bodies, and returns cited guidance. Index entries support discovery but +are not evidence of complete reads. Bounded helpers require every continuation +page; when unavailable, native file reads must continue through EOF. + +```mermaid +flowchart TD + Q["User or agent
Exact question and known context"] + S["Public skill entry point
Prepare knowledge-query
Resolve the BCQuality root"] + E["Entry
Prepare index when possible
Select the knowledge action"] + A["Internal al-knowledge action
Apply READ rules
Read complete articles"] + R["knowledge-response
Answer, outcome and citations"] + V{"Consumer validation
Schema, exact question
and complete citation reads"} + OK["Accept the unchanged response"] + BAD["Preserve the raw response
Record validation errors separately"] + STOP["Return Entry dispatch record
no-match or failed"] + Q --> S --> E + E -->|routed| A --> R --> V + E -->|no-match or failed| STOP + V -->|valid| OK + V -->|invalid| BAD +``` + +The host supplies the executing agent. It can follow these instructions inline +when no skill-invocation tool exists. Reading the skill entry point and Entry is +only preparation; the agent must execute the dispatched action and read articles. + +![Knowledge consultation: the question, skill entry point, Entry, article reading and cited response or no-knowledge branch](assets/al-knowledge-agent-diagram-en.png) + +[Editable explanatory diagram](assets/al-knowledge-diagram-en.mmd) and +[technical diagram source](assets/al-knowledge-technical-en.mmd). + +## Response and acceptance + +The action returns `knowledge-response` under [DO](../skills/do.md#knowledge-response-contract) +and its [schema](../schemas/knowledge-response.schema.json). `completed` requires +a useful answer and a verified citation. `no-knowledge` explains a gap with no +citations. `partial` explains unfinished work and cites only articles read. +`failed` explains a failure with no citations. Entry's `no-match` and `failed` +dispatch records remain separate and return unchanged. + +The consumer validates the schema, exact bound question and complete citation +reads before acceptance. It preserves invalid output and records validation +errors separately. The optional [consumer validator](knowledge-response-validation.md) +checks question fidelity and full-read evidence against live files. Its evidence +comes from successful tool results in the same run, not the answering agent's +assertions. It does not independently prove that every claim follows a source. +READ still governs applicability and interpretation. + +`al-code-review` remains the skill for review findings. Knowledge consultation +provides guidance, not a review verdict or a design approval gate. + +## Prerequisites and optional tooling + +The host agent needs access to the public skill entry point, Entry, READ, DO, +the internal action, response schema and enabled knowledge articles. It must +be able to read complete files and retain complete tool results. Authentication +to the agent provider follows the chosen host. No app, compiler or Business +Central tenant is required for a question-only consultation. + +PowerShell 7.2 or later runs the existing bounded retrieval helpers. READ already +permits path discovery and native complete file reads when PowerShell, a helper, +or a prepared index is unavailable. Index write permission is needed only when +running Entry's generator. + +Python with `jsonschema` is needed only for the proposed consumer validator. +Another consumer can enforce the same contract with its own implementation. +The read-evidence manifest is one proposed implementation, not a requirement +that all hosts adopt that format. Complete citation reads and response validation +remain required regardless of the tools selected. diff --git a/docs/knowledge-response-validation.md b/docs/knowledge-response-validation.md new file mode 100644 index 00000000..fdf3985d --- /dev/null +++ b/docs/knowledge-response-validation.md @@ -0,0 +1,117 @@ +# Validating a knowledge response + +`al-knowledge` returns guidance under the [DO knowledge-response contract](../skills/do.md#knowledge-response-contract). +The consumer checks its result before acceptance. This helper works independently +of consumer frameworks and hosts and does not execute the skill. + +## Optional tooling and existing contracts + +BCQuality already requires complete article reads and citation integrity under +READ and DO, and deterministic consumer validation for review reports. The new +knowledge-response contract applies these principles to knowledge consultation. + +This Python helper and its read-evidence manifest are proposed optional tooling, +not existing runtime requirements or a pre-existing BCQuality evidence standard. +Python with `jsonschema` is needed only to run this implementation. Consumers may +validate the same contract with another implementation and their native tools. +They still must verify the exact question and complete citation reads in the +current run; they do not have to adopt this script or its manifest format. + +The existing PowerShell retrieval helpers require PowerShell 7.2 or later. READ +already permits path discovery and native complete file reads when those helpers +or their prepared index are unavailable. The original repository's Python and +PyYAML frontmatter checks are development/CI tooling, not prerequisites for +question-only consultation. + +In an inline host, the answering agent checks its own successful full-body tool +reads. This is self-attestation, not independent citation verification. A +consumer with a trusted read collector can independently check the evidence +using the optional validator. Neither mode proves that every claim correctly +interprets its sources. Record the mode used in the consumer's own run record, +not in the `knowledge-response` JSON, whose schema has no mode field. Never +present inline checks as independent verification. + +## Consumer inputs + +Preserve the raw response as one JSON object, without Markdown fences or extra +text. Save the exact bound question as UTF-8. Spaces and CRLF/LF line endings are +part of the question; the validator does not trim or normalize them. A UTF-8 BOM +is an encoding marker and not part of the bound question. + +The consumer collects read evidence from successful tool results in the same +run, independently of the action's answer. A claim in `answer`, an attempted +tool call, an index row, or a displayed filename is insufficient. Mark an +article complete only after receiving its full body through EOF, and compute +its SHA-256 and UTF-8 byte count from the exact complete file bytes received. +Preserve source encoding and line endings when collecting that evidence. + +The evidence file has this shape (values below are placeholders): + +```json +{ + "runId": "consumer-assigned-run-id", + "questionSha256": "sha256-of-the-exact-question-encoded-as-utf8", + "reads": [ + { + "path": "microsoft/knowledge/performance/use-setloadfields-for-partial-records.md", + "complete": true, + "bytes": 1234, + "sha256": "sha256-of-the-complete-article-file-bytes" + } + ] +} +``` + +Use a consumer-assigned identifier for the current run. Supply it separately +to the validator and reject evidence from another run or question. Do not ask +the answering agent to invent this manifest. The validator trusts the evidence +collector; hashes and a boolean alone cannot independently prove a tool read. + +## Run the checks + +Install `jsonschema` in the consumer's Python environment if it is unavailable. +From the BCQuality root: + +```powershell +python tools/validate_knowledge_response.py response.json ` + --root . --question-file question.txt ` + --read-evidence reads.json --run-id consumer-assigned-run-id +``` + +The tool checks the response schema, exact question, existing canonical citation +paths within the corpus, and complete-read evidence with matching current file +hashes and byte counts. A changed article, truncated read, missing evidence, +different-layer path with the same filename, or mismatched run is rejected. +The tool never repairs or rewrites the response. + +Exit codes: `0` means these checks passed, `1` means validation failed, and `2` +means invalid JSON or an input/resource error prevented validation. Failed +validation is not a successful knowledge response. Preserve the raw payload and +record the validation errors separately. Do not accept or display its answer as +validated BCQuality guidance. A response with no citations does not need a read +manifest, but still must satisfy the schema and preserve the question. + +Passing does not establish that every claim follows from its references, that +target applicability is correct, or that the corpus covers the question fully. +Those checks require the [READ rules](../skills/read.md) and actual execution +evidence. Catalog discovery and schema validity alone do not prove execution. + +## Enabled layers + +Supply the consumer configuration through `--enabled-layers microsoft community` +(or all three by default). An explicit empty list disables every layer. +Citations and layer-precedence suppressions must belong to enabled layers. +A `suppressed` entry with reason `configuration` may identify an existing +article in a disabled layer; it is an exclusion record, never supporting evidence. +All these paths must be canonical existing corpus files. + +## Regression tests + +```powershell +python .github/scripts/test_knowledge_response.py +python .github/scripts/test_knowledge_response_semantics.py +``` + +The first checks JSON Schema cases. The second uses fixture evidence to exercise +question fidelity, complete reads, current body hashes, path identity, and CLI +behavior. Fixture tests are not live host executions. diff --git a/docs/using-bcquality.md b/docs/using-bcquality.md index f6c4a3a2..b6606767 100644 --- a/docs/using-bcquality.md +++ b/docs/using-bcquality.md @@ -16,6 +16,14 @@ and rendering; BCQuality does not install a BC extension or an agent. | Use your own agent or workflow | Supply selected articles as context, as described below, or use the [integration bootstrap](agent-consumption.md#try-a-minimal-integration) to execute BCQuality action skills without the plugin. | | Get guidance before coding (preview) | Ask the plugin's `al-plan-guidance` skill to enrich an existing plan, then review the finished change. See [plan guidance](../README.md#preview-guidance-before-implementation). | +### Knowledge consultation + +Use `al-knowledge` to support design and specification with guidance from the +articles the agent actually reads. A question needs no app folder, file or diff. +The response contains the exact question, cited guidance and applicability +conditions. See [knowledge consultation](knowledge-consultation.md) for the +request path, prerequisites and validation. + ### Read and reuse an article Start with a concern, such as `SetLoadFields`, and search within diff --git a/microsoft/skills/knowledge/al-knowledge.md b/microsoft/skills/knowledge/al-knowledge.md new file mode 100644 index 00000000..4f32c727 --- /dev/null +++ b/microsoft/skills/knowledge/al-knowledge.md @@ -0,0 +1,78 @@ +--- +kind: action-skill +id: al-knowledge +version: 1 +title: AL knowledge consultation +description: Answer a Business Central development question with cited knowledge, without reviewing source code. +inputs: [knowledge-query] +outputs: [knowledge-response] +bc-version: [all] +technologies: [al] +countries: [w1] +application-area: [all] +--- + +# AL knowledge consultation + +`knowledge-query` is the caller's exact question. This skill accepts optional +target context from Entry. It does not require an app, a file, or a diff. + +## Source + +Search `knowledge-index.json` for matching domains, titles, descriptions, and +keywords across enabled layers. Resolve the index against BCQuality's root. +When the index could not be generated, discover candidate article paths in +`/knowledge/**/` instead. The index is discovery metadata, +never the article body or evidence of applicability. + +## Relevance + +Apply READ's frontmatter matching rules for any known BC version, technology, +country, and application area. Exclude definite mismatches. Keep conditional +matches with their exact unknown dimensions. Interpret competing articles and +layer precedence under READ; record any displaced article in `suppressed`. + +Frontmatter applicability is not question relevance. A universal country or +version filter does not make an article answer every domain question. Candidate +keyword overlap is discovery only. After reading, retain as supporting evidence +only articles that address the requested subject and its essential constraints. + +## Worklist + +Choose the smallest set of articles that answers the actual question. Match +the question's concepts to article descriptions, filenames, and keywords. +Open each chosen article in full, including relevant sibling samples if cited +by that article. If the question spans domains, select each necessary domain; +do not turn a narrow question into an exhaustive corpus summary. Follow READ's +[bounded retrieval workflow](../../../skills/read.md#bounded-retrieval-for-review-skills), +including all continuation pages and complete bodies. Never truncate helper +output or treat an unread body as evidence. Do not cite an article merely +because it appeared in the index. + +## Action + +Answer only what the read articles support. State conditions, exceptions, +unknown target dimensions, and any unresolved conflict. Cite exact article +paths in the answer and `references`; do not infer a path from a title. This +is guidance, not a code review: do not inspect source to produce defects, +invoke review action skills, emit severities, or generate a findings-report. +If no applicable article answers the question, return `no-knowledge` instead +of filling the gap from general model knowledge under BCQuality's name. + +Before choosing the outcome, check whether the supporting articles answer the +bound question rather than a broader or adjacent question. Do not replace an +unsupported domain or target with general coding or localization advice merely +because some keywords overlap. If none answers the requested subject, return +`no-knowledge` with `references: []`; the answer may explain the coverage gap, +but must not append adjacent guidance with citations. Candidate reads may be +recorded separately in execution diagnostics. Use `partial` only when articles +support a genuine part of the requested question, with the uncovered part +explicitly identified; adjacent-topic advice alone is not partial coverage. + +## Output + +Return one JSON value following `schemas/knowledge-response.schema.json` and +DO's knowledge-response rules. `question` preserves the caller's question. +`completed` requires an answer to the actual question and at least one fully +read, checked supporting reference. `partial` +requires a reason and only cites articles actually read; `failed` cites none. diff --git a/plugin.json b/plugin.json index 021d9274..071d270f 100644 --- a/plugin.json +++ b/plugin.json @@ -1,6 +1,6 @@ { "name": "bcquality", - "description": "Quality skills and knowledge for Business Central development. Exposes AL code review and preview plan guidance skills backed by BCQuality's Entry router.", + "description": "Quality skills and knowledge for Business Central development. Exposes AL code review, cited knowledge consultation and preview plan guidance skills backed by BCQuality's Entry router.", "version": "1.9.0", "author": { "name": "microsoft/BCQuality", @@ -13,6 +13,7 @@ "al", "business-central", "code-review", + "knowledge", "quality" ], "skills": [ diff --git a/schemas/knowledge-response.schema.json b/schemas/knowledge-response.schema.json new file mode 100644 index 00000000..9fc45715 --- /dev/null +++ b/schemas/knowledge-response.schema.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/microsoft/BCQuality/schemas/knowledge-response.schema.json", + "title": "BCQuality knowledge consultation response", + "type": "object", + "additionalProperties": false, + "required": ["skill", "outcome", "question", "answer", "references", "suppressed"], + "properties": { + "skill": { + "type": "object", "additionalProperties": false, "required": ["id", "version"], + "properties": {"id": {"const": "al-knowledge"}, "version": {"type": "integer", "minimum": 1}} + }, + "outcome": {"enum": ["completed", "no-knowledge", "partial", "failed"]}, + "outcome-reason": {"type": "string", "minLength": 1}, + "question": {"type": "string", "minLength": 1}, + "answer": {"type": "string"}, + "references": { + "type": "array", "uniqueItems": true, + "items": { + "type": "object", "additionalProperties": false, + "required": ["path", "applicability", "unknown"], + "properties": { + "path": {"type": "string", "pattern": "^(microsoft|community|custom)/knowledge/.+\\.md$"}, + "sha": {"type": "string", "pattern": "^[a-f0-9]{40}$"}, + "applicability": {"enum": ["applicable", "conditional"]}, + "unknown": {"type": "array", "uniqueItems": true, "items": {"enum": ["bc-version", "technologies", "countries", "application-area"]}} + }, + "allOf": [ + {"if": {"properties": {"applicability": {"const": "applicable"}}}, "then": {"properties": {"unknown": {"maxItems": 0}}}}, + {"if": {"properties": {"applicability": {"const": "conditional"}}}, "then": {"properties": {"unknown": {"minItems": 1}}}} + ] + } + }, + "suppressed": { + "type": "array", "items": { + "type": "object", "additionalProperties": false, "required": ["path", "reason"], + "properties": { + "path": {"type": "string", "pattern": "^(microsoft|community|custom)/knowledge/.+\\.md$"}, + "reason": {"enum": ["layer-precedence", "configuration"]} + } + } + } + }, + "allOf": [ + {"if": {"properties": {"outcome": {"enum": ["partial", "failed"]}}}, "then": {"required": ["outcome-reason"]}}, + {"if": {"properties": {"outcome": {"enum": ["failed", "no-knowledge"]}}}, "then": {"properties": {"references": {"maxItems": 0}}}}, + {"if": {"properties": {"outcome": {"const": "completed"}}}, "then": {"properties": {"references": {"minItems": 1}, "answer": {"minLength": 1}}}} + ] +} diff --git a/schemas/skill-index.schema.json b/schemas/skill-index.schema.json index 8e11cca1..35088973 100644 --- a/schemas/skill-index.schema.json +++ b/schemas/skill-index.schema.json @@ -49,7 +49,7 @@ "minItems": 1, "maxItems": 1, "items": { - "enum": ["findings-report", "development-guidance-report"] + "enum": ["findings-report", "development-guidance-report", "knowledge-response"] } }, "filters": { diff --git a/skills/README.md b/skills/README.md index e47e5f86..5b0bd440 100644 --- a/skills/README.md +++ b/skills/README.md @@ -31,6 +31,7 @@ READ and DO are read on demand — typically by the first action skill the agent | Path | Capability | |---|---| | [`al-code-review/SKILL.md`](al-code-review/SKILL.md) | Review AL code: an app folder, files, or changes. | +| [`al-knowledge/SKILL.md`](al-knowledge/SKILL.md) | Answer a focused development question with cited knowledge, without code review. | | [`al-plan-guidance/SKILL.md`](al-plan-guidance/SKILL.md) | Preview. Enrich an existing AL development plan with applicable guidance, read-only. | Each adapter is deliberately thin. It translates the caller's request into an @@ -48,6 +49,10 @@ review: skills/al-code-review/SKILL.md -> review coordinator: microsoft/skills/review/al-code-review.md -> domain review leaves +consultation: skills/al-knowledge/SKILL.md + -> router: skills/entry.md + -> microsoft/skills/knowledge/al-knowledge.md + plan guidance: skills/al-plan-guidance/SKILL.md -> router: skills/entry.md -> microsoft/skills/development/al-development-plan.md diff --git a/skills/al-knowledge/SKILL.md b/skills/al-knowledge/SKILL.md new file mode 100644 index 00000000..5cdd6229 --- /dev/null +++ b/skills/al-knowledge/SKILL.md @@ -0,0 +1,51 @@ +--- +name: al-knowledge +description: Answer Business Central development questions from BCQuality's installed knowledge articles, with exact citations and no code review. Use for design, specification, or a focused knowledge question. +--- + +# AL knowledge consultation + +This is the public skill entry point: the file the host agent reads to start +the skill. It tells the agent how to prepare the question and follow +BCQuality's routing instructions. The same agent can execute all steps +inline; no separate program or agent is required. + +This file is distinct from `skills/entry.md`, which owns routing. Article +selection, precedence, and output policy belong to Entry, READ, DO, and the +dispatched internal action skill. + +## Execute + +1. Resolve `PLUGIN_ROOT` from the installed plugin's `plugin.json`; this file is + `PLUGIN_ROOT/skills/al-knowledge/SKILL.md`. Never assume the user's app root + is the plugin root. +2. Build Entry's `task-context`. Copy the caller's question verbatim into `goal`. + Set `inputs-available: [knowledge-query]` and bind `knowledge-query` to that + exact question. Pass `technologies`, `bc-version`, `countries`, and + `application-area` only when provided or reliably determined. Pass the + optional `BCQUALITY_ENABLED_LAYERS` and `BCQUALITY_DISABLED_SKILLS` settings + using the same parsing rules as `al-code-review`. +3. Read and execute `PLUGIN_ROOT/skills/entry.md`, including Preparation, + resolving all paths against `PLUGIN_ROOT`. If `pwsh` or index generation is + unavailable, use the documented path-based discovery fallback. +4. Execute only the dispatched action skills with their exact input subsets. + Read `PLUGIN_ROOT/skills/read.md` and `PLUGIN_ROOT/skills/do.md` on demand. + Do not call `al-code-review` for this knowledge question, synthesize a + dispatch, or turn an explanation into a finding. +5. Validate the action skill's `knowledge-response` under DO before accepting + it: check the schema, exact bound question, and current-run full reads of + every cited path. The consumer can use + `PLUGIN_ROOT/tools/validate_knowledge_response.py` with its own question + and read evidence. Preserve the raw result when validation fails; record + that failure separately instead of repairing or accepting the answer. + Inline checks by the answering agent are self-attestation, not independent + verification. Independent checks require a trusted consumer read collector. + Return a valid response unchanged, or Entry's `no-match` / `failed` + dispatch record unchanged. + +Where the host has no tool to invoke a skill and skills are instruction files the +agent reads (for example VS Code Copilot), invoking this skill means executing +steps 1–5 inline in the caller's own context. The `knowledge-response` the +dispatched action skill produces there is the response to return: producing it +that way is the protocol, not a fabrication. Lacking an invocation tool is never +a reason to skip this skill or to answer without its response. diff --git a/skills/do.md b/skills/do.md index 82b1985b..4a3721b7 100644 --- a/skills/do.md +++ b/skills/do.md @@ -63,7 +63,7 @@ application-area: [all] `inputs` is a list of abstract input types the skill **accepts**. Standard values: `pr-diff`, `object-list`, `file-path`, `folder-path`, `repository`, -`telemetry-query`, and `development-plan`. Semantics are any-of: the +`telemetry-query`, `development-plan`, and `knowledge-query` (a question without source-code scope). Semantics are any-of: the orchestrator supplies whichever listed input types it has, and the skill is invoked with a non-empty subset of its declared `inputs`. A skill that cannot proceed with the supplied subset MUST return `outcome: "not-applicable"`. @@ -71,6 +71,7 @@ proceed with the supplied subset MUST return `outcome: "not-applicable"`. `outputs` is always a single-element list naming the output kind: - `findings-report` — evaluates an input and reports defects or observations. +- `knowledge-response` — answers a bound development question with cited knowledge, without a code review. - `development-guidance-report` — selects and summarizes applicable BCQuality knowledge for an existing development plan without changing the target repository. `development-plan` and `development-guidance-report` are preview contract @@ -78,6 +79,10 @@ extensions. The standalone plugin exposes them through the preview `al-plan-guidance` adapter. Consumer-owner agreement and pilot evidence are required before they are treated as frozen public integration surfaces. +`knowledge-query` is a bound input value: its exact question travels to a +dispatched skill, while Entry's `goal` supports routing. Knowledge responses, +development guidance, and review findings must not be conflated. + `file-path` is one file. `folder-path` is a directory whose recursively contained files form the complete current-state input, such as a Business Central app folder containing `app.json` and AL source. The input value is the @@ -109,7 +114,7 @@ Every action skill MUST contain these five sections, in order: **Worklist.** Narrow the relevant candidates to the subset that applies to the current task. This is where the task-specific signal enters: the objects changed in the PR, the queries being audited, the existing plan being enriched. Typical moves: match `keywords` against task vocabulary, match file topics against changed objects, deduplicate by concern. -**Action.** Execute the skill's work against the worklist. Evaluate each item in the worklist against the task input and emit findings. The action step is where skill behavior differs; the preceding three steps are uniform. +**Action.** Execute the skill's work against the worklist and produce its declared output kind. Review actions evaluate items against the task input and emit findings; knowledge-consultation actions answer the question using the articles read. The action step is where skill behavior differs; the preceding three steps are uniform. Review and plan enrichment use different Worklist signals by design. Review leaves inspect existing source and use domain-specific code tokens to decide @@ -484,6 +489,48 @@ The skill is read-only with respect to the target repository: no edits, generate A reference `sha`, when present, is the commit SHA of the BCQuality checkout the file was read from, matching the findings-report `sha` semantics; it does not prove runtime pinning on its own. The consumer records and verifies the actual immutable BCQuality checkout used for both enrichment and final review, plus its filtering policy and run provenance outside the target repository. See [agent-consumption.md](../docs/agent-consumption.md). +## Knowledge-response contract + +An action skill declaring `outputs: [knowledge-response]` emits one strict JSON +object conforming to [`schemas/knowledge-response.schema.json`](../schemas/knowledge-response.schema.json). +This output represents cited guidance, never a finding, severity, code location, +or code-generation artifact. The `question` is the exact bound +`knowledge-query`; `answer` is readable guidance and identifies the exact +article paths used. `references[]` contains only paths to articles whose full +bodies were opened in this run. An optional `sha` is an observed commit, not +an expected pin. A conditional reference lists each unknown applicability +dimension in `unknown[]`; fully applicable references use `applicable` with an +empty `unknown[]`. READ defines matching and conflict precedence. Record +displaced articles in `suppressed[]` without citing them as supporting guidance. + +`completed` requires a useful answer to the bound question and at least one +verified supporting reference. Applicability under READ does not establish +question relevance. Articles about an adjacent topic do not justify `completed` +or `partial` when none addresses the requested subject. In that case use +`no-knowledge` with no references and explain only the coverage gap. +`no-knowledge` means no applicable article answers the question; `answer` +may explain the gap but `references` stays empty. `partial` records the reason +and cites only articles actually read; `failed` records the reason with no +references. Never fabricate an article path, infer a finding from article +applicability, or report advice without an applicable article under the +BCQuality name. A consumer MUST validate the schema and confirm every cited +path exists in its live corpus and was read in full before accepting a result. +If citation integrity cannot be verified, return `failed` with no citations. + +For deterministic consumer checks, see +[knowledge-response validation](../docs/knowledge-response-validation.md). +The helper checks schema, exact question, and complete-read evidence against +the live corpus. It does not prove comprehension, source support for every +claim, or applicability; those checks still follow READ and this contract. + +In an inline host, the answering agent checks its own successful full-body tool +reads. This is self-attestation, not independent citation verification. A +consumer with a trusted read collector can independently check the evidence +using the optional validator. Neither mode proves that every claim correctly +interprets its sources. Record the mode used in the consumer's own run record, +not in the `knowledge-response` JSON, whose schema has no mode field. Never +present inline checks as independent verification. + ## Composition (super-skills) A **super-skill** is an action skill whose frontmatter declares a non-empty `sub-skills: [...]`. A super-skill does not evaluate knowledge files directly; it invokes other action skills and composes their output. @@ -614,4 +661,12 @@ Conforms to the DO output contract. ## How orchestrators consume output -An orchestrator invokes an action skill with an input appropriate to the skill's declared `inputs` and uses the single output kind declared in frontmatter. It maps a `findings-report` to PR comments, build gates, or IDE diagnostics, and a `development-guidance-report` to additional constraints for its existing implementation workflow. These are the two output schemas defined by this contract; the consumer retains ownership of implementation and delivery. +An orchestrator invokes an action skill with an input appropriate to its +declared `inputs` and validates the JSON against its declared output schema. +It maps a `findings-report` to PR comments, build gates, or IDE diagnostics, +and a `development-guidance-report` to additional constraints for its existing +implementation workflow. It displays a `knowledge-response` as cited guidance, +without treating that response as findings or applying a review gate to it. +The consumer retains ownership of implementation and delivery. Skills needing +richer semantics must change a shared contract deliberately instead of adding +private fields to an output. diff --git a/skills/entry.md b/skills/entry.md index b99721a4..e05cb757 100644 --- a/skills/entry.md +++ b/skills/entry.md @@ -26,6 +26,7 @@ task-context: - pr-diff - file-path - folder-path + - knowledge-query # the caller's exact question without code scope technologies: [al] bc-version: 28 countries: [w1] @@ -187,6 +188,7 @@ Populated example (PR review on a repo where only `al-performance-review` is ena 1. Invoke Entry with the orchestrator-supplied task context. 2. Receive the dispatch record. 3. For each entry in `dispatch[]`, inspect `outputs` before invocation, read the referenced action skill, execute its Source → Relevance → Worklist → Action steps per DO, and produce the declared report kind. Verify the file's frontmatter output still equals the dispatch value; return `failed` on drift rather than executing a different contract. + A knowledge-only request with only `knowledge-query` cannot dispatch a review skill because its input types do not intersect. 4. Return the action-skill reports to the orchestrator. When Entry's `outcome` is `no-match` or `failed`, return the dispatch record itself so the orchestrator can log the reason. READ and DO are the contracts that govern what the dispatched skills do. An agent that has not yet read READ and DO reads them when it executes the first dispatched skill — they are not prerequisites for invoking Entry. diff --git a/skills/read.md b/skills/read.md index dddd7dbb..dc5d03db 100644 --- a/skills/read.md +++ b/skills/read.md @@ -185,7 +185,20 @@ returned `body`, then request `remainingPaths` with `continuation.snapshot` as `-Snapshot` until `complete` is `true`, preserving the other request settings. Continuation is confined to that chunk. Bodies are original strict UTF-8 text with source byte counts and SHA-256 content hashes; -they are never summarized or truncated. Samples are not loaded unless +they are never summarized or truncated. Never clip helper output with +`Substring`, `Select-Object -First`, or a file-reader display limit. Parse one +bounded JSON page and expose its article bodies as multiline text for reading: + +```powershell +$page = & (Join-Path $root 'tools\Get-KnowledgeArticles.ps1') -Paths @($exactPath) | ConvertFrom-Json +$page.articles | ForEach-Object { $_.path; $_.body } +``` + +Inspect `complete`, `remainingPaths`, and `continuation.snapshot` on that page +before requesting the next page as described above. Printing a body is not +proof that the host received it through EOF: if the host truncates the display, +continue with bounded native reads through EOF or use a smaller batch. Never +cite an article whose complete body could not be read. Samples are not loaded unless requested explicitly with `-Samples` and exact sibling paths; their sibling article must match its prepared hash and contain the exact READ link. diff --git a/tools/Build-SkillIndex.ps1 b/tools/Build-SkillIndex.ps1 index d4028c22..47ac729e 100644 --- a/tools/Build-SkillIndex.ps1 +++ b/tools/Build-SkillIndex.ps1 @@ -186,10 +186,10 @@ foreach ($layer in 'microsoft', 'community', 'custom') { inputs = [string[]]@($frontmatter['inputs']) outputs = [string[]]@($frontmatter['outputs']) filters = [ordered]@{ - 'bc-version' = [object[]]$(if ($frontmatter.Contains('bc-version')) { $frontmatter['bc-version'] }) - technologies = [string[]]$(if ($frontmatter.Contains('technologies')) { $frontmatter['technologies'] }) - countries = [string[]]$(if ($frontmatter.Contains('countries')) { $frontmatter['countries'] }) - 'application-area' = [string[]]$(if ($frontmatter.Contains('application-area')) { $frontmatter['application-area'] }) + 'bc-version' = [object[]]@(if ($frontmatter.Contains('bc-version')) { $frontmatter['bc-version'] }) + technologies = [string[]]@(if ($frontmatter.Contains('technologies')) { $frontmatter['technologies'] }) + countries = [string[]]@(if ($frontmatter.Contains('countries')) { $frontmatter['countries'] }) + 'application-area' = [string[]]@(if ($frontmatter.Contains('application-area')) { $frontmatter['application-area'] }) } subSkills = [string[]]$subSkills sourceSha256 = $sourceSha256 diff --git a/tools/validate_knowledge_response.py b/tools/validate_knowledge_response.py new file mode 100644 index 00000000..c4e783be --- /dev/null +++ b/tools/validate_knowledge_response.py @@ -0,0 +1,126 @@ +"""Validate a knowledge response against its question and trusted read evidence. + +Read evidence must come from the consumer's successful tool results in this run, +not from the answering agent's claims. This checks identity and complete-read +evidence; it does not prove comprehension or that every claim follows a source. +Requires jsonschema. Does not repair or rewrite the response. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +from jsonschema import Draft7Validator + + +def validate_response(response, *, question, root, evidence=None, run_id, enabled_layers=("microsoft", "community", "custom")): + root = Path(root).resolve() + schema = json.loads((root / "schemas/knowledge-response.schema.json").read_text(encoding="utf-8-sig")) + validator = Draft7Validator(schema) + errors = [f"Schema: {error.message}" for error in validator.iter_errors(response)] + if errors: + return errors + if response["question"] != question: + errors.append("question must equal the bound input exactly, including whitespace") + known_layers = {"microsoft", "community", "custom"} + enabled_layers = set(enabled_layers) + if not enabled_layers <= known_layers: + errors.append("Unknown enabled layers") + valid_paths = {} + for entry in response["references"] + response["suppressed"]: + name = entry["path"] + parts = name.split("/") + file = (root / name).resolve() + if "\\" in name or any(part in ("", ".", "..") for part in parts) or not file.is_relative_to(root): + errors.append(f"Noncanonical corpus path: {name}") + elif not file.is_file(): + errors.append(f"Corpus path does not exist: {name}") + else: + valid_paths[name] = file + if parts[0] not in enabled_layers: + # Configuration exclusions may identify disabled layers, never citations. + if entry in response["references"] or entry.get("reason") != "configuration": + errors.append(f"Path belongs to a disabled layer: {name}") + reads = {} + if evidence is not None: + if not isinstance(evidence, dict): + errors.append("Read evidence must be an object") + else: + if evidence.get("runId") != run_id: + errors.append("Read evidence belongs to a different run") + expected_hash = hashlib.sha256(question.encode("utf-8")).hexdigest() + if evidence.get("questionSha256") != expected_hash: + errors.append("Read evidence belongs to a different question") + records = evidence.get("reads") + if not isinstance(records, list): + errors.append("Read evidence must contain a reads array") + else: + for record in records: + if not isinstance(record, dict) or not isinstance(record.get("path"), str): + errors.append("Invalid read evidence record") + continue + if record["path"] in reads: + errors.append(f"Duplicate read evidence: {record['path']}") + reads[record["path"]] = record + for reference in response["references"]: + name = reference["path"] + if name not in valid_paths: + continue + file = valid_paths[name] + record = reads.get(name) + if record is None or record.get("complete") is not True: + errors.append(f"No complete read evidence for citation: {name}") + continue + content = file.read_bytes() + try: + content.decode("utf-8") + except UnicodeDecodeError: + errors.append(f"Citation is not strict UTF-8: {name}") + continue + if type(record.get("bytes")) is not int or record["bytes"] != len(content): + errors.append(f"Read byte count does not match the full article: {name}") + if record.get("sha256") != hashlib.sha256(content).hexdigest(): + errors.append(f"Read content hash does not match the live article: {name}") + return errors + + +def strict_json(file): + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError(f"Duplicate JSON key: {key}") + result[key] = value + return result + + def invalid_constant(value): + raise ValueError(f"Invalid JSON constant: {value}") + + return json.loads(Path(file).read_text(encoding="utf-8-sig"), object_pairs_hook=pairs, parse_constant=invalid_constant) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("response", type=Path) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--question-file", type=Path, required=True, help="Exact bound input as UTF-8; no whitespace trimming") + parser.add_argument("--read-evidence", type=Path, help="Consumer-collected evidence; required for cited responses") + parser.add_argument("--run-id", required=True, help="Current run identifier supplied independently by the consumer") + parser.add_argument("--enabled-layers", nargs="*", choices=("microsoft", "community", "custom"), default=["microsoft", "community", "custom"], help="Consumer configuration; an empty list disables every layer") + args = parser.parse_args() + try: + errors = validate_response(strict_json(args.response), + question=args.question_file.read_bytes().decode("utf-8-sig"), root=args.root, + evidence=strict_json(args.read_evidence) if args.read_evidence else None, + run_id=args.run_id, enabled_layers=args.enabled_layers) + except (OSError, ValueError) as error: + print(json.dumps({"valid": False, "errors": [str(error)]})) + return 2 + print(json.dumps({"valid": not errors, "errors": errors})) + return int(bool(errors)) + + +if __name__ == "__main__": + raise SystemExit(main())