laily/.pouch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ack): add project knowledge guardrails

Browse Source
This commit is contained in:
laily
2026-07-31 21:09:36 +08:00
parent 7d1994cf93
commit ee66dbe9ce
32 changed files with 8353 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_ack_tasks_validation.py
+705
View File
@@ -0,0 +1,705 @@
from __future__ import annotations
import json
import subprocess
import sys
import tempfile
import textwrap
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
VALIDATOR = REPO_ROOT / "skills" / "ack" / "scripts" / "validate_tasks.py"
EXAMPLE = REPO_ROOT / "skills" / "ack" / "examples" / "tasks.example.yaml"
def valid_knowledge_board() -> dict:
return {
"version": 1,
"project": {"name": "demo"},
"tasks": [
{
"id": "T-1",
"title": "validate knowledge fields",
"status": "open",
"knowledgeRefs": ["K-001@1"],
"knowledgeApplied": [
{
"ref": "K-001@1",
"result": "applied",
"evidence": "followed the guardrail",
}
],
"knowledgeCandidates": [
{
"kind": "pitfall",
"title": "candidate",
"claim": "the failure is reproducible",
"scope": {"components": ["web"]},
"appliesWhen": "the web component changes",
"directive": "run the reviewed check",
"rationale": "avoid the repeated failure",
"evidenceRefs": ["tasks.yaml#T-1"],
"proposedBy": "developer",
"proposedAt": "2026-07-31T10:00:00+08:00",
}
],
"knowledgeChecks": [
{
"ref": "K-001@1",
"result": "passed",
"evidence": "independently verified",
"checkedBy": "test",
"checkedAt": "2026-07-31T10:05:00+08:00",
}
],
}
],
}
class AckTaskValidationTests(unittest.TestCase):
def run_validator(
self,
content: str | None = None,
*extra_args: str,
no_site_packages: bool = False,
suffix: str = ".yaml",
) -> subprocess.CompletedProcess[str]:
command = [sys.executable]
if no_site_packages:
command.append("-S")
command.append(str(VALIDATOR))
if content is None:
return subprocess.run(
[*command, *extra_args, str(EXAMPLE)],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=False,
)
with tempfile.TemporaryDirectory() as temp_dir:
task_file = Path(temp_dir) / f"tasks{suffix}"
task_file.write_text(textwrap.dedent(content), encoding="utf-8")
return subprocess.run(
[*command, *extra_args, str(task_file)],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=False,
)
def assert_board_rejected_in_all_modes(
self,
board: dict,
*expected_messages: str,
) -> None:
for no_site_packages in (False, True):
with self.subTest(no_site_packages=no_site_packages):
result = self.run_validator(
json.dumps(board),
no_site_packages=no_site_packages,
suffix=".json",
)
self.assertEqual(result.returncode, 1, result.stdout)
for message in expected_messages:
self.assertIn(message, result.stderr)
if no_site_packages:
self.assertIn("内置语义规则", result.stderr)
self.assertNotIn("[schema]", result.stderr)
def assert_board_accepted_in_all_modes(self, board: dict) -> None:
for no_site_packages in (False, True):
with self.subTest(no_site_packages=no_site_packages):
result = self.run_validator(
json.dumps(board),
no_site_packages=no_site_packages,
suffix=".json",
)
self.assertEqual(result.returncode, 0, result.stderr)
def test_example_with_knowledge_fields_is_valid(self) -> None:
result = self.run_validator()
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("任务板校验通过", result.stdout)
def test_knowledge_applied_and_checks_must_reference_selected_knowledge(self) -> None:
result = self.run_validator(
"""
version: 1
project:
name: demo
tasks:
- id: T-1
title: invalid refs
status: verified
knowledgeRefs: ["K-001@1"]
knowledgeApplied:
- ref: "K-002@1"
result: applied
evidence: "used the rule"
knowledgeChecks:
- ref: "K-003@1"
result: failed
evidence: "still broken"
"""
)
self.assertEqual(result.returncode, 1)
self.assertIn("K-002@1 不在 knowledgeRefs 中", result.stderr)
self.assertIn("K-003@1 不在 knowledgeRefs 中", result.stderr)
self.assertIn("verified 任务不能保留失败", result.stderr)
def test_knowledge_refs_require_revision(self) -> None:
result = self.run_validator(
"""
version: 1
project:
name: demo
tasks:
- id: T-1
title: invalid ref
status: open
knowledgeRefs: ["K-001"]
"""
)
self.assertEqual(result.returncode, 1)
self.assertIn("K-<id>@<revision>", result.stderr)
def test_candidate_requires_actionable_scope_and_evidence(self) -> None:
result = self.run_validator(
"""
version: 1
project:
name: demo
tasks:
- id: T-1
title: invalid candidate
status: open
knowledgeCandidates:
- kind: guess
title: maybe
claim: uncertain
scope: {}
appliesWhen: sometimes
directive: retry
rationale: unknown
evidenceRefs: []
"""
)
self.assertEqual(result.returncode, 1)
self.assertIn(".kind: 必须是", result.stderr)
self.assertIn(".scope: 至少包含一个非空作用域", result.stderr)
self.assertIn(".evidenceRefs: 必须是非空字符串列表", result.stderr)
def test_forbidden_knowledge_properties_fail_with_and_without_jsonschema(self) -> None:
board = valid_knowledge_board()
task = board["tasks"][0]
task["knowledgeApplied"][0]["unexpectedApplication"] = True
task["knowledgeCandidates"][0]["unexpectedCandidate"] = True
task["knowledgeChecks"][0]["unexpectedCheck"] = True
self.assert_board_rejected_in_all_modes(
board,
"knowledgeApplied[0]: 未知字段 'unexpectedApplication'",
"knowledgeCandidates[0]: 未知字段 'unexpectedCandidate'",
"knowledgeChecks[0]: 未知字段 'unexpectedCheck'",
)
def test_valid_optional_knowledge_fields_pass_in_all_modes(self) -> None:
self.assert_board_accepted_in_all_modes(valid_knowledge_board())
def test_explicit_null_knowledge_collections_fail_in_all_modes(self) -> None:
board = {
"version": 1,
"project": {"name": "demo"},
"tasks": [
{
"id": "T-REFS",
"title": "null refs",
"status": "open",
"knowledgeRefs": None,
},
{
"id": "T-APPLIED",
"title": "null applications",
"status": "open",
"knowledgeApplied": None,
},
{
"id": "T-CANDIDATES",
"title": "null candidates",
"status": "open",
"knowledgeCandidates": None,
},
{
"id": "T-CHECKS",
"title": "null checks",
"status": "open",
"knowledgeChecks": None,
},
],
}
self.assert_board_rejected_in_all_modes(
board,
"knowledgeRefs: 必须是列表",
"knowledgeApplied: 必须是列表",
"knowledgeCandidates: 必须是列表",
"knowledgeChecks: 必须是列表",
)
def test_knowledge_item_types_and_blank_evidence_fail_in_all_modes(self) -> None:
board = valid_knowledge_board()
task = board["tasks"][0]
task["knowledgeApplied"][0].update(
{"ref": 1, "result": "unknown", "evidence": " "}
)
task["knowledgeChecks"][0].update(
{"ref": False, "result": "unknown", "evidence": "\t"}
)
self.assert_board_rejected_in_all_modes(
board,
"knowledgeApplied[0].ref: 必须使用 K-<id>@<revision> 格式",
"knowledgeApplied[0].result: 必须是 applied/not_applicable",
"knowledgeApplied[0].evidence: 必须提供非空证据",
"knowledgeChecks[0].ref: 必须使用 K-<id>@<revision> 格式",
"knowledgeChecks[0].result: 必须是",
"knowledgeChecks[0].evidence: 必须提供非空证据",
)
def test_candidate_text_scope_and_evidence_refs_fail_in_all_modes(self) -> None:
board = valid_knowledge_board()
candidate = board["tasks"][0]["knowledgeCandidates"][0]
candidate.update(
{
"title": 1,
"claim": " ",
"appliesWhen": [],
"directive": "",
"rationale": None,
"scope": {
"components": ["web", "web"],
"paths": [" "],
},
"evidenceRefs": ["tasks.yaml#T-1", "tasks.yaml#T-1"],
}
)
self.assert_board_rejected_in_all_modes(
board,
"knowledgeCandidates[0].title: 必须是非空字符串",
"knowledgeCandidates[0].claim: 必须是非空字符串",
"knowledgeCandidates[0].appliesWhen: 必须是非空字符串",
"knowledgeCandidates[0].directive: 必须是非空字符串",
"knowledgeCandidates[0].rationale: 必须是非空字符串",
"knowledgeCandidates[0].scope.components: 不能包含重复值",
"knowledgeCandidates[0].scope.paths: 必须是非空字符串列表",
"knowledgeCandidates[0].evidenceRefs: 不能包含重复值",
)
def test_candidate_evidence_refs_reject_blank_strings_in_all_modes(self) -> None:
board = valid_knowledge_board()
board["tasks"][0]["knowledgeCandidates"][0]["evidenceRefs"] = [" "]
self.assert_board_rejected_in_all_modes(
board,
"knowledgeCandidates[0].evidenceRefs: 必须是非空字符串列表",
)
def test_optional_knowledge_field_types_fail_in_all_modes(self) -> None:
board = valid_knowledge_board()
candidate = board["tasks"][0]["knowledgeCandidates"][0]
candidate["proposedBy"] = 1
candidate["proposedAt"] = []
check = board["tasks"][0]["knowledgeChecks"][0]
check["checkedBy"] = False
check["checkedAt"] = {}
self.assert_board_rejected_in_all_modes(
board,
"knowledgeCandidates[0].proposedBy: 必须是字符串",
"knowledgeCandidates[0].proposedAt: 必须是字符串",
"knowledgeChecks[0].checkedBy: 必须是字符串",
"knowledgeChecks[0].checkedAt: 必须是字符串",
)
def test_attempt_id_must_match_task_and_round_and_be_unique(self) -> None:
board = {
"version": 1,
"project": {"name": "demo"},
"tasks": [
{
"id": "BUG-017",
"title": "invalid attempt ids",
"status": "failed_retest",
"dispatch": {
"rounds": [
{
"round": 1,
"attemptId": "OTHER-A1",
"result": "failed",
},
{
"round": 1,
"attemptId": "OTHER-A1",
"result": "failed",
},
{
"round": 3,
"attemptId": "invalid/attempt",
"result": "failed",
},
]
},
}
],
}
self.assert_board_rejected_in_all_modes(
board,
"dispatch.rounds[0].attemptId: 应为 BUG-017-A1",
"dispatch.rounds[1].attemptId: 轮次内不能重复: OTHER-A1",
"dispatch.rounds[2].attemptId: 必须使用 <task-id>-A<round> 格式",
)
def test_valid_optional_attempt_ids_pass_in_all_modes(self) -> None:
board = {
"version": 1,
"project": {"name": "demo"},
"tasks": [
{
"id": "BUG-017",
"title": "valid attempt ids",
"status": "failed_retest",
"dispatch": {
"rounds": [
{
"round": 1,
"attemptId": "BUG-017-A1",
"result": "failed",
},
{
"round": 2,
"attemptId": "BUG-017-A2",
"result": "failed",
},
]
},
}
],
}
self.assert_board_accepted_in_all_modes(board)
def test_boolean_version_fails_in_all_modes(self) -> None:
board = valid_knowledge_board()
board["version"] = True
self.assert_board_rejected_in_all_modes(
board,
"version 必须是 >=1 的整数",
)
def test_knowledge_file_is_fixed_in_all_modes(self) -> None:
board = valid_knowledge_board()
board["project"]["knowledgeFile"] = "docs/ack/alternate.yaml"
self.assert_board_rejected_in_all_modes(
board,
"project.knowledgeFile 必须固定为 docs/ack/knowledge.yaml",
)
def test_basic_identifiers_must_be_nonempty_strings_in_all_modes(self) -> None:
board = valid_knowledge_board()
board["project"]["name"] = 7
board["tasks"][0]["id"] = 9
board["tasks"][0]["title"] = " "
self.assert_board_rejected_in_all_modes(
board,
"project.name 必须是非空字符串",
"id 必须是非空字符串",
"title 必须是非空字符串",
)
def test_root_project_and_summary_types_match_schema_in_all_modes(self) -> None:
board = valid_knowledge_board()
board.update(
{
"updatedAt": [],
"source": {},
"ackVersion": 1,
"kitVersion": False,
"summary": {
"verified": [1],
"open": {},
"failedRetest": [False],
"leftovers": None,
},
"statusReference": [],
}
)
board["project"].update(
{
"repoPath": [],
"baseUrl": {},
"devWorktree": 1,
"overlayFile": False,
}
)
self.assert_board_rejected_in_all_modes(
board,
"<root>.updatedAt: 必须是字符串",
"<root>.source: 必须是字符串",
"<root>.ackVersion: 必须是字符串",
"<root>.kitVersion: 必须是字符串",
"project.repoPath: 必须是字符串",
"project.baseUrl: 必须是字符串",
"project.devWorktree: 必须是字符串",
"project.overlayFile: 必须是字符串",
"summary.verified: 列表项必须是字符串",
"summary.open: 必须是列表",
"summary.failedRetest: 列表项必须是字符串",
"summary.leftovers: 必须是列表",
"statusReference 必须是对象",
)
def test_task_optional_types_match_schema_in_all_modes(self) -> None:
board = valid_knowledge_board()
board["tasks"][0].update(
{
"type": [],
"priority": {},
"assignee": False,
"component": 1,
"specRefs": {},
"testRefs": [1],
"description": [],
"stepsToReproduce": [{}],
"expected": False,
"actual": None,
"evidence": [],
"verification": [],
}
)
self.assert_board_rejected_in_all_modes(
board,
".type: 必须是字符串",
".priority: 必须是字符串",
".assignee: 必须是字符串",
".component: 必须是字符串",
".specRefs: 必须是列表",
".testRefs: 列表项必须是字符串",
".description: 必须是字符串",
".stepsToReproduce: 列表项必须是字符串",
".expected: 必须是字符串",
".actual: 必须是字符串",
".evidence: 必须是对象",
".verification: 必须是对象",
)
def test_dispatch_and_resolution_types_match_schema_in_all_modes(self) -> None:
board = valid_knowledge_board()
board["tasks"][0].update(
{
"dispatch": {
"taskId": [],
"dispatchId": {},
"worker": False,
"rounds": [
{
"round": 1,
"result": "failed",
"evidence": [],
}
],
},
"resolution": {
"fixedBy": [],
"verifiedBy": {},
"verifiedAt": False,
"leftoverReason": 1,
"evidence": [],
},
}
)
self.assert_board_rejected_in_all_modes(
board,
".dispatch.taskId: 必须是字符串或 null",
".dispatch.dispatchId: 必须是字符串或 null",
".dispatch.worker: 必须是字符串或 null",
".dispatch.rounds[0].evidence: 必须是字符串",
".resolution.fixedBy: 必须是字符串或 null",
".resolution.verifiedBy: 必须是字符串或 null",
".resolution.verifiedAt: 必须是字符串或 null",
".resolution.leftoverReason: 必须是字符串或 null",
".resolution.evidence: 必须是对象",
)
def test_explicit_null_structures_fail_in_all_modes(self) -> None:
board = {
"version": 1,
"project": {"name": "demo"},
"tasks": [
{
"id": "T-DISPATCH",
"title": "invalid dispatch",
"status": "open",
"dispatch": None,
},
{
"id": "T-ROUNDS",
"title": "invalid rounds",
"status": "open",
"dispatch": {"rounds": None},
},
{
"id": "T-RESOLUTION",
"title": "invalid resolution",
"status": "open",
"resolution": None,
},
],
}
self.assert_board_rejected_in_all_modes(
board,
"T-DISPATCH.dispatch: 必须是对象",
"T-ROUNDS.dispatch.rounds: 必须是列表",
"T-RESOLUTION.resolution: 必须是对象",
)
def test_valid_optional_schema_fields_pass_in_all_modes(self) -> None:
board = valid_knowledge_board()
board.update(
{
"updatedAt": "2026-07-31T10:00:00+08:00",
"source": "manual",
"ackVersion": "0.9.0",
"kitVersion": "0.8.0",
"summary": {
"verified": ["T-1"],
"open": [],
"failedRetest": [],
"leftovers": [],
},
"statusReference": {},
}
)
board["project"].update(
{
"repoPath": "/repo",
"baseUrl": "http://127.0.0.1:3000",
"devWorktree": "/repo-dev",
"overlayFile": "docs/ack/project.md",
"knowledgeFile": "docs/ack/knowledge.yaml",
}
)
board["tasks"][0].update(
{
"type": "bug",
"priority": "P1",
"assignee": "developer",
"component": "web",
"specRefs": ["spec.md"],
"testRefs": ["tests/test_web.py"],
"description": "description",
"stepsToReproduce": ["open page"],
"expected": "works",
"actual": "fails",
"evidence": {},
"verification": {},
"dispatch": {
"taskId": "orca-task",
"dispatchId": None,
"worker": "worker-1",
"rounds": [
{
"round": 1,
"attemptId": "T-1-A1",
"result": "failed",
"evidence": "test output",
}
],
},
"resolution": {
"fixedBy": "developer",
"verifiedBy": None,
"verifiedAt": None,
"leftoverReason": None,
"evidence": {},
},
}
)
self.assert_board_accepted_in_all_modes(board)
def test_round_numbers_must_be_contiguous_and_within_budget(self) -> None:
board = {
"version": 1,
"project": {"name": "demo"},
"tasks": [
{
"id": "BUG-017",
"title": "invalid round number",
"status": "failed_retest",
"dispatch": {
"rounds": [
{
"round": 999,
"attemptId": "BUG-017-A999",
"result": "failed",
}
]
},
}
],
}
self.assert_board_rejected_in_all_modes(
board,
"dispatch.rounds[0].round: 必须是 1..3 的整数",
"dispatch.rounds: round 必须从 1 连续递增且不重复",
)
def test_leftover_reason_must_be_nonempty_string_in_all_modes(self) -> None:
board = {
"version": 1,
"project": {"name": "demo"},
"tasks": [
{
"id": "T-1",
"title": "invalid leftover reason",
"status": "leftover",
"resolution": {"leftoverReason": True},
}
],
}
self.assert_board_rejected_in_all_modes(
board,
"leftover 必须填 resolution.leftoverReason",
)
def test_explicit_missing_schema_is_an_environment_error(self) -> None:
result = self.run_validator(None, "--schema", "/definitely/missing/schema.json")
self.assertEqual(result.returncode, 2)
self.assertIn("找不到指定的 schema 文件", result.stderr)
if __name__ == "__main__":
unittest.main()