laily/.pouch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ack): bind test env to deployer and add regression mode

Browse Source 
ACK 0.19.0 hands test-environment deploys to the deployer skill,
documents bug-fix as a first-class scenario, and adds
docs/ack/regression.yaml harvest plus a /ack regression run.
This commit is contained in:
laily
2026-08-25 14:41:05 +08:00
parent ee31278947
commit ad6695245b
39 changed files with 1919 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_ack_delivery.py
+19 -20
View File
@@ -314,7 +314,7 @@ class AckDeliveryValidationTests(unittest.TestCase):
self.assertEqual(validate_delivery.validate_builtin(contract), [])
def test_intents_must_point_at_matching_stop_points(self) -> None:
def test_test_environment_intent_requires_deployer_binding(self) -> None:
contract = valid_contract()
contract["intents"] = {
"testEnvironment": "review",
@@ -323,33 +323,32 @@ class AckDeliveryValidationTests(unittest.TestCase):
errors = validate_delivery.validate_builtin(contract)
self.assertTrue(
any("intents.testEnvironment" in item and "validation_ready" in item for item in errors)
any("已改为 deployer 绑定" in item for item in errors)
)
contract["intents"]["testEnvironment"] = "local-validation"
contract["profiles"]["local-validation"] = {
"stopAt": "validation_ready",
"steps": [
{"id": "build-local", "action": "build", "artifact": "service-deb"},
{
"id": "deploy-local",
"action": "deploy",
"artifact": "service-deb",
"environment": "test-server",
},
{
"id": "health-local",
"action": "health-check",
"environment": "test-server",
},
],
}
contract["intents"]["testEnvironment"] = {"via": "deployer", "env": "test"}
self.assertEqual(validate_delivery.validate_builtin(contract), [])
contract["intents"]["release"] = "missing-release"
errors = validate_delivery.validate_builtin(contract)
self.assertTrue(any("未定义 profile 'missing-release'" in item for item in errors))
def test_test_environment_binding_requires_deployer_env_directory(self) -> None:
contract = valid_contract()
contract["intents"] = {
"testEnvironment": {"via": "deployer", "env": "test"},
"release": None,
}
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
errors = validate_delivery.validate_builtin(contract, root)
self.assertTrue(
any(".skiff/deployer/test" in item for item in errors)
)
env_dir = root / ".skiff" / "deployer" / "test"
env_dir.mkdir(parents=True)
self.assertEqual(validate_delivery.validate_builtin(contract, root), [])
if __name__ == "__main__":
unittest.main()
tests/test_ack_regression.py
+149
View File
@@ -0,0 +1,149 @@
from __future__ import annotations
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
SCRIPTS_DIR = REPO_ROOT / "skills" / "ack" / "scripts"
VALIDATOR = SCRIPTS_DIR / "validate_regression.py"
SELECTOR = SCRIPTS_DIR / "select_regression.py"
EXAMPLE = REPO_ROOT / "skills" / "ack" / "examples" / "regression.example.yaml"
sys.path.insert(0, str(SCRIPTS_DIR))
import select_regression # noqa: E402
import validate_regression # noqa: E402
def empty_catalog() -> dict:
return {
"version": 1,
"updatedAt": "2026-08-25T10:00:00+08:00",
"project": {"name": "demo"},
"cases": [],
}
def sample_case(**overrides: object) -> dict:
case = {
"id": "REG-login-001",
"title": "登录后进入工作台",
"status": "active",
"source": {"taskId": "BUG-001", "kind": "bug"},
"suite": "smoke",
"surface": "browser",
"setup": "使用测试账号 A，未登录",
"steps": ["打开 /login", "输入账号 A 并提交"],
"expected": [
{"kind": "visible-text", "value": "工作台"},
{"kind": "url", "value": "/dashboard"},
],
}
case.update(overrides)
return case
class AckRegressionValidationTests(unittest.TestCase):
def test_example_and_template_are_valid(self) -> None:
for no_site_packages in (False, True):
command = [sys.executable]
if no_site_packages:
command.append("-S")
result = subprocess.run(
[*command, str(VALIDATOR), str(EXAMPLE)],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=False,
)
with self.subTest(no_site_packages=no_site_packages):
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("回归目录校验通过", result.stdout)
template = REPO_ROOT / "skills" / "ack" / "templates" / "regression.template.yaml"
rendered = template.read_text(encoding="utf-8").replace(
"<project_name>", "demo"
).replace("<YYYY-MM-DDTHH:mm:ss+TZ>", "2026-08-25T10:00:00+08:00")
with tempfile.TemporaryDirectory() as temp_dir:
path = Path(temp_dir) / "regression.yaml"
path.write_text(rendered, encoding="utf-8")
result = subprocess.run(
[sys.executable, str(VALIDATOR), str(path)],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=False,
)
self.assertEqual(result.returncode, 0, result.stderr)
def test_rejects_duplicate_ids_and_unknown_fields(self) -> None:
catalog = empty_catalog()
catalog["cases"] = [sample_case(), sample_case()]
errors = validate_regression.validate_builtin(catalog)
self.assertTrue(any("不能重复" in item for item in errors))
catalog["cases"] = [sample_case(extra="nope")]
errors = validate_regression.validate_builtin(catalog)
self.assertTrue(any("未知字段" in item for item in errors))
def test_selects_smoke_by_default_and_full_includes_all_active(self) -> None:
catalog = empty_catalog()
catalog["cases"] = [
sample_case(),
sample_case(
id="REG-notes-create-001",
suite="full",
surface="api",
source={"taskId": "FEAT-012", "kind": "feature"},
),
sample_case(id="REG-old-001", status="retired"),
]
smoke = select_regression.select_cases(
catalog, suite="smoke", case_ids=[], limit=50
)
full = select_regression.select_cases(
catalog, suite="full", case_ids=[], limit=50
)
self.assertEqual([case["id"] for case in smoke], ["REG-login-001"])
self.assertEqual(
[case["id"] for case in full],
["REG-login-001", "REG-notes-create-001"],
)
def test_selector_cli_and_task_link(self) -> None:
result = subprocess.run(
[sys.executable, str(SELECTOR), str(EXAMPLE), "--format", "ids"],
cwd=REPO_ROOT,
text=True,
capture_output=True,
check=False,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout.strip(), "REG-login-001")
tasks = {
"project": {
"name": "notes-web",
"regressionFile": "docs/ack/regression.yaml",
},
"regressionRuns": [],
"tasks": [
{
"id": "BUG-001",
"regressionRefs": ["REG-login-001"],
}
],
}
catalog = validate_regression.load_yaml(EXAMPLE, "回归目录")
self.assertEqual(validate_regression.validate_tasks_link(catalog, tasks), [])
tasks["project"]["regressionFile"] = "docs/ack/other.yaml"
errors = validate_regression.validate_tasks_link(catalog, tasks)
self.assertTrue(any("必须固定为 docs/ack/regression.yaml" in item for item in errors))
if __name__ == "__main__":
unittest.main()
tests/test_ack_skill.py
+13 -1
View File
@@ -18,18 +18,23 @@ class AckSkillContentTests(unittest.TestCase):
"docs/ack/tasks.yaml",
"docs/ack/knowledge.yaml",
"docs/ack/delivery.yaml",
"docs/ack/regression.yaml",
"tasks: []",
"validate_tasks.py",
"validate_knowledge.py",
"validate_delivery.py",
"validate_regression.py",
"select_tasks.py",
"select_knowledge.py",
"select_regression.py",
"references/kickoff.md",
"不要修改项目的 `AGENTS.md`",
"当前会话担任 Coordinator",
"intents.testEnvironment",
"运行测试环境",
"运行版本发布",
"运行回归",
"via: deployer",
):
self.assertIn(expected, content)
@@ -121,6 +126,12 @@ class AckSkillContentTests(unittest.TestCase):
"scripts/validate_knowledge.py",
"scripts/select_tasks.py",
"scripts/select_knowledge.py",
"scripts/select_regression.py",
"scripts/validate_regression.py",
"templates/regression.template.yaml",
"templates/regression.schema.json",
"examples/regression.example.yaml",
"references/regression.md",
"scripts/run_verification.py",
"scripts/worker_profiles.py",
"scripts/launch_worker.py",
@@ -131,7 +142,7 @@ class AckSkillContentTests(unittest.TestCase):
):
self.assertTrue((ack_dir / relative_path).is_file(), relative_path)
version = (ack_dir / "VERSION").read_text(encoding="utf-8").strip()
self.assertEqual(version, "0.18.0")
self.assertEqual(version, "0.19.0")
self.assertIn(
f'ackVersion: "{version}"',
(ack_dir / "examples" / "tasks.example.yaml").read_text(encoding="utf-8"),
@@ -157,6 +168,7 @@ class AckSkillContentTests(unittest.TestCase):
self.assertNotIn("allowedWorktrees:", template)
self.assertIn("allowedWorktrees", template)
self.assertIn('knowledgeFile: "docs/ack/knowledge.yaml"', template)
self.assertIn('regressionFile: "docs/ack/regression.yaml"', template)
if __name__ == "__main__":
tests/test_ack_tasks_validation.py
+8 -1
View File
@@ -302,7 +302,13 @@ class AckTaskValidationTests(unittest.TestCase):
with self.subTest(invalid_semver=invalid):
self.assertIsNone(ack_pattern.fullmatch(invalid))
current_gate, orchestration_gate, receipts_gate, delivery_gate = schema["allOf"]
(
current_gate,
orchestration_gate,
receipts_gate,
delivery_gate,
regression_gate,
) = schema["allOf"]
current_pattern = re.compile(
current_gate["if"]["properties"]["ackVersion"]["pattern"]
)
@@ -341,6 +347,7 @@ class AckTaskValidationTests(unittest.TestCase):
"#/definitions/launchableTasks",
)
self.assertIn("deliveryRuns", delivery_gate["then"]["required"])
self.assertIn("regressionRuns", regression_gate["then"]["required"])
delivery_run = schema["definitions"]["deliveryRun"]
revision_gate = delivery_run["allOf"][0]
tests/test_ack_yaml_fallback.py
+4 -2
View File
@@ -31,12 +31,14 @@ paths = (
Path('skills/ack/examples/knowledge.example.yaml'),
Path('skills/ack/templates/delivery.template.yaml'),
Path('skills/ack/examples/delivery.example.yaml'),
Path('skills/ack/templates/regression.template.yaml'),
Path('skills/ack/examples/regression.example.yaml'),
)
for path in paths:
value = load_yaml_subset(path.read_text(encoding='utf-8'))
if not isinstance(value, dict):
raise SystemExit(f'{path}: top-level value is not a mapping')
print('parsed=6')
print('parsed=8')
"""
result = subprocess.run(
[sys.executable, "-S", "-c", script],
@@ -47,7 +49,7 @@ print('parsed=6')
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout.strip(), "parsed=6")
self.assertEqual(result.stdout.strip(), "parsed=8")
def test_tasks_validator_runs_without_site_packages(self) -> None:
result = subprocess.run(
tests/test_skill_init.py
+98 -6
View File
@@ -65,10 +65,18 @@ class SkillInitTests(unittest.TestCase):
'profiles: {}\n',
encoding="utf-8",
)
(skill / "templates" / "regression.template.yaml").write_text(
'updatedAt: "<YYYY-MM-DDTHH:mm:ss+TZ>"\n'
'project:\n'
' name: "<project_name>"\n'
'cases: []\n',
encoding="utf-8",
)
for validator_name in (
"validate_tasks.py",
"validate_knowledge.py",
"validate_delivery.py",
"validate_regression.py",
):
(skill / "scripts" / validator_name).write_text(
"raise SystemExit(0)\n",
@@ -105,15 +113,19 @@ class SkillInitTests(unittest.TestCase):
tasks_content = (target / "tasks.yaml").read_text(encoding="utf-8")
knowledge_content = (target / "knowledge.yaml").read_text(encoding="utf-8")
delivery_content = (target / "delivery.yaml").read_text(encoding="utf-8")
regression_content = (target / "regression.yaml").read_text(encoding="utf-8")
self.assertIn("# sample-app", project_content)
self.assertIn("version=1.2.3", project_content)
self.assertIn(f'repoPath: "{project}"', tasks_content)
self.assertIn(f'repoPath: "{project}"', knowledge_content)
self.assertIn('name: "sample-app"', delivery_content)
self.assertIn("enabled: false", delivery_content)
self.assertIn('name: "sample-app"', regression_content)
self.assertIn("cases: []", regression_content)
self.assertNotIn("<project_name>", tasks_content)
self.assertNotIn("<project_name>", knowledge_content)
self.assertNotIn("<project_name>", delivery_content)
self.assertNotIn("<project_name>", regression_content)
def test_init_refuses_to_overwrite_existing_files(self) -> None:
project = self.home / "existing-app"
@@ -130,6 +142,7 @@ class SkillInitTests(unittest.TestCase):
self.assertFalse((target / "tasks.yaml").exists())
self.assertFalse((target / "knowledge.yaml").exists())
self.assertFalse((target / "delivery.yaml").exists())
self.assertFalse((target / "regression.yaml").exists())
def test_init_refuses_to_overwrite_existing_knowledge_file(self) -> None:
project = self.home / "existing-knowledge-app"
@@ -161,6 +174,22 @@ class SkillInitTests(unittest.TestCase):
self.assertFalse((target / "project.md").exists())
self.assertFalse((target / "tasks.yaml").exists())
self.assertFalse((target / "knowledge.yaml").exists())
self.assertFalse((target / "regression.yaml").exists())
def test_init_refuses_to_overwrite_existing_regression_file(self) -> None:
project = self.home / "existing-regression-app"
target = project / "docs" / "ack"
target.mkdir(parents=True)
existing = target / "regression.yaml"
existing.write_text("keep me", encoding="utf-8")
result = self.run_skiff("init", "ack", "--project", str(project))
self.assertNotEqual(result.returncode, 0)
self.assertIn("拒绝覆盖已有路径", result.stderr)
self.assertEqual(existing.read_text(encoding="utf-8"), "keep me")
self.assertFalse((target / "project.md").exists())
self.assertFalse((target / "tasks.yaml").exists())
def test_init_rejects_symlinked_destination_directories(self) -> None:
for symlink_level in ("docs", "ack"):
@@ -336,7 +365,13 @@ class SkillInitTests(unittest.TestCase):
self.assertEqual(list((project / "docs" / "ack").iterdir()), [])
self.assertEqual(
sorted(path.name for path in moved_target.iterdir()),
["delivery.yaml", "knowledge.yaml", "project.md", "tasks.yaml"],
[
"delivery.yaml",
"knowledge.yaml",
"project.md",
"regression.yaml",
"tasks.yaml",
],
)
def test_transaction_container_replacement_cannot_forge_payload(self) -> None:
@@ -395,7 +430,13 @@ class SkillInitTests(unittest.TestCase):
self.assertFalse((target / "marker").exists())
self.assertEqual(
sorted(path.name for path in target.iterdir()),
["delivery.yaml", "knowledge.yaml", "project.md", "tasks.yaml"],
[
"delivery.yaml",
"knowledge.yaml",
"project.md",
"regression.yaml",
"tasks.yaml",
],
)
def test_post_publish_fsync_failure_preserves_complete_state(self) -> None:
@@ -407,7 +448,7 @@ class SkillInitTests(unittest.TestCase):
def fail_directory_fsync_after_publish(file_descriptor: int) -> None:
nonlocal calls
calls += 1
if calls == 8:
if calls == 9:
raise OSError("simulated directory fsync failure")
real_fsync(file_descriptor)
@@ -433,7 +474,13 @@ class SkillInitTests(unittest.TestCase):
target = project / "docs" / "ack"
self.assertEqual(
sorted(path.name for path in target.iterdir()),
["delivery.yaml", "knowledge.yaml", "project.md", "tasks.yaml"],
[
"delivery.yaml",
"knowledge.yaml",
"project.md",
"regression.yaml",
"tasks.yaml",
],
)
def test_project_root_replacement_aborts_before_publish(self) -> None:
@@ -520,7 +567,13 @@ class SkillInitTests(unittest.TestCase):
target = moved_project / "docs" / "ack"
self.assertEqual(
sorted(path.name for path in target.iterdir()),
["delivery.yaml", "knowledge.yaml", "project.md", "tasks.yaml"],
[
"delivery.yaml",
"knowledge.yaml",
"project.md",
"regression.yaml",
"tasks.yaml",
],
)
def test_docs_replacement_aborts_before_publish(self) -> None:
@@ -615,7 +668,13 @@ class SkillInitTests(unittest.TestCase):
target = moved_docs / "ack"
self.assertEqual(
sorted(path.name for path in target.iterdir()),
["delivery.yaml", "knowledge.yaml", "project.md", "tasks.yaml"],
[
"delivery.yaml",
"knowledge.yaml",
"project.md",
"regression.yaml",
"tasks.yaml",
],
)
def test_ack_init_requires_knowledge_template(self) -> None:
@@ -652,11 +711,29 @@ class SkillInitTests(unittest.TestCase):
self.assertIn("delivery.template.yaml", result.stderr)
self.assertFalse((project / "docs" / "ack").exists())
def test_ack_init_requires_regression_template(self) -> None:
project = self.home / "missing-regression-template-app"
project.mkdir()
(
self.skills_home
/ "skills"
/ "ack"
/ "templates"
/ "regression.template.yaml"
).unlink()
result = self.run_skiff("init", "ack", "--project", str(project))
self.assertNotEqual(result.returncode, 0)
self.assertIn("regression.template.yaml", result.stderr)
self.assertFalse((project / "docs" / "ack").exists())
def test_ack_init_requires_all_validators(self) -> None:
for validator_name in (
"validate_tasks.py",
"validate_knowledge.py",
"validate_delivery.py",
"validate_regression.py",
):
with self.subTest(validator_name=validator_name):
project = self.home / f"missing-{validator_name}-app"
@@ -715,6 +792,21 @@ class SkillInitTests(unittest.TestCase):
self.assertIn("初始化交付契约校验失败", result.stderr)
self.assertFalse((project / "docs" / "ack").exists())
def test_regression_validator_failure_leaves_no_partial_initialization(self) -> None:
project = self.home / "invalid-regression-app"
project.mkdir()
validator = (
self.skills_home / "skills" / "ack" / "scripts" / "validate_regression.py"
)
validator.write_text("raise SystemExit(1)\n", encoding="utf-8")
result = self.run_skiff("init", "ack", "--project", str(project))
self.assertNotEqual(result.returncode, 0)
self.assertNotIn("Traceback", result.stderr)
self.assertIn("初始化回归目录校验失败", result.stderr)
self.assertFalse((project / "docs" / "ack").exists())
def test_validator_cannot_replace_staged_bytes_before_install(self) -> None:
project = self.home / "mutated-staging-app"
project.mkdir()