feat(tester): support isolated full regression workflow
This commit is contained in:
@@ -150,6 +150,86 @@ class TwoTierCatalogTests(unittest.TestCase):
|
||||
self.assertEqual(report_rows[0]["test_type"], "integration")
|
||||
self.assertEqual(report_rows[0]["suite"], "api-contract")
|
||||
|
||||
def test_isolated_full_keeps_risk_and_ignores_authorize_filter(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
write_mixed_catalog(root)
|
||||
catalog_path = root / ".pouch/tester/catalog.yaml"
|
||||
catalog = yaml.safe_load(catalog_path.read_text(encoding="utf-8"))
|
||||
catalog["authorize"] = {"mutating": "AUTHORIZE_MUTATING"}
|
||||
catalog["execution_profile"] = "isolated-full"
|
||||
catalog_path.write_text(yaml.safe_dump(catalog, sort_keys=False), encoding="utf-8")
|
||||
case_path = root / ".pouch/tester/cases/mixed.yaml"
|
||||
cases = yaml.safe_load(case_path.read_text(encoding="utf-8"))
|
||||
journey = cases["cases"][0]
|
||||
journey.update({"risk": "mutating", "tag": "full", "client_suite": "client-regression"})
|
||||
case_path.write_text(yaml.safe_dump(cases, sort_keys=False), encoding="utf-8")
|
||||
|
||||
selected = self.run_catalog(root, "select", "--suite", "client-regression")
|
||||
self.assertEqual(selected.returncode, 0, selected.stderr)
|
||||
payload = json.loads(selected.stdout)
|
||||
self.assertEqual(payload["execution_profile"], "isolated-full")
|
||||
self.assertEqual(payload["risk_filter"], "disabled")
|
||||
self.assertEqual(payload["count"], 1)
|
||||
self.assertEqual(payload["cases"][0]["risk"], "mutating")
|
||||
self.assertEqual(payload["skipped"], [])
|
||||
|
||||
all_active = self.run_catalog(root, "select")
|
||||
self.assertEqual(all_active.returncode, 0, all_active.stderr)
|
||||
all_payload = json.loads(all_active.stdout)
|
||||
self.assertEqual(all_payload["count"], 2)
|
||||
self.assertEqual(all_payload["test_types"], {"journey": 1, "integration": 1})
|
||||
|
||||
def test_isolated_full_run_script_does_not_use_authorize_switch(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
tester = root / ".pouch" / "tester"
|
||||
(tester / "cases").mkdir(parents=True)
|
||||
(tester / "local.yaml").write_text("HOST: http://127.0.0.1\nAUTHORIZE_MUTATING: false\n", encoding="utf-8")
|
||||
(tester / "catalog.yaml").write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"version": 2,
|
||||
"id_pattern": "^TC-[A-Z]+-[0-9]+$",
|
||||
"categories": [{"id": "SCRIPT"}],
|
||||
"risks": ["readonly", "mutating", "destructive"],
|
||||
"default_max_risk": "readonly",
|
||||
"execution_profile": "isolated-full",
|
||||
"authorize": {"mutating": "AUTHORIZE_MUTATING"},
|
||||
},
|
||||
sort_keys=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tester / "cases" / "script.yaml").write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"category": "SCRIPT",
|
||||
"cases": [{
|
||||
"id": "TC-SCRIPT-001",
|
||||
"title": "mutating script",
|
||||
"what": "run isolated script",
|
||||
"lifecycle": "active",
|
||||
"tag": "full",
|
||||
"risk": "mutating",
|
||||
"driver": "script",
|
||||
"how": {"run": "exit 0"},
|
||||
"cleanup": {"steps": ["reset"]},
|
||||
"expected": [{"kind": "script-exit", "value": 0}],
|
||||
}],
|
||||
},
|
||||
sort_keys=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(CATALOG), "run-script", "--root", str(root), "--id", "TC-SCRIPT-001"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(json.loads(result.stdout)["result"], "executed")
|
||||
|
||||
|
||||
class IntegrationFirstTests(unittest.TestCase):
|
||||
def test_preflight_block_does_not_create_session(self):
|
||||
@@ -177,9 +257,38 @@ class IntegrationFirstTests(unittest.TestCase):
|
||||
execute_journey=lambda session, case: "pass",
|
||||
)
|
||||
self.assertEqual(result["status"], "fail")
|
||||
self.assertEqual(calls, ["INT-1"])
|
||||
self.assertEqual(calls, ["INT-1", "INT-2"])
|
||||
self.assertFalse(result["session_created"])
|
||||
self.assertEqual([row["result"] for row in result["results"]], ["fail", "not-run", "not-run", "not-run"])
|
||||
self.assertEqual([row["result"] for row in result["results"]], ["fail", "pass", "not-run", "not-run"])
|
||||
self.assertEqual(result["results"][2]["blocked_by"], ["INT-1"])
|
||||
|
||||
def test_independent_integration_continues_and_dependency_is_blocked(self):
|
||||
result = run_integration_first(
|
||||
preflight_status="ready",
|
||||
integration_cases=[
|
||||
{"id": "INT-1"},
|
||||
{"id": "INT-2"},
|
||||
{"id": "INT-3", "depends_on": ["INT-1"]},
|
||||
],
|
||||
journey_cases=[],
|
||||
execute_integration=lambda case: "fail" if case["id"] == "INT-1" else "pass",
|
||||
create_journey_session=lambda: object(),
|
||||
execute_journey=lambda session, case: "pass",
|
||||
)
|
||||
self.assertEqual([row["result"] for row in result["results"]], ["fail", "pass", "not-run"])
|
||||
self.assertEqual(result["results"][2]["blocked_by"], ["INT-1"])
|
||||
|
||||
def test_preflight_block_creates_every_result_row(self):
|
||||
result = run_integration_first(
|
||||
preflight_status="blocked",
|
||||
integration_cases=[{"id": "INT-1"}],
|
||||
journey_cases=[{"id": "J-1"}],
|
||||
execute_integration=lambda case: "pass",
|
||||
create_journey_session=lambda: object(),
|
||||
execute_journey=lambda session, case: "pass",
|
||||
)
|
||||
self.assertEqual([row["result"] for row in result["results"]], ["not-run", "not-run"])
|
||||
self.assertEqual(result["results"][0]["blocked_by"], ["test-preflight"])
|
||||
|
||||
def test_service_unreachable_is_environment_failure(self):
|
||||
self.assertEqual(classify_integration(ok=False, service_reachable=False), "environment-fail")
|
||||
|
||||
Reference in New Issue
Block a user