300 lines
13 KiB
Python
300 lines
13 KiB
Python
import csv
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
|
CATALOG = SCRIPTS / "catalog.py"
|
|
CASES_CSV = SCRIPTS / "generate_cases_csv.py"
|
|
RUN_CSV = SCRIPTS / "generate_run_csv.py"
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
from two_tier import classify_integration, run_integration_first
|
|
|
|
|
|
def write_mixed_catalog(root: Path) -> None:
|
|
tester = root / ".pouch" / "tester"
|
|
(tester / "cases").mkdir(parents=True)
|
|
(tester / "runs").mkdir()
|
|
(tester / "catalog.yaml").write_text(
|
|
yaml.safe_dump(
|
|
{
|
|
"version": 2,
|
|
"id_pattern": "^TC-[A-Z]+-[0-9]+$",
|
|
"categories": [{"id": "MIXED"}],
|
|
"risks": ["readonly", "mutating", "destructive"],
|
|
"default_max_risk": "readonly",
|
|
"client_suites": {
|
|
"client-smoke": {"default": True},
|
|
"client-regression": {"default": False},
|
|
"safety": {"default": False},
|
|
},
|
|
"sessions": {"browser": {"kind": "browser", "reuse": "run"}},
|
|
},
|
|
sort_keys=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
journey = {
|
|
"id": "TC-JOURNEY-001",
|
|
"test_type": "journey",
|
|
"category": "MIXED",
|
|
"title": "journey",
|
|
"what": "client path",
|
|
"lifecycle": "active",
|
|
"tag": "smoke",
|
|
"risk": "readonly",
|
|
"driver": "browser",
|
|
"automation": "automated",
|
|
"session": "browser",
|
|
"client_suite": "client-smoke",
|
|
"platform": "android",
|
|
"journey": {
|
|
"goal": "goal",
|
|
"entry_state": "logged_out",
|
|
"actions": ["login"],
|
|
"assertions": ["visible"],
|
|
"final_state": "done",
|
|
"failure_policy": "stop",
|
|
},
|
|
"fixture": {"account": "test-role"},
|
|
"cleanup": {"required": False, "reason": "read-only"},
|
|
"how": {"steps": ["open"]},
|
|
"expected": [{"kind": "visible-text", "value": "done"}],
|
|
}
|
|
integration = {
|
|
"id": "TC-INTEGRATION-001",
|
|
"test_type": "integration",
|
|
"category": "MIXED",
|
|
"title": "contract",
|
|
"what": "API contract",
|
|
"lifecycle": "active",
|
|
"tag": "slow",
|
|
"risk": "readonly",
|
|
"driver": "script",
|
|
"automation": "automated",
|
|
"integration_suite": "api-contract",
|
|
"fixture": {"references": ["isolated-account"]},
|
|
"request": {"method": "GET", "path": "/health"},
|
|
"how": {"run": "true"},
|
|
"cleanup": {"required": False, "reason": "read-only"},
|
|
"expected": [{"kind": "api-status", "value": 200}],
|
|
}
|
|
(tester / "cases" / "mixed.yaml").write_text(
|
|
yaml.safe_dump({"category": "MIXED", "cases": [journey, integration]}, sort_keys=False),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
class TwoTierCatalogTests(unittest.TestCase):
|
|
def run_catalog(self, root: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
[sys.executable, str(CATALOG), *args, "--root", str(root)],
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
|
|
def test_default_selection_excludes_integration_and_explicit_selection_includes_it(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
write_mixed_catalog(root)
|
|
valid = self.run_catalog(root, "validate")
|
|
self.assertEqual(valid.returncode, 0, valid.stderr)
|
|
|
|
default = self.run_catalog(root, "select")
|
|
self.assertEqual(default.returncode, 0, default.stderr)
|
|
default_payload = json.loads(default.stdout)
|
|
self.assertEqual(default_payload["count"], 1)
|
|
self.assertEqual(default_payload["test_types"], {"journey": 1, "integration": 0})
|
|
|
|
integration = self.run_catalog(root, "select", "--test-type", "integration")
|
|
self.assertEqual(integration.returncode, 0, integration.stderr)
|
|
integration_payload = json.loads(integration.stdout)
|
|
self.assertEqual(integration_payload["count"], 1)
|
|
self.assertEqual(integration_payload["cases"][0]["test_type"], "integration")
|
|
|
|
csv_result = subprocess.run(
|
|
[sys.executable, str(CASES_CSV), "--root", str(root)],
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
self.assertEqual(csv_result.returncode, 0, csv_result.stderr)
|
|
with (root / ".pouch/tester/cases.csv").open(encoding="utf-8-sig", newline="") as handle:
|
|
rows = list(csv.DictReader(handle))
|
|
self.assertEqual({row["test_type"] for row in rows}, {"journey", "integration"})
|
|
self.assertEqual({row["suite"] for row in rows}, {"client-smoke", "api-contract"})
|
|
|
|
run_path = root / ".pouch/tester/runs/two-tier.yaml"
|
|
run_path.write_text(
|
|
yaml.safe_dump(
|
|
{"runId": "two-tier", "results": [
|
|
{"id": "TC-INTEGRATION-001", "result": "pass"},
|
|
{"id": "TC-JOURNEY-001", "result": "not-run"},
|
|
]},
|
|
sort_keys=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
run_csv = subprocess.run(
|
|
[sys.executable, str(RUN_CSV), "--root", str(root), "--run", str(run_path)],
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
self.assertEqual(run_csv.returncode, 0, run_csv.stderr)
|
|
with run_path.with_suffix(".csv").open(encoding="utf-8-sig", newline="") as handle:
|
|
report_rows = list(csv.DictReader(handle))
|
|
self.assertEqual(report_rows[0]["test_type"], "integration")
|
|
self.assertEqual(report_rows[0]["suite"], "api-contract")
|
|
|
|
def test_isolated_full_keeps_risk_and_ignores_authorize_filter(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
write_mixed_catalog(root)
|
|
catalog_path = root / ".pouch/tester/catalog.yaml"
|
|
catalog = yaml.safe_load(catalog_path.read_text(encoding="utf-8"))
|
|
catalog["authorize"] = {"mutating": "AUTHORIZE_MUTATING"}
|
|
catalog["execution_profile"] = "isolated-full"
|
|
catalog_path.write_text(yaml.safe_dump(catalog, sort_keys=False), encoding="utf-8")
|
|
case_path = root / ".pouch/tester/cases/mixed.yaml"
|
|
cases = yaml.safe_load(case_path.read_text(encoding="utf-8"))
|
|
journey = cases["cases"][0]
|
|
journey.update({"risk": "mutating", "tag": "full", "client_suite": "client-regression"})
|
|
case_path.write_text(yaml.safe_dump(cases, sort_keys=False), encoding="utf-8")
|
|
|
|
selected = self.run_catalog(root, "select", "--suite", "client-regression")
|
|
self.assertEqual(selected.returncode, 0, selected.stderr)
|
|
payload = json.loads(selected.stdout)
|
|
self.assertEqual(payload["execution_profile"], "isolated-full")
|
|
self.assertEqual(payload["risk_filter"], "disabled")
|
|
self.assertEqual(payload["count"], 1)
|
|
self.assertEqual(payload["cases"][0]["risk"], "mutating")
|
|
self.assertEqual(payload["skipped"], [])
|
|
|
|
all_active = self.run_catalog(root, "select")
|
|
self.assertEqual(all_active.returncode, 0, all_active.stderr)
|
|
all_payload = json.loads(all_active.stdout)
|
|
self.assertEqual(all_payload["count"], 2)
|
|
self.assertEqual(all_payload["test_types"], {"journey": 1, "integration": 1})
|
|
|
|
def test_isolated_full_run_script_does_not_use_authorize_switch(self):
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
tester = root / ".pouch" / "tester"
|
|
(tester / "cases").mkdir(parents=True)
|
|
(tester / "local.yaml").write_text("HOST: http://127.0.0.1\nAUTHORIZE_MUTATING: false\n", encoding="utf-8")
|
|
(tester / "catalog.yaml").write_text(
|
|
yaml.safe_dump(
|
|
{
|
|
"version": 2,
|
|
"id_pattern": "^TC-[A-Z]+-[0-9]+$",
|
|
"categories": [{"id": "SCRIPT"}],
|
|
"risks": ["readonly", "mutating", "destructive"],
|
|
"default_max_risk": "readonly",
|
|
"execution_profile": "isolated-full",
|
|
"authorize": {"mutating": "AUTHORIZE_MUTATING"},
|
|
},
|
|
sort_keys=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
(tester / "cases" / "script.yaml").write_text(
|
|
yaml.safe_dump(
|
|
{
|
|
"category": "SCRIPT",
|
|
"cases": [{
|
|
"id": "TC-SCRIPT-001",
|
|
"title": "mutating script",
|
|
"what": "run isolated script",
|
|
"lifecycle": "active",
|
|
"tag": "full",
|
|
"risk": "mutating",
|
|
"driver": "script",
|
|
"how": {"run": "exit 0"},
|
|
"cleanup": {"steps": ["reset"]},
|
|
"expected": [{"kind": "script-exit", "value": 0}],
|
|
}],
|
|
},
|
|
sort_keys=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
result = subprocess.run(
|
|
[sys.executable, str(CATALOG), "run-script", "--root", str(root), "--id", "TC-SCRIPT-001"],
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(json.loads(result.stdout)["result"], "executed")
|
|
|
|
|
|
class IntegrationFirstTests(unittest.TestCase):
|
|
def test_preflight_block_does_not_create_session(self):
|
|
calls: list[str] = []
|
|
result = run_integration_first(
|
|
preflight_status="blocked",
|
|
integration_cases=["INT-1"],
|
|
journey_cases=["J-1"],
|
|
execute_integration=lambda case: calls.append("integration") or "pass",
|
|
create_journey_session=lambda: calls.append("session"),
|
|
execute_journey=lambda session, case: "pass",
|
|
)
|
|
self.assertEqual(result["results"][0]["result"], "not-run")
|
|
self.assertFalse(result["session_created"])
|
|
self.assertEqual(calls, [])
|
|
|
|
def test_integration_failure_propagates_journey_not_run(self):
|
|
calls: list[str] = []
|
|
result = run_integration_first(
|
|
preflight_status="ready",
|
|
integration_cases=["INT-1", "INT-2"],
|
|
journey_cases=["J-1", "J-2"],
|
|
execute_integration=lambda case: calls.append(case) or ("fail" if case == "INT-1" else "pass"),
|
|
create_journey_session=lambda: calls.append("session"),
|
|
execute_journey=lambda session, case: "pass",
|
|
)
|
|
self.assertEqual(result["status"], "fail")
|
|
self.assertEqual(calls, ["INT-1", "INT-2"])
|
|
self.assertFalse(result["session_created"])
|
|
self.assertEqual([row["result"] for row in result["results"]], ["fail", "pass", "not-run", "not-run"])
|
|
self.assertEqual(result["results"][2]["blocked_by"], ["INT-1"])
|
|
|
|
def test_independent_integration_continues_and_dependency_is_blocked(self):
|
|
result = run_integration_first(
|
|
preflight_status="ready",
|
|
integration_cases=[
|
|
{"id": "INT-1"},
|
|
{"id": "INT-2"},
|
|
{"id": "INT-3", "depends_on": ["INT-1"]},
|
|
],
|
|
journey_cases=[],
|
|
execute_integration=lambda case: "fail" if case["id"] == "INT-1" else "pass",
|
|
create_journey_session=lambda: object(),
|
|
execute_journey=lambda session, case: "pass",
|
|
)
|
|
self.assertEqual([row["result"] for row in result["results"]], ["fail", "pass", "not-run"])
|
|
self.assertEqual(result["results"][2]["blocked_by"], ["INT-1"])
|
|
|
|
def test_preflight_block_creates_every_result_row(self):
|
|
result = run_integration_first(
|
|
preflight_status="blocked",
|
|
integration_cases=[{"id": "INT-1"}],
|
|
journey_cases=[{"id": "J-1"}],
|
|
execute_integration=lambda case: "pass",
|
|
create_journey_session=lambda: object(),
|
|
execute_journey=lambda session, case: "pass",
|
|
)
|
|
self.assertEqual([row["result"] for row in result["results"]], ["not-run", "not-run"])
|
|
self.assertEqual(result["results"][0]["blocked_by"], ["test-preflight"])
|
|
|
|
def test_service_unreachable_is_environment_failure(self):
|
|
self.assertEqual(classify_integration(ok=False, service_reachable=False), "environment-fail")
|
|
self.assertEqual(classify_integration(ok=False, service_reachable=True), "fail")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|