feat(tester): add integration-first journey execution
This commit is contained in:
@@ -0,0 +1,190 @@
|
||||
import csv
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
SCRIPTS = Path(__file__).resolve().parents[1] / "scripts"
|
||||
CATALOG = SCRIPTS / "catalog.py"
|
||||
CASES_CSV = SCRIPTS / "generate_cases_csv.py"
|
||||
RUN_CSV = SCRIPTS / "generate_run_csv.py"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
from two_tier import classify_integration, run_integration_first
|
||||
|
||||
|
||||
def write_mixed_catalog(root: Path) -> None:
|
||||
tester = root / ".pouch" / "tester"
|
||||
(tester / "cases").mkdir(parents=True)
|
||||
(tester / "runs").mkdir()
|
||||
(tester / "catalog.yaml").write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"version": 2,
|
||||
"id_pattern": "^TC-[A-Z]+-[0-9]+$",
|
||||
"categories": [{"id": "MIXED"}],
|
||||
"risks": ["readonly", "mutating", "destructive"],
|
||||
"default_max_risk": "readonly",
|
||||
"client_suites": {
|
||||
"client-smoke": {"default": True},
|
||||
"client-regression": {"default": False},
|
||||
"safety": {"default": False},
|
||||
},
|
||||
"sessions": {"browser": {"kind": "browser", "reuse": "run"}},
|
||||
},
|
||||
sort_keys=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
journey = {
|
||||
"id": "TC-JOURNEY-001",
|
||||
"test_type": "journey",
|
||||
"category": "MIXED",
|
||||
"title": "journey",
|
||||
"what": "client path",
|
||||
"lifecycle": "active",
|
||||
"tag": "smoke",
|
||||
"risk": "readonly",
|
||||
"driver": "browser",
|
||||
"automation": "automated",
|
||||
"session": "browser",
|
||||
"client_suite": "client-smoke",
|
||||
"platform": "android",
|
||||
"journey": {
|
||||
"goal": "goal",
|
||||
"entry_state": "logged_out",
|
||||
"actions": ["login"],
|
||||
"assertions": ["visible"],
|
||||
"final_state": "done",
|
||||
"failure_policy": "stop",
|
||||
},
|
||||
"fixture": {"account": "test-role"},
|
||||
"cleanup": {"required": False, "reason": "read-only"},
|
||||
"how": {"steps": ["open"]},
|
||||
"expected": [{"kind": "visible-text", "value": "done"}],
|
||||
}
|
||||
integration = {
|
||||
"id": "TC-INTEGRATION-001",
|
||||
"test_type": "integration",
|
||||
"category": "MIXED",
|
||||
"title": "contract",
|
||||
"what": "API contract",
|
||||
"lifecycle": "active",
|
||||
"tag": "slow",
|
||||
"risk": "readonly",
|
||||
"driver": "script",
|
||||
"automation": "automated",
|
||||
"integration_suite": "api-contract",
|
||||
"fixture": {"references": ["isolated-account"]},
|
||||
"request": {"method": "GET", "path": "/health"},
|
||||
"how": {"run": "true"},
|
||||
"cleanup": {"required": False, "reason": "read-only"},
|
||||
"expected": [{"kind": "api-status", "value": 200}],
|
||||
}
|
||||
(tester / "cases" / "mixed.yaml").write_text(
|
||||
yaml.safe_dump({"category": "MIXED", "cases": [journey, integration]}, sort_keys=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
class TwoTierCatalogTests(unittest.TestCase):
|
||||
def run_catalog(self, root: Path, *args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[sys.executable, str(CATALOG), *args, "--root", str(root)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
def test_default_selection_excludes_integration_and_explicit_selection_includes_it(self):
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
root = Path(temp)
|
||||
write_mixed_catalog(root)
|
||||
valid = self.run_catalog(root, "validate")
|
||||
self.assertEqual(valid.returncode, 0, valid.stderr)
|
||||
|
||||
default = self.run_catalog(root, "select")
|
||||
self.assertEqual(default.returncode, 0, default.stderr)
|
||||
default_payload = json.loads(default.stdout)
|
||||
self.assertEqual(default_payload["count"], 1)
|
||||
self.assertEqual(default_payload["test_types"], {"journey": 1, "integration": 0})
|
||||
|
||||
integration = self.run_catalog(root, "select", "--test-type", "integration")
|
||||
self.assertEqual(integration.returncode, 0, integration.stderr)
|
||||
integration_payload = json.loads(integration.stdout)
|
||||
self.assertEqual(integration_payload["count"], 1)
|
||||
self.assertEqual(integration_payload["cases"][0]["test_type"], "integration")
|
||||
|
||||
csv_result = subprocess.run(
|
||||
[sys.executable, str(CASES_CSV), "--root", str(root)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
self.assertEqual(csv_result.returncode, 0, csv_result.stderr)
|
||||
with (root / ".pouch/tester/cases.csv").open(encoding="utf-8-sig", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
self.assertEqual({row["test_type"] for row in rows}, {"journey", "integration"})
|
||||
self.assertEqual({row["suite"] for row in rows}, {"client-smoke", "api-contract"})
|
||||
|
||||
run_path = root / ".pouch/tester/runs/two-tier.yaml"
|
||||
run_path.write_text(
|
||||
yaml.safe_dump(
|
||||
{"runId": "two-tier", "results": [
|
||||
{"id": "TC-INTEGRATION-001", "result": "pass"},
|
||||
{"id": "TC-JOURNEY-001", "result": "not-run"},
|
||||
]},
|
||||
sort_keys=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
run_csv = subprocess.run(
|
||||
[sys.executable, str(RUN_CSV), "--root", str(root), "--run", str(run_path)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
self.assertEqual(run_csv.returncode, 0, run_csv.stderr)
|
||||
with run_path.with_suffix(".csv").open(encoding="utf-8-sig", newline="") as handle:
|
||||
report_rows = list(csv.DictReader(handle))
|
||||
self.assertEqual(report_rows[0]["test_type"], "integration")
|
||||
self.assertEqual(report_rows[0]["suite"], "api-contract")
|
||||
|
||||
|
||||
class IntegrationFirstTests(unittest.TestCase):
|
||||
def test_preflight_block_does_not_create_session(self):
|
||||
calls: list[str] = []
|
||||
result = run_integration_first(
|
||||
preflight_status="blocked",
|
||||
integration_cases=["INT-1"],
|
||||
journey_cases=["J-1"],
|
||||
execute_integration=lambda case: calls.append("integration") or "pass",
|
||||
create_journey_session=lambda: calls.append("session"),
|
||||
execute_journey=lambda session, case: "pass",
|
||||
)
|
||||
self.assertEqual(result["results"][0]["result"], "not-run")
|
||||
self.assertFalse(result["session_created"])
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_integration_failure_propagates_journey_not_run(self):
|
||||
calls: list[str] = []
|
||||
result = run_integration_first(
|
||||
preflight_status="ready",
|
||||
integration_cases=["INT-1", "INT-2"],
|
||||
journey_cases=["J-1", "J-2"],
|
||||
execute_integration=lambda case: calls.append(case) or ("fail" if case == "INT-1" else "pass"),
|
||||
create_journey_session=lambda: calls.append("session"),
|
||||
execute_journey=lambda session, case: "pass",
|
||||
)
|
||||
self.assertEqual(result["status"], "fail")
|
||||
self.assertEqual(calls, ["INT-1"])
|
||||
self.assertFalse(result["session_created"])
|
||||
self.assertEqual([row["result"] for row in result["results"]], ["fail", "not-run", "not-run", "not-run"])
|
||||
|
||||
def test_service_unreachable_is_environment_failure(self):
|
||||
self.assertEqual(classify_integration(ok=False, service_reachable=False), "environment-fail")
|
||||
self.assertEqual(classify_integration(ok=False, service_reachable=True), "fail")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user