116 lines
5.1 KiB
Python
116 lines
5.1 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import importlib.util
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
CASES = REPO_ROOT / "skills" / "tester" / "scripts" / "generate_cases_csv.py"
|
|
RUNS = REPO_ROOT / "skills" / "tester" / "scripts" / "generate_run_csv.py"
|
|
sys.path.insert(0, str(RUNS.parent))
|
|
from failure_feedback import attach_feedback
|
|
|
|
|
|
def load_module(name: str, path: Path):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
class TesterRunRecordTests(unittest.TestCase):
|
|
def test_cases_csv_contains_definitions_only(self) -> None:
|
|
cases = load_module("tester_cases_csv", CASES)
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
cases_dir = root / ".pouch" / "tester" / "cases"
|
|
cases_dir.mkdir(parents=True)
|
|
(root / ".pouch" / "tester" / "runs").mkdir()
|
|
(root / ".pouch" / "tester" / "catalog.yaml").write_text(
|
|
"categories:\n - id: smoke\n title: Smoke\n",
|
|
encoding="utf-8",
|
|
)
|
|
(cases_dir / "smoke.yaml").write_text(
|
|
"category: smoke\ncases:\n - id: TC-1\n title: One\n what: one\n lifecycle: active\n tag: smoke\n risk: readonly\n how: {steps: [open]}\n expected: [{kind: visible-text, value: ready}]\n",
|
|
encoding="utf-8",
|
|
)
|
|
output = cases.export_cases_csv(root)
|
|
with output.open(encoding="utf-8-sig", newline="") as handle:
|
|
rows = list(csv.DictReader(handle))
|
|
self.assertEqual(rows[0]["id"], "TC-1")
|
|
self.assertNotIn("last_result", rows[0])
|
|
self.assertNotIn("last_executed", rows[0])
|
|
|
|
def test_run_id_has_random_collision_suffix(self) -> None:
|
|
runs = load_module("tester_run_csv", RUNS)
|
|
first = runs.new_run_id("smoke")
|
|
second = runs.new_run_id("smoke")
|
|
self.assertNotEqual(first, second)
|
|
self.assertRegex(first, r"^\d{8}T\d{6}Z-smoke-[0-9a-f]{6}$")
|
|
|
|
def test_run_csv_preserves_all_results_and_structured_issue_fields(self) -> None:
|
|
runs = load_module("tester_run_csv_issues", RUNS)
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
root = Path(temp)
|
|
cases_dir = root / ".pouch" / "tester" / "cases"
|
|
cases_dir.mkdir(parents=True)
|
|
(root / ".pouch" / "tester" / "runs").mkdir()
|
|
(root / ".pouch" / "tester" / "catalog.yaml").write_text(
|
|
"categories:\n - id: smoke\n title: Smoke\n", encoding="utf-8"
|
|
)
|
|
(cases_dir / "smoke.yaml").write_text(
|
|
"category: smoke\ncases:\n"
|
|
" - id: TC-1\n title: One\n what: one\n lifecycle: active\n"
|
|
" tag: smoke\n risk: readonly\n how: {steps: [open]}\n"
|
|
" expected: [{kind: visible-text, value: ready}]\n"
|
|
" - id: TC-2\n title: Two\n what: two\n lifecycle: active\n"
|
|
" tag: smoke\n risk: readonly\n how: {steps: [open]}\n"
|
|
" expected: [{kind: visible-text, value: ready}]\n",
|
|
encoding="utf-8",
|
|
)
|
|
run = attach_feedback(
|
|
{
|
|
"runId": "issues",
|
|
"results": [
|
|
{
|
|
"id": "TC-1",
|
|
"result": "fail",
|
|
"issue": {
|
|
"expected": "ready",
|
|
"observed": "blank",
|
|
"artifact_hash": "sha256:abc",
|
|
"environment": "Android tester",
|
|
"evidence_path": "evidence/1.png",
|
|
"reproduction": "open",
|
|
"next_step": "targeted rerun",
|
|
},
|
|
},
|
|
{"id": "TC-2", "result": "not-run", "blocked_by": ["TC-1"]},
|
|
],
|
|
},
|
|
trigger="pr",
|
|
target="PR-1",
|
|
related_issue="ISSUE-1",
|
|
)
|
|
run_path = root / ".pouch" / "tester" / "runs" / "issues.yaml"
|
|
import yaml
|
|
run_path.write_text(yaml.safe_dump(run, sort_keys=False), encoding="utf-8")
|
|
out = runs.export_run_csv(root, run_path=run_path)
|
|
with out.open(encoding="utf-8-sig", newline="") as handle:
|
|
rows = list(csv.DictReader(handle))
|
|
self.assertEqual([row["id"] for row in rows], ["TC-1", "TC-2"])
|
|
self.assertEqual(rows[0]["issue_type"], "product")
|
|
self.assertEqual(rows[0]["expected"], "ready")
|
|
self.assertEqual(rows[0]["artifact_hash"], "sha256:abc")
|
|
self.assertEqual(rows[1]["blocked_by"], "TC-1")
|
|
self.assertEqual(run["feedback"]["route"], "pr-update-issue-dispatcher-reassign")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|