test(tester): validate CSV regression summary totals

2026-10-15 11:08:33 +08:00
parent 9ba9b3da0f
commit baed23d02a
4 changed files with 337 additions and 1 deletions
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+import csv
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+import summarize_run
+
+
+class RunSummaryTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp.name)
+        self.csv_path = self.root / "run.csv"
+        self.yaml_path = self.root / "run.yaml"
+        self.rows = [
+            {"id": "TC-001", "result": "pass"},
+            {"id": "TC-002", "result": "fail"},
+            {"id": "TC-003", "result": "environment-fail"},
+            {"id": "TC-004", "result": "skipped"},
+            {"id": "TC-005", "result": "not-run"},
+        ]
+        self.write_csv(self.rows)
+        self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows])
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def write_csv(self, rows):
+        with self.csv_path.open("w", encoding="utf-8", newline="") as handle:
+            writer = csv.DictWriter(handle, fieldnames=["id", "result"])
+            writer.writeheader()
+            writer.writerows(rows)
+
+    def write_yaml(self, results):
+        import yaml
+
+        self.yaml_path.write_text(
+            yaml.safe_dump(
+                {"counts": {"pass": 999}, "results": results},
+                sort_keys=False,
+            ),
+            encoding="utf-8",
+        )
+
+    def run_cli(self):
+        return subprocess.run(
+            [
+                sys.executable,
+                str(Path(summarize_run.__file__).resolve()),
+                "--csv",
+                str(self.csv_path),
+                "--yaml",
+                str(self.yaml_path),
+            ],
+            text=True,
+            capture_output=True,
+        )
+
+    def test_valid_summary_uses_csv_results(self):
+        result = self.run_cli()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("source=CSV.result", result.stdout)
+        self.assertIn("pass=1 fail=1 environment-fail=1 skipped=1 not-run=1 total=5", result.stdout)
+
+    def test_yaml_counts_do_not_drive_status_summary(self):
+        self.write_csv([{"id": "TC-001", "result": "fail"}])
+        self.write_yaml([{"id": "TC-001", "result": "fail"}])
+        result = self.run_cli()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("pass=0 fail=1", result.stdout)
+        self.assertNotIn("pass=999", result.stdout)
+
+    def test_yaml_result_count_mismatch_is_nonzero(self):
+        self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows[:-1]])
+        result = self.run_cli()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("YAML results 数量不一致", result.stderr)
+
+    def test_id_set_mismatch_is_nonzero(self):
+        self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows[:-1]] + [{"id": "TC-999"}])
+        result = self.run_cli()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("testcase id 集合不一致", result.stderr)
+
+    def test_duplicate_csv_id_is_nonzero(self):
+        rows = self.rows.copy()
+        rows[-1] = {"id": "TC-001", "result": "not-run"}
+        self.write_csv(rows)
+        result = self.run_cli()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("CSV testcase id 重复", result.stderr)
+
+    def test_missing_csv_id_is_nonzero(self):
+        rows = self.rows.copy()
+        rows[0] = {"id": "", "result": "pass"}
+        self.write_csv(rows)
+        result = self.run_cli()
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn("CSV 第 2 行 testcase id 缺失", result.stderr)
+
+    def test_duplicate_or_missing_yaml_id_is_nonzero(self):
+        cases = [
+            ([{"id": "TC-001"}, {"id": "TC-001"}] + [{"id": row["id"]} for row in self.rows[2:]], "YAML testcase id 重复"),
+            ([{"id": ""}] + [{"id": row["id"]} for row in self.rows[1:]], "YAML results[1] testcase id 缺失"),
+        ]
+        for results, expected in cases:
+            with self.subTest(expected=expected):
+                self.write_yaml(results)
+                result = self.run_cli()
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn(expected, result.stderr)
+
+    def test_unknown_or_missing_csv_result_is_nonzero(self):
+        for bad_result, expected in [("unexpected", "result 未知"), ("", "result 缺失")]:
+            with self.subTest(bad_result=bad_result):
+                rows = self.rows.copy()
+                rows[0] = {"id": "TC-001", "result": bad_result}
+                self.write_csv(rows)
+                result = self.run_cli()
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn(expected, result.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()