diff --git a/skills/tester/README.md b/skills/tester/README.md index 6b7fc08..8f67a03 100644 --- a/skills/tester/README.md +++ b/skills/tester/README.md @@ -85,3 +85,19 @@ APK/IPA,为本轮创建隔离 worktree,在 Android 和 iOS tester 节点安 - **列出**:能看到分类表和总条数;`.pouch/tester/cases.csv` 已由 skill 里的 `generate_cases_csv.py` 从 `cases/` 重新生成。每次运行的结果保存在独立的 `.pouch/tester/runs/.*` 文件中。 - **维护**:目标 id 能 `查看` 到,且检查通过。 - **运行**:有本次 `runs/*.yaml` 快照和同名 `.csv` 成绩单(`executed` 列区分已跑 / 未跑);没填 HOST 时会停,而不是编造失败。 + +### 汇总已有运行结果 + +对已经生成的同一运行 CSV/YAML,使用显式路径汇总: + +```bash +python3 /scripts/summarize_run.py \ + --csv .pouch/tester/runs/.csv \ + --yaml .pouch/tester/runs/.yaml +``` + +脚本只从 CSV 的 `result` 列计算 `pass`、`fail`、`environment-fail`、`skipped`、 +`not-run` 五类状态,并要求五类计数和等于 CSV 数据行数。YAML 只用于校验 +`results` 数量和 testcase id 集合;数量、集合、重复/缺失 id 或 CSV 缺失/未知状态 +时返回非零并输出诊断。校验成功才表示这份汇总可作为运行报告;非零表示报告不一致, +不能据此判定产品失败。 diff --git a/skills/tester/references/run.md b/skills/tester/references/run.md index 92c5716..f74427f 100644 --- a/skills/tester/references/run.md +++ b/skills/tester/references/run.md @@ -227,7 +227,23 @@ python3 /scripts/catalog.py run-script --root --id python3 /scripts/generate_run_csv.py --root --run ``` -这会写出与快照同名的 `.pouch/tester/runs/.csv`(`executed=yes|no`),不会修改 `cases.csv`。对话里两个路径都报。 +这会写出与快照同名的 `.pouch/tester/runs/.csv`(`executed=yes|no`),不会修改 +`cases.csv`。对话里两个路径都报。 + +需要复核已有成绩单时,显式同时传入同一次运行的 CSV 和 YAML: + +```bash +python3 /scripts/summarize_run.py \ + --csv .pouch/tester/runs/.csv \ + --yaml .pouch/tester/runs/.yaml +``` + +汇总状态只读取 CSV `result` 列,必须输出五类 `pass / fail / environment-fail / +skipped / not-run`,且计数和等于 CSV 数据行数。汇总前会检查 CSV 数据行数、CSV 唯一 +testcase id 数、YAML `results` 数量和 CSV/YAML testcase id 集合;重复或缺失 id、缺失或 +未知 CSV 状态、任一数量或集合不一致都会非零退出并给出诊断。YAML 的 `counts` 或其它 +历史汇总字段不会参与状态计数。退出码 0 表示汇总校验成功,非零表示报告不一致或输入 +无效,不等同于产品测试失败。 对话报告: diff --git a/skills/tester/scripts/summarize_run.py b/skills/tester/scripts/summarize_run.py new file mode 100644 index 0000000..9ce1791 --- /dev/null +++ b/skills/tester/scripts/summarize_run.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Validate a tester CSV/YAML pair and summarize statuses from CSV only.""" + +from __future__ import annotations + +import argparse +import csv +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + print("需要 PyYAML: python3 -m pip install pyyaml", file=sys.stderr) + raise SystemExit(2) + + +RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run") +RESULT_SET = frozenset(RESULTS) + + +class SummaryError(ValueError): + """Raised when the CSV/YAML pair cannot be safely summarized.""" + + +def _format_ids(ids: set[str]) -> str: + return ", ".join(sorted(ids)) or "(none)" + + +def _duplicate_ids(ids: list[str]) -> list[str]: + counts = Counter(ids) + return sorted(case_id for case_id, count in counts.items() if count > 1) + + +def load_csv(path: Path) -> list[dict[str, str]]: + """Load CSV rows and reject malformed IDs/results with diagnostics.""" + try: + handle = path.open(encoding="utf-8-sig", newline="") + except OSError as exc: + raise SummaryError(f"无法读取 CSV {path}: {exc}") from exc + + with handle: + reader = csv.DictReader(handle) + if not reader.fieldnames: + raise SummaryError(f"CSV {path} 缺少表头") + missing_fields = [field for field in ("id", "result") if field not in reader.fieldnames] + if missing_fields: + raise SummaryError(f"CSV {path} 缺少必需列: {', '.join(missing_fields)}") + rows = list(reader) + + ids: list[str] = [] + errors: list[str] = [] + for line_number, row in enumerate(rows, start=2): + case_id = (row.get("id") or "").strip() + result = (row.get("result") or "").strip() + ids.append(case_id) + if not case_id: + errors.append(f"CSV 第 {line_number} 行 testcase id 缺失") + if not result: + errors.append(f"CSV 第 {line_number} 行 result 缺失 (testcase id={case_id or '(missing)'})") + elif result not in RESULT_SET: + errors.append( + f"CSV 第 {line_number} 行 result 未知: {result!r} " + f"(testcase id={case_id or '(missing)'}; 允许值={', '.join(RESULTS)})" + ) + + duplicates = _duplicate_ids([case_id for case_id in ids if case_id]) + if duplicates: + errors.append(f"CSV testcase id 重复: {_format_ids(set(duplicates))}") + if errors: + raise SummaryError("CSV 校验失败:\n- " + "\n- ".join(errors)) + return rows + + +def load_yaml_results(path: Path) -> list[dict[str, Any]]: + """Load only YAML results structure needed for count/ID validation.""" + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except OSError as exc: + raise SummaryError(f"无法读取 YAML {path}: {exc}") from exc + except yaml.YAMLError as exc: + raise SummaryError(f"YAML 解析失败 {path}: {exc}") from exc + + if not isinstance(data, dict) or not isinstance(data.get("results"), list): + raise SummaryError(f"YAML {path} 缺少 results 列表") + + results = data["results"] + ids: list[str] = [] + errors: list[str] = [] + for index, item in enumerate(results, start=1): + if not isinstance(item, dict): + errors.append(f"YAML results[{index}] 不是 mapping") + continue + case_id = str(item.get("id") or "").strip() + ids.append(case_id) + if not case_id: + errors.append(f"YAML results[{index}] testcase id 缺失") + + duplicates = _duplicate_ids([case_id for case_id in ids if case_id]) + if duplicates: + errors.append(f"YAML testcase id 重复: {_format_ids(set(duplicates))}") + if errors: + raise SummaryError("YAML 校验失败:\n- " + "\n- ".join(errors)) + return results + + +def summarize(csv_path: Path, yaml_path: Path) -> tuple[Counter[str], dict[str, int]]: + """Validate the pair and return counts calculated from CSV result values.""" + rows = load_csv(csv_path) + yaml_results = load_yaml_results(yaml_path) + csv_ids = [(row.get("id") or "").strip() for row in rows] + yaml_ids = [str(item.get("id") or "").strip() for item in yaml_results] + csv_id_set = set(csv_ids) + yaml_id_set = set(yaml_ids) + diagnostics: list[str] = [] + + if len(rows) != len(csv_id_set): + diagnostics.append( + f"CSV 数据行总数与唯一 testcase id 数不一致: rows={len(rows)} unique_ids={len(csv_id_set)}" + ) + if len(rows) != len(yaml_results): + diagnostics.append( + f"CSV 数据行总数与 YAML results 数量不一致: csv_rows={len(rows)} yaml_results={len(yaml_results)}" + ) + if csv_id_set != yaml_id_set: + diagnostics.append( + "CSV/YAML testcase id 集合不一致: " + f"仅 CSV 有 [{_format_ids(csv_id_set - yaml_id_set)}]; " + f"仅 YAML 有 [{_format_ids(yaml_id_set - csv_id_set)}]" + ) + if diagnostics: + raise SummaryError("CSV/YAML 汇总校验失败:\n- " + "\n- ".join(diagnostics)) + + counts: Counter[str] = Counter((row.get("result") or "").strip() for row in rows) + total = sum(counts[name] for name in RESULTS) + if total != len(rows): + raise SummaryError( + "CSV 状态计数和不等于数据行总数: " + f"count_sum={total} csv_rows={len(rows)}" + ) + metadata = { + "csv_rows": len(rows), + "csv_unique_ids": len(csv_id_set), + "yaml_results": len(yaml_results), + "count_sum": total, + } + return counts, metadata + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="校验 tester CSV/YAML 并从 CSV result 汇总状态") + parser.add_argument("--csv", required=True, type=Path, help="运行成绩单 CSV") + parser.add_argument("--yaml", required=True, type=Path, help="对应运行快照 YAML") + args = parser.parse_args(argv) + try: + counts, metadata = summarize(args.csv, args.yaml) + except SummaryError as exc: + print(f"汇总失败: {exc}", file=sys.stderr) + return 1 + + bits = " ".join(f"{name}={counts[name]}" for name in RESULTS) + print(f"summary source=CSV.result {bits} total={metadata['count_sum']}") + print( + "validated " + f"csv_rows={metadata['csv_rows']} " + f"csv_unique_ids={metadata['csv_unique_ids']} " + f"yaml_results={metadata['yaml_results']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/tester/tests/test_run_summary.py b/skills/tester/tests/test_run_summary.py new file mode 100644 index 0000000..c2ed4aa --- /dev/null +++ b/skills/tester/tests/test_run_summary.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +import csv +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +import summarize_run + + +class RunSummaryTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.csv_path = self.root / "run.csv" + self.yaml_path = self.root / "run.yaml" + self.rows = [ + {"id": "TC-001", "result": "pass"}, + {"id": "TC-002", "result": "fail"}, + {"id": "TC-003", "result": "environment-fail"}, + {"id": "TC-004", "result": "skipped"}, + {"id": "TC-005", "result": "not-run"}, + ] + self.write_csv(self.rows) + self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows]) + + def tearDown(self): + self.temp.cleanup() + + def write_csv(self, rows): + with self.csv_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["id", "result"]) + writer.writeheader() + writer.writerows(rows) + + def write_yaml(self, results): + import yaml + + self.yaml_path.write_text( + yaml.safe_dump( + {"counts": {"pass": 999}, "results": results}, + sort_keys=False, + ), + encoding="utf-8", + ) + + def run_cli(self): + return subprocess.run( + [ + sys.executable, + str(Path(summarize_run.__file__).resolve()), + "--csv", + str(self.csv_path), + "--yaml", + str(self.yaml_path), + ], + text=True, + capture_output=True, + ) + + def test_valid_summary_uses_csv_results(self): + result = self.run_cli() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("source=CSV.result", result.stdout) + self.assertIn("pass=1 fail=1 environment-fail=1 skipped=1 not-run=1 total=5", result.stdout) + + def test_yaml_counts_do_not_drive_status_summary(self): + self.write_csv([{"id": "TC-001", "result": "fail"}]) + self.write_yaml([{"id": "TC-001", "result": "fail"}]) + result = self.run_cli() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("pass=0 fail=1", result.stdout) + self.assertNotIn("pass=999", result.stdout) + + def test_yaml_result_count_mismatch_is_nonzero(self): + self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows[:-1]]) + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn("YAML results 数量不一致", result.stderr) + + def test_id_set_mismatch_is_nonzero(self): + self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows[:-1]] + [{"id": "TC-999"}]) + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn("testcase id 集合不一致", result.stderr) + + def test_duplicate_csv_id_is_nonzero(self): + rows = self.rows.copy() + rows[-1] = {"id": "TC-001", "result": "not-run"} + self.write_csv(rows) + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn("CSV testcase id 重复", result.stderr) + + def test_missing_csv_id_is_nonzero(self): + rows = self.rows.copy() + rows[0] = {"id": "", "result": "pass"} + self.write_csv(rows) + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn("CSV 第 2 行 testcase id 缺失", result.stderr) + + def test_duplicate_or_missing_yaml_id_is_nonzero(self): + cases = [ + ([{"id": "TC-001"}, {"id": "TC-001"}] + [{"id": row["id"]} for row in self.rows[2:]], "YAML testcase id 重复"), + ([{"id": ""}] + [{"id": row["id"]} for row in self.rows[1:]], "YAML results[1] testcase id 缺失"), + ] + for results, expected in cases: + with self.subTest(expected=expected): + self.write_yaml(results) + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected, result.stderr) + + def test_unknown_or_missing_csv_result_is_nonzero(self): + for bad_result, expected in [("unexpected", "result 未知"), ("", "result 缺失")]: + with self.subTest(bad_result=bad_result): + rows = self.rows.copy() + rows[0] = {"id": "TC-001", "result": bad_result} + self.write_csv(rows) + result = self.run_cli() + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected, result.stderr) + + +if __name__ == "__main__": + unittest.main()