test(tester): validate CSV regression summary totals
This commit is contained in:
@@ -85,3 +85,19 @@ APK/IPA，为本轮创建隔离 worktree，在 Android 和 iOS tester 节点安
|
||||
- **列出**：能看到分类表和总条数；`.pouch/tester/cases.csv` 已由 skill 里的 `generate_cases_csv.py` 从 `cases/` 重新生成。每次运行的结果保存在独立的 `.pouch/tester/runs/<run-id>.*` 文件中。
|
||||
- **维护**：目标 id 能 `查看` 到，且检查通过。
|
||||
- **运行**：有本次 `runs/*.yaml` 快照和同名 `.csv` 成绩单（`executed` 列区分已跑 / 未跑）；没填 HOST 时会停，而不是编造失败。
|
||||
|
||||
### 汇总已有运行结果
|
||||
|
||||
对已经生成的同一运行 CSV/YAML，使用显式路径汇总：
|
||||
|
||||
```bash
|
||||
python3 <skill-dir>/scripts/summarize_run.py \
|
||||
--csv .pouch/tester/runs/<run-id>.csv \
|
||||
--yaml .pouch/tester/runs/<run-id>.yaml
|
||||
```
|
||||
|
||||
脚本只从 CSV 的 `result` 列计算 `pass`、`fail`、`environment-fail`、`skipped`、
|
||||
`not-run` 五类状态，并要求五类计数和等于 CSV 数据行数。YAML 只用于校验
|
||||
`results` 数量和 testcase id 集合；数量、集合、重复/缺失 id 或 CSV 缺失/未知状态
|
||||
时返回非零并输出诊断。校验成功才表示这份汇总可作为运行报告；非零表示报告不一致，
|
||||
不能据此判定产品失败。
|
||||
|
||||
@@ -227,7 +227,23 @@ python3 <skill-dir>/scripts/catalog.py run-script --root <root> --id <id>
|
||||
python3 <skill-dir>/scripts/generate_run_csv.py --root <root> --run <yaml>
|
||||
```
|
||||
|
||||
这会写出与快照同名的 `.pouch/tester/runs/<run-id>.csv`（`executed=yes|no`），不会修改 `cases.csv`。对话里两个路径都报。
|
||||
这会写出与快照同名的 `.pouch/tester/runs/<run-id>.csv`（`executed=yes|no`），不会修改
|
||||
`cases.csv`。对话里两个路径都报。
|
||||
|
||||
需要复核已有成绩单时，显式同时传入同一次运行的 CSV 和 YAML：
|
||||
|
||||
```bash
|
||||
python3 <skill-dir>/scripts/summarize_run.py \
|
||||
--csv .pouch/tester/runs/<run-id>.csv \
|
||||
--yaml .pouch/tester/runs/<run-id>.yaml
|
||||
```
|
||||
|
||||
汇总状态只读取 CSV `result` 列，必须输出五类 `pass / fail / environment-fail /
|
||||
skipped / not-run`，且计数和等于 CSV 数据行数。汇总前会检查 CSV 数据行数、CSV 唯一
|
||||
testcase id 数、YAML `results` 数量和 CSV/YAML testcase id 集合；重复或缺失 id、缺失或
|
||||
未知 CSV 状态、任一数量或集合不一致都会非零退出并给出诊断。YAML 的 `counts` 或其它
|
||||
历史汇总字段不会参与状态计数。退出码 0 表示汇总校验成功，非零表示报告不一致或输入
|
||||
无效，不等同于产品测试失败。
|
||||
|
||||
对话报告：
|
||||
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate a tester CSV/YAML pair and summarize statuses from CSV only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
print("需要 PyYAML: python3 -m pip install pyyaml", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run")
|
||||
RESULT_SET = frozenset(RESULTS)
|
||||
|
||||
|
||||
class SummaryError(ValueError):
|
||||
"""Raised when the CSV/YAML pair cannot be safely summarized."""
|
||||
|
||||
|
||||
def _format_ids(ids: set[str]) -> str:
|
||||
return ", ".join(sorted(ids)) or "(none)"
|
||||
|
||||
|
||||
def _duplicate_ids(ids: list[str]) -> list[str]:
|
||||
counts = Counter(ids)
|
||||
return sorted(case_id for case_id, count in counts.items() if count > 1)
|
||||
|
||||
|
||||
def load_csv(path: Path) -> list[dict[str, str]]:
|
||||
"""Load CSV rows and reject malformed IDs/results with diagnostics."""
|
||||
try:
|
||||
handle = path.open(encoding="utf-8-sig", newline="")
|
||||
except OSError as exc:
|
||||
raise SummaryError(f"无法读取 CSV {path}: {exc}") from exc
|
||||
|
||||
with handle:
|
||||
reader = csv.DictReader(handle)
|
||||
if not reader.fieldnames:
|
||||
raise SummaryError(f"CSV {path} 缺少表头")
|
||||
missing_fields = [field for field in ("id", "result") if field not in reader.fieldnames]
|
||||
if missing_fields:
|
||||
raise SummaryError(f"CSV {path} 缺少必需列: {', '.join(missing_fields)}")
|
||||
rows = list(reader)
|
||||
|
||||
ids: list[str] = []
|
||||
errors: list[str] = []
|
||||
for line_number, row in enumerate(rows, start=2):
|
||||
case_id = (row.get("id") or "").strip()
|
||||
result = (row.get("result") or "").strip()
|
||||
ids.append(case_id)
|
||||
if not case_id:
|
||||
errors.append(f"CSV 第 {line_number} 行 testcase id 缺失")
|
||||
if not result:
|
||||
errors.append(f"CSV 第 {line_number} 行 result 缺失 (testcase id={case_id or '(missing)'})")
|
||||
elif result not in RESULT_SET:
|
||||
errors.append(
|
||||
f"CSV 第 {line_number} 行 result 未知: {result!r} "
|
||||
f"(testcase id={case_id or '(missing)'}; 允许值={', '.join(RESULTS)})"
|
||||
)
|
||||
|
||||
duplicates = _duplicate_ids([case_id for case_id in ids if case_id])
|
||||
if duplicates:
|
||||
errors.append(f"CSV testcase id 重复: {_format_ids(set(duplicates))}")
|
||||
if errors:
|
||||
raise SummaryError("CSV 校验失败:\n- " + "\n- ".join(errors))
|
||||
return rows
|
||||
|
||||
|
||||
def load_yaml_results(path: Path) -> list[dict[str, Any]]:
|
||||
"""Load only YAML results structure needed for count/ID validation."""
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
raise SummaryError(f"无法读取 YAML {path}: {exc}") from exc
|
||||
except yaml.YAMLError as exc:
|
||||
raise SummaryError(f"YAML 解析失败 {path}: {exc}") from exc
|
||||
|
||||
if not isinstance(data, dict) or not isinstance(data.get("results"), list):
|
||||
raise SummaryError(f"YAML {path} 缺少 results 列表")
|
||||
|
||||
results = data["results"]
|
||||
ids: list[str] = []
|
||||
errors: list[str] = []
|
||||
for index, item in enumerate(results, start=1):
|
||||
if not isinstance(item, dict):
|
||||
errors.append(f"YAML results[{index}] 不是 mapping")
|
||||
continue
|
||||
case_id = str(item.get("id") or "").strip()
|
||||
ids.append(case_id)
|
||||
if not case_id:
|
||||
errors.append(f"YAML results[{index}] testcase id 缺失")
|
||||
|
||||
duplicates = _duplicate_ids([case_id for case_id in ids if case_id])
|
||||
if duplicates:
|
||||
errors.append(f"YAML testcase id 重复: {_format_ids(set(duplicates))}")
|
||||
if errors:
|
||||
raise SummaryError("YAML 校验失败:\n- " + "\n- ".join(errors))
|
||||
return results
|
||||
|
||||
|
||||
def summarize(csv_path: Path, yaml_path: Path) -> tuple[Counter[str], dict[str, int]]:
|
||||
"""Validate the pair and return counts calculated from CSV result values."""
|
||||
rows = load_csv(csv_path)
|
||||
yaml_results = load_yaml_results(yaml_path)
|
||||
csv_ids = [(row.get("id") or "").strip() for row in rows]
|
||||
yaml_ids = [str(item.get("id") or "").strip() for item in yaml_results]
|
||||
csv_id_set = set(csv_ids)
|
||||
yaml_id_set = set(yaml_ids)
|
||||
diagnostics: list[str] = []
|
||||
|
||||
if len(rows) != len(csv_id_set):
|
||||
diagnostics.append(
|
||||
f"CSV 数据行总数与唯一 testcase id 数不一致: rows={len(rows)} unique_ids={len(csv_id_set)}"
|
||||
)
|
||||
if len(rows) != len(yaml_results):
|
||||
diagnostics.append(
|
||||
f"CSV 数据行总数与 YAML results 数量不一致: csv_rows={len(rows)} yaml_results={len(yaml_results)}"
|
||||
)
|
||||
if csv_id_set != yaml_id_set:
|
||||
diagnostics.append(
|
||||
"CSV/YAML testcase id 集合不一致: "
|
||||
f"仅 CSV 有 [{_format_ids(csv_id_set - yaml_id_set)}]; "
|
||||
f"仅 YAML 有 [{_format_ids(yaml_id_set - csv_id_set)}]"
|
||||
)
|
||||
if diagnostics:
|
||||
raise SummaryError("CSV/YAML 汇总校验失败:\n- " + "\n- ".join(diagnostics))
|
||||
|
||||
counts: Counter[str] = Counter((row.get("result") or "").strip() for row in rows)
|
||||
total = sum(counts[name] for name in RESULTS)
|
||||
if total != len(rows):
|
||||
raise SummaryError(
|
||||
"CSV 状态计数和不等于数据行总数: "
|
||||
f"count_sum={total} csv_rows={len(rows)}"
|
||||
)
|
||||
metadata = {
|
||||
"csv_rows": len(rows),
|
||||
"csv_unique_ids": len(csv_id_set),
|
||||
"yaml_results": len(yaml_results),
|
||||
"count_sum": total,
|
||||
}
|
||||
return counts, metadata
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="校验 tester CSV/YAML 并从 CSV result 汇总状态")
|
||||
parser.add_argument("--csv", required=True, type=Path, help="运行成绩单 CSV")
|
||||
parser.add_argument("--yaml", required=True, type=Path, help="对应运行快照 YAML")
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
counts, metadata = summarize(args.csv, args.yaml)
|
||||
except SummaryError as exc:
|
||||
print(f"汇总失败: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
bits = " ".join(f"{name}={counts[name]}" for name in RESULTS)
|
||||
print(f"summary source=CSV.result {bits} total={metadata['count_sum']}")
|
||||
print(
|
||||
"validated "
|
||||
f"csv_rows={metadata['csv_rows']} "
|
||||
f"csv_unique_ids={metadata['csv_unique_ids']} "
|
||||
f"yaml_results={metadata['yaml_results']}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
import csv
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
|
||||
import summarize_run
|
||||
|
||||
|
||||
class RunSummaryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self.temp.name)
|
||||
self.csv_path = self.root / "run.csv"
|
||||
self.yaml_path = self.root / "run.yaml"
|
||||
self.rows = [
|
||||
{"id": "TC-001", "result": "pass"},
|
||||
{"id": "TC-002", "result": "fail"},
|
||||
{"id": "TC-003", "result": "environment-fail"},
|
||||
{"id": "TC-004", "result": "skipped"},
|
||||
{"id": "TC-005", "result": "not-run"},
|
||||
]
|
||||
self.write_csv(self.rows)
|
||||
self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows])
|
||||
|
||||
def tearDown(self):
|
||||
self.temp.cleanup()
|
||||
|
||||
def write_csv(self, rows):
|
||||
with self.csv_path.open("w", encoding="utf-8", newline="") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=["id", "result"])
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
def write_yaml(self, results):
|
||||
import yaml
|
||||
|
||||
self.yaml_path.write_text(
|
||||
yaml.safe_dump(
|
||||
{"counts": {"pass": 999}, "results": results},
|
||||
sort_keys=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
def run_cli(self):
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(Path(summarize_run.__file__).resolve()),
|
||||
"--csv",
|
||||
str(self.csv_path),
|
||||
"--yaml",
|
||||
str(self.yaml_path),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
def test_valid_summary_uses_csv_results(self):
|
||||
result = self.run_cli()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("source=CSV.result", result.stdout)
|
||||
self.assertIn("pass=1 fail=1 environment-fail=1 skipped=1 not-run=1 total=5", result.stdout)
|
||||
|
||||
def test_yaml_counts_do_not_drive_status_summary(self):
|
||||
self.write_csv([{"id": "TC-001", "result": "fail"}])
|
||||
self.write_yaml([{"id": "TC-001", "result": "fail"}])
|
||||
result = self.run_cli()
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("pass=0 fail=1", result.stdout)
|
||||
self.assertNotIn("pass=999", result.stdout)
|
||||
|
||||
def test_yaml_result_count_mismatch_is_nonzero(self):
|
||||
self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows[:-1]])
|
||||
result = self.run_cli()
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("YAML results 数量不一致", result.stderr)
|
||||
|
||||
def test_id_set_mismatch_is_nonzero(self):
|
||||
self.write_yaml([{"id": row["id"], "result": row["result"]} for row in self.rows[:-1]] + [{"id": "TC-999"}])
|
||||
result = self.run_cli()
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("testcase id 集合不一致", result.stderr)
|
||||
|
||||
def test_duplicate_csv_id_is_nonzero(self):
|
||||
rows = self.rows.copy()
|
||||
rows[-1] = {"id": "TC-001", "result": "not-run"}
|
||||
self.write_csv(rows)
|
||||
result = self.run_cli()
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("CSV testcase id 重复", result.stderr)
|
||||
|
||||
def test_missing_csv_id_is_nonzero(self):
|
||||
rows = self.rows.copy()
|
||||
rows[0] = {"id": "", "result": "pass"}
|
||||
self.write_csv(rows)
|
||||
result = self.run_cli()
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("CSV 第 2 行 testcase id 缺失", result.stderr)
|
||||
|
||||
def test_duplicate_or_missing_yaml_id_is_nonzero(self):
|
||||
cases = [
|
||||
([{"id": "TC-001"}, {"id": "TC-001"}] + [{"id": row["id"]} for row in self.rows[2:]], "YAML testcase id 重复"),
|
||||
([{"id": ""}] + [{"id": row["id"]} for row in self.rows[1:]], "YAML results[1] testcase id 缺失"),
|
||||
]
|
||||
for results, expected in cases:
|
||||
with self.subTest(expected=expected):
|
||||
self.write_yaml(results)
|
||||
result = self.run_cli()
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(expected, result.stderr)
|
||||
|
||||
def test_unknown_or_missing_csv_result_is_nonzero(self):
|
||||
for bad_result, expected in [("unexpected", "result 未知"), ("", "result 缺失")]:
|
||||
with self.subTest(bad_result=bad_result):
|
||||
rows = self.rows.copy()
|
||||
rows[0] = {"id": "TC-001", "result": bad_result}
|
||||
self.write_csv(rows)
|
||||
result = self.run_cli()
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(expected, result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user