#!/usr/bin/env python3 """tester catalog CLI: list / select / show / validate / next-id / run-script.""" from __future__ import annotations import argparse import json import os import re import subprocess import sys from datetime import datetime, timezone from pathlib import Path from typing import Any try: import yaml except ImportError: # pragma: no cover print("需要 PyYAML: python3 -m pip install pyyaml", file=sys.stderr) raise SystemExit(2) DRIVERS = ("script", "browser", "mobile") LIFECYCLES = ("active", "retired", "pending") LIFECYCLE_SKIP_REASON = { "retired": "retired", "pending": "pending(暂缓,见 notes)", } TAGS = ("smoke", "full", "slow") CANONICAL_RISKS = ("readonly", "mutating", "destructive") AUTOMATIONS = ("automated", "assisted", "manual") RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run") EXPECTED_KINDS = ( "api-status", "api-field", "script-exit", "visible-text", "url", "interaction", ) AGENT_DEFAULTS = { "environment": "", "host": "local", "project": "", "path": "", "role": "test", "cli": "codex", "tier": "standard", "model": None, "reasoningEffort": "medium", "permissionMode": "workspace-write", } def catalog_dir(root: Path) -> Path: return root / ".pouch" / "tester" def load_yaml(path: Path) -> Any: text = path.read_text(encoding="utf-8") data = yaml.safe_load(text) return data def dump_yaml(data: Any) -> str: return yaml.safe_dump( data, allow_unicode=True, sort_keys=False, default_flow_style=False, ) def truthy(value: Any) -> bool: if isinstance(value, bool): return value return str(value).strip().lower() in ("1", "true", "yes", "on") def load_catalog_meta(root: Path) -> dict[str, Any]: path = catalog_dir(root) / "catalog.yaml" if not path.is_file(): raise SystemExit(f"缺少 {path}") data = load_yaml(path) if not isinstance(data, dict): raise SystemExit(f"{path} 必须是 mapping") return data def load_local(root: Path) -> dict[str, Any]: path = catalog_dir(root) / "local.yaml" if not path.is_file(): return {} data = load_yaml(path) or {} if not isinstance(data, dict): raise SystemExit(f"{path} 必须是 mapping") return data def resolve_agent(root: Path, meta: dict[str, Any]) -> dict[str, Any]: """Resolve the structured agent target used by one tester run. Stable defaults may live in catalog.yaml. Machine-specific values belong in local.yaml and override the mapping shallowly. The old execution mapping is accepted as a compatibility input and is projected onto the agent target. """ agent: dict[str, Any] = dict(AGENT_DEFAULTS) local = load_local(root) found_agent = False for source_name, source in ( ("catalog.agent", meta.get("agent")), ("local.agent", local.get("agent")), ): if source is None: continue found_agent = True if not isinstance(source, dict): raise SystemExit(f"{source_name} 必须是 mapping") agent.update(source) if not found_agent: for source_name, source in ( ("catalog.execution", meta.get("execution")), ("local.execution", local.get("execution")), ): if source is None: continue if not isinstance(source, dict): raise SystemExit(f"{source_name} 必须是 mapping") for key in ("environment", "host", "path"): if key in source: agent[key] = source[key] return agent def case_files(root: Path) -> list[Path]: d = catalog_dir(root) / "cases" if not d.is_dir(): return [] return sorted(p for p in d.glob("*.yaml") if p.is_file()) def category_rank(root: Path) -> dict[str, int]: path = catalog_dir(root) / "catalog.yaml" order: list[str] = [] if path.is_file(): data = load_yaml(path) or {} if isinstance(data, dict): for item in data.get("categories") or []: if isinstance(item, dict) and item.get("id"): order.append(str(item["id"])) return {name: index for index, name in enumerate(order)} def sort_by_catalog(root: Path, cases: list[dict[str, Any]]) -> list[dict[str, Any]]: rank = category_rank(root) for case in cases: cat = str(case.get("category") or "") if cat and cat not in rank: rank[cat] = len(rank) return sorted(cases, key=lambda case: rank.get(str(case.get("category") or ""), 999)) def load_all_cases(root: Path) -> list[dict[str, Any]]: out: list[dict[str, Any]] = [] for path in case_files(root): data = load_yaml(path) or {} if not isinstance(data, dict): raise SystemExit(f"{path} 必须是 mapping") cat = data.get("category") title = data.get("title", "") cases = data.get("cases") or [] if not isinstance(cases, list): raise SystemExit(f"{path}: cases 必须是列表") for i, case in enumerate(cases): if not isinstance(case, dict): raise SystemExit(f"{path}: cases[{i}] 必须是 mapping") item = dict(case) item["_file"] = str(path.relative_to(root)) if "category" not in item and cat: item["category"] = cat if "_file_category" not in item: item["_file_category"] = cat item["_file_title"] = title out.append(item) return sort_by_catalog(root, out) def resolve_vars(root: Path, meta: dict[str, Any]) -> dict[str, str]: merged: dict[str, str] = {} for key, value in (meta.get("vars") or {}).items(): merged[str(key)] = "" if value is None else str(value) local = load_local(root) for key, value in local.items(): if value is None or isinstance(value, (dict, list)): continue merged[str(key)] = str(value).lower() if isinstance(value, bool) else str(value) auth_keys = set() for mapping in (meta.get("authorize") or {}, meta.get("gates") or {}): if isinstance(mapping, dict): auth_keys.update(str(value) for value in mapping.values() if value) for key, value in os.environ.items(): if key in merged or key in auth_keys or key.startswith("AUTHORIZE_") or key in { "HOST", "HX_TEST_HOST", "WEB_BASE", "API_BASE", "WS_URL", "STATUS_BASE", }: merged[key] = value if not merged.get("HOST") and merged.get("HX_TEST_HOST"): merged["HOST"] = merged["HX_TEST_HOST"] if os.environ.get("HX_TEST_HOST") and not os.environ.get("HOST"): merged["HOST"] = os.environ["HX_TEST_HOST"] for _ in range(10): changed = False for key, value in list(merged.items()): new = substitute(value, merged) if new != value: merged[key] = new changed = True if not changed: break return merged def substitute(text: str, vars_map: dict[str, str]) -> str: def repl(match: re.Match[str]) -> str: name = match.group(1) return vars_map.get(name, match.group(0)) return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", repl, text) def risk_index(meta: dict[str, Any], name: str) -> int: risks = list(meta.get("risks") or []) if name not in risks: raise SystemExit(f"未知风险: {name}(catalog risks={risks})") return risks.index(name) def host_set(vars_map: dict[str, str]) -> bool: host = (vars_map.get("HOST") or "").strip() return bool(host) and "${HOST}" not in host def export_csv_script() -> Path: return Path(__file__).resolve().parent / "generate_cases_csv.py" def run_export_csv(root: Path) -> int: proc = subprocess.run( [sys.executable, str(export_csv_script()), "--root", str(root)], check=False, ) return int(proc.returncode) def cmd_list(root: Path) -> int: meta = load_catalog_meta(root) cases = load_all_cases(root) risks = list(meta.get("risks") or []) cats_meta = {c["id"]: c for c in meta.get("categories") or [] if isinstance(c, dict) and "id" in c} active = [c for c in cases if c.get("lifecycle", "active") == "active"] retired = [c for c in cases if c.get("lifecycle") == "retired"] pending = [c for c in cases if c.get("lifecycle") == "pending"] print(f"## 测试目录 active {len(active)} / pending {len(pending)} / retired {len(retired)}") print() headers = ["分类", "标题", "条数", "smoke"] + risks + list(DRIVERS) print("| " + " | ".join(headers) + " |") print("| " + " | ".join("---" for _ in headers) + " |") cat_ids: list[str] = [] for c in meta.get("categories") or []: if isinstance(c, dict) and c.get("id"): cat_ids.append(str(c["id"])) for c in cases: cid = str(c.get("category") or "") if cid and cid not in cat_ids: cat_ids.append(cid) for cat in cat_ids: group = [c for c in active if c.get("category") == cat] title = (cats_meta.get(cat) or {}).get("title") or ( group[0].get("_file_title") if group else "" ) row = [cat, str(title), str(len(group)), str(sum(1 for c in group if c.get("tag") == "smoke"))] for risk in risks: row.append(str(sum(1 for c in group if c.get("risk") == risk))) for driver in DRIVERS: row.append(str(sum(1 for c in group if c.get("driver") == driver))) print("| " + " | ".join(row) + " |") print() print("跑法: 跑 <分类> | 跑 | 跑 smoke | 跑 <分类> 风险到 ") vars_map = resolve_vars(root, meta) print(f"HOST: {'已设' if host_set(vars_map) else '未设'}") code = run_export_csv(root) if code == 0: print(f"清单: {catalog_dir(root).relative_to(root) / 'cases.csv'}") run_csvs = sorted((catalog_dir(root) / "runs").glob("*.csv")) if run_csvs: print(f"成绩单: {run_csvs[-1].relative_to(root)}") return code def parse_ids(values: list[str]) -> list[str]: out: list[str] = [] for raw in values: for part in str(raw).split(","): item = part.strip() if item: out.append(item) return out def public_case(case: dict[str, Any]) -> dict[str, Any]: return {k: v for k, v in case.items() if not str(k).startswith("_")} PROBE_SOURCES = ("global", "category", "disabled", "none", "invalid") def effective_status_probe(meta: dict[str, Any], category: str) -> tuple[Any, str]: """分类生效的 status_probe(分类级前置条件)。 返回 (probe, source);probe 为 None 表示该分类不做探测。 source: global(继承顶层)/ category(分类覆盖)/ disabled(分类置 false)/ none(顶层和分类都没声明)/ invalid(分类值非法)。 """ cats = { str(c.get("id")): c for c in meta.get("categories") or [] if isinstance(c, dict) and c.get("id") } cat = cats.get(category) or {} if "status_probe" in cat: value = cat.get("status_probe") if value is False: return None, "disabled" if isinstance(value, dict): return value, "category" return None, "invalid" probe = meta.get("status_probe") if isinstance(probe, dict): return probe, "global" return None, "none" def resolve_probe(probe: Any, vars_map: dict[str, str]) -> Any: """把探测定义里的 ${VAR} 展开,Agent 拿到就能直接请求。""" if not isinstance(probe, dict): return probe out = dict(probe) url = out.get("url") if isinstance(url, str): out["url"] = substitute(url, vars_map) stop_if = out.get("stop_if") if isinstance(stop_if, dict): out["stop_if"] = { (substitute(str(key), vars_map) if "${" in str(key) else key): ( [ substitute(str(item), vars_map) if isinstance(item, str) and "${" in item else item for item in value ] if isinstance(value, list) else value ) for key, value in stop_if.items() } return out def probe_errors(where: str, probe: Any) -> list[str]: """校验一份 status_probe 定义。允许值为 mapping;false 由调用方处理。""" if not isinstance(probe, dict): return [f"{where}: status_probe 必须是 mapping 或 false"] errors: list[str] = [] url = probe.get("url") if not isinstance(url, str) or not url.strip(): errors.append(f"{where}.url 必填") stop_if = probe.get("stop_if") if stop_if is not None and not isinstance(stop_if, dict): errors.append(f"{where}.stop_if 必须是 mapping(字段 -> 命中值列表)") return errors def agent_errors(root: Path, meta: dict[str, Any]) -> list[str]: errors: list[str] = [] try: agent = resolve_agent(root, meta) except SystemExit as exc: return [str(exc)] for key in ( "environment", "host", "project", "path", "role", "cli", "tier", "model", "reasoningEffort", "permissionMode", ): if key not in agent: errors.append(f"agent.{key} 缺失") for key in ("environment", "host", "project", "path", "role", "cli", "tier", "permissionMode"): if not isinstance(agent.get(key), str): errors.append(f"agent.{key} 必须是字符串") if agent.get("model") is not None and not isinstance(agent.get("model"), str): errors.append("agent.model 必须是字符串或 null") if agent.get("role") != "test": errors.append("agent.role 必须是 test") if agent.get("cli") not in ("codex", "cursor-agent", "grok", "omp", "pi"): errors.append("agent.cli 只能是 codex/cursor-agent/grok/omp/pi") if agent.get("tier") not in ("standard", "strong"): errors.append("agent.tier 只能是 standard 或 strong") if agent.get("reasoningEffort") not in (None, "low", "medium", "high", "xhigh"): errors.append("agent.reasoningEffort 只能是 low/medium/high/xhigh 或 null") if agent.get("permissionMode") not in ("read-only", "workspace-write"): errors.append("agent.permissionMode 只能是 read-only 或 workspace-write") return errors def run_together_groups(meta: dict[str, Any]) -> list[dict[str, Any]]: raw = meta.get("run_together") or [] if not isinstance(raw, list): return [] groups: list[dict[str, Any]] = [] for item in raw: if not isinstance(item, dict): continue ids = [str(x) for x in (item.get("ids") or []) if str(x).strip()] if len(ids) < 2: continue groups.append( { "ids": ids, "continue_on_fail": bool(item.get("continue_on_fail")), } ) return groups def expand_run_together( meta: dict[str, Any], all_cases: list[dict[str, Any]], selected: list[dict[str, Any]], skipped: list[dict[str, str]], max_i: int, automation_filter: str = "automated", ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: groups = run_together_groups(meta) if not groups: return selected, [] by_id = {str(c.get("id") or ""): c for c in all_cases} selected_ids = [str(c.get("id") or "") for c in selected] selected_map = {str(c.get("id") or ""): c for c in selected} hit_groups: list[dict[str, Any]] = [] for group in groups: ids = group["ids"] if not any(cid in selected_map for cid in ids): continue hit_groups.append(group) first_pos = min( (selected_ids.index(cid) for cid in ids if cid in selected_ids), default=len(selected_ids), ) members: list[dict[str, Any]] = [] for cid in ids: if cid in selected_map: members.append(selected_map[cid]) continue case = by_id.get(cid) if not case or case.get("lifecycle", "active") != "active": skipped.append({"id": cid, "reason": "run_together 搭档不是 active"}) continue automation = str(case.get("automation") or "automated") if automation_filter != "any" and automation != automation_filter: skipped.append({"id": cid, "reason": f"run_together 搭档 automation={automation}(需人工)"}) continue risk = str(case.get("risk") or "readonly") try: r_i = risk_index(meta, risk) except SystemExit: skipped.append({"id": cid, "reason": f"未知风险 {risk}"}) continue if r_i > max_i: skipped.append({"id": cid, "reason": f"risk={risk} 超过上限"}) continue members.append(public_case(case)) drop = set(ids) new_selected: list[dict[str, Any]] = [] inserted = False for case in selected: cid = str(case.get("id") or "") if cid in drop: if not inserted: new_selected.extend(members) inserted = True continue new_selected.append(case) if not inserted: new_selected.extend(members) selected = new_selected selected_ids = [str(c.get("id") or "") for c in selected] selected_map = {str(c.get("id") or ""): c for c in selected} return selected, hit_groups def cmd_select(args: argparse.Namespace, root: Path) -> int: meta = load_catalog_meta(root) cases = load_all_cases(root) max_risk = args.max_risk or meta.get("default_max_risk") or "readonly" max_i = risk_index(meta, max_risk) automation_filter = args.automation or "automated" wanted = set(parse_ids(args.ids or [])) selected: list[dict[str, Any]] = [] skipped: list[dict[str, str]] = [] missing = set(wanted) for case in cases: cid = str(case.get("id") or "") if case.get("lifecycle", "active") != "active": if cid in wanted: lifecycle = str(case.get("lifecycle")) skipped.append({ "id": cid, "reason": LIFECYCLE_SKIP_REASON.get(lifecycle, lifecycle), }) missing.discard(cid) continue if wanted: if cid not in wanted: continue missing.discard(cid) else: if args.category and case.get("category") != args.category: continue if args.tag and case.get("tag") != args.tag: continue if args.driver and case.get("driver") != args.driver: continue automation = str(case.get("automation") or "automated") if automation_filter != "any" and automation != automation_filter: skipped.append({ "id": cid, "reason": f"automation={automation}(需人工),本轮只跑 automation={automation_filter}", }) continue risk = str(case.get("risk") or "readonly") try: r_i = risk_index(meta, risk) except SystemExit: skipped.append({"id": cid, "reason": f"未知风险 {risk}"}) continue if r_i > max_i: skipped.append({"id": cid, "reason": f"risk={risk} 超过上限 {max_risk}"}) continue selected.append(public_case(case)) if wanted and missing: print( json.dumps({"error": "找不到 active 用例: " + ", ".join(sorted(missing))}, ensure_ascii=False), file=sys.stderr, ) return 2 selected, together = expand_run_together(meta, cases, selected, skipped, max_i, automation_filter) # 按分类解析生效的 status_probe,让 Agent 不再自己按 catalog 猜作用范围。 vars_map = resolve_vars(root, meta) status_probes: dict[str, dict[str, Any]] = {} for case in selected: cat = str(case.get("category") or "") if cat and cat not in status_probes: probe, source = effective_status_probe(meta, cat) status_probes[cat] = {"source": source, "probe": resolve_probe(probe, vars_map)} print( json.dumps( { "max_risk": max_risk, "automation": automation_filter, "count": len(selected), "vars": vars_map, "skipped": skipped, "run_together": together, "status_probe": status_probes, "agent": resolve_agent(root, meta), "cases": selected, }, ensure_ascii=False, indent=2, ) ) return 0 def cmd_show(root: Path, case_id: str) -> int: for case in load_all_cases(root): if case.get("id") == case_id: public = {k: v for k, v in case.items() if not str(k).startswith("_")} print(dump_yaml(public), end="") return 0 print(f"找不到用例: {case_id}", file=sys.stderr) return 2 def validate(root: Path) -> list[str]: errors: list[str] = [] meta_path = catalog_dir(root) / "catalog.yaml" if not meta_path.is_file(): return [f"缺少 {meta_path}"] try: meta = load_catalog_meta(root) except SystemExit as exc: return [str(exc)] errors.extend(agent_errors(root, meta)) cats = meta.get("categories") if not isinstance(cats, list): errors.append("catalog.categories 必须是列表") cats = [] cat_ids = [] for i, item in enumerate(cats): if not isinstance(item, dict) or not item.get("id"): errors.append(f"catalog.categories[{i}] 需要 id") else: cat_ids.append(str(item["id"])) if isinstance(item, dict) and "status_probe" in item: value = item.get("status_probe") if value is not False: errors.extend(probe_errors(f"catalog.categories[{i}].status_probe", value)) if "status_probe" in meta: errors.extend(probe_errors("catalog.status_probe", meta.get("status_probe"))) risks = meta.get("risks") if not isinstance(risks, list) or not risks: errors.append("catalog.risks 必须是非空列表") risks = [] if risks and risks[0] != "readonly": errors.append("catalog.risks 必须以 readonly 开头") last_pos = -1 for name in risks: if name not in CANONICAL_RISKS: errors.append( f"非法 risk 名 {name!r},只能是 readonly/mutating/destructive" ) continue pos = CANONICAL_RISKS.index(name) if pos <= last_pos: errors.append("catalog.risks 必须按 readonly < mutating < destructive 升序且不重复") last_pos = pos default_max = meta.get("default_max_risk") if default_max not in risks: errors.append(f"default_max_risk={default_max!r} 不在 risks 中") gates = meta.get("gates") or {} if gates and not isinstance(gates, dict): errors.append("catalog.gates 必须是 mapping") gates = {} authorize = meta.get("authorize") or {} if authorize and not isinstance(authorize, dict): errors.append("catalog.authorize 必须是 mapping") authorize = {} sessions = meta.get("sessions") or {} if not isinstance(sessions, dict): errors.append("catalog.sessions 必须是 mapping") sessions = {} pattern = meta.get("id_pattern") or r"^.+$" try: id_re = re.compile(pattern) except re.error as exc: errors.append(f"id_pattern 无效: {exc}") id_re = re.compile(r"^.+$") seen: dict[str, str] = {} try: cases = load_all_cases(root) except SystemExit as exc: return errors + [str(exc)] together_raw = meta.get("run_together") if together_raw is None: together_raw = [] if not isinstance(together_raw, list): errors.append("catalog.run_together 必须是列表") together_raw = [] together_seen: set[str] = set() for i, item in enumerate(together_raw): where = f"catalog.run_together[{i}]" if not isinstance(item, dict): errors.append(f"{where}: 必须是 mapping,含 ids") continue ids = item.get("ids") if not isinstance(ids, list) or len(ids) < 2: errors.append(f"{where}: ids 至少两个") continue for cid in ids: cid_s = str(cid) if cid_s in together_seen: errors.append(f"{where}: id {cid_s} 已在另一组 run_together") together_seen.add(cid_s) for case in cases: cid = str(case.get("id") or "") where = f"{case.get('_file')}:{cid or '?'}" if not cid: errors.append(f"{where}: 缺少 id") continue if not id_re.match(cid): errors.append(f"{where}: id 不匹配 {pattern}") if cid in seen: errors.append(f"{where}: id 重复(也出现在 {seen[cid]})") seen[cid] = str(case.get("_file")) if cid in together_seen: together_seen.discard(cid) cat = str(case.get("category") or "") file_cat = str(case.get("_file_category") or "") if file_cat and cat and cat != file_cat: errors.append(f"{where}: category={cat} 与文件 category={file_cat} 不一致") if cat and cat_ids and cat not in cat_ids: errors.append(f"{where}: category={cat} 不在 catalog.categories") if case.get("lifecycle", "active") not in LIFECYCLES: errors.append(f"{where}: 非法 lifecycle") if case.get("tag") not in TAGS: errors.append(f"{where}: 非法 tag") if case.get("risk") not in risks: errors.append(f"{where}: 非法 risk={case.get('risk')}") if case.get("tag") == "smoke" and case.get("risk") != "readonly": errors.append(f"{where}: tag=smoke 只能 risk=readonly") gate = case.get("gate") if gate: if gate not in gates: errors.append(f"{where}: gate={gate} 未在 catalog.gates 声明") if case.get("risk") in (None, "readonly"): errors.append(f"{where}: risk=readonly 不要填 gate") risk = case.get("risk") if risk and risk != "readonly" and risk not in authorize and not gate: errors.append(f"{where}: {risk} 需要 authorize.{risk} 或 gate") automation = str(case.get("automation") or "automated") if automation not in AUTOMATIONS: errors.append(f"{where}: 非法 automation={case.get('automation')}") automation = "automated" if not str(case.get("title") or "").strip(): errors.append(f"{where}: 缺少 title") if not str(case.get("what") or "").strip(): errors.append(f"{where}: 缺少 what") how = case.get("how") if not isinstance(how, dict): errors.append(f"{where}: how 必须是 mapping") how = {} steps = how.get("steps") or [] if automation == "manual": if case.get("driver"): errors.append(f"{where}: automation=manual 不要填 driver") if how.get("run"): errors.append(f"{where}: automation=manual 不要填 how.run") if not steps: errors.append(f"{where}: automation=manual 需要 how.steps") else: driver = case.get("driver") if driver not in DRIVERS: errors.append(f"{where}: 非法 driver={driver}") if driver == "script" and not how.get("run") and not steps: errors.append(f"{where}: script 需要 how.run 或 how.steps") if driver in ("browser", "mobile"): session = case.get("session") if not session: errors.append(f"{where}: {driver} 需要 session") elif session not in sessions: errors.append(f"{where}: session={session} 未在 catalog.sessions 声明") if driver == "browser" and not how.get("url") and not steps: errors.append(f"{where}: browser 需要 how.url 或 how.steps") if driver == "mobile" and not steps: errors.append(f"{where}: mobile 需要 how.steps") if automation == "assisted": if not steps: errors.append(f"{where}: automation=assisted 需要 how.steps") elif not any(str(step).startswith("[人工]") for step in steps): errors.append(f"{where}: automation=assisted 的 how.steps 至少一步以 [人工] 开头") assume = case.get("assume") if assume is not None: if not isinstance(assume, str) or not assume.strip(): errors.append(f"{where}: assume 必须是非空字符串") else: forbidden: set[str] = set() risk_name = str(case.get("risk") or "") if risk_name and risk_name != "readonly": env = authorize.get(risk_name) if env: forbidden.add(str(env)) if gate and gate in gates: forbidden.add(str(gates[gate])) for var in sorted(forbidden): if re.search(rf"\b{re.escape(var)}\b", assume): errors.append(f"{where}: assume 重复了 {var}(已由 risk/gate 授权)") expected = case.get("expected") if not isinstance(expected, list) or not expected: errors.append(f"{where}: expected 必须是非空列表") else: for j, sig in enumerate(expected): if not isinstance(sig, dict) or not sig.get("kind") or not sig.get("value"): errors.append(f"{where}: expected[{j}] 需要 kind 与 value") elif sig.get("kind") not in EXPECTED_KINDS: errors.append(f"{where}: expected[{j}].kind 非法") if together_seen: errors.append("catalog.run_together 引用了不存在的 id: " + ", ".join(sorted(together_seen))) return errors def cmd_validate(root: Path) -> int: errors = validate(root) if errors: for err in errors: print(f"✗ {err}", file=sys.stderr) print(f"validate: {len(errors)} 个问题", file=sys.stderr) return 1 cases = load_all_cases(root) active = sum(1 for c in cases if c.get("lifecycle", "active") == "active") pending = sum(1 for c in cases if c.get("lifecycle") == "pending") print(f"✓ validate 通过 active={active} pending={pending} total={len(cases)}") return 0 def cmd_export_csv(root: Path) -> int: return run_export_csv(root) def cmd_next_id(root: Path, category: str) -> int: meta = load_catalog_meta(root) cat_ids = [c.get("id") for c in meta.get("categories") or [] if isinstance(c, dict)] if category not in cat_ids: print(f"未知分类: {category} 可选: {cat_ids}", file=sys.stderr) return 2 prefix = f"TC-{category}-" max_n = 0 for case in load_all_cases(root): cid = str(case.get("id") or "") if case.get("category") != category: continue match = re.match(rf"TC-{re.escape(category)}-(\d+)", cid) if match: max_n = max(max_n, int(match.group(1))) print(f"{prefix}{max_n + 1:03d}") return 0 def authorized(vars_map: dict[str, str], meta: dict[str, Any], case: dict[str, Any]) -> bool: risk = str(case.get("risk") or "readonly") risks = list(meta.get("risks") or ["readonly"]) lowest = risks[0] if risks else "readonly" if risk != lowest: env_name = (meta.get("authorize") or {}).get(risk) if env_name and not truthy(vars_map.get(str(env_name), "")): return False if not env_name and not case.get("gate"): return False gate = case.get("gate") if gate: gate_env = (meta.get("gates") or {}).get(gate) if not gate_env or not truthy(vars_map.get(str(gate_env), "")): return False return True def cmd_run_script(root: Path, case_id: str) -> int: meta = load_catalog_meta(root) vars_map = resolve_vars(root, meta) case = next((c for c in load_all_cases(root) if c.get("id") == case_id), None) if not case: print(json.dumps({"id": case_id, "result": "environment-fail", "evidence": "找不到用例"}, ensure_ascii=False)) return 2 public_id = case.get("id") if case.get("lifecycle", "active") != "active": lifecycle = str(case.get("lifecycle")) reason = LIFECYCLE_SKIP_REASON.get(lifecycle, lifecycle) print(json.dumps({"id": public_id, "result": "skipped", "evidence": reason}, ensure_ascii=False)) return 0 automation = str(case.get("automation") or "automated") if automation != "automated": print( json.dumps( {"id": public_id, "result": "not-run", "evidence": f"automation={automation},需人工,run-script 不执行"}, ensure_ascii=False, ) ) return 0 if case.get("driver") != "script": print( json.dumps( {"id": public_id, "result": "environment-fail", "evidence": "run-script 只能跑 driver=script"}, ensure_ascii=False, ) ) return 2 if not host_set(vars_map): print( json.dumps( {"id": public_id, "result": "environment-fail", "evidence": "HOST 未设"}, ensure_ascii=False, ) ) return 2 risk = str(case.get("risk") or "readonly") if not authorized(vars_map, meta, case): print( json.dumps( {"id": public_id, "result": "skipped", "evidence": f"风险 {risk} 未授权"}, ensure_ascii=False, ) ) return 0 how = case.get("how") or {} run = how.get("run") if not run: print( json.dumps( {"id": public_id, "result": "environment-fail", "evidence": "没有 how.run,改由 Agent 按 steps 执行"}, ensure_ascii=False, ) ) return 2 script = substitute(str(run), vars_map) timeout = float(how.get("timeout") or 60) env = os.environ.copy() env.update({k: v for k, v in vars_map.items() if v is not None}) try: proc = subprocess.run( script, shell=True, cwd=str(root), env=env, capture_output=True, text=True, timeout=timeout, check=False, ) except subprocess.TimeoutExpired: print( json.dumps( { "id": public_id, "result": "environment-fail", "evidence": f"timeout {timeout}s", }, ensure_ascii=False, ) ) return 1 output = ((proc.stdout or "") + ("\n" + proc.stderr if proc.stderr else "")).strip() evidence = output[-1200:] print( json.dumps( { "id": public_id, "result": "executed", "exit": proc.returncode, "evidence": evidence, }, ensure_ascii=False, ) ) return 0 def build_parser() -> argparse.ArgumentParser: parent = argparse.ArgumentParser(add_help=False) parent.add_argument("--root", default=".", help="项目根") parser = argparse.ArgumentParser(description="tester catalog") sub = parser.add_subparsers(dest="cmd", required=True) sub.add_parser("list", help="按分类统计", parents=[parent]) p_sel = sub.add_parser("select", help="筛选用例 JSON", parents=[parent]) p_sel.add_argument("--category") p_sel.add_argument("--id", action="append", dest="ids", default=[]) p_sel.add_argument("--tag", choices=TAGS) p_sel.add_argument("--max-risk") p_sel.add_argument("--driver", choices=DRIVERS) p_sel.add_argument( "--automation", choices=[*AUTOMATIONS, "any"], default=None, help="默认 automated;any 表示不按人工参与度筛选", ) p_show = sub.add_parser("show", help="打印一条用例", parents=[parent]) p_show.add_argument("--id", required=True, dest="case_id") sub.add_parser("validate", help="校验目录", parents=[parent]) sub.add_parser("export-csv", help="写出给人看的 cases.csv", parents=[parent]) p_next = sub.add_parser("next-id", help="分配下一个 TC--NNN", parents=[parent]) p_next.add_argument("--category", required=True) p_run = sub.add_parser("run-script", help="执行一条 how.run", parents=[parent]) p_run.add_argument("--id", required=True, dest="case_id") return parser def main(argv: list[str] | None = None) -> int: parser = build_parser() args = parser.parse_args(argv) root = Path(args.root).resolve() if args.cmd == "list": return cmd_list(root) if args.cmd == "select": return cmd_select(args, root) if args.cmd == "show": return cmd_show(root, args.case_id) if args.cmd == "validate": return cmd_validate(root) if args.cmd == "export-csv": return cmd_export_csv(root) if args.cmd == "next-id": return cmd_next_id(root, args.category) if args.cmd == "run-script": return cmd_run_script(root, args.case_id) parser.error(f"未知命令 {args.cmd}") return 2 if __name__ == "__main__": raise SystemExit(main())