1499 lines
56 KiB
Python
Executable File
1499 lines
56 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""tester catalog CLI: list / select / show / validate / next-id / run-script."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import contextlib
|
||
import io
|
||
import json
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import urllib.parse
|
||
import urllib.request
|
||
import zipfile
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from journey import (
|
||
CLIENT_SUITES,
|
||
INTEGRATION_SUITES,
|
||
TEST_TYPES,
|
||
contract_errors,
|
||
preflight as journey_preflight,
|
||
suite as case_suite,
|
||
test_type,
|
||
)
|
||
|
||
try:
|
||
import yaml
|
||
except ImportError: # pragma: no cover
|
||
print("需要 PyYAML: python3 -m pip install pyyaml", file=sys.stderr)
|
||
raise SystemExit(2)
|
||
|
||
DRIVERS = ("script", "browser", "mobile")
|
||
LIFECYCLES = ("active", "retired", "pending")
|
||
LIFECYCLE_SKIP_REASON = {
|
||
"retired": "retired",
|
||
"pending": "pending（暂缓，见 notes）",
|
||
}
|
||
TAGS = ("smoke", "full", "slow")
|
||
CANONICAL_RISKS = ("readonly", "mutating", "destructive")
|
||
EXECUTION_PROFILES = ("standard", "isolated-full")
|
||
AUTOMATIONS = ("automated", "assisted", "manual")
|
||
RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run")
|
||
EXPECTED_KINDS = (
|
||
"api-status",
|
||
"api-field",
|
||
"script-exit",
|
||
"visible-text",
|
||
"url",
|
||
"interaction",
|
||
)
|
||
AGENT_DEFAULTS = {
|
||
"environment": "",
|
||
"host": "local",
|
||
"project": "",
|
||
"path": "",
|
||
"role": "test",
|
||
"cli": "codex",
|
||
"tier": "standard",
|
||
"model": None,
|
||
"reasoningEffort": "medium",
|
||
"permissionMode": "workspace-write",
|
||
}
|
||
|
||
|
||
def catalog_dir(root: Path) -> Path:
|
||
return root / ".pouch" / "tester"
|
||
|
||
|
||
def catalog_path(root: Path) -> Path:
|
||
return catalog_dir(root) / "catalog.yaml"
|
||
|
||
|
||
def discover_or_self(root: Path) -> list[Path]:
|
||
"""Return all independent tester projects below a parent repository."""
|
||
root = root.resolve()
|
||
return discover_catalog_roots(root)
|
||
|
||
|
||
def cmd_fanout(args: argparse.Namespace, roots: list[Path]) -> int:
|
||
"""Run a catalog command independently for every discovered tester root."""
|
||
results: list[dict[str, Any]] = []
|
||
for child_root in roots:
|
||
output = io.StringIO()
|
||
with contextlib.redirect_stdout(output):
|
||
if args.cmd == "list":
|
||
exit_code = cmd_list(child_root)
|
||
elif args.cmd == "select":
|
||
exit_code = cmd_select(args, child_root)
|
||
elif args.cmd == "show":
|
||
exit_code = cmd_show(child_root, args.case_id)
|
||
elif args.cmd == "validate":
|
||
exit_code = cmd_validate(child_root)
|
||
elif args.cmd == "export-csv":
|
||
exit_code = cmd_export_csv(child_root)
|
||
elif args.cmd == "run-script":
|
||
exit_code = cmd_run_script(child_root, args.case_id)
|
||
else:
|
||
exit_code = 2
|
||
print(f"不支持从父仓库 fan-out 命令: {args.cmd}")
|
||
raw = output.getvalue().strip()
|
||
try:
|
||
result: Any = json.loads(raw) if raw else None
|
||
except json.JSONDecodeError:
|
||
result = raw
|
||
results.append({
|
||
"root": str(child_root),
|
||
"exit": exit_code,
|
||
"result": result,
|
||
})
|
||
|
||
if args.cmd == "select":
|
||
print(json.dumps({"mode": "fan-out", "roots": results}, ensure_ascii=False, indent=2))
|
||
else:
|
||
for item in results:
|
||
print(f"\n## tester root: {item['root']} (exit={item['exit']})")
|
||
result = item["result"]
|
||
if isinstance(result, (dict, list)):
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
elif result:
|
||
print(result)
|
||
return 0 if all(item["exit"] == 0 for item in results) else 2
|
||
|
||
|
||
FANOUT_COMMANDS = {"list", "select", "show", "validate", "export-csv", "run-script"}
|
||
|
||
|
||
def discover_catalog_roots(root: Path) -> list[Path]:
|
||
"""Find tester projects owned by a repository and its Git submodules.
|
||
|
||
A monorepo or a parent repository may contain more than one independent
|
||
tester catalog. Cases must stay scoped to the project that owns them; this
|
||
helper only discovers roots and never merges their cases.
|
||
"""
|
||
root = root.resolve()
|
||
candidates: list[Path] = []
|
||
if catalog_path(root).is_file():
|
||
candidates.append(root)
|
||
|
||
# Prefer the authoritative submodule list so generated directories and
|
||
# vendored dependencies are not accidentally treated as test projects.
|
||
try:
|
||
proc = subprocess.run(
|
||
["git", "-C", str(root), "config", "--file", ".gitmodules", "--get-regexp", r"^submodule\..*\.path$"],
|
||
capture_output=True,
|
||
text=True,
|
||
check=False,
|
||
)
|
||
for line in proc.stdout.splitlines():
|
||
_, _, raw_path = line.partition(" ")
|
||
subroot = (root / raw_path.strip()).resolve()
|
||
if raw_path.strip() and catalog_path(subroot).is_file() and subroot not in candidates:
|
||
candidates.append(subroot)
|
||
except OSError:
|
||
pass
|
||
|
||
# Also support a plain nested package that is not declared as a submodule.
|
||
# Only inspect a shallow set of project directories; callers can pass that
|
||
# directory explicitly when a deeper layout is intentional.
|
||
ignored = {".git", ".pouch", "node_modules", "vendor", "third_party", "dist", "build"}
|
||
try:
|
||
children = list(root.iterdir())
|
||
except OSError:
|
||
children = []
|
||
for child in children:
|
||
if not child.is_dir() or child.name in ignored or child.name.startswith("."):
|
||
continue
|
||
if catalog_path(child).is_file() and child.resolve() not in candidates:
|
||
candidates.append(child.resolve())
|
||
return sorted(candidates)
|
||
|
||
|
||
def load_yaml(path: Path) -> Any:
|
||
text = path.read_text(encoding="utf-8")
|
||
data = yaml.safe_load(text)
|
||
return data
|
||
|
||
|
||
def dump_yaml(data: Any) -> str:
|
||
return yaml.safe_dump(
|
||
data,
|
||
allow_unicode=True,
|
||
sort_keys=False,
|
||
default_flow_style=False,
|
||
)
|
||
|
||
|
||
def truthy(value: Any) -> bool:
|
||
if isinstance(value, bool):
|
||
return value
|
||
return str(value).strip().lower() in ("1", "true", "yes", "on")
|
||
|
||
|
||
def load_catalog_meta(root: Path) -> dict[str, Any]:
|
||
path = catalog_dir(root) / "catalog.yaml"
|
||
if not path.is_file():
|
||
raise SystemExit(f"缺少 {path}")
|
||
data = load_yaml(path)
|
||
if not isinstance(data, dict):
|
||
raise SystemExit(f"{path} 必须是 mapping")
|
||
return data
|
||
|
||
|
||
def load_local(root: Path) -> dict[str, Any]:
|
||
path = catalog_dir(root) / "local.yaml"
|
||
if not path.is_file():
|
||
return {}
|
||
data = load_yaml(path) or {}
|
||
if not isinstance(data, dict):
|
||
raise SystemExit(f"{path} 必须是 mapping")
|
||
return data
|
||
|
||
|
||
def resolve_agent(root: Path, meta: dict[str, Any]) -> dict[str, Any]:
|
||
"""Resolve the structured agent target used by one tester run.
|
||
|
||
Stable defaults may live in catalog.yaml. Machine-specific values belong in
|
||
local.yaml and override the mapping shallowly. The old execution mapping is
|
||
accepted as a compatibility input and is projected onto the agent target.
|
||
"""
|
||
agent: dict[str, Any] = dict(AGENT_DEFAULTS)
|
||
local = load_local(root)
|
||
found_agent = False
|
||
for source_name, source in (
|
||
("catalog.agent", meta.get("agent")),
|
||
("local.agent", local.get("agent")),
|
||
):
|
||
if source is None:
|
||
continue
|
||
found_agent = True
|
||
if not isinstance(source, dict):
|
||
raise SystemExit(f"{source_name} 必须是 mapping")
|
||
agent.update(source)
|
||
if not found_agent:
|
||
for source_name, source in (
|
||
("catalog.execution", meta.get("execution")),
|
||
("local.execution", local.get("execution")),
|
||
):
|
||
if source is None:
|
||
continue
|
||
if not isinstance(source, dict):
|
||
raise SystemExit(f"{source_name} 必须是 mapping")
|
||
for key in ("environment", "host", "path"):
|
||
if key in source:
|
||
agent[key] = source[key]
|
||
return agent
|
||
|
||
|
||
def case_files(root: Path) -> list[Path]:
|
||
d = catalog_dir(root) / "cases"
|
||
if not d.is_dir():
|
||
return []
|
||
return sorted(p for p in d.glob("*.yaml") if p.is_file())
|
||
|
||
|
||
def category_rank(root: Path) -> dict[str, int]:
|
||
path = catalog_dir(root) / "catalog.yaml"
|
||
order: list[str] = []
|
||
if path.is_file():
|
||
data = load_yaml(path) or {}
|
||
if isinstance(data, dict):
|
||
for item in data.get("categories") or []:
|
||
if isinstance(item, dict) and item.get("id"):
|
||
order.append(str(item["id"]))
|
||
return {name: index for index, name in enumerate(order)}
|
||
|
||
|
||
def sort_by_catalog(root: Path, cases: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
rank = category_rank(root)
|
||
for case in cases:
|
||
cat = str(case.get("category") or "")
|
||
if cat and cat not in rank:
|
||
rank[cat] = len(rank)
|
||
return sorted(cases, key=lambda case: rank.get(str(case.get("category") or ""), 999))
|
||
|
||
|
||
def load_all_cases(root: Path) -> list[dict[str, Any]]:
|
||
out: list[dict[str, Any]] = []
|
||
for path in case_files(root):
|
||
data = load_yaml(path) or {}
|
||
if not isinstance(data, dict):
|
||
raise SystemExit(f"{path} 必须是 mapping")
|
||
cat = data.get("category")
|
||
title = data.get("title", "")
|
||
cases = data.get("cases") or []
|
||
if not isinstance(cases, list):
|
||
raise SystemExit(f"{path}: cases 必须是列表")
|
||
for i, case in enumerate(cases):
|
||
if not isinstance(case, dict):
|
||
raise SystemExit(f"{path}: cases[{i}] 必须是 mapping")
|
||
item = dict(case)
|
||
item["_file"] = str(path.relative_to(root))
|
||
if "category" not in item and cat:
|
||
item["category"] = cat
|
||
if "_file_category" not in item:
|
||
item["_file_category"] = cat
|
||
item["_file_title"] = title
|
||
out.append(item)
|
||
return sort_by_catalog(root, out)
|
||
|
||
|
||
def resolve_vars(root: Path, meta: dict[str, Any]) -> dict[str, str]:
|
||
merged: dict[str, str] = {}
|
||
for key, value in (meta.get("vars") or {}).items():
|
||
merged[str(key)] = "" if value is None else str(value)
|
||
local = load_local(root)
|
||
for key, value in local.items():
|
||
if value is None or isinstance(value, (dict, list)):
|
||
continue
|
||
merged[str(key)] = str(value).lower() if isinstance(value, bool) else str(value)
|
||
auth_keys = set()
|
||
for mapping in (meta.get("authorize") or {}, meta.get("gates") or {}):
|
||
if isinstance(mapping, dict):
|
||
auth_keys.update(str(value) for value in mapping.values() if value)
|
||
for key, value in os.environ.items():
|
||
if key in merged or key in auth_keys or key.startswith("AUTHORIZE_") or key in {
|
||
"HOST",
|
||
"WEB_BASE",
|
||
"API_BASE",
|
||
"WS_URL",
|
||
"STATUS_BASE",
|
||
}:
|
||
merged[key] = value
|
||
for _ in range(10):
|
||
changed = False
|
||
for key, value in list(merged.items()):
|
||
new = substitute(value, merged)
|
||
if new != value:
|
||
merged[key] = new
|
||
changed = True
|
||
if not changed:
|
||
break
|
||
return merged
|
||
|
||
|
||
def substitute(text: str, vars_map: dict[str, str]) -> str:
|
||
def repl(match: re.Match[str]) -> str:
|
||
name = match.group(1)
|
||
return vars_map.get(name, match.group(0))
|
||
|
||
return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", repl, text)
|
||
|
||
|
||
def risk_index(meta: dict[str, Any], name: str) -> int:
|
||
risks = list(meta.get("risks") or [])
|
||
if name not in risks:
|
||
raise SystemExit(f"未知风险: {name}（catalog risks={risks}）")
|
||
return risks.index(name)
|
||
|
||
|
||
def resolve_execution_profile(meta: dict[str, Any], requested: str | None = None) -> str:
|
||
"""Resolve the run profile without changing the meaning of ``risk``.
|
||
|
||
``isolated-full`` is an execution policy for a resettable test environment:
|
||
it disables risk/authorization filtering, but leaves the catalog risk value
|
||
on every case so reports still describe the side effect that was exercised.
|
||
"""
|
||
profile = requested or meta.get("execution_profile") or "standard"
|
||
if profile not in EXECUTION_PROFILES:
|
||
raise SystemExit(
|
||
f"未知 execution_profile: {profile!r}（可选: {', '.join(EXECUTION_PROFILES)}）"
|
||
)
|
||
return str(profile)
|
||
|
||
|
||
def host_set(vars_map: dict[str, str]) -> bool:
|
||
host = (vars_map.get("HOST") or "").strip()
|
||
return bool(host) and "${HOST}" not in host
|
||
|
||
|
||
def export_csv_script() -> Path:
|
||
return Path(__file__).resolve().parent / "generate_cases_csv.py"
|
||
|
||
|
||
def run_export_csv(root: Path) -> int:
|
||
proc = subprocess.run(
|
||
[sys.executable, str(export_csv_script()), "--root", str(root)],
|
||
check=False,
|
||
)
|
||
return int(proc.returncode)
|
||
|
||
|
||
def cmd_list(root: Path) -> int:
|
||
meta = load_catalog_meta(root)
|
||
cases = load_all_cases(root)
|
||
risks = list(meta.get("risks") or [])
|
||
cats_meta = {c["id"]: c for c in meta.get("categories") or [] if isinstance(c, dict) and "id" in c}
|
||
active = [c for c in cases if c.get("lifecycle", "active") == "active"]
|
||
retired = [c for c in cases if c.get("lifecycle") == "retired"]
|
||
pending = [c for c in cases if c.get("lifecycle") == "pending"]
|
||
print(f"## 测试目录 active {len(active)} / pending {len(pending)} / retired {len(retired)}")
|
||
print()
|
||
headers = ["分类", "标题", "条数", "journey", "integration", "smoke"] + risks + list(DRIVERS)
|
||
print("| " + " | ".join(headers) + " |")
|
||
print("| " + " | ".join("---" for _ in headers) + " |")
|
||
|
||
cat_ids: list[str] = []
|
||
for c in meta.get("categories") or []:
|
||
if isinstance(c, dict) and c.get("id"):
|
||
cat_ids.append(str(c["id"]))
|
||
for c in cases:
|
||
cid = str(c.get("category") or "")
|
||
if cid and cid not in cat_ids:
|
||
cat_ids.append(cid)
|
||
|
||
for cat in cat_ids:
|
||
group = [c for c in active if c.get("category") == cat]
|
||
title = (cats_meta.get(cat) or {}).get("title") or (
|
||
group[0].get("_file_title") if group else ""
|
||
)
|
||
row = [
|
||
cat,
|
||
str(title),
|
||
str(len(group)),
|
||
str(sum(1 for c in group if test_type(c) == "journey")),
|
||
str(sum(1 for c in group if test_type(c) == "integration")),
|
||
str(sum(1 for c in group if c.get("tag") == "smoke")),
|
||
]
|
||
for risk in risks:
|
||
row.append(str(sum(1 for c in group if c.get("risk") == risk)))
|
||
for driver in DRIVERS:
|
||
row.append(str(sum(1 for c in group if c.get("driver") == driver)))
|
||
print("| " + " | ".join(row) + " |")
|
||
print()
|
||
print("跑法: 跑 <分类> | 跑 <id,id> | 跑 smoke | 跑 <分类> 风险到 <risk>")
|
||
vars_map = resolve_vars(root, meta)
|
||
print(f"HOST: {'已设' if host_set(vars_map) else '未设'}")
|
||
code = run_export_csv(root)
|
||
if code == 0:
|
||
print(f"清单: {catalog_dir(root).relative_to(root) / 'cases.csv'}")
|
||
run_csvs = sorted((catalog_dir(root) / "runs").glob("*.csv"))
|
||
if run_csvs:
|
||
print(f"成绩单: {run_csvs[-1].relative_to(root)}")
|
||
return code
|
||
|
||
|
||
def parse_ids(values: list[str]) -> list[str]:
|
||
out: list[str] = []
|
||
for raw in values:
|
||
for part in str(raw).split(","):
|
||
item = part.strip()
|
||
if item:
|
||
out.append(item)
|
||
return out
|
||
|
||
|
||
def public_case(case: dict[str, Any]) -> dict[str, Any]:
|
||
return {k: v for k, v in case.items() if not str(k).startswith("_")}
|
||
|
||
|
||
PROBE_SOURCES = ("global", "category", "disabled", "none", "invalid")
|
||
|
||
|
||
def effective_status_probe(meta: dict[str, Any], category: str) -> tuple[Any, str]:
|
||
"""分类生效的 status_probe（分类级前置条件）。
|
||
|
||
返回 (probe, source)；probe 为 None 表示该分类不做探测。
|
||
source: global（继承顶层）/ category（分类覆盖）/ disabled（分类置 false）/
|
||
none（顶层和分类都没声明）/ invalid（分类值非法）。
|
||
"""
|
||
cats = {
|
||
str(c.get("id")): c
|
||
for c in meta.get("categories") or []
|
||
if isinstance(c, dict) and c.get("id")
|
||
}
|
||
cat = cats.get(category) or {}
|
||
if "status_probe" in cat:
|
||
value = cat.get("status_probe")
|
||
if value is False:
|
||
return None, "disabled"
|
||
if isinstance(value, dict):
|
||
return value, "category"
|
||
return None, "invalid"
|
||
probe = meta.get("status_probe")
|
||
if isinstance(probe, dict):
|
||
return probe, "global"
|
||
return None, "none"
|
||
|
||
|
||
def resolve_probe(probe: Any, vars_map: dict[str, str]) -> Any:
|
||
"""把探测定义里的 ${VAR} 展开，Agent 拿到就能直接请求。"""
|
||
if not isinstance(probe, dict):
|
||
return probe
|
||
out = dict(probe)
|
||
url = out.get("url")
|
||
if isinstance(url, str):
|
||
out["url"] = substitute(url, vars_map)
|
||
stop_if = out.get("stop_if")
|
||
if isinstance(stop_if, dict):
|
||
out["stop_if"] = {
|
||
(substitute(str(key), vars_map) if "${" in str(key) else key): (
|
||
[
|
||
substitute(str(item), vars_map)
|
||
if isinstance(item, str) and "${" in item
|
||
else item
|
||
for item in value
|
||
]
|
||
if isinstance(value, list)
|
||
else value
|
||
)
|
||
for key, value in stop_if.items()
|
||
}
|
||
return out
|
||
|
||
|
||
def probe_errors(where: str, probe: Any) -> list[str]:
|
||
"""校验一份 status_probe 定义。允许值为 mapping；false 由调用方处理。"""
|
||
if not isinstance(probe, dict):
|
||
return [f"{where}: status_probe 必须是 mapping 或 false"]
|
||
errors: list[str] = []
|
||
url = probe.get("url")
|
||
if not isinstance(url, str) or not url.strip():
|
||
errors.append(f"{where}.url 必填")
|
||
stop_if = probe.get("stop_if")
|
||
if stop_if is not None and not isinstance(stop_if, dict):
|
||
errors.append(f"{where}.stop_if 必须是 mapping（字段 -> 命中值列表）")
|
||
return errors
|
||
|
||
|
||
def agent_errors(root: Path, meta: dict[str, Any]) -> list[str]:
|
||
errors: list[str] = []
|
||
try:
|
||
agent = resolve_agent(root, meta)
|
||
except SystemExit as exc:
|
||
return [str(exc)]
|
||
for key in (
|
||
"environment",
|
||
"host",
|
||
"project",
|
||
"path",
|
||
"role",
|
||
"cli",
|
||
"tier",
|
||
"model",
|
||
"reasoningEffort",
|
||
"permissionMode",
|
||
):
|
||
if key not in agent:
|
||
errors.append(f"agent.{key} 缺失")
|
||
for key in ("environment", "host", "project", "path", "role", "cli", "tier", "permissionMode"):
|
||
if not isinstance(agent.get(key), str):
|
||
errors.append(f"agent.{key} 必须是字符串")
|
||
if agent.get("model") is not None and not isinstance(agent.get("model"), str):
|
||
errors.append("agent.model 必须是字符串或 null")
|
||
if agent.get("role") != "test":
|
||
errors.append("agent.role 必须是 test")
|
||
if agent.get("cli") not in ("codex", "cursor-agent", "grok", "omp", "pi"):
|
||
errors.append("agent.cli 只能是 codex/cursor-agent/grok/omp/pi")
|
||
if agent.get("tier") not in ("standard", "strong"):
|
||
errors.append("agent.tier 只能是 standard 或 strong")
|
||
if agent.get("reasoningEffort") not in (None, "low", "medium", "high", "xhigh"):
|
||
errors.append("agent.reasoningEffort 只能是 low/medium/high/xhigh 或 null")
|
||
if agent.get("permissionMode") not in ("read-only", "workspace-write"):
|
||
errors.append("agent.permissionMode 只能是 read-only 或 workspace-write")
|
||
return errors
|
||
|
||
|
||
def run_together_groups(meta: dict[str, Any]) -> list[dict[str, Any]]:
|
||
raw = meta.get("run_together") or []
|
||
if not isinstance(raw, list):
|
||
return []
|
||
groups: list[dict[str, Any]] = []
|
||
for item in raw:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
ids = [str(x) for x in (item.get("ids") or []) if str(x).strip()]
|
||
if len(ids) < 2:
|
||
continue
|
||
groups.append(
|
||
{
|
||
"ids": ids,
|
||
"continue_on_fail": bool(item.get("continue_on_fail")),
|
||
}
|
||
)
|
||
return groups
|
||
|
||
|
||
def expand_run_together(
|
||
meta: dict[str, Any],
|
||
all_cases: list[dict[str, Any]],
|
||
selected: list[dict[str, Any]],
|
||
skipped: list[dict[str, str]],
|
||
max_i: int,
|
||
automation_filter: str = "automated",
|
||
bypass_risk: bool = False,
|
||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||
groups = run_together_groups(meta)
|
||
if not groups:
|
||
return selected, []
|
||
by_id = {str(c.get("id") or ""): c for c in all_cases}
|
||
selected_ids = [str(c.get("id") or "") for c in selected]
|
||
selected_map = {str(c.get("id") or ""): c for c in selected}
|
||
hit_groups: list[dict[str, Any]] = []
|
||
|
||
for group in groups:
|
||
ids = group["ids"]
|
||
if not any(cid in selected_map for cid in ids):
|
||
continue
|
||
hit_groups.append(group)
|
||
first_pos = min(
|
||
(selected_ids.index(cid) for cid in ids if cid in selected_ids),
|
||
default=len(selected_ids),
|
||
)
|
||
members: list[dict[str, Any]] = []
|
||
for cid in ids:
|
||
if cid in selected_map:
|
||
members.append(selected_map[cid])
|
||
continue
|
||
case = by_id.get(cid)
|
||
if not case or case.get("lifecycle", "active") != "active":
|
||
skipped.append({"id": cid, "reason": "run_together 搭档不是 active"})
|
||
continue
|
||
automation = str(case.get("automation") or "automated")
|
||
if automation_filter != "any" and automation != automation_filter:
|
||
skipped.append({"id": cid, "reason": f"run_together 搭档 automation={automation}（需人工）"})
|
||
continue
|
||
risk = str(case.get("risk") or "readonly")
|
||
try:
|
||
r_i = risk_index(meta, risk)
|
||
except SystemExit:
|
||
skipped.append({"id": cid, "reason": f"未知风险 {risk}"})
|
||
continue
|
||
if not bypass_risk and r_i > max_i:
|
||
skipped.append({"id": cid, "reason": f"risk={risk} 超过上限"})
|
||
continue
|
||
members.append(public_case(case))
|
||
drop = set(ids)
|
||
new_selected: list[dict[str, Any]] = []
|
||
inserted = False
|
||
for case in selected:
|
||
cid = str(case.get("id") or "")
|
||
if cid in drop:
|
||
if not inserted:
|
||
new_selected.extend(members)
|
||
inserted = True
|
||
continue
|
||
new_selected.append(case)
|
||
if not inserted:
|
||
new_selected.extend(members)
|
||
selected = new_selected
|
||
selected_ids = [str(c.get("id") or "") for c in selected]
|
||
selected_map = {str(c.get("id") or ""): c for c in selected}
|
||
|
||
return selected, hit_groups
|
||
|
||
|
||
def parse_platform_pairs(values: list[str], label: str) -> tuple[dict[str, str], list[str]]:
|
||
pairs: dict[str, str] = {}
|
||
errors: list[str] = []
|
||
for raw in values:
|
||
platform, sep, value = raw.partition("=")
|
||
platform = platform.strip().lower()
|
||
value = value.strip()
|
||
if not sep or platform not in ("android", "ios") or not value:
|
||
errors.append(f"{label} 格式必须是 android=<值> 或 ios=<值>: {raw}")
|
||
continue
|
||
if platform in pairs:
|
||
errors.append(f"{label} 重复平台: {platform}")
|
||
continue
|
||
pairs[platform] = value
|
||
return pairs, errors
|
||
|
||
|
||
def safe_locator(locator: str) -> str:
|
||
parsed = urllib.parse.urlsplit(locator)
|
||
if parsed.scheme in ("http", "https"):
|
||
path = parsed.path or "/"
|
||
return f"{parsed.scheme}://{parsed.netloc}{path}"
|
||
return locator
|
||
|
||
|
||
def artifact_ready(locator: str) -> tuple[bool, str]:
|
||
parsed = urllib.parse.urlsplit(locator)
|
||
if parsed.scheme in ("http", "https"):
|
||
try:
|
||
request = urllib.request.Request(locator, method="HEAD")
|
||
with urllib.request.urlopen(request, timeout=10) as response:
|
||
if 200 <= response.status < 400:
|
||
return True, f"HTTP {response.status}"
|
||
return False, f"HTTP {response.status}"
|
||
except Exception as exc:
|
||
return False, f"下载地址不可达: {type(exc).__name__}: {exc}"
|
||
|
||
path = Path(locator).expanduser()
|
||
if not path.is_file():
|
||
return False, "文件不存在"
|
||
if path.stat().st_size <= 0:
|
||
return False, "文件为空"
|
||
if path.suffix.lower() in (".apk", ".ipa") and not zipfile.is_zipfile(path):
|
||
return False, "不是有效的 APK/IPA zip 产物"
|
||
return True, f"文件 {path.stat().st_size} bytes"
|
||
|
||
|
||
def runtime_binding(root: Path, binding_id: str) -> tuple[bool, str]:
|
||
config_path = root / ".pouch" / "agent.yaml"
|
||
if not config_path.is_file():
|
||
return False, f"缺少 {config_path}"
|
||
try:
|
||
config = load_yaml(config_path) or {}
|
||
except Exception as exc:
|
||
return False, f"读取 agent.yaml 失败: {exc}"
|
||
binding = (config.get("bindings") or {}).get(binding_id)
|
||
if not isinstance(binding, dict):
|
||
return False, "binding 不存在"
|
||
if binding.get("adapter") != "tester":
|
||
return False, f"adapter={binding.get('adapter')!r}，不是 tester"
|
||
location = binding.get("location")
|
||
if not location:
|
||
return False, "未配置 location"
|
||
return True, f"location={location}"
|
||
|
||
|
||
def mobile_preflight(
|
||
root: Path,
|
||
meta: dict[str, Any],
|
||
selected: list[dict[str, Any]],
|
||
platforms_raw: list[str],
|
||
artifacts_raw: list[str],
|
||
bindings_raw: list[str],
|
||
) -> dict[str, Any]:
|
||
mobile_selected = [case for case in selected if case.get("driver") == "mobile"]
|
||
if not mobile_selected:
|
||
return {"required": False, "status": "not-needed", "checks": [], "errors": []}
|
||
|
||
platforms = [item.strip().lower() for item in platforms_raw if item.strip()]
|
||
errors: list[str] = []
|
||
if not platforms:
|
||
errors.append("选中了 mobile 用例，必须显式提供 --platform android/ios")
|
||
if len(platforms) != len(set(platforms)):
|
||
errors.append("--platform 不能重复")
|
||
invalid_platforms = [item for item in platforms if item not in ("android", "ios")]
|
||
if invalid_platforms:
|
||
errors.append("非法平台: " + ", ".join(invalid_platforms))
|
||
|
||
artifacts, pair_errors = parse_platform_pairs(artifacts_raw, "--artifact")
|
||
errors.extend(pair_errors)
|
||
bindings, pair_errors = parse_platform_pairs(bindings_raw, "--binding")
|
||
errors.extend(pair_errors)
|
||
contract = meta.get("artifacts")
|
||
if not isinstance(contract, dict) or not contract:
|
||
errors.append("catalog 缺少 artifacts 产物契约")
|
||
|
||
checks: list[dict[str, Any]] = []
|
||
for platform in platforms:
|
||
if platform not in ("android", "ios"):
|
||
continue
|
||
spec = (contract or {}).get(platform) if isinstance(contract, dict) else None
|
||
if not isinstance(spec, dict):
|
||
errors.append(f"{platform}: catalog.artifacts 未声明")
|
||
if platform not in artifacts:
|
||
errors.append(f"{platform}: 缺少 --artifact {platform}=<路径或下载地址>")
|
||
else:
|
||
ok, detail = artifact_ready(artifacts[platform])
|
||
checks.append({
|
||
"platform": platform,
|
||
"artifact": safe_locator(artifacts[platform]),
|
||
"artifact_ok": ok,
|
||
"artifact_detail": detail,
|
||
})
|
||
if not ok:
|
||
errors.append(f"{platform}: 产物不可用: {detail}")
|
||
|
||
binding_id = bindings.get(platform, f"tester.{platform}")
|
||
ok, detail = runtime_binding(root, binding_id)
|
||
checks.append({
|
||
"platform": platform,
|
||
"binding": binding_id,
|
||
"binding_ok": ok,
|
||
"binding_detail": detail,
|
||
})
|
||
if not ok:
|
||
errors.append(f"{platform}: binding {binding_id} 不可用: {detail}")
|
||
|
||
return {
|
||
"required": True,
|
||
"status": "blocked" if errors else "ready",
|
||
"platforms": platforms,
|
||
"checks": checks,
|
||
"errors": errors,
|
||
}
|
||
|
||
|
||
def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||
meta = load_catalog_meta(root)
|
||
cases = load_all_cases(root)
|
||
execution_profile = resolve_execution_profile(meta, args.execution_profile)
|
||
bypass_risk = execution_profile == "isolated-full"
|
||
max_risk = args.max_risk or meta.get("default_max_risk") or "readonly"
|
||
max_i = risk_index(meta, max_risk)
|
||
automation_filter = args.automation or ("any" if bypass_risk else "automated")
|
||
wanted = set(parse_ids(args.ids or []))
|
||
selected: list[dict[str, Any]] = []
|
||
skipped: list[dict[str, str]] = []
|
||
missing = set(wanted)
|
||
|
||
for case in cases:
|
||
cid = str(case.get("id") or "")
|
||
if case.get("lifecycle", "active") != "active":
|
||
if cid in wanted:
|
||
lifecycle = str(case.get("lifecycle"))
|
||
skipped.append({
|
||
"id": cid,
|
||
"reason": LIFECYCLE_SKIP_REASON.get(lifecycle, lifecycle),
|
||
})
|
||
missing.discard(cid)
|
||
continue
|
||
if wanted:
|
||
if cid not in wanted:
|
||
continue
|
||
missing.discard(cid)
|
||
else:
|
||
if args.category and case.get("category") != args.category:
|
||
continue
|
||
if args.tag and case.get("tag") != args.tag:
|
||
continue
|
||
if args.test_type and test_type(case) != args.test_type:
|
||
continue
|
||
if args.suite and case_suite(case) != args.suite:
|
||
continue
|
||
# A mixed catalog keeps legacy cases runnable. Standard keeps the
|
||
# historical client-only implicit default; isolated-full means
|
||
# exactly what it says and includes every active test type.
|
||
if (
|
||
not bypass_risk
|
||
and not args.test_type
|
||
and not args.suite
|
||
and test_type(case) == "integration"
|
||
):
|
||
continue
|
||
if args.driver and case.get("driver") != args.driver:
|
||
continue
|
||
automation = str(case.get("automation") or "automated")
|
||
if automation_filter != "any" and automation != automation_filter:
|
||
skipped.append({
|
||
"id": cid,
|
||
"reason": f"automation={automation}（需人工），本轮只跑 automation={automation_filter}",
|
||
})
|
||
continue
|
||
risk = str(case.get("risk") or "readonly")
|
||
try:
|
||
r_i = risk_index(meta, risk)
|
||
except SystemExit:
|
||
skipped.append({"id": cid, "reason": f"未知风险 {risk}"})
|
||
continue
|
||
if not bypass_risk and r_i > max_i:
|
||
skipped.append({"id": cid, "reason": f"risk={risk} 超过上限 {max_risk}"})
|
||
continue
|
||
selected.append(public_case(case))
|
||
|
||
if wanted and missing:
|
||
print(
|
||
json.dumps({"error": "找不到 active 用例: " + ", ".join(sorted(missing))}, ensure_ascii=False),
|
||
file=sys.stderr,
|
||
)
|
||
return 2
|
||
|
||
selected, together = expand_run_together(
|
||
meta,
|
||
cases,
|
||
selected,
|
||
skipped,
|
||
max_i,
|
||
automation_filter,
|
||
bypass_risk=bypass_risk,
|
||
)
|
||
|
||
# The historical key is kept in the JSON, but the gate now validates both
|
||
# test types before any caller can start a phase.
|
||
journey_gate = journey_preflight(selected, sessions=meta.get("sessions") or {})
|
||
if journey_gate["status"] == "blocked":
|
||
print(json.dumps({"preflight": journey_gate, "journey_preflight": journey_gate}, ensure_ascii=False, indent=2))
|
||
return 2
|
||
|
||
preflight = mobile_preflight(
|
||
root,
|
||
meta,
|
||
selected,
|
||
args.platform or [],
|
||
args.artifacts or [],
|
||
args.bindings or [],
|
||
)
|
||
if preflight["status"] == "blocked":
|
||
print(json.dumps({"preflight": preflight}, ensure_ascii=False, indent=2))
|
||
return 2
|
||
|
||
# 按分类解析生效的 status_probe，让 Agent 不再自己按 catalog 猜作用范围。
|
||
vars_map = resolve_vars(root, meta)
|
||
status_probes: dict[str, dict[str, Any]] = {}
|
||
for case in selected:
|
||
cat = str(case.get("category") or "")
|
||
if cat and cat not in status_probes:
|
||
probe, source = effective_status_probe(meta, cat)
|
||
status_probes[cat] = {"source": source, "probe": resolve_probe(probe, vars_map)}
|
||
|
||
print(
|
||
json.dumps(
|
||
{
|
||
"max_risk": max_risk,
|
||
"execution_profile": execution_profile,
|
||
"risk_filter": "disabled" if bypass_risk else "enforced",
|
||
"automation": automation_filter,
|
||
"count": len(selected),
|
||
"vars": vars_map,
|
||
"artifacts": meta.get("artifacts") or {},
|
||
"preflight": preflight,
|
||
"journey_preflight": journey_gate,
|
||
"test_types": {
|
||
"journey": sum(1 for case in selected if test_type(case) == "journey"),
|
||
"integration": sum(1 for case in selected if test_type(case) == "integration"),
|
||
},
|
||
"skipped": skipped,
|
||
"run_together": together,
|
||
"status_probe": status_probes,
|
||
"agent": resolve_agent(root, meta),
|
||
"cases": selected,
|
||
},
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
)
|
||
)
|
||
return 0
|
||
|
||
|
||
def cmd_show(root: Path, case_id: str) -> int:
|
||
for case in load_all_cases(root):
|
||
if case.get("id") == case_id:
|
||
public = {k: v for k, v in case.items() if not str(k).startswith("_")}
|
||
print(dump_yaml(public), end="")
|
||
return 0
|
||
print(f"找不到用例: {case_id}", file=sys.stderr)
|
||
return 2
|
||
|
||
|
||
def validate(root: Path) -> list[str]:
|
||
errors: list[str] = []
|
||
meta_path = catalog_dir(root) / "catalog.yaml"
|
||
if not meta_path.is_file():
|
||
return [f"缺少 {meta_path}"]
|
||
try:
|
||
meta = load_catalog_meta(root)
|
||
except SystemExit as exc:
|
||
return [str(exc)]
|
||
errors.extend(agent_errors(root, meta))
|
||
|
||
cats = meta.get("categories")
|
||
if not isinstance(cats, list):
|
||
errors.append("catalog.categories 必须是列表")
|
||
cats = []
|
||
cat_ids = []
|
||
for i, item in enumerate(cats):
|
||
if not isinstance(item, dict) or not item.get("id"):
|
||
errors.append(f"catalog.categories[{i}] 需要 id")
|
||
else:
|
||
cat_ids.append(str(item["id"]))
|
||
if isinstance(item, dict) and "status_probe" in item:
|
||
value = item.get("status_probe")
|
||
if value is not False:
|
||
errors.extend(probe_errors(f"catalog.categories[{i}].status_probe", value))
|
||
if "status_probe" in meta:
|
||
errors.extend(probe_errors("catalog.status_probe", meta.get("status_probe")))
|
||
risks = meta.get("risks")
|
||
if not isinstance(risks, list) or not risks:
|
||
errors.append("catalog.risks 必须是非空列表")
|
||
risks = []
|
||
if risks and risks[0] != "readonly":
|
||
errors.append("catalog.risks 必须以 readonly 开头")
|
||
last_pos = -1
|
||
for name in risks:
|
||
if name not in CANONICAL_RISKS:
|
||
errors.append(
|
||
f"非法 risk 名 {name!r}，只能是 readonly/mutating/destructive"
|
||
)
|
||
continue
|
||
pos = CANONICAL_RISKS.index(name)
|
||
if pos <= last_pos:
|
||
errors.append("catalog.risks 必须按 readonly < mutating < destructive 升序且不重复")
|
||
last_pos = pos
|
||
default_max = meta.get("default_max_risk")
|
||
if default_max not in risks:
|
||
errors.append(f"default_max_risk={default_max!r} 不在 risks 中")
|
||
profile = meta.get("execution_profile", "standard")
|
||
if profile not in EXECUTION_PROFILES:
|
||
errors.append(
|
||
f"execution_profile={profile!r} 非法，只能是 {', '.join(EXECUTION_PROFILES)}"
|
||
)
|
||
isolated_full = profile == "isolated-full"
|
||
gates = meta.get("gates") or {}
|
||
if gates and not isinstance(gates, dict):
|
||
errors.append("catalog.gates 必须是 mapping")
|
||
gates = {}
|
||
authorize = meta.get("authorize") or {}
|
||
if authorize and not isinstance(authorize, dict):
|
||
errors.append("catalog.authorize 必须是 mapping")
|
||
authorize = {}
|
||
sessions = meta.get("sessions") or {}
|
||
if not isinstance(sessions, dict):
|
||
errors.append("catalog.sessions 必须是 mapping")
|
||
sessions = {}
|
||
client_suites = meta.get("client_suites")
|
||
if client_suites is not None:
|
||
if not isinstance(client_suites, dict):
|
||
errors.append("catalog.client_suites 必须是 mapping")
|
||
else:
|
||
unknown = sorted(set(client_suites) - set(CLIENT_SUITES))
|
||
if unknown:
|
||
errors.append("catalog.client_suites 含非法套件: " + ", ".join(unknown))
|
||
missing_suites = sorted(set(CLIENT_SUITES) - set(client_suites))
|
||
if missing_suites:
|
||
errors.append("catalog.client_suites 缺少标准套件: " + ", ".join(missing_suites))
|
||
integration_suites = meta.get("integration_suites")
|
||
if integration_suites is not None:
|
||
if not isinstance(integration_suites, dict):
|
||
errors.append("catalog.integration_suites 必须是 mapping")
|
||
else:
|
||
unknown = sorted(set(integration_suites) - set(INTEGRATION_SUITES))
|
||
if unknown:
|
||
errors.append("catalog.integration_suites 含非法套件: " + ", ".join(unknown))
|
||
missing_suites = sorted(set(INTEGRATION_SUITES) - set(integration_suites))
|
||
if missing_suites:
|
||
errors.append("catalog.integration_suites 缺少标准套件: " + ", ".join(missing_suites))
|
||
artifacts = meta.get("artifacts")
|
||
if artifacts is not None:
|
||
if not isinstance(artifacts, dict):
|
||
errors.append("catalog.artifacts 必须是 mapping")
|
||
else:
|
||
for platform, spec in artifacts.items():
|
||
where = f"catalog.artifacts.{platform}"
|
||
if platform not in ("android", "ios"):
|
||
errors.append(f"{where}: 平台只能是 android/ios")
|
||
if not isinstance(spec, dict):
|
||
errors.append(f"{where}: 必须是 mapping")
|
||
continue
|
||
if "required" in spec and not isinstance(spec["required"], bool):
|
||
errors.append(f"{where}.required 必须是 boolean")
|
||
if platform == "android" and not spec.get("package"):
|
||
errors.append(f"{where}: Android 需要 package")
|
||
if platform == "ios" and not spec.get("bundle_id"):
|
||
errors.append(f"{where}: iOS 需要 bundle_id")
|
||
if platform == "android" and spec.get("bundle_id"):
|
||
errors.append(f"{where}: Android 不应填 bundle_id")
|
||
if platform == "ios" and spec.get("package"):
|
||
errors.append(f"{where}: iOS 不应填 package")
|
||
pattern = meta.get("id_pattern") or r"^.+$"
|
||
try:
|
||
id_re = re.compile(pattern)
|
||
except re.error as exc:
|
||
errors.append(f"id_pattern 无效: {exc}")
|
||
id_re = re.compile(r"^.+$")
|
||
|
||
seen: dict[str, str] = {}
|
||
try:
|
||
cases = load_all_cases(root)
|
||
except SystemExit as exc:
|
||
return errors + [str(exc)]
|
||
|
||
together_raw = meta.get("run_together")
|
||
if together_raw is None:
|
||
together_raw = []
|
||
if not isinstance(together_raw, list):
|
||
errors.append("catalog.run_together 必须是列表")
|
||
together_raw = []
|
||
together_seen: set[str] = set()
|
||
for i, item in enumerate(together_raw):
|
||
where = f"catalog.run_together[{i}]"
|
||
if not isinstance(item, dict):
|
||
errors.append(f"{where}: 必须是 mapping，含 ids")
|
||
continue
|
||
ids = item.get("ids")
|
||
if not isinstance(ids, list) or len(ids) < 2:
|
||
errors.append(f"{where}: ids 至少两个")
|
||
continue
|
||
for cid in ids:
|
||
cid_s = str(cid)
|
||
if cid_s in together_seen:
|
||
errors.append(f"{where}: id {cid_s} 已在另一组 run_together")
|
||
together_seen.add(cid_s)
|
||
|
||
for case in cases:
|
||
cid = str(case.get("id") or "")
|
||
where = f"{case.get('_file')}:{cid or '?'}"
|
||
if not cid:
|
||
errors.append(f"{where}: 缺少 id")
|
||
continue
|
||
if not id_re.match(cid):
|
||
errors.append(f"{where}: id 不匹配 {pattern}")
|
||
if cid in seen:
|
||
errors.append(f"{where}: id 重复（也出现在 {seen[cid]}）")
|
||
seen[cid] = str(case.get("_file"))
|
||
if cid in together_seen:
|
||
together_seen.discard(cid)
|
||
cat = str(case.get("category") or "")
|
||
file_cat = str(case.get("_file_category") or "")
|
||
if file_cat and cat and cat != file_cat:
|
||
errors.append(f"{where}: category={cat} 与文件 category={file_cat} 不一致")
|
||
if cat and cat_ids and cat not in cat_ids:
|
||
errors.append(f"{where}: category={cat} 不在 catalog.categories")
|
||
errors.extend(contract_errors(case, sessions=sessions))
|
||
if case.get("lifecycle", "active") not in LIFECYCLES:
|
||
errors.append(f"{where}: 非法 lifecycle")
|
||
if case.get("tag") not in TAGS:
|
||
errors.append(f"{where}: 非法 tag")
|
||
if case.get("risk") not in risks:
|
||
errors.append(f"{where}: 非法 risk={case.get('risk')}")
|
||
if case.get("tag") == "smoke" and case.get("risk") != "readonly":
|
||
errors.append(f"{where}: tag=smoke 只能 risk=readonly")
|
||
gate = case.get("gate")
|
||
if gate:
|
||
if gate not in gates:
|
||
errors.append(f"{where}: gate={gate} 未在 catalog.gates 声明")
|
||
if case.get("risk") in (None, "readonly"):
|
||
errors.append(f"{where}: risk=readonly 不要填 gate")
|
||
risk = case.get("risk")
|
||
if risk and risk != "readonly" and risk not in authorize and not gate and not isolated_full:
|
||
errors.append(f"{where}: {risk} 需要 authorize.{risk} 或 gate")
|
||
if isolated_full and case.get("lifecycle", "active") == "active" and risk != "readonly":
|
||
cleanup = case.get("cleanup")
|
||
cleanup_ok = isinstance(cleanup, dict) and (
|
||
bool(cleanup.get("steps"))
|
||
or bool(str(cleanup.get("run") or "").strip())
|
||
or (cleanup.get("required") is False and bool(str(cleanup.get("reason") or "").strip()))
|
||
)
|
||
if not cleanup_ok:
|
||
errors.append(f"{where}: isolated-full 的写入型用例必须声明 cleanup/rollback steps/run")
|
||
automation = str(case.get("automation") or "automated")
|
||
if automation not in AUTOMATIONS:
|
||
errors.append(f"{where}: 非法 automation={case.get('automation')}")
|
||
automation = "automated"
|
||
if not str(case.get("title") or "").strip():
|
||
errors.append(f"{where}: 缺少 title")
|
||
if not str(case.get("what") or "").strip():
|
||
errors.append(f"{where}: 缺少 what")
|
||
how = case.get("how")
|
||
if not isinstance(how, dict):
|
||
errors.append(f"{where}: how 必须是 mapping")
|
||
how = {}
|
||
steps = how.get("steps") or []
|
||
if automation == "manual":
|
||
if case.get("driver"):
|
||
errors.append(f"{where}: automation=manual 不要填 driver")
|
||
if how.get("run"):
|
||
errors.append(f"{where}: automation=manual 不要填 how.run")
|
||
if not steps:
|
||
errors.append(f"{where}: automation=manual 需要 how.steps")
|
||
else:
|
||
driver = case.get("driver")
|
||
if driver not in DRIVERS:
|
||
errors.append(f"{where}: 非法 driver={driver}")
|
||
if driver == "script" and not how.get("run") and not steps:
|
||
errors.append(f"{where}: script 需要 how.run 或 how.steps")
|
||
if driver in ("browser", "mobile"):
|
||
session = case.get("session")
|
||
if not session:
|
||
errors.append(f"{where}: {driver} 需要 session")
|
||
elif session not in sessions:
|
||
errors.append(f"{where}: session={session} 未在 catalog.sessions 声明")
|
||
if driver == "browser" and not how.get("url") and not steps:
|
||
errors.append(f"{where}: browser 需要 how.url 或 how.steps")
|
||
if driver == "mobile" and not steps:
|
||
errors.append(f"{where}: mobile 需要 how.steps")
|
||
if automation == "assisted":
|
||
if not steps:
|
||
errors.append(f"{where}: automation=assisted 需要 how.steps")
|
||
elif not any(str(step).startswith("[人工]") for step in steps):
|
||
errors.append(f"{where}: automation=assisted 的 how.steps 至少一步以 [人工] 开头")
|
||
assume = case.get("assume")
|
||
if assume is not None:
|
||
if not isinstance(assume, str) or not assume.strip():
|
||
errors.append(f"{where}: assume 必须是非空字符串")
|
||
else:
|
||
forbidden: set[str] = set()
|
||
risk_name = str(case.get("risk") or "")
|
||
if risk_name and risk_name != "readonly":
|
||
env = authorize.get(risk_name)
|
||
if env:
|
||
forbidden.add(str(env))
|
||
if gate and gate in gates:
|
||
forbidden.add(str(gates[gate]))
|
||
for var in sorted(forbidden):
|
||
if re.search(rf"\b{re.escape(var)}\b", assume):
|
||
errors.append(f"{where}: assume 重复了 {var}（已由 risk/gate 授权）")
|
||
expected = case.get("expected")
|
||
if not isinstance(expected, list) or not expected:
|
||
errors.append(f"{where}: expected 必须是非空列表")
|
||
else:
|
||
for j, sig in enumerate(expected):
|
||
if not isinstance(sig, dict) or not sig.get("kind") or not sig.get("value"):
|
||
errors.append(f"{where}: expected[{j}] 需要 kind 与 value")
|
||
elif sig.get("kind") not in EXPECTED_KINDS:
|
||
errors.append(f"{where}: expected[{j}].kind 非法")
|
||
if together_seen:
|
||
errors.append("catalog.run_together 引用了不存在的 id: " + ", ".join(sorted(together_seen)))
|
||
return errors
|
||
|
||
|
||
def cmd_validate(root: Path) -> int:
|
||
errors = validate(root)
|
||
if errors:
|
||
for err in errors:
|
||
print(f"✗ {err}", file=sys.stderr)
|
||
print(f"validate: {len(errors)} 个问题", file=sys.stderr)
|
||
return 1
|
||
cases = load_all_cases(root)
|
||
active = sum(1 for c in cases if c.get("lifecycle", "active") == "active")
|
||
pending = sum(1 for c in cases if c.get("lifecycle") == "pending")
|
||
print(f"✓ validate 通过 active={active} pending={pending} total={len(cases)}")
|
||
return 0
|
||
|
||
|
||
def cmd_export_csv(root: Path) -> int:
|
||
return run_export_csv(root)
|
||
|
||
|
||
def cmd_roots(root: Path) -> int:
|
||
"""List independent tester catalog roots below a repository."""
|
||
roots = discover_catalog_roots(root)
|
||
entries: list[dict[str, str]] = []
|
||
for item in roots:
|
||
project = ""
|
||
try:
|
||
meta = load_catalog_meta(item)
|
||
project = str((meta.get("project") or {}).get("name") or "")
|
||
except (SystemExit, AttributeError):
|
||
pass
|
||
entries.append(
|
||
{
|
||
"root": str(item),
|
||
"catalog": str(catalog_path(item)),
|
||
"project": project,
|
||
"relative": str(item.relative_to(root.resolve())) if item != root.resolve() else ".",
|
||
}
|
||
)
|
||
print(json.dumps({"base": str(root.resolve()), "count": len(entries), "roots": entries}, ensure_ascii=False, indent=2))
|
||
return 0
|
||
|
||
|
||
def cmd_next_id(root: Path, category: str) -> int:
|
||
meta = load_catalog_meta(root)
|
||
cat_ids = [c.get("id") for c in meta.get("categories") or [] if isinstance(c, dict)]
|
||
if category not in cat_ids:
|
||
print(f"未知分类: {category} 可选: {cat_ids}", file=sys.stderr)
|
||
return 2
|
||
prefix = f"TC-{category}-"
|
||
max_n = 0
|
||
for case in load_all_cases(root):
|
||
cid = str(case.get("id") or "")
|
||
if case.get("category") != category:
|
||
continue
|
||
match = re.match(rf"TC-{re.escape(category)}-(\d+)", cid)
|
||
if match:
|
||
max_n = max(max_n, int(match.group(1)))
|
||
print(f"{prefix}{max_n + 1:03d}")
|
||
return 0
|
||
|
||
|
||
def authorized(
|
||
vars_map: dict[str, str],
|
||
meta: dict[str, Any],
|
||
case: dict[str, Any],
|
||
*,
|
||
execution_profile: str | None = None,
|
||
) -> bool:
|
||
profile = resolve_execution_profile(meta, execution_profile)
|
||
if profile == "isolated-full":
|
||
# AUTHORIZE_* and gate variables are intentionally not run-time
|
||
# switches for an isolated resettable environment. Capability and
|
||
# cleanup preflight are the safety boundary for this profile.
|
||
return True
|
||
risk = str(case.get("risk") or "readonly")
|
||
risks = list(meta.get("risks") or ["readonly"])
|
||
lowest = risks[0] if risks else "readonly"
|
||
if risk != lowest:
|
||
env_name = (meta.get("authorize") or {}).get(risk)
|
||
if env_name and not truthy(vars_map.get(str(env_name), "")):
|
||
return False
|
||
if not env_name and not case.get("gate"):
|
||
return False
|
||
gate = case.get("gate")
|
||
if gate:
|
||
gate_env = (meta.get("gates") or {}).get(gate)
|
||
if not gate_env or not truthy(vars_map.get(str(gate_env), "")):
|
||
return False
|
||
return True
|
||
|
||
|
||
def cmd_run_script(root: Path, case_id: str) -> int:
|
||
meta = load_catalog_meta(root)
|
||
execution_profile = resolve_execution_profile(meta)
|
||
vars_map = resolve_vars(root, meta)
|
||
case = next((c for c in load_all_cases(root) if c.get("id") == case_id), None)
|
||
if not case:
|
||
print(json.dumps({"id": case_id, "result": "environment-fail", "evidence": "找不到用例"}, ensure_ascii=False))
|
||
return 2
|
||
public_id = case.get("id")
|
||
if case.get("lifecycle", "active") != "active":
|
||
lifecycle = str(case.get("lifecycle"))
|
||
reason = LIFECYCLE_SKIP_REASON.get(lifecycle, lifecycle)
|
||
print(json.dumps({"id": public_id, "result": "skipped", "evidence": reason}, ensure_ascii=False))
|
||
return 0
|
||
automation = str(case.get("automation") or "automated")
|
||
if automation != "automated":
|
||
print(
|
||
json.dumps(
|
||
{"id": public_id, "result": "not-run", "evidence": f"automation={automation}，需人工，run-script 不执行"},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
return 0
|
||
if case.get("driver") != "script":
|
||
print(
|
||
json.dumps(
|
||
{"id": public_id, "result": "environment-fail", "evidence": "run-script 只能跑 driver=script"},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
return 2
|
||
if not host_set(vars_map):
|
||
print(
|
||
json.dumps(
|
||
{"id": public_id, "result": "environment-fail", "evidence": "HOST 未设"},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
return 2
|
||
risk = str(case.get("risk") or "readonly")
|
||
if not authorized(vars_map, meta, case, execution_profile=execution_profile):
|
||
print(
|
||
json.dumps(
|
||
{"id": public_id, "result": "skipped", "evidence": f"风险 {risk} 未授权", "execution_profile": execution_profile},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
return 0
|
||
how = case.get("how") or {}
|
||
run = how.get("run")
|
||
if not run:
|
||
print(
|
||
json.dumps(
|
||
{"id": public_id, "result": "environment-fail", "evidence": "没有 how.run，改由 Agent 按 steps 执行"},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
return 2
|
||
script = substitute(str(run), vars_map)
|
||
timeout = float(how.get("timeout") or 60)
|
||
env = os.environ.copy()
|
||
env.update({k: v for k, v in vars_map.items() if v is not None})
|
||
try:
|
||
proc = subprocess.run(
|
||
script,
|
||
shell=True,
|
||
cwd=str(root),
|
||
env=env,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=timeout,
|
||
check=False,
|
||
)
|
||
except subprocess.TimeoutExpired:
|
||
print(
|
||
json.dumps(
|
||
{
|
||
"id": public_id,
|
||
"result": "environment-fail",
|
||
"evidence": f"timeout {timeout}s",
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
return 1
|
||
output = ((proc.stdout or "") + ("\n" + proc.stderr if proc.stderr else "")).strip()
|
||
evidence = output[-1200:]
|
||
print(
|
||
json.dumps(
|
||
{
|
||
"id": public_id,
|
||
"result": "executed",
|
||
"execution_profile": execution_profile,
|
||
"exit": proc.returncode,
|
||
"evidence": evidence,
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
return 0
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parent = argparse.ArgumentParser(add_help=False)
|
||
parent.add_argument("--root", default=".", help="项目根")
|
||
parser = argparse.ArgumentParser(description="tester catalog")
|
||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||
|
||
sub.add_parser("list", help="按分类统计", parents=[parent])
|
||
p_sel = sub.add_parser("select", help="筛选用例 JSON", parents=[parent])
|
||
p_sel.add_argument("--category")
|
||
p_sel.add_argument("--id", action="append", dest="ids", default=[])
|
||
p_sel.add_argument("--tag", choices=TAGS)
|
||
p_sel.add_argument(
|
||
"--suite", "--client-suite", dest="suite",
|
||
choices=[*CLIENT_SUITES, *INTEGRATION_SUITES],
|
||
help="客户端或集成测试套件；不传时默认不选 integration",
|
||
)
|
||
p_sel.add_argument(
|
||
"--test-type", choices=TEST_TYPES, default=None,
|
||
help="按顶层 test_type 筛选；integration 必须显式选择",
|
||
)
|
||
p_sel.add_argument("--max-risk")
|
||
p_sel.add_argument(
|
||
"--execution-profile",
|
||
choices=EXECUTION_PROFILES,
|
||
default=None,
|
||
help="执行策略；isolated-full 在隔离可重置环境中不按 risk/authorize 筛选",
|
||
)
|
||
p_sel.add_argument("--driver", choices=DRIVERS)
|
||
p_sel.add_argument(
|
||
"--automation",
|
||
choices=[*AUTOMATIONS, "any"],
|
||
default=None,
|
||
help="默认 automated；any 表示不按人工参与度筛选",
|
||
)
|
||
p_sel.add_argument(
|
||
"--platform",
|
||
action="append",
|
||
choices=["android", "ios"],
|
||
default=[],
|
||
help="mobile 运行平台，可重复；选中 mobile 用例时必填",
|
||
)
|
||
p_sel.add_argument(
|
||
"--artifact",
|
||
action="append",
|
||
dest="artifacts",
|
||
default=[],
|
||
metavar="PLATFORM=PATH_OR_URL",
|
||
help="本轮移动产物，可重复，例如 android=dist/app.apk",
|
||
)
|
||
p_sel.add_argument(
|
||
"--binding",
|
||
action="append",
|
||
dest="bindings",
|
||
default=[],
|
||
metavar="PLATFORM=TESTER_BINDING",
|
||
help="平台 tester binding，可重复；默认 tester.<platform>",
|
||
)
|
||
p_show = sub.add_parser("show", help="打印一条用例", parents=[parent])
|
||
p_show.add_argument("--id", required=True, dest="case_id")
|
||
sub.add_parser("validate", help="校验目录", parents=[parent])
|
||
sub.add_parser("roots", help="发现当前仓库及子项目的 tester 根目录", parents=[parent])
|
||
sub.add_parser("export-csv", help="写出给人看的 cases.csv", parents=[parent])
|
||
p_next = sub.add_parser("next-id", help="分配下一个 TC-<CAT>-NNN", parents=[parent])
|
||
p_next.add_argument("--category", required=True)
|
||
p_run = sub.add_parser("run-script", help="执行一条 how.run", parents=[parent])
|
||
p_run.add_argument("--id", required=True, dest="case_id")
|
||
return parser
|
||
|
||
|
||
def main(argv: list[str] | None = None) -> int:
|
||
parser = build_parser()
|
||
args = parser.parse_args(argv)
|
||
root = Path(args.root).resolve()
|
||
if args.cmd != "roots":
|
||
roots = discover_or_self(root)
|
||
if len(roots) > 1:
|
||
if args.cmd in FANOUT_COMMANDS:
|
||
return cmd_fanout(args, roots)
|
||
print(
|
||
f"发现多个 tester 根目录，命令 {args.cmd} 不是可 fan-out 的执行命令；"
|
||
"请对具体项目根目录执行该维护命令。",
|
||
file=sys.stderr,
|
||
)
|
||
return 2
|
||
if len(roots) == 1:
|
||
root = roots[0]
|
||
if args.cmd == "list":
|
||
return cmd_list(root)
|
||
if args.cmd == "select":
|
||
return cmd_select(args, root)
|
||
if args.cmd == "show":
|
||
return cmd_show(root, args.case_id)
|
||
if args.cmd == "validate":
|
||
return cmd_validate(root)
|
||
if args.cmd == "roots":
|
||
return cmd_roots(root)
|
||
if args.cmd == "export-csv":
|
||
return cmd_export_csv(root)
|
||
if args.cmd == "next-id":
|
||
return cmd_next_id(root, args.category)
|
||
if args.cmd == "run-script":
|
||
return cmd_run_script(root, args.case_id)
|
||
parser.error(f"未知命令 {args.cmd}")
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|