feat(tester): support isolated full regression workflow
This commit is contained in:
@@ -42,6 +42,7 @@ LIFECYCLE_SKIP_REASON = {
|
||||
}
|
||||
TAGS = ("smoke", "full", "slow")
|
||||
CANONICAL_RISKS = ("readonly", "mutating", "destructive")
|
||||
EXECUTION_PROFILES = ("standard", "isolated-full")
|
||||
AUTOMATIONS = ("automated", "assisted", "manual")
|
||||
RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run")
|
||||
EXPECTED_KINDS = (
|
||||
@@ -351,6 +352,21 @@ def risk_index(meta: dict[str, Any], name: str) -> int:
|
||||
return risks.index(name)
|
||||
|
||||
|
||||
def resolve_execution_profile(meta: dict[str, Any], requested: str | None = None) -> str:
|
||||
"""Resolve the run profile without changing the meaning of ``risk``.
|
||||
|
||||
``isolated-full`` is an execution policy for a resettable test environment:
|
||||
it disables risk/authorization filtering, but leaves the catalog risk value
|
||||
on every case so reports still describe the side effect that was exercised.
|
||||
"""
|
||||
profile = requested or meta.get("execution_profile") or "standard"
|
||||
if profile not in EXECUTION_PROFILES:
|
||||
raise SystemExit(
|
||||
f"未知 execution_profile: {profile!r}(可选: {', '.join(EXECUTION_PROFILES)})"
|
||||
)
|
||||
return str(profile)
|
||||
|
||||
|
||||
def host_set(vars_map: dict[str, str]) -> bool:
|
||||
host = (vars_map.get("HOST") or "").strip()
|
||||
return bool(host) and "${HOST}" not in host
|
||||
@@ -570,6 +586,7 @@ def expand_run_together(
|
||||
skipped: list[dict[str, str]],
|
||||
max_i: int,
|
||||
automation_filter: str = "automated",
|
||||
bypass_risk: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
groups = run_together_groups(meta)
|
||||
if not groups:
|
||||
@@ -607,7 +624,7 @@ def expand_run_together(
|
||||
except SystemExit:
|
||||
skipped.append({"id": cid, "reason": f"未知风险 {risk}"})
|
||||
continue
|
||||
if r_i > max_i:
|
||||
if not bypass_risk and r_i > max_i:
|
||||
skipped.append({"id": cid, "reason": f"risk={risk} 超过上限"})
|
||||
continue
|
||||
members.append(public_case(case))
|
||||
@@ -770,9 +787,11 @@ def mobile_preflight(
|
||||
def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
meta = load_catalog_meta(root)
|
||||
cases = load_all_cases(root)
|
||||
execution_profile = resolve_execution_profile(meta, args.execution_profile)
|
||||
bypass_risk = execution_profile == "isolated-full"
|
||||
max_risk = args.max_risk or meta.get("default_max_risk") or "readonly"
|
||||
max_i = risk_index(meta, max_risk)
|
||||
automation_filter = args.automation or "automated"
|
||||
automation_filter = args.automation or ("any" if bypass_risk else "automated")
|
||||
wanted = set(parse_ids(args.ids or []))
|
||||
selected: list[dict[str, Any]] = []
|
||||
skipped: list[dict[str, str]] = []
|
||||
@@ -802,9 +821,15 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
continue
|
||||
if args.suite and case_suite(case) != args.suite:
|
||||
continue
|
||||
# A mixed catalog keeps legacy cases runnable, but the implicit
|
||||
# default is the client journey surface. Integration is opt-in.
|
||||
if not args.test_type and not args.suite and test_type(case) == "integration":
|
||||
# A mixed catalog keeps legacy cases runnable. Standard keeps the
|
||||
# historical client-only implicit default; isolated-full means
|
||||
# exactly what it says and includes every active test type.
|
||||
if (
|
||||
not bypass_risk
|
||||
and not args.test_type
|
||||
and not args.suite
|
||||
and test_type(case) == "integration"
|
||||
):
|
||||
continue
|
||||
if args.driver and case.get("driver") != args.driver:
|
||||
continue
|
||||
@@ -821,7 +846,7 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
except SystemExit:
|
||||
skipped.append({"id": cid, "reason": f"未知风险 {risk}"})
|
||||
continue
|
||||
if r_i > max_i:
|
||||
if not bypass_risk and r_i > max_i:
|
||||
skipped.append({"id": cid, "reason": f"risk={risk} 超过上限 {max_risk}"})
|
||||
continue
|
||||
selected.append(public_case(case))
|
||||
@@ -833,7 +858,15 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
)
|
||||
return 2
|
||||
|
||||
selected, together = expand_run_together(meta, cases, selected, skipped, max_i, automation_filter)
|
||||
selected, together = expand_run_together(
|
||||
meta,
|
||||
cases,
|
||||
selected,
|
||||
skipped,
|
||||
max_i,
|
||||
automation_filter,
|
||||
bypass_risk=bypass_risk,
|
||||
)
|
||||
|
||||
# The historical key is kept in the JSON, but the gate now validates both
|
||||
# test types before any caller can start a phase.
|
||||
@@ -867,6 +900,8 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
json.dumps(
|
||||
{
|
||||
"max_risk": max_risk,
|
||||
"execution_profile": execution_profile,
|
||||
"risk_filter": "disabled" if bypass_risk else "enforced",
|
||||
"automation": automation_filter,
|
||||
"count": len(selected),
|
||||
"vars": vars_map,
|
||||
@@ -947,6 +982,12 @@ def validate(root: Path) -> list[str]:
|
||||
default_max = meta.get("default_max_risk")
|
||||
if default_max not in risks:
|
||||
errors.append(f"default_max_risk={default_max!r} 不在 risks 中")
|
||||
profile = meta.get("execution_profile", "standard")
|
||||
if profile not in EXECUTION_PROFILES:
|
||||
errors.append(
|
||||
f"execution_profile={profile!r} 非法,只能是 {', '.join(EXECUTION_PROFILES)}"
|
||||
)
|
||||
isolated_full = profile == "isolated-full"
|
||||
gates = meta.get("gates") or {}
|
||||
if gates and not isinstance(gates, dict):
|
||||
errors.append("catalog.gates 必须是 mapping")
|
||||
@@ -1073,8 +1114,17 @@ def validate(root: Path) -> list[str]:
|
||||
if case.get("risk") in (None, "readonly"):
|
||||
errors.append(f"{where}: risk=readonly 不要填 gate")
|
||||
risk = case.get("risk")
|
||||
if risk and risk != "readonly" and risk not in authorize and not gate:
|
||||
if risk and risk != "readonly" and risk not in authorize and not gate and not isolated_full:
|
||||
errors.append(f"{where}: {risk} 需要 authorize.{risk} 或 gate")
|
||||
if isolated_full and case.get("lifecycle", "active") == "active" and risk != "readonly":
|
||||
cleanup = case.get("cleanup")
|
||||
cleanup_ok = isinstance(cleanup, dict) and (
|
||||
bool(cleanup.get("steps"))
|
||||
or bool(str(cleanup.get("run") or "").strip())
|
||||
or (cleanup.get("required") is False and bool(str(cleanup.get("reason") or "").strip()))
|
||||
)
|
||||
if not cleanup_ok:
|
||||
errors.append(f"{where}: isolated-full 的写入型用例必须声明 cleanup/rollback steps/run")
|
||||
automation = str(case.get("automation") or "automated")
|
||||
if automation not in AUTOMATIONS:
|
||||
errors.append(f"{where}: 非法 automation={case.get('automation')}")
|
||||
@@ -1206,7 +1256,19 @@ def cmd_next_id(root: Path, category: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def authorized(vars_map: dict[str, str], meta: dict[str, Any], case: dict[str, Any]) -> bool:
|
||||
def authorized(
|
||||
vars_map: dict[str, str],
|
||||
meta: dict[str, Any],
|
||||
case: dict[str, Any],
|
||||
*,
|
||||
execution_profile: str | None = None,
|
||||
) -> bool:
|
||||
profile = resolve_execution_profile(meta, execution_profile)
|
||||
if profile == "isolated-full":
|
||||
# AUTHORIZE_* and gate variables are intentionally not run-time
|
||||
# switches for an isolated resettable environment. Capability and
|
||||
# cleanup preflight are the safety boundary for this profile.
|
||||
return True
|
||||
risk = str(case.get("risk") or "readonly")
|
||||
risks = list(meta.get("risks") or ["readonly"])
|
||||
lowest = risks[0] if risks else "readonly"
|
||||
@@ -1226,6 +1288,7 @@ def authorized(vars_map: dict[str, str], meta: dict[str, Any], case: dict[str, A
|
||||
|
||||
def cmd_run_script(root: Path, case_id: str) -> int:
|
||||
meta = load_catalog_meta(root)
|
||||
execution_profile = resolve_execution_profile(meta)
|
||||
vars_map = resolve_vars(root, meta)
|
||||
case = next((c for c in load_all_cases(root) if c.get("id") == case_id), None)
|
||||
if not case:
|
||||
@@ -1263,10 +1326,10 @@ def cmd_run_script(root: Path, case_id: str) -> int:
|
||||
)
|
||||
return 2
|
||||
risk = str(case.get("risk") or "readonly")
|
||||
if not authorized(vars_map, meta, case):
|
||||
if not authorized(vars_map, meta, case, execution_profile=execution_profile):
|
||||
print(
|
||||
json.dumps(
|
||||
{"id": public_id, "result": "skipped", "evidence": f"风险 {risk} 未授权"},
|
||||
{"id": public_id, "result": "skipped", "evidence": f"风险 {risk} 未授权", "execution_profile": execution_profile},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
@@ -1315,6 +1378,7 @@ def cmd_run_script(root: Path, case_id: str) -> int:
|
||||
{
|
||||
"id": public_id,
|
||||
"result": "executed",
|
||||
"execution_profile": execution_profile,
|
||||
"exit": proc.returncode,
|
||||
"evidence": evidence,
|
||||
},
|
||||
@@ -1345,6 +1409,12 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="按顶层 test_type 筛选;integration 必须显式选择",
|
||||
)
|
||||
p_sel.add_argument("--max-risk")
|
||||
p_sel.add_argument(
|
||||
"--execution-profile",
|
||||
choices=EXECUTION_PROFILES,
|
||||
default=None,
|
||||
help="执行策略;isolated-full 在隔离可重置环境中不按 risk/authorize 筛选",
|
||||
)
|
||||
p_sel.add_argument("--driver", choices=DRIVERS)
|
||||
p_sel.add_argument(
|
||||
"--automation",
|
||||
|
||||
Reference in New Issue
Block a user