feat(tester): support isolated full regression workflow
This commit is contained in:
@@ -42,6 +42,7 @@ LIFECYCLE_SKIP_REASON = {
|
||||
}
|
||||
TAGS = ("smoke", "full", "slow")
|
||||
CANONICAL_RISKS = ("readonly", "mutating", "destructive")
|
||||
EXECUTION_PROFILES = ("standard", "isolated-full")
|
||||
AUTOMATIONS = ("automated", "assisted", "manual")
|
||||
RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run")
|
||||
EXPECTED_KINDS = (
|
||||
@@ -351,6 +352,21 @@ def risk_index(meta: dict[str, Any], name: str) -> int:
|
||||
return risks.index(name)
|
||||
|
||||
|
||||
def resolve_execution_profile(meta: dict[str, Any], requested: str | None = None) -> str:
|
||||
"""Resolve the run profile without changing the meaning of ``risk``.
|
||||
|
||||
``isolated-full`` is an execution policy for a resettable test environment:
|
||||
it disables risk/authorization filtering, but leaves the catalog risk value
|
||||
on every case so reports still describe the side effect that was exercised.
|
||||
"""
|
||||
profile = requested or meta.get("execution_profile") or "standard"
|
||||
if profile not in EXECUTION_PROFILES:
|
||||
raise SystemExit(
|
||||
f"未知 execution_profile: {profile!r}(可选: {', '.join(EXECUTION_PROFILES)})"
|
||||
)
|
||||
return str(profile)
|
||||
|
||||
|
||||
def host_set(vars_map: dict[str, str]) -> bool:
|
||||
host = (vars_map.get("HOST") or "").strip()
|
||||
return bool(host) and "${HOST}" not in host
|
||||
@@ -570,6 +586,7 @@ def expand_run_together(
|
||||
skipped: list[dict[str, str]],
|
||||
max_i: int,
|
||||
automation_filter: str = "automated",
|
||||
bypass_risk: bool = False,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
groups = run_together_groups(meta)
|
||||
if not groups:
|
||||
@@ -607,7 +624,7 @@ def expand_run_together(
|
||||
except SystemExit:
|
||||
skipped.append({"id": cid, "reason": f"未知风险 {risk}"})
|
||||
continue
|
||||
if r_i > max_i:
|
||||
if not bypass_risk and r_i > max_i:
|
||||
skipped.append({"id": cid, "reason": f"risk={risk} 超过上限"})
|
||||
continue
|
||||
members.append(public_case(case))
|
||||
@@ -770,9 +787,11 @@ def mobile_preflight(
|
||||
def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
meta = load_catalog_meta(root)
|
||||
cases = load_all_cases(root)
|
||||
execution_profile = resolve_execution_profile(meta, args.execution_profile)
|
||||
bypass_risk = execution_profile == "isolated-full"
|
||||
max_risk = args.max_risk or meta.get("default_max_risk") or "readonly"
|
||||
max_i = risk_index(meta, max_risk)
|
||||
automation_filter = args.automation or "automated"
|
||||
automation_filter = args.automation or ("any" if bypass_risk else "automated")
|
||||
wanted = set(parse_ids(args.ids or []))
|
||||
selected: list[dict[str, Any]] = []
|
||||
skipped: list[dict[str, str]] = []
|
||||
@@ -802,9 +821,15 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
continue
|
||||
if args.suite and case_suite(case) != args.suite:
|
||||
continue
|
||||
# A mixed catalog keeps legacy cases runnable, but the implicit
|
||||
# default is the client journey surface. Integration is opt-in.
|
||||
if not args.test_type and not args.suite and test_type(case) == "integration":
|
||||
# A mixed catalog keeps legacy cases runnable. Standard keeps the
|
||||
# historical client-only implicit default; isolated-full means
|
||||
# exactly what it says and includes every active test type.
|
||||
if (
|
||||
not bypass_risk
|
||||
and not args.test_type
|
||||
and not args.suite
|
||||
and test_type(case) == "integration"
|
||||
):
|
||||
continue
|
||||
if args.driver and case.get("driver") != args.driver:
|
||||
continue
|
||||
@@ -821,7 +846,7 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
except SystemExit:
|
||||
skipped.append({"id": cid, "reason": f"未知风险 {risk}"})
|
||||
continue
|
||||
if r_i > max_i:
|
||||
if not bypass_risk and r_i > max_i:
|
||||
skipped.append({"id": cid, "reason": f"risk={risk} 超过上限 {max_risk}"})
|
||||
continue
|
||||
selected.append(public_case(case))
|
||||
@@ -833,7 +858,15 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
)
|
||||
return 2
|
||||
|
||||
selected, together = expand_run_together(meta, cases, selected, skipped, max_i, automation_filter)
|
||||
selected, together = expand_run_together(
|
||||
meta,
|
||||
cases,
|
||||
selected,
|
||||
skipped,
|
||||
max_i,
|
||||
automation_filter,
|
||||
bypass_risk=bypass_risk,
|
||||
)
|
||||
|
||||
# The historical key is kept in the JSON, but the gate now validates both
|
||||
# test types before any caller can start a phase.
|
||||
@@ -867,6 +900,8 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
json.dumps(
|
||||
{
|
||||
"max_risk": max_risk,
|
||||
"execution_profile": execution_profile,
|
||||
"risk_filter": "disabled" if bypass_risk else "enforced",
|
||||
"automation": automation_filter,
|
||||
"count": len(selected),
|
||||
"vars": vars_map,
|
||||
@@ -947,6 +982,12 @@ def validate(root: Path) -> list[str]:
|
||||
default_max = meta.get("default_max_risk")
|
||||
if default_max not in risks:
|
||||
errors.append(f"default_max_risk={default_max!r} 不在 risks 中")
|
||||
profile = meta.get("execution_profile", "standard")
|
||||
if profile not in EXECUTION_PROFILES:
|
||||
errors.append(
|
||||
f"execution_profile={profile!r} 非法,只能是 {', '.join(EXECUTION_PROFILES)}"
|
||||
)
|
||||
isolated_full = profile == "isolated-full"
|
||||
gates = meta.get("gates") or {}
|
||||
if gates and not isinstance(gates, dict):
|
||||
errors.append("catalog.gates 必须是 mapping")
|
||||
@@ -1073,8 +1114,17 @@ def validate(root: Path) -> list[str]:
|
||||
if case.get("risk") in (None, "readonly"):
|
||||
errors.append(f"{where}: risk=readonly 不要填 gate")
|
||||
risk = case.get("risk")
|
||||
if risk and risk != "readonly" and risk not in authorize and not gate:
|
||||
if risk and risk != "readonly" and risk not in authorize and not gate and not isolated_full:
|
||||
errors.append(f"{where}: {risk} 需要 authorize.{risk} 或 gate")
|
||||
if isolated_full and case.get("lifecycle", "active") == "active" and risk != "readonly":
|
||||
cleanup = case.get("cleanup")
|
||||
cleanup_ok = isinstance(cleanup, dict) and (
|
||||
bool(cleanup.get("steps"))
|
||||
or bool(str(cleanup.get("run") or "").strip())
|
||||
or (cleanup.get("required") is False and bool(str(cleanup.get("reason") or "").strip()))
|
||||
)
|
||||
if not cleanup_ok:
|
||||
errors.append(f"{where}: isolated-full 的写入型用例必须声明 cleanup/rollback steps/run")
|
||||
automation = str(case.get("automation") or "automated")
|
||||
if automation not in AUTOMATIONS:
|
||||
errors.append(f"{where}: 非法 automation={case.get('automation')}")
|
||||
@@ -1206,7 +1256,19 @@ def cmd_next_id(root: Path, category: str) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def authorized(vars_map: dict[str, str], meta: dict[str, Any], case: dict[str, Any]) -> bool:
|
||||
def authorized(
|
||||
vars_map: dict[str, str],
|
||||
meta: dict[str, Any],
|
||||
case: dict[str, Any],
|
||||
*,
|
||||
execution_profile: str | None = None,
|
||||
) -> bool:
|
||||
profile = resolve_execution_profile(meta, execution_profile)
|
||||
if profile == "isolated-full":
|
||||
# AUTHORIZE_* and gate variables are intentionally not run-time
|
||||
# switches for an isolated resettable environment. Capability and
|
||||
# cleanup preflight are the safety boundary for this profile.
|
||||
return True
|
||||
risk = str(case.get("risk") or "readonly")
|
||||
risks = list(meta.get("risks") or ["readonly"])
|
||||
lowest = risks[0] if risks else "readonly"
|
||||
@@ -1226,6 +1288,7 @@ def authorized(vars_map: dict[str, str], meta: dict[str, Any], case: dict[str, A
|
||||
|
||||
def cmd_run_script(root: Path, case_id: str) -> int:
|
||||
meta = load_catalog_meta(root)
|
||||
execution_profile = resolve_execution_profile(meta)
|
||||
vars_map = resolve_vars(root, meta)
|
||||
case = next((c for c in load_all_cases(root) if c.get("id") == case_id), None)
|
||||
if not case:
|
||||
@@ -1263,10 +1326,10 @@ def cmd_run_script(root: Path, case_id: str) -> int:
|
||||
)
|
||||
return 2
|
||||
risk = str(case.get("risk") or "readonly")
|
||||
if not authorized(vars_map, meta, case):
|
||||
if not authorized(vars_map, meta, case, execution_profile=execution_profile):
|
||||
print(
|
||||
json.dumps(
|
||||
{"id": public_id, "result": "skipped", "evidence": f"风险 {risk} 未授权"},
|
||||
{"id": public_id, "result": "skipped", "evidence": f"风险 {risk} 未授权", "execution_profile": execution_profile},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
@@ -1315,6 +1378,7 @@ def cmd_run_script(root: Path, case_id: str) -> int:
|
||||
{
|
||||
"id": public_id,
|
||||
"result": "executed",
|
||||
"execution_profile": execution_profile,
|
||||
"exit": proc.returncode,
|
||||
"evidence": evidence,
|
||||
},
|
||||
@@ -1345,6 +1409,12 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
help="按顶层 test_type 筛选;integration 必须显式选择",
|
||||
)
|
||||
p_sel.add_argument("--max-risk")
|
||||
p_sel.add_argument(
|
||||
"--execution-profile",
|
||||
choices=EXECUTION_PROFILES,
|
||||
default=None,
|
||||
help="执行策略;isolated-full 在隔离可重置环境中不按 risk/authorize 筛选",
|
||||
)
|
||||
p_sel.add_argument("--driver", choices=DRIVERS)
|
||||
p_sel.add_argument(
|
||||
"--automation",
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build structured tester issues and the development feedback route.
|
||||
|
||||
This module only writes the run contract. Updating a remote PR/issue or
|
||||
notifying a dispatcher remains an explicit caller-side integration step; no
|
||||
network mutation is hidden in the tester.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
|
||||
ISSUE_TYPES = ("product", "test")
|
||||
TRIGGERS = ("pr", "requirement", "bug")
|
||||
ISSUE_RESULTS = frozenset({"fail", "environment-fail"})
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def issue_type_for(result: str) -> str:
|
||||
if result == "fail":
|
||||
return "product"
|
||||
if result == "environment-fail":
|
||||
return "test"
|
||||
raise ValueError(f"只有 fail/environment-fail 可以生成 issue: {result}")
|
||||
|
||||
|
||||
def _value(row: Mapping[str, Any], key: str, defaults: Mapping[str, Any]) -> Any:
|
||||
value = row.get(key)
|
||||
return value if value not in (None, "") else defaults.get(key, "")
|
||||
|
||||
|
||||
def build_issue(
|
||||
row: Mapping[str, Any],
|
||||
*,
|
||||
defaults: Mapping[str, Any] | None = None,
|
||||
issue_type: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create the stable issue shape required by the run report."""
|
||||
defaults = defaults or {}
|
||||
result = str(row.get("result") or "")
|
||||
if result not in ISSUE_RESULTS:
|
||||
raise ValueError(f"不能为 {result!r} 结果创建 defect: {row.get('id')}")
|
||||
selected_type = issue_type or row.get("issue_type") or issue_type_for(result)
|
||||
if selected_type not in ISSUE_TYPES:
|
||||
raise ValueError(f"issue_type 必须是 product/test: {selected_type}")
|
||||
testcase = str(row.get("testcase") or row.get("id") or "")
|
||||
if not testcase:
|
||||
raise ValueError("issue 缺少 testcase")
|
||||
return {
|
||||
"id": str(row.get("issue_id") or f"TEST-{testcase}-{result}"),
|
||||
"testcase": testcase,
|
||||
"issue_type": selected_type,
|
||||
"result": result,
|
||||
"phase": _value(row, "phase", defaults),
|
||||
"expected": _value(row, "expected", defaults),
|
||||
"observed": _value(row, "observed", defaults),
|
||||
"artifact_hash": _value(row, "artifact_hash", defaults),
|
||||
"environment": _value(row, "environment", defaults),
|
||||
"evidence_path": _value(row, "evidence_path", defaults),
|
||||
"reproduction": _value(row, "reproduction", defaults),
|
||||
"next_step": _value(row, "next_step", defaults),
|
||||
"created_at": str(row.get("created_at") or _now()),
|
||||
}
|
||||
|
||||
|
||||
def collect_issues(
|
||||
results: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
defaults: Mapping[str, Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
issues: list[dict[str, Any]] = []
|
||||
for row in results:
|
||||
if not isinstance(row, Mapping) or str(row.get("result") or "") not in ISSUE_RESULTS:
|
||||
continue
|
||||
explicit = row.get("issue")
|
||||
source = dict(row)
|
||||
if isinstance(explicit, Mapping):
|
||||
source.update(explicit)
|
||||
issues.append(build_issue(source, defaults=defaults))
|
||||
return issues
|
||||
|
||||
|
||||
def feedback_plan(
|
||||
trigger: str,
|
||||
*,
|
||||
target: str = "",
|
||||
related_issue: str = "",
|
||||
dispatcher: str = "dispatcher",
|
||||
) -> dict[str, Any]:
|
||||
"""Describe the required handoff without performing remote mutations."""
|
||||
if trigger not in TRIGGERS:
|
||||
raise ValueError(f"trigger 必须是 {', '.join(TRIGGERS)}")
|
||||
if trigger == "pr":
|
||||
actions = [
|
||||
{"action": "update_pr", "target": target, "status": "required"},
|
||||
{"action": "update_related_issue", "target": related_issue, "status": "required"},
|
||||
{"action": "notify_dispatcher", "target": dispatcher, "status": "required"},
|
||||
{"action": "reassign_development", "target": dispatcher, "status": "pending"},
|
||||
]
|
||||
route = "pr-update-issue-dispatcher-reassign"
|
||||
else:
|
||||
actions = [
|
||||
{"action": "return_to_development", "target": target or trigger, "status": "required"},
|
||||
]
|
||||
route = "development"
|
||||
return {
|
||||
"trigger": trigger,
|
||||
"route": route,
|
||||
"target": target,
|
||||
"related_issue": related_issue,
|
||||
"actions": actions,
|
||||
"remote_mutation": "caller-required",
|
||||
"created_at": _now(),
|
||||
}
|
||||
|
||||
|
||||
def attach_feedback(
|
||||
run: Mapping[str, Any],
|
||||
*,
|
||||
trigger: str | None = None,
|
||||
target: str = "",
|
||||
related_issue: str = "",
|
||||
dispatcher: str = "dispatcher",
|
||||
defaults: Mapping[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return a new run mapping with results, defects and routing preserved."""
|
||||
output = deepcopy(dict(run))
|
||||
results = output.get("results")
|
||||
if not isinstance(results, list):
|
||||
raise ValueError("run 缺少 results 列表")
|
||||
issues = collect_issues(results, defaults=defaults)
|
||||
output["defects"] = issues
|
||||
# Compatibility name for consumers that call the same structure
|
||||
# ``test-issues``. It is deliberately a copy, not a second source of truth.
|
||||
output["test_issues"] = deepcopy(issues)
|
||||
if trigger:
|
||||
output["feedback"] = feedback_plan(
|
||||
trigger,
|
||||
target=target,
|
||||
related_issue=related_issue,
|
||||
dispatcher=dispatcher,
|
||||
)
|
||||
return output
|
||||
|
||||
@@ -40,6 +40,16 @@ FIELDS = (
|
||||
"risk",
|
||||
"driver",
|
||||
"maps_to",
|
||||
"blocked_by",
|
||||
"issue_id",
|
||||
"issue_type",
|
||||
"expected",
|
||||
"observed",
|
||||
"artifact_hash",
|
||||
"environment",
|
||||
"evidence_path",
|
||||
"reproduction",
|
||||
"next_step",
|
||||
)
|
||||
|
||||
|
||||
@@ -112,6 +122,23 @@ def case_index(root: Path) -> dict[str, dict[str, Any]]:
|
||||
return out
|
||||
|
||||
|
||||
def issue_index(run: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
"""Index canonical defects and compatibility test_issues by testcase."""
|
||||
indexed: dict[str, dict[str, Any]] = {}
|
||||
raw = run.get("defects")
|
||||
if not isinstance(raw, list):
|
||||
raw = run.get("test_issues")
|
||||
if not isinstance(raw, list):
|
||||
return indexed
|
||||
for issue in raw:
|
||||
if not isinstance(issue, dict):
|
||||
continue
|
||||
testcase = str(issue.get("testcase") or issue.get("id") or "")
|
||||
if testcase:
|
||||
indexed.setdefault(testcase, issue)
|
||||
return indexed
|
||||
|
||||
|
||||
def export_run_csv(
|
||||
root: Path,
|
||||
run_path: Path | None = None,
|
||||
@@ -123,6 +150,7 @@ def export_run_csv(
|
||||
path = path.resolve()
|
||||
run = load_run(path)
|
||||
cases = case_index(root)
|
||||
issues = issue_index(run)
|
||||
dest = out or path.with_suffix(".csv")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with dest.open("w", encoding="utf-8-sig", newline="") as handle:
|
||||
@@ -134,6 +162,9 @@ def export_run_csv(
|
||||
cid = str(item["id"])
|
||||
result = str(item.get("result") or "")
|
||||
case = cases.get(cid) or {}
|
||||
issue = dict(issues.get(cid, {}))
|
||||
if isinstance(item.get("issue"), dict):
|
||||
issue.update(item["issue"])
|
||||
client_suite_value = case.get("client_suite") or (
|
||||
case.get("suite") if test_type(case) == "journey" else ""
|
||||
)
|
||||
@@ -154,6 +185,16 @@ def export_run_csv(
|
||||
"risk": case.get("risk") or "",
|
||||
"driver": case.get("driver") or "",
|
||||
"maps_to": case.get("maps_to") or "",
|
||||
"blocked_by": one_line(item.get("blocked_by")),
|
||||
"issue_id": issue.get("id") or item.get("issue_id") or "",
|
||||
"issue_type": issue.get("issue_type") or item.get("issue_type") or "",
|
||||
"expected": one_line(issue.get("expected") or item.get("expected")),
|
||||
"observed": one_line(issue.get("observed") or item.get("observed")),
|
||||
"artifact_hash": issue.get("artifact_hash") or item.get("artifact_hash") or "",
|
||||
"environment": one_line(issue.get("environment") or item.get("environment")),
|
||||
"evidence_path": one_line(issue.get("evidence_path") or item.get("evidence_path")),
|
||||
"reproduction": one_line(issue.get("reproduction") or item.get("reproduction")),
|
||||
"next_step": one_line(issue.get("next_step") or item.get("next_step")),
|
||||
}
|
||||
)
|
||||
return dest
|
||||
|
||||
@@ -254,6 +254,7 @@ def propagate_results(
|
||||
skipped: Mapping[str, str] | None = None,
|
||||
continue_on_fail: bool = False,
|
||||
policies: Mapping[str, str] | None = None,
|
||||
blocked_by: Mapping[str, Iterable[str]] | None = None,
|
||||
) -> list[dict[str, str]]:
|
||||
"""Build ordered result rows and mark cases after a blocking failure not-run.
|
||||
|
||||
@@ -263,15 +264,21 @@ def propagate_results(
|
||||
"""
|
||||
skipped = skipped or {}
|
||||
policies = policies or {}
|
||||
blocked_by = blocked_by or {}
|
||||
rows: list[dict[str, str]] = []
|
||||
blocked = False
|
||||
blocking_case = ""
|
||||
for raw_id in case_ids:
|
||||
case_id = str(raw_id)
|
||||
if case_id in skipped:
|
||||
rows.append({"id": case_id, "result": "skipped", "notes": skipped[case_id]})
|
||||
continue
|
||||
if blocked:
|
||||
rows.append({"id": case_id, "result": "not-run", "notes": "前置用例失败,未轮到执行"})
|
||||
if blocking_case:
|
||||
rows.append({
|
||||
"id": case_id,
|
||||
"result": "not-run",
|
||||
"notes": "前置用例失败,未轮到执行",
|
||||
"blocked_by": ",".join(blocked_by.get(case_id) or (blocking_case,)),
|
||||
})
|
||||
continue
|
||||
result = observed.get(case_id, "not-run")
|
||||
if result not in RESULTS:
|
||||
@@ -279,5 +286,5 @@ def propagate_results(
|
||||
rows.append({"id": case_id, "result": result})
|
||||
policy = policies.get(case_id)
|
||||
if result in ("fail", "environment-fail") and not (continue_on_fail or policy == "continue"):
|
||||
blocked = True
|
||||
blocking_case = case_id
|
||||
return rows
|
||||
|
||||
@@ -390,6 +390,27 @@ def write_run(root: Path, record: dict[str, Any]) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def all_active_result_rows(root: Path, *, blocked_by: str) -> list[dict[str, Any]]:
|
||||
"""Keep a testcase row for every active catalog entry when preflight blocks."""
|
||||
try:
|
||||
from generate_cases_csv import load_cases
|
||||
|
||||
cases = load_cases(root)
|
||||
except (ImportError, OSError, ValueError, SystemExit):
|
||||
cases = []
|
||||
rows: list[dict[str, Any]] = []
|
||||
for case in cases:
|
||||
if case.get("lifecycle", "active") != "active":
|
||||
continue
|
||||
rows.append({
|
||||
"id": str(case.get("id") or ""),
|
||||
"result": "not-run",
|
||||
"notes": "test-preflight 未通过,未创建移动 session",
|
||||
"blocked_by": [blocked_by],
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def run_test_preflight(
|
||||
config_path: Path,
|
||||
*,
|
||||
@@ -507,6 +528,7 @@ def execute(args: argparse.Namespace) -> int:
|
||||
record["platforms"][platform].update({"status": "not-run", "reason": "全量 preflight 闸门被另一平台阻断"})
|
||||
overall = record.get("preflight", {}).get("status") if "preflight" in record else "environment-fail"
|
||||
record.update({"status": overall if overall in {"blocked", "environment-fail"} else "environment-fail", "failure_class": "test-preflight" if "preflight" in record else "automatic-repair-failed", "next_action": "修复列出的平台环境后重跑;默认不会静默降级"})
|
||||
record["results"] = all_active_result_rows(context.root, blocked_by="test-preflight")
|
||||
else:
|
||||
for platform, (result, node, identity) in prepared.items():
|
||||
try:
|
||||
@@ -527,7 +549,18 @@ def execute(args: argparse.Namespace) -> int:
|
||||
record["evidence"] = context.evidence
|
||||
except (EnvironmentFailure, ArtifactFailure) as exc:
|
||||
record.update({"status": "environment-fail", "failure_class": "automatic-repair-failed", "error": str(exc), "evidence": context.evidence})
|
||||
if record.get("status") != "ready" and "results" not in record:
|
||||
record["results"] = all_active_result_rows(context.root, blocked_by="test-preflight")
|
||||
path = write_run(context.root, record)
|
||||
try:
|
||||
from generate_run_csv import export_run_csv
|
||||
|
||||
record["csv"] = str(export_run_csv(context.root, run_path=path))
|
||||
except (OSError, SystemExit, ValueError):
|
||||
# The YAML run remains the source of truth when a partial/provisioning
|
||||
# root has no catalog cases yet.
|
||||
record["csv"] = ""
|
||||
path.write_text(json.dumps(record, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output = {"status": record["status"], "run": str(path), "platforms": record["platforms"], "worktree": record.get("worktree")}
|
||||
if record.get("error"):
|
||||
output["error"] = record["error"]
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Append structured tester issue evidence to an existing run snapshot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from failure_feedback import attach_feedback
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="将 tester 失败结构化回流到 run YAML/CSV")
|
||||
parser.add_argument("--run", required=True, type=Path)
|
||||
parser.add_argument("--trigger", choices=("pr", "requirement", "bug"))
|
||||
parser.add_argument("--target", default="")
|
||||
parser.add_argument("--related-issue", default="")
|
||||
parser.add_argument("--dispatcher", default="dispatcher")
|
||||
parser.add_argument("--testcase", default="", help="为一条失败结果补充 testcase issue 字段")
|
||||
parser.add_argument("--issue-type", choices=("product", "test"))
|
||||
parser.add_argument("--expected", default="")
|
||||
parser.add_argument("--observed", default="")
|
||||
parser.add_argument("--artifact-hash", default="")
|
||||
parser.add_argument("--environment", default="")
|
||||
parser.add_argument("--evidence-path", default="")
|
||||
parser.add_argument("--reproduction", default="")
|
||||
parser.add_argument("--next-step", default="")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
data: dict[str, Any] = yaml.safe_load(args.run.read_text(encoding="utf-8")) or {}
|
||||
if args.testcase:
|
||||
rows = data.get("results")
|
||||
if not isinstance(rows, list):
|
||||
raise SystemExit("run 缺少 results 列表")
|
||||
row = next((item for item in rows if isinstance(item, dict) and str(item.get("id")) == args.testcase), None)
|
||||
if row is None:
|
||||
raise SystemExit(f"找不到 testcase: {args.testcase}")
|
||||
row["issue"] = {
|
||||
key: value
|
||||
for key, value in {
|
||||
"issue_type": args.issue_type,
|
||||
"expected": args.expected,
|
||||
"observed": args.observed,
|
||||
"artifact_hash": args.artifact_hash,
|
||||
"environment": args.environment,
|
||||
"evidence_path": args.evidence_path,
|
||||
"reproduction": args.reproduction,
|
||||
"next_step": args.next_step,
|
||||
}.items()
|
||||
if value
|
||||
}
|
||||
output = attach_feedback(
|
||||
data,
|
||||
trigger=args.trigger,
|
||||
target=args.target,
|
||||
related_issue=args.related_issue,
|
||||
dispatcher=args.dispatcher,
|
||||
)
|
||||
args.run.write_text(yaml.safe_dump(output, allow_unicode=True, sort_keys=False), encoding="utf-8")
|
||||
csv_path = ""
|
||||
try:
|
||||
from generate_run_csv import export_run_csv
|
||||
|
||||
csv_path = str(export_run_csv(args.run.parents[3], run_path=args.run))
|
||||
except (ImportError, OSError, SystemExit, ValueError):
|
||||
pass
|
||||
print(f"updated {args.run} defects={len(output.get('defects') or [])} csv={csv_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -103,6 +103,27 @@ def load_yaml_results(path: Path) -> list[dict[str, Any]]:
|
||||
errors.append(f"YAML testcase id 重复: {_format_ids(set(duplicates))}")
|
||||
if errors:
|
||||
raise SummaryError("YAML 校验失败:\n- " + "\n- ".join(errors))
|
||||
issues = data.get("defects", data.get("test_issues"))
|
||||
if issues is not None:
|
||||
if not isinstance(issues, list):
|
||||
raise SummaryError("YAML defects/test_issues 必须是列表")
|
||||
result_ids = set(ids)
|
||||
issue_errors: list[str] = []
|
||||
for index, issue in enumerate(issues, start=1):
|
||||
if not isinstance(issue, dict):
|
||||
issue_errors.append(f"YAML defects[{index}] 不是 mapping")
|
||||
continue
|
||||
testcase = str(issue.get("testcase") or "")
|
||||
issue_type = str(issue.get("issue_type") or "")
|
||||
if not testcase or testcase not in result_ids:
|
||||
issue_errors.append(f"YAML defects[{index}] testcase 无效: {testcase or '(missing)'}")
|
||||
if issue_type not in {"product", "test"}:
|
||||
issue_errors.append(f"YAML defects[{index}] issue_type 无效: {issue_type or '(missing)'}")
|
||||
for field in ("expected", "observed", "artifact_hash", "environment", "evidence_path", "reproduction", "next_step"):
|
||||
if field not in issue:
|
||||
issue_errors.append(f"YAML defects[{index}] 缺少 {field}")
|
||||
if issue_errors:
|
||||
raise SummaryError("YAML defects 校验失败:\n- " + "\n- ".join(issue_errors))
|
||||
return results
|
||||
|
||||
|
||||
|
||||
@@ -12,6 +12,8 @@ from __future__ import annotations
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from failure_feedback import collect_issues
|
||||
|
||||
RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run")
|
||||
PHASES = ("preflight", "integration", "journey")
|
||||
|
||||
@@ -29,10 +31,18 @@ def _case_id(case: Mapping[str, Any] | str) -> str:
|
||||
return str(case)
|
||||
|
||||
|
||||
def _not_run(cases: Iterable[Mapping[str, Any] | str], reason: str) -> list[dict[str, str]]:
|
||||
def _not_run(
|
||||
cases: Iterable[Mapping[str, Any] | str],
|
||||
reason: str,
|
||||
*,
|
||||
blocked_by: Iterable[str] = (),
|
||||
) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, str]] = []
|
||||
blockers = [str(item) for item in blocked_by if str(item)]
|
||||
for case in cases:
|
||||
row = {"id": _case_id(case), "result": "not-run", "notes": reason}
|
||||
row: dict[str, Any] = {"id": _case_id(case), "result": "not-run", "notes": reason}
|
||||
if blockers:
|
||||
row["blocked_by"] = blockers
|
||||
if isinstance(case, Mapping):
|
||||
row["test_type"] = str(case.get("test_type") or "")
|
||||
row["suite"] = str(
|
||||
@@ -45,6 +55,37 @@ def _not_run(cases: Iterable[Mapping[str, Any] | str], reason: str) -> list[dict
|
||||
return rows
|
||||
|
||||
|
||||
def _dependencies(
|
||||
case: Mapping[str, Any] | str,
|
||||
*,
|
||||
default: Iterable[str] = (),
|
||||
) -> list[str]:
|
||||
if not isinstance(case, Mapping):
|
||||
return [str(item) for item in default]
|
||||
declared = case.get("depends_on", case.get("dependencies"))
|
||||
if declared is None:
|
||||
return [str(item) for item in default]
|
||||
if isinstance(declared, str):
|
||||
return [declared]
|
||||
if isinstance(declared, Iterable):
|
||||
return [str(item) for item in declared if str(item)]
|
||||
raise ValueError(f"depends_on 必须是字符串或列表: {_case_id(case)}")
|
||||
|
||||
|
||||
def _failed_dependencies(dependencies: Iterable[str], results: Mapping[str, Mapping[str, Any]]) -> list[str]:
|
||||
return [
|
||||
dependency
|
||||
for dependency in dependencies
|
||||
if str((results.get(dependency) or {}).get("result"))
|
||||
in {"fail", "environment-fail", "not-run"}
|
||||
]
|
||||
|
||||
|
||||
def _issue_structures(results: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
|
||||
issues = collect_issues(results)
|
||||
return {"defects": issues, "test_issues": [dict(issue) for issue in issues]}
|
||||
|
||||
|
||||
def run_integration_first(
|
||||
*,
|
||||
preflight_status: str,
|
||||
@@ -65,18 +106,29 @@ def run_integration_first(
|
||||
journey = list(journey_cases)
|
||||
if preflight_status != "ready":
|
||||
reason = f"test-preflight={preflight_status},未创建客户端 session"
|
||||
result_rows = _not_run(integration, reason, blocked_by=("test-preflight",)) + _not_run(journey, reason, blocked_by=("test-preflight",))
|
||||
return {
|
||||
"status": preflight_status,
|
||||
"phase": "preflight",
|
||||
"session_created": False,
|
||||
"results": _not_run(integration, reason) + _not_run(journey, reason),
|
||||
"results": result_rows,
|
||||
**_issue_structures(result_rows),
|
||||
}
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
integration_failed = False
|
||||
for index, case in enumerate(integration):
|
||||
if integration_failed:
|
||||
results.extend(_not_run([case], "前置 integration 失败,未轮到执行"))
|
||||
results_by_id: dict[str, dict[str, Any]] = {}
|
||||
for case in integration:
|
||||
case_id = _case_id(case)
|
||||
dependencies = _dependencies(case)
|
||||
blocked_by = _failed_dependencies(dependencies, results_by_id)
|
||||
if blocked_by:
|
||||
rows = _not_run(
|
||||
[case],
|
||||
"依赖的 integration 失败,未执行",
|
||||
blocked_by=blocked_by,
|
||||
)
|
||||
results.extend(rows)
|
||||
results_by_id[case_id] = rows[0]
|
||||
continue
|
||||
outcome = execute_integration(case)
|
||||
row = dict(outcome) if isinstance(outcome, Mapping) else {"result": outcome}
|
||||
@@ -89,20 +141,26 @@ def run_integration_first(
|
||||
raise ValueError(f"未知 integration 结果: {row['id']}={result}")
|
||||
row["result"] = result
|
||||
results.append(row)
|
||||
if result in ("fail", "environment-fail"):
|
||||
integration_failed = True
|
||||
results_by_id[case_id] = row
|
||||
|
||||
if integration_failed:
|
||||
results.extend(_not_run(journey, "integration 失败,依赖的 journey 未执行"))
|
||||
return {
|
||||
"status": "fail" if any(row["result"] == "fail" for row in results) else "environment-fail",
|
||||
"phase": "integration",
|
||||
"session_created": False,
|
||||
"results": results,
|
||||
}
|
||||
|
||||
session = create_journey_session() if journey else None
|
||||
integration_ids = [_case_id(case) for case in integration]
|
||||
has_runnable_journey = any(
|
||||
not _failed_dependencies(_dependencies(case, default=integration_ids), results_by_id)
|
||||
for case in journey
|
||||
)
|
||||
session = create_journey_session() if has_runnable_journey else None
|
||||
for case in journey:
|
||||
dependencies = _dependencies(case, default=integration_ids)
|
||||
blocked_by = _failed_dependencies(dependencies, results_by_id)
|
||||
if blocked_by:
|
||||
row = _not_run(
|
||||
[case],
|
||||
"依赖的用例失败,journey 未执行",
|
||||
blocked_by=blocked_by,
|
||||
)[0]
|
||||
results.append(row)
|
||||
results_by_id[_case_id(case)] = row
|
||||
continue
|
||||
outcome = execute_journey(session, case)
|
||||
row = dict(outcome) if isinstance(outcome, Mapping) else {"result": outcome}
|
||||
row["id"] = _case_id(case)
|
||||
@@ -114,12 +172,14 @@ def run_integration_first(
|
||||
raise ValueError(f"未知 journey 结果: {row['id']}={result}")
|
||||
row["result"] = result
|
||||
results.append(row)
|
||||
results_by_id[_case_id(case)] = row
|
||||
status = "fail" if any(row["result"] == "fail" for row in results) else (
|
||||
"environment-fail" if any(row["result"] == "environment-fail" for row in results) else "pass"
|
||||
)
|
||||
return {
|
||||
"status": status,
|
||||
"phase": "journey" if journey else "integration",
|
||||
"session_created": bool(journey),
|
||||
"session_created": has_runnable_journey,
|
||||
"results": results,
|
||||
**_issue_structures(results),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user