feat(tester): add client journey testing standard

2026-10-15 16:48:54 +08:00
parent baed23d02a
commit ffa7cbd63e
12 changed files with 518 additions and 0 deletions
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""Shared client-journey contract and result lifecycle helpers.
+
+The mobile/browser driver still owns the real UI interaction.  This module
+owns the parts that must be identical for every client catalog: contract
+validation, preflight blocking, and result propagation.  It deliberately
+returns the existing tester result vocabulary instead of introducing a second
+report format.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Mapping
+
+
+CLIENT_SUITES = ("client-smoke", "client-regression", "safety")
+PLATFORMS = ("android", "ios")
+FAILURE_POLICIES = ("stop", "continue")
+RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run")
+
+
+def client_suite(case: Mapping[str, Any]) -> str:
+    """Return the canonical client suite, accepting ``suite`` as a migration alias."""
+    value = case.get("client_suite")
+    alias = case.get("suite")
+    if value and alias and str(value) != str(alias):
+        return ""
+    return str(value or alias or "")
+
+
+def normalize_platforms(value: Any) -> list[str]:
+    if isinstance(value, str):
+        values = [value]
+    elif isinstance(value, (list, tuple)):
+        values = list(value)
+    else:
+        return []
+    return [str(item).strip().lower() for item in values if str(item).strip()]
+
+
+def _non_empty_list(value: Any) -> bool:
+    return isinstance(value, list) and bool(value) and all(str(item).strip() for item in value)
+
+
+def _has_account(fixture: Mapping[str, Any]) -> bool:
+    account = fixture.get("account")
+    if isinstance(account, Mapping):
+        return bool(str(account.get("role") or account.get("username") or "").strip())
+    return bool(str(account or "").strip())
+
+
+def contract_errors(case: Mapping[str, Any], sessions: Mapping[str, Any] | None = None) -> list[str]:
+    """Validate a client journey without requiring app-specific field names.
+
+    A case becomes a client journey when it declares ``client_suite`` (or the
+    temporary ``suite`` alias).  Legacy non-client cases keep the existing
+    schema.  Once opted in, all lifecycle inputs are explicit so a missing
+    fixture/cleanup can stop the run before any case result is fabricated.
+    """
+    suite = client_suite(case)
+    journey = case.get("journey")
+    if not suite and journey is None:
+        return []
+    errors: list[str] = []
+    cid = str(case.get("id") or "?")
+    where = f"{cid}: client journey"
+    if case.get("client_suite") and case.get("suite") and str(case["client_suite"]) != str(case["suite"]):
+        errors.append(f"{where}: client_suite 与 suite 不一致")
+    if suite not in CLIENT_SUITES:
+        errors.append(f"{where}: client_suite 必须是 {', '.join(CLIENT_SUITES)}")
+
+    if not isinstance(journey, Mapping):
+        errors.append(f"{where}: journey 必须是 mapping")
+        journey = {}
+    for key in ("goal", "entry_state", "final_state"):
+        if not str(journey.get(key) or "").strip():
+            errors.append(f"{where}: journey.{key} 必填")
+    if not _non_empty_list(journey.get("actions")):
+        errors.append(f"{where}: journey.actions 必须是非空列表")
+    if not _non_empty_list(journey.get("assertions")):
+        errors.append(f"{where}: journey.assertions 必须是非空列表")
+    policy = str(journey.get("failure_policy") or "")
+    if policy not in FAILURE_POLICIES:
+        errors.append(f"{where}: journey.failure_policy 必须是 stop 或 continue")
+
+    fixture = case.get("fixture")
+    if not isinstance(fixture, Mapping):
+        errors.append(f"{where}: fixture 缺失，必须在 preflight 前声明")
+    elif not _has_account(fixture):
+        errors.append(f"{where}: fixture.account 缺失，必须声明测试账号或角色")
+
+    cleanup = case.get("cleanup")
+    if not isinstance(cleanup, Mapping):
+        errors.append(f"{where}: cleanup 缺失，必须在 preflight 前声明")
+    else:
+        steps = cleanup.get("steps")
+        command = cleanup.get("run")
+        explicitly_noop = cleanup.get("required") is False and str(cleanup.get("reason") or "").strip()
+        if not explicitly_noop and not _non_empty_list(steps) and not str(command or "").strip():
+            errors.append(f"{where}: cleanup 必须有非空 steps/run；只读用例也要显式声明 reason")
+
+    platforms = normalize_platforms(case.get("platform"))
+    if not platforms:
+        errors.append(f"{where}: platform 必须声明 android/ios")
+    elif any(platform not in PLATFORMS for platform in platforms):
+        errors.append(f"{where}: platform 只能是 {', '.join(PLATFORMS)}")
+    elif len(platforms) != len(set(platforms)):
+        errors.append(f"{where}: platform 不能重复")
+
+    driver = case.get("driver")
+    if driver not in ("mobile", "browser"):
+        errors.append(f"{where}: 客户端旅程 driver 必须是 mobile 或 browser")
+    if driver and driver in ("mobile", "browser"):
+        session = case.get("session")
+        if not str(session or "").strip():
+            errors.append(f"{where}: session 必须声明可复用客户端会话")
+        elif sessions is not None and session not in sessions:
+            errors.append(f"{where}: session={session} 未在 catalog.sessions 声明")
+    if suite == "safety" and str(case.get("tag") or "") == "smoke":
+        errors.append(f"{where}: safety 不得标记为 smoke")
+    return errors
+
+
+def preflight(cases: Iterable[Mapping[str, Any]], sessions: Mapping[str, Any] | None = None) -> dict[str, Any]:
+    """Return a contract preflight result; no per-case result is created on block."""
+    cases = list(cases)
+    errors: list[str] = []
+    for case in cases:
+        errors.extend(contract_errors(case, sessions=sessions))
+    return {
+        "required": bool(errors or any(client_suite(case) for case in cases)),
+        "status": "blocked" if errors else "ready",
+        "failure_class": "contract" if errors else None,
+        "errors": errors,
+    }
+
+
+def classify_outcome(
+    *,
+    ok: bool,
+    environment_ready: bool = True,
+    phase: str = "journey",
+) -> str:
+    """Classify one observed phase using the tester's five result states.
+
+    A valid test account is part of the client journey: a failed login is a
+    product ``fail``.  Only an unavailable environment is ``environment-fail``.
+    """
+    if not environment_ready:
+        return "environment-fail"
+    if not ok:
+        return "fail"
+    return "pass"
+
+
+def propagate_results(
+    case_ids: Iterable[str],
+    observed: Mapping[str, str],
+    *,
+    skipped: Mapping[str, str] | None = None,
+    continue_on_fail: bool = False,
+    policies: Mapping[str, str] | None = None,
+) -> list[dict[str, str]]:
+    """Build ordered result rows and mark cases after a blocking failure not-run.
+
+    ``skipped`` is intentional selection (risk/lifecycle/assumption); it is
+    never converted to ``not-run``.  A product or environment failure stops
+    subsequent cases unless the caller explicitly opts into continuation.
+    """
+    skipped = skipped or {}
+    policies = policies or {}
+    rows: list[dict[str, str]] = []
+    blocked = False
+    for raw_id in case_ids:
+        case_id = str(raw_id)
+        if case_id in skipped:
+            rows.append({"id": case_id, "result": "skipped", "notes": skipped[case_id]})
+            continue
+        if blocked:
+            rows.append({"id": case_id, "result": "not-run", "notes": "前置用例失败，未轮到执行"})
+            continue
+        result = observed.get(case_id, "not-run")
+        if result not in RESULTS:
+            raise ValueError(f"未知结果: {case_id}={result}")
+        rows.append({"id": case_id, "result": result})
+        policy = policies.get(case_id)
+        if result in ("fail", "environment-fail") and not (continue_on_fail or policy == "continue"):
+            blocked = True
+    return rows