From ffa7cbd63e8e73d0a3d9d046da7a94c3d6465974 Mon Sep 17 00:00:00 2001 From: laily Date: Thu, 15 Oct 2026 16:48:54 +0800 Subject: [PATCH] feat(tester): add client journey testing standard --- skills/tester/README.md | 14 ++ skills/tester/SKILL.md | 26 +++ skills/tester/references/concepts.md | 18 ++ skills/tester/references/run.md | 9 + skills/tester/references/schema.md | 50 ++++++ skills/tester/scripts/catalog.py | 24 +++ skills/tester/scripts/generate_cases_csv.py | 4 + skills/tester/scripts/generate_run_csv.py | 4 + skills/tester/scripts/journey.py | 189 ++++++++++++++++++++ skills/tester/templates/catalog.yaml | 7 + skills/tester/tests/test_catalog_journey.py | 104 +++++++++++ skills/tester/tests/test_journey.py | 69 +++++++ 12 files changed, 518 insertions(+) create mode 100644 skills/tester/scripts/journey.py create mode 100644 skills/tester/tests/test_catalog_journey.py create mode 100644 skills/tester/tests/test_journey.py diff --git a/skills/tester/README.md b/skills/tester/README.md index 8f67a03..9743d00 100644 --- a/skills/tester/README.md +++ b/skills/tester/README.md @@ -86,6 +86,20 @@ APK/IPA，为本轮创建隔离 worktree，在 Android 和 iOS tester 节点安 - **维护**：目标 id 能 `查看` 到，且检查通过。 - **运行**：有本次 `runs/*.yaml` 快照和同名 `.csv` 成绩单（`executed` 列区分已跑 / 未跑）；没填 HOST 时会停，而不是编造失败。 +### 客户端用户旅程 + +客户端 E2E 用例使用 `client_suite: client-smoke | client-regression | safety`，并明确 +`journey`、`fixture`、`session`、`cleanup`、`platform`。标准生命周期是： + +```text +preflight → 启动 → 登录 → 旅程 → 断言 → 清理 → 报告 +``` + +正确测试账号登录失败记 `fail`；设备、产物或服务不可达记 `environment-fail`。fixture 或 +cleanup 缺失会在开始前阻断；前置失败后的剩余用例是 `not-run`，明确排除才是 `skipped`。 +`safety` 必须显式选择和授权，不进入默认 smoke/full。它与普通 tester 用例共用原有 +CSV/YAML 运行记录和汇总校验。 + ### 汇总已有运行结果 对已经生成的同一运行 CSV/YAML，使用显式路径汇总： diff --git a/skills/tester/SKILL.md b/skills/tester/SKILL.md index 06860ec..2cbc002 100644 --- a/skills/tester/SKILL.md +++ b/skills/tester/SKILL.md @@ -145,3 +145,29 @@ python3 /scripts/catalog.py show --root --id 4. 只有矩阵所有行都可用，才创建本轮测试 terminal/session，按 `select` 结果执行并记录成绩。 测试环境由本机 deployer 触发，正式环境由 app00 Agent terminal 委派。随后按 agent 解析目标项目、目录和 Agent，更新配置并输出生效列表。移动端每个平台的产物位置必须由调用方给出，值为本地文件或 URL；缺了就停，不要猜 builder、CI release 或本机旧包。矩阵行记产物位置、身份和 `tester.` binding，并用 `/tester run --artifact android= --artifact ios= --platform android,ios` 下载或取用、安装并准备本轮隔离 worktree。Android 在 `tester.android`、iOS 在 `tester.ios` 节点执行。未指定范围时等同「tag=smoke 且风险上限=默认且 automation=automated」。`select` 的 `run_together` 组按 [run.md](references/run.md) 整组执行。无论部署或测试成功、失败、超时还是中断，都要清理各自本轮创建的 terminal。 + +### 客户端用户旅程标准 + +声明 `client_suite`（迁移期也接受 `suite`）的用例采用固定的三套客户端集合： +`client-smoke`、`client-regression`、`safety`。前两者分别覆盖最短只读主路径和隔离测试 +环境中的普通用户回归；`safety` 覆盖危险/权限边界场景，必须显式选择和授权，不能被 +默认 smoke/full 扩入。选择某套客户端旅程可用： + +```bash +python3 /scripts/catalog.py select --root --suite client-smoke +python3 /scripts/catalog.py select --root --suite client-regression --max-risk mutating +python3 /scripts/catalog.py select --root --suite safety --max-risk destructive +``` + +客户端用例必须声明 `journey`、`fixture`、`session`、`cleanup` 和 `platform`。旅程包含 +`goal`、`entry_state`、`actions`、`assertions`、`final_state`、`failure_policy`；fixture +至少声明测试账号；session 指向 catalog 中可复用的 browser/mobile session；cleanup 要有 +步骤或命令（只读也要显式说明无需清理）；platform 只能是 Android/iOS。具体字段见 +`references/schema.md`。平台适配只能改操作层，不能把真实客户端操作换成 API 调用。 + +客户端运行器统一遵守 `preflight → 启动 → 登录 → 旅程 → 断言 → 清理 → 报告`。正确测试 +账号登录失败是产品 `fail`；设备、产物、服务或 session 不可达才是 `environment-fail`。 +fixture 或 cleanup 缺失会在 `catalog.py select` 的 preflight 阶段阻断，不产生虚假逐条结果。 +前置用例失败后剩余条目写 `not-run`，主动排除/未授权写 `skipped`。成绩仍写现有 +`runs/.yaml` 并用 `generate_run_csv.py` 生成同名 CSV；清理失败作为 cleanup 证据 +单独记录，不添加新的 result 值。 diff --git a/skills/tester/references/concepts.md b/skills/tester/references/concepts.md index 9336a2d..d303754 100644 --- a/skills/tester/references/concepts.md +++ b/skills/tester/references/concepts.md @@ -40,6 +40,11 @@ catalog（整个测试集） | `how` | 怎么测：`steps` / `run` / `url` / `timeout` | test steps | 不是判定标准 | | `expected` | 怎么算过：`{kind, value}` 列表 | pytest `assert`；Robot `Should …` | 不是步骤 | | `run_together` | 必须成组执行的用例（有序、可选 continue_on_fail） | 场景组（scenario） | 不是 suite，不是 tag | +| `client_suite` | 客户端用户旅程套件：`client-smoke` / `client-regression` / `safety` | mobile/browser E2E suite | 不是 `category`，也不是通用 tag；迁移期可读 `suite` 别名 | +| `journey` | 用户目标、入口、操作、UI 断言、最终状态和失败继续策略 | Cucumber scenario / Robot task | 不是 API contract | +| `fixture` | 旅程开始前声明的测试账号与隔离数据 | pytest fixture | 不是运行中临时猜出的数据 | +| `cleanup` | 本轮数据的 teardown 契约；缺失会在 preflight 阻断 | pytest teardown / Robot teardown | 不是测试通过后的可选备注 | +| `platform` | 旅程适用的 `android`/`ios` 平台列表 | device matrix | 不是 tester Agent binding；binding 仍在命令参数/运行环境 | ### 结果词 @@ -99,6 +104,19 @@ catalog（整个测试集） - 因此它是**分类级前置条件**：`catalog.status_probe` 是全局默认，`categories[].status_probe` 可覆盖（缺省继承 / `false` 关闭 / mapping 整体替换）。对应 pytest 的 session fixture、Robot 的 Suite Setup。 - 命中或不可达只影响用它的分类；只有所有选中分类都被挡住，整轮结论才是 `environment-fail`。`select` 会按分类返回解析后的生效值，Agent 不要自己读 catalog 猜。 +### 3.5 客户端套件和生命周期 + +客户端套件固定为： + +- `client-smoke`：启动、正确账号登录和少量只读主路径，通常配 `tag=smoke`、`risk=readonly`。 +- `client-regression`：完整的普通用户旅程，可包含隔离环境中的 `mutating` 操作；不等于所有 full 用例。 +- `safety`：删除、权限边界、改密、停用或故障注入等危险旅程；必须显式选择和授权，不进入默认运行。 + +每条旅程按 `preflight → 启动 → 登录 → 旅程 → 断言 → 清理 → 报告` 执行。登录失败在 +客户端和正确测试账号都已就绪时记 `fail`；设备、产物、服务或 session 不可达才记 +`environment-fail`。fixture/cleanup 缺失是 contract block，不伪装成逐条结果。前置失败 +后的剩余条目是 `not-run`，主动排除仍是 `skipped`。 + ## 4. 不采用的词 | 词 | 不用它的原因 | diff --git a/skills/tester/references/run.md b/skills/tester/references/run.md index f74427f..bd86131 100644 --- a/skills/tester/references/run.md +++ b/skills/tester/references/run.md @@ -81,6 +81,15 @@ python3 /scripts/catalog.py select --root \ [--binding android=tester.android] [--binding ios=tester.ios] ``` +客户端旅程还可按标准套件选择： + +```bash +python3 /scripts/catalog.py select --root --suite client-smoke +``` + +`client-smoke`、`client-regression`、`safety` 是 `client_suite` 的固定值，不替换既有 +`tag=smoke|full|slow`。`safety` 不参与未指定范围的默认选择。 + 选中任一 `driver=mobile` 用例时，`--platform`、每个平台的 `--artifact` 和对应 `tester` binding 是硬性参数；`catalog.py select` 会检查产物文件/下载地址和 `.pouch/agent.yaml` 中的 binding。缺任何一项直接返回非零并打印 `preflight: blocked`， diff --git a/skills/tester/references/schema.md b/skills/tester/references/schema.md index d1d0b00..b2e8bb1 100644 --- a/skills/tester/references/schema.md +++ b/skills/tester/references/schema.md @@ -14,6 +14,11 @@ runs/.yaml 每次运行快照（执行源）；run-id 必须唯一 runs/*.csv 同一次运行的成绩单；generate_run_csv.py 从对应 yaml 生成 ``` +客户端旅程在上述目录中仍使用同一套 cases/runs 和五态报告，不另造成绩单。启用旅程 +契约的用例声明 `client_suite`（兼容迁移期别名 `suite`）；catalog 可用 +`client_suites` 显式列出三个标准套件：`client-smoke`、`client-regression`、`safety`。 +`category` 仍是文件/执行顺序容器，不被套件字段替代。 + ## catalog.yaml | 字段 | 含义 | @@ -29,6 +34,7 @@ runs/*.csv 同一次运行的成绩单；generate_run_csv.py 从对 | `id_pattern` | 用例 id 正则 | | `run_together` | 可选。成组执行：`[{ids: [TC-…, TC-…], continue_on_fail: true}]`。点到组内任一条，`select` 会按 `ids` 顺序把整组扩进命中列表。`continue_on_fail: true` 时组内前一条产品 fail 仍跑后一条（返回/收尾）。一个 id 只能属于一组。 | | `status_probe` | 可选。顶层默认值，一个**分类级前置条件**：`{url, stop_if: {字段: [命中值]}}`。运行前请求 `url`，响应命中任一 `stop_if` 条件即视为前置不满足。分类项可用同名键覆盖：缺省继承顶层、`false` 关闭、mapping 整体替换。命中或不可达只影响用它的分类，不是产品失败。业界对应 pytest 的 session fixture / Robot 的 Suite Setup。`select` 会按分类返回解析后的生效值，`url` 已用 `vars` 展开 | +| `client_suites` | 可选的标准套件声明；出现时必须完整包含 `client-smoke`、`client-regression`、`safety`。缺省运行仍按 `tag` 兼容旧目录 | `local.yaml` 与环境变量覆盖 `HOST` 和授权开关。模板中的 `HOST` 默认是 `127.0.0.1`；远程目标必须显式覆盖。查找顺序：环境变量 > `local.yaml` > @@ -88,6 +94,50 @@ terminal；运行结束用 `orca terminal close --terminal ` 关闭本 | `teardown` | 否 | 跑完回退 | | `notes` | 否 | 备注 | +### 客户端旅程用例契约 + +声明 `client_suite` 后，以下字段都必须存在；`catalog.py validate` 和 `select` 使用同一 +套检查。契约缺失是 `preflight: blocked`，不生成逐条产品失败。 + +| 字段 | 必填 | 约束 | +|------|------|------| +| `client_suite` | 是 | `client-smoke`、`client-regression` 或 `safety`；安全套件必须显式选择，不属于默认 smoke/full | +| `journey.goal` | 是 | 用户目标，不写接口目标 | +| `journey.entry_state` | 是 | 启动/登录前的可观察入口状态 | +| `journey.actions` | 是 | 非空用户操作列表；真实 App/browser 驱动从这里开始 | +| `journey.assertions` | 是 | 非空可观察 UI 断言列表；最终业务状态可另由 `expected` 的 api-field/api-status 佐证 | +| `journey.final_state` | 是 | 旅程完成后的业务状态 | +| `journey.failure_policy` | 是 | `stop` 或 `continue`；默认不隐式吞掉前置失败 | +| `fixture` | 是 | mapping，至少有 `account`；账号和数据由 app catalog 提供，skill 不写具体账号 | +| `session` | 是 | 复用的 browser/mobile session 名，必须在 catalog.sessions 声明 | +| `cleanup` | 是 | mapping，必须有非空 `steps`/`run`；只读旅程也要用 `required: false` + `reason` 显式说明无需清理 | +| `platform` | 是 | `android`/`ios` 的字符串或列表，不能重复；平台差异只能进入执行适配层 | + +一个最小形状如下（值是占位符，不是 app 规则）： + +```yaml +client_suite: client-smoke +platform: [android, ios] +journey: + goal: "用户完成一个可观察的客户端目标" + entry_state: logged_out + actions: ["启动客户端", "输入测试账号并登录", "完成目标操作"] + assertions: ["看到目标页面", "看到成功提示"] + final_state: "目标状态已保存" + failure_policy: stop +fixture: + account: test-role + data: [] +session: mobile +cleanup: + steps: ["删除本轮 run-id 创建的数据"] +``` + +账号登录属于 session 初始化：正确测试账号登录失败是产品 `fail`；设备、产物或服务 +不可达才是 `environment-fail`。fixture/cleanup 缺失必须在启动、登录和任何旅程操作前 +阻断。旅程失败后，尚未轮到的用例记 `not-run`；风险、生命周期或明确范围排除记 +`skipped`。清理失败单独写入运行快照的 cleanup 证据，不改造为一个新的 result 值。 + ## how 按 `driver` 分派；`automation=manual` 时没有 `driver`，只有 `how.steps`（人工步骤）。 diff --git a/skills/tester/scripts/catalog.py b/skills/tester/scripts/catalog.py index 327cdf1..00974fa 100755 --- a/skills/tester/scripts/catalog.py +++ b/skills/tester/scripts/catalog.py @@ -18,6 +18,8 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +from journey import CLIENT_SUITES, client_suite, contract_errors, preflight as journey_preflight + try: import yaml except ImportError: # pragma: no cover @@ -781,6 +783,8 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int: continue if args.tag and case.get("tag") != args.tag: continue + if args.suite and client_suite(case) != args.suite: + continue if args.driver and case.get("driver") != args.driver: continue automation = str(case.get("automation") or "automated") @@ -810,6 +814,11 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int: selected, together = expand_run_together(meta, cases, selected, skipped, max_i, automation_filter) + journey_gate = journey_preflight(selected, sessions=meta.get("sessions") or {}) + if journey_gate["status"] == "blocked": + print(json.dumps({"preflight": journey_gate, "journey_preflight": journey_gate}, ensure_ascii=False, indent=2)) + return 2 + preflight = mobile_preflight( root, meta, @@ -840,6 +849,7 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int: "vars": vars_map, "artifacts": meta.get("artifacts") or {}, "preflight": preflight, + "journey_preflight": journey_gate, "skipped": skipped, "run_together": together, "status_probe": status_probes, @@ -922,6 +932,17 @@ def validate(root: Path) -> list[str]: if not isinstance(sessions, dict): errors.append("catalog.sessions 必须是 mapping") sessions = {} + client_suites = meta.get("client_suites") + if client_suites is not None: + if not isinstance(client_suites, dict): + errors.append("catalog.client_suites 必须是 mapping") + else: + unknown = sorted(set(client_suites) - set(CLIENT_SUITES)) + if unknown: + errors.append("catalog.client_suites 含非法套件: " + ", ".join(unknown)) + missing_suites = sorted(set(CLIENT_SUITES) - set(client_suites)) + if missing_suites: + errors.append("catalog.client_suites 缺少标准套件: " + ", ".join(missing_suites)) artifacts = meta.get("artifacts") if artifacts is not None: if not isinstance(artifacts, dict): @@ -998,6 +1019,7 @@ def validate(root: Path) -> list[str]: errors.append(f"{where}: category={cat} 与文件 category={file_cat} 不一致") if cat and cat_ids and cat not in cat_ids: errors.append(f"{where}: category={cat} 不在 catalog.categories") + errors.extend(contract_errors(case, sessions=sessions)) if case.get("lifecycle", "active") not in LIFECYCLES: errors.append(f"{where}: 非法 lifecycle") if case.get("tag") not in TAGS: @@ -1275,6 +1297,8 @@ def build_parser() -> argparse.ArgumentParser: p_sel.add_argument("--category") p_sel.add_argument("--id", action="append", dest="ids", default=[]) p_sel.add_argument("--tag", choices=TAGS) + p_sel.add_argument("--suite", "--client-suite", dest="suite", choices=CLIENT_SUITES, + help="客户端旅程套件") p_sel.add_argument("--max-risk") p_sel.add_argument("--driver", choices=DRIVERS) p_sel.add_argument( diff --git a/skills/tester/scripts/generate_cases_csv.py b/skills/tester/scripts/generate_cases_csv.py index c90d11b..f4c47b3 100755 --- a/skills/tester/scripts/generate_cases_csv.py +++ b/skills/tester/scripts/generate_cases_csv.py @@ -22,6 +22,8 @@ except ImportError: # pragma: no cover FIELDS = ( "category", "id", + "client_suite", + "platform", "maps_to", "title", "what", @@ -160,6 +162,8 @@ def export_cases_csv(root: Path, out: Path | None = None) -> Path: { "category": case.get("category") or "", "id": cid, + "client_suite": case.get("client_suite") or case.get("suite") or "", + "platform": one_line(case.get("platform")), "maps_to": case.get("maps_to") or "", "title": case.get("title") or "", "what": one_line(case.get("what")), diff --git a/skills/tester/scripts/generate_run_csv.py b/skills/tester/scripts/generate_run_csv.py index 40518fe..20bc1c5 100644 --- a/skills/tester/scripts/generate_run_csv.py +++ b/skills/tester/scripts/generate_run_csv.py @@ -25,6 +25,8 @@ EXECUTED_YES = frozenset({"pass", "fail", "environment-fail"}) EXECUTED_NO = frozenset({"skipped", "not-run"}) FIELDS = ( "id", + "client_suite", + "platform", "result", "executed", "evidence", @@ -132,6 +134,8 @@ def export_run_csv( writer.writerow( { "id": cid, + "client_suite": case.get("client_suite") or case.get("suite") or "", + "platform": one_line(case.get("platform")), "result": result, "executed": executed_cell(result), "evidence": one_line(item.get("evidence")), diff --git a/skills/tester/scripts/journey.py b/skills/tester/scripts/journey.py new file mode 100644 index 0000000..a09aee2 --- /dev/null +++ b/skills/tester/scripts/journey.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Shared client-journey contract and result lifecycle helpers. + +The mobile/browser driver still owns the real UI interaction. This module +owns the parts that must be identical for every client catalog: contract +validation, preflight blocking, and result propagation. It deliberately +returns the existing tester result vocabulary instead of introducing a second +report format. +""" + +from __future__ import annotations + +from typing import Any, Iterable, Mapping + + +CLIENT_SUITES = ("client-smoke", "client-regression", "safety") +PLATFORMS = ("android", "ios") +FAILURE_POLICIES = ("stop", "continue") +RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run") + + +def client_suite(case: Mapping[str, Any]) -> str: + """Return the canonical client suite, accepting ``suite`` as a migration alias.""" + value = case.get("client_suite") + alias = case.get("suite") + if value and alias and str(value) != str(alias): + return "" + return str(value or alias or "") + + +def normalize_platforms(value: Any) -> list[str]: + if isinstance(value, str): + values = [value] + elif isinstance(value, (list, tuple)): + values = list(value) + else: + return [] + return [str(item).strip().lower() for item in values if str(item).strip()] + + +def _non_empty_list(value: Any) -> bool: + return isinstance(value, list) and bool(value) and all(str(item).strip() for item in value) + + +def _has_account(fixture: Mapping[str, Any]) -> bool: + account = fixture.get("account") + if isinstance(account, Mapping): + return bool(str(account.get("role") or account.get("username") or "").strip()) + return bool(str(account or "").strip()) + + +def contract_errors(case: Mapping[str, Any], sessions: Mapping[str, Any] | None = None) -> list[str]: + """Validate a client journey without requiring app-specific field names. + + A case becomes a client journey when it declares ``client_suite`` (or the + temporary ``suite`` alias). Legacy non-client cases keep the existing + schema. Once opted in, all lifecycle inputs are explicit so a missing + fixture/cleanup can stop the run before any case result is fabricated. + """ + suite = client_suite(case) + journey = case.get("journey") + if not suite and journey is None: + return [] + errors: list[str] = [] + cid = str(case.get("id") or "?") + where = f"{cid}: client journey" + if case.get("client_suite") and case.get("suite") and str(case["client_suite"]) != str(case["suite"]): + errors.append(f"{where}: client_suite 与 suite 不一致") + if suite not in CLIENT_SUITES: + errors.append(f"{where}: client_suite 必须是 {', '.join(CLIENT_SUITES)}") + + if not isinstance(journey, Mapping): + errors.append(f"{where}: journey 必须是 mapping") + journey = {} + for key in ("goal", "entry_state", "final_state"): + if not str(journey.get(key) or "").strip(): + errors.append(f"{where}: journey.{key} 必填") + if not _non_empty_list(journey.get("actions")): + errors.append(f"{where}: journey.actions 必须是非空列表") + if not _non_empty_list(journey.get("assertions")): + errors.append(f"{where}: journey.assertions 必须是非空列表") + policy = str(journey.get("failure_policy") or "") + if policy not in FAILURE_POLICIES: + errors.append(f"{where}: journey.failure_policy 必须是 stop 或 continue") + + fixture = case.get("fixture") + if not isinstance(fixture, Mapping): + errors.append(f"{where}: fixture 缺失，必须在 preflight 前声明") + elif not _has_account(fixture): + errors.append(f"{where}: fixture.account 缺失，必须声明测试账号或角色") + + cleanup = case.get("cleanup") + if not isinstance(cleanup, Mapping): + errors.append(f"{where}: cleanup 缺失，必须在 preflight 前声明") + else: + steps = cleanup.get("steps") + command = cleanup.get("run") + explicitly_noop = cleanup.get("required") is False and str(cleanup.get("reason") or "").strip() + if not explicitly_noop and not _non_empty_list(steps) and not str(command or "").strip(): + errors.append(f"{where}: cleanup 必须有非空 steps/run；只读用例也要显式声明 reason") + + platforms = normalize_platforms(case.get("platform")) + if not platforms: + errors.append(f"{where}: platform 必须声明 android/ios") + elif any(platform not in PLATFORMS for platform in platforms): + errors.append(f"{where}: platform 只能是 {', '.join(PLATFORMS)}") + elif len(platforms) != len(set(platforms)): + errors.append(f"{where}: platform 不能重复") + + driver = case.get("driver") + if driver not in ("mobile", "browser"): + errors.append(f"{where}: 客户端旅程 driver 必须是 mobile 或 browser") + if driver and driver in ("mobile", "browser"): + session = case.get("session") + if not str(session or "").strip(): + errors.append(f"{where}: session 必须声明可复用客户端会话") + elif sessions is not None and session not in sessions: + errors.append(f"{where}: session={session} 未在 catalog.sessions 声明") + if suite == "safety" and str(case.get("tag") or "") == "smoke": + errors.append(f"{where}: safety 不得标记为 smoke") + return errors + + +def preflight(cases: Iterable[Mapping[str, Any]], sessions: Mapping[str, Any] | None = None) -> dict[str, Any]: + """Return a contract preflight result; no per-case result is created on block.""" + cases = list(cases) + errors: list[str] = [] + for case in cases: + errors.extend(contract_errors(case, sessions=sessions)) + return { + "required": bool(errors or any(client_suite(case) for case in cases)), + "status": "blocked" if errors else "ready", + "failure_class": "contract" if errors else None, + "errors": errors, + } + + +def classify_outcome( + *, + ok: bool, + environment_ready: bool = True, + phase: str = "journey", +) -> str: + """Classify one observed phase using the tester's five result states. + + A valid test account is part of the client journey: a failed login is a + product ``fail``. Only an unavailable environment is ``environment-fail``. + """ + if not environment_ready: + return "environment-fail" + if not ok: + return "fail" + return "pass" + + +def propagate_results( + case_ids: Iterable[str], + observed: Mapping[str, str], + *, + skipped: Mapping[str, str] | None = None, + continue_on_fail: bool = False, + policies: Mapping[str, str] | None = None, +) -> list[dict[str, str]]: + """Build ordered result rows and mark cases after a blocking failure not-run. + + ``skipped`` is intentional selection (risk/lifecycle/assumption); it is + never converted to ``not-run``. A product or environment failure stops + subsequent cases unless the caller explicitly opts into continuation. + """ + skipped = skipped or {} + policies = policies or {} + rows: list[dict[str, str]] = [] + blocked = False + for raw_id in case_ids: + case_id = str(raw_id) + if case_id in skipped: + rows.append({"id": case_id, "result": "skipped", "notes": skipped[case_id]}) + continue + if blocked: + rows.append({"id": case_id, "result": "not-run", "notes": "前置用例失败，未轮到执行"}) + continue + result = observed.get(case_id, "not-run") + if result not in RESULTS: + raise ValueError(f"未知结果: {case_id}={result}") + rows.append({"id": case_id, "result": result}) + policy = policies.get(case_id) + if result in ("fail", "environment-fail") and not (continue_on_fail or policy == "continue"): + blocked = True + return rows diff --git a/skills/tester/templates/catalog.yaml b/skills/tester/templates/catalog.yaml index 87e98bb..5a8ed2a 100644 --- a/skills/tester/templates/catalog.yaml +++ b/skills/tester/templates/catalog.yaml @@ -22,6 +22,13 @@ risks: - mutating - destructive default_max_risk: readonly +client_suites: + client-smoke: + default: true + client-regression: + default: false + safety: + default: false vars: {} authorize: mutating: AUTHORIZE_MUTATING diff --git a/skills/tester/tests/test_catalog_journey.py b/skills/tester/tests/test_catalog_journey.py new file mode 100644 index 0000000..24def5b --- /dev/null +++ b/skills/tester/tests/test_catalog_journey.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +CATALOG = ROOT / "scripts" / "catalog.py" + + +def write_fixture(root: Path, *, complete: bool = True) -> None: + tester = root / ".pouch" / "tester" + (tester / "cases").mkdir(parents=True) + (tester / "runs").mkdir() + (tester / "catalog.yaml").write_text( + yaml.safe_dump( + { + "version": 2, + "id_pattern": "^TC-[A-Z]+-[0-9]+$", + "categories": [{"id": "CLIENT"}], + "risks": ["readonly", "mutating", "destructive"], + "default_max_risk": "readonly", + "client_suites": { + "client-smoke": {"default": True}, + "client-regression": {"default": False}, + "safety": {"default": False}, + }, + "sessions": {"browser": {"kind": "browser", "reuse": "run"}}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + case = { + "id": "TC-CLIENT-001", + "category": "CLIENT", + "title": "journey", + "what": "client journey", + "lifecycle": "active", + "tag": "smoke", + "risk": "readonly", + "driver": "browser", + "session": "browser", + "client_suite": "client-smoke", + "platform": ["android", "ios"], + "journey": { + "goal": "goal", + "entry_state": "logged_out", + "actions": ["start", "login"], + "assertions": ["visible"], + "final_state": "done", + "failure_policy": "stop", + }, + "fixture": {"account": "test-role"}, + "cleanup": {"steps": ["cleanup"]}, + "how": {"steps": ["start", "login"]}, + "expected": [{"kind": "visible-text", "value": "done"}], + } + if not complete: + case.pop("fixture") + case.pop("cleanup") + (tester / "cases" / "client.yaml").write_text( + yaml.safe_dump({"category": "CLIENT", "cases": [case]}, sort_keys=False), + encoding="utf-8", + ) + + +class CatalogJourneyTests(unittest.TestCase): + def run_catalog(self, root: Path, *args: str): + return subprocess.run( + [sys.executable, str(CATALOG), *args, "--root", str(root)], + text=True, + capture_output=True, + ) + + def test_valid_client_catalog_and_suite_filter(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + write_fixture(root) + valid = self.run_catalog(root, "validate") + self.assertEqual(valid.returncode, 0, valid.stderr) + selected = self.run_catalog(root, "select", "--suite", "client-smoke") + self.assertEqual(selected.returncode, 0, selected.stderr) + self.assertEqual(json.loads(selected.stdout)["count"], 1) + + def test_missing_fixture_or_cleanup_blocks_before_results(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + write_fixture(root, complete=False) + selected = self.run_catalog(root, "select", "--suite", "client-smoke") + self.assertNotEqual(selected.returncode, 0) + payload = json.loads(selected.stdout) + self.assertEqual(payload["preflight"]["failure_class"], "contract") + self.assertTrue(any("fixture 缺失" in error for error in payload["preflight"]["errors"])) + self.assertEqual(payload["preflight"]["status"], "blocked") + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/tester/tests/test_journey.py b/skills/tester/tests/test_journey.py new file mode 100644 index 0000000..61e4e98 --- /dev/null +++ b/skills/tester/tests/test_journey.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from journey import classify_outcome, contract_errors, propagate_results + + +def valid_case(**overrides): + case = { + "id": "TC-CLIENT-001", + "client_suite": "client-smoke", + "driver": "browser", + "session": "browser", + "platform": ["android", "ios"], + "journey": { + "goal": "完成客户端目标", + "entry_state": "logged_out", + "actions": ["启动", "登录", "完成目标"], + "assertions": ["看到目标页面"], + "final_state": "目标已完成", + "failure_policy": "stop", + }, + "fixture": {"account": "test-role", "data": []}, + "cleanup": {"steps": ["清理本轮数据"]}, + } + case.update(overrides) + return case + + +class JourneyContractTests(unittest.TestCase): + def test_contract_accepts_shared_client_shape(self): + self.assertEqual(contract_errors(valid_case(), {"browser": {}}), []) + + def test_missing_fixture_and_cleanup_are_contract_errors(self): + errors = contract_errors( + valid_case(fixture=None, cleanup=None), + {"browser": {}}, + ) + self.assertTrue(any("fixture 缺失" in error for error in errors)) + self.assertTrue(any("cleanup 缺失" in error for error in errors)) + + def test_login_failure_is_product_failure_but_unavailable_environment_is_not(self): + self.assertEqual(classify_outcome(ok=False, phase="login"), "fail") + self.assertEqual( + classify_outcome(ok=False, phase="login", environment_ready=False), + "environment-fail", + ) + + def test_failed_case_blocks_remaining_cases_and_keeps_explicit_skip(self): + rows = propagate_results( + ["TC-1", "TC-2", "TC-3"], + {"TC-1": "fail", "TC-2": "pass", "TC-3": "pass"}, + skipped={"TC-3": "未授权"}, + ) + self.assertEqual([row["result"] for row in rows], ["fail", "not-run", "skipped"]) + + def test_continue_policy_allows_next_case(self): + rows = propagate_results( + ["TC-1", "TC-2"], + {"TC-1": "fail", "TC-2": "pass"}, + policies={"TC-1": "continue"}, + ) + self.assertEqual([row["result"] for row in rows], ["fail", "pass"]) + + +if __name__ == "__main__": + unittest.main()