From 3a18a27dc60e857f1af1fc78d6881c08e67d40c0 Mon Sep 17 00:00:00 2001 From: laily Date: Thu, 15 Oct 2026 18:11:28 +0800 Subject: [PATCH] feat(tester): add integration-first journey execution --- skills/tester/README.md | 17 ++ skills/tester/SKILL.md | 21 +++ skills/tester/references/concepts.md | 2 + skills/tester/references/run.md | 36 +++- skills/tester/references/schema.md | 21 ++- skills/tester/scripts/catalog.py | 57 +++++- skills/tester/scripts/generate_cases_csv.py | 11 +- skills/tester/scripts/generate_run_csv.py | 10 +- skills/tester/scripts/journey.py | 104 ++++++++++- skills/tester/scripts/mobile_bundle.py | 90 +++++++++- skills/tester/scripts/two_tier.py | 125 +++++++++++++ skills/tester/templates/catalog.yaml | 7 + skills/tester/tests/test_two_tier.py | 190 ++++++++++++++++++++ 13 files changed, 662 insertions(+), 29 deletions(-) create mode 100644 skills/tester/scripts/two_tier.py create mode 100644 skills/tester/tests/test_two_tier.py diff --git a/skills/tester/README.md b/skills/tester/README.md index 9743d00..12e0238 100644 --- a/skills/tester/README.md +++ b/skills/tester/README.md @@ -88,6 +88,23 @@ APK/IPA，为本轮创建隔离 worktree，在 Android 和 iOS tester 节点安 ### 客户端用户旅程 +测试用例支持两种顶层类型：`test_type: integration` 和 `test_type: journey`。集成套件 +至少包括 `api-contract`、`authorization`、`build-contract`；客户端套件仍是 +`client-smoke`、`client-regression`、`safety`。没有显式范围时默认不选择 integration。 + +执行顺序固定为 `test-preflight -> integration -> journey`。preflight 状态为 blocked 或 +environment-fail 时不创建客户端 session；integration 失败时依赖 journey 为 `not-run`。 +服务不可达是 environment-fail，契约断言错误是 integration fail，正确账号登录失败仍是 +journey fail。`tester run`/bundle 必须传入 caller 提供的 matrix： + +```bash +/tester run --preflight-config /abs/preflight.yaml \ + --artifact android=/abs/app.apk --artifact ios=/abs/app.ipa +``` + +没有真实 deployer、Mac 或设备时，只能使用 test-preflight 的 command-contract mock fixture +验证协议，不能把 mock 结果写成真实运行成功。 + 客户端 E2E 用例使用 `client_suite: client-smoke | client-regression | safety`，并明确 `journey`、`fixture`、`session`、`cleanup`、`platform`。标准生命周期是： diff --git a/skills/tester/SKILL.md b/skills/tester/SKILL.md index 2cbc002..fd90440 100644 --- a/skills/tester/SKILL.md +++ b/skills/tester/SKILL.md @@ -148,6 +148,27 @@ python3 /scripts/catalog.py show --root --id ### 客户端用户旅程标准 +### 双层测试与执行顺序 + +用例顶层类型是 `test_type: integration | journey`。旧目录没有该字段时保持兼容：已有 +`client_suite`/`journey` 视为 journey，已有 `integration_suite` 视为 integration。集成 +套件至少包括 `api-contract`、`authorization`、`build-contract`；旅程套件仍固定为 +`client-smoke`、`client-regression`、`safety`。集成用例不需要客户端 session、platform +或 journey 字段，不能被客户端套件选中。 + +一次运行严格遵守： + +```text +test-preflight (blocked/environment-fail -> 不创建 session) + -> integration + -> journey (integration fail -> not-run，不创建 session) +``` + +`test-preflight` 的报告是环境证据，不计入 testcase 总数。服务不可达记 +`environment-fail`；服务已达但契约断言错误记 integration `fail`；正确测试账号登录失败 +仍记 journey `fail`。tester 的 `run`/bundle 入口必须接收 caller 提供的 preflight matrix， +没有 matrix 不得绕过闸门或伪造成功。 + 声明 `client_suite`（迁移期也接受 `suite`）的用例采用固定的三套客户端集合： `client-smoke`、`client-regression`、`safety`。前两者分别覆盖最短只读主路径和隔离测试 环境中的普通用户回归；`safety` 覆盖危险/权限边界场景，必须显式选择和授权，不能被 diff --git a/skills/tester/references/concepts.md b/skills/tester/references/concepts.md index d303754..67412d8 100644 --- a/skills/tester/references/concepts.md +++ b/skills/tester/references/concepts.md @@ -41,6 +41,8 @@ catalog（整个测试集） | `expected` | 怎么算过：`{kind, value}` 列表 | pytest `assert`；Robot `Should …` | 不是步骤 | | `run_together` | 必须成组执行的用例（有序、可选 continue_on_fail） | 场景组（scenario） | 不是 suite，不是 tag | | `client_suite` | 客户端用户旅程套件：`client-smoke` / `client-regression` / `safety` | mobile/browser E2E suite | 不是 `category`，也不是通用 tag；迁移期可读 `suite` 别名 | +| `test_type` | 顶层测试类型：`journey` 或 `integration` | E2E vs service/contract test | 不是 driver；旧目录可由既有字段推断 | +| `integration_suite` | 集成套件：`api-contract` / `authorization` / `build-contract` | API/contract test suite | 不是客户端 `client_suite`；迁移期可读 `suite` 别名 | | `journey` | 用户目标、入口、操作、UI 断言、最终状态和失败继续策略 | Cucumber scenario / Robot task | 不是 API contract | | `fixture` | 旅程开始前声明的测试账号与隔离数据 | pytest fixture | 不是运行中临时猜出的数据 | | `cleanup` | 本轮数据的 teardown 契约；缺失会在 preflight 阻断 | pytest teardown / Robot teardown | 不是测试通过后的可选备注 | diff --git a/skills/tester/references/run.md b/skills/tester/references/run.md index bd86131..3383277 100644 --- a/skills/tester/references/run.md +++ b/skills/tester/references/run.md @@ -9,6 +9,11 @@ Agent、执行目标和临时 terminal 的完整协议见 [execution.md](executi ## 0. 建立测试矩阵并完成全量 preflight +`test-preflight` 是 integration 和 journey 的共同硬闸。tester 必须先调用其 `verify` +或 `prepare`，保存独立的 preflight YAML/CSV；状态为 `blocked` 或 `environment-fail` +时不创建 browser/mobile session，所有尚未执行的 testcase 记 `not-run`。preflight 证据 +不算 testcase，也不进入 testcase CSV/YAML 的总数。 + 测试不是“找到一个能跑的设备就开始”。每轮先建立一份测试矩阵；移动端每个目标产物 必须单独占一行： @@ -76,6 +81,7 @@ python3 /scripts/catalog.py select --root \ [--category CAT] [--id ID]... [--tag smoke|full|slow] \ [--max-risk ] [--driver script|browser|mobile] \ [--automation automated|assisted|manual|any] \ + [--test-type integration|journey] [--suite ] \ [--platform android|ios]... \ [--artifact android=] [--artifact ios=] \ [--binding android=tester.android] [--binding ios=tester.ios] @@ -90,6 +96,15 @@ python3 /scripts/catalog.py select --root --suite client-smoke `client-smoke`、`client-regression`、`safety` 是 `client_suite` 的固定值，不替换既有 `tag=smoke|full|slow`。`safety` 不参与未指定范围的默认选择。 +集成测试显式选择： + +```bash +python3 /scripts/catalog.py select --root --test-type integration +python3 /scripts/catalog.py select --root --suite api-contract +``` + +未指定 `--test-type`/`--suite` 时，默认选择不会包含 `test_type=integration`。 + 选中任一 `driver=mobile` 用例时，`--platform`、每个平台的 `--artifact` 和对应 `tester` binding 是硬性参数；`catalog.py select` 会检查产物文件/下载地址和 `.pouch/agent.yaml` 中的 binding。缺任何一项直接返回非零并打印 `preflight: blocked`， @@ -106,7 +121,18 @@ python3 /scripts/catalog.py select --root --suite client-smoke 无命中：停下说明筛选条件，不要扩大范围。 -## 2. 运行前闸 +## 2. 双层执行 + +preflight 通过后，先按 catalog 顺序执行 `integration`，再按客户端套件顺序执行 +`journey`。集成服务不可达是 `environment-fail`；服务已达但 API/权限/数据契约断言错误 +是 `fail`。任何 integration `fail` 都阻断依赖的 journey，并把它们记为 `not-run`；此时 +不得创建客户端 session。正确测试账号登录失败发生在客户端已启动后，仍是产品 `fail`。 + +运行器可复用 `scripts/two_tier.py` 的 `run_integration_first`，它只接收外部 driver +回调，不实现 API、浏览器或移动执行器；mock fixture 只能验证调用顺序和失败传播，不能 +冒充真实部署、Mac 或设备成功。 + +## 3. 运行前闸 - `HOST` 未设（环境变量 / `local.yaml` 都没有）：全部停，environment-fail。 - 命中条的 `risk` 在 catalog `authorize` 里有对应变量：变量不是 `true` 则该条 `skipped`。有 `gate` 时还要 `catalog.gates` 对应变量为 `true`；`assume` 不成立同样 `skipped`。即使用户点了 id 也不执行。 @@ -123,7 +149,7 @@ python3 /scripts/catalog.py select --root --suite client-smoke - 探测不可达或命中 `stop_if`：在 preflight 阶段记录对应分类/平台的不可用项并停止整轮； 先返回用户处理，不创建后续测试 session。只有所有本轮目标环境通过，才进入用例执行。 -## 2.1 移动端平台环境闸门与用户引导 +## 3.1 移动端平台环境闸门与用户引导 `driver=mobile` 不是一个可以互换的平台。运行前必须根据 catalog session、用例变量或用户指定，明确本轮是 Android 还是 iOS；无法判断时先停下询问平台，不得猜测。两类平台分别执行以下闸门： @@ -184,7 +210,7 @@ Android+iOS 默认是全量 preflight 闸门：任一项不满足，另一平台 环境引导只负责让用户补齐前置条件；除 catalog 已明确授权的 session 初始化外，不自动安装 SDK、配对真机、修改系统安全设置或上传产物。 -## 3. Session +## 4. Session 看命中条用到的 `session` 键： @@ -193,7 +219,7 @@ Android+iOS 默认是全量 preflight 闸门：任一项不满足，另一平台 - `reuse: run`：整轮结束再关。`persist`：不关。 - 中途 session 挂了：后续同 driver 标 environment-fail，不要标产品失败。 -## 4. 执行 +## 5. 执行 按 select 返回的顺序，一条一条来。select 按 `catalog.yaml` 的 `categories` 顺序，同一分类内按该 yaml 列表顺序（不是文件名字母序）。默认失败即停（产品 `fail` 或探测到的不安全状态）。用户说继续 / keep-going 才往后跑。 @@ -225,7 +251,7 @@ python3 /scripts/catalog.py run-script --root --id 挂到已有 session，执行 `how.steps`，对照 `expected`。必须留下可见文本、URL 或交互证据。 -## 5. 记录与报告 +## 6. 记录与报告 每条：`id, result, evidence, notes`。`result` 只能是 `pass` / `fail` / `environment-fail` / `skipped` / `not-run`。`skipped` 是主动不跑（retired / pending / 超风险上限 / 未授权 / `assume` 不成立），`not-run` 是没轮到（失败即停后剩下的 / 被前置整组挡住的 / 需人工而无人值守的）。 diff --git a/skills/tester/references/schema.md b/skills/tester/references/schema.md index b2e8bb1..0c51fb1 100644 --- a/skills/tester/references/schema.md +++ b/skills/tester/references/schema.md @@ -14,10 +14,9 @@ runs/.yaml 每次运行快照（执行源）；run-id 必须唯一 runs/*.csv 同一次运行的成绩单；generate_run_csv.py 从对应 yaml 生成 ``` -客户端旅程在上述目录中仍使用同一套 cases/runs 和五态报告，不另造成绩单。启用旅程 -契约的用例声明 `client_suite`（兼容迁移期别名 `suite`）；catalog 可用 -`client_suites` 显式列出三个标准套件：`client-smoke`、`client-regression`、`safety`。 -`category` 仍是文件/执行顺序容器，不被套件字段替代。 +两类用例共用同一套 cases/runs 和五态报告，不另造成绩单。新目录显式声明顶层 +`test_type: journey | integration`；旧目录没有该字段时按已有 `client_suite`/`journey` +或 `integration_suite` 推断。`category` 仍是文件/执行顺序容器，不被套件字段替代。 ## catalog.yaml @@ -35,6 +34,7 @@ runs/*.csv 同一次运行的成绩单；generate_run_csv.py 从对 | `run_together` | 可选。成组执行：`[{ids: [TC-…, TC-…], continue_on_fail: true}]`。点到组内任一条，`select` 会按 `ids` 顺序把整组扩进命中列表。`continue_on_fail: true` 时组内前一条产品 fail 仍跑后一条（返回/收尾）。一个 id 只能属于一组。 | | `status_probe` | 可选。顶层默认值，一个**分类级前置条件**：`{url, stop_if: {字段: [命中值]}}`。运行前请求 `url`，响应命中任一 `stop_if` 条件即视为前置不满足。分类项可用同名键覆盖：缺省继承顶层、`false` 关闭、mapping 整体替换。命中或不可达只影响用它的分类，不是产品失败。业界对应 pytest 的 session fixture / Robot 的 Suite Setup。`select` 会按分类返回解析后的生效值，`url` 已用 `vars` 展开 | | `client_suites` | 可选的标准套件声明；出现时必须完整包含 `client-smoke`、`client-regression`、`safety`。缺省运行仍按 `tag` 兼容旧目录 | +| `integration_suites` | 可选的集成套件声明；标准值为 `api-contract`、`authorization`、`build-contract` | `local.yaml` 与环境变量覆盖 `HOST` 和授权开关。模板中的 `HOST` 默认是 `127.0.0.1`；远程目标必须显式覆盖。查找顺序：环境变量 > `local.yaml` > @@ -76,6 +76,7 @@ terminal；运行结束用 `orca terminal close --terminal ` 关闭本 | 字段 | 必填 | 含义 | |------|------|------| | `id` | 是 | 匹配 `id_pattern` | +| `test_type` | 新用例是 | `journey` 或 `integration`；缺省兼容旧目录并按字段推断 | | `category` | 是 | 必须等于文件声明的 category | | `title` | 是 | 短名 | | `what` | 是 | 一句可观察的「测什么」 | @@ -94,6 +95,18 @@ terminal；运行结束用 `orca terminal close --terminal ` 关闭本 | `teardown` | 否 | 跑完回退 | | `notes` | 否 | 备注 | +### 集成测试用例契约 + +`test_type: integration` 的用例不创建 browser/mobile session，也不能声明 `platform` 或 +客户端 `journey`。必须有 `integration_suite`（迁移期可用 `suite`）且值为 +`api-contract`、`authorization` 或 `build-contract`；`fixture` 必须是非空 mapping； +`request`、`input`、`call`、`integration` 之一或旧式 `how` 用于描述调用；`expected` 是 +协议/数据断言。`mutating`/`destructive` 集成用例必须提供 `cleanup.steps` 或 +`cleanup.run`（只读探针可用 `required: false` + `reason`）。 + +选择 `--suite api-contract` 或 `--test-type integration` 才会进入集成阶段；无显式类型 +时的默认选择不会包含 integration。 + ### 客户端旅程用例契约 声明 `client_suite` 后，以下字段都必须存在；`catalog.py validate` 和 `select` 使用同一 diff --git a/skills/tester/scripts/catalog.py b/skills/tester/scripts/catalog.py index 00974fa..7c64323 100755 --- a/skills/tester/scripts/catalog.py +++ b/skills/tester/scripts/catalog.py @@ -18,7 +18,15 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any -from journey import CLIENT_SUITES, client_suite, contract_errors, preflight as journey_preflight +from journey import ( + CLIENT_SUITES, + INTEGRATION_SUITES, + TEST_TYPES, + contract_errors, + preflight as journey_preflight, + suite as case_suite, + test_type, +) try: import yaml @@ -370,7 +378,7 @@ def cmd_list(root: Path) -> int: pending = [c for c in cases if c.get("lifecycle") == "pending"] print(f"## 测试目录 active {len(active)} / pending {len(pending)} / retired {len(retired)}") print() - headers = ["分类", "标题", "条数", "smoke"] + risks + list(DRIVERS) + headers = ["分类", "标题", "条数", "journey", "integration", "smoke"] + risks + list(DRIVERS) print("| " + " | ".join(headers) + " |") print("| " + " | ".join("---" for _ in headers) + " |") @@ -388,7 +396,14 @@ def cmd_list(root: Path) -> int: title = (cats_meta.get(cat) or {}).get("title") or ( group[0].get("_file_title") if group else "" ) - row = [cat, str(title), str(len(group)), str(sum(1 for c in group if c.get("tag") == "smoke"))] + row = [ + cat, + str(title), + str(len(group)), + str(sum(1 for c in group if test_type(c) == "journey")), + str(sum(1 for c in group if test_type(c) == "integration")), + str(sum(1 for c in group if c.get("tag") == "smoke")), + ] for risk in risks: row.append(str(sum(1 for c in group if c.get("risk") == risk))) for driver in DRIVERS: @@ -783,7 +798,13 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int: continue if args.tag and case.get("tag") != args.tag: continue - if args.suite and client_suite(case) != args.suite: + if args.test_type and test_type(case) != args.test_type: + continue + if args.suite and case_suite(case) != args.suite: + continue + # A mixed catalog keeps legacy cases runnable, but the implicit + # default is the client journey surface. Integration is opt-in. + if not args.test_type and not args.suite and test_type(case) == "integration": continue if args.driver and case.get("driver") != args.driver: continue @@ -814,6 +835,8 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int: selected, together = expand_run_together(meta, cases, selected, skipped, max_i, automation_filter) + # The historical key is kept in the JSON, but the gate now validates both + # test types before any caller can start a phase. journey_gate = journey_preflight(selected, sessions=meta.get("sessions") or {}) if journey_gate["status"] == "blocked": print(json.dumps({"preflight": journey_gate, "journey_preflight": journey_gate}, ensure_ascii=False, indent=2)) @@ -850,6 +873,10 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int: "artifacts": meta.get("artifacts") or {}, "preflight": preflight, "journey_preflight": journey_gate, + "test_types": { + "journey": sum(1 for case in selected if test_type(case) == "journey"), + "integration": sum(1 for case in selected if test_type(case) == "integration"), + }, "skipped": skipped, "run_together": together, "status_probe": status_probes, @@ -943,6 +970,17 @@ def validate(root: Path) -> list[str]: missing_suites = sorted(set(CLIENT_SUITES) - set(client_suites)) if missing_suites: errors.append("catalog.client_suites 缺少标准套件: " + ", ".join(missing_suites)) + integration_suites = meta.get("integration_suites") + if integration_suites is not None: + if not isinstance(integration_suites, dict): + errors.append("catalog.integration_suites 必须是 mapping") + else: + unknown = sorted(set(integration_suites) - set(INTEGRATION_SUITES)) + if unknown: + errors.append("catalog.integration_suites 含非法套件: " + ", ".join(unknown)) + missing_suites = sorted(set(INTEGRATION_SUITES) - set(integration_suites)) + if missing_suites: + errors.append("catalog.integration_suites 缺少标准套件: " + ", ".join(missing_suites)) artifacts = meta.get("artifacts") if artifacts is not None: if not isinstance(artifacts, dict): @@ -1297,8 +1335,15 @@ def build_parser() -> argparse.ArgumentParser: p_sel.add_argument("--category") p_sel.add_argument("--id", action="append", dest="ids", default=[]) p_sel.add_argument("--tag", choices=TAGS) - p_sel.add_argument("--suite", "--client-suite", dest="suite", choices=CLIENT_SUITES, - help="客户端旅程套件") + p_sel.add_argument( + "--suite", "--client-suite", dest="suite", + choices=[*CLIENT_SUITES, *INTEGRATION_SUITES], + help="客户端或集成测试套件；不传时默认不选 integration", + ) + p_sel.add_argument( + "--test-type", choices=TEST_TYPES, default=None, + help="按顶层 test_type 筛选；integration 必须显式选择", + ) p_sel.add_argument("--max-risk") p_sel.add_argument("--driver", choices=DRIVERS) p_sel.add_argument( diff --git a/skills/tester/scripts/generate_cases_csv.py b/skills/tester/scripts/generate_cases_csv.py index f4c47b3..636396c 100755 --- a/skills/tester/scripts/generate_cases_csv.py +++ b/skills/tester/scripts/generate_cases_csv.py @@ -13,6 +13,8 @@ import sys from pathlib import Path from typing import Any +from journey import suite as case_suite, test_type + try: import yaml except ImportError: # pragma: no cover @@ -22,6 +24,8 @@ except ImportError: # pragma: no cover FIELDS = ( "category", "id", + "test_type", + "suite", "client_suite", "platform", "maps_to", @@ -158,11 +162,16 @@ def export_cases_csv(root: Path, out: Path | None = None) -> Path: writer.writeheader() for case in cases: cid = str(case.get("id") or "") + client_suite_value = case.get("client_suite") or ( + case.get("suite") if test_type(case) == "journey" else "" + ) writer.writerow( { "category": case.get("category") or "", "id": cid, - "client_suite": case.get("client_suite") or case.get("suite") or "", + "test_type": test_type(case), + "suite": case_suite(case), + "client_suite": client_suite_value or "", "platform": one_line(case.get("platform")), "maps_to": case.get("maps_to") or "", "title": case.get("title") or "", diff --git a/skills/tester/scripts/generate_run_csv.py b/skills/tester/scripts/generate_run_csv.py index 20bc1c5..e7272d3 100644 --- a/skills/tester/scripts/generate_run_csv.py +++ b/skills/tester/scripts/generate_run_csv.py @@ -18,6 +18,7 @@ except ImportError: # pragma: no cover raise SystemExit(2) from generate_cases_csv import load_cases, one_line, resolve_root, tester_dir +from journey import suite as case_suite, test_type RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run") RUN_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") @@ -25,6 +26,8 @@ EXECUTED_YES = frozenset({"pass", "fail", "environment-fail"}) EXECUTED_NO = frozenset({"skipped", "not-run"}) FIELDS = ( "id", + "test_type", + "suite", "client_suite", "platform", "result", @@ -131,10 +134,15 @@ def export_run_csv( cid = str(item["id"]) result = str(item.get("result") or "") case = cases.get(cid) or {} + client_suite_value = case.get("client_suite") or ( + case.get("suite") if test_type(case) == "journey" else "" + ) writer.writerow( { "id": cid, - "client_suite": case.get("client_suite") or case.get("suite") or "", + "test_type": test_type(case), + "suite": case_suite(case), + "client_suite": client_suite_value or "", "platform": one_line(case.get("platform")), "result": result, "executed": executed_cell(result), diff --git a/skills/tester/scripts/journey.py b/skills/tester/scripts/journey.py index a09aee2..10eca98 100644 --- a/skills/tester/scripts/journey.py +++ b/skills/tester/scripts/journey.py @@ -13,7 +13,9 @@ from __future__ import annotations from typing import Any, Iterable, Mapping +TEST_TYPES = ("journey", "integration") CLIENT_SUITES = ("client-smoke", "client-regression", "safety") +INTEGRATION_SUITES = ("api-contract", "authorization", "build-contract") PLATFORMS = ("android", "ios") FAILURE_POLICIES = ("stop", "continue") RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run") @@ -28,6 +30,33 @@ def client_suite(case: Mapping[str, Any]) -> str: return str(value or alias or "") +def integration_suite(case: Mapping[str, Any]) -> str: + """Return the integration suite, accepting ``suite`` during migration.""" + value = case.get("integration_suite") + alias = case.get("suite") + if value and alias and str(value) != str(alias): + return "" + return str(value or alias or "") + + +def test_type(case: Mapping[str, Any]) -> str: + """Return an explicit type or infer one for an old catalog entry.""" + declared = str(case.get("test_type") or "").strip().lower() + if declared: + return declared + if client_suite(case) or case.get("journey") is not None: + return "journey" + if integration_suite(case): + return "integration" + return "" + + +def suite(case: Mapping[str, Any]) -> str: + """Return the report/selection suite for either test type.""" + kind = test_type(case) + return client_suite(case) if kind == "journey" else integration_suite(case) + + def normalize_platforms(value: Any) -> list[str]: if isinstance(value, str): values = [value] @@ -57,16 +86,23 @@ def contract_errors(case: Mapping[str, Any], sessions: Mapping[str, Any] | None schema. Once opted in, all lifecycle inputs are explicit so a missing fixture/cleanup can stop the run before any case result is fabricated. """ - suite = client_suite(case) - journey = case.get("journey") - if not suite and journey is None: + kind = test_type(case) + declared_kind = str(case.get("test_type") or "").strip().lower() + if declared_kind and declared_kind not in TEST_TYPES: + return [f"{case.get('id') or '?'}: test_type 必须是 journey 或 integration"] + if not kind: return [] + if kind == "integration": + return integration_contract_errors(case) + + suite_name = client_suite(case) + journey = case.get("journey") errors: list[str] = [] cid = str(case.get("id") or "?") where = f"{cid}: client journey" if case.get("client_suite") and case.get("suite") and str(case["client_suite"]) != str(case["suite"]): errors.append(f"{where}: client_suite 与 suite 不一致") - if suite not in CLIENT_SUITES: + if suite_name not in CLIENT_SUITES: errors.append(f"{where}: client_suite 必须是 {', '.join(CLIENT_SUITES)}") if not isinstance(journey, Mapping): @@ -116,11 +152,69 @@ def contract_errors(case: Mapping[str, Any], sessions: Mapping[str, Any] | None errors.append(f"{where}: session 必须声明可复用客户端会话") elif sessions is not None and session not in sessions: errors.append(f"{where}: session={session} 未在 catalog.sessions 声明") - if suite == "safety" and str(case.get("tag") or "") == "smoke": + if suite_name == "safety" and str(case.get("tag") or "") == "smoke": errors.append(f"{where}: safety 不得标记为 smoke") return errors +def integration_contract_errors(case: Mapping[str, Any]) -> list[str]: + """Validate the non-client half of the tester catalog. + + Integration cases deliberately do not require a browser/mobile session. A + ``how`` block is the legacy call declaration; newer catalogs may use + ``request``/``input``/``call`` or an ``integration`` mapping as well. + ``expected`` remains the common protocol/data assertion list. + """ + cid = str(case.get("id") or "?") + where = f"{cid}: integration" + errors: list[str] = [] + value = case.get("integration_suite") + alias = case.get("suite") + if value and alias and str(value) != str(alias): + errors.append(f"{where}: integration_suite 与 suite 不一致") + suite_name = integration_suite(case) + if suite_name not in INTEGRATION_SUITES: + errors.append( + f"{where}: integration_suite 必须是 {', '.join(INTEGRATION_SUITES)}" + ) + + fixture = case.get("fixture") + if not isinstance(fixture, Mapping) or not fixture: + errors.append(f"{where}: fixture 缺失，必须在 preflight 前声明") + + declared_call = any( + key in case and case.get(key) not in (None, "", [], {}) + for key in ("request", "input", "call", "integration") + ) + how = case.get("how") + if not declared_call and not isinstance(how, Mapping): + errors.append(f"{where}: 必须声明 request/input/call/integration 或 how") + + # Integration writes need an explicit rollback/cleanup contract. Read-only + # probes may state why cleanup is unnecessary, matching test-preflight. + risk = str(case.get("risk") or "") + cleanup = case.get("cleanup") + cleanup_steps = isinstance(cleanup, Mapping) and ( + bool(cleanup.get("steps")) or bool(str(cleanup.get("run") or "").strip()) + ) + cleanup_noop = isinstance(cleanup, Mapping) and cleanup.get("required") is False and bool( + str(cleanup.get("reason") or "").strip() + ) + if risk != "readonly" and not (cleanup_steps or cleanup_noop): + errors.append(f"{where}: 写入型 integration 必须声明 cleanup/rollback steps/run") + elif cleanup is not None and not (cleanup_steps or cleanup_noop): + errors.append(f"{where}: cleanup 必须有 steps/run 或 required=false + reason") + + forbidden = [] + if case.get("client_suite") or case.get("journey") is not None: + forbidden.append("client_suite/journey") + if case.get("session") or case.get("platform"): + forbidden.append("session/platform") + if forbidden: + errors.append(f"{where}: integration 不应声明 " + ", ".join(forbidden)) + return errors + + def preflight(cases: Iterable[Mapping[str, Any]], sessions: Mapping[str, Any] | None = None) -> dict[str, Any]: """Return a contract preflight result; no per-case result is created on block.""" cases = list(cases) diff --git a/skills/tester/scripts/mobile_bundle.py b/skills/tester/scripts/mobile_bundle.py index 3a01054..3b4c8a9 100755 --- a/skills/tester/scripts/mobile_bundle.py +++ b/skills/tester/scripts/mobile_bundle.py @@ -11,6 +11,7 @@ from __future__ import annotations import argparse import hashlib +import importlib.util import json import os import plistlib @@ -33,6 +34,7 @@ from typing import Any COMMIT = re.compile(r"^[0-9a-fA-F]{40}$") PLATFORMS = {"android", "ios"} +PREFLIGHT_ROOT = Path(__file__).resolve().parents[2] / "test-preflight" / "scripts" / "preflight.py" class EnvironmentFailure(RuntimeError): @@ -388,6 +390,33 @@ def write_run(root: Path, record: dict[str, Any]) -> Path: return path +def run_test_preflight( + config_path: Path, + *, + mode: str = "verify", + yaml_path: Path | None = None, + csv_path: Path | None = None, +) -> dict[str, Any]: + """Run the shared preflight contract before any client session work.""" + if mode not in {"verify", "prepare"}: + raise ArtifactFailure("preflight mode 只能是 verify 或 prepare") + if not PREFLIGHT_ROOT.is_file(): + raise EnvironmentFailure(f"找不到 test-preflight skill: {PREFLIGHT_ROOT}") + spec = importlib.util.spec_from_file_location("pouch_test_preflight", PREFLIGHT_ROOT) + if spec is None or spec.loader is None: + raise EnvironmentFailure("无法加载 test-preflight skill") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + try: + config = module.load_config(config_path) + evidence = module.run(config, mode) + yaml_path = yaml_path or config_path.with_name("preflight.yaml") + csv_path = csv_path or config_path.with_name("preflight.csv") + return module.write_evidence(evidence, yaml_path, csv_path) + except (OSError, ValueError, module.yaml.YAMLError) as exc: + return {"status": "blocked", "error": str(exc), "yaml": str(yaml_path or ""), "csv": str(csv_path or "")} + + def execute(args: argparse.Namespace) -> int: requested = platforms(args.platform) context = RuntimeContext(Path(args.root).resolve()) @@ -417,7 +446,7 @@ def execute(args: argparse.Namespace) -> int: artifact_dir = Path(tempfile.mkdtemp(prefix="pouch-tester-artifacts-")) record["artifact_dir"] = str(artifact_dir) expected = catalog_identities(context.root) - prepared: dict[str, tuple[dict[str, Any], dict[str, str], str]] = {} + prepared: dict[str, tuple[dict[str, Any], dict[str, str] | None, str]] = {} failures: dict[str, str] = {} for platform in requested: result: dict[str, Any] = {"node": "tester.android" if platform == "android" else "tester.ios", "status": "preparing"} @@ -429,22 +458,65 @@ def execute(args: argparse.Namespace) -> int: wanted = expected.get(platform) if wanted and identity != wanted: raise EnvironmentFailure(f"{platform} 身份不匹配: expected={wanted} actual={identity}") - node = android_preflight(context) if platform == "android" else ios_preflight(context) - check_network(context, node) - result.update({"identity": identity, "node_preflight": node, "status": "preflight-passed"}) - prepared[platform] = (result, node, identity) + result.update({"identity": identity, "status": "artifact-ready"}) + prepared[platform] = (result, None, identity) except (EnvironmentFailure, ArtifactFailure) as exc: failures[platform] = str(exc) result.update({"status": "environment-fail", "failure_class": "automatic-repair-failed", "error": str(exc)}) + + # This is the hard gate shared by integration and journey callers. + # There is no safe implicit matrix: without a caller-supplied config we + # cannot prove HOST, fixtures, tester bindings, or node reachability. + preflight_config = Path(args.preflight_config).expanduser() if args.preflight_config else None + if not failures: + if preflight_config is None: + preflight_result = { + "status": "blocked", + "error": "缺少 --preflight-config；不能跳过 test-preflight", + } + else: + preflight_result = run_test_preflight( + preflight_config, + mode=args.preflight_mode, + yaml_path=Path(args.preflight_yaml).expanduser() if args.preflight_yaml else None, + csv_path=Path(args.preflight_csv).expanduser() if args.preflight_csv else None, + ) + record["preflight"] = preflight_result + if preflight_result.get("status") != "ready": + failures["preflight"] = str(preflight_result.get("error") or preflight_result.get("status")) + + if not failures and args.preflight_mode == "prepare": + # test-preflight prepare owns platform probing, installation and + # launch; do not install/start the same artifact a second time. + for result, _, _ in prepared.values(): + result.update({"status": "preflight-prepared"}) + elif not failures: + for platform in requested: + result, _, identity = prepared[platform] + try: + node = android_preflight(context) if platform == "android" else ios_preflight(context) + check_network(context, node) + result.update({"node_preflight": node, "status": "preflight-passed"}) + prepared[platform] = (result, node, identity) + except EnvironmentFailure as exc: + failures[platform] = str(exc) + result.update({"status": "environment-fail", "failure_class": "automatic-repair-failed", "error": str(exc)}) if failures and not args.partial: for platform in requested: if platform not in failures: record["platforms"][platform].update({"status": "not-run", "reason": "全量 preflight 闸门被另一平台阻断"}) - record.update({"status": "environment-fail", "failure_class": "automatic-repair-failed", "next_action": "修复列出的平台环境后重跑；默认不会静默降级"}) + overall = record.get("preflight", {}).get("status") if "preflight" in record else "environment-fail" + record.update({"status": overall if overall in {"blocked", "environment-fail"} else "environment-fail", "failure_class": "test-preflight" if "preflight" in record else "automatic-repair-failed", "next_action": "修复列出的平台环境后重跑；默认不会静默降级"}) else: for platform, (result, node, identity) in prepared.items(): try: - installed = install_android(node, Path(result["artifact"]["path"]), identity, context) if platform == "android" else install_ios(node, Path(result["artifact"]["path"]), identity, context) + installed = ( + {"delegated": "test-preflight.prepare"} + if args.preflight_mode == "prepare" + else install_android(node, Path(result["artifact"]["path"]), identity, context) + if platform == "android" + else install_ios(node, Path(result["artifact"]["path"]), identity, context) + ) result.update({"status": "ready", "install": installed, "failure_class": None}) except EnvironmentFailure as exc: failures[platform] = str(exc) @@ -476,6 +548,10 @@ def parser() -> argparse.ArgumentParser: run.add_argument("--root", default=".") run.add_argument("--worktree-root", default="") run.add_argument("--binding", default="") + run.add_argument("--preflight-config", default="", help="test-preflight matrix YAML（必填，不得跳过）") + run.add_argument("--preflight-mode", choices=("verify", "prepare"), default="verify") + run.add_argument("--preflight-yaml", default="", help="preflight YAML 证据输出路径") + run.add_argument("--preflight-csv", default="", help="preflight CSV 证据输出路径") run.set_defaults(func=execute) return root diff --git a/skills/tester/scripts/two_tier.py b/skills/tester/scripts/two_tier.py new file mode 100644 index 0000000..521bb30 --- /dev/null +++ b/skills/tester/scripts/two_tier.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Shared execution policy for tester's integration-first runs. + +This module intentionally has no browser, mobile, deployment, or API driver. +Those adapters belong to the caller. It only controls the phase gate and the +stable five-state testcase result propagation so a caller cannot create a +journey session before preflight and integration have passed. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable, Mapping +from typing import Any + +RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run") +PHASES = ("preflight", "integration", "journey") + + +def classify_integration(*, ok: bool, service_reachable: bool = True) -> str: + """Separate a contract assertion failure from an unavailable service.""" + if not service_reachable: + return "environment-fail" + return "pass" if ok else "fail" + + +def _case_id(case: Mapping[str, Any] | str) -> str: + if isinstance(case, Mapping): + return str(case.get("id") or "") + return str(case) + + +def _not_run(cases: Iterable[Mapping[str, Any] | str], reason: str) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + for case in cases: + row = {"id": _case_id(case), "result": "not-run", "notes": reason} + if isinstance(case, Mapping): + row["test_type"] = str(case.get("test_type") or "") + row["suite"] = str( + case.get("client_suite") + or case.get("integration_suite") + or case.get("suite") + or "" + ) + rows.append(row) + return rows + + +def run_integration_first( + *, + preflight_status: str, + integration_cases: Iterable[Mapping[str, Any] | str], + journey_cases: Iterable[Mapping[str, Any] | str], + execute_integration: Callable[[Mapping[str, Any] | str], str | Mapping[str, Any]], + create_journey_session: Callable[[], Any], + execute_journey: Callable[[Any, Mapping[str, Any] | str], str | Mapping[str, Any]], +) -> dict[str, Any]: + """Execute integration cases before creating any journey session. + + ``preflight_status`` must be the status emitted by test-preflight. A + blocked/environment-fail gate records only ``not-run`` testcase rows and + never invokes either adapter. An integration failure stops the phase and + propagates ``not-run`` to all journey cases. + """ + integration = list(integration_cases) + journey = list(journey_cases) + if preflight_status != "ready": + reason = f"test-preflight={preflight_status}，未创建客户端 session" + return { + "status": preflight_status, + "phase": "preflight", + "session_created": False, + "results": _not_run(integration, reason) + _not_run(journey, reason), + } + + results: list[dict[str, Any]] = [] + integration_failed = False + for index, case in enumerate(integration): + if integration_failed: + results.extend(_not_run([case], "前置 integration 失败，未轮到执行")) + continue + outcome = execute_integration(case) + row = dict(outcome) if isinstance(outcome, Mapping) else {"result": outcome} + row["id"] = _case_id(case) + if isinstance(case, Mapping): + row.setdefault("test_type", str(case.get("test_type") or "integration")) + row.setdefault("suite", str(case.get("integration_suite") or case.get("suite") or "")) + result = str(row.get("result") or "not-run") + if result not in RESULTS: + raise ValueError(f"未知 integration 结果: {row['id']}={result}") + row["result"] = result + results.append(row) + if result in ("fail", "environment-fail"): + integration_failed = True + + if integration_failed: + results.extend(_not_run(journey, "integration 失败，依赖的 journey 未执行")) + return { + "status": "fail" if any(row["result"] == "fail" for row in results) else "environment-fail", + "phase": "integration", + "session_created": False, + "results": results, + } + + session = create_journey_session() if journey else None + for case in journey: + outcome = execute_journey(session, case) + row = dict(outcome) if isinstance(outcome, Mapping) else {"result": outcome} + row["id"] = _case_id(case) + if isinstance(case, Mapping): + row.setdefault("test_type", str(case.get("test_type") or "journey")) + row.setdefault("suite", str(case.get("client_suite") or case.get("suite") or "")) + result = str(row.get("result") or "not-run") + if result not in RESULTS: + raise ValueError(f"未知 journey 结果: {row['id']}={result}") + row["result"] = result + results.append(row) + status = "fail" if any(row["result"] == "fail" for row in results) else ( + "environment-fail" if any(row["result"] == "environment-fail" for row in results) else "pass" + ) + return { + "status": status, + "phase": "journey" if journey else "integration", + "session_created": bool(journey), + "results": results, + } diff --git a/skills/tester/templates/catalog.yaml b/skills/tester/templates/catalog.yaml index 5a8ed2a..864ac8c 100644 --- a/skills/tester/templates/catalog.yaml +++ b/skills/tester/templates/catalog.yaml @@ -29,6 +29,13 @@ client_suites: default: false safety: default: false +integration_suites: + api-contract: + default: false + authorization: + default: false + build-contract: + default: false vars: {} authorize: mutating: AUTHORIZE_MUTATING diff --git a/skills/tester/tests/test_two_tier.py b/skills/tester/tests/test_two_tier.py new file mode 100644 index 0000000..4ebb6aa --- /dev/null +++ b/skills/tester/tests/test_two_tier.py @@ -0,0 +1,190 @@ +import csv +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import yaml + +SCRIPTS = Path(__file__).resolve().parents[1] / "scripts" +CATALOG = SCRIPTS / "catalog.py" +CASES_CSV = SCRIPTS / "generate_cases_csv.py" +RUN_CSV = SCRIPTS / "generate_run_csv.py" +sys.path.insert(0, str(SCRIPTS)) +from two_tier import classify_integration, run_integration_first + + +def write_mixed_catalog(root: Path) -> None: + tester = root / ".pouch" / "tester" + (tester / "cases").mkdir(parents=True) + (tester / "runs").mkdir() + (tester / "catalog.yaml").write_text( + yaml.safe_dump( + { + "version": 2, + "id_pattern": "^TC-[A-Z]+-[0-9]+$", + "categories": [{"id": "MIXED"}], + "risks": ["readonly", "mutating", "destructive"], + "default_max_risk": "readonly", + "client_suites": { + "client-smoke": {"default": True}, + "client-regression": {"default": False}, + "safety": {"default": False}, + }, + "sessions": {"browser": {"kind": "browser", "reuse": "run"}}, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + journey = { + "id": "TC-JOURNEY-001", + "test_type": "journey", + "category": "MIXED", + "title": "journey", + "what": "client path", + "lifecycle": "active", + "tag": "smoke", + "risk": "readonly", + "driver": "browser", + "automation": "automated", + "session": "browser", + "client_suite": "client-smoke", + "platform": "android", + "journey": { + "goal": "goal", + "entry_state": "logged_out", + "actions": ["login"], + "assertions": ["visible"], + "final_state": "done", + "failure_policy": "stop", + }, + "fixture": {"account": "test-role"}, + "cleanup": {"required": False, "reason": "read-only"}, + "how": {"steps": ["open"]}, + "expected": [{"kind": "visible-text", "value": "done"}], + } + integration = { + "id": "TC-INTEGRATION-001", + "test_type": "integration", + "category": "MIXED", + "title": "contract", + "what": "API contract", + "lifecycle": "active", + "tag": "slow", + "risk": "readonly", + "driver": "script", + "automation": "automated", + "integration_suite": "api-contract", + "fixture": {"references": ["isolated-account"]}, + "request": {"method": "GET", "path": "/health"}, + "how": {"run": "true"}, + "cleanup": {"required": False, "reason": "read-only"}, + "expected": [{"kind": "api-status", "value": 200}], + } + (tester / "cases" / "mixed.yaml").write_text( + yaml.safe_dump({"category": "MIXED", "cases": [journey, integration]}, sort_keys=False), + encoding="utf-8", + ) + + +class TwoTierCatalogTests(unittest.TestCase): + def run_catalog(self, root: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(CATALOG), *args, "--root", str(root)], + text=True, + capture_output=True, + ) + + def test_default_selection_excludes_integration_and_explicit_selection_includes_it(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + write_mixed_catalog(root) + valid = self.run_catalog(root, "validate") + self.assertEqual(valid.returncode, 0, valid.stderr) + + default = self.run_catalog(root, "select") + self.assertEqual(default.returncode, 0, default.stderr) + default_payload = json.loads(default.stdout) + self.assertEqual(default_payload["count"], 1) + self.assertEqual(default_payload["test_types"], {"journey": 1, "integration": 0}) + + integration = self.run_catalog(root, "select", "--test-type", "integration") + self.assertEqual(integration.returncode, 0, integration.stderr) + integration_payload = json.loads(integration.stdout) + self.assertEqual(integration_payload["count"], 1) + self.assertEqual(integration_payload["cases"][0]["test_type"], "integration") + + csv_result = subprocess.run( + [sys.executable, str(CASES_CSV), "--root", str(root)], + text=True, + capture_output=True, + ) + self.assertEqual(csv_result.returncode, 0, csv_result.stderr) + with (root / ".pouch/tester/cases.csv").open(encoding="utf-8-sig", newline="") as handle: + rows = list(csv.DictReader(handle)) + self.assertEqual({row["test_type"] for row in rows}, {"journey", "integration"}) + self.assertEqual({row["suite"] for row in rows}, {"client-smoke", "api-contract"}) + + run_path = root / ".pouch/tester/runs/two-tier.yaml" + run_path.write_text( + yaml.safe_dump( + {"runId": "two-tier", "results": [ + {"id": "TC-INTEGRATION-001", "result": "pass"}, + {"id": "TC-JOURNEY-001", "result": "not-run"}, + ]}, + sort_keys=False, + ), + encoding="utf-8", + ) + run_csv = subprocess.run( + [sys.executable, str(RUN_CSV), "--root", str(root), "--run", str(run_path)], + text=True, + capture_output=True, + ) + self.assertEqual(run_csv.returncode, 0, run_csv.stderr) + with run_path.with_suffix(".csv").open(encoding="utf-8-sig", newline="") as handle: + report_rows = list(csv.DictReader(handle)) + self.assertEqual(report_rows[0]["test_type"], "integration") + self.assertEqual(report_rows[0]["suite"], "api-contract") + + +class IntegrationFirstTests(unittest.TestCase): + def test_preflight_block_does_not_create_session(self): + calls: list[str] = [] + result = run_integration_first( + preflight_status="blocked", + integration_cases=["INT-1"], + journey_cases=["J-1"], + execute_integration=lambda case: calls.append("integration") or "pass", + create_journey_session=lambda: calls.append("session"), + execute_journey=lambda session, case: "pass", + ) + self.assertEqual(result["results"][0]["result"], "not-run") + self.assertFalse(result["session_created"]) + self.assertEqual(calls, []) + + def test_integration_failure_propagates_journey_not_run(self): + calls: list[str] = [] + result = run_integration_first( + preflight_status="ready", + integration_cases=["INT-1", "INT-2"], + journey_cases=["J-1", "J-2"], + execute_integration=lambda case: calls.append(case) or ("fail" if case == "INT-1" else "pass"), + create_journey_session=lambda: calls.append("session"), + execute_journey=lambda session, case: "pass", + ) + self.assertEqual(result["status"], "fail") + self.assertEqual(calls, ["INT-1"]) + self.assertFalse(result["session_created"]) + self.assertEqual([row["result"] for row in result["results"]], ["fail", "not-run", "not-run", "not-run"]) + + def test_service_unreachable_is_environment_failure(self): + self.assertEqual(classify_integration(ok=False, service_reachable=False), "environment-fail") + self.assertEqual(classify_integration(ok=False, service_reachable=True), "fail") + + +if __name__ == "__main__": + unittest.main()