7 Commits

27 changed files with 1117 additions and 78 deletions
+3 -2
View File
@@ -2,8 +2,9 @@
Reusable preflight for tester matrices. It supports read-only `verify` and
test-environment-only `prepare`, and writes per-check YAML/CSV evidence for
artifacts, tester nodes, devices, network, services, and fixture/cleanup
contracts.
artifacts, tester nodes, devices, network, services, and fixture/cleanup contracts.
This project assumes mobile album/photo/IME capabilities; journey cases exercise them
and preflight does not gate them or read authorization environment variables.
Run the self-tests with:
+9 -2
View File
@@ -3,7 +3,8 @@ name: test-preflight
description: >-
Verify or prepare a mobile/integration test environment from a declared matrix:
artifacts, tester bindings, HOST/PUBLIC_HOST, service probes, and fixture/cleanup
contracts. Use before tester runs; it is not a deployment, build, or business-data skill.
contracts. Use before tester runs; it is not a deployment,
build, or business-data skill.
---
# test-preflight
@@ -47,5 +48,11 @@ Overall status is `ready`, `blocked`, or `environment-fail`:
- `ready` means every recorded check passed. A ready preflight permits the
caller to continue with `integration`; this skill does not run test cases.
This project assumes the client has the required album picker, photo permission, and Chinese
IME capabilities. Preflight does not probe or gate these capabilities, and does not read
`AUTHORIZE_ALBUM` or `IME_PINYIN`. Their real behavior is exercised by journey cases; a failure
there is a product test failure, not an environment block.
Do not treat a preflight report as a product result. A preflight failure means
no client session may be created; the tester owns `not-run` propagation.
no client session may be created; the tester owns `not-run` propagation and must write
one result row per selected testcase with `blocked_by: [test-preflight]`.
@@ -18,6 +18,9 @@ fixtures:
cleanup:
required: true
steps: [delete-run-data]
# Optional project metadata. It is not a preflight gate.
capabilities:
assumed: [album_picker, photo_permission, ime_pinyin]
matrix:
- platform: android
tester:
@@ -48,6 +51,13 @@ needs non-empty `references`; a required cleanup needs non-empty `steps` or
`run`. For a read-only run, use `required: false` with a non-empty `reason`.
The skill never executes these values.
## Client capabilities
The project assumes the mobile test nodes provide the album picker, photo permission,
test image, and Chinese IME. These are not preflight declarations or gates. Journey
cases exercise the real UI; failures are recorded as product failures. `AUTHORIZE_ALBUM`
and `IME_PINYIN` are not read.
## Tester-node contract
`tester.runner.kind: local` executes platform and network probes on the current
+5 -1
View File
@@ -391,7 +391,8 @@ def check_network(evidence: Evidence, config: Mapping[str, Any], row: Mapping[st
statuses.append(status)
evidence.add(f"{platform_name}-{index}-network-{probe_index}-{detail['name']}", platform=platform_name, node=node, target=url, observed=detail["observed"], expected=detail["expected"], status=status, remediation=detail.get("remediation", ""))
elif isinstance(runner, dict) and runner.get("kind") == "contract":
status, observed, result = runner_call(runner, {"operation": "network_probe", "platform": platform_name, "node": node, "url": url, "probe": probe})
binding = text(tester.get("binding")) if isinstance(tester, dict) else ""
status, observed, result = runner_call(runner, {"operation": "network_probe", "platform": platform_name, "node": node, "binding": binding, "url": url, "probe": probe})
result = result or {"status": status, "observed": observed}
complete = require_runner_checks(evidence, result, prefix=f"{platform_name}-{index}-network-{probe_index}", platform_name=platform_name, node=node, required={"dns", "tcp", "http"})
network_status = add_runner_checks(evidence, result, prefix=f"{platform_name}-{index}-network-{probe_index}", platform_name=platform_name, node=node, default_expected="DNS/TCP/HTTP reachable from actual tester node")
@@ -479,6 +480,9 @@ def run(config: Mapping[str, Any], mode: str) -> Evidence:
evidence.add(f"{platform_name}-{index}-platform-runner", platform=platform_name, node=node, target=node, expected="node runner contract", status="blocked", remediation="远端 tester 必须由 agent-runtime 提供可执行 runner")
platform_ok = False
network_ok = check_network(evidence, config, row, platform_name, node, index)
# This isolated project assumes the client capabilities used by its
# journeys. Their real behavior is a product assertion, not a preflight
# environment gate.
row_ok = artifact_ok and fixture_ok and platform_ok and network_ok
if mode == "prepare":
artifact = row.get("artifact") or {}
+19
View File
@@ -50,6 +50,25 @@ elif operation == "network_probe":
"tcp": {"status": "ready", "observed": "443 open", "expected": "TCP reachable"},
"http": {"status": "ready", "observed": 200, "expected": "HTTP 2xx-4xx"},
}}))
elif operation == "capability_probe":
capability = request.get("capability")
checks = {
"album_picker": {
"picker": {"status": "ready", "observed": "real picker opened and returned image-001"},
},
"photo_permission": {
"permission": {"status": "ready", "observed": "permission state checked on device"},
"test_image": {"status": "ready", "observed": "image-001 available in test album"},
},
"ime_pinyin": {
"composition": {"status": "ready", "observed": "nihao composed in Chinese IME"},
"candidates": {"status": "ready", "observed": "你好 candidate selected"},
},
}
if scenario == "capability-missing" and capability == "album_picker":
print(json.dumps({"status": "ready", "checks": {}}))
else:
print(json.dumps({"status": "ready", "checks": checks.get(capability, {})}))
elif operation in {"install", "start"}:
print(json.dumps({"status": "ready", "observed": operation + "ed mock artifact"}))
else:
@@ -140,6 +140,21 @@ class PreflightTests(unittest.TestCase):
self.assertTrue(any(row["check_id"].endswith("-fixture") and row["status"] == "blocked" for row in result.rows))
self.assertTrue(any(row["check_id"].endswith("-cleanup") and row["status"] == "blocked" for row in result.rows))
def test_mobile_capabilities_are_assumed_and_not_a_preflight_gate(self):
data = self.config()
config_path = self.write_config(data)
env = os.environ.copy()
env["PREFLIGHT_RUNNER_SCENARIO"] = "capability-missing"
output = self.root / "capabilities.yaml"
process = subprocess.run([
sys.executable, str(ROOT / "scripts" / "preflight.py"), "verify",
"--config", str(config_path), "--yaml", str(output), "--csv", str(self.root / "capabilities.csv"),
], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
report = __import__("yaml").safe_load(output.read_text(encoding="utf-8"))
self.assertEqual(process.returncode, 0)
self.assertEqual(report["status"], "ready")
self.assertFalse(any("capability-" in row["check_id"] for row in report["checks"]))
def test_host_unreachable_from_actual_runner_is_environment_fail(self):
data = self.config()
config_path = self.write_config(data)
@@ -203,6 +218,7 @@ class PreflightTests(unittest.TestCase):
self.assertEqual(result.overall(), "ready")
self.assertNotIn("install", calls)
self.assertNotIn("start", calls)
self.assertNotIn("capability_probe", calls)
if __name__ == "__main__":
+16 -2
View File
@@ -27,7 +27,12 @@
每轮测试会创建新的 terminal,结束后自动关闭;不会复用或关闭已有 terminal。
- 测试环境部署由 deployer 在本机 Compose 中触发;正式环境部署由本机 `~/code/app00`
项目的 fresh Agent terminal 触发,tester 不直接 SSH 或执行正式 Compose。
- 会动硬件、会改数据的用例带风险级别(`readonly` / `mutating` / `destructive`)。默认只跑最低档 `readonly`。要跑更高风险,需要你明确说,例如「风险到 mutating」;碰哪个子系统另由 `gate`(如 hardware / weld)控制。
- 会动硬件、会改数据的用例带风险级别(`readonly` / `mutating` / `destructive`)。默认
`standard` 只跑最低档 `readonly`。隔离、可重置的 Writes 类环境在 catalog 设置
`execution_profile: isolated-full` 后,所有 active 自动化用例都进入执行,不因
`AUTHORIZE_MUTATING`、`AUTHORIZE_DESTRUCTIVE`、`AUTHORIZE_ALBUM` 或
`AUTHORIZE_PASSWORD` skipped;risk 原值仍保留。设备、产物、服务和 cleanup/rollback
由 test-preflight 保证;客户端相册、权限和中文输入能力默认具备,由 journey 用例验证。
## 使用示例
@@ -76,7 +81,8 @@ APK/IPA,为本轮创建隔离 worktree,在 Android 和 iOS tester 节点安
不执行可用平台;只有全部可用才按目标环境走 deployer 路由、解析 Agent、输出生效
列表、准备 Orca 项目并创建临时 terminal。脚本类、浏览器和移动模拟器都在对应
Agent 环境中执行。`builder.<platform>` 不能替代 `tester.<platform>`。
4. 超出你允许的风险级别会跳过,不会偷偷执行。
4. `standard` 下超出你允许的风险级别会跳过;`isolated-full` 由 catalog profile 明确
全量 active 执行,不把风险悄悄改成 readonly。
5. 无论测试成功、失败或异常,都会关闭本轮创建的 terminal;跑完给出按分类计数的
结果,以及 `pass / fail / environment-fail / skipped / not-run`。
@@ -117,6 +123,14 @@ cleanup 缺失会在开始前阻断;前置失败后的剩余用例是 `not-run
`safety` 必须显式选择和授权,不进入默认 smoke/full。它与普通 tester 用例共用原有
CSV/YAML 运行记录和汇总校验。
移动 preflight 还必须在实际 tester 节点验证相册 picker、照片权限与测试图片,以及中文
拼音的真实 composition 和候选词选择;缺失时所有 testcase 都保留 `not-run` +
`blocked_by`,不会静默 `skipped`。
失败回流使用 run YAML 顶层 `defects`/`test_issues` 与 `feedback`:PR 触发先更新 PR/关联
issue、通知 dispatcher 重新分配开发;需求/bug 触发直接回开发流程。可用
`scripts/record_issues.py` 生成同名 CSV 的结构化 issue 列。
### 汇总已有运行结果
对已经生成的同一运行 CSV/YAML,使用显式路径汇总:
+12 -5
View File
@@ -74,10 +74,10 @@ tester agent 配置。
- `local.yaml.example` 的 HOST 默认是 `127.0.0.1`;如果目标不是本机,必须在
`local.yaml` 或环境变量中显式覆盖。`local.yaml` 不存在且没有环境变量时仍不猜
HOST,停下并记 environment-fail。
- 默认风险上限是 catalog 里的 `default_max_risk`。超出上限的条目标 `skipped`,禁止执行。
- 默认执行 profile 是 `standard`:风险上限来自 catalog 的 `default_max_risk`,超出上限的条目标 `skipped`,禁止执行。若 catalog 声明 `execution_profile: isolated-full`,只允许用于隔离且可重置的测试环境:所有 active 自动化用例都进入执行矩阵,不因 `AUTHORIZE_MUTATING`、`AUTHORIZE_DESTRUCTIVE`、`AUTHORIZE_ALBUM` 或 `AUTHORIZE_PASSWORD` skipped;原有 `risk` 值必须保留并进入报告。设备、产物、服务和清理契约由 test-preflight 检查;相册、照片权限/测试图片和中文拼音能力在本项目中默认具备,由 journey 用例真实验证。
- `risk` 只能是 `readonly` / `mutating` / `destructive`,默认只跑 `readonly`。`tag=smoke` 只能配 `risk=readonly`。规则见 [levels.md](references/levels.md)。
- `driver=browser` 必须开真实页面;`driver=mobile` 必须挂模拟器 session。二者都不得改成只打 API。
- `automation=assisted|manual` 需要人在场:无人值守时记 `not-run`,不要自己代跑人工步骤。见 [concepts.md](references/concepts.md)。
- `automation=assisted|manual` 需要人在场:无人值守时记 `not-run`,不要自己代跑人工步骤。isolated-full 取消的是风险/授权过滤,不会伪造人工操作。见 [concepts.md](references/concepts.md)。
- 浏览器、移动模拟器按 catalog `sessions` 初始化一次,同一次运行内复用。
- Agent、环境、项目目录和 Orca terminal 生命周期见 [execution.md](references/execution.md):可以从 project id/名称或 terminal id 自动解析,更新后输出配置列表;每轮都新建 terminal,结束后关闭本轮创建的 terminal,不复用或关闭已有 terminal。
- 环境准备见 [deployment.md](references/deployment.md):测试环境只调用 deployer 的本机 Compose 路径;正式环境只委派给本机 `~/code/app00` 的 fresh Agent terminal,tester 不直接 SSH 或执行生产 Compose。
@@ -189,6 +189,13 @@ python3 <skill-dir>/scripts/catalog.py select --root <root> --suite safety --max
客户端运行器统一遵守 `preflight → 启动 → 登录 → 旅程 → 断言 → 清理 → 报告`。正确测试
账号登录失败是产品 `fail`;设备、产物、服务或 session 不可达才是 `environment-fail`。
fixture 或 cleanup 缺失会在 `catalog.py select` 的 preflight 阶段阻断,不产生虚假逐条结果。
前置用例失败后剩余条目写 `not-run`,主动排除/未授权写 `skipped`。成绩仍写现有
`runs/<run-id>.yaml` 并用 `generate_run_csv.py` 生成同名 CSV;清理失败作为 cleanup 证据
单独记录,不添加新的 result 值。
设备、产物、服务或 session 能力缺失时 preflight 必须明确 blocked/environment-fail,并为所有选中的 testcase 生成
`not-run` 结果行;本项目默认具备相册、照片权限、测试图片或中文拼音 composition/candidate
能力,不在 preflight 中校验或静默变成 `skipped`,由 journey 真实验证并记录产品失败。独立用例继续执行,依赖失败的用例写 `not-run` 和 `blocked_by`;
修复后按 targeted rerun,再执行 full integration → journey。成绩仍写现有
`runs/<run-id>.yaml` 并用 `generate_run_csv.py` 生成同名 CSV;YAML 可带 `defects`/
`test_issues` 与 `feedback`,记录 testcase、product/test issue 类型、expected/observed、
artifact hash、environment、证据路径、复现和下一步,不添加新的 result 值。
PR 触发的失败回流必须先更新 PR/关联 issue,再通知 dispatcher 重新分配开发;需求/bug
触发的失败直接返回开发流程。tester 只生成结构化回流计划,不在本地偷偷修改远端。
+5 -1
View File
@@ -35,6 +35,7 @@ catalog(整个测试集)
| `session` | 可复用会话(浏览器 / 模拟器) | pytest fixture;Robot suite variable | 不是一条用例 |
| `gate` | 项目细分闸门名 → 环境变量(`motion` / `hardware` / `weld`) | 授权开关 | 不是 `risk`,不要拿它代替风险等级 |
| `authorize` | 风险档 → 环境变量。该档执行前必须为 `true` | pytest `skipif` 条件 | 不是认证(auth),是**授权** |
| `execution_profile` | 执行环境策略:`standard` 或隔离可重置的 `isolated-full` | pytest marker/fixture policy | 不是风险等级;isolated-full 不会抹掉 `risk` |
| `assume` | 额外跳过条件(gate / authorize 之外的环境表达式) | JUnit `assumeTrue`;pytest `skipif` | 不是授权开关的重复写法 |
| `status_probe` | 分类级前置条件:请求一个 URL,命中即视为前置不满足 | pytest session fixture;Robot Suite Setup | 不是产品失败,不是整轮硬闸 |
| `how` | 怎么测:`steps` / `run` / `url` / `timeout` | test steps | 不是判定标准 |
@@ -55,7 +56,7 @@ catalog(整个测试集)
| `pass` | 通过 | passed |
| `fail` | 产品缺陷 | failed |
| `environment-fail` | 前置/环境导致无法执行,**永远不是产品失败** | pytest error;JUnit aborted |
| `skipped` | **主动**不跑:`retired` / `pending` / 超风险上限 / 未授权 / `assume` 不成立 | pytest skipped;JUnit disabled |
| `skipped` | **主动**不跑:standard 下的 `retired` / `pending` / 超风险上限 / 未授权 / `assume` 不成立 | pytest skipped;JUnit disabled |
| `not-run` | **被动**没轮到:失败即停后剩下的;分类被 `status_probe` 挡住整组没执行;`automation` 需人工而无人值守 | pytest not run;deselected |
## 2. 命名迁移对照(v1 → v2)
@@ -82,6 +83,8 @@ catalog(整个测试集)
- 三级**单调**:`readonly ⊂ mutating ⊂ destructive`,所以只需要一个上限 `default_max_risk` 和一个 `--max-risk`,不用组合矩阵。
- 与 `gate` 正交:`risk` 说「影响多大」,`gate` 说「碰哪个子系统」。点动 vs 焊接这种细分授权用 `gate`,不要再造 risk 名。
- `tag=smoke` 只能配 `risk=readonly`:冒烟测试不许留下变化。
- 隔离可重置环境可设置 `execution_profile: isolated-full` 跑所有 active 用例;安全前置由
test-preflight 的设备、服务与 cleanup/rollback 契约保证,原始 risk 仍必须报告;客户端能力由 journey 用例验证。
### 3.2 `automation`:为什么是 `automated / assisted / manual`
@@ -93,6 +96,7 @@ catalog(整个测试集)
- `assisted`:Agent 驱动,但至少一步需要人在场;该步在 `how.steps` 里以 `[人工]` 开头(可被 `validate` 机器判定)。
- `manual`:只能人工执行;禁止 `driver`、禁止 `how.run`,必须有 `how.steps`。
- 默认只跑 `automated`。无人值守时 `assisted` / `manual` 记 `not-run`;有人在场时 `assisted` 暂停等人确认,`manual` 由人工回报结果。
- isolated-full 只取消风险/授权过滤,不伪造 assisted/manual 的人工步骤。
### 3.3 结果为什么拆出 `not-run`
+10
View File
@@ -25,6 +25,16 @@ bindings:
上执行测试。`tester.test` 只能用于明确的单平台或非移动测试;同一轮包含 Android
和 iOS 时必须分别解析两个 tester 绑定。
移动 bundle 入口必须把平台绑定显式传入:
```bash
--binding android=tester.android --binding ios=tester.ios
```
入口会校验 preflight matrix 的每个平台 binding、node 和 runner。远程 iOS 必须是
`contract` runner;多平台运行不会读取旧的单一 `local.yaml.agent` 来决定执行节点,
也不会在协调节点直接执行 Android/iOS 平台命令。
`locations` 和 `profiles` 的定义见共享 `agent-runtime`。tester adapter 只把它 materialize
到本轮兼容配置;每轮 terminal 仍然新建并在结束时关闭,不能因为 binding 是 persistent
就复用测试 terminal。
+5 -1
View File
@@ -36,7 +36,11 @@
| `destructive` | 难恢复或伤人/伤钱/伤环境 | 删库、真实扣费 | 清数据、内购 | 急停、断电、真焊 |
默认只跑 `readonly`。项目细分闸(点动 vs 焊接)用可选 `gate`,不要再发明 risk 名。
隔离且可重置的项目可在 catalog 设置 `execution_profile: isolated-full`,此时所有 active
用例按原始 risk 执行;`risk` 仍用于报告和 cleanup 设计,不能改写为 readonly。
`tag=smoke` 且 `risk!=readonly` 为非法。
不想跑某一档时,`authorize.<risk>` 指向的环境变量保持 false 即可——该档用例记 `skipped`,不是 `not-run`。
standard 下不想跑某一档时,`authorize.<risk>` 指向的环境变量保持 false 即可——该档用例记
`skipped`,不是 `not-run`。isolated-full 不使用这些授权开关,改由 preflight 能力探针和
cleanup/rollback 契约阻断不安全环境。
+13 -7
View File
@@ -72,7 +72,11 @@ iOS | … | … | page.writes | tester.ios | remote-ios
## 1. 选范围
未指定范围 → `--tag smoke`,风险上限 = `default_max_risk`,`automation` 默认只跑 `automated`。
未指定范围 → `--tag smoke`,执行 profile = catalog 的 `execution_profile`(默认 `standard`),
`standard` 的风险上限 = `default_max_risk`,`automation` 默认只跑 `automated`。
`execution_profile=isolated-full` 只用于隔离可重置环境:不按风险上限或 `AUTHORIZE_*`/gate
授权变量过滤 active 用例,保留每条用例原始 `risk`;安全门禁由 test-preflight 能力探针和
cleanup/rollback 契约承担。
用户说分类、id、tag、风险上限时,原样传给 select:
@@ -80,6 +84,7 @@ iOS | … | … | page.writes | tester.ios | remote-ios
python3 <skill-dir>/scripts/catalog.py select --root <root> \
[--category CAT] [--id ID]... [--tag smoke|full|slow] \
[--max-risk <risk>] [--driver script|browser|mobile] \
[--execution-profile standard|isolated-full] \
[--automation automated|assisted|manual|any] \
[--test-type integration|journey] [--suite <client-or-integration-suite>] \
[--platform android|ios]... \
@@ -111,11 +116,11 @@ python3 <skill-dir>/scripts/catalog.py select --root <root> --suite api-contract
不能拿到 cases 后绕过闸门开始测试。默认 binding 是 `tester.<platform>`,所以 iOS
会强制检查 `tester.ios`,不会回退到 `builder.ios` 或通用 `tester.test`。
`--automation` 默认 `automated`(需人工的条目会被跳过,并在 JSON `skipped` 里说明);用户明确要求现场配合时用 `assisted`,`any` 表示不按人工参与度筛。
`--automation` 默认 `automated`(需人工的条目会被记为 `not-run`,不会伪造人工步骤);用户明确要求现场配合时用 `assisted`,`any` 表示不按人工参与度筛。
单根 `select` 的 JSON 字段:`max_risk` / `automation` / `count` / `vars`(已展开的变量,用来替换 `how` 里的 `${…}`)/ `artifacts` / `preflight` / `skipped` / `run_together` / `status_probe`(按分类,`url` 已展开)/ `cases`。从父仓库 fan-out 时返回 `mode: fan-out` 和每个根的独立结果,不能把多个根的 cases 拼成一个列表。
`select` 已排除 `retired` 和风险高于上限的条目(后者若用户点了具体 id,会在 JSON 的 `skipped` 里说明,不执行)。
`standard` 的 `select` 已排除 `retired` 和风险高于上限的条目(后者若用户点了具体 id,会在 JSON 的 `skipped` 里说明);`isolated-full` 保留所有 active 条目,不因风险或授权变量 skipped。
若 catalog 有 `run_together`,命中组内任一条就把整组按 `ids` 顺序扩进列表(JSON 带 `run_together`)。这不是擅自扩大范围。
@@ -135,8 +140,9 @@ preflight 通过后,先按 catalog 顺序执行 `integration`,再按客户
## 3. 运行前闸
- `HOST` 未设(环境变量 / `local.yaml` 都没有):全部停,environment-fail。
- 命中条的 `risk` 在 catalog `authorize` 里有对应变量:变量不是 `true` 则该条 `skipped`。有 `gate` 时还要 `catalog.gates` 对应变量为 `true`;`assume` 不成立同样 `skipped`。即使用户点了 id 也不执行。
- 授权/假设不成立是**主动不跑**(`skipped`),不是 `not-run`:这是我们决定不跑,不是没轮到。
- `standard`:命中条的 `risk` 在 catalog `authorize` 里有对应变量时,变量不是 `true` 则该条 `skipped`;有 `gate` 时还要对应变量为 `true`,`assume` 不成立同样 `skipped`。即使用户点了 id 也不执行。
- `isolated-full`:不读取 `AUTHORIZE_MUTATING`、`AUTHORIZE_DESTRUCTIVE`、`AUTHORIZE_ALBUM`、`AUTHORIZE_PASSWORD` 等人工开关;相册、照片权限/测试图片和中文拼音能力默认具备,由 journey 用例真实验证,失败记为产品失败。
- standard 中授权/假设不成立是**主动不跑**(`skipped`);isolated-full 不因客户端能力授权变量停止,设备、产物、服务或清理前置失败时才为全部 testcase 写 `not-run` + `blocked_by: [test-preflight]`。
- `status_probe` 是**分类级前置条件**(业界对应 pytest 的 session fixture / Robot 的 Suite Setup)。用 `select` 返回的 `status_probe` 解析生效值,不要自己读 catalog 猜。返回的 `probe.url` 已用 `vars` 展开,直接请求即可;`select` 同批返回 `vars`:
| `categories[].status_probe` | 生效探测 |
@@ -221,7 +227,7 @@ Android+iOS 默认是全量 preflight 闸门:任一项不满足,另一平台
## 5. 执行
按 select 返回的顺序,一条一条来。select 按 `catalog.yaml` 的 `categories` 顺序,同一分类内按该 yaml 列表顺序(不是文件名字母序)。默认失败即停(产品 `fail` 或探测到的不安全状态)。用户说继续 / keep-going 才往后跑。
按 select 返回的顺序,一条一条来。select 按 `catalog.yaml` 的 `categories` 顺序,同一分类内按该 yaml 列表顺序(不是文件名字母序)。同一 phase 的独立用例继续执行;只有显式 `depends_on` 的失败依赖才写 `not-run` + `blocked_by`。用户说 targeted rerun 时只重跑受影响用例,修复后再 full integration → journey。
例外:当前条和下一条同属 `run_together` 且该组 `continue_on_fail: true` 时,前一条产品 `fail` 仍跑后一条。`environment-fail` 仍整轮停。
@@ -253,7 +259,7 @@ python3 <skill-dir>/scripts/catalog.py run-script --root <root> --id <id>
## 6. 记录与报告
每条:`id, result, evidence, notes`。`result` 只能是 `pass` / `fail` / `environment-fail` / `skipped` / `not-run`。`skipped` 是主动不跑(retired / pending / 超风险上限 / 未授权 / `assume` 不成立),`not-run` 是没轮到(失败即停后剩下的 / 被前置整组挡住的 / 需人工而无人值守的)。
每条:`id, result, evidence, notes`,依赖阻断时另有 `blocked_by`。`result` 只能是 `pass` / `fail` / `environment-fail` / `skipped` / `not-run`。`skipped` 是 standard 中主动不跑(retired / pending / 超风险上限 / 未授权 / `assume` 不成立);`not-run` 是没轮到或 preflight/依赖阻断。失败用例可在 YAML 顶层 `defects`(兼容 `test_issues`)记录 testcase、`issue_type=product|test`、expected/observed、artifact hash、environment、evidence path、reproduction、next step。
写入 `.pouch/tester/runs/<UTC时间>-<范围>-<随机后缀>.yaml`,可用
`generate_run_csv.new_run_id(<范围>)` 生成同名 `runId`。记录包含唯一 `runId`、筛选条件、HOST 是否已设(可以写 HOST 值)、session 是否复用、每个分类的 `status_probe` 生效值与观测结果(`source` / `disabled` 原因 / `hit` / `observed`)、逐条结果。记录文件只创建一次,不覆盖已有文件。立刻用同一快照生成 CSV,不要手改:
+27
View File
@@ -25,6 +25,7 @@ runs/*.csv 同一次运行的成绩单;generate_run_csv.py 从对
| `categories` | `{id, title, status_probe?}` 列表,id 与 `cases/<id小写>.yaml` 对应;列表顺序即 `select` / 执行顺序。`status_probe` 见下 |
| `risks` | 从低到高,只能是 `readonly` → `mutating` → `destructive` 的一段,且以 `readonly` 开头。含义见 [levels.md](levels.md) |
| `default_max_risk` | 未指定时的运行上限,必须是 `risks` 里的值 |
| `execution_profile` | `standard`(默认)或 `isolated-full`。后者只用于隔离可重置环境,保留 risk 语义但不按 risk/authorize/gate 过滤 active 用例;设备、服务和 cleanup/rollback 契约仍由 preflight 检查,客户端能力由 journey 真实验证 |
| `vars` | 可替换变量,如 `API_BASE: http://${HOST}:8095` |
| `artifacts` | 可选的移动产物身份契约。按 `android` / `ios` 声明 `required` 和 `package` / `bundle_id`;每轮文件路径或下载地址由调用方提供,不提交临时产物路径。 |
| `authorize` | 风险名 → 环境变量名。该档执行前变量必须为 `true`(`readonly` 不需要) |
@@ -36,6 +37,10 @@ runs/*.csv 同一次运行的成绩单;generate_run_csv.py 从对
| `client_suites` | 可选的标准套件声明;出现时必须完整包含 `client-smoke`、`client-regression`、`safety`。缺省运行仍按 `tag` 兼容旧目录 |
| `integration_suites` | 可选的集成套件声明;标准值为 `api-contract`、`authorization`、`build-contract` |
`isolated-full` 不得把风险字段改成 `readonly`,也不得把 `AUTHORIZE_ALBUM`、
`AUTHORIZE_PASSWORD`、`IME_PINYIN` 等变量当成门禁。移动用例必须通过 test-preflight
的设备、产物、服务和网络检查;相册、权限和输入法能力默认具备,交由 journey 用例验证。
`local.yaml` 与环境变量覆盖 `HOST` 和授权开关。模板中的 `HOST` 默认是
`127.0.0.1`;远程目标必须显式覆盖。查找顺序:环境变量 > `local.yaml` >
`catalog.yaml` 的 vars。
@@ -91,6 +96,7 @@ terminal;运行结束用 `orca terminal close --terminal <handle>` 关闭本
| `expected` | 是 | 怎么过,至少一条 `{kind, value}` |
| `maps_to` | 否 | 对人文档的功能项 id |
| `assume` | 否 | 额外跳过条件(环境变量表达式)。不要重复 `risk` / `gate` 已覆盖的授权变量,否则 `validate` 报错 |
| `depends_on` | 否 | 依赖的 testcase id 列表;依赖 fail/environment-fail/not-run 时本条写 `not-run` 和 `blocked_by`,独立用例继续 |
| `setup` | 否 | 人话前置 |
| `teardown` | 否 | 跑完回退 |
| `notes` | 否 | 备注 |
@@ -170,3 +176,24 @@ cleanup:
## 判定
`pass` / `fail` / `environment-fail` / `skipped` / `not-run`
## 运行失败回流
run YAML 的 `results` 必须保留每条 testcase 行。失败可在顶层 `defects`(兼容别名
`test_issues`)追加:
```yaml
defects:
- testcase: TC-CLIENT-001
issue_type: product # product | test
expected: "候选词可选择"
observed: "composition 成功但候选列表为空"
artifact_hash: sha256:...
environment: "Android emulator API 35 / tester.android"
evidence_path: .pouch/tester/evidence/TC-CLIENT-001.png
reproduction: "打开相册后输入 nihao"
next_step: "开发修复候选词渲染后 targeted rerun"
```
PR 触发的 run 可带 `feedback` 计划:先更新 PR/关联 issue,再通知 dispatcher 重新分配开发;
需求/bug 触发则 route 直接为 development。tester 只记录计划,不隐式调用远端写操作。
+7 -3
View File
@@ -7,11 +7,15 @@ CI release、builder 绑定或本机旧文件猜测产物。
<skill-dir>/tester run \
--artifact android=<path-or-url> \
--artifact ios=<path-or-url> \
--platform android,ios
--platform android,ios \
--binding android=tester.android \
--binding ios=tester.ios
```
URL 就下载,本地文件就直接用。每个平台都在自己的 `tester.<platform>` 节点校验身份并安装,
随后由现有 tester catalog 驱动真实移动 session。默认先完成 Android+iOS 双平台
URL 就下载,本地文件就直接用。每个平台都通过自己的 `tester.<platform>` binding
校验、探测和安装;`agent-runtime` 负责把 Android 请求送到 Android 节点、把 iOS
请求送到 Mac 节点。旧的单一 `local.yaml.agent` 只能作为兼容配置,不能作为双平台
运行目标。随后由现有 tester catalog 驱动真实移动 session。默认先完成 Android+iOS 双平台
preflight;任一平台无法准备时整轮停止。只有显式加 `--partial` 才允许运行可用平台。
每轮另外创建隔离 Git worktree,不切用户当前分支。`--commit` 缺省为当前 HEAD。
+81 -11
View File
@@ -42,6 +42,7 @@ LIFECYCLE_SKIP_REASON = {
}
TAGS = ("smoke", "full", "slow")
CANONICAL_RISKS = ("readonly", "mutating", "destructive")
EXECUTION_PROFILES = ("standard", "isolated-full")
AUTOMATIONS = ("automated", "assisted", "manual")
RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run")
EXPECTED_KINDS = (
@@ -351,6 +352,21 @@ def risk_index(meta: dict[str, Any], name: str) -> int:
return risks.index(name)
def resolve_execution_profile(meta: dict[str, Any], requested: str | None = None) -> str:
"""Resolve the run profile without changing the meaning of ``risk``.
``isolated-full`` is an execution policy for a resettable test environment:
it disables risk/authorization filtering, but leaves the catalog risk value
on every case so reports still describe the side effect that was exercised.
"""
profile = requested or meta.get("execution_profile") or "standard"
if profile not in EXECUTION_PROFILES:
raise SystemExit(
f"未知 execution_profile: {profile!r}(可选: {', '.join(EXECUTION_PROFILES)})"
)
return str(profile)
def host_set(vars_map: dict[str, str]) -> bool:
host = (vars_map.get("HOST") or "").strip()
return bool(host) and "${HOST}" not in host
@@ -570,6 +586,7 @@ def expand_run_together(
skipped: list[dict[str, str]],
max_i: int,
automation_filter: str = "automated",
bypass_risk: bool = False,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
groups = run_together_groups(meta)
if not groups:
@@ -607,7 +624,7 @@ def expand_run_together(
except SystemExit:
skipped.append({"id": cid, "reason": f"未知风险 {risk}"})
continue
if r_i > max_i:
if not bypass_risk and r_i > max_i:
skipped.append({"id": cid, "reason": f"risk={risk} 超过上限"})
continue
members.append(public_case(case))
@@ -770,9 +787,11 @@ def mobile_preflight(
def cmd_select(args: argparse.Namespace, root: Path) -> int:
meta = load_catalog_meta(root)
cases = load_all_cases(root)
execution_profile = resolve_execution_profile(meta, args.execution_profile)
bypass_risk = execution_profile == "isolated-full"
max_risk = args.max_risk or meta.get("default_max_risk") or "readonly"
max_i = risk_index(meta, max_risk)
automation_filter = args.automation or "automated"
automation_filter = args.automation or ("any" if bypass_risk else "automated")
wanted = set(parse_ids(args.ids or []))
selected: list[dict[str, Any]] = []
skipped: list[dict[str, str]] = []
@@ -802,9 +821,15 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
continue
if args.suite and case_suite(case) != args.suite:
continue
# A mixed catalog keeps legacy cases runnable, but the implicit
# default is the client journey surface. Integration is opt-in.
if not args.test_type and not args.suite and test_type(case) == "integration":
# A mixed catalog keeps legacy cases runnable. Standard keeps the
# historical client-only implicit default; isolated-full means
# exactly what it says and includes every active test type.
if (
not bypass_risk
and not args.test_type
and not args.suite
and test_type(case) == "integration"
):
continue
if args.driver and case.get("driver") != args.driver:
continue
@@ -821,7 +846,7 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
except SystemExit:
skipped.append({"id": cid, "reason": f"未知风险 {risk}"})
continue
if r_i > max_i:
if not bypass_risk and r_i > max_i:
skipped.append({"id": cid, "reason": f"risk={risk} 超过上限 {max_risk}"})
continue
selected.append(public_case(case))
@@ -833,7 +858,15 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
)
return 2
selected, together = expand_run_together(meta, cases, selected, skipped, max_i, automation_filter)
selected, together = expand_run_together(
meta,
cases,
selected,
skipped,
max_i,
automation_filter,
bypass_risk=bypass_risk,
)
# The historical key is kept in the JSON, but the gate now validates both
# test types before any caller can start a phase.
@@ -867,6 +900,8 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
json.dumps(
{
"max_risk": max_risk,
"execution_profile": execution_profile,
"risk_filter": "disabled" if bypass_risk else "enforced",
"automation": automation_filter,
"count": len(selected),
"vars": vars_map,
@@ -947,6 +982,12 @@ def validate(root: Path) -> list[str]:
default_max = meta.get("default_max_risk")
if default_max not in risks:
errors.append(f"default_max_risk={default_max!r} 不在 risks 中")
profile = meta.get("execution_profile", "standard")
if profile not in EXECUTION_PROFILES:
errors.append(
f"execution_profile={profile!r} 非法,只能是 {', '.join(EXECUTION_PROFILES)}"
)
isolated_full = profile == "isolated-full"
gates = meta.get("gates") or {}
if gates and not isinstance(gates, dict):
errors.append("catalog.gates 必须是 mapping")
@@ -1073,8 +1114,17 @@ def validate(root: Path) -> list[str]:
if case.get("risk") in (None, "readonly"):
errors.append(f"{where}: risk=readonly 不要填 gate")
risk = case.get("risk")
if risk and risk != "readonly" and risk not in authorize and not gate:
if risk and risk != "readonly" and risk not in authorize and not gate and not isolated_full:
errors.append(f"{where}: {risk} 需要 authorize.{risk} 或 gate")
if isolated_full and case.get("lifecycle", "active") == "active" and risk != "readonly":
cleanup = case.get("cleanup")
cleanup_ok = isinstance(cleanup, dict) and (
bool(cleanup.get("steps"))
or bool(str(cleanup.get("run") or "").strip())
or (cleanup.get("required") is False and bool(str(cleanup.get("reason") or "").strip()))
)
if not cleanup_ok:
errors.append(f"{where}: isolated-full 的写入型用例必须声明 cleanup/rollback steps/run")
automation = str(case.get("automation") or "automated")
if automation not in AUTOMATIONS:
errors.append(f"{where}: 非法 automation={case.get('automation')}")
@@ -1206,7 +1256,19 @@ def cmd_next_id(root: Path, category: str) -> int:
return 0
def authorized(vars_map: dict[str, str], meta: dict[str, Any], case: dict[str, Any]) -> bool:
def authorized(
vars_map: dict[str, str],
meta: dict[str, Any],
case: dict[str, Any],
*,
execution_profile: str | None = None,
) -> bool:
profile = resolve_execution_profile(meta, execution_profile)
if profile == "isolated-full":
# AUTHORIZE_* and gate variables are intentionally not run-time
# switches for an isolated resettable environment. Capability and
# cleanup preflight are the safety boundary for this profile.
return True
risk = str(case.get("risk") or "readonly")
risks = list(meta.get("risks") or ["readonly"])
lowest = risks[0] if risks else "readonly"
@@ -1226,6 +1288,7 @@ def authorized(vars_map: dict[str, str], meta: dict[str, Any], case: dict[str, A
def cmd_run_script(root: Path, case_id: str) -> int:
meta = load_catalog_meta(root)
execution_profile = resolve_execution_profile(meta)
vars_map = resolve_vars(root, meta)
case = next((c for c in load_all_cases(root) if c.get("id") == case_id), None)
if not case:
@@ -1263,10 +1326,10 @@ def cmd_run_script(root: Path, case_id: str) -> int:
)
return 2
risk = str(case.get("risk") or "readonly")
if not authorized(vars_map, meta, case):
if not authorized(vars_map, meta, case, execution_profile=execution_profile):
print(
json.dumps(
{"id": public_id, "result": "skipped", "evidence": f"风险 {risk} 未授权"},
{"id": public_id, "result": "skipped", "evidence": f"风险 {risk} 未授权", "execution_profile": execution_profile},
ensure_ascii=False,
)
)
@@ -1315,6 +1378,7 @@ def cmd_run_script(root: Path, case_id: str) -> int:
{
"id": public_id,
"result": "executed",
"execution_profile": execution_profile,
"exit": proc.returncode,
"evidence": evidence,
},
@@ -1345,6 +1409,12 @@ def build_parser() -> argparse.ArgumentParser:
help="按顶层 test_type 筛选;integration 必须显式选择",
)
p_sel.add_argument("--max-risk")
p_sel.add_argument(
"--execution-profile",
choices=EXECUTION_PROFILES,
default=None,
help="执行策略;isolated-full 在隔离可重置环境中不按 risk/authorize 筛选",
)
p_sel.add_argument("--driver", choices=DRIVERS)
p_sel.add_argument(
"--automation",
+150
View File
@@ -0,0 +1,150 @@
#!/usr/bin/env python3
"""Build structured tester issues and the development feedback route.
This module only writes the run contract. Updating a remote PR/issue or
notifying a dispatcher remains an explicit caller-side integration step; no
network mutation is hidden in the tester.
"""
from __future__ import annotations
from copy import deepcopy
from datetime import datetime, timezone
from typing import Any, Iterable, Mapping
ISSUE_TYPES = ("product", "test")
TRIGGERS = ("pr", "requirement", "bug")
ISSUE_RESULTS = frozenset({"fail", "environment-fail"})
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
def issue_type_for(result: str) -> str:
if result == "fail":
return "product"
if result == "environment-fail":
return "test"
raise ValueError(f"只有 fail/environment-fail 可以生成 issue: {result}")
def _value(row: Mapping[str, Any], key: str, defaults: Mapping[str, Any]) -> Any:
value = row.get(key)
return value if value not in (None, "") else defaults.get(key, "")
def build_issue(
row: Mapping[str, Any],
*,
defaults: Mapping[str, Any] | None = None,
issue_type: str | None = None,
) -> dict[str, Any]:
"""Create the stable issue shape required by the run report."""
defaults = defaults or {}
result = str(row.get("result") or "")
if result not in ISSUE_RESULTS:
raise ValueError(f"不能为 {result!r} 结果创建 defect: {row.get('id')}")
selected_type = issue_type or row.get("issue_type") or issue_type_for(result)
if selected_type not in ISSUE_TYPES:
raise ValueError(f"issue_type 必须是 product/test: {selected_type}")
testcase = str(row.get("testcase") or row.get("id") or "")
if not testcase:
raise ValueError("issue 缺少 testcase")
return {
"id": str(row.get("issue_id") or f"TEST-{testcase}-{result}"),
"testcase": testcase,
"issue_type": selected_type,
"result": result,
"phase": _value(row, "phase", defaults),
"expected": _value(row, "expected", defaults),
"observed": _value(row, "observed", defaults),
"artifact_hash": _value(row, "artifact_hash", defaults),
"environment": _value(row, "environment", defaults),
"evidence_path": _value(row, "evidence_path", defaults),
"reproduction": _value(row, "reproduction", defaults),
"next_step": _value(row, "next_step", defaults),
"created_at": str(row.get("created_at") or _now()),
}
def collect_issues(
results: Iterable[Mapping[str, Any]],
*,
defaults: Mapping[str, Any] | None = None,
) -> list[dict[str, Any]]:
issues: list[dict[str, Any]] = []
for row in results:
if not isinstance(row, Mapping) or str(row.get("result") or "") not in ISSUE_RESULTS:
continue
explicit = row.get("issue")
source = dict(row)
if isinstance(explicit, Mapping):
source.update(explicit)
issues.append(build_issue(source, defaults=defaults))
return issues
def feedback_plan(
trigger: str,
*,
target: str = "",
related_issue: str = "",
dispatcher: str = "dispatcher",
) -> dict[str, Any]:
"""Describe the required handoff without performing remote mutations."""
if trigger not in TRIGGERS:
raise ValueError(f"trigger 必须是 {', '.join(TRIGGERS)}")
if trigger == "pr":
actions = [
{"action": "update_pr", "target": target, "status": "required"},
{"action": "update_related_issue", "target": related_issue, "status": "required"},
{"action": "notify_dispatcher", "target": dispatcher, "status": "required"},
{"action": "reassign_development", "target": dispatcher, "status": "pending"},
]
route = "pr-update-issue-dispatcher-reassign"
else:
actions = [
{"action": "return_to_development", "target": target or trigger, "status": "required"},
]
route = "development"
return {
"trigger": trigger,
"route": route,
"target": target,
"related_issue": related_issue,
"actions": actions,
"remote_mutation": "caller-required",
"created_at": _now(),
}
def attach_feedback(
run: Mapping[str, Any],
*,
trigger: str | None = None,
target: str = "",
related_issue: str = "",
dispatcher: str = "dispatcher",
defaults: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Return a new run mapping with results, defects and routing preserved."""
output = deepcopy(dict(run))
results = output.get("results")
if not isinstance(results, list):
raise ValueError("run 缺少 results 列表")
issues = collect_issues(results, defaults=defaults)
output["defects"] = issues
# Compatibility name for consumers that call the same structure
# ``test-issues``. It is deliberately a copy, not a second source of truth.
output["test_issues"] = deepcopy(issues)
if trigger:
output["feedback"] = feedback_plan(
trigger,
target=target,
related_issue=related_issue,
dispatcher=dispatcher,
)
return output
+41
View File
@@ -40,6 +40,16 @@ FIELDS = (
"risk",
"driver",
"maps_to",
"blocked_by",
"issue_id",
"issue_type",
"expected",
"observed",
"artifact_hash",
"environment",
"evidence_path",
"reproduction",
"next_step",
)
@@ -112,6 +122,23 @@ def case_index(root: Path) -> dict[str, dict[str, Any]]:
return out
def issue_index(run: dict[str, Any]) -> dict[str, dict[str, Any]]:
"""Index canonical defects and compatibility test_issues by testcase."""
indexed: dict[str, dict[str, Any]] = {}
raw = run.get("defects")
if not isinstance(raw, list):
raw = run.get("test_issues")
if not isinstance(raw, list):
return indexed
for issue in raw:
if not isinstance(issue, dict):
continue
testcase = str(issue.get("testcase") or issue.get("id") or "")
if testcase:
indexed.setdefault(testcase, issue)
return indexed
def export_run_csv(
root: Path,
run_path: Path | None = None,
@@ -123,6 +150,7 @@ def export_run_csv(
path = path.resolve()
run = load_run(path)
cases = case_index(root)
issues = issue_index(run)
dest = out or path.with_suffix(".csv")
dest.parent.mkdir(parents=True, exist_ok=True)
with dest.open("w", encoding="utf-8-sig", newline="") as handle:
@@ -134,6 +162,9 @@ def export_run_csv(
cid = str(item["id"])
result = str(item.get("result") or "")
case = cases.get(cid) or {}
issue = dict(issues.get(cid, {}))
if isinstance(item.get("issue"), dict):
issue.update(item["issue"])
client_suite_value = case.get("client_suite") or (
case.get("suite") if test_type(case) == "journey" else ""
)
@@ -154,6 +185,16 @@ def export_run_csv(
"risk": case.get("risk") or "",
"driver": case.get("driver") or "",
"maps_to": case.get("maps_to") or "",
"blocked_by": one_line(item.get("blocked_by")),
"issue_id": issue.get("id") or item.get("issue_id") or "",
"issue_type": issue.get("issue_type") or item.get("issue_type") or "",
"expected": one_line(issue.get("expected") or item.get("expected")),
"observed": one_line(issue.get("observed") or item.get("observed")),
"artifact_hash": issue.get("artifact_hash") or item.get("artifact_hash") or "",
"environment": one_line(issue.get("environment") or item.get("environment")),
"evidence_path": one_line(issue.get("evidence_path") or item.get("evidence_path")),
"reproduction": one_line(issue.get("reproduction") or item.get("reproduction")),
"next_step": one_line(issue.get("next_step") or item.get("next_step")),
}
)
return dest
+11 -4
View File
@@ -254,6 +254,7 @@ def propagate_results(
skipped: Mapping[str, str] | None = None,
continue_on_fail: bool = False,
policies: Mapping[str, str] | None = None,
blocked_by: Mapping[str, Iterable[str]] | None = None,
) -> list[dict[str, str]]:
"""Build ordered result rows and mark cases after a blocking failure not-run.
@@ -263,15 +264,21 @@ def propagate_results(
"""
skipped = skipped or {}
policies = policies or {}
blocked_by = blocked_by or {}
rows: list[dict[str, str]] = []
blocked = False
blocking_case = ""
for raw_id in case_ids:
case_id = str(raw_id)
if case_id in skipped:
rows.append({"id": case_id, "result": "skipped", "notes": skipped[case_id]})
continue
if blocked:
rows.append({"id": case_id, "result": "not-run", "notes": "前置用例失败,未轮到执行"})
if blocking_case:
rows.append({
"id": case_id,
"result": "not-run",
"notes": "前置用例失败,未轮到执行",
"blocked_by": ",".join(blocked_by.get(case_id) or (blocking_case,)),
})
continue
result = observed.get(case_id, "not-run")
if result not in RESULTS:
@@ -279,5 +286,5 @@ def propagate_results(
rows.append({"id": case_id, "result": result})
policy = policies.get(case_id)
if result in ("fail", "environment-fail") and not (continue_on_fail or policy == "continue"):
blocked = True
blocking_case = case_id
return rows
+230 -14
View File
@@ -54,6 +54,7 @@ class CommandResult:
@dataclass
class RuntimeContext:
root: Path
bindings: dict[str, str] = field(default_factory=dict)
retries: dict[str, int] = field(default_factory=lambda: {"download": 3, "start": 3, "install": 3})
evidence: list[dict[str, Any]] = field(default_factory=list)
@@ -126,6 +127,120 @@ def parse_artifacts(values: list[str]) -> dict[str, str]:
return pairs
def parse_bindings(values: list[str], requested: list[str]) -> dict[str, str]:
"""Resolve one tester binding per requested platform.
A single legacy ``agent`` mapping cannot safely represent a multi-platform
run. Keep the platform distinction in the command contract all the way to
agent-runtime instead of treating binding as report metadata.
"""
pairs: dict[str, str] = {}
for raw in values:
platform, sep, value = raw.partition("=")
platform = platform.strip().lower()
value = value.strip()
if not sep or platform not in PLATFORMS or not value or not re.fullmatch(r"[a-z][a-z0-9-]*(?:\.[a-z][a-z0-9-]*)+", value):
raise ArtifactFailure(f"--binding 格式必须是 android=tester.<name> 或 ios=tester.<name>: {raw}")
if platform in pairs:
raise ArtifactFailure(f"--binding 重复平台: {platform}")
pairs[platform] = value
resolved = {platform: pairs.get(platform, f"tester.{platform}") for platform in requested}
if len(set(resolved.values())) != len(resolved) and len(resolved) > 1:
raise ArtifactFailure("多平台运行不能复用同一个 tester binding;请分别配置 tester.android/tester.ios")
return resolved
def load_agent_runtime():
candidates = [
Path.home() / ".codex" / "skills" / "agent-runtime" / "scripts" / "agent.py",
Path.home() / ".pouch" / "skills" / "agent-runtime" / "scripts" / "agent.py",
Path(__file__).resolve().parents[2] / "agent-runtime" / "scripts" / "agent.py",
]
runtime_path = next((item for item in candidates if item.is_file()), None)
if runtime_path is None:
raise EnvironmentFailure("找不到共享 agent-runtime: " + ", ".join(str(item) for item in candidates))
spec = importlib.util.spec_from_file_location("pouch_agent_runtime_for_mobile_bundle", runtime_path)
if spec is None or spec.loader is None:
raise EnvironmentFailure(f"无法加载共享 agent-runtime: {runtime_path}")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def binding_location(root: Path, binding_id: str) -> dict[str, Any]:
runtime = load_agent_runtime()
try:
value = runtime.binding(runtime.load_config(root), binding_id)
except (ValueError, OSError) as exc:
raise EnvironmentFailure(f"无法解析 {binding_id}: {exc}") from exc
if value.get("adapter") != "tester":
raise EnvironmentFailure(f"{binding_id} 不是 tester binding")
return value
def agent_runner(root: Path, request: dict[str, Any]) -> dict[str, Any]:
"""Dispatch a platform operation through the resolved tester binding."""
runtime = load_agent_runtime()
return runtime.execute_runner_request(root, request)
def platform_node(root: Path, binding_id: str) -> str:
location = binding_location(root, binding_id)["location"]
return "local" if location.get("environment") == "local" else str(location.get("environment"))
def validate_preflight_matrix(root: Path, config_path: Path, bindings: dict[str, str], requested: list[str]) -> None:
"""Fail closed when a matrix routes a platform to the wrong node/runner."""
try:
import yaml
raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
except (OSError, ValueError, ImportError) as exc:
raise EnvironmentFailure(f"无法读取 preflight matrix: {config_path}: {exc}") from exc
rows = raw.get("matrix") if isinstance(raw, dict) else None
if not isinstance(rows, list):
raise EnvironmentFailure("preflight matrix 缺少 matrix;不能推断平台执行节点")
by_platform = {str(row.get("platform", "")).lower(): row for row in rows if isinstance(row, dict)}
for platform in requested:
expected_binding = bindings[platform]
row = by_platform.get(platform)
if not row:
raise EnvironmentFailure(f"preflight matrix 缺少 {platform} 行;不能创建该平台 session")
tester = row.get("tester") if isinstance(row, dict) else None
actual_binding = tester.get("binding") if isinstance(tester, dict) else None
if actual_binding != expected_binding:
raise EnvironmentFailure(f"{platform} preflight binding={actual_binding!r},期望 {expected_binding!r}")
expected_node = platform_node(root, expected_binding)
actual_node = tester.get("node") if isinstance(tester, dict) else None
if expected_node == "local" and actual_node not in {"local", "localhost", "127.0.0.1"}:
raise EnvironmentFailure(f"{platform} binding 是 local,但 preflight node={actual_node!r}")
runner = tester.get("runner") if isinstance(tester, dict) else None
if expected_node != "local" and (not isinstance(runner, dict) or runner.get("kind") != "contract"):
raise EnvironmentFailure(f"{platform} 绑定到 {expected_node},必须使用 contract runner;禁止在协调节点执行")
def preflight_remote_artifacts(config_path: Path, requested: list[str]) -> dict[str, str]:
"""Return caller-declared artifact paths readable by remote tester nodes."""
try:
import yaml
raw = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
except (OSError, ValueError, ImportError) as exc:
raise EnvironmentFailure(f"无法读取 preflight matrix: {config_path}: {exc}") from exc
rows = raw.get("matrix") if isinstance(raw, dict) else None
if not isinstance(rows, list):
raise EnvironmentFailure("preflight matrix 缺少 matrix;不能解析远程产物路径")
by_platform = {str(row.get("platform", "")).lower(): row for row in rows if isinstance(row, dict)}
paths: dict[str, str] = {}
for platform in requested:
row = by_platform.get(platform) or {}
artifact = row.get("artifact") if isinstance(row, dict) else None
remote_path = artifact.get("remote_path") if isinstance(artifact, dict) else None
if isinstance(remote_path, str) and remote_path.strip():
paths[platform] = remote_path.strip()
return paths
def local_path(locator: str) -> Path | None:
parsed = urllib.parse.urlsplit(locator)
if parsed.scheme in ("http", "https"):
@@ -211,6 +326,10 @@ def android_identity(path: Path, context: RuntimeContext) -> str:
result = run_command(command, context=context)
if result.code == 0:
match = re.search(r"(?:name|application-id)[=:]['\"]?([A-Za-z0-9_.]+)", result.output)
if not match:
# apkanalyzer manifest application-id prints the bare package
# name, while aapt/aapt2 print a package: name=... record.
match = re.fullmatch(r"\s*([A-Za-z0-9_.]+)\s*", result.output)
if match:
return match.group(1)
raise EnvironmentFailure("APK 身份读取失败;请检查 Android SDK build-tools")
@@ -390,6 +509,27 @@ def write_run(root: Path, record: dict[str, Any]) -> Path:
return path
def all_active_result_rows(root: Path, *, blocked_by: str) -> list[dict[str, Any]]:
"""Keep a testcase row for every active catalog entry when preflight blocks."""
try:
from generate_cases_csv import load_cases
cases = load_cases(root)
except (ImportError, OSError, ValueError, SystemExit):
cases = []
rows: list[dict[str, Any]] = []
for case in cases:
if case.get("lifecycle", "active") != "active":
continue
rows.append({
"id": str(case.get("id") or ""),
"result": "not-run",
"notes": "test-preflight 未通过,未创建移动 session",
"blocked_by": [blocked_by],
})
return rows
def run_test_preflight(
config_path: Path,
*,
@@ -419,18 +559,21 @@ def run_test_preflight(
def execute(args: argparse.Namespace) -> int:
requested = platforms(args.platform)
context = RuntimeContext(Path(args.root).resolve())
root = Path(args.root).resolve()
bindings = parse_bindings(args.bindings, requested)
context = RuntimeContext(root, bindings=bindings)
record: dict[str, Any] = {
"schema_version": 1,
"started_at": datetime.now(timezone.utc).isoformat(),
"platforms_requested": requested,
"partial": bool(args.partial),
"binding": args.binding,
"bindings": bindings,
"node": os.environ.get("TESTER_NODE", "local"),
"terminal": {"handle": os.environ.get("ORCA_TERMINAL_HANDLE"), "created": os.environ.get("TESTER_TERMINAL_CREATED", "false").lower() == "true", "closed": os.environ.get("TESTER_TERMINAL_CLOSED", "unknown")},
"product_test": {"status": "not-run", "failure_class": "automatic-repair-failed", "note": "准备完成后由 tester catalog 驱动真实移动 session;本工具不以 API 替代移动测试"},
"platforms": {},
}
remote_artifacts: dict[str, str] = {}
try:
locators = parse_artifacts(args.artifacts)
missing = [platform for platform in requested if platform not in locators]
@@ -475,6 +618,8 @@ def execute(args: argparse.Namespace) -> int:
"error": "缺少 --preflight-config;不能跳过 test-preflight",
}
else:
validate_preflight_matrix(context.root, preflight_config, bindings, requested)
remote_artifacts = preflight_remote_artifacts(preflight_config, requested)
preflight_result = run_test_preflight(
preflight_config,
mode=args.preflight_mode,
@@ -494,10 +639,28 @@ def execute(args: argparse.Namespace) -> int:
for platform in requested:
result, _, identity = prepared[platform]
try:
node = android_preflight(context) if platform == "android" else ios_preflight(context)
check_network(context, node)
result.update({"node_preflight": node, "status": "preflight-passed"})
prepared[platform] = (result, node, identity)
binding_id = bindings[platform]
node_name = platform_node(context.root, binding_id)
probe = agent_runner(context.root, {
"operation": "platform_probe",
"platform": platform,
"node": node_name,
"binding": binding_id,
})
if probe.get("status") != "ready":
raise EnvironmentFailure(str(probe.get("observed") or probe))
host = os.environ.get("HOST", "").strip()
network = agent_runner(context.root, {
"operation": "network_probe",
"platform": platform,
"node": node_name,
"binding": binding_id,
"url": f"http://{host}/v1/me",
}) if host else {"status": "blocked", "observed": "HOST 未设置"}
if network.get("status") != "ready":
raise EnvironmentFailure(str(network.get("observed") or network))
result.update({"node_preflight": probe, "network_preflight": network, "status": "preflight-passed"})
prepared[platform] = (result, None, identity)
except EnvironmentFailure as exc:
failures[platform] = str(exc)
result.update({"status": "environment-fail", "failure_class": "automatic-repair-failed", "error": str(exc)})
@@ -507,16 +670,51 @@ def execute(args: argparse.Namespace) -> int:
record["platforms"][platform].update({"status": "not-run", "reason": "全量 preflight 闸门被另一平台阻断"})
overall = record.get("preflight", {}).get("status") if "preflight" in record else "environment-fail"
record.update({"status": overall if overall in {"blocked", "environment-fail"} else "environment-fail", "failure_class": "test-preflight" if "preflight" in record else "automatic-repair-failed", "next_action": "修复列出的平台环境后重跑;默认不会静默降级"})
record["results"] = all_active_result_rows(context.root, blocked_by="test-preflight")
else:
for platform, (result, node, identity) in prepared.items():
try:
installed = (
{"delegated": "test-preflight.prepare"}
if args.preflight_mode == "prepare"
else install_android(node, Path(result["artifact"]["path"]), identity, context)
if platform == "android"
else install_ios(node, Path(result["artifact"]["path"]), identity, context)
)
if args.preflight_mode == "prepare":
# test-preflight already ran platform_probe, network_probe,
# install and start through each matrix binding.
installed = {"delegated": "test-preflight.prepare", "binding": bindings[platform]}
else:
binding_id = bindings[platform]
node_name = platform_node(context.root, binding_id)
probe = agent_runner(context.root, {
"operation": "platform_probe",
"platform": platform,
"node": node_name,
"binding": binding_id,
})
if probe.get("status") != "ready":
raise EnvironmentFailure(str(probe.get("observed") or probe))
host = os.environ.get("HOST", "").strip()
if host:
network = agent_runner(context.root, {
"operation": "network_probe",
"platform": platform,
"node": node_name,
"binding": binding_id,
"url": f"http://{host}/v1/me",
})
if network.get("status") != "ready":
raise EnvironmentFailure(str(network.get("observed") or network))
artifact_path = result["artifact"]["path"]
if node_name != "local":
artifact_path = remote_artifacts.get(platform, "")
if not artifact_path:
raise EnvironmentFailure(f"远程 {platform} 安装必须在 matrix.artifact.remote_path 声明节点可读产物")
installed = agent_runner(context.root, {
"operation": "install",
"platform": platform,
"node": node_name,
"binding": binding_id,
"artifact": artifact_path,
"identity": identity,
})
if installed.get("status") != "ready":
raise EnvironmentFailure(str(installed.get("observed") or installed))
result.update({"status": "ready", "install": installed, "failure_class": None})
except EnvironmentFailure as exc:
failures[platform] = str(exc)
@@ -527,7 +725,18 @@ def execute(args: argparse.Namespace) -> int:
record["evidence"] = context.evidence
except (EnvironmentFailure, ArtifactFailure) as exc:
record.update({"status": "environment-fail", "failure_class": "automatic-repair-failed", "error": str(exc), "evidence": context.evidence})
if record.get("status") != "ready" and "results" not in record:
record["results"] = all_active_result_rows(context.root, blocked_by="test-preflight")
path = write_run(context.root, record)
try:
from generate_run_csv import export_run_csv
record["csv"] = str(export_run_csv(context.root, run_path=path))
except (OSError, SystemExit, ValueError):
# The YAML run remains the source of truth when a partial/provisioning
# root has no catalog cases yet.
record["csv"] = ""
path.write_text(json.dumps(record, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output = {"status": record["status"], "run": str(path), "platforms": record["platforms"], "worktree": record.get("worktree")}
if record.get("error"):
output["error"] = record["error"]
@@ -547,7 +756,14 @@ def parser() -> argparse.ArgumentParser:
run.add_argument("--partial", action="store_true")
run.add_argument("--root", default=".")
run.add_argument("--worktree-root", default="")
run.add_argument("--binding", default="")
run.add_argument(
"--binding",
action="append",
dest="bindings",
default=[],
metavar="PLATFORM=TESTER_BINDING",
help="每个平台的 tester binding,可重复;默认 tester.<platform>",
)
run.add_argument("--preflight-config", default="", help="test-preflight matrix YAML(必填,不得跳过)")
run.add_argument("--preflight-mode", choices=("verify", "prepare"), default="verify")
run.add_argument("--preflight-yaml", default="", help="preflight YAML 证据输出路径")
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""Append structured tester issue evidence to an existing run snapshot."""
from __future__ import annotations
import argparse
from pathlib import Path
from typing import Any
import yaml
from failure_feedback import attach_feedback
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="将 tester 失败结构化回流到 run YAML/CSV")
parser.add_argument("--run", required=True, type=Path)
parser.add_argument("--trigger", choices=("pr", "requirement", "bug"))
parser.add_argument("--target", default="")
parser.add_argument("--related-issue", default="")
parser.add_argument("--dispatcher", default="dispatcher")
parser.add_argument("--testcase", default="", help="为一条失败结果补充 testcase issue 字段")
parser.add_argument("--issue-type", choices=("product", "test"))
parser.add_argument("--expected", default="")
parser.add_argument("--observed", default="")
parser.add_argument("--artifact-hash", default="")
parser.add_argument("--environment", default="")
parser.add_argument("--evidence-path", default="")
parser.add_argument("--reproduction", default="")
parser.add_argument("--next-step", default="")
args = parser.parse_args(argv)
data: dict[str, Any] = yaml.safe_load(args.run.read_text(encoding="utf-8")) or {}
if args.testcase:
rows = data.get("results")
if not isinstance(rows, list):
raise SystemExit("run 缺少 results 列表")
row = next((item for item in rows if isinstance(item, dict) and str(item.get("id")) == args.testcase), None)
if row is None:
raise SystemExit(f"找不到 testcase: {args.testcase}")
row["issue"] = {
key: value
for key, value in {
"issue_type": args.issue_type,
"expected": args.expected,
"observed": args.observed,
"artifact_hash": args.artifact_hash,
"environment": args.environment,
"evidence_path": args.evidence_path,
"reproduction": args.reproduction,
"next_step": args.next_step,
}.items()
if value
}
output = attach_feedback(
data,
trigger=args.trigger,
target=args.target,
related_issue=args.related_issue,
dispatcher=args.dispatcher,
)
args.run.write_text(yaml.safe_dump(output, allow_unicode=True, sort_keys=False), encoding="utf-8")
csv_path = ""
try:
from generate_run_csv import export_run_csv
csv_path = str(export_run_csv(args.run.parents[3], run_path=args.run))
except (ImportError, OSError, SystemExit, ValueError):
pass
print(f"updated {args.run} defects={len(output.get('defects') or [])} csv={csv_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+8 -3
View File
@@ -180,9 +180,14 @@ def clean_config(config: dict[str, Any]) -> dict[str, Any]:
def config_from_binding(root: Path, selector: str) -> dict[str, Any]:
"""Materialize a common agent-runtime tester binding for legacy local.yaml."""
runtime_path = Path(__file__).resolve().parents[3] / "skills" / "agent-runtime" / "scripts" / "agent.py"
if not runtime_path.is_file():
raise RuntimeError(f"找不到共享 agent-runtime: {runtime_path}")
candidates = [
Path.home() / ".codex" / "skills" / "agent-runtime" / "scripts" / "agent.py",
Path.home() / ".pouch" / "skills" / "agent-runtime" / "scripts" / "agent.py",
Path(__file__).resolve().parents[3] / "skills" / "agent-runtime" / "scripts" / "agent.py",
]
runtime_path = next((item for item in candidates if item.is_file()), None)
if runtime_path is None:
raise RuntimeError("找不到共享 agent-runtime: " + ", ".join(str(item) for item in candidates))
spec = importlib.util.spec_from_file_location("pouch_agent_runtime", runtime_path)
if spec is None or spec.loader is None:
raise RuntimeError(f"无法加载共享 agent-runtime: {runtime_path}")
+21
View File
@@ -103,6 +103,27 @@ def load_yaml_results(path: Path) -> list[dict[str, Any]]:
errors.append(f"YAML testcase id 重复: {_format_ids(set(duplicates))}")
if errors:
raise SummaryError("YAML 校验失败:\n- " + "\n- ".join(errors))
issues = data.get("defects", data.get("test_issues"))
if issues is not None:
if not isinstance(issues, list):
raise SummaryError("YAML defects/test_issues 必须是列表")
result_ids = set(ids)
issue_errors: list[str] = []
for index, issue in enumerate(issues, start=1):
if not isinstance(issue, dict):
issue_errors.append(f"YAML defects[{index}] 不是 mapping")
continue
testcase = str(issue.get("testcase") or "")
issue_type = str(issue.get("issue_type") or "")
if not testcase or testcase not in result_ids:
issue_errors.append(f"YAML defects[{index}] testcase 无效: {testcase or '(missing)'}")
if issue_type not in {"product", "test"}:
issue_errors.append(f"YAML defects[{index}] issue_type 无效: {issue_type or '(missing)'}")
for field in ("expected", "observed", "artifact_hash", "environment", "evidence_path", "reproduction", "next_step"):
if field not in issue:
issue_errors.append(f"YAML defects[{index}] 缺少 {field}")
if issue_errors:
raise SummaryError("YAML defects 校验失败:\n- " + "\n- ".join(issue_errors))
return results
+80 -20
View File
@@ -12,6 +12,8 @@ from __future__ import annotations
from collections.abc import Callable, Iterable, Mapping
from typing import Any
from failure_feedback import collect_issues
RESULTS = ("pass", "fail", "environment-fail", "skipped", "not-run")
PHASES = ("preflight", "integration", "journey")
@@ -29,10 +31,18 @@ def _case_id(case: Mapping[str, Any] | str) -> str:
return str(case)
def _not_run(cases: Iterable[Mapping[str, Any] | str], reason: str) -> list[dict[str, str]]:
def _not_run(
cases: Iterable[Mapping[str, Any] | str],
reason: str,
*,
blocked_by: Iterable[str] = (),
) -> list[dict[str, Any]]:
rows: list[dict[str, str]] = []
blockers = [str(item) for item in blocked_by if str(item)]
for case in cases:
row = {"id": _case_id(case), "result": "not-run", "notes": reason}
row: dict[str, Any] = {"id": _case_id(case), "result": "not-run", "notes": reason}
if blockers:
row["blocked_by"] = blockers
if isinstance(case, Mapping):
row["test_type"] = str(case.get("test_type") or "")
row["suite"] = str(
@@ -45,6 +55,37 @@ def _not_run(cases: Iterable[Mapping[str, Any] | str], reason: str) -> list[dict
return rows
def _dependencies(
case: Mapping[str, Any] | str,
*,
default: Iterable[str] = (),
) -> list[str]:
if not isinstance(case, Mapping):
return [str(item) for item in default]
declared = case.get("depends_on", case.get("dependencies"))
if declared is None:
return [str(item) for item in default]
if isinstance(declared, str):
return [declared]
if isinstance(declared, Iterable):
return [str(item) for item in declared if str(item)]
raise ValueError(f"depends_on 必须是字符串或列表: {_case_id(case)}")
def _failed_dependencies(dependencies: Iterable[str], results: Mapping[str, Mapping[str, Any]]) -> list[str]:
return [
dependency
for dependency in dependencies
if str((results.get(dependency) or {}).get("result"))
in {"fail", "environment-fail", "not-run"}
]
def _issue_structures(results: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
issues = collect_issues(results)
return {"defects": issues, "test_issues": [dict(issue) for issue in issues]}
def run_integration_first(
*,
preflight_status: str,
@@ -65,18 +106,29 @@ def run_integration_first(
journey = list(journey_cases)
if preflight_status != "ready":
reason = f"test-preflight={preflight_status},未创建客户端 session"
result_rows = _not_run(integration, reason, blocked_by=("test-preflight",)) + _not_run(journey, reason, blocked_by=("test-preflight",))
return {
"status": preflight_status,
"phase": "preflight",
"session_created": False,
"results": _not_run(integration, reason) + _not_run(journey, reason),
"results": result_rows,
**_issue_structures(result_rows),
}
results: list[dict[str, Any]] = []
integration_failed = False
for index, case in enumerate(integration):
if integration_failed:
results.extend(_not_run([case], "前置 integration 失败,未轮到执行"))
results_by_id: dict[str, dict[str, Any]] = {}
for case in integration:
case_id = _case_id(case)
dependencies = _dependencies(case)
blocked_by = _failed_dependencies(dependencies, results_by_id)
if blocked_by:
rows = _not_run(
[case],
"依赖的 integration 失败,未执行",
blocked_by=blocked_by,
)
results.extend(rows)
results_by_id[case_id] = rows[0]
continue
outcome = execute_integration(case)
row = dict(outcome) if isinstance(outcome, Mapping) else {"result": outcome}
@@ -89,20 +141,26 @@ def run_integration_first(
raise ValueError(f"未知 integration 结果: {row['id']}={result}")
row["result"] = result
results.append(row)
if result in ("fail", "environment-fail"):
integration_failed = True
results_by_id[case_id] = row
if integration_failed:
results.extend(_not_run(journey, "integration 失败,依赖的 journey 未执行"))
return {
"status": "fail" if any(row["result"] == "fail" for row in results) else "environment-fail",
"phase": "integration",
"session_created": False,
"results": results,
}
session = create_journey_session() if journey else None
integration_ids = [_case_id(case) for case in integration]
has_runnable_journey = any(
not _failed_dependencies(_dependencies(case, default=integration_ids), results_by_id)
for case in journey
)
session = create_journey_session() if has_runnable_journey else None
for case in journey:
dependencies = _dependencies(case, default=integration_ids)
blocked_by = _failed_dependencies(dependencies, results_by_id)
if blocked_by:
row = _not_run(
[case],
"依赖的用例失败,journey 未执行",
blocked_by=blocked_by,
)[0]
results.append(row)
results_by_id[_case_id(case)] = row
continue
outcome = execute_journey(session, case)
row = dict(outcome) if isinstance(outcome, Mapping) else {"result": outcome}
row["id"] = _case_id(case)
@@ -114,12 +172,14 @@ def run_integration_first(
raise ValueError(f"未知 journey 结果: {row['id']}={result}")
row["result"] = result
results.append(row)
results_by_id[_case_id(case)] = row
status = "fail" if any(row["result"] == "fail" for row in results) else (
"environment-fail" if any(row["result"] == "environment-fail" for row in results) else "pass"
)
return {
"status": status,
"phase": "journey" if journey else "integration",
"session_created": bool(journey),
"session_created": has_runnable_journey,
"results": results,
**_issue_structures(results),
}
+4
View File
@@ -22,6 +22,10 @@ risks:
- mutating
- destructive
default_max_risk: readonly
# standard enforces the default_max_risk/authorize policy. Writes-style isolated
# resettable environments may set this to isolated-full; risk values remain on
# cases and are still reported.
execution_profile: standard
client_suites:
client-smoke:
default: true
+88
View File
@@ -50,6 +50,84 @@ class MobileBundleTests(unittest.TestCase):
with self.assertRaises(bundle.ArtifactFailure):
bundle.parse_artifacts(["android=/tmp/a.apk", "android=/tmp/b.apk"])
def test_parse_bindings_keeps_platforms_separate(self):
self.assertEqual(
bundle.parse_bindings(["android=tester.android", "ios=tester.ios"], ["android", "ios"]),
{"android": "tester.android", "ios": "tester.ios"},
)
self.assertEqual(
bundle.parse_bindings([], ["android"]),
{"android": "tester.android"},
)
with self.assertRaises(bundle.ArtifactFailure):
bundle.parse_bindings(["android=tester.test", "ios=tester.test"], ["android", "ios"])
def test_preflight_matrix_rejects_remote_platform_using_local_runner(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
agent_dir = root / ".pouch"
agent_dir.mkdir()
(agent_dir / "agent.yaml").write_text(
"""
schemaVersion: 1
locations:
local-app:
environment: local
host: local
project: test
path: /tmp/test
remote-ios:
environment: macmini
host: runtime:12345678-1234-1234-1234-123456789012
project: test
path: /tmp/test-ios
profiles:
test-agent:
role: test
cli: codex
tier: standard
model: gpt-5
reasoningEffort: medium
permissionMode: workspace-write
bindings:
tester.android: {adapter: tester, location: local-app, profile: test-agent}
tester.ios: {adapter: tester, location: remote-ios, profile: test-agent}
""".strip()
+ "\n",
encoding="utf-8",
)
config = root / "matrix.yaml"
config.write_text(
"""
matrix:
- platform: android
tester: {binding: tester.android, node: local, runner: {kind: local}}
- platform: ios
tester: {binding: tester.ios, node: local, runner: {kind: local}}
""".strip()
+ "\n",
encoding="utf-8",
)
with self.assertRaisesRegex(bundle.EnvironmentFailure, "ios.*contract runner"):
bundle.validate_preflight_matrix(
root,
config,
{"android": "tester.android", "ios": "tester.ios"},
["android", "ios"],
)
def test_remote_artifact_path_comes_from_preflight_matrix(self):
with tempfile.TemporaryDirectory() as temp:
config = Path(temp) / "matrix.yaml"
config.write_text(
"matrix:\n - platform: ios\n artifact:\n remote_path: /Users/tester/app.ipa\n",
encoding="utf-8",
)
self.assertEqual(
bundle.preflight_remote_artifacts(config, ["ios"]),
{"ios": "/Users/tester/app.ipa"},
)
def test_new_worktree_each_run(self):
with tempfile.TemporaryDirectory() as temp:
repo, worktrees = Path(temp) / "repo", Path(temp) / "worktrees"
@@ -80,6 +158,16 @@ class MobileBundleTests(unittest.TestCase):
self.assertTrue(result["verified"])
self.assertTrue(any("install" in call for call in calls))
def test_android_identity_accepts_bare_apkanalyzer_output(self):
with tempfile.TemporaryDirectory() as temp:
apk = Path(temp) / "app.apk"
apk.write_bytes(b"apk")
context = bundle.RuntimeContext(Path(temp))
with mock.patch.object(bundle.shutil, "which", side_effect=lambda name: "apkanalyzer" if name == "apkanalyzer" else None), mock.patch.object(
bundle, "run_command", return_value=bundle.CommandResult(0, "page.writes\n")
):
self.assertEqual(bundle.android_identity(apk, context), "page.writes")
def test_android_missing_device_is_environment_failure(self):
context = bundle.RuntimeContext(Path("."))
with mock.patch.object(bundle.shutil, "which", return_value="/sdk/adb"), mock.patch.object(
+111 -2
View File
@@ -150,6 +150,86 @@ class TwoTierCatalogTests(unittest.TestCase):
self.assertEqual(report_rows[0]["test_type"], "integration")
self.assertEqual(report_rows[0]["suite"], "api-contract")
def test_isolated_full_keeps_risk_and_ignores_authorize_filter(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
write_mixed_catalog(root)
catalog_path = root / ".pouch/tester/catalog.yaml"
catalog = yaml.safe_load(catalog_path.read_text(encoding="utf-8"))
catalog["authorize"] = {"mutating": "AUTHORIZE_MUTATING"}
catalog["execution_profile"] = "isolated-full"
catalog_path.write_text(yaml.safe_dump(catalog, sort_keys=False), encoding="utf-8")
case_path = root / ".pouch/tester/cases/mixed.yaml"
cases = yaml.safe_load(case_path.read_text(encoding="utf-8"))
journey = cases["cases"][0]
journey.update({"risk": "mutating", "tag": "full", "client_suite": "client-regression"})
case_path.write_text(yaml.safe_dump(cases, sort_keys=False), encoding="utf-8")
selected = self.run_catalog(root, "select", "--suite", "client-regression")
self.assertEqual(selected.returncode, 0, selected.stderr)
payload = json.loads(selected.stdout)
self.assertEqual(payload["execution_profile"], "isolated-full")
self.assertEqual(payload["risk_filter"], "disabled")
self.assertEqual(payload["count"], 1)
self.assertEqual(payload["cases"][0]["risk"], "mutating")
self.assertEqual(payload["skipped"], [])
all_active = self.run_catalog(root, "select")
self.assertEqual(all_active.returncode, 0, all_active.stderr)
all_payload = json.loads(all_active.stdout)
self.assertEqual(all_payload["count"], 2)
self.assertEqual(all_payload["test_types"], {"journey": 1, "integration": 1})
def test_isolated_full_run_script_does_not_use_authorize_switch(self):
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
tester = root / ".pouch" / "tester"
(tester / "cases").mkdir(parents=True)
(tester / "local.yaml").write_text("HOST: http://127.0.0.1\nAUTHORIZE_MUTATING: false\n", encoding="utf-8")
(tester / "catalog.yaml").write_text(
yaml.safe_dump(
{
"version": 2,
"id_pattern": "^TC-[A-Z]+-[0-9]+$",
"categories": [{"id": "SCRIPT"}],
"risks": ["readonly", "mutating", "destructive"],
"default_max_risk": "readonly",
"execution_profile": "isolated-full",
"authorize": {"mutating": "AUTHORIZE_MUTATING"},
},
sort_keys=False,
),
encoding="utf-8",
)
(tester / "cases" / "script.yaml").write_text(
yaml.safe_dump(
{
"category": "SCRIPT",
"cases": [{
"id": "TC-SCRIPT-001",
"title": "mutating script",
"what": "run isolated script",
"lifecycle": "active",
"tag": "full",
"risk": "mutating",
"driver": "script",
"how": {"run": "exit 0"},
"cleanup": {"steps": ["reset"]},
"expected": [{"kind": "script-exit", "value": 0}],
}],
},
sort_keys=False,
),
encoding="utf-8",
)
result = subprocess.run(
[sys.executable, str(CATALOG), "run-script", "--root", str(root), "--id", "TC-SCRIPT-001"],
text=True,
capture_output=True,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(json.loads(result.stdout)["result"], "executed")
class IntegrationFirstTests(unittest.TestCase):
def test_preflight_block_does_not_create_session(self):
@@ -177,9 +257,38 @@ class IntegrationFirstTests(unittest.TestCase):
execute_journey=lambda session, case: "pass",
)
self.assertEqual(result["status"], "fail")
self.assertEqual(calls, ["INT-1"])
self.assertEqual(calls, ["INT-1", "INT-2"])
self.assertFalse(result["session_created"])
self.assertEqual([row["result"] for row in result["results"]], ["fail", "not-run", "not-run", "not-run"])
self.assertEqual([row["result"] for row in result["results"]], ["fail", "pass", "not-run", "not-run"])
self.assertEqual(result["results"][2]["blocked_by"], ["INT-1"])
def test_independent_integration_continues_and_dependency_is_blocked(self):
result = run_integration_first(
preflight_status="ready",
integration_cases=[
{"id": "INT-1"},
{"id": "INT-2"},
{"id": "INT-3", "depends_on": ["INT-1"]},
],
journey_cases=[],
execute_integration=lambda case: "fail" if case["id"] == "INT-1" else "pass",
create_journey_session=lambda: object(),
execute_journey=lambda session, case: "pass",
)
self.assertEqual([row["result"] for row in result["results"]], ["fail", "pass", "not-run"])
self.assertEqual(result["results"][2]["blocked_by"], ["INT-1"])
def test_preflight_block_creates_every_result_row(self):
result = run_integration_first(
preflight_status="blocked",
integration_cases=[{"id": "INT-1"}],
journey_cases=[{"id": "J-1"}],
execute_integration=lambda case: "pass",
create_journey_session=lambda: object(),
execute_journey=lambda session, case: "pass",
)
self.assertEqual([row["result"] for row in result["results"]], ["not-run", "not-run"])
self.assertEqual(result["results"][0]["blocked_by"], ["test-preflight"])
def test_service_unreachable_is_environment_failure(self):
self.assertEqual(classify_integration(ok=False, service_reachable=False), "environment-fail")
+59
View File
@@ -12,6 +12,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
CASES = REPO_ROOT / "skills" / "tester" / "scripts" / "generate_cases_csv.py"
RUNS = REPO_ROOT / "skills" / "tester" / "scripts" / "generate_run_csv.py"
sys.path.insert(0, str(RUNS.parent))
from failure_feedback import attach_feedback
def load_module(name: str, path: Path):
@@ -29,6 +30,7 @@ class TesterRunRecordTests(unittest.TestCase):
root = Path(temp)
cases_dir = root / ".pouch" / "tester" / "cases"
cases_dir.mkdir(parents=True)
(root / ".pouch" / "tester" / "runs").mkdir()
(root / ".pouch" / "tester" / "catalog.yaml").write_text(
"categories:\n - id: smoke\n title: Smoke\n",
encoding="utf-8",
@@ -51,6 +53,63 @@ class TesterRunRecordTests(unittest.TestCase):
self.assertNotEqual(first, second)
self.assertRegex(first, r"^\d{8}T\d{6}Z-smoke-[0-9a-f]{6}$")
def test_run_csv_preserves_all_results_and_structured_issue_fields(self) -> None:
runs = load_module("tester_run_csv_issues", RUNS)
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
cases_dir = root / ".pouch" / "tester" / "cases"
cases_dir.mkdir(parents=True)
(root / ".pouch" / "tester" / "runs").mkdir()
(root / ".pouch" / "tester" / "catalog.yaml").write_text(
"categories:\n - id: smoke\n title: Smoke\n", encoding="utf-8"
)
(cases_dir / "smoke.yaml").write_text(
"category: smoke\ncases:\n"
" - id: TC-1\n title: One\n what: one\n lifecycle: active\n"
" tag: smoke\n risk: readonly\n how: {steps: [open]}\n"
" expected: [{kind: visible-text, value: ready}]\n"
" - id: TC-2\n title: Two\n what: two\n lifecycle: active\n"
" tag: smoke\n risk: readonly\n how: {steps: [open]}\n"
" expected: [{kind: visible-text, value: ready}]\n",
encoding="utf-8",
)
run = attach_feedback(
{
"runId": "issues",
"results": [
{
"id": "TC-1",
"result": "fail",
"issue": {
"expected": "ready",
"observed": "blank",
"artifact_hash": "sha256:abc",
"environment": "Android tester",
"evidence_path": "evidence/1.png",
"reproduction": "open",
"next_step": "targeted rerun",
},
},
{"id": "TC-2", "result": "not-run", "blocked_by": ["TC-1"]},
],
},
trigger="pr",
target="PR-1",
related_issue="ISSUE-1",
)
run_path = root / ".pouch" / "tester" / "runs" / "issues.yaml"
import yaml
run_path.write_text(yaml.safe_dump(run, sort_keys=False), encoding="utf-8")
out = runs.export_run_csv(root, run_path=run_path)
with out.open(encoding="utf-8-sig", newline="") as handle:
rows = list(csv.DictReader(handle))
self.assertEqual([row["id"] for row in rows], ["TC-1", "TC-2"])
self.assertEqual(rows[0]["issue_type"], "product")
self.assertEqual(rows[0]["expected"], "ready")
self.assertEqual(rows[0]["artifact_hash"], "sha256:abc")
self.assertEqual(rows[1]["blocked_by"], "TC-1")
self.assertEqual(run["feedback"]["route"], "pr-update-issue-dispatcher-reassign")
if __name__ == "__main__":
unittest.main()