fix(tester): assume mobile capabilities in preflight

This commit is contained in:
2026-10-16 14:33:55 +08:00
parent 5b9a693fda
commit 3a757e3376
10 changed files with 35 additions and 217 deletions
+3 -4
View File
@@ -2,10 +2,9 @@
Reusable preflight for tester matrices. It supports read-only `verify` and
test-environment-only `prepare`, and writes per-check YAML/CSV evidence for
artifacts, tester nodes, devices, network, services, mobile album/photo/pinyin
capabilities, and fixture/cleanup contracts. Capability checks run on the actual
Android/iOS tester node; authorization environment variables are not accepted as
a substitute.
artifacts, tester nodes, devices, network, services, and fixture/cleanup contracts.
This project assumes mobile album/photo/IME capabilities; journey cases exercise them
and preflight does not gate them or read authorization environment variables.
Run the self-tests with:
+7 -9
View File
@@ -2,8 +2,8 @@
name: test-preflight
description: >-
Verify or prepare a mobile/integration test environment from a declared matrix:
artifacts, tester bindings, HOST/PUBLIC_HOST, service probes, mobile capability
probes, and fixture/cleanup contracts. Use before tester runs; it is not a deployment,
artifacts, tester bindings, HOST/PUBLIC_HOST, service probes, and fixture/cleanup
contracts. Use before tester runs; it is not a deployment,
build, or business-data skill.
---
@@ -41,19 +41,17 @@ Mac or that a deployment happened.
Overall status is `ready`, `blocked`, or `environment-fail`:
- `blocked` means a required declaration, artifact, binding, runner, capability, fixture,
- `blocked` means a required declaration, artifact, binding, runner, fixture,
cleanup contract, or authorization is absent/unsafe.
- `environment-fail` means a declared environment exists but DNS/TCP/HTTP,
device/toolchain, installation, launch, or deployment execution failed.
- `ready` means every recorded check passed. A ready preflight permits the
caller to continue with `integration`; this skill does not run test cases.
For every mobile matrix row, preflight requires real tester-node capability evidence for
`album_picker`, `photo_permission` (including a usable test image), and `ime_pinyin`
(composition and candidate selection). Android evidence must exercise the Android picker,
permission state and real Chinese IME composition; iOS evidence must exercise the corresponding
Photos picker/permission and text-input capabilities on the iOS node. `AUTHORIZE_ALBUM` and
`IME_PINYIN` are never accepted as substitutes for these probes.
This project assumes the client has the required album picker, photo permission, and Chinese
IME capabilities. Preflight does not probe or gate these capabilities, and does not read
`AUTHORIZE_ALBUM` or `IME_PINYIN`. Their real behavior is exercised by journey cases; a failure
there is a product test failure, not an environment block.
Do not treat a preflight report as a product result. A preflight failure means
no client session may be created; the tester owns `not-run` propagation and must write
+7 -30
View File
@@ -18,10 +18,9 @@ fixtures:
cleanup:
required: true
steps: [delete-run-data]
# Optional project metadata. It is not a preflight gate.
capabilities:
# These baseline checks are mandatory for mobile rows and cannot be disabled
# with AUTHORIZE_ALBUM or IME_PINYIN.
required: [album_picker, photo_permission, ime_pinyin]
assumed: [album_picker, photo_permission, ime_pinyin]
matrix:
- platform: android
tester:
@@ -52,34 +51,12 @@ needs non-empty `references`; a required cleanup needs non-empty `steps` or
`run`. For a read-only run, use `required: false` with a non-empty `reason`.
The skill never executes these values.
## Mobile capability contract
## Client capabilities
Every Android/iOS matrix row checks these capabilities on the actual tester node:
| capability | required evidence |
|---|---|
| `album_picker` | `checks.picker`: the real platform album/Photos picker opened and returned a selectable asset |
| `photo_permission` | `checks.permission` and `checks.test_image`: permission state was exercised and the declared test image was available |
| `ime_pinyin` | `checks.composition` and `checks.candidates`: real Chinese pinyin composition and candidate selection succeeded |
The default list is always applied; `capabilities.required` can add project-specific
checks but cannot remove these three. A contract runner receives one request per
capability:
```json
{
"operation": "capability_probe",
"platform": "android",
"node": "tester.android",
"capability": "ime_pinyin",
"required_checks": ["composition", "candidates"]
}
```
It must return every requested check with `status`, `observed`, and (when useful)
`expected`/`remediation`. Missing evidence is `blocked`; a real probe failure is
`environment-fail`. Environment variables such as `AUTHORIZE_ALBUM` and
`IME_PINYIN` are not capability evidence.
The project assumes the mobile test nodes provide the album picker, photo permission,
test image, and Chinese IME. These are not preflight declarations or gates. Journey
cases exercise the real UI; failures are recorded as product failures. `AUTHORIZE_ALBUM`
and `IME_PINYIN` are not read.
## Tester-node contract
+4 -158
View File
@@ -31,17 +31,6 @@ except ImportError as exc: # pragma: no cover
PLATFORMS = {"android", "ios"}
STATUSES = {"ready", "blocked", "environment-fail"}
CAPABILITY_CHECKS = {
"album_picker": ("picker",),
"photo_permission": ("permission", "test_image"),
"ime_pinyin": ("composition", "candidates"),
}
CAPABILITY_EXPECTED = {
"album_picker": "real platform album picker opens and returns a selectable asset",
"photo_permission": "photo permission flow is observable and the declared test image is available",
"ime_pinyin": "real Chinese pinyin composition produces and selects candidate words",
}
DEFAULT_CAPABILITIES = tuple(CAPABILITY_CHECKS)
EVIDENCE_FIELDS = [
"check_id", "mode", "platform", "node", "target", "observed", "expected",
"status", "remediation", "timestamp",
@@ -172,27 +161,6 @@ def validate_shape(config: Mapping[str, Any], evidence: Evidence) -> bool:
if not isinstance(config.get("host"), str) or not config["host"].strip():
evidence.add("config-host", target="HOST", expected="non-empty host URL", status="blocked", remediation="提供 HOST")
ok = False
capabilities = config.get("capabilities")
if capabilities is not None and not isinstance(capabilities, (list, dict)):
evidence.add(
"config-capabilities",
target="capabilities",
observed=capabilities,
expected="list or mapping with required list",
status="blocked",
remediation="声明 capabilities.required;不能用 AUTHORIZE_ALBUM/IME_PINYIN 代替能力检查",
)
ok = False
elif isinstance(capabilities, dict) and capabilities.get("required") is not None and not isinstance(capabilities.get("required"), list):
evidence.add(
"config-capabilities-required",
target="capabilities.required",
observed=capabilities.get("required"),
expected="list of capability ids",
status="blocked",
remediation="将 capabilities.required 写成列表",
)
ok = False
probes = config.get("service_probes")
if not isinstance(probes, list) or not probes:
evidence.add("config-service-probes", target="service_probes", expected="at least one probe", status="blocked", remediation="声明服务 URL 探针")
@@ -454,122 +422,6 @@ def check_fixture_cleanup(evidence: Evidence, config: Mapping[str, Any], platfor
return ok
def capability_ids(config: Mapping[str, Any]) -> list[str]:
"""Return mandatory mobile capabilities.
The three baseline capabilities are never disabled by an authorization
environment variable. A config may add more checks, but it cannot remove
the album/photo/pinyin checks required for real client journeys.
"""
raw = config.get("capabilities")
requested: Any = raw.get("required") if isinstance(raw, dict) else raw
ids = list(DEFAULT_CAPABILITIES)
if isinstance(requested, list):
for item in requested:
name = text(item).strip()
if name and name not in ids:
ids.append(name)
return ids
def check_capabilities(
evidence: Evidence,
config: Mapping[str, Any],
row: Mapping[str, Any],
platform_name: str,
node: str,
index: int,
*,
platform_ok: bool,
) -> bool:
"""Run real capability probes on the resolved tester node.
A contract runner must return each required sub-check. We intentionally do
not infer album, permission, image, or IME behavior from adb/xcrun alone:
the adapter must exercise the real picker/permission/UI composition path.
"""
tester = row.get("tester") or {}
runner = tester.get("runner") if isinstance(tester, dict) else None
is_local = node in {"local", "localhost", "127.0.0.1"}
capabilities_ok = True
for capability in capability_ids(config):
required = CAPABILITY_CHECKS.get(capability, ("capability",))
prefix = f"{platform_name}-{index}-capability-{capability}"
if not platform_ok:
for check in required:
evidence.add(
f"{prefix}-{check}",
platform=platform_name,
node=node,
target=capability,
observed="platform prerequisite failed",
expected=CAPABILITY_EXPECTED.get(capability, capability),
status="environment-fail",
remediation="先修复平台/设备闸门,再由 tester 节点重跑真实能力探针",
)
capabilities_ok = False
continue
if not (isinstance(runner, dict) and runner.get("kind") == "contract"):
for check in required:
evidence.add(
f"{prefix}-{check}",
platform=platform_name,
node=node,
target=capability,
observed="no real capability runner",
expected=CAPABILITY_EXPECTED.get(capability, capability),
status="blocked",
remediation="配置 tester 节点的 contract runner,真实打开 picker/权限 UI 并完成拼音 composition/candidate 检查",
)
capabilities_ok = False
continue
status, observed, result = runner_call(
runner,
{
"operation": "capability_probe",
"platform": platform_name,
"node": node,
"capability": capability,
"required_checks": list(required),
"artifact": (row.get("artifact") or {}).get("path"),
},
)
checks = result.get("checks") if isinstance(result, dict) else None
if not isinstance(checks, dict):
checks = {}
for check in required:
detail = checks.get(check)
if detail is None:
evidence.add(
f"{prefix}-{check}",
platform=platform_name,
node=node,
target=capability,
observed=observed or "runner omitted required evidence",
expected=CAPABILITY_EXPECTED.get(capability, capability),
status="blocked",
remediation=f"runner 必须返回 checks.{check},且证据来自真实 {platform_name} 能力操作",
)
capabilities_ok = False
continue
detail = detail if isinstance(detail, dict) else {"observed": detail}
detail_status = status_from(detail.get("status"), status)
evidence.add(
f"{prefix}-{check}",
platform=platform_name,
node=node,
target=text(detail.get("target", capability)),
observed=detail.get("observed", observed),
expected=detail.get("expected", CAPABILITY_EXPECTED.get(capability, capability)),
status=detail_status,
remediation=text(detail.get("remediation", "")),
)
if detail_status != "ready":
capabilities_ok = False
return capabilities_ok
def run(config: Mapping[str, Any], mode: str) -> Evidence:
evidence = Evidence(mode)
shape_ok = validate_shape(config, evidence)
@@ -627,16 +479,10 @@ def run(config: Mapping[str, Any], mode: str) -> Evidence:
evidence.add(f"{platform_name}-{index}-platform-runner", platform=platform_name, node=node, target=node, expected="node runner contract", status="blocked", remediation="远端 tester 必须由 agent-runtime 提供可执行 runner")
platform_ok = False
network_ok = check_network(evidence, config, row, platform_name, node, index)
capabilities_ok = check_capabilities(
evidence,
config,
row,
platform_name,
node,
index,
platform_ok=platform_ok,
)
row_ok = artifact_ok and fixture_ok and platform_ok and network_ok and capabilities_ok
# This isolated project assumes the client capabilities used by its
# journeys. Their real behavior is a product assertion, not a preflight
# environment gate.
row_ok = artifact_ok and fixture_ok and platform_ok and network_ok
if mode == "prepare":
artifact = row.get("artifact") or {}
identity = artifact.get("package" if platform_name == "android" else "bundle_id")
@@ -140,7 +140,7 @@ class PreflightTests(unittest.TestCase):
self.assertTrue(any(row["check_id"].endswith("-fixture") and row["status"] == "blocked" for row in result.rows))
self.assertTrue(any(row["check_id"].endswith("-cleanup") and row["status"] == "blocked" for row in result.rows))
def test_mobile_capabilities_are_mandatory_and_missing_evidence_blocks(self):
def test_mobile_capabilities_are_assumed_and_not_a_preflight_gate(self):
data = self.config()
config_path = self.write_config(data)
env = os.environ.copy()
@@ -151,12 +151,9 @@ class PreflightTests(unittest.TestCase):
"--config", str(config_path), "--yaml", str(output), "--csv", str(self.root / "capabilities.csv"),
], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
report = __import__("yaml").safe_load(output.read_text(encoding="utf-8"))
self.assertEqual(process.returncode, 1)
self.assertEqual(report["status"], "blocked")
capability_rows = [row for row in report["checks"] if "capability-album_picker" in row["check_id"]]
self.assertTrue(capability_rows)
self.assertTrue(all(row["status"] == "blocked" for row in capability_rows))
self.assertTrue(any("photo_permission" in row["check_id"] for row in report["checks"]))
self.assertEqual(process.returncode, 0)
self.assertEqual(report["status"], "ready")
self.assertFalse(any("capability-" in row["check_id"] for row in report["checks"]))
def test_host_unreachable_from_actual_runner_is_environment_fail(self):
data = self.config()
@@ -221,6 +218,7 @@ class PreflightTests(unittest.TestCase):
self.assertEqual(result.overall(), "ready")
self.assertNotIn("install", calls)
self.assertNotIn("start", calls)
self.assertNotIn("capability_probe", calls)
if __name__ == "__main__":