feat(tester): initialize agent target from Orca selectors

This commit is contained in:
2026-10-12 01:14:52 +08:00
parent 5ba1d8e3d6
commit a35cc916cb
9 changed files with 688 additions and 10 deletions
+88 -1
View File
@@ -37,6 +37,18 @@ EXPECTED_KINDS = (
"url",
"interaction",
)
AGENT_DEFAULTS = {
"environment": "",
"host": "local",
"project": "",
"path": "",
"role": "test",
"cli": "codex",
"tier": "standard",
"model": None,
"reasoningEffort": "medium",
"permissionMode": "workspace-write",
}
def catalog_dir(root: Path) -> Path:
@@ -84,6 +96,41 @@ def load_local(root: Path) -> dict[str, Any]:
return data
def resolve_agent(root: Path, meta: dict[str, Any]) -> dict[str, Any]:
"""Resolve the structured agent target used by one tester run.
Stable defaults may live in catalog.yaml. Machine-specific values belong in
local.yaml and override the mapping shallowly. The old execution mapping is
accepted as a compatibility input and is projected onto the agent target.
"""
agent: dict[str, Any] = dict(AGENT_DEFAULTS)
local = load_local(root)
found_agent = False
for source_name, source in (
("catalog.agent", meta.get("agent")),
("local.agent", local.get("agent")),
):
if source is None:
continue
found_agent = True
if not isinstance(source, dict):
raise SystemExit(f"{source_name} 必须是 mapping")
agent.update(source)
if not found_agent:
for source_name, source in (
("catalog.execution", meta.get("execution")),
("local.execution", local.get("execution")),
):
if source is None:
continue
if not isinstance(source, dict):
raise SystemExit(f"{source_name} 必须是 mapping")
for key in ("environment", "host", "path"):
if key in source:
agent[key] = source[key]
return agent
def case_files(root: Path) -> list[Path]:
d = catalog_dir(root) / "cases"
if not d.is_dir():
@@ -143,7 +190,7 @@ def resolve_vars(root: Path, meta: dict[str, Any]) -> dict[str, str]:
merged[str(key)] = "" if value is None else str(value)
local = load_local(root)
for key, value in local.items():
if value is None:
if value is None or isinstance(value, (dict, list)):
continue
merged[str(key)] = str(value).lower() if isinstance(value, bool) else str(value)
auth_keys = set()
@@ -338,6 +385,44 @@ def probe_errors(where: str, probe: Any) -> list[str]:
return errors
def agent_errors(root: Path, meta: dict[str, Any]) -> list[str]:
errors: list[str] = []
try:
agent = resolve_agent(root, meta)
except SystemExit as exc:
return [str(exc)]
for key in (
"environment",
"host",
"project",
"path",
"role",
"cli",
"tier",
"model",
"reasoningEffort",
"permissionMode",
):
if key not in agent:
errors.append(f"agent.{key} 缺失")
for key in ("environment", "host", "project", "path", "role", "cli", "tier", "permissionMode"):
if not isinstance(agent.get(key), str):
errors.append(f"agent.{key} 必须是字符串")
if agent.get("model") is not None and not isinstance(agent.get("model"), str):
errors.append("agent.model 必须是字符串或 null")
if agent.get("role") != "test":
errors.append("agent.role 必须是 test")
if agent.get("cli") not in ("codex", "cursor-agent", "grok", "omp", "pi"):
errors.append("agent.cli 只能是 codex/cursor-agent/grok/omp/pi")
if agent.get("tier") not in ("standard", "strong"):
errors.append("agent.tier 只能是 standard 或 strong")
if agent.get("reasoningEffort") not in (None, "low", "medium", "high", "xhigh"):
errors.append("agent.reasoningEffort 只能是 low/medium/high/xhigh 或 null")
if agent.get("permissionMode") not in ("read-only", "workspace-write"):
errors.append("agent.permissionMode 只能是 read-only 或 workspace-write")
return errors
def run_together_groups(meta: dict[str, Any]) -> list[dict[str, Any]]:
raw = meta.get("run_together") or []
if not isinstance(raw, list):
@@ -505,6 +590,7 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
"skipped": skipped,
"run_together": together,
"status_probe": status_probes,
"agent": resolve_agent(root, meta),
"cases": selected,
},
ensure_ascii=False,
@@ -533,6 +619,7 @@ def validate(root: Path) -> list[str]:
meta = load_catalog_meta(root)
except SystemExit as exc:
return [str(exc)]
errors.extend(agent_errors(root, meta))
cats = meta.get("categories")
if not isinstance(cats, list):