feat(tester): initialize agent target from Orca selectors
This commit is contained in:
@@ -37,6 +37,18 @@ EXPECTED_KINDS = (
|
||||
"url",
|
||||
"interaction",
|
||||
)
|
||||
AGENT_DEFAULTS = {
|
||||
"environment": "",
|
||||
"host": "local",
|
||||
"project": "",
|
||||
"path": "",
|
||||
"role": "test",
|
||||
"cli": "codex",
|
||||
"tier": "standard",
|
||||
"model": None,
|
||||
"reasoningEffort": "medium",
|
||||
"permissionMode": "workspace-write",
|
||||
}
|
||||
|
||||
|
||||
def catalog_dir(root: Path) -> Path:
|
||||
@@ -84,6 +96,41 @@ def load_local(root: Path) -> dict[str, Any]:
|
||||
return data
|
||||
|
||||
|
||||
def resolve_agent(root: Path, meta: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Resolve the structured agent target used by one tester run.
|
||||
|
||||
Stable defaults may live in catalog.yaml. Machine-specific values belong in
|
||||
local.yaml and override the mapping shallowly. The old execution mapping is
|
||||
accepted as a compatibility input and is projected onto the agent target.
|
||||
"""
|
||||
agent: dict[str, Any] = dict(AGENT_DEFAULTS)
|
||||
local = load_local(root)
|
||||
found_agent = False
|
||||
for source_name, source in (
|
||||
("catalog.agent", meta.get("agent")),
|
||||
("local.agent", local.get("agent")),
|
||||
):
|
||||
if source is None:
|
||||
continue
|
||||
found_agent = True
|
||||
if not isinstance(source, dict):
|
||||
raise SystemExit(f"{source_name} 必须是 mapping")
|
||||
agent.update(source)
|
||||
if not found_agent:
|
||||
for source_name, source in (
|
||||
("catalog.execution", meta.get("execution")),
|
||||
("local.execution", local.get("execution")),
|
||||
):
|
||||
if source is None:
|
||||
continue
|
||||
if not isinstance(source, dict):
|
||||
raise SystemExit(f"{source_name} 必须是 mapping")
|
||||
for key in ("environment", "host", "path"):
|
||||
if key in source:
|
||||
agent[key] = source[key]
|
||||
return agent
|
||||
|
||||
|
||||
def case_files(root: Path) -> list[Path]:
|
||||
d = catalog_dir(root) / "cases"
|
||||
if not d.is_dir():
|
||||
@@ -143,7 +190,7 @@ def resolve_vars(root: Path, meta: dict[str, Any]) -> dict[str, str]:
|
||||
merged[str(key)] = "" if value is None else str(value)
|
||||
local = load_local(root)
|
||||
for key, value in local.items():
|
||||
if value is None:
|
||||
if value is None or isinstance(value, (dict, list)):
|
||||
continue
|
||||
merged[str(key)] = str(value).lower() if isinstance(value, bool) else str(value)
|
||||
auth_keys = set()
|
||||
@@ -338,6 +385,44 @@ def probe_errors(where: str, probe: Any) -> list[str]:
|
||||
return errors
|
||||
|
||||
|
||||
def agent_errors(root: Path, meta: dict[str, Any]) -> list[str]:
|
||||
errors: list[str] = []
|
||||
try:
|
||||
agent = resolve_agent(root, meta)
|
||||
except SystemExit as exc:
|
||||
return [str(exc)]
|
||||
for key in (
|
||||
"environment",
|
||||
"host",
|
||||
"project",
|
||||
"path",
|
||||
"role",
|
||||
"cli",
|
||||
"tier",
|
||||
"model",
|
||||
"reasoningEffort",
|
||||
"permissionMode",
|
||||
):
|
||||
if key not in agent:
|
||||
errors.append(f"agent.{key} 缺失")
|
||||
for key in ("environment", "host", "project", "path", "role", "cli", "tier", "permissionMode"):
|
||||
if not isinstance(agent.get(key), str):
|
||||
errors.append(f"agent.{key} 必须是字符串")
|
||||
if agent.get("model") is not None and not isinstance(agent.get("model"), str):
|
||||
errors.append("agent.model 必须是字符串或 null")
|
||||
if agent.get("role") != "test":
|
||||
errors.append("agent.role 必须是 test")
|
||||
if agent.get("cli") not in ("codex", "cursor-agent", "grok", "omp", "pi"):
|
||||
errors.append("agent.cli 只能是 codex/cursor-agent/grok/omp/pi")
|
||||
if agent.get("tier") not in ("standard", "strong"):
|
||||
errors.append("agent.tier 只能是 standard 或 strong")
|
||||
if agent.get("reasoningEffort") not in (None, "low", "medium", "high", "xhigh"):
|
||||
errors.append("agent.reasoningEffort 只能是 low/medium/high/xhigh 或 null")
|
||||
if agent.get("permissionMode") not in ("read-only", "workspace-write"):
|
||||
errors.append("agent.permissionMode 只能是 read-only 或 workspace-write")
|
||||
return errors
|
||||
|
||||
|
||||
def run_together_groups(meta: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
raw = meta.get("run_together") or []
|
||||
if not isinstance(raw, list):
|
||||
@@ -505,6 +590,7 @@ def cmd_select(args: argparse.Namespace, root: Path) -> int:
|
||||
"skipped": skipped,
|
||||
"run_together": together,
|
||||
"status_probe": status_probes,
|
||||
"agent": resolve_agent(root, meta),
|
||||
"cases": selected,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
@@ -533,6 +619,7 @@ def validate(root: Path) -> list[str]:
|
||||
meta = load_catalog_meta(root)
|
||||
except SystemExit as exc:
|
||||
return [str(exc)]
|
||||
errors.extend(agent_errors(root, meta))
|
||||
|
||||
cats = meta.get("categories")
|
||||
if not isinstance(cats, list):
|
||||
|
||||
Reference in New Issue
Block a user