feat(orc): add tiered engineering orchestration
This commit is contained in:
@@ -0,0 +1,405 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
ORC_DIR = REPO_ROOT / "skills" / "orc"
|
||||
SCRIPT = ORC_DIR / "scripts" / "resolve_profile.py"
|
||||
CONFIG = ORC_DIR / "templates" / "config.template.yaml"
|
||||
|
||||
spec = importlib.util.spec_from_file_location("orc_resolve_profile", SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
orc_profiles = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(orc_profiles)
|
||||
|
||||
|
||||
class OrcSkillTests(unittest.TestCase):
|
||||
def test_orc_is_explicit_and_routes_to_independent_skills(self) -> None:
|
||||
skill = (ORC_DIR / "SKILL.md").read_text(encoding="utf-8")
|
||||
routing = (ORC_DIR / "references" / "routing.md").read_text(encoding="utf-8")
|
||||
metadata = (ORC_DIR / "agents" / "openai.yaml").read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("allow_implicit_invocation: false", metadata)
|
||||
self.assertIn("low", skill)
|
||||
self.assertIn("mid", skill)
|
||||
self.assertIn("high", skill)
|
||||
self.assertIn("只支持 `cli: codex`", skill)
|
||||
for child in ("$ack", "$manage-release", "$deb-publisher", "$publish-docker-image"):
|
||||
self.assertIn(child, routing)
|
||||
|
||||
def test_children_do_not_reference_orc(self) -> None:
|
||||
for child in ("ack", "manage-release", "deb-publisher", "publish-docker-image"):
|
||||
for path in (REPO_ROOT / "skills" / child).rglob("*"):
|
||||
if not path.is_file() or "__pycache__" in path.parts:
|
||||
continue
|
||||
content = path.read_text(encoding="utf-8", errors="ignore")
|
||||
self.assertNotIn("$orc", content, str(path))
|
||||
self.assertNotRegex(
|
||||
content,
|
||||
r"(?<![A-Za-z0-9_-])/orc(?![A-Za-z0-9_-])",
|
||||
str(path),
|
||||
)
|
||||
self.assertNotRegex(content, r"\bORC\b", str(path))
|
||||
self.assertNotIn("skills/orc", content, str(path))
|
||||
|
||||
def test_routing_pins_normal_pr_and_unsupported_release_gate(self) -> None:
|
||||
routing = (ORC_DIR / "references" / "routing.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
self.assertIn("普通非发布 PR/MR", routing)
|
||||
self.assertIn("最多到", routing)
|
||||
self.assertIn("review_ready", routing)
|
||||
self.assertIn("当前 task 粒度无法安全表达", routing)
|
||||
|
||||
def test_template_and_cli_validate(self) -> None:
|
||||
config = orc_profiles.load_config(CONFIG)
|
||||
self.assertEqual(set(config["profiles"]), {"low", "mid", "high"})
|
||||
completed = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "validate", str(CONFIG)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(completed.returncode, 0, completed.stderr)
|
||||
self.assertIn('"ok": true', completed.stdout)
|
||||
|
||||
clean_python = subprocess.run(
|
||||
[sys.executable, "-I", "-S", str(SCRIPT), "validate", str(CONFIG)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(clean_python.returncode, 0, clean_python.stderr)
|
||||
|
||||
def test_level_precedence_is_stage_then_global_then_config(self) -> None:
|
||||
config = orc_profiles.load_config(CONFIG)
|
||||
stage = orc_profiles.resolve_profile(
|
||||
config, stage="docker", global_level="mid", stage_level="high"
|
||||
)
|
||||
global_choice = orc_profiles.resolve_profile(
|
||||
config, stage="docker", global_level="mid"
|
||||
)
|
||||
configured = orc_profiles.resolve_profile(config, stage="docker")
|
||||
fallback_config = copy.deepcopy(config)
|
||||
del fallback_config["stageDefaults"]["docker"]
|
||||
fallback = orc_profiles.resolve_profile(fallback_config, stage="docker")
|
||||
|
||||
self.assertEqual((stage["level"], stage["selectionSource"]), ("high", "request.stage"))
|
||||
self.assertEqual((global_choice["level"], global_choice["selectionSource"]), ("mid", "request.global"))
|
||||
self.assertEqual((configured["level"], configured["selectionSource"]), ("low", "config.stageDefaults.docker"))
|
||||
self.assertEqual((fallback["level"], fallback["selectionSource"]), ("mid", "config.defaultLevel"))
|
||||
|
||||
def test_resolver_builds_fixed_worker_args_without_free_command_fields(self) -> None:
|
||||
config = orc_profiles.load_config(CONFIG)
|
||||
plan = orc_profiles.resolve_profile(config, stage="release")
|
||||
|
||||
self.assertEqual(plan["workerArgs"][:2], ["--model", "gpt-5.6-terra"])
|
||||
self.assertIn("--strict-config", plan["workerArgs"])
|
||||
self.assertNotIn("danger-full-access", " ".join(plan["workerArgs"]))
|
||||
self.assertNotIn("env", plan["profile"])
|
||||
self.assertNotIn("command", plan["profile"])
|
||||
|
||||
def test_worker_environment_selects_one_model_and_remote_auth(self) -> None:
|
||||
ambient = {
|
||||
"OPENAI_API_KEY": "openai-secret",
|
||||
"AZURE_OPENAI_API_KEY": "azure-secret",
|
||||
"GITHUB_TOKEN": "github-secret",
|
||||
"GITLAB_TOKEN": "gitlab-secret",
|
||||
"GITEA_TOKEN": "gitea-secret",
|
||||
"FORGEJO_TOKEN": "forgejo-secret",
|
||||
"DEB_TOKEN": "deb-secret",
|
||||
"SSH_AUTH_SOCK": "/tmp/agent.sock",
|
||||
"HTTPS_PROXY": "https://user:secret@proxy.example",
|
||||
}
|
||||
with mock.patch.dict(os.environ, ambient, clear=True):
|
||||
environment = orc_profiles.worker_environment(
|
||||
"release",
|
||||
model_auth="openai",
|
||||
remote_auth="forgejo-token",
|
||||
)
|
||||
|
||||
self.assertEqual(environment["OPENAI_API_KEY"], "openai-secret")
|
||||
self.assertEqual(environment["FORGEJO_TOKEN"], "forgejo-secret")
|
||||
for rejected in (
|
||||
"AZURE_OPENAI_API_KEY",
|
||||
"GITHUB_TOKEN",
|
||||
"GITLAB_TOKEN",
|
||||
"GITEA_TOKEN",
|
||||
"DEB_TOKEN",
|
||||
"SSH_AUTH_SOCK",
|
||||
"HTTPS_PROXY",
|
||||
):
|
||||
self.assertNotIn(rejected, environment)
|
||||
|
||||
with self.assertRaisesRegex(orc_profiles.ConfigError, "deb remote auth"):
|
||||
orc_profiles.worker_environment(
|
||||
"deb",
|
||||
model_auth="codex-login",
|
||||
remote_auth="forgejo-token",
|
||||
)
|
||||
|
||||
with mock.patch.dict(os.environ, {}, clear=True):
|
||||
with self.assertRaisesRegex(
|
||||
orc_profiles.ConfigError,
|
||||
"selected authentication variable is unavailable",
|
||||
):
|
||||
orc_profiles.worker_environment(
|
||||
"release",
|
||||
model_auth="openai",
|
||||
remote_auth="none",
|
||||
)
|
||||
|
||||
with mock.patch.dict(os.environ, {"OPENAI_API_KEY": ""}, clear=True):
|
||||
with self.assertRaisesRegex(
|
||||
orc_profiles.ConfigError,
|
||||
"selected authentication variable is unavailable",
|
||||
):
|
||||
orc_profiles.worker_environment(
|
||||
"release",
|
||||
model_auth="openai",
|
||||
remote_auth="none",
|
||||
)
|
||||
|
||||
def test_ssh_auth_requires_a_trusted_user_socket(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
socket_path = root / "agent.sock"
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as agent:
|
||||
agent.bind(str(socket_path))
|
||||
socket_path.chmod(0o600)
|
||||
with mock.patch.dict(
|
||||
os.environ,
|
||||
{"SSH_AUTH_SOCK": str(socket_path)},
|
||||
clear=True,
|
||||
):
|
||||
environment = orc_profiles.worker_environment(
|
||||
"release",
|
||||
model_auth="codex-login",
|
||||
remote_auth="ssh-agent",
|
||||
)
|
||||
self.assertEqual(environment["SSH_AUTH_SOCK"], str(socket_path))
|
||||
|
||||
regular_file = root / "not-a-socket"
|
||||
regular_file.write_text("not a socket", encoding="utf-8")
|
||||
regular_file.chmod(0o600)
|
||||
with mock.patch.dict(
|
||||
os.environ,
|
||||
{"SSH_AUTH_SOCK": str(regular_file)},
|
||||
clear=True,
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
orc_profiles.ConfigError,
|
||||
"not a trusted user socket",
|
||||
):
|
||||
orc_profiles.worker_environment(
|
||||
"release",
|
||||
model_auth="codex-login",
|
||||
remote_auth="ssh-agent",
|
||||
)
|
||||
|
||||
def test_trusted_orca_ignores_ambient_path(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
trusted = root / "trusted"
|
||||
hostile = root / "hostile"
|
||||
trusted.mkdir()
|
||||
hostile.mkdir()
|
||||
trusted_orca = trusted / "orca"
|
||||
hostile_orca = hostile / "orca"
|
||||
for executable in (trusted_orca, hostile_orca):
|
||||
executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
||||
executable.chmod(0o700)
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
orc_profiles,
|
||||
"trusted_path_entries",
|
||||
return_value=[trusted],
|
||||
),
|
||||
mock.patch.dict(os.environ, {"PATH": str(hostile)}, clear=True),
|
||||
):
|
||||
resolved = orc_profiles.resolve_trusted_executable("orca")
|
||||
|
||||
self.assertEqual(resolved, trusted_orca)
|
||||
|
||||
def test_config_rejects_unsafe_or_ambiguous_profiles(self) -> None:
|
||||
base = orc_profiles.load_config(CONFIG)
|
||||
|
||||
free_command = copy.deepcopy(base)
|
||||
free_command["profiles"]["low"]["command"] = "codex --dangerously-bypass"
|
||||
with self.assertRaisesRegex(orc_profiles.ConfigError, "unknown fields"):
|
||||
orc_profiles.validate_config(free_command)
|
||||
|
||||
full_access = copy.deepcopy(base)
|
||||
full_access["profiles"]["high"]["permissionMode"] = "danger-full-access"
|
||||
with self.assertRaisesRegex(orc_profiles.ConfigError, "workspace-write in ORC v1"):
|
||||
orc_profiles.validate_config(full_access)
|
||||
|
||||
read_only = copy.deepcopy(base)
|
||||
read_only["profiles"]["low"]["permissionMode"] = "read-only"
|
||||
with self.assertRaisesRegex(orc_profiles.ConfigError, "workspace-write in ORC v1"):
|
||||
orc_profiles.validate_config(read_only)
|
||||
|
||||
missing_level = copy.deepcopy(base)
|
||||
del missing_level["profiles"]["mid"]
|
||||
with self.assertRaisesRegex(orc_profiles.ConfigError, "missing fields: mid"):
|
||||
orc_profiles.validate_config(missing_level)
|
||||
|
||||
option_model = copy.deepcopy(base)
|
||||
option_model["profiles"]["low"]["model"] = "--model"
|
||||
with self.assertRaisesRegex(orc_profiles.ConfigError, "safe exact model ID"):
|
||||
orc_profiles.validate_config(option_model)
|
||||
|
||||
list_policy = copy.deepcopy(base)
|
||||
list_policy["profiles"]["low"]["approvalPolicy"] = ["never"]
|
||||
with self.assertRaisesRegex(orc_profiles.ConfigError, "approvalPolicy"):
|
||||
orc_profiles.validate_config(list_policy)
|
||||
|
||||
def test_config_reader_rejects_symlinks_and_redacts_parser_input(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
real = root / "real.yaml"
|
||||
real.write_text(CONFIG.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
linked = root / "linked.yaml"
|
||||
linked.symlink_to(real)
|
||||
with self.assertRaisesRegex(orc_profiles.ConfigError, "safely read"):
|
||||
orc_profiles.load_config(linked)
|
||||
|
||||
malformed = root / "malformed.yaml"
|
||||
secret_marker = "PRIVATE_MATERIAL_MUST_NOT_APPEAR"
|
||||
malformed.write_text(f"{secret_marker}: [unterminated\n", encoding="utf-8")
|
||||
completed = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "validate", str(malformed)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(completed.returncode, 1)
|
||||
self.assertNotIn(secret_marker, completed.stderr)
|
||||
|
||||
def test_bound_plan_enforces_registered_allowlisted_worktree(self) -> None:
|
||||
git = shutil.which("git")
|
||||
self.assertIsNotNone(git)
|
||||
assert git is not None
|
||||
git_path = Path(git).resolve()
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="orc shell $(id) ' ") as temporary:
|
||||
temporary_root = Path(temporary).resolve()
|
||||
root = temporary_root / "project"
|
||||
root.mkdir()
|
||||
subprocess.run(
|
||||
[str(git_path), "init", "-q", str(root)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
config_path = root / "docs" / "orc" / "config.yaml"
|
||||
config_path.parent.mkdir(parents=True)
|
||||
config_path.write_text(CONFIG.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
fake_codex = root / "trusted-codex" / "codex"
|
||||
fake_codex.parent.mkdir()
|
||||
fake_codex.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
||||
fake_codex.chmod(0o700)
|
||||
fake_orca = root / "trusted-orca" / "orca"
|
||||
fake_orca.parent.mkdir()
|
||||
fake_orca.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
||||
fake_orca.chmod(0o700)
|
||||
|
||||
def executable(name: str) -> Path:
|
||||
return {
|
||||
"git": git_path,
|
||||
"codex": fake_codex,
|
||||
"orca": fake_orca,
|
||||
}[name]
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
orc_profiles,
|
||||
"resolve_trusted_executable",
|
||||
side_effect=executable,
|
||||
),
|
||||
mock.patch.object(
|
||||
orc_profiles,
|
||||
"resolve_trusted_python",
|
||||
return_value=Path(sys.executable).resolve(),
|
||||
),
|
||||
mock.patch.object(
|
||||
orc_profiles,
|
||||
"_executable_facts",
|
||||
return_value={
|
||||
"path": str(fake_codex),
|
||||
"device": 1,
|
||||
"inode": 2,
|
||||
"size": 3,
|
||||
"mtimeNs": 4,
|
||||
"version": "codex-test 1",
|
||||
},
|
||||
),
|
||||
):
|
||||
plan = orc_profiles.build_launch_plan(
|
||||
config_path,
|
||||
project_root=root,
|
||||
worktree=root,
|
||||
stage="code",
|
||||
stage_level="high",
|
||||
)
|
||||
|
||||
self.assertEqual(plan["argv"][0], str(fake_codex))
|
||||
self.assertEqual(plan["terminalCreateArgv"][0], str(fake_orca))
|
||||
self.assertEqual(plan["worktree"], str(root))
|
||||
self.assertEqual(plan["selectionSource"], "request.stage")
|
||||
self.assertTrue(plan["launchFingerprint"].startswith("sha256:"))
|
||||
self.assertIn("_launch", plan["launcherArgv"])
|
||||
self.assertEqual(plan["launcherArgv"][1:3], ["-I", "-S"])
|
||||
self.assertIn("python", plan)
|
||||
self.assertEqual(plan["worktreeSelector"], f"path:{root}")
|
||||
self.assertNotIn("PRIVATE", json.dumps(plan))
|
||||
self.assertEqual(shlex.split(plan["terminalCommand"]), plan["launcherArgv"])
|
||||
self.assertEqual(
|
||||
shlex.split(plan["terminalCreateShellCommand"]),
|
||||
plan["terminalCreateArgv"],
|
||||
)
|
||||
command_index = plan["terminalCreateArgv"].index("--command")
|
||||
self.assertEqual(
|
||||
plan["terminalCreateArgv"][command_index + 1],
|
||||
plan["terminalCommand"],
|
||||
)
|
||||
|
||||
outside = temporary_root / "outside"
|
||||
outside.mkdir()
|
||||
subprocess.run(
|
||||
[str(git_path), "init", "-q", str(outside)],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
with mock.patch.object(
|
||||
orc_profiles,
|
||||
"resolve_trusted_executable",
|
||||
side_effect=executable,
|
||||
):
|
||||
with self.assertRaisesRegex(
|
||||
orc_profiles.ConfigError,
|
||||
"not in allowedWorktrees",
|
||||
):
|
||||
orc_profiles.validate_worktree(
|
||||
orc_profiles.load_config(config_path),
|
||||
project_root_value=root,
|
||||
worktree_value=outside,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user