fix(ack): bound worker polling and detect stale heartbeats

This commit is contained in:
2026-09-19 19:30:20 +08:00
parent 2917708b10
commit 94c66bc695
5 changed files with 132 additions and 39 deletions
+1 -1
View File
@@ -84,7 +84,7 @@ python3 <ack-skill-dir>/scripts/validate_agents.py .pouch/ack/agents.yaml
2. 读 `project.md`;若存在 `agents.yaml`,先用 `validate_agents.py` 校验并读取其 `defaults`;用 `select_tasks.py`(已知任务加 `--task-id`)和 `select_knowledge.py` 取当前任务相关 `active` 条目。超预算时缩小选择,不回退为完整 yaml。 2. 读 `project.md`;若存在 `agents.yaml`,先用 `validate_agents.py` 校验并读取其 `defaults`;用 `select_tasks.py`(已知任务加 `--task-id`)和 `select_knowledge.py` 取当前任务相关 `active` 条目。超预算时缩小选择,不回退为完整 yaml。
3. 新需求或尚未确认的计划:读 [kickoff.md](references/kickoff.md)。修 bug 写短问题说明、复现和可观测验收,不写大 PRD。 3. 新需求或尚未确认的计划:读 [kickoff.md](references/kickoff.md)。修 bug 写短问题说明、复现和可观测验收,不写大 PRD。
4. 用户已确认后,先读取并按 [closed-loop.md](references/closed-loop.md) 执行;使用 Orca 时同时读取 [orca-adapter.md](references/orca-adapter.md)。当前会话担任 Coordinator,不亲自写代码或跑测试;worker profile 按 `agents.yaml`(若存在)否则按任务板配置选择。 4. 用户已确认后,先读取并按 [closed-loop.md](references/closed-loop.md) 执行;使用 Orca 时同时读取 [orca-adapter.md](references/orca-adapter.md)。当前会话担任 Coordinator,不亲自写代码或跑测试;worker profile 按 `agents.yaml`(若存在)否则按任务板配置选择。
这不是 fire-and-forget:Coordinator 不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话。每个任务都必须在同一轮持续执行 `check --wait` 与 `worker_probe.py`,直到 Developer 回报 `worker_done` 或 `escalation`;随后写回 `fixed_by_dev`,主动派发 Test,并继续等待 `retest_result`。只有完成 Test 终检、重试/`leftover` 决策、回归决定和 worker 回收后,才可以结束本轮 ACK。 这不是 fire-and-forget:Coordinator 不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话。每个任务都必须在同一轮持续执行默认 90 秒窗口的 `check --wait`,并按 60–120 秒节奏运行 `worker_probe.py`,直到 Developer 回报 `worker_done` 或 `escalation`;随后写回 `fixed_by_dev`,主动派发 Test,并继续等待 `retest_result`。只有完成 Test 终检、重试/`leftover` 决策、回归决定和 worker 回收后,才可以结束本轮 ACK。
5. 存在 `project.bugIntake` 时先完成「飞书收件」。创建或更换 worker 走「启动 worker」。若 `intents.testEnvironment` 已启用,派 Test 前先走「运行测试环境」。 5. 存在 `project.bugIntake` 时先完成「飞书收件」。创建或更换 worker 走「启动 worker」。若 `intents.testEnvironment` 已启用,派 Test 前先走「运行测试环境」。
6. 知识 `verification.ref` 只经 `run_verification.py`。不把 `worker_done` 或 Test 自报成功当作完成。环境失败先有界恢复并报告 `userAction`;三轮产品失败记 `leftover`。 6. 知识 `verification.ref` 只经 `run_verification.py`。不把 `worker_done` 或 Test 自报成功当作完成。环境失败先有界恢复并报告 `userAction`;三轮产品失败记 `leftover`。
7. 任务 `verified` 且改了可见行为或 API 后,转入「运行回归」收获用例。若本次计划含交付,再走对应交付模式;默认最多到 `validation_ready` 或 `review_ready`。 7. 任务 `verified` 且改了可见行为或 API 后,转入「运行回归」收获用例。若本次计划含交付,再走对应交付模式;默认最多到 `validation_ready` 或 `review_ready`。
+5 -3
View File
@@ -215,7 +215,7 @@ EOF
且终端进入工作指示(Working / Running)。 且终端进入工作指示(Working / Running)。
- 手动投递路径:`orca terminal send` 必须带 `--enter`;投递后同样 read 确认。 - 手动投递路径:`orca terminal send` 必须带 `--enter`;投递后同样 read 确认。
- 确认失败或终端仍停在欢迎提示:按「消息未投递」记录环境失败,不消耗产品轮次。 - 确认失败或终端仍停在欢迎提示:按「消息未投递」记录环境失败,不消耗产品轮次。
2. 等待期间滚动 probe(每 60–120 秒一次): 2. 等待期间滚动 probe(每 60–120 秒一次;不要在 30 秒窗口后每次都 probe):
```bash ```bash
python3 <ack-skill-dir>/scripts/worker_probe.py \ python3 <ack-skill-dir>/scripts/worker_probe.py \
@@ -229,8 +229,10 @@ EOF
- `not-started`:检查是否漏投递或未回车;重新投递或记录环境失败,不占轮次。 - `not-started`:检查是否漏投递或未回车;重新投递或记录环境失败,不占轮次。
- `running` / `progress`:继续滚动 wait。 - `running` / `progress`:继续滚动 wait。
- `unknown`:按 `dispatch-show` 与 Orca live state 人工核对,不自动重试。 - `unknown`:按 `dispatch-show` 与 Orca live state 人工核对,不自动重试。
4. `check --wait` 使用短窗口(60–90 秒)而不是 15 分钟:窗口超时是检查点,先 probe 4. `check --wait` 默认使用 90 秒窗口而不是 30 秒或 15 分钟:窗口超时是检查点,
再决定继续等待、恢复或上报。 仅在达到 probe 间隔后再探测,然后决定继续等待、恢复或上报。新鲜 heartbeat 才能
说明 progress;按 worker 每 5 分钟一次的约定,连续三个周期(超过 15 分钟)未更新、
格式非法或时间在未来的 heartbeat 只能报告 `unknown`,不得当作持续进展。
```bash ```bash
orca orchestration check \ orca orchestration check \
+101 -32
View File
@@ -16,6 +16,7 @@ Output (single JSON document on stdout):
"status": "running | progress | stall | not-started | unknown", "status": "running | progress | stall | not-started | unknown",
"stallReason": "<label> | null", "stallReason": "<label> | null",
"heartbeatAt": "<value> | null", "heartbeatAt": "<value> | null",
"heartbeatAgeSeconds": "<number> | null",
"evidence": "<bounded terminal tail>" "evidence": "<bounded terminal tail>"
} }
@@ -29,6 +30,7 @@ import argparse
import json import json
import re import re
import sys import sys
from datetime import datetime
from pathlib import Path from pathlib import Path
@@ -36,6 +38,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
from launch_worker import ( # noqa: E402 from launch_worker import ( # noqa: E402
LaunchError, LaunchError,
parse_timestamp,
resolve_executable, resolve_executable,
run_json, run_json,
utc_now, utc_now,
@@ -53,48 +56,113 @@ WORKING_PATTERN = re.compile(r"(?i)working|•working|running|执行中|正在")
IDLE_TAIL_PATTERN = re.compile(r"(?i)welcome to|type help|fish, the friendly|>\\s*$") IDLE_TAIL_PATTERN = re.compile(r"(?i)welcome to|type help|fish, the friendly|>\\s*$")
MAX_EVIDENCE_CHARS = 500 MAX_EVIDENCE_CHARS = 500
HEARTBEAT_INTERVAL_SECONDS = 5 * 60
HEARTBEAT_MAX_AGE_SECONDS = HEARTBEAT_INTERVAL_SECONDS * 3
def classify(tail: str | list[str], heartbeat: object) -> dict[str, object]: def heartbeat_age_seconds(
heartbeat: object, now: datetime | None = None
) -> tuple[float | None, str | None]:
"""Return heartbeat age and a diagnostic for malformed/stale timestamps."""
if heartbeat is None:
return None, None
try:
heartbeat_at = parse_timestamp(heartbeat, "last_heartbeat_at")
except LaunchError:
return None, "invalid-heartbeat"
current = now if now is not None else utc_now()
age = (current - heartbeat_at).total_seconds()
if age < -60:
return None, "future-heartbeat"
return max(age, 0.0), None
def _result(
*,
status: str,
stall_reason: str | None,
heartbeat: object,
heartbeat_age: float | None,
evidence: str,
) -> dict[str, object]:
return {
"status": status,
"stallReason": stall_reason,
"heartbeatAt": heartbeat,
"heartbeatAgeSeconds": heartbeat_age,
"evidence": evidence[:MAX_EVIDENCE_CHARS],
}
def classify(
tail: str | list[str],
heartbeat: object,
*,
now: datetime | None = None,
) -> dict[str, object]:
if isinstance(tail, list): if isinstance(tail, list):
tail = "\n".join(tail) tail = "\n".join(tail)
heartbeat_age, heartbeat_error = heartbeat_age_seconds(heartbeat, now)
for label, pattern in STALL_PATTERNS: for label, pattern in STALL_PATTERNS:
if pattern.search(tail): if pattern.search(tail):
return { return _result(
"status": "stall", status="stall",
"stallReason": label, stall_reason=label,
"heartbeatAt": heartbeat, heartbeat=heartbeat,
"evidence": tail[:MAX_EVIDENCE_CHARS], heartbeat_age=heartbeat_age,
} evidence=tail,
if heartbeat: )
return { if heartbeat_error:
"status": "progress", return _result(
"stallReason": None, status="unknown",
"heartbeatAt": heartbeat, stall_reason=heartbeat_error,
"evidence": tail[:MAX_EVIDENCE_CHARS], heartbeat=heartbeat,
} heartbeat_age=heartbeat_age,
evidence=tail,
)
if heartbeat_age is not None and heartbeat_age <= HEARTBEAT_MAX_AGE_SECONDS:
return _result(
status="progress",
stall_reason=None,
heartbeat=heartbeat,
heartbeat_age=heartbeat_age,
evidence=tail,
)
if heartbeat_age is not None:
return _result(
status="unknown",
stall_reason="stale-heartbeat",
heartbeat=heartbeat,
heartbeat_age=heartbeat_age,
evidence=tail,
)
if WORKING_PATTERN.search(tail): if WORKING_PATTERN.search(tail):
return { return _result(
"status": "running", status="running",
"stallReason": None, stall_reason=None,
"heartbeatAt": None, heartbeat=heartbeat,
"evidence": tail[:MAX_EVIDENCE_CHARS], heartbeat_age=heartbeat_age,
} evidence=tail,
)
# No heartbeat and no working marker: the terminal may still be sitting at # No heartbeat and no working marker: the terminal may still be sitting at
# a welcome/idle prompt (task never started) or have unclassified output. # a welcome/idle prompt (task never started) or have unclassified output.
if IDLE_TAIL_PATTERN.search(tail) or not tail.strip(): if IDLE_TAIL_PATTERN.search(tail) or not tail.strip():
return { return _result(
"status": "not-started", status="not-started",
"stallReason": None, stall_reason=None,
"heartbeatAt": None, heartbeat=heartbeat,
"evidence": tail[:MAX_EVIDENCE_CHARS], heartbeat_age=heartbeat_age,
} evidence=tail,
return { )
"status": "unknown", return _result(
"stallReason": None, status="unknown",
"heartbeatAt": None, stall_reason=None,
"evidence": tail[:MAX_EVIDENCE_CHARS], heartbeat=heartbeat,
} heartbeat_age=heartbeat_age,
evidence=tail,
)
def main(argv: list[str] | None = None) -> int: def main(argv: list[str] | None = None) -> int:
@@ -126,6 +194,7 @@ def main(argv: list[str] | None = None) -> int:
"status": "unknown", "status": "unknown",
"stallReason": None, "stallReason": None,
"heartbeatAt": None, "heartbeatAt": None,
"heartbeatAgeSeconds": None,
"evidence": f"probe failed: {type(exc).__name__}: {exc}", "evidence": f"probe failed: {type(exc).__name__}: {exc}",
}, },
ensure_ascii=False, ensure_ascii=False,
+2
View File
@@ -131,6 +131,8 @@ class AckSkillContentTests(unittest.TestCase):
"不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话", "不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话",
"`check --wait`", "`check --wait`",
"`worker_probe.py`", "`worker_probe.py`",
"默认 90 秒窗口",
"60–120 秒节奏",
"Developer 回报 `worker_done` 或 `escalation`", "Developer 回报 `worker_done` 或 `escalation`",
"写回 `fixed_by_dev`", "写回 `fixed_by_dev`",
"主动派发 Test", "主动派发 Test",
+23 -3
View File
@@ -5,6 +5,7 @@ import io
import json import json
import sys import sys
import unittest import unittest
from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from unittest import mock from unittest import mock
@@ -62,12 +63,30 @@ class WorkerProbeClassifyTests(unittest.TestCase):
self.assertEqual(result["status"], "stall") self.assertEqual(result["status"], "stall")
self.assertEqual(result["stallReason"], "usage-limit") self.assertEqual(result["stallReason"], "usage-limit")
def test_heartbeat_means_progress_even_without_tail_markers(self) -> None: def test_fresh_heartbeat_means_progress_even_without_tail_markers(self) -> None:
result = worker_probe.classify( result = worker_probe.classify(
"some unclassified output", "2026-08-23T12:00:00Z" "some unclassified output",
"2026-08-23T12:00:00Z",
now=datetime(2026, 8, 23, 12, 2, tzinfo=timezone.utc),
) )
self.assertEqual(result["status"], "progress") self.assertEqual(result["status"], "progress")
self.assertEqual(result["heartbeatAt"], "2026-08-23T12:00:00Z") self.assertEqual(result["heartbeatAt"], "2026-08-23T12:00:00Z")
self.assertEqual(result["heartbeatAgeSeconds"], 120.0)
def test_stale_heartbeat_is_unknown_even_with_working_tail(self) -> None:
result = worker_probe.classify(
["• Working (18m)"],
"2026-08-23T12:00:00Z",
now=datetime(2026, 8, 23, 12, 16, tzinfo=timezone.utc),
)
self.assertEqual(result["status"], "unknown")
self.assertEqual(result["stallReason"], "stale-heartbeat")
self.assertEqual(result["heartbeatAgeSeconds"], 960.0)
def test_invalid_heartbeat_is_unknown(self) -> None:
result = worker_probe.classify("some output", "not-a-timestamp")
self.assertEqual(result["status"], "unknown")
self.assertEqual(result["stallReason"], "invalid-heartbeat")
def test_working_marker_means_running_without_heartbeat(self) -> None: def test_working_marker_means_running_without_heartbeat(self) -> None:
result = worker_probe.classify(["• Working (12s)"], None) result = worker_probe.classify(["• Working (12s)"], None)
@@ -87,6 +106,7 @@ class WorkerProbeClassifyTests(unittest.TestCase):
class WorkerProbeMainTests(unittest.TestCase): class WorkerProbeMainTests(unittest.TestCase):
def test_main_emits_single_json_with_progress(self) -> None: def test_main_emits_single_json_with_progress(self) -> None:
fresh_heartbeat = worker_probe.utc_now().isoformat().replace("+00:00", "Z")
with ( with (
mock.patch.object( mock.patch.object(
worker_probe, worker_probe,
@@ -97,7 +117,7 @@ class WorkerProbeMainTests(unittest.TestCase):
worker_probe, worker_probe,
"run_json", "run_json",
side_effect=[ side_effect=[
dispatch_show(heartbeat="2026-08-23T12:00:00Z"), dispatch_show(heartbeat=fresh_heartbeat),
terminal_read(["Working (5s)"]), terminal_read(["Working (5s)"]),
], ],
), ),