fix(ack): bound worker polling and detect stale heartbeats
This commit is contained in:
+1
-1
@@ -84,7 +84,7 @@ python3 <ack-skill-dir>/scripts/validate_agents.py .pouch/ack/agents.yaml
|
|||||||
2. 读 `project.md`;若存在 `agents.yaml`,先用 `validate_agents.py` 校验并读取其 `defaults`;用 `select_tasks.py`(已知任务加 `--task-id`)和 `select_knowledge.py` 取当前任务相关 `active` 条目。超预算时缩小选择,不回退为完整 yaml。
|
2. 读 `project.md`;若存在 `agents.yaml`,先用 `validate_agents.py` 校验并读取其 `defaults`;用 `select_tasks.py`(已知任务加 `--task-id`)和 `select_knowledge.py` 取当前任务相关 `active` 条目。超预算时缩小选择,不回退为完整 yaml。
|
||||||
3. 新需求或尚未确认的计划:读 [kickoff.md](references/kickoff.md)。修 bug 写短问题说明、复现和可观测验收,不写大 PRD。
|
3. 新需求或尚未确认的计划:读 [kickoff.md](references/kickoff.md)。修 bug 写短问题说明、复现和可观测验收,不写大 PRD。
|
||||||
4. 用户已确认后,先读取并按 [closed-loop.md](references/closed-loop.md) 执行;使用 Orca 时同时读取 [orca-adapter.md](references/orca-adapter.md)。当前会话担任 Coordinator,不亲自写代码或跑测试;worker profile 按 `agents.yaml`(若存在)否则按任务板配置选择。
|
4. 用户已确认后,先读取并按 [closed-loop.md](references/closed-loop.md) 执行;使用 Orca 时同时读取 [orca-adapter.md](references/orca-adapter.md)。当前会话担任 Coordinator,不亲自写代码或跑测试;worker profile 按 `agents.yaml`(若存在)否则按任务板配置选择。
|
||||||
这不是 fire-and-forget:Coordinator 不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话。每个任务都必须在同一轮持续执行 `check --wait` 与 `worker_probe.py`,直到 Developer 回报 `worker_done` 或 `escalation`;随后写回 `fixed_by_dev`,主动派发 Test,并继续等待 `retest_result`。只有完成 Test 终检、重试/`leftover` 决策、回归决定和 worker 回收后,才可以结束本轮 ACK。
|
这不是 fire-and-forget:Coordinator 不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话。每个任务都必须在同一轮持续执行默认 90 秒窗口的 `check --wait`,并按 60–120 秒节奏运行 `worker_probe.py`,直到 Developer 回报 `worker_done` 或 `escalation`;随后写回 `fixed_by_dev`,主动派发 Test,并继续等待 `retest_result`。只有完成 Test 终检、重试/`leftover` 决策、回归决定和 worker 回收后,才可以结束本轮 ACK。
|
||||||
5. 存在 `project.bugIntake` 时先完成「飞书收件」。创建或更换 worker 走「启动 worker」。若 `intents.testEnvironment` 已启用,派 Test 前先走「运行测试环境」。
|
5. 存在 `project.bugIntake` 时先完成「飞书收件」。创建或更换 worker 走「启动 worker」。若 `intents.testEnvironment` 已启用,派 Test 前先走「运行测试环境」。
|
||||||
6. 知识 `verification.ref` 只经 `run_verification.py`。不把 `worker_done` 或 Test 自报成功当作完成。环境失败先有界恢复并报告 `userAction`;三轮产品失败记 `leftover`。
|
6. 知识 `verification.ref` 只经 `run_verification.py`。不把 `worker_done` 或 Test 自报成功当作完成。环境失败先有界恢复并报告 `userAction`;三轮产品失败记 `leftover`。
|
||||||
7. 任务 `verified` 且改了可见行为或 API 后,转入「运行回归」收获用例。若本次计划含交付,再走对应交付模式;默认最多到 `validation_ready` 或 `review_ready`。
|
7. 任务 `verified` 且改了可见行为或 API 后,转入「运行回归」收获用例。若本次计划含交付,再走对应交付模式;默认最多到 `validation_ready` 或 `review_ready`。
|
||||||
|
|||||||
@@ -215,7 +215,7 @@ EOF
|
|||||||
且终端进入工作指示(Working / Running)。
|
且终端进入工作指示(Working / Running)。
|
||||||
- 手动投递路径:`orca terminal send` 必须带 `--enter`;投递后同样 read 确认。
|
- 手动投递路径:`orca terminal send` 必须带 `--enter`;投递后同样 read 确认。
|
||||||
- 确认失败或终端仍停在欢迎提示:按「消息未投递」记录环境失败,不消耗产品轮次。
|
- 确认失败或终端仍停在欢迎提示:按「消息未投递」记录环境失败,不消耗产品轮次。
|
||||||
2. 等待期间滚动 probe(每 60–120 秒一次):
|
2. 等待期间滚动 probe(每 60–120 秒一次;不要在 30 秒窗口后每次都 probe):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python3 <ack-skill-dir>/scripts/worker_probe.py \
|
python3 <ack-skill-dir>/scripts/worker_probe.py \
|
||||||
@@ -229,8 +229,10 @@ EOF
|
|||||||
- `not-started`:检查是否漏投递或未回车;重新投递或记录环境失败,不占轮次。
|
- `not-started`:检查是否漏投递或未回车;重新投递或记录环境失败,不占轮次。
|
||||||
- `running` / `progress`:继续滚动 wait。
|
- `running` / `progress`:继续滚动 wait。
|
||||||
- `unknown`:按 `dispatch-show` 与 Orca live state 人工核对,不自动重试。
|
- `unknown`:按 `dispatch-show` 与 Orca live state 人工核对,不自动重试。
|
||||||
4. `check --wait` 使用短窗口(60–90 秒)而不是 15 分钟:窗口超时是检查点,先 probe
|
4. `check --wait` 默认使用 90 秒窗口而不是 30 秒或 15 分钟:窗口超时是检查点,
|
||||||
再决定继续等待、恢复或上报。
|
仅在达到 probe 间隔后再探测,然后决定继续等待、恢复或上报。新鲜 heartbeat 才能
|
||||||
|
说明 progress;按 worker 每 5 分钟一次的约定,连续三个周期(超过 15 分钟)未更新、
|
||||||
|
格式非法或时间在未来的 heartbeat 只能报告 `unknown`,不得当作持续进展。
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
orca orchestration check \
|
orca orchestration check \
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ Output (single JSON document on stdout):
|
|||||||
"status": "running | progress | stall | not-started | unknown",
|
"status": "running | progress | stall | not-started | unknown",
|
||||||
"stallReason": "<label> | null",
|
"stallReason": "<label> | null",
|
||||||
"heartbeatAt": "<value> | null",
|
"heartbeatAt": "<value> | null",
|
||||||
|
"heartbeatAgeSeconds": "<number> | null",
|
||||||
"evidence": "<bounded terminal tail>"
|
"evidence": "<bounded terminal tail>"
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -29,6 +30,7 @@ import argparse
|
|||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
@@ -36,6 +38,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|||||||
|
|
||||||
from launch_worker import ( # noqa: E402
|
from launch_worker import ( # noqa: E402
|
||||||
LaunchError,
|
LaunchError,
|
||||||
|
parse_timestamp,
|
||||||
resolve_executable,
|
resolve_executable,
|
||||||
run_json,
|
run_json,
|
||||||
utc_now,
|
utc_now,
|
||||||
@@ -53,48 +56,113 @@ WORKING_PATTERN = re.compile(r"(?i)working|•working|running|执行中|正在")
|
|||||||
IDLE_TAIL_PATTERN = re.compile(r"(?i)welcome to|type help|fish, the friendly|>\\s*$")
|
IDLE_TAIL_PATTERN = re.compile(r"(?i)welcome to|type help|fish, the friendly|>\\s*$")
|
||||||
|
|
||||||
MAX_EVIDENCE_CHARS = 500
|
MAX_EVIDENCE_CHARS = 500
|
||||||
|
HEARTBEAT_INTERVAL_SECONDS = 5 * 60
|
||||||
|
HEARTBEAT_MAX_AGE_SECONDS = HEARTBEAT_INTERVAL_SECONDS * 3
|
||||||
|
|
||||||
|
|
||||||
def classify(tail: str | list[str], heartbeat: object) -> dict[str, object]:
|
def heartbeat_age_seconds(
|
||||||
|
heartbeat: object, now: datetime | None = None
|
||||||
|
) -> tuple[float | None, str | None]:
|
||||||
|
"""Return heartbeat age and a diagnostic for malformed/stale timestamps."""
|
||||||
|
|
||||||
|
if heartbeat is None:
|
||||||
|
return None, None
|
||||||
|
try:
|
||||||
|
heartbeat_at = parse_timestamp(heartbeat, "last_heartbeat_at")
|
||||||
|
except LaunchError:
|
||||||
|
return None, "invalid-heartbeat"
|
||||||
|
|
||||||
|
current = now if now is not None else utc_now()
|
||||||
|
age = (current - heartbeat_at).total_seconds()
|
||||||
|
if age < -60:
|
||||||
|
return None, "future-heartbeat"
|
||||||
|
return max(age, 0.0), None
|
||||||
|
|
||||||
|
|
||||||
|
def _result(
|
||||||
|
*,
|
||||||
|
status: str,
|
||||||
|
stall_reason: str | None,
|
||||||
|
heartbeat: object,
|
||||||
|
heartbeat_age: float | None,
|
||||||
|
evidence: str,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"status": status,
|
||||||
|
"stallReason": stall_reason,
|
||||||
|
"heartbeatAt": heartbeat,
|
||||||
|
"heartbeatAgeSeconds": heartbeat_age,
|
||||||
|
"evidence": evidence[:MAX_EVIDENCE_CHARS],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def classify(
|
||||||
|
tail: str | list[str],
|
||||||
|
heartbeat: object,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> dict[str, object]:
|
||||||
if isinstance(tail, list):
|
if isinstance(tail, list):
|
||||||
tail = "\n".join(tail)
|
tail = "\n".join(tail)
|
||||||
|
heartbeat_age, heartbeat_error = heartbeat_age_seconds(heartbeat, now)
|
||||||
for label, pattern in STALL_PATTERNS:
|
for label, pattern in STALL_PATTERNS:
|
||||||
if pattern.search(tail):
|
if pattern.search(tail):
|
||||||
return {
|
return _result(
|
||||||
"status": "stall",
|
status="stall",
|
||||||
"stallReason": label,
|
stall_reason=label,
|
||||||
"heartbeatAt": heartbeat,
|
heartbeat=heartbeat,
|
||||||
"evidence": tail[:MAX_EVIDENCE_CHARS],
|
heartbeat_age=heartbeat_age,
|
||||||
}
|
evidence=tail,
|
||||||
if heartbeat:
|
)
|
||||||
return {
|
if heartbeat_error:
|
||||||
"status": "progress",
|
return _result(
|
||||||
"stallReason": None,
|
status="unknown",
|
||||||
"heartbeatAt": heartbeat,
|
stall_reason=heartbeat_error,
|
||||||
"evidence": tail[:MAX_EVIDENCE_CHARS],
|
heartbeat=heartbeat,
|
||||||
}
|
heartbeat_age=heartbeat_age,
|
||||||
|
evidence=tail,
|
||||||
|
)
|
||||||
|
if heartbeat_age is not None and heartbeat_age <= HEARTBEAT_MAX_AGE_SECONDS:
|
||||||
|
return _result(
|
||||||
|
status="progress",
|
||||||
|
stall_reason=None,
|
||||||
|
heartbeat=heartbeat,
|
||||||
|
heartbeat_age=heartbeat_age,
|
||||||
|
evidence=tail,
|
||||||
|
)
|
||||||
|
if heartbeat_age is not None:
|
||||||
|
return _result(
|
||||||
|
status="unknown",
|
||||||
|
stall_reason="stale-heartbeat",
|
||||||
|
heartbeat=heartbeat,
|
||||||
|
heartbeat_age=heartbeat_age,
|
||||||
|
evidence=tail,
|
||||||
|
)
|
||||||
if WORKING_PATTERN.search(tail):
|
if WORKING_PATTERN.search(tail):
|
||||||
return {
|
return _result(
|
||||||
"status": "running",
|
status="running",
|
||||||
"stallReason": None,
|
stall_reason=None,
|
||||||
"heartbeatAt": None,
|
heartbeat=heartbeat,
|
||||||
"evidence": tail[:MAX_EVIDENCE_CHARS],
|
heartbeat_age=heartbeat_age,
|
||||||
}
|
evidence=tail,
|
||||||
|
)
|
||||||
# No heartbeat and no working marker: the terminal may still be sitting at
|
# No heartbeat and no working marker: the terminal may still be sitting at
|
||||||
# a welcome/idle prompt (task never started) or have unclassified output.
|
# a welcome/idle prompt (task never started) or have unclassified output.
|
||||||
if IDLE_TAIL_PATTERN.search(tail) or not tail.strip():
|
if IDLE_TAIL_PATTERN.search(tail) or not tail.strip():
|
||||||
return {
|
return _result(
|
||||||
"status": "not-started",
|
status="not-started",
|
||||||
"stallReason": None,
|
stall_reason=None,
|
||||||
"heartbeatAt": None,
|
heartbeat=heartbeat,
|
||||||
"evidence": tail[:MAX_EVIDENCE_CHARS],
|
heartbeat_age=heartbeat_age,
|
||||||
}
|
evidence=tail,
|
||||||
return {
|
)
|
||||||
"status": "unknown",
|
return _result(
|
||||||
"stallReason": None,
|
status="unknown",
|
||||||
"heartbeatAt": None,
|
stall_reason=None,
|
||||||
"evidence": tail[:MAX_EVIDENCE_CHARS],
|
heartbeat=heartbeat,
|
||||||
}
|
heartbeat_age=heartbeat_age,
|
||||||
|
evidence=tail,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
def main(argv: list[str] | None = None) -> int:
|
||||||
@@ -126,6 +194,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
"status": "unknown",
|
"status": "unknown",
|
||||||
"stallReason": None,
|
"stallReason": None,
|
||||||
"heartbeatAt": None,
|
"heartbeatAt": None,
|
||||||
|
"heartbeatAgeSeconds": None,
|
||||||
"evidence": f"probe failed: {type(exc).__name__}: {exc}",
|
"evidence": f"probe failed: {type(exc).__name__}: {exc}",
|
||||||
},
|
},
|
||||||
ensure_ascii=False,
|
ensure_ascii=False,
|
||||||
|
|||||||
@@ -131,6 +131,8 @@ class AckSkillContentTests(unittest.TestCase):
|
|||||||
"不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话",
|
"不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话",
|
||||||
"`check --wait`",
|
"`check --wait`",
|
||||||
"`worker_probe.py`",
|
"`worker_probe.py`",
|
||||||
|
"默认 90 秒窗口",
|
||||||
|
"60–120 秒节奏",
|
||||||
"Developer 回报 `worker_done` 或 `escalation`",
|
"Developer 回报 `worker_done` 或 `escalation`",
|
||||||
"写回 `fixed_by_dev`",
|
"写回 `fixed_by_dev`",
|
||||||
"主动派发 Test",
|
"主动派发 Test",
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import io
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import unittest
|
import unittest
|
||||||
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
@@ -62,12 +63,30 @@ class WorkerProbeClassifyTests(unittest.TestCase):
|
|||||||
self.assertEqual(result["status"], "stall")
|
self.assertEqual(result["status"], "stall")
|
||||||
self.assertEqual(result["stallReason"], "usage-limit")
|
self.assertEqual(result["stallReason"], "usage-limit")
|
||||||
|
|
||||||
def test_heartbeat_means_progress_even_without_tail_markers(self) -> None:
|
def test_fresh_heartbeat_means_progress_even_without_tail_markers(self) -> None:
|
||||||
result = worker_probe.classify(
|
result = worker_probe.classify(
|
||||||
"some unclassified output", "2026-08-23T12:00:00Z"
|
"some unclassified output",
|
||||||
|
"2026-08-23T12:00:00Z",
|
||||||
|
now=datetime(2026, 8, 23, 12, 2, tzinfo=timezone.utc),
|
||||||
)
|
)
|
||||||
self.assertEqual(result["status"], "progress")
|
self.assertEqual(result["status"], "progress")
|
||||||
self.assertEqual(result["heartbeatAt"], "2026-08-23T12:00:00Z")
|
self.assertEqual(result["heartbeatAt"], "2026-08-23T12:00:00Z")
|
||||||
|
self.assertEqual(result["heartbeatAgeSeconds"], 120.0)
|
||||||
|
|
||||||
|
def test_stale_heartbeat_is_unknown_even_with_working_tail(self) -> None:
|
||||||
|
result = worker_probe.classify(
|
||||||
|
["• Working (18m)"],
|
||||||
|
"2026-08-23T12:00:00Z",
|
||||||
|
now=datetime(2026, 8, 23, 12, 16, tzinfo=timezone.utc),
|
||||||
|
)
|
||||||
|
self.assertEqual(result["status"], "unknown")
|
||||||
|
self.assertEqual(result["stallReason"], "stale-heartbeat")
|
||||||
|
self.assertEqual(result["heartbeatAgeSeconds"], 960.0)
|
||||||
|
|
||||||
|
def test_invalid_heartbeat_is_unknown(self) -> None:
|
||||||
|
result = worker_probe.classify("some output", "not-a-timestamp")
|
||||||
|
self.assertEqual(result["status"], "unknown")
|
||||||
|
self.assertEqual(result["stallReason"], "invalid-heartbeat")
|
||||||
|
|
||||||
def test_working_marker_means_running_without_heartbeat(self) -> None:
|
def test_working_marker_means_running_without_heartbeat(self) -> None:
|
||||||
result = worker_probe.classify(["• Working (12s)"], None)
|
result = worker_probe.classify(["• Working (12s)"], None)
|
||||||
@@ -87,6 +106,7 @@ class WorkerProbeClassifyTests(unittest.TestCase):
|
|||||||
|
|
||||||
class WorkerProbeMainTests(unittest.TestCase):
|
class WorkerProbeMainTests(unittest.TestCase):
|
||||||
def test_main_emits_single_json_with_progress(self) -> None:
|
def test_main_emits_single_json_with_progress(self) -> None:
|
||||||
|
fresh_heartbeat = worker_probe.utc_now().isoformat().replace("+00:00", "Z")
|
||||||
with (
|
with (
|
||||||
mock.patch.object(
|
mock.patch.object(
|
||||||
worker_probe,
|
worker_probe,
|
||||||
@@ -97,7 +117,7 @@ class WorkerProbeMainTests(unittest.TestCase):
|
|||||||
worker_probe,
|
worker_probe,
|
||||||
"run_json",
|
"run_json",
|
||||||
side_effect=[
|
side_effect=[
|
||||||
dispatch_show(heartbeat="2026-08-23T12:00:00Z"),
|
dispatch_show(heartbeat=fresh_heartbeat),
|
||||||
terminal_read(["Working (5s)"]),
|
terminal_read(["Working (5s)"]),
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
|
|||||||
Reference in New Issue
Block a user