fix(ack): bound worker polling and detect stale heartbeats

2026-09-19 19:30:20 +08:00
parent 2917708b10
commit 94c66bc695
5 changed files with 132 additions and 39 deletions
@@ -84,7 +84,7 @@ python3 <ack-skill-dir>/scripts/validate_agents.py .pouch/ack/agents.yaml
 2. 读 `project.md`；若存在 `agents.yaml`，先用 `validate_agents.py` 校验并读取其 `defaults`；用 `select_tasks.py`（已知任务加 `--task-id`）和 `select_knowledge.py` 取当前任务相关 `active` 条目。超预算时缩小选择，不回退为完整 yaml。
 3. 新需求或尚未确认的计划：读 [kickoff.md](references/kickoff.md)。修 bug 写短问题说明、复现和可观测验收，不写大 PRD。
 4. 用户已确认后，先读取并按 [closed-loop.md](references/closed-loop.md) 执行；使用 Orca 时同时读取 [orca-adapter.md](references/orca-adapter.md)。当前会话担任 Coordinator，不亲自写代码或跑测试；worker profile 按 `agents.yaml`（若存在）否则按任务板配置选择。
-   这不是 fire-and-forget：Coordinator 不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话。每个任务都必须在同一轮持续执行 `check --wait` 与 `worker_probe.py`，直到 Developer 回报 `worker_done` 或 `escalation`；随后写回 `fixed_by_dev`，主动派发 Test，并继续等待 `retest_result`。只有完成 Test 终检、重试/`leftover` 决策、回归决定和 worker 回收后，才可以结束本轮 ACK。
+   这不是 fire-and-forget：Coordinator 不得在 `launch` / `dispatch` 后只回复“已派发”并结束当前会话。每个任务都必须在同一轮持续执行默认 90 秒窗口的 `check --wait`，并按 60–120 秒节奏运行 `worker_probe.py`，直到 Developer 回报 `worker_done` 或 `escalation`；随后写回 `fixed_by_dev`，主动派发 Test，并继续等待 `retest_result`。只有完成 Test 终检、重试/`leftover` 决策、回归决定和 worker 回收后，才可以结束本轮 ACK。
 5. 存在 `project.bugIntake` 时先完成「飞书收件」。创建或更换 worker 走「启动 worker」。若 `intents.testEnvironment` 已启用，派 Test 前先走「运行测试环境」。
 6. 知识 `verification.ref` 只经 `run_verification.py`。不把 `worker_done` 或 Test 自报成功当作完成。环境失败先有界恢复并报告 `userAction`；三轮产品失败记 `leftover`。
 7. 任务 `verified` 且改了可见行为或 API 后，转入「运行回归」收获用例。若本次计划含交付，再走对应交付模式；默认最多到 `validation_ready` 或 `review_ready`。
@@ -215,7 +215,7 @@ EOF
     且终端进入工作指示（Working / Running）。
   - 手动投递路径：`orca terminal send` 必须带 `--enter`；投递后同样 read 确认。
   - 确认失败或终端仍停在欢迎提示：按「消息未投递」记录环境失败，不消耗产品轮次。
-2. 等待期间滚动 probe（每 60–120 秒一次）：
+2. 等待期间滚动 probe（每 60–120 秒一次；不要在 30 秒窗口后每次都 probe）：

   ```bash
   python3 <ack-skill-dir>/scripts/worker_probe.py \
@@ -229,8 +229,10 @@ EOF
   - `not-started`：检查是否漏投递或未回车；重新投递或记录环境失败，不占轮次。
   - `running` / `progress`：继续滚动 wait。
   - `unknown`：按 `dispatch-show` 与 Orca live state 人工核对，不自动重试。
-4. `check --wait` 使用短窗口（60–90 秒）而不是 15 分钟：窗口超时是检查点，先 probe
-   再决定继续等待、恢复或上报。
+4. `check --wait` 默认使用 90 秒窗口而不是 30 秒或 15 分钟：窗口超时是检查点，
+   仅在达到 probe 间隔后再探测，然后决定继续等待、恢复或上报。新鲜 heartbeat 才能
+   说明 progress；按 worker 每 5 分钟一次的约定，连续三个周期（超过 15 分钟）未更新、
+   格式非法或时间在未来的 heartbeat 只能报告 `unknown`，不得当作持续进展。

 ```bash
 orca orchestration check \
@@ -16,6 +16,7 @@ Output (single JSON document on stdout):
    "status": "running | progress | stall | not-started | unknown",
    "stallReason": "<label> | null",
    "heartbeatAt": "<value> | null",
+    "heartbeatAgeSeconds": "<number> | null",
    "evidence": "<bounded terminal tail>"
  }

@@ -29,6 +30,7 @@ import argparse
 import json
 import re
 import sys
+from datetime import datetime
 from pathlib import Path


@@ -36,6 +38,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))

 from launch_worker import (  # noqa: E402
    LaunchError,
+    parse_timestamp,
    resolve_executable,
    run_json,
    utc_now,
@@ -53,48 +56,113 @@ WORKING_PATTERN = re.compile(r"(?i)working|•working|running|执行中|正在")
 IDLE_TAIL_PATTERN = re.compile(r"(?i)welcome to|type help|fish, the friendly|>\\s*$")

 MAX_EVIDENCE_CHARS = 500
+HEARTBEAT_INTERVAL_SECONDS = 5 * 60
+HEARTBEAT_MAX_AGE_SECONDS = HEARTBEAT_INTERVAL_SECONDS * 3


-def classify(tail: str | list[str], heartbeat: object) -> dict[str, object]:
+def heartbeat_age_seconds(
+    heartbeat: object, now: datetime | None = None
+) -> tuple[float | None, str | None]:
+    """Return heartbeat age and a diagnostic for malformed/stale timestamps."""
+
+    if heartbeat is None:
+        return None, None
+    try:
+        heartbeat_at = parse_timestamp(heartbeat, "last_heartbeat_at")
+    except LaunchError:
+        return None, "invalid-heartbeat"
+
+    current = now if now is not None else utc_now()
+    age = (current - heartbeat_at).total_seconds()
+    if age < -60:
+        return None, "future-heartbeat"
+    return max(age, 0.0), None
+
+
+def _result(
+    *,
+    status: str,
+    stall_reason: str | None,
+    heartbeat: object,
+    heartbeat_age: float | None,
+    evidence: str,
+) -> dict[str, object]:
+    return {
+        "status": status,
+        "stallReason": stall_reason,
+        "heartbeatAt": heartbeat,
+        "heartbeatAgeSeconds": heartbeat_age,
+        "evidence": evidence[:MAX_EVIDENCE_CHARS],
+    }
+
+
+def classify(
+    tail: str | list[str],
+    heartbeat: object,
+    *,
+    now: datetime | None = None,
+) -> dict[str, object]:
    if isinstance(tail, list):
        tail = "\n".join(tail)
+    heartbeat_age, heartbeat_error = heartbeat_age_seconds(heartbeat, now)
    for label, pattern in STALL_PATTERNS:
        if pattern.search(tail):
-            return {
-                "status": "stall",
-                "stallReason": label,
-                "heartbeatAt": heartbeat,
-                "evidence": tail[:MAX_EVIDENCE_CHARS],
-            }
-    if heartbeat:
-        return {
-            "status": "progress",
-            "stallReason": None,
-            "heartbeatAt": heartbeat,
-            "evidence": tail[:MAX_EVIDENCE_CHARS],
-        }
+            return _result(
+                status="stall",
+                stall_reason=label,
+                heartbeat=heartbeat,
+                heartbeat_age=heartbeat_age,
+                evidence=tail,
+            )
+    if heartbeat_error:
+        return _result(
+            status="unknown",
+            stall_reason=heartbeat_error,
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
+    if heartbeat_age is not None and heartbeat_age <= HEARTBEAT_MAX_AGE_SECONDS:
+        return _result(
+            status="progress",
+            stall_reason=None,
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
+    if heartbeat_age is not None:
+        return _result(
+            status="unknown",
+            stall_reason="stale-heartbeat",
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
    if WORKING_PATTERN.search(tail):
-        return {
-            "status": "running",
-            "stallReason": None,
-            "heartbeatAt": None,
-            "evidence": tail[:MAX_EVIDENCE_CHARS],
-        }
+        return _result(
+            status="running",
+            stall_reason=None,
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
    # No heartbeat and no working marker: the terminal may still be sitting at
    # a welcome/idle prompt (task never started) or have unclassified output.
    if IDLE_TAIL_PATTERN.search(tail) or not tail.strip():
-        return {
-            "status": "not-started",
-            "stallReason": None,
-            "heartbeatAt": None,
-            "evidence": tail[:MAX_EVIDENCE_CHARS],
-        }
-    return {
-        "status": "unknown",
-        "stallReason": None,
-        "heartbeatAt": None,
-        "evidence": tail[:MAX_EVIDENCE_CHARS],
-    }
+        return _result(
+            status="not-started",
+            stall_reason=None,
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
+    return _result(
+        status="unknown",
+        stall_reason=None,
+        heartbeat=heartbeat,
+        heartbeat_age=heartbeat_age,
+        evidence=tail,
+    )


 def main(argv: list[str] | None = None) -> int:
@@ -126,6 +194,7 @@ def main(argv: list[str] | None = None) -> int:
                    "status": "unknown",
                    "stallReason": None,
                    "heartbeatAt": None,
+                    "heartbeatAgeSeconds": None,
                    "evidence": f"probe failed: {type(exc).__name__}: {exc}",
                },
                ensure_ascii=False,