fix(ack): bound worker polling and detect stale heartbeats

skills/ack/scripts/worker_probe.py

@@ -16,6 +16,7 @@ Output (single JSON document on stdout):
     "status": "running | progress | stall | not-started | unknown",
     "stallReason": "<label> | null",
     "heartbeatAt": "<value> | null",
+    "heartbeatAgeSeconds": "<number> | null",
     "evidence": "<bounded terminal tail>"
   }
 
@@ -29,6 +30,7 @@ import argparse
 import json
 import re
 import sys
+from datetime import datetime
 from pathlib import Path
 
 
@@ -36,6 +38,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
 
 from launch_worker import (  # noqa: E402
     LaunchError,
+    parse_timestamp,
     resolve_executable,
     run_json,
     utc_now,
@@ -53,48 +56,113 @@ WORKING_PATTERN = re.compile(r"(?i)working|•working|running|执行中|正在")
 IDLE_TAIL_PATTERN = re.compile(r"(?i)welcome to|type help|fish, the friendly|>\\s*$")
 
 MAX_EVIDENCE_CHARS = 500
+HEARTBEAT_INTERVAL_SECONDS = 5 * 60
+HEARTBEAT_MAX_AGE_SECONDS = HEARTBEAT_INTERVAL_SECONDS * 3
 
 
-def classify(tail: str | list[str], heartbeat: object) -> dict[str, object]:
+def heartbeat_age_seconds(
+    heartbeat: object, now: datetime | None = None
+) -> tuple[float | None, str | None]:
+    """Return heartbeat age and a diagnostic for malformed/stale timestamps."""
+
+    if heartbeat is None:
+        return None, None
+    try:
+        heartbeat_at = parse_timestamp(heartbeat, "last_heartbeat_at")
+    except LaunchError:
+        return None, "invalid-heartbeat"
+
+    current = now if now is not None else utc_now()
+    age = (current - heartbeat_at).total_seconds()
+    if age < -60:
+        return None, "future-heartbeat"
+    return max(age, 0.0), None
+
+
+def _result(
+    *,
+    status: str,
+    stall_reason: str | None,
+    heartbeat: object,
+    heartbeat_age: float | None,
+    evidence: str,
+) -> dict[str, object]:
+    return {
+        "status": status,
+        "stallReason": stall_reason,
+        "heartbeatAt": heartbeat,
+        "heartbeatAgeSeconds": heartbeat_age,
+        "evidence": evidence[:MAX_EVIDENCE_CHARS],
+    }
+
+
+def classify(
+    tail: str | list[str],
+    heartbeat: object,
+    *,
+    now: datetime | None = None,
+) -> dict[str, object]:
     if isinstance(tail, list):
         tail = "\n".join(tail)
+    heartbeat_age, heartbeat_error = heartbeat_age_seconds(heartbeat, now)
     for label, pattern in STALL_PATTERNS:
         if pattern.search(tail):
-            return {
-                "status": "stall",
-                "stallReason": label,
-                "heartbeatAt": heartbeat,
-                "evidence": tail[:MAX_EVIDENCE_CHARS],
-            }
-    if heartbeat:
-        return {
-            "status": "progress",
-            "stallReason": None,
-            "heartbeatAt": heartbeat,
-            "evidence": tail[:MAX_EVIDENCE_CHARS],
-        }
+            return _result(
+                status="stall",
+                stall_reason=label,
+                heartbeat=heartbeat,
+                heartbeat_age=heartbeat_age,
+                evidence=tail,
+            )
+    if heartbeat_error:
+        return _result(
+            status="unknown",
+            stall_reason=heartbeat_error,
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
+    if heartbeat_age is not None and heartbeat_age <= HEARTBEAT_MAX_AGE_SECONDS:
+        return _result(
+            status="progress",
+            stall_reason=None,
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
+    if heartbeat_age is not None:
+        return _result(
+            status="unknown",
+            stall_reason="stale-heartbeat",
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
     if WORKING_PATTERN.search(tail):
-        return {
-            "status": "running",
-            "stallReason": None,
-            "heartbeatAt": None,
-            "evidence": tail[:MAX_EVIDENCE_CHARS],
-        }
+        return _result(
+            status="running",
+            stall_reason=None,
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
     # No heartbeat and no working marker: the terminal may still be sitting at
     # a welcome/idle prompt (task never started) or have unclassified output.
     if IDLE_TAIL_PATTERN.search(tail) or not tail.strip():
-        return {
-            "status": "not-started",
-            "stallReason": None,
-            "heartbeatAt": None,
-            "evidence": tail[:MAX_EVIDENCE_CHARS],
-        }
-    return {
-        "status": "unknown",
-        "stallReason": None,
-        "heartbeatAt": None,
-        "evidence": tail[:MAX_EVIDENCE_CHARS],
-    }
+        return _result(
+            status="not-started",
+            stall_reason=None,
+            heartbeat=heartbeat,
+            heartbeat_age=heartbeat_age,
+            evidence=tail,
+        )
+    return _result(
+        status="unknown",
+        stall_reason=None,
+        heartbeat=heartbeat,
+        heartbeat_age=heartbeat_age,
+        evidence=tail,
+    )
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -126,6 +194,7 @@ def main(argv: list[str] | None = None) -> int:
                     "status": "unknown",
                     "stallReason": None,
                     "heartbeatAt": None,
+                    "heartbeatAgeSeconds": None,
                     "evidence": f"probe failed: {type(exc).__name__}: {exc}",
                 },
                 ensure_ascii=False,