From 26e133e3e85f2d892de2ed869281730442d12f8e Mon Sep 17 00:00:00 2001 From: nesquena-hermes Date: Mon, 8 Jun 2026 22:36:18 -0700 Subject: [PATCH] =?UTF-8?q?[HELD=20=E2=80=94=20independent=20review=20pend?= =?UTF-8?q?ing]=20Release=20v0.51.340=20=E2=80=94=20bg=5Ftask=20agent=20wa?= =?UTF-8?q?keup=20(trio=20#2968+#2971+#2979)=20(#3867)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * stage bg_task trio combined (#2979 superset) on master for deep review * fix(bg_task): unsubscribe SessionChannel on header-write failure (Codex deep-review catch) + regression test * test: realign on-subscribe-recovery anchor to subscribe_to_session_channel after leak fix * CHANGELOG: bg_task trio as v0.51.340 LD (HELD pending independent review) * bg_task trio: apply 3 independent-review (greptile) fixes 1. start_session_turn now threads the session PROFILE model defaults (_read_profile_model_config) into the wakeup model-resolve, so a brand-new session with an empty model falls back to the profile default not global DEFAULT_MODEL. Updated the white-box spy test signature accordingly. 2. /api/session/stream omits the Connection header (HTTP/1.1 keep-alive default) to match the #3103 long-lived-SSE pattern. 3. Reaper now prunes _LAST_EMIT_TS for collected sessions so the coalesce timestamp map can't grow one permanent entry per session forever. nesquena APPROVED the PR; these are the 3 non-blocking greptile suggestions. * test: realign _start_session_turn adapter stub lambda to new profile-defaults signature --- CHANGELOG.md | 23 +- api/background_process.py | 1248 +++++++++++++++++ api/config.py | 368 ++++- api/routes.py | 567 +++++++- api/streaming.py | 215 ++- server.py | 35 + static/messages.js | 257 ++++ static/sessions.js | 63 +- tests/_wakeup_helpers.py | 96 ++ tests/manual/repro_wakeup_hang.py | 166 +++ tests/test_bg_task_complete_ab_coexistence.py | 319 +++++ tests/test_bg_task_complete_focus_drop.py | 215 +++ ...ask_complete_loadsession_stream_restart.py | 135 ++ tests/test_bg_task_complete_ring_buffer.py | 170 +++ tests/test_bg_task_complete_throttle.py | 190 +++ tests/test_bg_task_complete_wakeup.py | 323 +++++ tests/test_bugbatch_apr2026.py | 2 +- ...est_live_rebuild_budget_warn_rate_limit.py | 59 + tests/test_optionz_liveview_perf.py | 553 ++++++++ tests/test_provider_mismatch.py | 7 +- tests/test_runtime_adapter_seam.py | 54 +- tests/test_session_channel_option_x.py | 921 ++++++++++++ ...ession_display_resolver_no_live_rebuild.py | 133 ++ ...test_start_session_turn_runtime_adapter.py | 130 ++ tests/test_wakeup_defer_race.py | 621 ++++++++ tests/test_wakeup_model_resolve_hang.py | 283 ++++ tests/test_xsession_wakeup_misroute.py | 259 ++++ 27 files changed, 7317 insertions(+), 95 deletions(-) create mode 100644 api/background_process.py create mode 100644 tests/_wakeup_helpers.py create mode 100644 tests/manual/repro_wakeup_hang.py create mode 100644 tests/test_bg_task_complete_ab_coexistence.py create mode 100644 tests/test_bg_task_complete_focus_drop.py create mode 100644 tests/test_bg_task_complete_loadsession_stream_restart.py create mode 100644 tests/test_bg_task_complete_ring_buffer.py create mode 100644 tests/test_bg_task_complete_throttle.py create mode 100644 tests/test_bg_task_complete_wakeup.py create mode 100644 tests/test_live_rebuild_budget_warn_rate_limit.py create mode 100644 tests/test_optionz_liveview_perf.py create mode 100644 tests/test_session_channel_option_x.py create mode 100644 tests/test_session_display_resolver_no_live_rebuild.py create mode 100644 tests/test_start_session_turn_runtime_adapter.py create mode 100644 tests/test_wakeup_defer_race.py create mode 100644 tests/test_wakeup_model_resolve_hang.py create mode 100644 tests/test_xsession_wakeup_misroute.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fe406d3f..cbc6e623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,13 +3,30 @@ ## [Unreleased] -## [v0.51.339] — 2026-06-09 — Release LC (targeted workspace create actions) +## [v0.51.340] — 2026-06-09 — Release LD (background-task agent wakeup in WebUI) — ⛔ HELD pending independent review ### Added -- **Create files and folders where you right-click in the workspace.** The workspace context menus (root heading, directory rows, file rows) now offer **New File** / **New Folder** that target the thing you clicked — a directory row creates inside that directory, a file row creates alongside it — instead of always targeting the current directory. The create prompt names the target (e.g. "New file name in `src`:"). (#3855, #3843, @b3nw) + +- Background tasks started with `terminal(notify_on_complete=true)` now wake the WebUI agent turn server-side, so the wakeup fires even when no browser tab is open. Closed-tab parity with CLI / Telegram / gateway hosts. (#2968) +- New `bg_task_complete` SSE event with trimmed `{session_id, task_id, completed_at, summary?, event_id}` payload and per-emit `event_id`. The legacy `process_complete` event name is dual-emitted with the same `event_id` for one PR cycle so in-flight WebUI builds keep working, then removed by #2971. (#2968, #2971) +- WebUI surfaces a small toast on background task completion, suppressed when the user is already focused on the target session. (#2971, #2979) +- New `GET /api/session/stream` per-session SSE channel for live-view of server-initiated wakeup turns; the browser's chat-stream renderer is reused (no second renderer). (#2968, #2971) +- New `POST /api/bg-task-complete-ack` diagnostic endpoint accepts `task_id` (canonical) and `process_id` (transitional alias; alias responses include a `Deprecation` header). (#2971, #2979) + +### Changed + +- WebUI subscribes to `bg_task_complete`; the legacy `process_complete` listener is removed and the browser no longer re-POSTs `wakeup_prompt` (wakeup is server-driven). Dedupe uses a 60s TTL ring buffer keyed `(session_id, event_id)`. (#2971) +- `POST /api/process-complete-ack` is replaced by `/api/bg-task-complete-ack`; the legacy path returns HTTP 410 with `X-Replaced-By: /api/bg-task-complete-ack`. Wired ahead of the CSRF gate so stale tabs see a discoverable hard error instead of a silent 403/404. (#2968) ### Fixed -- **The "Add as space?" prompt after creating a folder now has a clear "No" option** instead of an unlabeled cancel. (#3858, #3856, @b3nw) + +- Cross-session `notify_on_complete` wakeup no longer misroutes between concurrent WebUI sessions. Per-turn session identity is bound to a `contextvars.ContextVar` (`gateway.session_context` + `tools.approval`) inside the turn worker thread so concurrent background spawns inherit task/thread-local identity instead of racing on a process-global env slot, with a completion-time owner cross-check as defense-in-depth. (#2968) +- Fast (sub-teardown) background tasks completing while the WebUI session is technically still in its turn-teardown window now have their wakeup persisted in a deferred-wakeup map and drained at the turn's active→idle transition, so autonomous agent loops no longer lose the wakeup. (#2968) +- A background-task wakeup is no longer lost when the server-side wakeup turn races a human turn during session teardown. The teardown idle-hook atomically claims the deferred wakeup and discards the pending marker before starting the turn; if that turn then `409`s on a concurrent `/api/chat/start`, the wakeup is re-queued (idempotent per `process_id`) so a later teardown or the next-turn drain redelivers it instead of dropping it. (#2971) +- Server-initiated wakeup turn now renders live in an open tab by fanning a `server_turn_started` frame onto the per-session live-view channel; no more "needs manual refresh" after a server-driven wakeup. (#2968) +- SSE handlers now arm a 20s socket write deadline once per connection so a slow/backgrounded tab whose recv window fills no longer pins its HTTP worker thread indefinitely, applied uniformly to the 6 long-lived SSE endpoints (chat-stream, terminal, gateway, approval, clarify, session). (#2968) +- Focused background-task completion viewers still emit `/api/bg-task-complete-ack` for server cleanup/diagnostics while keeping the focused-session toast suppressed. (#2979) +- A failed session load (network error / server 4xx/5xx) no longer permanently silences the per-session SSE stream. `loadSession` stops the stream on entry but previously only restarted it on the success path; a mid-load failure left the on-screen session's stream closed, silently dropping `bg_task_complete` events until the user navigated again. The metadata-fetch error path now restarts the stream for the session still on screen, skipping the restart when a newer load is in flight or when the current session 404'd and self-healed away. (#2979) ## [v0.51.338] — 2026-06-09 — Release LB (saved prompts library) diff --git a/api/background_process.py b/api/background_process.py new file mode 100644 index 00000000..46801949 --- /dev/null +++ b/api/background_process.py @@ -0,0 +1,1248 @@ +"""Drain thread for terminal(notify_on_complete=true) agent wakeup. + +The hermes-agent ``tools.process_registry.ProcessRegistry`` exposes a thread-safe +``completion_queue`` (a ``queue.Queue``) that any background process pushes onto +when it exits or matches a ``watch_patterns`` rule. In the CLI and in the +gateway adapter this queue is drained by the host's main loop; in WebUI the +queue was never read, so the agent never woke up from a ``notify_on_complete`` +finish. This module restores that behavior. + +The drain thread: + 1. blocks on ``completion_queue.get()`` in a worker thread, + 2. looks up the WebUI session_id from ``PROCESS_SESSION_INDEX`` (keyed on + the per-process ``session_key`` env var captured at spawn time), + 3. formats a synthetic ``[IMPORTANT: ...]`` wakeup prompt, identical in + intent to ``cli._format_process_notification`` and + ``gateway.run._format_gateway_process_notification`` so the agent sees + the same payload regardless of host, + 4. emits a canonical ``bg_task_complete`` SSE event (plus a temporary + ``process_complete`` alias for the migration window) on the active + stream(s) for that session (DEMOTED to pure live-view — an open tab + streams the turn live), records a server-side marker in + ``PENDING_BG_TASK_COMPLETIONS``, + and — Option Z PIVOT — starts the agent wakeup turn **directly + server-side** when the session is idle (``_start_server_side_wakeup_turn`` + → ``routes.start_session_turn``). This needs NO browser round-trip, so + the closed-tab case works exactly like CLI / Telegram / gateway + self-wake. When a turn is already active the wakeup is NOT started here; + the ``PENDING_BG_TASK_COMPLETIONS`` marker is left for PR #2279's + next-turn drain (``api/streaming._drain_webui_process_notifications``). + +The marker is *not* required for delivery — it's a telemetry-style flag the +turn handler can read to know "this stream is a process_complete wakeup, not a +human-typed prompt". It also lets the PR #2279 next-turn drain deliver the +wakeup when a turn was active at completion time; the marker drains harmlessly +on the next turn for the session. + +Watch-pattern events share the same queue but produce a different SSE payload; +this module routes them to the same listener so the frontend's single +``process_complete`` handler can re-POST either flavor verbatim. +""" + +from __future__ import annotations + +import logging +import queue +import threading +import time +import uuid +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +_DRAIN_THREAD: Optional[threading.Thread] = None +_DRAIN_STOP = threading.Event() + +_REAPER_THREAD: Optional[threading.Thread] = None +_REAPER_STOP = threading.Event() +_REAPER_INTERVAL_SECS = 60.0 + +# T3: per-session coalesce gate for the public bg_task_complete SSE emit. +# The server-side wakeup path remains immediate; only the browser-observation +# frame is throttled so a burst of background task completions does not flood an +# open tab. The first emit for a session fires immediately, then any further +# emits inside the 1s window are payload-replaced and flushed after 1s of quiet. +_EMIT_COALESCE_WINDOW_SECS = 1.0 +_EMIT_COALESCE_LOCK = threading.Lock() +_LAST_EMIT_TS: dict[str, float] = {} +_PENDING_EMIT_PAYLOADS: dict[str, dict] = {} +_PENDING_EMIT_TIMERS: dict[str, threading.Timer] = {} + + +# ── Persistent per-session SSE channel (Option X) ────────────────────────── +# SESSION_CHANNELS maps WebUI session_id -> SessionChannel. Each channel owns +# zero or more queue.Queue subscribers (one per active EventSource tab) and +# is collected by ``_reaper_loop`` after the last subscriber drops + a grace +# period, or after the session has been idle for SESSION_CHANNEL_IDLE_TTL_SECS. +# +# Why a sibling registry to STREAMS: +# STREAMS is keyed on stream_id (one per agent turn) and is torn down by +# /api/chat/stream's `finally` when the turn ends. process_complete events +# from background processes that exit BETWEEN turns therefore have no live +# STREAMS channel to ride. SESSION_CHANNELS is keyed on session_id, lives +# across turns, and gives the frontend a stable subscription that survives +# stream_end / cancel / reconnect. +SESSION_CHANNELS: dict[str, "SessionChannel"] = {} +SESSION_CHANNELS_LOCK = threading.Lock() + + +class SessionChannel: + """A long-lived multi-subscriber SSE channel for one WebUI session. + + Subscribers are ``queue.Queue`` instances owned by the SSE route + handler — one per active EventSource (tab). ``emit`` broadcasts to every + live subscriber; subscribers whose buffer is full silently drop the + event (the tab will reconnect on disconnect and the SSE-level disconnect + detection will tear it down). + + Lifecycle: + - Created on demand by ``get_or_create_session_channel`` when the first + tab subscribes. + - ``subscribe`` / ``unsubscribe`` are refcount-style: zero subscribers + does NOT immediately collect the channel; the reaper waits a 60s + grace so a quick navigation away/back doesn't churn the registry. + - The reaper collects the channel when subscribers stay empty past the + grace period, OR when subscribers are empty AND ``created_at`` is + older than SESSION_CHANNEL_IDLE_TTL_SECS (zombie cap — applies only + when nobody is subscribed; a live subscriber keeps the channel even + past the idle TTL). + """ + + def __init__(self, session_id: str): + self.session_id = session_id + self._lock = threading.Lock() + self._subscribers: list[queue.Queue] = [] + now = time.time() + self.created_at = now + self.last_event_at = now + self.last_subscriber_drop_at: float | None = None + + def subscribe(self, maxsize: int = 16) -> queue.Queue: + q: queue.Queue = queue.Queue(maxsize=maxsize) + with self._lock: + self._subscribers.append(q) + # Cancel any pending subscribers-empty grace timer. + self.last_subscriber_drop_at = None + return q + + def unsubscribe(self, q: queue.Queue) -> None: + with self._lock: + try: + self._subscribers.remove(q) + except ValueError: + pass + if not self._subscribers: + self.last_subscriber_drop_at = time.time() + + def subscriber_count(self) -> int: + with self._lock: + return len(self._subscribers) + + def emit(self, event: str, data: Any) -> int: + """Broadcast (event, data) to all live subscribers. Returns delivered count.""" + delivered = 0 + with self._lock: + subs = list(self._subscribers) + self.last_event_at = time.time() + for q in subs: + try: + q.put_nowait((event, data)) + delivered += 1 + except queue.Full: + # Slow tab: drop this event for that tab. SSE-level disconnect + # detection will eventually tear the connection down and the + # browser will reconnect, replaying the live stream from + # whatever fires next. process_complete is intrinsically + # idempotent (frontend dedupes by ``(session_id, event_id)`` + # using a small ring-buffer in static/messages.js — see the + # bg_task_complete consumer-side dedupe introduced in PR #2971). + logger.debug("SessionChannel emit: subscriber buffer full, dropping") + except Exception: + logger.debug("SessionChannel emit failed", exc_info=True) + return delivered + + def reaper_should_collect(self, now: float) -> bool: + """True when the reaper should remove this channel. + + Two collection conditions (per Option X spec): + 1. Subscribers empty AND last_subscriber_drop_at is older than + SESSION_CHANNEL_SUBSCRIBER_GRACE_SECS (normal teardown). + 2. created_at older than SESSION_CHANNEL_IDLE_TTL_SECS AND + subscribers empty (zombie cap — survived too long). + """ + from api import config as _cfg + + with self._lock: + sub_count = len(self._subscribers) + drop_at = self.last_subscriber_drop_at + created_at = self.created_at + + if sub_count > 0: + # Live subscriber — never collect, even past idle TTL (a tab is + # genuinely listening). The browser will close on its own. + return False + # No subscribers — check grace period. + grace = float(getattr(_cfg, "SESSION_CHANNEL_SUBSCRIBER_GRACE_SECS", 60)) + if drop_at is not None and (now - drop_at) >= grace: + return True + # Hard cap on lifetime (even if subscribers oscillated): if created + # long ago AND nobody's subscribed right now, sweep. + ttl = float(getattr(_cfg, "SESSION_CHANNEL_IDLE_TTL_SECS", 14400)) + if (now - created_at) >= ttl: + return True + return False + + +def get_or_create_session_channel(session_id: str) -> SessionChannel: + """Return the channel for ``session_id``, creating it on first access.""" + with SESSION_CHANNELS_LOCK: + ch = SESSION_CHANNELS.get(session_id) + if ch is None: + ch = SessionChannel(session_id) + SESSION_CHANNELS[session_id] = ch + return ch + + +def get_session_channel(session_id: str) -> Optional[SessionChannel]: + """Return an existing channel or None — does NOT auto-create.""" + with SESSION_CHANNELS_LOCK: + return SESSION_CHANNELS.get(session_id) + + +def subscribe_to_session_channel( + session_id: str, maxsize: int = 16 +) -> tuple["SessionChannel", "queue.Queue"]: + """Atomically get-or-create the channel for ``session_id`` AND register a + subscriber on it, both under ``SESSION_CHANNELS_LOCK``. + + Closes a TOCTOU race flagged on PR #2971: callers that did + ``ch = get_or_create_session_channel(sid); q = ch.subscribe()`` released + ``SESSION_CHANNELS_LOCK`` between the two steps. The reaper acquires that + same lock and evaluates+collects channels entirely within it + (``_reaper_loop``), so a previously-idle channel with 0 subscribers past + the grace/TTL window could be collected in the gap — leaving the SSE + handler subscribed to a channel no longer in ``SESSION_CHANNELS``. Future + ``bg_task_complete`` emits resolve the session via ``get_session_channel`` + and reach a different (or absent) channel, so the orphaned subscriber's + queue never fills: keepalives keep flowing, ``onerror`` never fires, no + auto-reconnect, and only a manual refresh recovers. + + Holding ``SESSION_CHANNELS_LOCK`` across both the get-or-create and the + ``subscribe`` makes the two steps indivisible w.r.t. the reaper: the reaper + cannot run its critical section concurrently, and the next time it does the + channel already has ``sub_count >= 1`` so ``reaper_should_collect`` refuses + to collect it. + + Lock order is ``SESSION_CHANNELS_LOCK`` → ``SessionChannel._lock`` (taken + inside ``subscribe``), identical to the order the reaper uses + (``SESSION_CHANNELS_LOCK`` → ``reaper_should_collect`` → ``_lock``), so no + new lock-ordering hazard is introduced. ``emit`` only takes ``_lock``. + + Returns ``(channel, queue)``. The caller still owns the subscriber slot and + MUST ``channel.unsubscribe(queue)`` on every exit path. + """ + with SESSION_CHANNELS_LOCK: + ch = SESSION_CHANNELS.get(session_id) + if ch is None: + ch = SessionChannel(session_id) + SESSION_CHANNELS[session_id] = ch + q = ch.subscribe(maxsize=maxsize) + return ch, q + + +def active_stream_id_for_session(session_id: str) -> Optional[str]: + """Return the stream_id of the live run for *session_id*, or None. + + Used by the per-session SSE handler's on-subscribe recovery: the + ``server_turn_started`` fan-out in ``routes.start_session_turn`` is a + fire-and-forget broadcast with NO replay buffer (SessionChannel.emit + drops to whoever is subscribed *at that instant*). A tab whose + ``/api/session/stream`` EventSource is momentarily absent at the emit + instant — a transient SSE drop, a reverse-proxy idle-timeout, or browser + connection-pool starvation — misses the frame permanently and the + server-initiated wakeup turn never renders live (the user must hard- + refresh). The server-side wakeup itself still ran and persisted; only the + live-view was lost. The handler replays a synthetic ``server_turn_started`` + to a freshly-subscribed tab using this lookup so the open tab self-heals. + + Keys on ACTIVE_RUNS (worker-lifecycle registry) — the same source + ``_session_has_active_turn`` / ``_emit_to_session_streams`` already trust + to map a stream back to its owning session. Returns the first matching + stream_id (a session has at most one live run; cancel/reconnect can + briefly hold two — either is a valid attach target, the frontend dedupes + by stream_id). + """ + from api import config as _cfg + + try: + with _cfg.ACTIVE_RUNS_LOCK: + for _stream_id, meta in (_cfg.ACTIVE_RUNS or {}).items(): + if isinstance(meta, dict) and meta.get("session_id") == session_id: + return str(_stream_id) + except Exception: + logger.debug( + "active_stream_id_for_session lookup failed for %s", + session_id, + exc_info=True, + ) + return None + + +def _reaper_loop() -> None: + logger.info("SessionChannel reaper thread started") + while not _REAPER_STOP.is_set(): + try: + now = time.time() + collected: list[str] = [] + with SESSION_CHANNELS_LOCK: + for sid, ch in list(SESSION_CHANNELS.items()): + if ch.reaper_should_collect(now): + SESSION_CHANNELS.pop(sid, None) + collected.append(sid) + if collected: + # Prune the per-session coalesce timestamp map for any collected + # session so _LAST_EMIT_TS does not grow one permanent entry per + # session that ever fired a bg task (greptile flag). The pending + # payload/timer maps self-clean when their timers fire, but + # _LAST_EMIT_TS is only ever written, never deleted — sweep it + # here alongside the channel it belongs to. A session that fires + # again after collection simply re-seeds its entry (first emit in + # the new window fires immediately, which is correct). + with _EMIT_COALESCE_LOCK: + for sid in collected: + _LAST_EMIT_TS.pop(sid, None) + logger.debug("SessionChannel reaper collected: %s", collected) + except Exception: + logger.warning("SessionChannel reaper iteration failed", exc_info=True) + # Wait but wake up promptly on stop. + if _REAPER_STOP.wait(_REAPER_INTERVAL_SECS): + break + + +def start_session_channel_reaper() -> bool: + """Start the SessionChannel reaper thread. Idempotent; returns True on first start.""" + global _REAPER_THREAD + if _REAPER_THREAD is not None and _REAPER_THREAD.is_alive(): + return False + _REAPER_STOP.clear() + _REAPER_THREAD = threading.Thread( + target=_reaper_loop, + name="hermes-webui-session-channel-reaper", + daemon=True, + ) + _REAPER_THREAD.start() + return True + + +def stop_session_channel_reaper(timeout: float = 2.0) -> None: + _REAPER_STOP.set() + th = _REAPER_THREAD + if th is not None and th.is_alive(): + th.join(timeout=timeout) + + +def _truncate(text: str, limit: int) -> str: + if text is None: + return "" + s = str(text) + if len(s) <= limit: + return s + return s[:limit] + "\n…(truncated)" + + +def format_wakeup_prompt(evt: dict) -> str: + """Build the synthetic [IMPORTANT: …] message the agent will see. + + Mirrors ``cli._format_process_notification`` so wakeup payloads look the + same in CLI and WebUI sessions. + """ + evt_type = evt.get("type", "completion") + sid = evt.get("session_id", "unknown") + cmd = evt.get("command", "unknown") + if evt_type == "watch_disabled": + return f"[IMPORTANT: {evt.get('message', '')}]" + if evt_type == "watch_match": + pat = evt.get("pattern", "?") + out = _truncate(evt.get("output", ""), 4000) + sup = evt.get("suppressed", 0) + body = ( + f"[IMPORTANT: Background process {sid} matched watch pattern \"{pat}\".\n" + f"Command: {cmd}\n" + f"Matched output:\n{out}" + ) + if sup: + body += f"\n({sup} earlier matches were suppressed by rate limit)" + return body + "]" + # Default: completion event + exit_code = evt.get("exit_code", "?") + out = _truncate(evt.get("output", ""), 4000) + return ( + f"[IMPORTANT: Background process {sid} completed (exit_code={exit_code}).\n" + f"Command: {cmd}\n" + f"Output:\n{out}]" + ) + + +def _build_payload(evt: dict, session_id: str) -> dict: + """Build the SSE data payload. + + Shape per maintainer decision on PR #2242 (R2 §Q1): + ``{session_id, task_id, completed_at, summary?, event_id}`` + Minimal by design — consumers re-fetch task detail by ``task_id`` if + they need ``command`` / ``exit_code`` / ``stdout_preview`` etc. + + - ``task_id``: the background process id (registry uuid). Stable across + the process's lifetime; was previously surfaced as ``process_id``. + - ``completed_at``: float wall-clock seconds; was ``emitted_at``. + - ``summary``: optional one-liner derived from the completion event when + available (e.g. ``[IMPORTANT: …]`` synthetic body's first line); omitted + otherwise to honour "keep the payload minimal". + - ``event_id``: server-generated uuid hex — per-emit, so a re-emit for the + same ``task_id`` (shouldn't happen today, but the spec is forward-looking) + still produces a distinct id. The cross-A/B dedupe stays keyed on + ``task_id`` (it prevents double-emit at the source, which is the right + layer); the WebUI consumer-side ring buffer (PR (b)) keys on + ``(session_id, event_id)`` to dedupe across reconnects. + """ + # ProcessRegistry completion events use the field name ``session_id`` for + # the process id. Alias it locally before exposing it as payload ``task_id`` + # to avoid confusing that wire-format name with the WebUI session id. + process_id = str(evt.get("session_id") or "") + payload: dict[str, Any] = { + "session_id": str(session_id), + "task_id": process_id, + "completed_at": time.time(), + "event_id": uuid.uuid4().hex, + } + # Best-effort optional summary: the first non-empty line of the synthetic + # wakeup body, trimmed. Omitted entirely when nothing useful is available. + try: + wakeup_body = format_wakeup_prompt(evt) + if wakeup_body: + # Strip leading "[IMPORTANT: " marker noise — take the first + # informative line, cap length. + first_line = next( + ( + ln.strip().lstrip("[").rstrip("]").strip() + for ln in wakeup_body.splitlines() + if ln.strip() + ), + "", + ) + if first_line: + payload["summary"] = _truncate(first_line, 200) + except Exception: + # Summary is optional; never let its derivation block the emit. + logger.debug("summary derivation failed", exc_info=True) + return payload + + +def _emit_to_session_streams(session_id: str, event: str, data: dict) -> int: + """Push (event, data) to every active SSE channel for *session_id*. + + Streams in WebUI are keyed by ``stream_id`` (not session_id) — a single + session can have at most one active stream at a time, but cancel/reconnect + flows can briefly hold two. We push to every channel whose tracked + ``session_id`` matches; the channel implementation broadcasts to all live + subscribers and buffers when offline. + """ + from api import config as _cfg + + emitted = 0 + # Snapshot ACTIVE_RUNS under its lock before STREAMS_LOCK so owner_sid + # lookups are consistent without nesting the two independent locks. + if hasattr(_cfg, "ACTIVE_RUNS") and hasattr(_cfg, "ACTIVE_RUNS_LOCK"): + with _cfg.ACTIVE_RUNS_LOCK: + active_runs_snapshot: dict = dict(_cfg.ACTIVE_RUNS) + elif hasattr(_cfg, "ACTIVE_RUNS"): + active_runs_snapshot = dict(_cfg.ACTIVE_RUNS) + else: + active_runs_snapshot = {} + with _cfg.STREAMS_LOCK: + items = list(_cfg.STREAMS.items()) + for stream_id, channel in items: + meta = active_runs_snapshot.get(stream_id) + owner_sid = (meta or {}).get("session_id") if isinstance(meta, dict) else None + # Copilot review #3: skip non-matching and owner-unknown STREAMS + # channels. Cross-turn delivery is handled by SESSION_CHANNELS below. + if owner_sid != session_id: + continue + try: + channel.put_nowait((event, data)) + emitted += 1 + except Exception: + logger.debug("process_complete emit failed for stream %s", stream_id, exc_info=True) + # Option X: also emit to the persistent per-session SSE channel. This is + # the path that survives between turns (when STREAMS is torn down). We + # keep the STREAMS emit above as defense-in-depth — if a turn IS active + # the frontend dedupes by process_id, so a double-delivery is harmless. + ch = get_session_channel(session_id) + if ch is not None: + try: + delivered = ch.emit(event, data) + emitted += delivered + except Exception: + logger.debug("SessionChannel emit failed for session %s", session_id, exc_info=True) + return emitted + + +def _emit_bg_task_complete_events_now(session_id: str, payload: dict) -> int: + """Emit the canonical bg_task_complete event and temporary legacy alias.""" + # T1 emit rename: the canonical event name is now ``bg_task_complete`` + # (per maintainer decision on PR #2242). Until PR (b) lands the new + # consumer-side listener, we ALSO emit the legacy ``process_complete`` + # name so any in-flight WebUI build (subscribed to the old listener) + # still receives the wakeup. PR (b) will: + # 1. Add `bg_task_complete` listeners on the WebUI side. + # 2. Remove this dual-emit shim (drop the legacy alias). + # Both emits carry the SAME trimmed payload + the SAME event_id, so a + # consumer that ever sees both can dedupe by ``event_id``. + # + # Greptile P2: hand each emit its OWN shallow copy of the payload. The + # same dict object would otherwise be referenced by every subscriber queue + # across STREAMS and SESSION_CHANNELS for BOTH event names; a downstream + # consumer that mutates it in place would silently corrupt all other + # concurrent consumers' views. Shallow copies are sufficient — the payload + # is a flat trimmed dict of scalars. + return ( + _emit_to_session_streams(session_id, "bg_task_complete", dict(payload)) + + _emit_to_session_streams(session_id, "process_complete", dict(payload)) + ) + + +def _flush_coalesced_bg_task_complete(session_id: str) -> None: + """Timer callback: flush the latest pending payload for one session.""" + payload: dict | None = None + with _EMIT_COALESCE_LOCK: + payload = _PENDING_EMIT_PAYLOADS.pop(session_id, None) + _PENDING_EMIT_TIMERS.pop(session_id, None) + if payload is not None: + _LAST_EMIT_TS[session_id] = time.time() + if payload is None: + return + try: + _emit_bg_task_complete_events_now(session_id, payload) + except Exception: + logger.debug( + "coalesced bg_task_complete flush failed for session %s", + session_id, + exc_info=True, + ) + + +def _emit_bg_task_complete_events_coalesced(session_id: str, payload: dict) -> int: + """Per-session 1s coalesce gate around the bg_task_complete dual emit. + + The first payload for a session emits immediately. If another payload for + that session arrives within ``_EMIT_COALESCE_WINDOW_SECS`` of the last emit, + replace the pending payload and reset the quiet timer. The deferred flush + therefore uses the latest payload from a burst. + """ + if not session_id: + return 0 + should_emit_now = False + now = time.time() + with _EMIT_COALESCE_LOCK: + last = _LAST_EMIT_TS.get(session_id) + has_pending = session_id in _PENDING_EMIT_TIMERS + if last is None or (now - last) >= _EMIT_COALESCE_WINDOW_SECS: + _LAST_EMIT_TS[session_id] = now + should_emit_now = True + if has_pending: + _PENDING_EMIT_PAYLOADS.pop(session_id, None) + old_timer = _PENDING_EMIT_TIMERS.pop(session_id, None) + if old_timer is not None: + try: + old_timer.cancel() + except Exception: + logger.debug( + "coalesced bg_task_complete timer cancel failed for session %s", + session_id, + exc_info=True, + ) + else: + _PENDING_EMIT_PAYLOADS[session_id] = payload + + if not should_emit_now: + old_timer = _PENDING_EMIT_TIMERS.get(session_id) + if old_timer is not None: + try: + old_timer.cancel() + except Exception: + logger.debug( + "coalesced bg_task_complete timer cancel failed for session %s", + session_id, + exc_info=True, + ) + timer = threading.Timer( + _EMIT_COALESCE_WINDOW_SECS, + _flush_coalesced_bg_task_complete, + args=(session_id,), + ) + timer.daemon = True + _PENDING_EMIT_TIMERS[session_id] = timer + timer.start() + + if should_emit_now: + return _emit_bg_task_complete_events_now(session_id, payload) + return 0 + + +# ── Coupling contract: agent ProcessRegistry cross-A/B dedupe key ────────── +# This WebUI drain (B) and the merged upstream PR #2279 next-turn drain (A) +# dedupe a process_id against a SINGLE shared key inside the agent's +# ``tools.process_registry.ProcessRegistry``: +# +# * READ side: the PUBLIC ``is_completion_consumed(process_id)`` method +# (used in ``_process_one`` above) — stable public API. +# * WRITE side: there is NO public ``mark_completion_consumed`` upstream, so +# B must reach into the registry's private ``_completion_consumed`` set +# (guarded by its private ``_lock``) to set the shared marker A reads. +# +# That private WRITE coupling is what Copilot review #2242 comment #4 flagged. +# The long-term fix is an upstream PUBLIC ``mark_completion_consumed`` — see +# the test ``test_registry_completion_consumed_contract`` which fails CI LOUD +# the moment ``_completion_consumed`` / ``_lock`` / ``is_completion_consumed`` +# is renamed or retyped upstream, instead of a future rename silently +# reintroducing the double-wakeup bug. ``_mark_registry_completion_consumed`` +# narrows the exception handling so a rename is logged at ERROR (visible in +# errors.log + monitoring) rather than swallowed by a broad ``except`` at +# DEBUG. ImportError stays best-effort (the registry is legitimately absent in +# non-agent unit-test contexts; this module's own locked +# ``BG_TASK_COMPLETE_EVENTS_SEEN`` gate still dedupes B's own duplicates there). +_REGISTRY_CONSUMED_CONTRACT = ("_lock", "_completion_consumed", "is_completion_consumed") + + +def _mark_registry_completion_consumed(process_id: str) -> None: + """Set the shared cross-A/B dedupe marker on the agent ProcessRegistry. + + Couples to ``ProcessRegistry`` privates (``_lock`` / + ``_completion_consumed``) because no public ``mark_completion_consumed`` + exists upstream (the read side uses the public + ``is_completion_consumed``). A future upstream rename must FAIL LOUD, not + silently reintroduce the double-wakeup: an ``AttributeError`` / ``TypeError`` + from the private access is logged at ERROR with a contract-violation + message (and ``test_registry_completion_consumed_contract`` breaks CI at + test time). Only ``ImportError`` is treated as best-effort/expected (the + registry is absent in pure unit-test contexts). + """ + try: + from tools.process_registry import process_registry as _pr + except ImportError: + # Agent registry not importable (e.g. isolated unit test) — B's own + # locked BG_TASK_COMPLETE_EVENTS_SEEN gate still prevents this module's + # duplicates; cross-A/B dedupe is moot when A isn't running either. + logger.debug( + "tools.process_registry not importable; skipping shared " + "completion-consumed marker (best-effort, expected off-agent)", + exc_info=True, + ) + return + try: + lock = _pr._lock + consumed = _pr._completion_consumed + except AttributeError: + logger.error( + "ProcessRegistry coupling contract VIOLATED: expected private " + "attrs %s for cross-A/B wakeup dedupe are missing — an upstream " + "rename has broken the shared marker; process_complete wakeups may " + "now double-fire. A public mark_completion_consumed() upstream is " + "the durable fix (Copilot #2242 review #4).", + _REGISTRY_CONSUMED_CONTRACT, + exc_info=True, + ) + return + try: + with lock: + consumed.add(process_id) + except (AttributeError, TypeError): + logger.error( + "ProcessRegistry coupling contract VIOLATED: _lock/" + "_completion_consumed changed shape (not a Lock / not a set) — " + "cross-A/B wakeup dedupe is broken; wakeups may double-fire. " + "Upstream public mark_completion_consumed() is the durable fix " + "(Copilot #2242 review #4).", + exc_info=True, + ) + + +# ── xsession wakeup misroute defense-in-depth (Option 3) ─────────────────── +# Option 1 (api/streaming._set_turn_session_identity) is the ROOT fix: it binds +# the per-turn session identity to a contextvar so a notify_on_complete spawn +# can no longer capture a concurrent turn's process-global env. Option 3 is an +# INDEPENDENT completion-time safety net at the wakeup-routing layer: even if +# some future regression reintroduces a capture race, a positively-detected +# mismatch must not wake the wrong session. +# +# The proc->owner link the WebUI drain trusts is ProcessSession.session_key, +# which the terminal tool captured from the (historically racy) env at spawn. +# An env-IMMUNE spawn-time owner would be the authoritative cross-check, but +# adding such a field to the core ProcessSession is out of scope for this +# WebUI-only change. So this resolver is forward-compatible by DUCK-TYPING: +# if a future core grows an env-immune spawn-owner attribute (any of the +# names below) AND it positively disagrees with the session_key-resolved +# target, re-route to the env-immune owner and log ERROR. When the owner is +# absent/empty/unknown (today's core, cron/CLI processes sharing the +# registry, pre-Option-1 spawns) it is a PURE PASS-THROUGH — it never +# suppresses a legitimate Option Z wakeup on uncertainty (Option Z must keep +# working). +_ENV_IMMUNE_OWNER_ATTRS = ("spawn_session_id", "owner_session_id", "turn_session_id") + + +def _env_immune_spawn_owner(proc_session) -> str: + """Return the env-immune spawn-time owner sid from the ProcessSession, or + "" when none is available (the only contract today; forward-compatible).""" + if proc_session is None: + return "" + for attr in _ENV_IMMUNE_OWNER_ATTRS: + try: + val = getattr(proc_session, attr, "") + except Exception: + val = "" + if val: + return str(val) + return "" + + +def _resolve_wakeup_target( + *, + process_id: str, + session_key_resolved_sid: str, + proc_session, +) -> str: + """Cross-check the session_key-resolved wakeup target against the + env-immune spawn owner. Returns the sid the server-side wakeup turn should + actually target. + + - Owner unknown/empty -> pass-through (return session_key_resolved_sid). + - Owner == resolved -> pass-through (the normal + post-Option-1 case). + - Owner != resolved -> POSITIVE mismatch: log ERROR and RE-ROUTE to the + env-immune owner (do NOT wake the wrong session — this is the exact + agent.log:6632 cross-session misroute). + """ + resolved = str(session_key_resolved_sid or "") + owner = _env_immune_spawn_owner(proc_session) + if not owner or owner == resolved: + return resolved + logger.error( + "xsession wakeup misroute BLOCKED (Option 3 safety net): process %r " + "session_key resolved to session %r but the env-immune spawn owner " + "is %r — re-routing the server-side wakeup to the true owner. This " + "means a per-turn session-identity capture race occurred upstream " + "(Option 1 should have prevented it); investigate streaming.py " + "_set_turn_session_identity coverage.", + process_id, resolved, owner, + ) + return owner + + +def _process_one(evt: dict) -> None: + """Route a single completion_queue event to the matching WebUI session.""" + from api import config as _cfg + + # Hoist the process-registry import once per event: it was imported in + # three separate blocks below (session_key recovery, env-immune owner + # cross-check, upstream is_completion_consumed dedupe) on every completion + # event, paying repeated import-system overhead. Single local rebind keeps + # the ImportError fallback contract (process_registry may be missing in + # cut-down vendoring) while collapsing to one lookup per call. + try: + from tools.process_registry import process_registry as _process_registry + except Exception: + _process_registry = None + + process_id = str(evt.get("session_id") or "") + session_key = str(evt.get("session_key") or "") + # Root-cause fix (t_0f447014): the notify_on_complete completion event + # enqueued by ProcessRegistry._move_to_finished() carries NO "session_key" + # field — only the watch_match enqueue includes one. Without it the old + # `evt.get("session_key") or process_id` fell back to the process id + # ("proc_xxxx"), which is never a PROCESS_SESSION_INDEX key (only + # webui_session_id -> webui_session_id is registered at chat-start), so + # every wakeup was silently dropped here and the frontend never POSTed an + # ack. Recover the spawn-time session_key from the process registry's + # ProcessSession: the terminal tool captured it synchronously at spawn + # (while the turn's env was active), so it survives the turn-end env + # restore and is the WebUI session_id for WebUI-spawned processes. + if not session_key and process_id: + try: + if _process_registry is not None: + _ps = _process_registry.get(process_id) + if _ps is not None and getattr(_ps, "session_key", ""): + session_key = str(_ps.session_key) + except Exception: + logger.debug( + "session_key recovery from process registry failed for %r", + process_id, + exc_info=True, + ) + if not session_key: + logger.debug( + "process_complete drop: no recoverable session_key for process_id=%r", + process_id, + ) + return + with _cfg.PROCESS_SESSION_INDEX_LOCK: + session_id = _cfg.PROCESS_SESSION_INDEX.get(session_key) + if not session_id: + # No mapping — could be a cron/gateway process that uses the same + # registry but a non-WebUI session_key. Ignore. + logger.debug("process_complete drop: no session mapping for key=%r", session_key) + return + # ── xsession wakeup misroute defense-in-depth (Option 3) ────────────── + # session_id above came from PROCESS_SESSION_INDEX.get(session_key), and + # session_key was captured by the terminal tool from the (historically + # racy) process-global env at spawn. Option 1 binds the per-turn identity + # to a contextvar so that capture is no longer racy — but as an INDEPENDENT + # safety net, cross-check the resolved target against the env-immune + # spawn owner (when the core ProcessSession exposes one). On a positive + # mismatch this re-routes the wakeup (and the live-view emit + dedupe + # markers below) to the TRUE owner instead of waking the wrong session. + # Pure pass-through when no env-immune owner is available (today's core, + # cron/CLI procs, pre-Option-1 spawns) — never suppresses a valid wakeup. + try: + _ps_xs = _process_registry.get(process_id) if (_process_registry is not None and process_id) else None + except Exception: + _ps_xs = None + session_id = _resolve_wakeup_target( + process_id=process_id, + session_key_resolved_sid=session_id, + proc_session=_ps_xs, + ) + # ── Idempotency vs the REAL merged upstream #2279 (shared dedupe key) ── + # The real merged #2279 next-turn drain + # (api/streaming._drain_webui_process_notifications) dedupes ONLY via + # process_registry.is_completion_consumed() / _completion_consumed — it + # does NOT populate BG_TASK_COMPLETE_EVENTS_SEEN (that set is ours-original + # and private to this module). So the cross-A/B shared dedupe contract is + # process_registry._completion_consumed, NOT BG_TASK_COMPLETE_EVENTS_SEEN. + # If the upstream A-drain already delivered this process_id (A-first + # order), it marked _completion_consumed; B must early-return here or it + # would double-fire a wakeup. This guard aligns our B-drain to the real + # upstream key (verified against origin/master streaming.py). + if process_id: + try: + if _process_registry is not None and _process_registry.is_completion_consumed(process_id): + return + except Exception: + logger.debug( + "is_completion_consumed check failed on B drain; " + "falling back to BG_TASK_COMPLETE_EVENTS_SEEN gate", + exc_info=True, + ) + # Secondary (ours-original) idempotency: if we've already emitted for this + # (session_id, process_id) pair via THIS module, skip the duplicate. Two + # _move_to_finished() callers (kill_process racing the reader thread) can + # occasionally enqueue twice despite the process_registry guard. + with _cfg.BG_TASK_COMPLETE_EVENTS_SEEN_LOCK: + seen = _cfg.BG_TASK_COMPLETE_EVENTS_SEEN.setdefault(session_id, set()) + if process_id and process_id in seen: + return + if process_id: + seen.add(process_id) + payload = _build_payload(evt, session_id) + _emit_bg_task_complete_events_coalesced(session_id, payload) + _cfg.PENDING_BG_TASK_COMPLETIONS.add(session_id) + # Mark the event consumed in the agent's process registry so the REAL + # merged PR #2279's next-turn drain + # (api/streaming._drain_webui_process_notifications) treats this process_id + # as already-delivered and does not re-fire a wakeup (B-first order). + # This is the SHARED upstream dedupe key (see _mark_registry_completion_ + # consumed for the coupling contract + why a future rename now fails loud). + if process_id: + _mark_registry_completion_consumed(process_id) + + # ── Option Z (PRIMARY): server-side wakeup, NO browser round-trip ────── + # The SSE emit above is now demoted to a pure live-view layer (an open tab + # streams the turn live via the per-session SSE channel). The ACTUAL agent + # wakeup is started HERE, server-side, so a CLOSED tab still gets the turn + # — parity with how CLI / Telegram / gateway self-wake from a + # notify_on_complete completion. This is the fix for the structural flaw: + # "fire a long background task, close the tab, come back later" is THE + # primary background-task use case and browser-mediated wakeup could never + # serve it. + # + # - turn ACTIVE → do NOT start a turn. Leave the PENDING_PROCESS_ + # COMPLETIONS marker so PR #2279's next-turn drain + # (api/streaming._drain_webui_process_notifications) injects the wakeup + # when the active turn ends. (That path already works when a turn is + # active — it was never the gap.) + # - turn IDLE → start a new server-side turn directly with wakeup_prompt + # as the user message (the real gap Option Z closes). + # + # Idempotency is already guaranteed above: BG_TASK_COMPLETE_EVENTS_SEEN + + # the registry _completion_consumed marker mean this process_id reached + # here at most once, so the wakeup turn starts at most once. + try: + # ``wakeup_prompt`` is server-internal state used only by the + # Option Z server-side wakeup; it was previously surfaced on the + # SSE payload but T1 trimmed the payload to the minimal shape + # `{session_id, task_id, completed_at, summary?, event_id}`, so + # we derive the prompt directly from the evt here (same source the + # prior _build_payload used). + wakeup_prompt = format_wakeup_prompt(evt).strip() + if wakeup_prompt: + if _session_has_active_turn(session_id): + # Defer-path fix: persist the prompt so a turn-teardown + # idle-hook can redeliver it once the session goes idle. + # The OLD behavior only logged + left a bare + # PENDING_BG_TASK_COMPLETIONS session flag; the prompt was + # discarded and the next-turn drain reads completion_queue + # (already emptied by THIS drain thread), so for an + # autonomous agent with no next user turn the wakeup was + # lost forever. process_id is already in + # BG_TASK_COMPLETE_EVENTS_SEEN + the registry + # _completion_consumed marker (set above), so persisting it + # here cannot cause a double-fire — the atomic claim in + # ``claim_deferred_wakeups`` guarantees exactly one delivery. + record_deferred_wakeup(session_id, process_id, wakeup_prompt) + logger.debug( + "server-side wakeup deferred: turn active for session %s " + "(persisted for turn-teardown idle-hook redelivery)", + session_id, + ) + else: + # Idle-path sibling of the F1 (409/teardown) fix: pass + # ``process_id`` so that if this idle wakeup's daemon thread + # loses the per-session lock race and 409s, the re-defer in + # ``_start_server_side_wakeup_turn`` records the entry WITH its + # process_id — keeping the ``record_deferred_wakeup`` dedup + # guard (``if process_id and any(...)``) live on that re-defer + # path so a second 409 race cannot accumulate a duplicate + # deferred entry (which would deliver the same wakeup twice). + _start_server_side_wakeup_turn( + session_id, wakeup_prompt, process_id=process_id + ) + except Exception: + logger.warning( + "server-side wakeup dispatch failed for session %s", session_id, exc_info=True + ) + + +def record_deferred_wakeup(session_id: str, process_id: str, wakeup_prompt: str) -> None: + """Persist a deferred process-completion wakeup for later redelivery. + + Called from ``_process_one`` when a completion arrives while a turn is + active (the Option Z drain branch cannot start a turn — it would 409). + The turn-teardown idle-hook (``drain_deferred_wakeups_for_session``) + redelivers it once the session goes idle, OR the PR #2279 next-turn drain + claims it if a user turn comes first. Whoever claims first wins (atomic + pop in ``claim_deferred_wakeups``); the other finds nothing. + + Idempotent per process_id: if the same process_id is already queued for + this session (kill_process racing the reader thread), it is not appended + twice. Best-effort — never raises into the drain loop. + """ + if not session_id or not wakeup_prompt: + return + from api import config as _cfg + + try: + with _cfg.DEFERRED_PROCESS_WAKEUPS_LOCK: + entries = _cfg.DEFERRED_PROCESS_WAKEUPS.setdefault(session_id, []) + if process_id and any( + e.get("process_id") == process_id for e in entries + ): + return + entries.append( + {"process_id": process_id, "wakeup_prompt": wakeup_prompt} + ) + except Exception: + logger.debug( + "record_deferred_wakeup failed for session %s", session_id, exc_info=True + ) + + +def claim_deferred_wakeups(session_id: str) -> list[dict]: + """Atomically remove and return all deferred wakeups for *session_id*. + + The single-delivery guarantee for the defer path: the dict ``pop`` under + ``DEFERRED_PROCESS_WAKEUPS_LOCK`` means whichever caller runs first + (turn-teardown idle-hook OR PR #2279 next-turn drain) gets the entries and + delivers them; every subsequent caller gets ``[]``. This is what makes the + teardown hook idempotent with the next-turn drain (no double-fire) AND + prevents a wakeup loop (the wakeup turn's own teardown re-runs the hook, + finds nothing already-claimed → no re-fire). + """ + if not session_id: + return [] + from api import config as _cfg + + try: + with _cfg.DEFERRED_PROCESS_WAKEUPS_LOCK: + return _cfg.DEFERRED_PROCESS_WAKEUPS.pop(session_id, []) or [] + except Exception: + logger.debug( + "claim_deferred_wakeups failed for session %s", session_id, exc_info=True + ) + return [] + + +def drain_deferred_wakeups_for_session(session_id: str) -> int: + """Turn-teardown idle-hook: redeliver deferred wakeups once idle. + + Called from ``api/streaming`` right AFTER ``unregister_active_run`` so + ``_session_has_active_turn`` no longer counts the just-ended stream. This + makes the active-at-completion case symmetric with the idle-at-completion + case: idle now → fire now (Option Z idle branch); busy now → fire here + when the turn ends and the session goes idle. + + Multi-stream / cancel-reconnect guard: if ANY other ACTIVE_RUNS row still + exists for this session (a second stream from cancel/reconnect), the + session is NOT yet idle — leave the deferred entries untouched so a later + teardown (or the next-turn drain) delivers them. Only the teardown of the + LAST active stream for the session claims + fires. + + Returns the number of wakeup turns started (0 when nothing pending or the + session is still busy). Best-effort — never raises into the streaming + teardown thread; the actual turn is started on the same throwaway daemon + thread the idle branch uses, so this never blocks teardown. + """ + if not session_id: + return 0 + from api import config as _cfg + + try: + # Multi-stream guard: only fire when the session is TRULY idle. + if _session_has_active_turn(session_id): + return 0 + # Peek without claiming: avoid taking the entries then discovering + # there is nothing to do under contention. + with _cfg.DEFERRED_PROCESS_WAKEUPS_LOCK: + if not _cfg.DEFERRED_PROCESS_WAKEUPS.get(session_id): + return 0 + # Atomic claim — exactly one caller gets the entries. + entries = claim_deferred_wakeups(session_id) + if not entries: + return 0 + # The session-level PENDING marker is server-internal telemetry; the + # real delivery is the prompt(s) we just claimed. Discard it now that + # the deferred wakeups are owned by this teardown. + try: + _cfg.PENDING_BG_TASK_COMPLETIONS.discard(session_id) + except Exception: + logger.debug( + "PENDING discard failed for session %s", session_id, exc_info=True + ) + started = 0 + # Greptile P1 fix: do NOT fire one daemon-threaded wakeup per entry in + # a tight loop. Each ``_start_server_side_wakeup_turn`` spawns a daemon + # thread that races for the per-session agent lock; only ONE can win, + # the rest 409. Since we already claimed + popped every entry (line + # ~938) and discarded the PENDING marker, the losers' prompts would be + # permanently lost. Instead: start exactly the FIRST prompt, and + # re-defer the remaining entries so each subsequent turn-teardown + # (or next-turn drain) delivers the next one — one wakeup per turn, + # which matches the single-prompt-per-turn design and the + # BG_TASK_COMPLETE_EVENTS_SEEN dedup (no double-fire). + leftover = [e for e in entries if str((e or {}).get("wakeup_prompt") or "").strip()] + if leftover: + first = leftover[0] + # Re-defer entries 2..N BEFORE starting the first turn, so they are + # already persisted if the first wakeup's own teardown re-runs this + # hook and tries to claim them. + for entry in leftover[1:]: + record_deferred_wakeup( + session_id, + str((entry or {}).get("process_id") or ""), + str((entry or {}).get("wakeup_prompt") or "").strip(), + ) + _start_server_side_wakeup_turn( + session_id, + str((first or {}).get("wakeup_prompt") or "").strip(), + process_id=str((first or {}).get("process_id") or ""), + ) + started = 1 + if started: + logger.info( + "turn-teardown idle-hook redelivered %d deferred wakeup(s) " + "for session %s", + started, + session_id, + ) + return started + except Exception: + logger.warning( + "drain_deferred_wakeups_for_session failed for session %s", + session_id, + exc_info=True, + ) + return 0 + + +def _session_has_active_turn(session_id: str) -> bool: + """True if a foreground/streaming agent turn is currently active for *session_id*. + + The drain thread has no Session object, so we key on ACTIVE_RUNS — the + worker-lifecycle registry that this module already uses (see + ``_emit_to_session_streams``) to map a stream back to its owning session. + ACTIVE_RUNS is registered at agent-worker start and removed in the worker's + outer ``finally``, so it survives cancel/reconnect races better than + STREAMS. There is a brief window where ``_start_chat_stream_for_session`` + has populated STREAMS but the worker thread has not yet called + ``register_active_run``; in that window this returns False and the + subsequent ``start_session_turn`` is rejected with a 409 by + ``_start_chat_stream_for_session``'s own active-stream guard — i.e. the + same lock /api/chat/start uses is the authoritative race backstop. + """ + from api import config as _cfg + + try: + with _cfg.ACTIVE_RUNS_LOCK: + for _stream_id, meta in (_cfg.ACTIVE_RUNS or {}).items(): + if isinstance(meta, dict) and meta.get("session_id") == session_id: + return True + except Exception: + logger.debug("ACTIVE_RUNS active-turn check failed", exc_info=True) + return False + + +def _start_server_side_wakeup_turn( + session_id: str, wakeup_prompt: str, *, process_id: str = "" +) -> None: + """Start an agent turn server-side for a process_complete wakeup (Option Z). + + Runs on a short-lived daemon thread so the drain loop NEVER blocks: + ``start_session_turn`` itself spawns the agent worker thread, but does + synchronous session-load / workspace / model resolution first, which must + not stall the single drain thread shared by every WebUI session. + + Concurrency + idempotency are enforced by the layers below, not here: + - ``start_session_turn`` → ``_start_chat_stream_for_session`` serializes + on the per-session agent lock and returns ``_status=409`` if a turn is + already active. A human ``/api/chat/start`` racing this wakeup wins + (one starts, the other 409s). On 409 we re-queue the prompt via + ``record_deferred_wakeup`` (see below) so the racing turn's own + teardown idle-hook — or PR #2279's next-turn drain — redelivers it. + - ``BG_TASK_COMPLETE_EVENTS_SEEN`` already deduped this process_id in + ``_process_one`` before we were called, so a process wakes at most once. + + Why re-queue on 409 instead of trusting the PENDING marker: this helper is + called from two sites. The idle branch of ``_process_one`` leaves the + PENDING_BG_TASK_COMPLETIONS marker intact, but the teardown-hook caller + ``drain_deferred_wakeups_for_session`` has ALREADY atomically claimed the + deferred entry and discarded the marker before spawning this thread. So in + the teardown path a 409 would otherwise lose the wakeup permanently — + nothing is left to drain. Re-queuing here is uniformly correct: it is + idempotent per ``process_id`` (``record_deferred_wakeup`` dedupes) and the + claim in ``claim_deferred_wakeups`` is atomic, so re-queue can never cause + a double delivery. + """ + + def _runner() -> None: + try: + from api.routes import start_session_turn + + resp = start_session_turn( + session_id, wakeup_prompt, source="process_wakeup" + ) + status = int((resp or {}).get("_status", 200) or 200) + if status == 409: + # Raced an active turn (e.g. a human /api/chat/start, or a + # sibling deferred-wakeup thread). Re-defer this prompt so it + # is delivered by the winning turn's teardown / next-turn drain + # instead of being lost. The atomic claim in + # ``claim_deferred_wakeups`` still guarantees exactly-once + # delivery, and BG_TASK_COMPLETE_EVENTS_SEEN already deduped + # this process_id, so re-recording cannot double-fire. + if wakeup_prompt: + record_deferred_wakeup(session_id, process_id, wakeup_prompt) + logger.debug( + "server-side wakeup raced an active turn for session %s; " + "re-deferred for redelivery on next teardown/turn", + session_id, + ) + elif status >= 400: + logger.warning( + "server-side wakeup failed for session %s: status=%s err=%r", + session_id, + status, + (resp or {}).get("error"), + ) + else: + logger.info( + "server-side wakeup turn started for session %s (stream_id=%s)", + session_id, + (resp or {}).get("stream_id"), + ) + except Exception: + logger.warning( + "server-side wakeup turn raised for session %s", + session_id, + exc_info=True, + ) + + threading.Thread( + target=_runner, + name=f"hermes-webui-process-wakeup-{str(session_id)[:8]}", + daemon=True, + ).start() + + +def _drain_loop() -> None: + try: + from tools import process_registry as _pr_mod # noqa: F401 + from tools.process_registry import process_registry + except Exception as exc: + logger.warning("bg_task_complete drain unavailable: %s", exc) + return + logger.info("bg_task_complete drain thread started") + while not _DRAIN_STOP.is_set(): + try: + evt = process_registry.completion_queue.get(timeout=1.0) + except Exception: + # queue.Empty or transient — re-check stop flag and continue. + continue + if not isinstance(evt, dict): + continue + try: + _process_one(evt) + except Exception: + logger.warning("bg_task_complete event handling failed", exc_info=True) + + +def register_process_session(session_key: str, session_id: str) -> None: + """Bind a process-registry session_key to a WebUI session_id. + + Called at chat-start time, before the agent thread spawns any background + processes. The same ``session_key`` is exported to the child via + ``HERMES_SESSION_KEY`` (already done by streaming.py), so when the child + pushes onto ``completion_queue`` it carries the key we registered. + """ + if not session_key or not session_id: + return + from api import config as _cfg + + with _cfg.PROCESS_SESSION_INDEX_LOCK: + _cfg.PROCESS_SESSION_INDEX[str(session_key)] = str(session_id) + + +def unregister_process_session(session_key: str) -> None: + if not session_key: + return + from api import config as _cfg + + with _cfg.PROCESS_SESSION_INDEX_LOCK: + _cfg.PROCESS_SESSION_INDEX.pop(str(session_key), None) + + +def start_drain_thread() -> bool: + """Start the background drain thread idempotently. Returns True on first start.""" + global _DRAIN_THREAD + if _DRAIN_THREAD is not None and _DRAIN_THREAD.is_alive(): + return False + _DRAIN_STOP.clear() + _DRAIN_THREAD = threading.Thread( + target=_drain_loop, + name="hermes-webui-bg-task-complete-drain", + daemon=True, + ) + _DRAIN_THREAD.start() + return True + + +def stop_drain_thread(timeout: float = 2.0) -> None: + _DRAIN_STOP.set() + th = _DRAIN_THREAD + if th is not None and th.is_alive(): + th.join(timeout=timeout) diff --git a/api/config.py b/api/config.py index a2e6d0ec..b7ebaa6d 100644 --- a/api/config.py +++ b/api/config.py @@ -2791,6 +2791,142 @@ _available_models_cache_lock = threading.RLock() # must be RLock: cold path ref _cache_build_cv = threading.Condition(_available_models_cache_lock) # shares underlying RLock so notify_all() is safe inside with _available_models_cache_lock _cache_build_in_progress = False # True while a cold path is actively building +# Hard wall-clock budget for a COLD live provider-catalog rebuild when it is +# run from a foreground request path. The live rebuild does one network probe +# per detected provider (Copilot token-exchange HTTPS, OpenRouter /v1/models, +# Nous /models, ...). On a flaky / corp / WSL network any single probe can +# stall for its full per-call timeout (Copilot urllib timeout=10s) and, summed +# across N providers, block the request thread for tens of seconds. This bounds +# the time a foreground caller will wait: past the budget it returns a usable +# fallback (last-known disk cache or a network-free minimal catalog) and lets +# the rebuild finish out-of-band and populate the cache for the next call. +# Set HERMES_WEBUI_MODELS_REBUILD_BUDGET=0 to restore the legacy synchronous +# (unbounded) behaviour. +try: + _LIVE_REBUILD_BUDGET_SECONDS: float = float( + os.getenv("HERMES_WEBUI_MODELS_REBUILD_BUDGET", "4") or "4" + ) +except (TypeError, ValueError): + _LIVE_REBUILD_BUDGET_SECONDS = 4.0 + + +# ── Budget-exceeded warning rate-limit ─────────────────────────────────────── +# Q-2979-A3 / Copilot discussion_r3305864400: the live-rebuild-budget-exceeded +# warning at _invoke_models_rebuild's slow-path is potentially high-volume — +# every provider catalog refresh that runs past _LIVE_REBUILD_BUDGET_SECONDS +# emits one, so a hung upstream probe (or a sustained burst of cold callers) +# could flood the log at warning level. Rate-limit per reason: the FIRST +# occurrence in a cooldown window logs at warning; subsequent occurrences in +# the same window log at info (so log signal stays useful but volume bounded). +# Override the default cooldown via HERMES_WEBUI_BUDGET_WARN_COOLDOWN (seconds). +try: + _BUDGET_WARN_COOLDOWN_SECONDS: float = float( + os.getenv("HERMES_WEBUI_BUDGET_WARN_COOLDOWN", "300") or "300" + ) +except (TypeError, ValueError): + _BUDGET_WARN_COOLDOWN_SECONDS = 300.0 + +_BUDGET_WARN_STATE: dict[str, float] = {} +_BUDGET_WARN_LOCK = threading.Lock() + + +def _should_warn_budget(reason: str, cooldown_s: float | None = None) -> bool: + """Return True iff the budget warning for ``reason`` should log at + warning level (first hit, or last warn-level emit was more than + ``cooldown_s`` seconds ago). Otherwise False — the caller should demote + to info for the same payload so the signal is retained but the noise is + capped. Thread-safe; the cooldown is shared across all live-rebuild + callers in this process. + """ + cooldown = ( + _BUDGET_WARN_COOLDOWN_SECONDS if cooldown_s is None else float(cooldown_s) + ) + now = time.monotonic() + with _BUDGET_WARN_LOCK: + last = _BUDGET_WARN_STATE.get(reason) + if last is None or (now - last) >= cooldown: + _BUDGET_WARN_STATE[reason] = now + return True + return False + + +def _invoke_models_rebuild(builder): + """Indirection seam around the cold catalog rebuild. + + Production simply calls ``builder()``. Exists so tests can simulate a + slow / hanging provider probe without having to reach the closure that + actually does the per-provider network calls. + """ + return builder() + + +def _minimal_static_models_catalog() -> dict: + """Return a network-free /api/models catalog derived from config + auth. + + Used as the fast fallback when a foreground caller must NOT pay the live + provider probe: server-initiated wakeup turns (Option Z) and the + bounded-rebuild timeout path. It is enough for + ``_resolve_compatible_session_model_state`` (which only needs + ``default_model`` / ``active_provider`` plus the persisted session model) + and keeps the picker non-empty. Intentionally NOT written to the 24h + cache so a subsequent human ``/api/models`` still triggers a real rebuild. + """ + try: + active_provider = None + cfg_base_url = "" + model_cfg = cfg.get("model", {}) if isinstance(cfg, dict) else {} + if isinstance(model_cfg, dict): + active_provider = model_cfg.get("provider") + cfg_base_url = model_cfg.get("base_url", "") or "" + if active_provider: + try: + active_provider = _resolve_configured_provider_id( + active_provider, cfg, base_url=cfg_base_url + ) + except Exception: + active_provider = str(active_provider or "").strip() or None + if not active_provider: + try: + _ap = _get_auth_store_path() + if _ap.exists(): + _store = json.loads(_ap.read_text(encoding="utf-8")) + active_provider = ( + _resolve_configured_provider_id( + _store.get("active_provider"), cfg, base_url=cfg_base_url + ) + or None + ) + except Exception: + pass + default_model = get_effective_default_model(cfg) + groups: list[dict] = [] + if default_model: + try: + label = _get_label_for_model(default_model, []) + except Exception: + label = default_model + groups.append( + { + "provider": "Default", + "provider_id": active_provider or "default", + "models": [{"id": default_model, "label": label}], + } + ) + return { + "active_provider": active_provider, + "default_model": default_model, + "configured_model_badges": {}, + "groups": groups, + } + except Exception: + logger.debug("minimal static models catalog build failed", exc_info=True) + return { + "active_provider": None, + "default_model": "", + "configured_model_badges": {}, + "groups": [], + } + # Cache for credential pool results -- calling load_pool() per-provider per-server # session is expensive (~10s for zai due to endpoint probing). The credential pool # only changes when the user adds/removes credentials, which is rare; a 24h TTL @@ -3435,7 +3571,7 @@ def _read_visible_codex_cache_model_ids() -> list[str]: return ordered -def get_available_models() -> dict: +def get_available_models(*, prefer_cache: bool = False) -> dict: """ Return available models grouped by provider. @@ -3450,6 +3586,15 @@ def get_available_models() -> dict: 'default_model': str, 'groups': [{'provider': str, 'models': [{'id': str, 'label': str}]}] } + + ``prefer_cache=True`` resolves WITHOUT ever triggering a live provider + probe: it serves the warm in-memory cache, then the last-known on-disk + cache, and only as a last resort a network-free minimal catalog + (config/auth derived). It NEVER does the per-provider live rebuild (the + Copilot token-exchange HTTPS call et al.). This is the path a + server-initiated wakeup turn (Option Z) takes so a cold catalog can never + block the wakeup chat/start on a flaky network. A normal human request + leaves this False and keeps the full live-discovery behaviour. """ global _cache_build_in_progress, _available_models_cache, _available_models_cache_ts, _available_models_cache_source_fingerprint, _cache_build_cv # Config mtime check — must come before any config reads. @@ -4867,25 +5012,175 @@ def get_available_models() -> dict: _save_models_cache_to_disk(disk_groups) return copy.deepcopy(disk_groups) + # ── prefer_cache: NEVER run the live provider rebuild ──────────────── + # Server-initiated wakeup turns (Option Z) reach here with a cold + # cache (the drain thread fires while idle; the catalog warmed by a + # human's /api/models has expired or was never built). The live + # rebuild does a Copilot token-exchange HTTPS call per the proven + # thread-stack; on this WSL/corp network it stalls the wakeup + # chat/start indefinitely. A wakeup turn does NOT need the full live + # catalog — _resolve_compatible_session_model_state only needs + # default_model/active_provider and trusts the persisted session + # model. Serve a network-free minimal catalog instead and let a later + # human request do the real live rebuild. + if prefer_cache: + # NOTE (Greptile P1): do NOT touch _cache_build_in_progress here. + # This branch never set the flag (only the cold path below does), + # and `should_wait` is sampled outside the lock (line ~4964). A + # concurrent cold-path caller can flip the flag to True after our + # sample but before we acquire the lock; clearing it here would + # prematurely release that rebuild's serialization, waking waiters + # to an empty cache and triggering a second live rebuild. Just + # serve the network-free minimal catalog and leave the flag alone. + return copy.deepcopy(_minimal_static_models_catalog()) + # Cold path: full rebuild — only one thread reaches here at a time with _cache_build_cv: _cache_build_in_progress = True - try: - result = _build_available_models_uncached() - except Exception: - # Always reset the flag so waiting threads don't block for 60s + + # Legacy synchronous (unbounded) rebuild — opt-in via budget<=0. + if _LIVE_REBUILD_BUDGET_SECONDS <= 0: + try: + result = _invoke_models_rebuild(_build_available_models_uncached) + except BaseException: + # Always reset the flag so waiting threads don't block for 60s + with _cache_build_cv: + _cache_build_in_progress = False + _cache_build_cv.notify_all() + raise + with _cache_build_cv: + _available_models_cache = result + _available_models_cache_ts = time.monotonic() + _available_models_cache_source_fingerprint = _models_cache_source_fingerprint() + _cache_build_in_progress = False + _cache_build_cv.notify_all() + _save_models_cache_to_disk(result) + return copy.deepcopy(result) + + # ── Bounded rebuild (defense-in-depth) ─────────────────────────────── + # The live rebuild does a network probe per provider (Copilot token + # exchange over HTTPS, OpenRouter/Nous /models, ...). On a flaky / corp + # / WSL network any single probe can stall for its full per-call + # timeout and, summed across providers, pin a foreground request + # thread for tens of seconds (the wakeup-turn / chat-start hang). + # + # Run the rebuild on a daemon worker; the foreground waits at most + # _LIVE_REBUILD_BUDGET_SECONDS. + # + # WITHIN budget (the normal fast case): the FOREGROUND publishes the + # result synchronously and only then returns — preserving the exact + # pre-existing contract (cache + on-disk file populated by the time + # get_available_models() returns). The worker stays hands-off. + # + # OVER budget (a provider probe is slow/hung): the foreground returns + # the best fallback immediately and the still-running worker publishes + # its result out-of-band when it finally finishes, so the next caller + # gets a warm cache instead of paying the cold rebuild again. + # + # ``_publish_models_result`` / ``box["published"]`` ensure exactly one + # publisher even at the budget boundary (no double write, no lost + # refresh). The worker only touches _cache_build_cv after the + # foreground releases the RLock by returning, so no lock inversion. + build_done = threading.Event() + budget_exceeded = threading.Event() + publish_lock = threading.Lock() + box: dict = {} + + def _publish_models_result(result): + global _cache_build_in_progress, _available_models_cache + global _available_models_cache_ts, _available_models_cache_source_fingerprint + with _cache_build_cv: + _available_models_cache = result + _available_models_cache_ts = time.monotonic() + _available_models_cache_source_fingerprint = ( + _models_cache_source_fingerprint() + ) + _cache_build_in_progress = False + _cache_build_cv.notify_all() + try: + _save_models_cache_to_disk(result) + except Exception: + logger.debug("models cache disk save failed", exc_info=True) + + def _clear_build_in_progress(): + global _cache_build_in_progress with _cache_build_cv: _cache_build_in_progress = False _cache_build_cv.notify_all() - raise - with _cache_build_cv: - _available_models_cache = result - _available_models_cache_ts = time.monotonic() - _available_models_cache_source_fingerprint = _models_cache_source_fingerprint() - _cache_build_in_progress = False - _cache_build_cv.notify_all() - _save_models_cache_to_disk(result) - return copy.deepcopy(result) + + def _claim_publish() -> bool: + """Return True iff the caller won the right to publish.""" + with publish_lock: + if box.get("published"): + return False + box["published"] = True + return True + + def _rebuild_worker(): + try: + box["result"] = _invoke_models_rebuild(_build_available_models_uncached) + except Exception as exc: # noqa: BLE001 — propagated to caller + box["error"] = exc + finally: + build_done.set() + # Only publish out-of-band if the foreground already gave up + # (over budget). Within budget the foreground publishes + # synchronously, so the worker must NOT touch the cache. + if budget_exceeded.is_set() and _claim_publish(): + if "result" in box: + _publish_models_result(box["result"]) + else: + _clear_build_in_progress() + + _worker = threading.Thread( + target=_rebuild_worker, + name="models-catalog-rebuild", + daemon=True, + ) + _worker.start() + + if build_done.wait(timeout=_LIVE_REBUILD_BUDGET_SECONDS): + # Build finished within budget — foreground publishes + # synchronously, exactly like the legacy path. + if "error" in box: + _clear_build_in_progress() + raise box["error"] + if _claim_publish(): + _publish_models_result(box["result"]) + return copy.deepcopy(box["result"]) + + # Budget elapsed. Mark it so the worker knows it owns out-of-band + # publication. Handle the tiny race where the build completed between + # wait() returning False and here: if so, still publish synchronously + # so this caller honours the cache contract. + budget_exceeded.set() + if build_done.is_set() and "error" not in box and "result" in box: + if _claim_publish(): + _publish_models_result(box["result"]) + return copy.deepcopy(box["result"]) + + # Genuinely slow/hung probe: serve the best fallback now; the worker + # keeps going and refreshes the cache for the next caller. + # Rate-limit the warning per Q-2979-A3 — see _should_warn_budget; a + # sustained budget breach demotes to info after the first emit in + # each cooldown window so log volume stays bounded. + _budget_log_msg = ( + "live provider-catalog rebuild exceeded %.1fs budget — serving " + "fallback, refreshing catalog out-of-band" + ) + if _should_warn_budget("live_rebuild_budget_exceeded"): + logger.warning(_budget_log_msg, _LIVE_REBUILD_BUDGET_SECONDS) + else: + logger.info(_budget_log_msg, _LIVE_REBUILD_BUDGET_SECONDS) + # Note: ``disk_groups``, if non-None, was already consumed by the + # cold-path early-return at the "Cold path: disk cache hit" branch + # above (line ~4608). Any execution that reaches HERE necessarily + # took the live-rebuild branch, which means ``disk_groups is None`` + # at this point — so we don't re-check it. Per Copilot review on + # PR #2971: the previous ``if disk_groups is not None`` branch + # here was dead code. Fall back directly to the static minimal + # catalog (no second disk read). + return copy.deepcopy(_minimal_static_models_catalog()) # ── Static file path ───────────────────────────────────────────────────────── @@ -4987,6 +5282,51 @@ STREAM_GOAL_RELATED: dict = {} # stream_id -> bool: only evaluate goal for goal STREAM_LAST_EVENT_ID: dict = {} # stream_id -> latest journal event_id for `id:` field on live SSE frames (stage-364) PENDING_GOAL_CONTINUATION: set = set() # session_ids awaiting a goal continuation turn (#1932) +# ── notify_on_complete agent-wakeup wiring ───────────────────────────────── +# When terminal(notify_on_complete=true, background=true) fires, the process +# registry pushes a completion event onto tools.process_registry.completion_queue. +# A drain task spawned at WebUI startup (api/background_process.py) reads that +# queue and emits an SSE `process_complete` event to the matching session. +# PROCESS_SESSION_INDEX maps the per-process "session_key" (set in the spawned +# subprocess via HERMES_SESSION_KEY) back to the WebUI session_id that owns it, +# so the drain task can route the event to the right SSE channel. +# PENDING_BG_TASK_COMPLETIONS mirrors PENDING_GOAL_CONTINUATION: server-side +# marker discarded atomically by routes.py when the frontend re-POSTs the +# wakeup_prompt as the next user turn. (process_complete event, agent wakeup fix) +PROCESS_SESSION_INDEX: dict = {} # process_registry session_key -> WebUI session_id +PROCESS_SESSION_INDEX_LOCK = threading.Lock() +PENDING_BG_TASK_COMPLETIONS: set = set() # session_ids awaiting a process_complete wakeup turn +BG_TASK_COMPLETE_EVENTS_SEEN: dict = {} # session_id -> set[process_id] for idempotency +BG_TASK_COMPLETE_EVENTS_SEEN_LOCK = threading.Lock() + +# Defer-path fix (fast-bg-task wakeup race): when a completion arrives while a +# turn is active, Option Z's drain branch CANNOT start a turn (would 409). The +# pre-existing PENDING_BG_TASK_COMPLETIONS marker was a bare session_id flag — +# the wakeup_prompt was DISCARDED, and the only consumer (PR #2279 next-turn +# drain) reads completion_queue, which the Option Z drain thread already +# emptied. So for an autonomous agent (no next user turn) the deferred wakeup +# was lost forever. DEFERRED_PROCESS_WAKEUPS persists the actual prompt(s) so a +# turn-teardown idle-hook (api/streaming) can redeliver them once the session +# goes idle — symmetric with the idle branch (idle now → fire now; busy now → +# fire at turn-end). Atomic claim (pop under lock) guarantees single delivery: +# whoever claims first (teardown hook OR next-turn drain) fires; the other +# finds nothing → no double-fire, no wakeup loop. +DEFERRED_PROCESS_WAKEUPS: dict = {} # session_id -> list[{"process_id", "wakeup_prompt"}] +DEFERRED_PROCESS_WAKEUPS_LOCK = threading.Lock() + +# ── Persistent per-session SSE channel (Option X) ────────────────────────── +# A long-lived SSE channel scoped to a WebUI session_id rather than a single +# agent turn (stream_id). Subscribed to by the frontend on session mount, +# torn down on session unmount, and refcounted across tabs. Used to deliver +# events (currently process_complete) that fire while no agent turn is +# active — bridging the gap that PR #2242 + #2279 left when STREAMS has +# already been torn down. The registry lives in api.background_process; this +# constant is the idle-cap before the reaper collects an unsubscribed +# channel. 4h is a defensive ceiling against zombie connections; the +# subscribers-empty grace path (60s) handles ordinary tab-close traffic. +SESSION_CHANNEL_IDLE_TTL_SECS: int = 14400 # 4 hours +SESSION_CHANNEL_SUBSCRIBER_GRACE_SECS: int = 60 # subscribers-empty grace + # Active agent-run registry. This intentionally tracks worker lifecycle rather # than SSE lifecycle: cancel/reconnect may remove STREAMS while the worker is # still unwinding, blocked in a provider call, or waiting for delegated work. diff --git a/api/routes.py b/api/routes.py index c4a09c6c..f078ad2f 100644 --- a/api/routes.py +++ b/api/routes.py @@ -1105,6 +1105,7 @@ from api.config import ( _save_yaml_config_file, reload_config, _cfg_lock, + PENDING_BG_TASK_COMPLETIONS, ) from api.helpers import ( require, @@ -2255,6 +2256,7 @@ def _resolve_compatible_session_model_state( profile_provider: str | None = None, profile_default_model: str | None = None, explicit_model_pick: bool = False, + prefer_cached_catalog: bool = False, ) -> tuple[str, str | None, bool]: """Return (effective_model, effective_provider, model_was_normalized). @@ -2276,6 +2278,16 @@ def _resolve_compatible_session_model_state( OpenRouter ``/models``, LM Studio ``/models``, credential pool refresh) — those used to wedge the handler for >100s and trigger 502s on default-60s reverse proxies, even though the WebUI itself eventually responded. + + ``prefer_cached_catalog=True`` (ours-original) makes the catalog lookup + non-blocking: it resolves from the warm/disk cache or a network-free + minimal catalog and NEVER triggers a live per-provider rebuild (the + Copilot token-exchange HTTPS call that hangs a server-initiated wakeup + turn — see rebase report §1/§3/model-resolve-hang). Human-initiated + chat/start leaves this False to keep full live discovery; a session that + already has a persisted model still resolves correctly because the + persisted model wins over the catalog and the catalog is only consulted + for the default-model backstop. """ model = str(model_id or "").strip() requested_provider = _clean_session_model_provider(model_provider) @@ -2292,7 +2304,33 @@ def _resolve_compatible_session_model_state( if not explicit_provider and not stale_codex_openai_slash_id: return model, requested_provider, False - catalog = get_available_models() + # Default (human chat/start) path calls get_available_models() with NO + # kwargs so it stays signature-compatible with the many tests that stub + # get_available_models as a zero-arg callable. Only the server-side wakeup + # path (prefer_cached_catalog=True) opts into the cache-only mode. Some + # tests monkeypatch get_available_models as a zero-arg callable, so probe + # the (possibly monkeypatched) signature for ``prefer_cache`` rather than + # catching TypeError — a blanket ``except TypeError`` would also swallow a + # genuine TypeError raised *inside* get_available_models(prefer_cache=True) + # and silently fall back to the slow live provider rebuild that + # prefer_cached_catalog=True is meant to avoid. + if prefer_cached_catalog: + import inspect as _inspect + + try: + _gam_accepts_prefer_cache = ( + "prefer_cache" in _inspect.signature(get_available_models).parameters + ) + except (TypeError, ValueError): + # Builtins / C-callables can refuse introspection; assume the + # zero-arg stub shape in that case. + _gam_accepts_prefer_cache = False + if _gam_accepts_prefer_cache: + catalog = get_available_models(prefer_cache=True) + else: + catalog = get_available_models() + else: + catalog = get_available_models() default_model = str(catalog.get("default_model") or DEFAULT_MODEL or "").strip() # Profile-aware resolution: when the caller supplies profile context @@ -2547,6 +2585,17 @@ def _resolve_effective_session_model_for_display(session) -> str: requested_provider, profile_provider=_pp_provider, profile_default_model=_pp_default, + # GET /api/session is a hot, side-effect-free per-tab/per-poll path. + # It must never pay the cold live provider-catalog rebuild (a + # botocore IMDS probe that cannot resolve on a non-AWS / WSL / corp + # network, plus anthropic/openrouter /models). That rebuild is + # un-cacheable here (auth.json fingerprint churn) so every cold call + # cost ~10s and, run concurrently across browser tabs, serialized on + # the models-cache lock and starved SSE/streaming -> BrokenPipe storm + # (#multi-tab-streaming-interlock). The persisted session model is + # authoritative; the catalog is only a default-model backstop, which + # the network-free minimal catalog already provides. + prefer_cached_catalog=True, ) return effective_model or original_model @@ -2559,6 +2608,11 @@ def _resolve_effective_session_model_provider_for_display(session) -> str | None requested_provider, profile_provider=_pp_provider, profile_default_model=_pp_default, + # See _resolve_effective_session_model_for_display: same hot + # side-effect-free GET /api/session path; must not trigger the cold + # live rebuild. prefer_cached_catalog resolves from warm/disk cache + # or the network-free minimal catalog. + prefer_cached_catalog=True, ) return provider @@ -3408,6 +3462,7 @@ from api.upload import ( ) from api.streaming import ( _sse, + _sse_set_write_deadline, _run_agent_streaming, cancel_stream, _materialize_pending_user_turn_before_error, @@ -6208,6 +6263,9 @@ def handle_get(handler, parsed) -> bool: if parsed.path == "/api/clarify/stream": return _handle_clarify_sse_stream(handler, parsed) + if parsed.path == "/api/session/stream": + return _handle_session_sse_stream(handler, parsed) + if parsed.path == "/api/clarify/inject_test": # Loopback-only: used by automated tests; blocked from any remote client if handler.client_address[0] != "127.0.0.1": @@ -6662,6 +6720,37 @@ def handle_post(handler, parsed) -> bool: finally: if diag: diag.finish() + # T1 deprecation alias for the legacy ack endpoint that the pre-rename + # WebUI used to POST to after handling ``process_complete``. The new + # canonical SSE event is ``bg_task_complete`` and the new ack endpoint + # will be ``/api/bg-task-complete-ack`` (introduced by PR (b), the WebUI + # half of the split). Until PR (b) lands we keep the old path responding + # with HTTP 410 Gone + ``X-Replaced-By`` so any stale tab posting under + # the old name fails loudly with a discoverable hint. The handler runs + # BEFORE the CSRF gate on purpose: an old tab will not carry a CSRF token + # for the deprecated path, and surfacing 410 (not 403) is the correct + # contract here. + if parsed.path == "/api/process-complete-ack": + if diag: + diag.stage("process_complete_ack_deprecated") + try: + j( + handler, + { + "error": ( + "gone: /api/process-complete-ack was replaced by " + "/api/bg-task-complete-ack as part of the " + "process_complete -> bg_task_complete event rename" + ), + "replaced_by": "/api/bg-task-complete-ack", + }, + status=410, + extra_headers={"X-Replaced-By": "/api/bg-task-complete-ack"}, + ) + return True + finally: + if diag: + diag.finish() # CSRF: reject cross-origin or tokenless authenticated browser requests. # /api/auth/login has no authenticated session token yet, and /api/csp-report # is intentionally unauthenticated for browser-generated violation reports. @@ -7625,6 +7714,9 @@ def handle_post(handler, parsed) -> bool: if parsed.path == "/api/goal": return _handle_goal_command(handler, body) + if parsed.path == "/api/bg-task-complete-ack": + return _handle_bg_task_complete_ack(handler, body) + if parsed.path == "/api/chat/start": return _handle_chat_start(handler, body, diag=diag) @@ -9139,6 +9231,7 @@ def _handle_sse_stream(handler, parsed): handler.send_header("X-Accel-Buffering", "no") handler.send_header("Connection", "close") handler.end_headers() + _sse_set_write_deadline(handler) # Defect A: slow tab can't pin this thread replay_cutoff_seq = None if qs.get("replay", [""])[0] or qs.get("after_seq", [None])[0] not in (None, "") or qs.get("after_event_id", [None])[0]: snapshot_cutoff_seq = _run_journal_same_run_seq( @@ -9318,6 +9411,7 @@ def _handle_terminal_output(handler, parsed): handler.send_header("X-Accel-Buffering", "no") handler.send_header("Connection", "close") handler.end_headers() + _sse_set_write_deadline(handler) # Defect A: slow tab can't pin this thread try: while True: try: @@ -9403,6 +9497,7 @@ def _handle_gateway_sse_stream(handler, parsed): # session list every ~1s. Letting the server close the socket # naturally after the stream ends is sufficient. handler.end_headers() + _sse_set_write_deadline(handler) # Defect A: slow tab can't pin this thread q = watcher.subscribe() try: @@ -10478,6 +10573,7 @@ def _handle_approval_sse_stream(handler, parsed): handler.send_header('X-Accel-Buffering', 'no') handler.send_header('Connection', 'close') handler.end_headers() + _sse_set_write_deadline(handler) # Defect A: slow tab can't pin this thread from api.streaming import _sse @@ -10579,6 +10675,7 @@ def _handle_clarify_sse_stream(handler, parsed): handler.send_header('X-Accel-Buffering', 'no') handler.send_header('Connection', 'close') handler.end_headers() + _sse_set_write_deadline(handler) # Defect A: slow tab can't pin this thread from api.streaming import _sse @@ -10602,6 +10699,126 @@ def _handle_clarify_sse_stream(handler, parsed): clarify_sse_unsubscribe(sid, q) +def _handle_session_sse_stream(handler, parsed): + """SSE endpoint for the persistent per-session channel (Option X). + + Subscribes to ``api.background_process.SESSION_CHANNELS[sid]`` — a channel + that lives across agent turns (unlike STREAMS, which is torn down at + end-of-turn). Used to deliver ``bg_task_complete`` events that fire while + no agent turn is active. + + Lifecycle: opened by the frontend at session mount, closed at unmount or + on tab close. Multiple tabs share one SessionChannel (refcounted via + subscribe/unsubscribe). 30s SSE keepalive comments keep the proxy alive. + Reaper-driven idle TTL (default 4h) prevents zombie channels. + """ + sid = parse_qs(parsed.query).get("session_id", [""])[0] + if not sid: + return bad(handler, "session_id is required") + + from api.background_process import ( + subscribe_to_session_channel, + active_stream_id_for_session, + ) + + # Atomic get-or-create + subscribe under SESSION_CHANNELS_LOCK. Doing these + # two steps separately (get_or_create_session_channel then ch.subscribe) + # left a TOCTOU gap where the reaper — which also holds + # SESSION_CHANNELS_LOCK and collects idle 0-subscriber channels in one + # critical section — could collect the channel between the two calls, + # orphaning this subscriber on a channel no longer in SESSION_CHANNELS. + # bg_task_complete emits would then never reach this queue. See + # subscribe_to_session_channel for the full rationale (PR #2971 Greptile P1). + ch, q = subscribe_to_session_channel(sid, maxsize=64) + + # NOTE: ``subscribe_to_session_channel`` above acquires a subscriber slot + # that MUST be released on every exit path. Header setup + # (``send_response`` / ``send_header`` / ``end_headers`` / + # ``_sse_set_write_deadline``) and the initial-frame + on-subscribe + # recovery writes below all touch the socket and can raise a member of + # ``_CLIENT_DISCONNECT_ERRORS`` (BrokenPipeError / ConnectionResetError) if + # the client drops immediately after subscribing. If that happened outside + # this try/finally the ``ch.unsubscribe(q)`` cleanup would be skipped, + # permanently leaking a subscriber. Because + # ``SessionChannel.reaper_should_collect()`` refuses to collect any channel + # with ``sub_count > 0``, a single ghost subscriber blocks the reaper + # forever and the channel zombies in SESSION_CHANNELS. So EVERYTHING from + # the subscribe onward — header setup included — runs inside one + # try/finally that unconditionally unsubscribes. + try: + handler.send_response(200) + handler.send_header('Content-Type', 'text/event-stream; charset=utf-8') + handler.send_header('Cache-Control', 'no-cache') + handler.send_header('X-Accel-Buffering', 'no') + # #3103: omit the Connection header — rely on the HTTP/1.1 keep-alive + # default, matching the other long-lived SSE handlers (gateway/session + # events) that fixed the reconnect-storm. An explicit value here is a + # third, inconsistent approach (greptile flag). + handler.end_headers() + _sse_set_write_deadline(handler) # Defect A: slow tab can't pin this thread + + from api.streaming import _sse + + # Push an initial frame so the client has confirmation the channel is + # live (mirrors approval/clarify which send an 'initial' frame). No + # snapshot data is needed — this channel only carries forward-looking + # events, not pending state. + _sse(handler, 'initial', {"session_id": sid}) + + # ── Open-tab live-view self-heal (root cause: lost server_turn_started) ── + # The `server_turn_started` fan-out (routes.start_session_turn) is a + # fire-and-forget SessionChannel.emit with NO replay buffer: it reaches + # only the subscribers connected at the exact emit instant. A tab whose + # per-session EventSource was momentarily absent at that instant — a + # transient SSE drop, a reverse-proxy idle-timeout, or browser + # connection-pool starvation (all common behind a corporate proxy) — + # misses the frame permanently, so a SERVER-initiated wakeup turn never + # renders live and the user must hard-refresh (the reported defect). The + # server-side wakeup itself ran and persisted fine; only the live-view + # was lost. On (re)subscribe, if the session has a live run RIGHT NOW, + # replay a synthetic `server_turn_started` to THIS new subscriber so the + # open tab attaches its existing chat-stream renderer (attachLiveStream) + # and self-heals with no refresh. `recovered: True` lets the frontend + # use the replay (reconnecting) attach so the renderer picks up the + # in-progress stream from the run journal rather than expecting token 0. + # Idempotent: the frontend dedupes by (session_id, stream_id) — if the + # original frame WAS delivered this is a harmless no-op there. + try: + recover_stream_id = active_stream_id_for_session(sid) + if recover_stream_id: + _sse(handler, 'server_turn_started', { + "session_id": sid, + "stream_id": recover_stream_id, + "source": "subscribe_recovery", + "recovered": True, + }) + except _CLIENT_DISCONNECT_ERRORS: + # Client vanished mid-recovery — re-raise so the outer handler + # treats it as a normal disconnect and the finally still cleans up. + raise + except Exception: + logger.debug( + "session-stream on-subscribe recovery failed for %s", sid, + exc_info=True, + ) + + while True: + try: + payload = q.get(timeout=_SSE_HEARTBEAT_INTERVAL_SECONDS) + except queue.Empty: + handler.wfile.write(b': keepalive\n\n') + handler.wfile.flush() + continue + if payload is None: + break + event_name, data = payload + _sse(handler, event_name, data) + except _CLIENT_DISCONNECT_ERRORS: + pass # client went away — normal for long-lived connections + finally: + ch.unsubscribe(q) + + def _handle_clarify_inject(handler, parsed): """Inject a fake pending clarify prompt -- loopback-only, used by automated tests.""" qs = parse_qs(parsed.query) @@ -10800,15 +11017,15 @@ def _handle_live_models(handler, parsed): ids = [m.get("id", "") for m in _data if m.get("id")] elif isinstance(_body, list): ids = [m.get("id", m) if isinstance(m, dict) else m for m in _body] - + if ids: logger.debug("Live-fetched %d models from custom provider %s", len(ids), _base_url) else: logger.debug("Custom provider returned no models from %s", _base_url) - + except Exception as _fetch_err: logger.debug("Live fetch from custom provider failed: %s", _fetch_err) - + # If live fetch succeeded, merge with config entries (live takes # priority). If live fetch failed, fall back to config-only list. if ids: @@ -11597,6 +11814,14 @@ def _start_chat_stream_for_session( goal_related = True PENDING_GOAL_CONTINUATION.discard(s.session_id) + # process_complete wakeup (ours-original, Option B): if this session has a + # pending process_complete marker (set by api/background_process.py drain), + # discard it atomically here. Mirrors the goal_continue pattern (#1932). + # The marker is server-internal telemetry; the actual wakeup is delivered + # either server-side (Option Z) or via the PR #2279 next-turn drain. + if s.session_id in PENDING_BG_TASK_COMPLETIONS: + PENDING_BG_TASK_COMPLETIONS.discard(s.session_id) + session_lock = _get_session_agent_lock(s.session_id) diag.stage("session_lock_wait") if diag else None while True: @@ -11754,6 +11979,263 @@ def _runtime_adapter_goal_action(goal_args: str) -> str: return "set" +def _start_run( + s, + *, + msg: str, + attachments, + workspace: str, + model, + model_provider, + normalized_model, + source: str, + route: str, + diag=None, +): + """Shared start-run helper for /api/chat/start and start_session_turn. + + Centralizes the runtime-adapter selection block (Q-2979-A2 / Copilot + discussion_r3305864087/r3305864173) so both entrypoints honor + ``runtime_adapter_enabled()`` / ``runtime_adapter_runner_enabled()`` the + same way. Prior to this helper ``start_session_turn`` bypassed the + adapter path entirely, so a process-wakeup turn skipped the adapter that + a human-typed turn would have hit — a behavioral divergence. + + ``source`` is the StartRunRequest.source (``"webui"`` for browser POSTs, + ``"process_wakeup"`` for the drain-thread wakeup). ``route`` is the + metadata.route label that lands on the run record for observability. + + Returns a dict with ``_status`` plus the legacy chat-start response + fields (``stream_id``, ``session_id``, etc.). Adapter selection that + returns no adapter is surfaced as ``{"error": str(exc), "_status": 501}`` + so both call sites can map it onto their own HTTP shape. + """ + from api.runtime_adapter import ( + LegacyJournalRuntimeAdapter, + StartRunRequest, + build_runtime_adapter, + runtime_adapter_enabled, + runtime_adapter_runner_enabled, + ) + + if runtime_adapter_enabled() or runtime_adapter_runner_enabled(): + def _legacy_start_run(request: StartRunRequest) -> dict: + return _start_chat_stream_for_session( + s, + msg=request.message, + attachments=request.attachments, + workspace=request.workspace or workspace, + model=request.model or model, + model_provider=request.provider or model_provider, + normalized_model=normalized_model, + diag=diag, + ) + + def _legacy_adapter_factory(): + return LegacyJournalRuntimeAdapter(start_run_delegate=_legacy_start_run) + + try: + adapter = build_runtime_adapter( + legacy_adapter_factory=_legacy_adapter_factory, + runner_client_factory=_runtime_runner_client_factory, + ) + if adapter is None: + raise NotImplementedError("runtime adapter selection returned no adapter") + result = adapter.start_run( + StartRunRequest( + session_id=s.session_id, + message=msg, + attachments=attachments, + workspace=workspace, + profile=getattr(s, "profile", None), + provider=model_provider, + model=model, + source=source, + metadata={"route": route}, + ) + ) + except NotImplementedError as exc: + return {"error": str(exc), "_status": 501} + return _chat_start_response_from_run_start(result) + + return _start_chat_stream_for_session( + s, + msg=msg, + attachments=attachments, + workspace=workspace, + model=model, + model_provider=model_provider, + normalized_model=normalized_model, + diag=diag, + ) + + +def start_session_turn( + session_id: str, + message: str, + *, + source: str = "process_wakeup", +): + """Start a server-side agent turn for ``session_id`` with ``message``. + + Option Z primary wakeup entrypoint. This is the minimal, HTTP-handler-free + core that ``/api/chat/start`` already reaches via ``_handle_chat_start`` → + ``_start_chat_stream_for_session``. The drain thread + (``api/background_process._process_one``) calls this directly with a + synthetic ``[IMPORTANT: …]`` wakeup_prompt so a background process can wake + the agent server-side with NO browser round-trip — exactly how CLI / + gateway self-wake from a ``notify_on_complete`` completion. + + Contract: + - Resolves the session record (profile/workspace/model/model_provider are + already persisted on it; no user auth needed — same trust level as + gateway/cron starting a turn). + - Resolves workspace + model/provider through the SAME helpers + ``_handle_chat_start`` uses, so a process-wakeup turn is constructed + identically to a human-typed turn. If the session record has no model + persisted, ``_resolve_compatible_session_model_state`` falls back to the + configured default model/provider (documented in the impl report §1). + - Delegates to ``_start_chat_stream_for_session`` which spawns the agent + on a daemon worker thread (the drain thread NEVER blocks) and serializes + on the per-session agent lock + active-stream guard, so a concurrent + human ``/api/chat/start`` cannot double-start (one wins, the other gets + the existing 409 "session already has an active stream"). + + Returns the same dict ``_start_chat_stream_for_session`` returns, including + ``_status`` (200 on start, 409 when a turn is already active). On 409 the + caller must leave the ``PENDING_BG_TASK_COMPLETIONS`` marker in place so the + PR #2279 next-turn drain delivers the wakeup when the active turn ends. + """ + msg = str(message or "").strip() + if not msg: + return {"error": "message is required", "_status": 400} + try: + s = get_session(session_id) + except KeyError: + return {"error": "Session not found", "_status": 404} + + try: + workspace = _resolve_chat_workspace_with_recovery(s, None) + except ValueError as e: + return {"error": str(e), "_status": 400} + + requested_model = s.model + requested_provider = getattr(s, "model_provider", None) + # Server-initiated wakeup (Option Z): resolve persisted model via the + # standard helper in cache-only mode so wakeups never trigger a cold + # catalog rebuild. Thread the session's PROFILE model defaults through too + # (mirrors _handle_chat_start) — a brand-new session that spawned a + # background task before its first human turn has an empty s.model, and + # without the profile defaults the resolver would fall back to the global + # DEFAULT_MODEL instead of the profile's configured default (greptile flag). + _pp_provider, _pp_default = _read_profile_model_config(s, requested_provider) + model, model_provider, normalized_model = _resolve_compatible_session_model_state( + requested_model, + requested_provider, + profile_provider=_pp_provider, + profile_default_model=_pp_default, + prefer_cached_catalog=True, + ) + resp = _start_run( + s, + msg=msg, + attachments=[], + workspace=workspace, + model=model, + model_provider=model_provider, + normalized_model=normalized_model, + source="process_wakeup", + route="start_session_turn", + ) + + # ── Defect B: live-view of server-initiated turns ────────────────────── + # Option Z starts this turn server-side, so NO browser EventSource is + # attached to the new STREAMS[stream_id] (the browser only opens + # /api/chat/stream when IT POSTs /api/chat/start). An already-open tab + # would therefore see nothing until a manual refresh re-reads persisted + # state. Fix: fan a lightweight `server_turn_started` {stream_id} frame + # onto the persistent per-session live-view channel. messages.js handles + # it by attaching its EXISTING chat-stream renderer (attachLiveStream) to + # that stream_id — no second renderer, no chat/start POST. + # + # Idempotent with the closed-tab path: get_session_channel() is the + # NON-creating accessor, so when no tab is open this is a pure no-op and + # the server-side wakeup (the Option Z headline) is completely unaffected. + # If the user also has the per-turn chat-stream open, the frontend dedupes + # by stream_id so there is no double-render. + try: + status = int((resp or {}).get("_status", 200) or 200) + stream_id = (resp or {}).get("stream_id") + if status < 400 and stream_id: + from api.background_process import get_session_channel + + ch = get_session_channel(session_id) + if ch is not None: + ch.emit( + "server_turn_started", + { + "session_id": str(session_id), + "stream_id": str(stream_id), + "source": source, + }, + ) + except Exception: + logger.debug( + "server_turn_started fan-out failed for session %s", session_id, exc_info=True + ) + return resp + + +def _handle_bg_task_complete_ack(handler, body): + """Acknowledge a bg_task_complete SSE event (diagnostic only). + + Option Z PIVOT: the agent wakeup is now started SERVER-SIDE by the drain + thread (``api/background_process._process_one`` → ``start_session_turn``) + with NO browser round-trip — the closed-tab case works (parity with + CLI/Telegram). The frontend no longer re-POSTs ``wakeup_prompt`` to + /api/chat/start; the per-session SSE channel is demoted to pure live-view. + + This endpoint is therefore a pure no-op for state — it exists so an open + tab can confirm receipt of the live-view event and so a future follow-up + (analytics, telemetry) has a stable hook. ``PENDING_BG_TASK_COMPLETIONS`` + is consumed by ``_start_chat_stream_for_session`` when the server-side + wakeup turn (or the next human turn / PR #2279 next-turn drain) runs. + """ + from api.helpers import j + + try: + require(body, "session_id") + except ValueError as e: + return bad(handler, str(e)) + sid = str(body.get("session_id") or "").strip() + try: + s = get_session(sid) + except KeyError: + return bad(handler, "Session not found", 404) + # process_id accepted as transitional alias; see Deprecation response header + # + maintainer decision on removal milestone / future Sunset header. Only + # flag Deprecation when the alias was ACTUALLY used (i.e. process_id present + # and not empty), even if task_id is also present. + _task_id_present = bool(str(body.get("task_id") or "").strip()) + _process_id_present = bool(str(body.get("process_id") or "").strip()) + legacy_process_id_used = _process_id_present + pid = str(body.get("task_id") or body.get("process_id") or "").strip() + # Post Option-Z pivot this endpoint owns no state: the server-side drain + # thread starts the wakeup turn, the browser never re-POSTs /api/chat/start. + # `noop` is returned so the diagnostic shape stays explicit about that and + # matches the docstring ("pure no-op for state"). + return j( + handler, + { + "ok": True, + "session_id": s.session_id, + "task_id": pid, + "noop": True, + }, + extra_headers={"Deprecation": "true"} if legacy_process_id_used else {}, + ) + + def _handle_goal_command(handler, body): """Handle WebUI /goal command controls and optional kickoff stream.""" try: @@ -11959,64 +12441,27 @@ def _handle_chat_start(handler, body, diag=None): profile_default_model=_pp_default, explicit_model_pick=explicit_model_pick, ) - from api.runtime_adapter import ( - LegacyJournalRuntimeAdapter, - StartRunRequest, - build_runtime_adapter, - runtime_adapter_enabled, - runtime_adapter_runner_enabled, + # NOTE: runtime-adapter selection is delegated to _start_run (shared + # with start_session_turn so both entry points behave identically + # under runtime_adapter_enabled() / runtime_adapter_runner_enabled() + # — Q-2979-A2 / Copilot discussion_r3305864087/r3305864173). + response = _start_run( + s, + msg=msg, + attachments=attachments, + workspace=workspace, + model=model, + model_provider=model_provider, + normalized_model=normalized_model, + source="webui", + route="/api/chat/start", + diag=diag, ) - - if runtime_adapter_enabled() or runtime_adapter_runner_enabled(): - def _legacy_start_run(request: StartRunRequest) -> dict: - return _start_chat_stream_for_session( - s, - msg=request.message, - attachments=request.attachments, - workspace=request.workspace or workspace, - model=request.model or model, - model_provider=request.provider or model_provider, - normalized_model=normalized_model, - diag=diag, - ) - - def _legacy_adapter_factory(): - return LegacyJournalRuntimeAdapter(start_run_delegate=_legacy_start_run) - - try: - adapter = build_runtime_adapter( - legacy_adapter_factory=_legacy_adapter_factory, - runner_client_factory=_runtime_runner_client_factory, - ) - if adapter is None: - raise NotImplementedError("runtime adapter selection returned no adapter") - result = adapter.start_run( - StartRunRequest( - session_id=s.session_id, - message=msg, - attachments=attachments, - workspace=workspace, - profile=getattr(s, "profile", None), - provider=model_provider, - model=model, - source="webui", - metadata={"route": "/api/chat/start"}, - ) - ) - except NotImplementedError as exc: - return j(handler, {"error": str(exc)}, status=501) - response = _chat_start_response_from_run_start(result) - else: - response = _start_chat_stream_for_session( - s, - msg=msg, - attachments=attachments, - workspace=workspace, - model=model, - model_provider=model_provider, - normalized_model=normalized_model, - diag=diag, - ) + # Map adapter-selection NotImplementedError (501) onto the legacy + # bad-request response shape that this route exposed historically + # before the helper extraction. + if response.get("_status") == 501 and "error" in response: + return j(handler, {"error": response["error"]}, status=501) status = int(response.pop("_status", 200) or 200) diag.stage("response_write") if diag else None return j(handler, response, status=status) diff --git a/api/streaming.py b/api/streaming.py index 76a39960..9c80b664 100644 --- a/api/streaming.py +++ b/api/streaming.py @@ -1255,11 +1255,113 @@ def _build_agent_thread_env(profile_runtime_env: dict | None, workspace: str, se 'HERMES_SESSION_KEY': session_id, 'HERMES_SESSION_ID': session_id, 'HERMES_SESSION_PLATFORM': 'webui', + # process_complete agent-wakeup wiring (ours-original, Option B): the + # terminal_tool watcher routing gate (terminal_tool.py:~1940) reads + # HERMES_SESSION_CHAT_ID to populate pending_watchers for WebUI + # sessions so notify_on_complete completions enqueue and the agent + # can be woken. HERMES_SESSION_ID/PLATFORM come from upstream #2279. + 'HERMES_SESSION_CHAT_ID': str(session_id), 'HERMES_HOME': profile_home, }) return env +# ── Per-turn session identity (xsession wakeup misroute root fix — Option 1) ─ +# WebUI bound per-turn session identity ONLY to the process-global +# os.environ['HERMES_SESSION_KEY'] (turn-start, line ~3263) and released the +# env lock BEFORE the agent ran. WebUI never called any contextvar setter, so +# gateway.session_context._SESSION_KEY stayed _UNSET and +# tools.approval.get_current_session_key (the EXACT call a +# notify_on_complete background spawn makes in terminal_tool.py:~1928) fell +# back to that racy process-global slot. Two concurrent WebUI turns therefore +# raced on one slot: session A's spawn could capture session B's id, and at +# completion the server-side wakeup turn started for the WRONG session +# (RCA t_f62ff1e8, agent.log:6632). The agent worker runs synchronously inside +# the _run_agent_streaming thread (concurrent tool batches use +# contextvars.copy_context() so children inherit this binding); binding the +# context-local here makes the capture task/thread-local and race-immune. +def _set_turn_session_identity(session_id: str): + """Bind THIS turn's session identity to the current (task/thread-local) + context and return an opaque token for _reset_turn_session_identity. + + Binds two context-locals so every session-key consumer is covered without + a race: + * ``tools.approval._approval_session_key`` — checked FIRST by + ``get_current_session_key`` (the exact call terminal_tool.py makes for + a notify_on_complete background spawn: the bug path). + * ``gateway.session_context._SESSION_KEY`` — read by direct + ``get_session_env("HERMES_SESSION_KEY")`` consumers (e.g. the sudo + password cache scope, terminal_tool.py:272). + + It deliberately does NOT call ``gateway.session_context.set_session_vars``: + that blanket setter also zeroes the platform/chat_id/user contextvars, + flipping ``HERMES_SESSION_PLATFORM`` from its env fallback (``'webui'``, + still written to os.environ at turn-start) to an explicit ``""`` — which + would break the ``notify_on_complete`` watcher registration gate in + terminal_tool.py:~1966. Only the session-key identity is bound; every + other session var keeps its existing os.environ fallback (CLI/cron compat + preserved — when these contextvars are _UNSET, get_session_env still falls + back to os.environ). + """ + sid = str(session_id or "") + tokens: dict = {} + try: + from tools.approval import set_current_session_key + tokens["approval"] = set_current_session_key(sid) + except Exception: + logger.debug("per-turn approval session-key bind failed", exc_info=True) + try: + from gateway.session_context import _SESSION_KEY as _SK + tokens["session_key"] = _SK.set(sid) + except Exception: + logger.debug("per-turn _SESSION_KEY bind failed", exc_info=True) + return tokens + + +def _reset_turn_session_identity(tokens) -> None: + """Restore the context-locals bound by ``_set_turn_session_identity`` via + contextvars reset-token semantics. + + Reset-token (not a blanket clear) is the canonical idiom: it composes + correctly under nesting and restores ``_UNSET`` for the top-level turn so + a reused thread-pool worker leaks no identity and CLI/cron env fallback + resumes. Order mirrors the bind in reverse. + """ + if not tokens: + return + tok = tokens.get("session_key") + if tok is not None: + try: + from gateway.session_context import _SESSION_KEY as _SK + _SK.reset(tok) + except Exception: + logger.debug("per-turn _SESSION_KEY reset failed", exc_info=True) + tok = tokens.get("approval") + if tok is not None: + try: + from tools.approval import reset_current_session_key + reset_current_session_key(tok) + except Exception: + logger.debug("per-turn approval session-key reset failed", exc_info=True) + + +@contextlib.contextmanager +def _bind_turn_session_identity(session_id: str): + """Context-manager form of the per-turn session-identity binding. + + The ``_run_agent_streaming`` worker uses the explicit ``_set``/``_reset`` + pair directly because its single ``try/finally`` already spans the whole + turn (~2k lines) and the binding must cover every mid-turn background + spawn; this wrapper is the canonical single-call API for other callers and + for tests, and shares the exact same code path. + """ + tokens = _set_turn_session_identity(session_id) + try: + yield + finally: + _reset_turn_session_identity(tokens) + + def _format_process_notification(evt: dict) -> str: """Format a completed background process notification for agent input.""" if not isinstance(evt, dict): @@ -4226,6 +4328,56 @@ def _sse(handler, event, data): handler.wfile.flush() +# ── SSE write deadline (Defect A: per-connection thread exhaustion) ───────── +# server.py runs QuietHTTPServer(ThreadingHTTPServer): one OS thread per +# connection, no pool cap (request_queue_size=64). Every SSE endpoint holds +# its thread for the connection's whole lifetime. If a tab is slow or +# backgrounded its TCP receive window fills; the next handler.wfile.write()/ +# flush() then blocks *indefinitely* (sockets have no write timeout by +# default). That thread is pinned forever — it never reaches its +# `finally: unsubscribe`, so the SessionChannel reaper can never reclaim the +# channel either. N such tabs * M sessions pile threads up until new +# requests queue past request_queue_size and the UI shows "streaming +# pending". +# +# Fix: arm a socket-level timeout on the connection. A genuinely healthy +# keepalive/event write completes in well under a millisecond, so a +# multi-second deadline never trips for a live tab; only a backpressured +# (stuck) socket blocks past it. When it trips, the write raises +# socket.timeout — which on Python 3.10+ *is* TimeoutError, already a member +# of api.routes._CLIENT_DISCONNECT_ERRORS — so each SSE handler's existing +# `except _CLIENT_DISCONNECT_ERRORS:` breaks the loop, `finally` drops the +# subscriber, the browser's EventSource auto-reconnects, and the OS thread +# is released. SessionChannel already supports reconnect + offline buffer, +# so no events are lost for a tab that comes back. Operators behind unusual +# proxies can tune the deadline without code changes. +try: + _raw_deadline = os.getenv("HERMES_WEBUI_SSE_WRITE_DEADLINE") or os.getenv("HERMES_SSE_WRITE_DEADLINE") + SSE_WRITE_DEADLINE_SECONDS = float(_raw_deadline or "20.0") +except (TypeError, ValueError): + SSE_WRITE_DEADLINE_SECONDS = 20.0 +if SSE_WRITE_DEADLINE_SECONDS <= 0: + SSE_WRITE_DEADLINE_SECONDS = 20.0 + + +def _sse_set_write_deadline(handler, seconds=None): + """Best-effort: arm a socket write deadline on an SSE handler. + + Call once, right after end_headers(), in every long-lived SSE endpoint. + Never raises — an unusual/missing transport just keeps the pre-fix + (no-deadline) behaviour for that single connection rather than breaking + the stream setup. + """ + if seconds is None: + seconds = SSE_WRITE_DEADLINE_SECONDS + try: + conn = getattr(handler, "connection", None) + if conn is not None and hasattr(conn, "settimeout"): + conn.settimeout(seconds) + except Exception: + logger.debug("Failed to arm SSE write deadline", exc_info=True) + + def _materialize_pending_user_turn_before_error(session) -> bool: """Persist the pending user prompt before clearing runtime stream state. @@ -4970,6 +5122,11 @@ def _run_agent_streaming( if _is_fallback_notice: put('warning', {'type': 'fallback', 'message': _message}) + # xsession wakeup misroute root fix (Option 1): pre-init so the outer + # finally can always reset even if an exception fires before the bind. + # Placed ABOVE the _checkpoint_stop cluster so that cluster stays adjacent + # to the `try:` (preserves the Issue #765 static-locator invariant). + _turn_session_identity_tokens = None # Initialised here (before any code that may raise) so the outer `finally` # block can safely check `if _checkpoint_stop is not None` even when an # exception fires before the checkpoint thread is created (Issue #765). @@ -4977,6 +5134,12 @@ def _run_agent_streaming( _ckpt_thread = None _agent_lock = None try: + # Bind THIS turn's session identity to the worker thread/context BEFORE + # any agent work (so every mid-turn notify_on_complete background spawn + # captures THIS session, not a concurrent turn's process-global env). + # Co-located with the existing env-restore lifecycle: set here, reset + # in the outer finally next to _clear_thread_env(). + _turn_session_identity_tokens = _set_turn_session_identity(session_id) s = get_session(session_id) update_active_run(stream_id, phase="running", session_id=session_id) s.workspace = str(Path(workspace).expanduser().resolve()) @@ -5052,7 +5215,15 @@ def _run_agent_streaming( _profile_home, ) _set_thread_env(**_thread_env) - # Prewarm skill-tool imports *before* acquiring the lock so that + # process_complete agent-wakeup wiring (ours-original, Option B): bind + # this session's HERMES_SESSION_KEY to its WebUI session_id so the + # drain thread can route notify_on_complete events back to the right + # SSE channel / server-side wakeup. + try: + from api.background_process import register_process_session + register_process_session(session_id, session_id) + except Exception: + logger.debug("register_process_session failed", exc_info=True) # first-time module initialisation (which can be slow) does not # block other concurrent sessions waiting on _ENV_LOCK (#2024). _prewarm_skill_tool_modules() @@ -5067,6 +5238,7 @@ def _run_agent_streaming( old_session_key = os.environ.get('HERMES_SESSION_KEY') old_session_id = os.environ.get('HERMES_SESSION_ID') old_session_platform = os.environ.get('HERMES_SESSION_PLATFORM') + old_session_chat_id = os.environ.get('HERMES_SESSION_CHAT_ID') old_hermes_home = os.environ.get('HERMES_HOME') os.environ.update(_profile_runtime_env) os.environ['TERMINAL_CWD'] = str(s.workspace) @@ -5074,6 +5246,9 @@ def _run_agent_streaming( os.environ['HERMES_SESSION_KEY'] = session_id os.environ['HERMES_SESSION_ID'] = session_id os.environ['HERMES_SESSION_PLATFORM'] = 'webui' + # process_complete wiring (ours-original, Option B): see + # _build_agent_thread_env above. + os.environ['HERMES_SESSION_CHAT_ID'] = str(session_id) if _profile_home: os.environ['HERMES_HOME'] = _profile_home # Patch module-level caches to match the active profile. @@ -7361,6 +7536,8 @@ def _run_agent_streaming( else: os.environ['HERMES_SESSION_ID'] = old_session_id if old_session_platform is None: os.environ.pop('HERMES_SESSION_PLATFORM', None) else: os.environ['HERMES_SESSION_PLATFORM'] = old_session_platform + if old_session_chat_id is None: os.environ.pop('HERMES_SESSION_CHAT_ID', None) + else: os.environ['HERMES_SESSION_CHAT_ID'] = old_session_chat_id if old_hermes_home is None: os.environ.pop('HERMES_HOME', None) else: os.environ['HERMES_HOME'] = old_hermes_home @@ -7588,6 +7765,12 @@ def _run_agent_streaming( update_active_run(stream_id, phase="finalizing") _last_resort_sync_from_core(s, stream_id, _agent_lock) _clear_thread_env() # TD1: always clear thread-local context + # xsession wakeup misroute root fix (Option 1): restore the per-turn + # session-identity context-locals (reset-token semantics). MUST run on + # every exit path so a reused thread-pool worker leaks no identity and + # CLI/cron env fallback resumes — same lifecycle slot as the env + # restore above. + _reset_turn_session_identity(_turn_session_identity_tokens) with STREAMS_LOCK: STREAMS.pop(stream_id, None) CANCEL_FLAGS.pop(stream_id, None) @@ -7608,6 +7791,36 @@ def _run_agent_streaming( # the next stream can read it, breaking the goal-continuation # chain. Stage-326 critical fix per Opus advisor review. + # ── Defer-path fix: turn-teardown idle-hook ──────────────────────── + # The session has just transitioned active→idle: unregister_active_run + # above cleared this stream's ACTIVE_RUNS row (under ACTIVE_RUNS_LOCK, + # independent of STREAMS_LOCK), so _session_has_active_turn() is now + # False for this session unless a *different* stream is still active + # (cancel/reconnect — drain_deferred_wakeups_for_session guards on + # that and leaves the marker for the later teardown). A FAST + # background task that completed while this turn was tearing down was + # deferred by api/background_process._process_one (it could not start + # a turn → would 409) and its wakeup_prompt persisted in + # DEFERRED_PROCESS_WAKEUPS. For an autonomous agent there is no next + # user turn, so the PR #2279 next-turn drain never runs; without this + # hook the deferred wakeup is lost forever (the Test B failure). This + # makes the busy-at-completion case symmetric with the idle case: + # idle now → fire now (Option Z idle branch); busy now → fire here at + # turn-end. claim_deferred_wakeups pops atomically, so this is + # idempotent with the next-turn drain (no double-fire) and the wakeup + # turn's own teardown finds nothing claimed (no wakeup loop). The + # drain spawns its own daemon thread, so teardown never blocks. + try: + from api.background_process import drain_deferred_wakeups_for_session + + drain_deferred_wakeups_for_session(session_id) + except Exception: + logger.debug( + "turn-teardown deferred-wakeup drain failed for session %s", + session_id, + exc_info=True, + ) + # ============================================================ # SECTION: HTTP Request Handler # do_GET: read-only API endpoints + SSE stream + static HTML diff --git a/server.py b/server.py index 8064c953..fc8f4360 100644 --- a/server.py +++ b/server.py @@ -586,6 +586,27 @@ def main() -> None: except Exception as e: print(f'[!!] WARNING: Gateway watcher failed to start: {e}', flush=True) + # Start the bg_task_complete drain thread for terminal(notify_on_complete=true) + # agent wakeup. Reads tools.process_registry.completion_queue and emits SSE + # bg_task_complete events (canonical name; legacy process_complete alias is + # still emitted for back-compat) to the matching session's stream. + try: + from api.background_process import start_drain_thread + if start_drain_thread(): + print('[ok] bg_task_complete drain thread started', flush=True) + except Exception as e: + print(f'[!!] WARNING: bg_task_complete drain failed to start: {e}', flush=True) + + # Start the SessionChannel reaper for the persistent per-session SSE + # endpoint (/api/session/stream). Runs every 60s, collects channels with + # no subscribers past the grace period or past the idle TTL cap. + try: + from api.background_process import start_session_channel_reaper + if start_session_channel_reaper(): + print('[ok] SessionChannel reaper thread started', flush=True) + except Exception as e: + print(f'[!!] WARNING: SessionChannel reaper failed to start: {e}', flush=True) + # Load WebUI dashboard plugins try: from api.plugins import load_plugins @@ -633,6 +654,20 @@ def main() -> None: drain_all_on_shutdown() except Exception: logger.debug("Failed to drain lifecycle on shutdown", exc_info=True) + # Stop bg_task_complete drain + SessionChannel reaper (ours-original). + # The drain thread emits the canonical ``bg_task_complete`` event + # (with ``process_complete`` kept as a temporary backward-compat + # alias for older clients — see start_drain_thread comment above). + try: + from api.background_process import stop_drain_thread + stop_drain_thread() + except Exception: + logger.debug("Failed to stop bg_task_complete drain thread during shutdown", exc_info=True) + try: + from api.background_process import stop_session_channel_reaper + stop_session_channel_reaper() + except Exception: + logger.debug("Failed to stop SessionChannel reaper during shutdown", exc_info=True) if __name__ == '__main__': main() diff --git a/static/messages.js b/static/messages.js index b921e15c..befa81f3 100644 --- a/static/messages.js +++ b/static/messages.js @@ -4,6 +4,52 @@ function _markSessionViewed(sid, messageCount) { _setSessionViewedCount(sid, next); } +function _apiUrl(path) { + return new URL(path, document.baseURI || location.href).href; +} + +// Module-scope dedupe ring buffer for bg_task_complete events. Shared between +// the in-turn STREAMS path (per-turn EventSource inside the chat-stream wirer) +// and the persistent session-scoped path (/api/session/stream), so the +// frontend never double-fires a toast or ack for the same (session_id, +// event_id) regardless of which channel delivered it first. (Option X) +// +// Keyed by `${session_id}|${event_id}` → expiry timestamp (ms since epoch). +// Bounded by a 60-second TTL plus a 256-entry soft cap with insertion-order +// eviction on overflow. Events without `event_id` are ignored by the caller +// (the server contract guarantees `event_id` on every completion emit). +const _BG_TASK_COMPLETE_TTL_MS = 60000; +const _BG_TASK_COMPLETE_CAP = 256; +const _bgTaskCompleteSeenIds = new Map(); + +function _bgTaskCompleteRingBufferAdd(sid, evt_id) { + // Missing key → treat as "seen/skip" (return true). The sole caller already + // guards with `if (!evt_id) return;` before invoking this, so this branch is + // defensive: returning true (skip) rather than false (proceed) means a + // future call site that forgets that guard drops the un-keyable event + // instead of processing a completion with no dedupe key. + if (!sid || !evt_id) return true; + const key = sid + '|' + evt_id; + const now = Date.now(); + // Lazy purge: walk insertion-order; drop any entry whose expiry has passed. + // Map iteration is insertion-order so this also surfaces the oldest entries + // first when we need to evict for the soft cap below. + for (const [k, exp] of _bgTaskCompleteSeenIds) { + if (exp <= now) { + _bgTaskCompleteSeenIds.delete(k); + } + } + if (_bgTaskCompleteSeenIds.has(key)) return true; // duplicate + _bgTaskCompleteSeenIds.set(key, now + _BG_TASK_COMPLETE_TTL_MS); + // Soft cap: insertion-order eviction. + while (_bgTaskCompleteSeenIds.size > _BG_TASK_COMPLETE_CAP) { + const firstKey = _bgTaskCompleteSeenIds.keys().next().value; + if (firstKey === undefined) break; + _bgTaskCompleteSeenIds.delete(firstKey); + } + return false; +} + function _isDocumentVisibleAndFocused() { if(typeof document!=='undefined' && document.visibilityState && document.visibilityState!=='visible') return false; if(typeof document!=='undefined' && typeof document.hasFocus==='function' && !document.hasFocus()) return false; @@ -2971,6 +3017,29 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){ }catch(_){} }); + // bg_task_complete: terminal(notify_on_complete=true) background process + // exited. Option Z PIVOT: the agent wakeup is started SERVER-SIDE by the + // drain thread (api/background_process._process_one → + // routes.start_session_turn) with NO browser round-trip — so the + // closed-tab case works (parity with CLI/Telegram). The browser does NOT + // re-POST /api/chat/start anymore. This SSE event is pure LIVE-VIEW: if + // a tab is open the server-initiated turn streams live via the normal + // /api/chat/stream EventSource; if the tab is closed the turn still runs + // server-side and persists to the session store. + // + // Idempotency: dedupe by (session_id, event_id) via a Map+TTL ring + // buffer (`_bgTaskCompleteRingBufferAdd`). + // + // Option X: this handler is the in-turn (STREAMS-bound) path. The server + // dual-emits to the persistent session-scoped channel too — the + // `_handleBgTaskCompleteEvent` function below is shared between both + // paths (dedupe only; the wakeup itself is server-side). + source.addEventListener('bg_task_complete',e=>{ + if(typeof _handleBgTaskCompleteEvent==='function'){ + _handleBgTaskCompleteEvent(e, activeSid, {source:'stream'}); + } + }); + source.addEventListener('done',e=>{ if(_streamFinalized) return; if(_bailOutOfTerminalEventsFromStaleStream(source)) return; @@ -4056,6 +4125,194 @@ function stopApprovalPolling() { _approvalPollingSessionId = null; } +// ── Session-scoped SSE stream (Option X) ────────────────────────────────── +// Long-lived EventSource bound to /api/session/stream?session_id=. +// Lives across agent turns (unlike the per-turn /api/chat/stream which is +// torn down at end-of-turn). Carries bg_task_complete events fired while no +// turn is active — the architectural fix for the notify_on_complete wakeup +// gap that #2242 + #2279 papered over. +// +// Lifecycle: opened on session mount (loadSession / newSession), closed on +// session switch / unmount. The browser closes it implicitly on tab close +// (server detects disconnect via the SSE read-loop and unsubscribes). +let _sessionEventSource = null; +let _sessionStreamSessionId = null; +let _sessionStreamReconnectTimer = null; + +function startSessionStream(sid) { + if (!sid) return; + // Already on this session? No-op (loadSession is a no-op when re-selecting + // the same session; this defends against external re-callers). + if (_sessionStreamSessionId === sid && _sessionEventSource) return; + stopSessionStream(); + _sessionStreamSessionId = sid; + try { + const es = new EventSource(_apiUrl('api/session/stream?session_id=' + encodeURIComponent(sid))); + _sessionEventSource = es; + es.addEventListener('initial', () => { /* connection confirmed */ }); + es.addEventListener('bg_task_complete', e => { + // Shared handler — same dedupe set as the in-turn STREAMS path. + if (typeof _handleBgTaskCompleteEvent === 'function') { + _handleBgTaskCompleteEvent(e, sid, {source: 'session'}); + } + }); + // ── Defect B: live-view of server-initiated (Option Z) turns ────────── + // The drain thread starts the wakeup turn server-side and the server + // fans a `server_turn_started` {stream_id} frame onto this per-session + // channel. No browser POSTed /api/chat/start, so nothing is attached to + // that STREAMS[stream_id] yet. Attach the EXISTING chat-stream renderer + // (attachLiveStream — the exact path /api/chat/start uses) to the + // server-created stream so the open tab renders the turn live. Reuses + // the one renderer; does NOT hand-roll a second one. + es.addEventListener('server_turn_started', e => { + try { + const d = JSON.parse(e.data || '{}'); + const evSid = d.session_id || sid; + const streamId = String(d.stream_id || ''); + if (!streamId || evSid !== sid) return; + // `recovered` marks an on-subscribe replay from the server: the tab + // (re)connected to /api/session/stream AFTER the original + // fire-and-forget server_turn_started had already been broadcast, so + // the live stream is mid-flight. Attach via the reconnecting (replay) + // path so the renderer rebuilds from the run journal instead of + // expecting token 0 (which would render a truncated turn). A fresh + // (non-recovered) frame still attaches from the first token. + const recovered = !!d.recovered; + // Only drive the renderer when this session is the one on screen. + const isCurrent = (typeof _isSessionCurrentPane === 'function') + ? _isSessionCurrentPane(sid) + : (S.session && S.session.session_id === sid); + if (!isCurrent) return; + // A turn is already rendering in this tab (user-initiated, or we + // already attached to this very stream). attachLiveStream is + // idempotent per (sid, streamId); bail if we're already on it. + if (S.activeStreamId === streamId) return; + const existingLive = (typeof LIVE_STREAMS !== 'undefined') ? LIVE_STREAMS[sid] : null; + if (existingLive && existingLive.streamId === streamId) return; + // Mirror the loadSession reattach setup. For a fresh frame the turn + // renders from its first token; for a recovered (replay) frame + // attachLiveStream reconstructs the in-progress stream. + S.busy = true; + S.activeStreamId = streamId; + if (S.session && S.session.session_id === sid) S.session.active_stream_id = streamId; + if (typeof updateSendBtn === 'function') updateSendBtn(); + if (typeof setComposerStatus === 'function') setComposerStatus(''); + if (typeof syncTopbar === 'function') syncTopbar(); + if (typeof appendThinking === 'function') appendThinking(); + if (typeof startApprovalPolling === 'function') startApprovalPolling(sid); + if (typeof startClarifyPolling === 'function') startClarifyPolling(sid); + if (typeof attachLiveStream === 'function') { + attachLiveStream( + sid, streamId, + (S.session && S.session.pending_attachments) || [], + recovered ? {reconnecting: true} : {}, + ); + } + if (typeof renderSessionList === 'function') void renderSessionList(); + } catch (_) {} + }); + es.onerror = () => { + // Browser already auto-reconnects EventSource on most transient + // failures. We only intervene if the connection has been closed for + // good (readyState === 2) — schedule a one-shot re-open after 5s. + if (es.readyState === 2 && _sessionStreamSessionId === sid) { + if (_sessionStreamReconnectTimer) clearTimeout(_sessionStreamReconnectTimer); + // The CLOSED EventSource (readyState === 2) will never reconnect on + // its own, and startSessionStream's top guard + // (`_sessionStreamSessionId === sid && _sessionEventSource`) would + // short-circuit the re-open while this dead object is still pinned. + // Drop our reference (and close it for good measure) so the timer's + // startSessionStream() reaches stopSessionStream() and builds a FRESH + // EventSource instead of reusing the closed one. Only clear if `es` + // is still the active source — a newer connection may have replaced + // it in the interim (stale onerror from a superseded stream), in + // which case we must not stomp the live one. + if (_sessionEventSource === es) { + try { es.close(); } catch (_) {} + _sessionEventSource = null; + } + _sessionStreamReconnectTimer = setTimeout(() => { + _sessionStreamReconnectTimer = null; + if (_sessionStreamSessionId === sid) startSessionStream(sid); + }, 5000); + } + }; + } catch(_) { + // EventSource ctor threw — silently disabled; the in-turn STREAMS path + // still works for events that fire during an active turn. + _sessionEventSource = null; + } +} + +function stopSessionStream() { + if (_sessionStreamReconnectTimer) { clearTimeout(_sessionStreamReconnectTimer); _sessionStreamReconnectTimer = null; } + if (_sessionEventSource) { + try { _sessionEventSource.close(); } catch(_){} + _sessionEventSource = null; + } + _sessionStreamSessionId = null; +} + +// Shared bg_task_complete handler — invoked from BOTH the in-turn STREAMS +// channel (legacy path, still kept as defense-in-depth) AND the session- +// scoped channel (Option X primary path). Dedupes by (session_id, event_id) +// via the Map+TTL ring buffer declared at the top of this module. +// Events without `event_id` are ignored — the server contract guarantees one +// on every completion emit, so a missing key signals a malformed or replayed +// payload we should not surface or ack. +// PR (c) UX surface: post-dedupe the handler marks the session viewed (when +// the session pane is current and the doc is visible+focused), then runs the +// T4 drop-when-focused gate; only out-of-focus or off-pane completions spawn +// a toast. The diagnostic ack POST still fires for both focused and +// unfocused viewers so the server receives the delivery/cleanup signal; +// the focus gate suppresses UI noise only. +function _handleBgTaskCompleteEvent(e, expectedSid, opts) { + try { + const d = JSON.parse(e.data || '{}'); + const sid = d.session_id || expectedSid; + if (sid !== expectedSid) return; + const evt_id = d.event_id ? String(d.event_id) : ''; + if (!evt_id) return; // server contract requires event_id; ignore otherwise + if (_bgTaskCompleteRingBufferAdd(sid, evt_id)) return; // duplicate + const pid = String(d.task_id || ''); + const _viewed = typeof _isSessionActivelyViewed === 'function' && _isSessionActivelyViewed(sid); + if (_viewed) { + try { _markSessionViewed(sid, (S&&S.session&&S.session.session_id===sid)?(S.session.message_count??(S.messages&&S.messages.length)??0):0); } catch(_){} + try { if(typeof _clearSessionCompletionUnread==='function') _clearSessionCompletionUnread(sid); } catch(_){} + } else { + // T4 drop-when-focused: suppress toast only; ack below still fires. + try { + const tid = (d.task_id || '').slice(0, 8) || '?'; + const tail = d.summary ? `: ${String(d.summary).slice(0, 80)}` : ''; + showToast(`Task ${tid} done${tail}`, 2600); + } catch (_) {} + } + + // Fire-and-forget ack (diagnostic only — Option Z made this a no-op for + // state. The agent wakeup is now started SERVER-SIDE by the drain thread + // in api/background_process._process_one → start_session_turn; the + // browser is no longer in the wakeup path at all.) + try { + fetch(_apiUrl('api/bg-task-complete-ack'), { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + credentials: 'include', + body: JSON.stringify({session_id: sid, task_id: pid, event_id: evt_id}), + }).catch(() => {}); + } catch(_) {} + + // Option Z PIVOT: the browser NO LONGER re-POSTs the chat-start endpoint + // to wake the agent. Server-side wakeup is the PRIMARY mechanism — the + // drain thread starts the turn directly (no tab required), so the + // closed-tab case works (parity with CLI/Telegram). The per-session SSE + // channel this handler is wired into is DEMOTED to a pure live-view + // layer: if a tab is open the server-initiated turn streams live via the + // existing chat-stream EventSource; if the tab is closed the turn still + // runs server-side and the result is persisted to the session store. + // The user-facing toast + drop-when-focused gate land in PR (c). + } catch(_) {} +} + // ── Clarify polling ── let _clarifyPollTimer = null; let _clarifyHideTimer = null; diff --git a/static/sessions.js b/static/sessions.js index 282c8d24..36409307 100644 --- a/static/sessions.js +++ b/static/sessions.js @@ -666,6 +666,7 @@ async function newSession(flash, options={}){ if(flash)S.session._flash=true; try{localStorage.setItem('hermes-webui-session',S.session.session_id);}catch(_){} _setActiveSessionUrl(S.session.session_id); + if(typeof startSessionStream==='function') startSessionStream(S.session.session_id); _setSessionViewedCount(S.session.session_id, S.session.message_count || 0); // Sync chat-header dropdown to the session's model/provider so the UI reflects // the default route the server actually used (#872). Compare provider state too: @@ -731,6 +732,25 @@ async function newSession(flash, options={}){ } } +// #2971 (Greptile P1 r3377162160): loadSession() tears down the live +// per-session SSE at the top via stopSessionStream() (line ~754), but only the +// success path re-arms it via startSessionStream() (line ~875). Every +// early-return exit (fetch error, auth-redirect undefined) — and the +// same-session no-op guard, which returns BEFORE the teardown — could leave +// the session the user actually remains on with a permanently null +// EventSource, silently dropping bg_task_complete delivery until a full page +// reload or a forced loadSession. This helper re-arms the stream for whatever +// session is currently on screen (S.session). startSessionStream() is +// idempotent — it no-ops when already live for that sid (top guard +// `_sessionStreamSessionId === sid && _sessionEventSource`) — so this never +// double-arms the success path, which arms the *newly assigned* S.session +// only after this point. +function _rearmActiveSessionStream(){ + if(typeof startSessionStream!=='function') return; + const activeSid = S.session ? S.session.session_id : null; + if(activeSid) startSessionStream(activeSid); +} + async function loadSession(sid){ const opts = arguments[1] || {}; if(!opts.skipLineageResolve && typeof _resolveSessionIdFromSidebarLineage==='function'){ @@ -747,11 +767,15 @@ async function loadSession(sid){ // Do not no-op a same-session click while another load is in flight: the // previous transcript may already have been cleared for the pending switch. // Static force-reload invariant: if(currentSid===sid && !forceReload) return; + // #2971: idempotent re-arm before the no-op guard revives a stream a prior + // failed loadSession killed; no-ops on real switches. + _rearmActiveSessionStream(); if(currentSid===sid && !forceReload && !_loadingSessionId) return; // Mark this session as the in-flight load. Subsequent loadSession() calls // will overwrite this; stale awaits use the mismatch to bail out (#1060). _loadingSessionId = sid; stopApprovalPolling();hideApprovalCard(forceReload); + if(typeof stopSessionStream==='function') stopSessionStream(); _yoloEnabled=false;_updateYoloPill(); if(typeof stopClarifyPolling==='function') stopClarifyPolling(); if(typeof hideClarifyCard==='function') hideClarifyCard(forceReload, forceReload?'external-refresh':'dismissed'); @@ -841,7 +865,32 @@ async function loadSession(sid){ } } _clearSameSessionForceReloadHint(sid); + // Capture whether this failure self-healed away the current session (a + // 404 on the *current* session whose sidecar was deleted server-side). + // In that case there is no live session left to stream for, so we must + // NOT restart — doing so would spin the SSE reconnect loop against a dead + // session_id. + const _selfHealedCurrent = (e.status===404) && (currentSid===sid); if (_loadingSessionId === sid) _loadingSessionId = null; + // The session stream was stopped unconditionally at the top of this load + // (mirroring stopApprovalPolling). On the happy path it's restarted ~120 + // lines below, but this failure exit never reaches that point — leaving + // the session still on screen permanently silenced. bg_task_complete + // events (the new feature's primary delivery path) would be dropped until + // the user explicitly navigates to a session again. Restart the stream for + // the session that remains on screen. Skip when a newer load is already in + // flight (_loadingSessionId !== null after the reset above): that load owns + // the stream and starts its own. Skip the self-healed-current case (no live + // session to stream). + // #2971: this fetch-error path keeps its bespoke guarded restart (rather + // than the shared _rearmActiveSessionStream helper used on the other + // early-returns) because only here can the current session have just + // self-healed away — re-arming a 404'd/deleted session_id would spin the + // SSE reconnect loop against a dead session. + if (currentSid && !_selfHealedCurrent && _loadingSessionId === null + && typeof startSessionStream === 'function') { + startSessionStream(currentSid); + } return; } // Guard: api() may have redirected (401) and returned undefined; in that case @@ -849,10 +898,21 @@ async function loadSession(sid){ if (!data) { _clearSameSessionForceReloadHint(sid); if (_loadingSessionId === sid) _loadingSessionId = null; + // #2971: re-arm the still-displayed session's stream (defensive — harmless + // if the 401 redirect is already tearing the page down). Idempotent. + _rearmActiveSessionStream(); return; } // Stale response? A newer loadSession() call has already started (#1060). - if (_loadingSessionId !== sid) return; + if (_loadingSessionId !== sid) { + // #2971: a newer in-flight load owns the final stream arming, but until it + // assigns S.session and reaches startSessionStream() the currently-shown + // session must not be left stream-dead by our top-of-function teardown. + // Re-arm the genuinely-displayed S.session (idempotent — no-ops once the + // newer load arms its own sid). + _rearmActiveSessionStream(); + return; + } S.session=data.session; if(typeof _hydrateTodosFromSession==='function') _hydrateTodosFromSession(S.session); S.session._modelResolutionDeferred=true; @@ -872,6 +932,7 @@ async function loadSession(sid){ _clearSessionCompletionUnread(S.session.session_id); try{localStorage.setItem('hermes-webui-session',S.session.session_id);}catch(_){} _setActiveSessionUrl(S.session.session_id); + if(typeof startSessionStream==='function') startSessionStream(S.session.session_id); const activeStreamId=S.session.active_stream_id||null; // If the server says the session is idle, discard any browser-side inflight diff --git a/tests/_wakeup_helpers.py b/tests/_wakeup_helpers.py new file mode 100644 index 00000000..5b3757ee --- /dev/null +++ b/tests/_wakeup_helpers.py @@ -0,0 +1,96 @@ +"""Shared test helpers for the server-side wakeup test suites. + +Consolidates the ``_install_fake_start_session_turn`` / ``_wait_for_wakeup`` +pair that ``test_session_channel_option_x.py`` and ``test_wakeup_defer_race.py`` +both need so the two suites can't drift. Per Copilot review on PR #2971 +(r3305700944). +""" +from __future__ import annotations + +import queue +import sys +import threading +import types + + +class FakeProcessRegistry: + """Minimal stand-in for ``tools.process_registry.process_registry``. + + Consolidated from the verbatim ``_FakeProcessRegistry`` copies that lived + in ``test_bg_task_complete_wakeup.py``, ``test_bg_task_complete_throttle.py`` + and ``test_bg_task_complete_ab_coexistence.py`` (Greptile review on PR #2979). + Keeping one definition prevents the three suites from drifting to subtly + different stub shapes. + """ + + def __init__(self): + self._lock = threading.Lock() + self._completion_consumed: set[str] = set() + self.completion_queue: queue.Queue = queue.Queue() + self._procs: dict[str, types.SimpleNamespace] = {} + + def register(self, process_id: str, session_key: str) -> None: + self._procs[process_id] = types.SimpleNamespace(session_key=session_key) + + def get(self, process_id: str): + return self._procs.get(process_id) + + def is_completion_consumed(self, process_id: str) -> bool: + with self._lock: + return process_id in self._completion_consumed + + +def install_fake_registry(monkeypatch, fake) -> None: + """Inject ``fake`` under ``tools.process_registry`` for the test. + + IMPORTANT (rebase isolation): uses ONLY ``monkeypatch.setitem`` so both + ``sys.modules`` entries are restored to their real/absent state on + teardown. A prior implementation used ``sys.modules.setdefault("tools", …)`` + which is an UNTRACKED mutation — when the real ``tools`` package was not yet + imported it permanently leaked a non-package fake ``tools`` into + ``sys.modules``, breaking any later test doing + ``from tools.process_registry import …``. Both setitem calls below are + monkeypatch-tracked: on teardown each key is restored to its prior value, + or deleted if it was absent — no leak. + """ + mod = types.ModuleType("tools.process_registry") + mod.process_registry = fake # type: ignore[attr-defined] + tools_mod = types.ModuleType("tools") + tools_mod.process_registry = mod # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "tools", tools_mod) + monkeypatch.setitem(sys.modules, "tools.process_registry", mod) + + +def install_fake_start_session_turn(monkeypatch, *, status: int = 200): + """Patch ``api.routes.start_session_turn`` to record calls instead of + running a real agent turn. + + The drain helper does ``from api.routes import start_session_turn`` + inside a daemon thread, so patching the attribute on the ``api.routes`` + module is what the thread resolves at call time. + + Returns a ``holder`` dict with ``calls`` (list of recorded call kwargs) + and ``event`` (a ``threading.Event`` set on first call) — pair it with + ``wait_for_wakeup`` below. + """ + import api.routes as _routes + + holder = {"calls": [], "event": threading.Event()} + + def _fake(session_id, message, *, source="process_wakeup"): + holder["calls"].append( + {"session_id": session_id, "message": message, "source": source} + ) + holder["event"].set() + return {"stream_id": "fake-stream", "session_id": session_id, "_status": status} + + monkeypatch.setattr(_routes, "start_session_turn", _fake, raising=True) + return holder + + +def wait_for_wakeup(holder, timeout: float = 3.0) -> bool: + """Block until the server-side wakeup runner thread recorded a call. + + Returns True if the holder's event fired within ``timeout`` seconds. + """ + return holder["event"].wait(timeout=timeout) diff --git a/tests/manual/repro_wakeup_hang.py b/tests/manual/repro_wakeup_hang.py new file mode 100644 index 00000000..be0829fe --- /dev/null +++ b/tests/manual/repro_wakeup_hang.py @@ -0,0 +1,166 @@ +"""Deterministic BEFORE/AFTER repro for the wakeup model-resolve hang. + +Simulates the proven thread-stack: a server-initiated Option-Z wakeup turn +reaches start_session_turn with a COLD provider catalog; the live rebuild's +per-provider probe (the Copilot token-exchange HTTPS call) is monkeypatched to +hang. We drive start_session_turn directly (no browser, no real agent) and +measure how long model-resolution takes. + +BEFORE (legacy behaviour: HERMES_WEBUI_MODELS_REBUILD_BUDGET=0 AND +prefer_cache forced off): the wakeup blocks on the hung probe — exactly the +"stuck at resolve_model_provider" symptom. + +AFTER (shipped behaviour): start_session_turn resolves with +prefer_cached_catalog=True (never touches the live rebuild) AND the rebuild is +budget-bounded as defense-in-depth -- the wakeup turn starts in well under a +second using the persisted session model. + +Run (from the repo root, with the Hermes Agent venv python -- any interpreter +that can import this repo's ``api`` package works): + + python tests/manual/repro_wakeup_hang.py + +Exits 0 on PASS, 1 on FAIL. Not collected by pytest (see +tests/manual/conftest.py); this is an operator-run reproduction, not a test. +""" + +from __future__ import annotations + +import os as _os +import sys as _sys + +# Repo root is two levels up: /tests/manual/repro_wakeup_hang.py +_REPO_ROOT = _os.path.dirname( + _os.path.dirname(_os.path.dirname(_os.path.abspath(__file__))) +) +if _REPO_ROOT not in _sys.path: + _sys.path.insert(0, _REPO_ROOT) + +import sys +import time + +HANG_SECONDS = 30.0 # stand-in for an unreachable Copilot endpoint + + +def _install_fakes(): + import api.config as cfg + import api.routes as routes + + cfg.invalidate_models_cache() + + # Inject the hang at the cold-rebuild seam. This is faithful to the proven + # root cause (the live per-provider rebuild — which on the real host does + # the Copilot token-exchange HTTPS call — is what blocks) and is + # deterministic regardless of which providers the isolated HERMES_HOME has + # configured (an isolated home with no providers can't reach the real + # _read_live_provider_model_ids path; precedent t_9f0184cf). + def _hung_rebuild(_builder): + time.sleep(HANG_SECONDS) + return { + "active_provider": "anthropic", + "default_model": "anthropic/claude-sonnet-4", + "configured_model_badges": {}, + "groups": [], + } + + cfg._invoke_models_rebuild = _hung_rebuild + # No disk cache → forces the cold rebuild branch. + cfg._load_models_cache_from_disk = lambda: None + + fake_stream = "stream-repro-1" + captured = {} + + def _fake_start(s, **kwargs): + captured["model"] = kwargs.get("model") + return {"stream_id": fake_stream, "session_id": s.session_id, "_status": 200} + + class _FakeSession: + session_id = "sess-repro" + model = "anthropic/claude-sonnet-4" + model_provider = "anthropic" + + routes._start_chat_stream_for_session = _fake_start + routes.get_session = lambda _s: _FakeSession() + routes._resolve_chat_workspace_with_recovery = lambda s, w: "/tmp/ws" + return routes, captured, fake_stream + + +def _time_call(label, fn, timeout): + import threading + + box = {} + + def _run(): + box["resp"] = fn() + + t = threading.Thread(target=_run, daemon=True) + t0 = time.monotonic() + t.start() + t.join(timeout=timeout) + elapsed = time.monotonic() - t0 + if t.is_alive(): + print(f" {label}: STUCK — still blocked after {elapsed:.1f}s " + f"(the wakeup turn never starts; matches the bug symptom)") + return None, elapsed, True + print(f" {label}: started in {elapsed:.3f}s " + f"(model={box.get('resp', {}).get('stream_id')!r})") + return box.get("resp"), elapsed, False + + +def main(): + print("=== BEFORE (legacy: budget=0, prefer_cache forced OFF) ===") + import os + os.environ["HERMES_WEBUI_MODELS_REBUILD_BUDGET"] = "0" + # Reimport config so the budget constant picks up env=0. + for m in ("api.config", "api.routes"): + sys.modules.pop(m, None) + routes, captured, _ = _install_fakes() + + # Force the legacy code path: bypass the prefer_cache short-circuit by + # calling resolve with prefer_cached_catalog=False, like the OLD code did. + _orig_resolve = routes._resolve_compatible_session_model_state + routes._resolve_compatible_session_model_state = ( + lambda m, p, **_k: _orig_resolve(m, p, prefer_cached_catalog=False) + ) + _, _, stuck_before = _time_call( + "wakeup chat/start", + lambda: routes.start_session_turn( + "sess-repro", "[IMPORTANT: bg done]", source="process_wakeup" + ), + timeout=8.0, + ) + + print() + print("=== AFTER (shipped: prefer_cached_catalog=True + bounded rebuild) ===") + os.environ["HERMES_WEBUI_MODELS_REBUILD_BUDGET"] = "4" + for m in ("api.config", "api.routes"): + sys.modules.pop(m, None) + import api.config as cfg2 # noqa: F401 + routes2, captured2, fake_stream = _install_fakes() + resp, elapsed_after, stuck_after = _time_call( + "wakeup chat/start", + lambda: routes2.start_session_turn( + "sess-repro", "[IMPORTANT: bg done]", source="process_wakeup" + ), + timeout=8.0, + ) + + print() + print("=== RESULT ===") + ok = ( + stuck_before is True + and stuck_after is False + and resp is not None + and resp.get("stream_id") == fake_stream + and captured2.get("model") == "anthropic/claude-sonnet-4" + and elapsed_after < 2.0 + ) + print(f" BEFORE stuck on hung probe: {stuck_before}") + print(f" AFTER started fast: {not stuck_after} " + f"({elapsed_after:.3f}s, persisted model={captured2.get('model')!r})") + print(f" REPRO {'PASS' if ok else 'FAIL'}") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_bg_task_complete_ab_coexistence.py b/tests/test_bg_task_complete_ab_coexistence.py new file mode 100644 index 00000000..a389146d --- /dev/null +++ b/tests/test_bg_task_complete_ab_coexistence.py @@ -0,0 +1,319 @@ +"""Integration tests: the merged upstream PR #2279 (next-turn drain, A) + +our-original Option B SSE/server-side drain coexist without duplicating +wakeups for the same background process_id. + +These tests verify the shared dedupe contract via the REAL merged upstream +key — process_registry._completion_consumed (checked by +process_registry.is_completion_consumed()): +- If B's drain fires first (proactive case), it marks the registry + consumed-marker so A's next-turn drain skips the same process_id. +- If A's (real merged #2279) drain fires first (SSE-disconnected case), it + marks the same registry consumed-marker so B's drain early-returns. + +api.config.BG_TASK_COMPLETE_EVENTS_SEEN remains as B's own private +secondary dedupe (duplicate enqueue within this module) but is NOT the +cross-A/B contract — the real merged #2279 never writes it. + +The two paths run in *different* hot paths (background thread vs. agent turn +start) but share process_registry._completion_consumed, so a wakeup can only +happen once. +""" +from __future__ import annotations + +import threading + +import pytest + +# The fake process-registry stub + its installer were duplicated verbatim in +# three bg_task_complete suites; they now live once in tests/_wakeup_helpers.py +# (Greptile review on PR #2979). Import under the legacy local names so the +# rest of this module is unchanged. ``threading`` is still imported above for +# the _RenamedRegistry stub further down. +from tests._wakeup_helpers import FakeProcessRegistry as _FakeProcessRegistry +from tests._wakeup_helpers import install_fake_registry as _install_fake_registry + + +def _reset_cfg_state(): + from api import config as _cfg + from api import background_process as bp + with _cfg.PROCESS_SESSION_INDEX_LOCK: + _cfg.PROCESS_SESSION_INDEX.clear() + _cfg.PENDING_BG_TASK_COMPLETIONS.clear() + _cfg.BG_TASK_COMPLETE_EVENTS_SEEN.clear() + with _cfg.STREAMS_LOCK: + _cfg.STREAMS.clear() + if hasattr(_cfg, "ACTIVE_RUNS"): + with _cfg.ACTIVE_RUNS_LOCK: + _cfg.ACTIVE_RUNS.clear() + if hasattr(bp, "_LAST_EMIT_TS"): + bp._LAST_EMIT_TS.clear() + if hasattr(bp, "_PENDING_EMIT_PAYLOADS"): + bp._PENDING_EMIT_PAYLOADS.clear() + if hasattr(bp, "_PENDING_EMIT_TIMERS"): + bp._PENDING_EMIT_TIMERS.clear() + + +def test_b_sse_first_then_a_drain_skips_same_process_id(monkeypatch): + """B emits SSE for process_id=p1, then user types a new turn — A must skip p1.""" + fake = _FakeProcessRegistry() + fake.register("p1", "sess-1") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + + from api import background_process as bp + from api import streaming as st + from api import config as _cfg + + # Map session_key -> WebUI session_id + bp.register_process_session("sess-1", "sess-1") + + evt = { + "type": "completion", + "session_id": "p1", + "session_key": "sess-1", + "command": "sleep 1", + "exit_code": 0, + "output": "done", + } + # B path: process the event + bp._process_one(evt) + + # B must have marked the (session, process) seen and registry-consumed + assert "p1" in _cfg.BG_TASK_COMPLETE_EVENTS_SEEN["sess-1"] + assert fake.is_completion_consumed("p1") + + # Now simulate A's next-turn drain. Put a *new* event onto the queue for the + # same process_id (e.g. a kill_process race). A must skip because B already + # delivered. + fake.completion_queue.put(evt) + notifications = st._drain_webui_process_notifications("sess-1") + assert notifications == [], "A must NOT re-fire when B already woke the agent for p1" + + +def test_a_drain_first_marks_seen_so_b_would_skip(monkeypatch): + """A (the REAL merged upstream #2279 next-turn drain) drains and wakes the + agent; later B's queue read of the same id is a no-op because the SHARED + upstream dedupe key (process_registry._completion_consumed) already + contains it. + + Re-pointed for the rebase: the real merged #2279 drain dedupes ONLY via + process_registry.is_completion_consumed() — it does NOT populate + api.config.BG_TASK_COMPLETE_EVENTS_SEEN (that set is ours-original and + private to api.background_process). So the cross-A/B contract is the + registry consumed-marker, not BG_TASK_COMPLETE_EVENTS_SEEN. + """ + fake = _FakeProcessRegistry() + fake.register("p2", "sess-2") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + + from api import background_process as bp + from api import streaming as st + from api import config as _cfg + + bp.register_process_session("sess-2", "sess-2") + + evt = { + "type": "completion", + "session_id": "p2", + "session_key": "sess-2", + "command": "echo hi", + "exit_code": 0, + "output": "hi", + } + # A path: queue carried over from a closed-tab session, drain at next turn + fake.completion_queue.put(evt) + notifications = st._drain_webui_process_notifications("sess-2") + assert len(notifications) == 1 + assert "Background process p2 completed" in notifications[0] + + # The REAL merged #2279 A-drain marks the SHARED upstream dedupe key + # (registry consumed-marker) — NOT our private BG_TASK_COMPLETE_EVENTS_SEEN. + assert fake.is_completion_consumed("p2") + assert "sess-2" not in _cfg.BG_TASK_COMPLETE_EVENTS_SEEN, ( + "real upstream #2279 A-drain must NOT populate our private " + "BG_TASK_COMPLETE_EVENTS_SEEN set" + ) + + # Now if B's drain thread sees another spurious event for the same id + # (duplicate enqueue), _process_one must early-return on the SHARED + # registry consumed-marker that A set — no double wakeup. + bp._process_one(evt) # second time + assert fake.is_completion_consumed("p2") + # B early-returned on the shared key BEFORE reaching its own seen-set, so + # BG_TASK_COMPLETE_EVENTS_SEEN stays unpopulated for this session (proves + # the cross-A/B dedupe used the real upstream key, not ours). + assert "sess-2" not in _cfg.BG_TASK_COMPLETE_EVENTS_SEEN + # And no duplicate wakeup marker was queued by the second B pass. + assert "sess-2" not in _cfg.PENDING_BG_TASK_COMPLETIONS + + +def test_registry_completion_consumed_contract(): + """Copilot #2242 review #4 — fail CI LOUD if the agent ProcessRegistry + private cross-A/B dedupe surface is renamed/retyped upstream. + + The WebUI B-drain has no public ``mark_completion_consumed`` to call, so + it reaches into ``ProcessRegistry._completion_consumed`` (under + ``._lock``) to set the shared marker that the public + ``is_completion_consumed`` reads. If a future upstream refactor renames + any of these, the double-wakeup bug would silently come back. This test + pins the contract so the rename breaks HERE (visibly) instead. + """ + pytest.importorskip("tools.process_registry", reason="hermes-agent not installed") + from tools.process_registry import ProcessRegistry + from api import background_process as bp + + pr = ProcessRegistry() + for attr in bp._REGISTRY_CONSUMED_CONTRACT: + assert hasattr(pr, attr), ( + f"ProcessRegistry.{attr} is gone — the WebUI cross-A/B wakeup " + f"dedupe coupling (Copilot #2242 #4) is broken. Either restore it " + f"or add a PUBLIC mark_completion_consumed() upstream and switch " + f"api/background_process._mark_registry_completion_consumed to it." + ) + # Shape contract: the write target must be a set-like (supports .add) and + # the guard must be a usable context manager (supports `with`). + assert hasattr(pr._completion_consumed, "add"), ( + "ProcessRegistry._completion_consumed is no longer a set-like " + "(.add gone) — cross-A/B wakeup dedupe write would fail." + ) + assert hasattr(pr._lock, "__enter__") and hasattr(pr._lock, "__exit__"), ( + "ProcessRegistry._lock is no longer a context manager — the guarded " + "marker write in _mark_registry_completion_consumed would fail." + ) + assert callable(pr.is_completion_consumed), ( + "ProcessRegistry.is_completion_consumed must stay a public method " + "(the cross-A/B dedupe READ side depends on it)." + ) + + # End-to-end: the public read sees what the guarded private write sets + # (the exact mechanism _mark_registry_completion_consumed relies on). + pid = "proc_contract_test" + assert pr.is_completion_consumed(pid) is False + with pr._lock: + pr._completion_consumed.add(pid) + assert pr.is_completion_consumed(pid) is True + + +def test_mark_registry_completion_consumed_fails_loud_on_rename(monkeypatch, caplog): + """A renamed private attr must log ERROR (contract violation), NOT be + swallowed silently at DEBUG (the pre-Copilot-#4 behavior).""" + import logging + + class _RenamedRegistry: + # Simulates an upstream rename: _completion_consumed -> _consumed_v2. + def __init__(self): + self._lock = threading.Lock() + self._consumed_v2: set[str] = set() + + def is_completion_consumed(self, pid: str) -> bool: + return pid in self._consumed_v2 + + fake = _RenamedRegistry() + _install_fake_registry(monkeypatch, fake) + + from api import background_process as bp + + with caplog.at_level(logging.ERROR, logger="api.background_process"): + bp._mark_registry_completion_consumed("p-renamed") + + assert any( + "coupling contract VIOLATED" in r.message and r.levelno >= logging.ERROR + for r in caplog.records + ), "a renamed registry private attr must surface as an ERROR, not a silent DEBUG" + # The marker was NOT set (the bug it guards against), but it failed LOUD so + # CI / monitoring catches it instead of double-firing wakeups silently. + assert not fake.is_completion_consumed("p-renamed") + + +def test_emit_uses_new_event_name_with_trimmed_payload_and_event_id(monkeypatch): + """T1 + T2 contract: emit is named ``bg_task_complete`` (canonical) AND + ``process_complete`` (dual-emit shim until PR (b)); payload matches the + minimal shape ``{session_id, task_id, completed_at, summary?, event_id}``; + both emits carry the same payload + the same ``event_id``. + """ + fake = _FakeProcessRegistry() + fake.register("task-evt-1", "sess-evt-1") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + + from api import background_process as bp + + bp.register_process_session("sess-evt-1", "sess-evt-1") + + # Capture every (event, data) tuple the emitter pushes to streams. + emits: list[tuple[str, dict]] = [] + + def _capture(session_id: str, event: str, data: dict) -> int: + emits.append((event, data)) + return 1 + + monkeypatch.setattr(bp, "_emit_to_session_streams", _capture) + + evt = { + "type": "completion", + "session_id": "task-evt-1", + "session_key": "sess-evt-1", + "command": "sleep 1", + "exit_code": 0, + "output": "done", + } + bp._process_one(evt) + + # Dual-emit shim: both names fire, same payload, same event_id. + names = [e[0] for e in emits] + assert "bg_task_complete" in names, f"canonical event missing: {names}" + assert "process_complete" in names, f"dual-emit shim missing: {names}" + + payloads = [e[1] for e in emits if e[0] in ("bg_task_complete", "process_complete")] + assert len({p["event_id"] for p in payloads}) == 1, ( + "dual-emit must share a single event_id so consumers can dedupe" + ) + + payload = payloads[0] + # Minimal shape per maintainer (R2 §Q1). + expected_required = {"session_id", "task_id", "completed_at", "event_id"} + allowed = expected_required | {"summary"} + assert expected_required <= set(payload), f"missing required keys: {payload}" + assert set(payload) <= allowed, f"unexpected keys in trimmed payload: {payload}" + + # Dropped keys must NOT be present. + for dropped in ("command", "exit_code", "type", "stdout_preview", "wakeup_prompt", "emitted_at", "process_id"): + assert dropped not in payload, f"{dropped!r} should be dropped by T1 trim" + + # Field-rename invariants: + assert payload["session_id"] == "sess-evt-1" + assert payload["task_id"] == "task-evt-1" # was process_id + assert isinstance(payload["completed_at"], float) # was emitted_at + assert isinstance(payload["event_id"], str) and len(payload["event_id"]) >= 8 + + +def test_event_id_is_unique_per_emit(monkeypatch): + """T2: every emit gets a fresh event_id; two completions for two distinct + processes produce two distinct ids. + """ + fake = _FakeProcessRegistry() + fake.register("task-a", "sess-evt-2") + fake.register("task-b", "sess-evt-2") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + + from api import background_process as bp + + bp.register_process_session("sess-evt-2", "sess-evt-2") + monkeypatch.setattr(bp, "_EMIT_COALESCE_WINDOW_SECS", 0.0) + + emits: list[tuple[str, dict]] = [] + + def _capture(session_id: str, event: str, data: dict) -> int: + emits.append((event, data)) + return 1 + + monkeypatch.setattr(bp, "_emit_to_session_streams", _capture) + + bp._process_one({"type": "completion", "session_id": "task-a", "session_key": "sess-evt-2", "exit_code": 0}) + bp._process_one({"type": "completion", "session_id": "task-b", "session_key": "sess-evt-2", "exit_code": 0}) + + canonical_payloads = [d for ev, d in emits if ev == "bg_task_complete"] + assert len(canonical_payloads) == 2 + assert canonical_payloads[0]["event_id"] != canonical_payloads[1]["event_id"] diff --git a/tests/test_bg_task_complete_focus_drop.py b/tests/test_bg_task_complete_focus_drop.py new file mode 100644 index 00000000..2c5b9aeb --- /dev/null +++ b/tests/test_bg_task_complete_focus_drop.py @@ -0,0 +1,215 @@ +"""Structural assertions for the PR(c) UX surface on bg_task_complete. + +Per P-bc §3.3 / §3.4: ``_handleBgTaskCompleteEvent`` in ``static/messages.js`` +gains a toast surface and a T4 drop-when-focused gate stacked on top of the +PR(b) ring-buffer dedupe. The insertion order is contractual: + + 1. JSON parse + sid guard (existing) + 2. ring-buffer dedup check (existing post-PR(b)) + 3. mark-as-seen + clear-unread bookkeeping (NEW) + 4. T4 ``_isSessionActivelyViewed(sid)`` toast gate (NEW) + 5. ``showToast(...)`` inside unfocused branch (NEW per Q-c-1) + 6. diagnostic ack POST outside the T4 gate (existing) + +We can't drive JS from pytest (the repo intentionally avoids a node/jsdom dep +per AGENTS.md), so this file does string-grep + relative-index assertions on +``static/messages.js`` — the same convention the rest of the WEBUI-SUB suite +uses. Each grep is precise so a behavioural regression (e.g. moving the ack +inside the focus gate, or leaking ``d.command`` / ``d.exit_code`` into the +toast copy per Rc-2) trips a hard failure. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _read_messages_js() -> str: + return (REPO_ROOT / "static" / "messages.js").read_text() + + +def _handler_body() -> str: + """Return the source slice of ``_handleBgTaskCompleteEvent`` start → + next top-level ``function`` declaration.""" + js = _read_messages_js() + start = js.index("function _handleBgTaskCompleteEvent(") + # Next top-level function declaration after the handler. + rest = js[start + 1 :] + m = re.search(r"\n(function |// ──)", rest) + end = start + 1 + (m.start() if m else len(rest)) + return js[start:end] + + +# --------------------------------------------------------------------------- +# Insertion-order contract +# --------------------------------------------------------------------------- + + +def _focus_gate_match(body: str): + """Locate the T4 focus gate. + + Supported shapes: + - legacy: ``if (_isSessionActivelyViewed(sid)) return;`` + - current: ``const _viewed = ... _isSessionActivelyViewed(sid) ...;`` + followed by ``if (_viewed) { ... } else { showToast(...) }``. + Returns an object exposing ``.start()`` / ``.end()`` over the gate block. + """ + # Direct legacy early-return form. + m = re.search( + r"if\s*\([^)]*_isSessionActivelyViewed\s*\(\s*sid\s*\)[^)]*\)\s*return", + body, + ) + if m is not None: + return m + # Indirect form: const _viewed = ... _isSessionActivelyViewed(sid) ...; + # if (_viewed) { ... } else { ... } — brace-balance walk to find the gate block end. + flag_decl = re.search( + r"const\s+_viewed\s*=[^;]*_isSessionActivelyViewed\s*\(\s*sid\s*\)[^;]*;", + body, + ) + if flag_decl is None: + return None + gate_head = re.search(r"if\s*\(\s*_viewed\s*\)\s*\{", body[flag_decl.end():]) + if gate_head is None: + return None + open_abs = flag_decl.end() + gate_head.end() - 1 # index of '{' + depth = 0 + close_abs = None + for i in range(open_abs, len(body)): + c = body[i] + if c == '{': + depth += 1 + elif c == '}': + depth -= 1 + if depth == 0: + close_abs = i + break + if close_abs is None: + return None + # If the gate has an `else`, include that branch because it owns the toast branch. + else_match = re.match(r"\s*else\s*\{", body[close_abs + 1 :]) + if else_match is not None: + else_open_abs = close_abs + 1 + else_match.end() - 1 + depth = 0 + else_close_abs = None + for i in range(else_open_abs, len(body)): + c = body[i] + if c == '{': + depth += 1 + elif c == '}': + depth -= 1 + if depth == 0: + else_close_abs = i + break + if else_close_abs is None: + return None + close_abs = else_close_abs + + class _M: + def __init__(self, s, e): + self._s, self._e = s, e + def start(self): + return self._s + def end(self): + return self._e + return _M(flag_decl.start(), close_abs + 1) + + +def test_focus_gate_is_after_ring_buffer_dedup(): + """T4 ``_isSessionActivelyViewed(sid)`` gate MUST appear AFTER the + ``_bgTaskCompleteRingBufferAdd`` dedupe call so duplicates never reach the + focus gate (avoids touching mark-as-seen twice on a duplicate event).""" + body = _handler_body() + dedupe_idx = body.index("_bgTaskCompleteRingBufferAdd(sid, evt_id)") + gate_match = _focus_gate_match(body) + assert gate_match is not None, "T4 focus gate missing" + assert gate_match.start() > dedupe_idx, ( + "T4 focus gate must follow the ring-buffer dedup check" + ) + + +def test_mark_as_seen_is_after_dedup_and_inside_focus_gate(): + """Mark-as-seen + clear-unread bookkeeping MUST run after dedup and inside + the focused-viewer branch so an actively-viewed session clears its unread + counter even though the toast is suppressed.""" + body = _handler_body() + dedupe_idx = body.index("_bgTaskCompleteRingBufferAdd(sid, evt_id)") + mark_idx = body.index("_markSessionViewed") + clear_idx = body.index("_clearSessionCompletionUnread") + gate_match = _focus_gate_match(body) + assert gate_match is not None + assert dedupe_idx < gate_match.start() <= mark_idx < gate_match.end(), ( + "_markSessionViewed must sit after dedupe inside the T4 focus gate" + ) + assert dedupe_idx < gate_match.start() <= clear_idx < gate_match.end(), ( + "_clearSessionCompletionUnread must sit after dedupe inside the T4 focus gate" + ) + + +def test_toast_call_is_inside_unfocused_gate_branch(): + """The ``showToast`` call MUST sit inside the T4 else/unfocused branch so a + focused viewer never sees a toast for the session they are watching.""" + body = _handler_body() + gate_match = _focus_gate_match(body) + assert gate_match is not None + toast_idx = body.index("showToast(") + assert gate_match.start() < toast_idx < gate_match.end(), ( + "showToast must be gated by the T4 drop-when-focused branch" + ) + assert "} else {" in body[gate_match.start() : toast_idx], ( + "showToast must be in the unfocused `else` branch, not the focused branch" + ) + + +def test_ack_post_is_after_focus_gate_and_outside_toast_branch(): + """The diagnostic ack POST MUST run after the T4 gate and outside the toast + branch so both focused and unfocused viewers emit the server cleanup signal. + For unfocused viewers this preserves the existing toast-before-ack order.""" + body = _handler_body() + gate_match = _focus_gate_match(body) + assert gate_match is not None + toast_idx = body.index("showToast(") + ack_idx = body.index("api/bg-task-complete-ack") + assert toast_idx < ack_idx, "unfocused toast must still precede diagnostic ack POST" + assert ack_idx > gate_match.end(), "diagnostic ack POST must live outside the T4 focus gate" + + +# --------------------------------------------------------------------------- +# Toast copy guards (Rc-2: minimal-payload-safe) +# --------------------------------------------------------------------------- + + +def _toast_block() -> str: + """Slice the source from the toast comment to the showToast call's end.""" + body = _handler_body() + # The toast block is the try { ... } that contains showToast(. + start = body.rindex("try", 0, body.index("showToast(")) + # Find the matching close — toast block ends at the next "} catch (_) {}". + end_marker = body.index("} catch (_) {}", start) + return body[start : end_marker + len("} catch (_) {}")] + + +def test_toast_block_uses_only_minimal_payload_fields(): + """Per Rc-2 the toast copy may reference ONLY ``d.task_id`` and the + optional ``d.summary`` — never ``d.command`` or ``d.exit_code`` (those + fields are not guaranteed on the minimal payload shipped by the server).""" + block = _toast_block() + assert "d.task_id" in block, "toast must reference d.task_id" + assert "d.summary" in block, "toast must reference d.summary" + assert "d.command" not in block, "toast must NOT reference d.command (Rc-2)" + assert "d.exit_code" not in block, "toast must NOT reference d.exit_code (Rc-2)" + + +def test_toast_template_pins_copy(): + """The toast template (P-bc §3.3 Q-c-1 verbatim) wraps the task id in the + 8-char prefix and falls back to ``''`` (empty tail — just ``Task done``) + when ``d.summary`` is absent. Pin both literals so a future drift in copy is + caught loud.""" + block = _toast_block() + assert "slice(0, 8)" in block + assert "slice(0, 80)" in block + assert "Task ${tid} done${tail}" in block + assert "2600" in block, "toast duration must be 2600ms per Q-c-1" diff --git a/tests/test_bg_task_complete_loadsession_stream_restart.py b/tests/test_bg_task_complete_loadsession_stream_restart.py new file mode 100644 index 00000000..6a052fcd --- /dev/null +++ b/tests/test_bg_task_complete_loadsession_stream_restart.py @@ -0,0 +1,135 @@ +"""Regression: ``loadSession`` must restart the session SSE stream on an early +failure exit (Greptile review on PR #2979). + +Context +------- +``loadSession`` in ``static/sessions.js`` stops the per-session SSE stream +unconditionally near the top (mirroring ``stopApprovalPolling``): + + if(typeof stopSessionStream==='function') stopSessionStream(); + +On the happy path it is restarted ~120 lines later at the success tail: + + if(typeof startSessionStream==='function') startSessionStream(S.session.session_id); + +But the metadata-fetch ``catch`` block (network error / 4xx / 5xx) returns +early WITHOUT reaching that restart. The session stream is the new feature's +primary delivery path for ``bg_task_complete`` events, so leaving it stopped +silently drops every completion event for the session still on screen until +the user explicitly navigates to a session again. + +The fix restarts the stream for the session that remains on screen +(``currentSid``) inside the ``catch`` block, guarded so it does NOT fire when: + - a newer load is already in flight (``_loadingSessionId`` reset to a newer + sid owns the restart), or + - the failure self-healed away the current session (404 on the current + session) — there is no live session to stream for. + +We can't drive JS from pytest (the repo intentionally avoids a node/jsdom dep +per AGENTS.md), so this file does string-grep + brace-balance assertions on +``static/sessions.js`` — the same convention the rest of the WEBUI suite uses. +""" +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _read_sessions_js() -> str: + return (REPO_ROOT / "static" / "sessions.js").read_text() + + +def _load_session_body() -> str: + """Return the source slice of ``async function loadSession(`` start → + next top-level ``function`` / ``async function`` declaration.""" + js = _read_sessions_js() + start = js.index("async function loadSession(") + rest = js[start + 1 :] + m = re.search(r"\n(async function |function )", rest) + end = start + 1 + (m.start() if m else len(rest)) + return js[start:end] + + +def _catch_block() -> str: + """Return the metadata-fetch ``catch(e){ ... }`` slice within loadSession. + + Anchors on the ``data = await api(`/api/session?...messages=0...`)`` try and + walks brace balance over the following ``catch (e) { ... }`` block. + """ + body = _load_session_body() + # The metadata fetch is the first `catch(` after the messages=0 api() call. + anchor = re.search(r"messages=0[^\n]*resolve_model=0", body) + assert anchor is not None, "metadata fetch (messages=0&resolve_model=0) not found" + cm = re.search(r"catch\s*\(\s*e\s*\)\s*\{", body[anchor.end():]) + assert cm is not None, "catch(e){ for metadata fetch not found" + open_abs = anchor.end() + cm.end() - 1 # index of '{' + depth = 0 + for i in range(open_abs, len(body)): + c = body[i] + if c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + return body[open_abs : i + 1] + raise AssertionError("unbalanced catch block in loadSession") + + +def test_load_session_stops_stream_on_entry(): + """Sanity: the unconditional stop at the top of loadSession still exists — + it is the precondition that makes the restart-on-failure necessary.""" + body = _load_session_body() + assert "stopSessionStream()" in body, ( + "loadSession no longer stops the session stream on entry; the restart " + "guard's precondition has changed — re-derive this test." + ) + + +def test_catch_restarts_session_stream_for_current_sid(): + """The metadata-fetch catch block must restart the session stream for the + session still on screen (currentSid).""" + catch = _catch_block() + m = re.search(r"startSessionStream\s*\(\s*currentSid\s*\)", catch) + assert m is not None, ( + "loadSession's metadata-fetch catch block does not restart " + "startSessionStream(currentSid); bg_task_complete events would be " + "silently dropped for the on-screen session after a failed load." + ) + + +def test_restart_is_guarded_against_newer_inflight_load(): + """The restart must be gated on ``_loadingSessionId === null`` so a newer + in-flight load (rapid session switch) owns the stream instead — the newer + load starts its own stream and must not be clobbered.""" + catch = _catch_block() + # The guard and the restart call live in the same if-condition; assert the + # null-check precedes the startSessionStream(currentSid) call. + restart = re.search(r"startSessionStream\s*\(\s*currentSid\s*\)", catch) + assert restart is not None + guard = re.search(r"_loadingSessionId\s*===\s*null", catch[: restart.start()]) + assert guard is not None, ( + "restart of startSessionStream(currentSid) is not guarded by " + "_loadingSessionId === null; a newer in-flight load could be clobbered." + ) + + +def test_restart_skipped_when_current_session_self_healed(): + """A 404 on the *current* session self-heals it away (clears localStorage + + URL). There is then no live session to stream for, so the restart must be + skipped to avoid spinning the SSE reconnect loop against a dead id.""" + catch = _catch_block() + # A self-heal guard distinguishing the 404-on-current case must exist and + # gate the restart (negated in the restart condition). + assert re.search(r"_selfHealedCurrent", catch), ( + "no _selfHealedCurrent guard found; a 404 on the current session would " + "wrongly restart a stream against a dead session id." + ) + restart = re.search(r"startSessionStream\s*\(\s*currentSid\s*\)", catch) + assert restart is not None + neg_guard = re.search(r"!\s*_selfHealedCurrent", catch[: restart.start()]) + assert neg_guard is not None, ( + "restart is not gated on !_selfHealedCurrent; the self-healed-current " + "case would wrongly re-open a stream." + ) diff --git a/tests/test_bg_task_complete_ring_buffer.py b/tests/test_bg_task_complete_ring_buffer.py new file mode 100644 index 00000000..0393fc52 --- /dev/null +++ b/tests/test_bg_task_complete_ring_buffer.py @@ -0,0 +1,170 @@ +"""Structural assertions for the bg_task_complete dedupe ring buffer. + +Per P-bc §2.5: the WebUI consumer dedupe graduates from an unbounded ``Set`` +keyed by ``task_id`` to a bounded ``Map``-backed ring buffer keyed by +``(session_id, event_id)``. The new structure carries: + + * A 60-second TTL (``_BG_TASK_COMPLETE_TTL_MS = 60000``). + * A 256-entry soft cap (``_BG_TASK_COMPLETE_CAP = 256``). + * Lazy purge: every insert walks the Map in insertion order and drops + entries whose expiry has passed. + * Insertion-order eviction on overflow (oldest entry dropped first). + * A helper ``_bgTaskCompleteRingBufferAdd(sid, event_id)`` returning + ``true`` on duplicate, ``false`` on first-seen. + +We can't exercise JS at runtime from pytest (the repo intentionally avoids a +node/jsdom dep per AGENTS.md), so this file does structural / string-grep +assertions on ``static/messages.js`` — the same convention every other +WEBUI-SUB test uses. The grep targets are intentionally precise so a +behavioural regression (e.g. silently switching the dedupe key back to +``task_id`` or removing lazy purge) shows up as a hard test failure. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + + +def _read_messages_js() -> str: + return (REPO_ROOT / "static" / "messages.js").read_text() + + +# --------------------------------------------------------------------------- +# Module-scope declarations +# --------------------------------------------------------------------------- + + +def test_ring_buffer_constants_declared_module_scope(): + """TTL and cap constants live at module scope, not buried inside a + function body, so the dedupe state is shared across every EventSource + (in-turn STREAMS + per-session SSE channel).""" + js = _read_messages_js() + assert "const _BG_TASK_COMPLETE_TTL_MS = 60000;" in js + assert "const _BG_TASK_COMPLETE_CAP = 256;" in js + # Map (not Set) carries the (key -> expiry) entries. + assert "const _bgTaskCompleteSeenIds = new Map();" in js + # Set form must be gone — D-b-2 replaced it. + assert "_bgTaskCompleteSeenIds = new Set()" not in js + assert "_seenProcessCompleteIds" not in js + + +def test_ring_buffer_helper_declared(): + """The add-and-dedupe helper exists and lives at module scope.""" + js = _read_messages_js() + assert "function _bgTaskCompleteRingBufferAdd(sid, evt_id)" in js + + +# --------------------------------------------------------------------------- +# Helper-body internals +# --------------------------------------------------------------------------- + + +def _helper_source(js: str) -> str: + ix = js.index("function _bgTaskCompleteRingBufferAdd(sid, evt_id)") + # Helper body is short (~25-35 LOC); 2000 chars is comfortable headroom + # without bleeding into the next function. + return js[ix : ix + 2000] + + +def test_helper_ignores_missing_event_id(): + """Events without an event_id are ignored — server contract guarantees one. + + Missing key returns ``true`` (treat as seen/skip) rather than ``false`` + (proceed): if a future call site forgets the caller-side ``if (!evt_id)`` + guard, an un-keyable completion is dropped instead of processed without a + dedupe key. + """ + body = _helper_source(_read_messages_js()) + assert "if (!sid || !evt_id) return true;" in body + + +def test_helper_key_construction_uses_event_id_not_task_id(): + """Dedupe key is (session_id, event_id), the canonical contract surface.""" + body = _helper_source(_read_messages_js()) + # The composite key uses evt_id, not pid / task_id / process_id. + assert "const key = sid + '|' + evt_id;" in body + # Negative guards on the previous keying schemes. + assert "sid + '|' + pid" not in body + assert "sid + '|' + task_id" not in body + assert "process_id" not in body # legacy payload key, gone + + +def test_helper_lazy_purges_expired_entries(): + """Each add walks the Map in insertion order and drops expired entries. + Without lazy purge the soft cap eviction would silently drop live entries + while expired ones squat in the Map indefinitely.""" + body = _helper_source(_read_messages_js()) + assert "const now = Date.now();" in body + assert "for (const [k, exp] of _bgTaskCompleteSeenIds)" in body + assert "if (exp <= now)" in body + assert "_bgTaskCompleteSeenIds.delete(k);" in body + + +def test_helper_returns_true_on_duplicate(): + """Duplicate detection short-circuits — returns true without inserting.""" + body = _helper_source(_read_messages_js()) + assert "if (_bgTaskCompleteSeenIds.has(key)) return true;" in body + + +def test_helper_inserts_with_expiry_in_future(): + """First-seen entries are stamped with now + TTL_MS.""" + body = _helper_source(_read_messages_js()) + assert "_bgTaskCompleteSeenIds.set(key, now + _BG_TASK_COMPLETE_TTL_MS);" in body + + +def test_helper_soft_cap_evicts_oldest(): + """Soft cap enforcement uses insertion-order eviction (Map.keys().next()).""" + body = _helper_source(_read_messages_js()) + assert "while (_bgTaskCompleteSeenIds.size > _BG_TASK_COMPLETE_CAP)" in body + assert "_bgTaskCompleteSeenIds.keys().next().value" in body + assert "_bgTaskCompleteSeenIds.delete(firstKey);" in body + + +def test_helper_returns_false_on_first_seen(): + """First-seen path returns false so the caller proceeds to surface/ack.""" + body = _helper_source(_read_messages_js()) + # The trailing `return false;` of the helper. + assert re.search(r"return false;\s*\n\}", body), ( + "helper must end with `return false;` after the soft-cap loop" + ) + + +# --------------------------------------------------------------------------- +# Call-site integration in _handleBgTaskCompleteEvent +# --------------------------------------------------------------------------- + + +def _handler_source(js: str) -> str: + ix = js.index("function _handleBgTaskCompleteEvent") + return js[ix : ix + 2200] + + +def test_handler_calls_ring_buffer_helper(): + """The shared handler calls the helper with (sid, event_id) — not (sid, task_id).""" + body = _handler_source(_read_messages_js()) + assert "_bgTaskCompleteRingBufferAdd(sid, evt_id)" in body + # Old call shape must be gone. + assert "_bgTaskCompleteSeenIds.has(dedupeKey)" not in body + assert "_bgTaskCompleteSeenIds.add(dedupeKey)" not in body + + +def test_handler_extracts_event_id_from_payload(): + """The handler pulls event_id out of the parsed payload.""" + body = _handler_source(_read_messages_js()) + assert "d.event_id" in body + # Missing event_id short-circuits before the dedupe / ack. + assert "if (!evt_id) return;" in body + + +def test_handler_dedup_runs_before_ack_post(): + """Dedupe gate must precede the fire-and-forget ack POST so duplicates + don't generate a flood of ack traffic.""" + body = _handler_source(_read_messages_js()) + dedupe_ix = body.index("_bgTaskCompleteRingBufferAdd(sid, evt_id)") + ack_ix = body.index("api/bg-task-complete-ack") + assert dedupe_ix < ack_ix, ( + "ring-buffer dedupe must execute before the ack POST" + ) diff --git a/tests/test_bg_task_complete_throttle.py b/tests/test_bg_task_complete_throttle.py new file mode 100644 index 00000000..65e818b5 --- /dev/null +++ b/tests/test_bg_task_complete_throttle.py @@ -0,0 +1,190 @@ +"""T3 throttle tests for bg_task_complete emits. + +The backend emits a canonical ``bg_task_complete`` SSE frame and a temporary +``process_complete`` alias for the same payload. T3 adds a per-session 1s +coalesce gate around that dual emit so rapid completion bursts do not flood a +live WebUI tab; the deferred emit must carry the latest payload from the burst. +""" +from __future__ import annotations + +# The fake process-registry stub + its installer were duplicated verbatim in +# three bg_task_complete suites; they now live once in tests/_wakeup_helpers.py +# (Greptile review on PR #2979). Import under the legacy local names so the +# rest of this module is unchanged. +from tests._wakeup_helpers import FakeProcessRegistry as _FakeProcessRegistry +from tests._wakeup_helpers import install_fake_registry as _install_fake_registry + + +def _reset_state(bp): + from api import config as _cfg + + with _cfg.PROCESS_SESSION_INDEX_LOCK: + _cfg.PROCESS_SESSION_INDEX.clear() + _cfg.PENDING_BG_TASK_COMPLETIONS.clear() + _cfg.BG_TASK_COMPLETE_EVENTS_SEEN.clear() + if hasattr(_cfg, "DEFERRED_PROCESS_WAKEUPS"): + with _cfg.DEFERRED_PROCESS_WAKEUPS_LOCK: + _cfg.DEFERRED_PROCESS_WAKEUPS.clear() + with _cfg.STREAMS_LOCK: + _cfg.STREAMS.clear() + if hasattr(_cfg, "ACTIVE_RUNS"): + with _cfg.ACTIVE_RUNS_LOCK: + _cfg.ACTIVE_RUNS.clear() + # T3 module-level throttle state. + if hasattr(bp, "_LAST_EMIT_TS"): + bp._LAST_EMIT_TS.clear() + if hasattr(bp, "_PENDING_EMIT_PAYLOADS"): + bp._PENDING_EMIT_PAYLOADS.clear() + if hasattr(bp, "_PENDING_EMIT_TIMERS"): + bp._PENDING_EMIT_TIMERS.clear() + + +class _FakeClock: + def __init__(self): + self.now = 1000.0 + + def time(self) -> float: + return self.now + + def advance(self, seconds: float) -> None: + self.now += seconds + + +class _ManualTimer: + def __init__(self, timers: list["_ManualTimer"], delay: float, callback, args=()): + self.timers = timers + self.delay = delay + self.callback = callback + self.args = args + self.daemon = False + self.cancelled = False + self.started = False + + def start(self): + self.started = True + self.timers.append(self) + + def cancel(self): + self.cancelled = True + + def fire(self): + if not self.cancelled: + self.callback(*self.args) + + +def _install_emit_harness(monkeypatch, *, session_id: str = "sess-throttle"): + from api import background_process as bp + + fake = _FakeProcessRegistry() + _install_fake_registry(monkeypatch, fake) + _reset_state(bp) + bp.register_process_session(session_id, session_id) + + emits: list[tuple[str, dict]] = [] + + def _capture_emit(sid: str, event: str, data: dict) -> int: + emits.append((event, dict(data))) + return 1 + + monkeypatch.setattr(bp, "_emit_to_session_streams", _capture_emit) + monkeypatch.setattr(bp, "_start_server_side_wakeup_turn", lambda sid, prompt: None) + monkeypatch.setattr(bp, "_session_has_active_turn", lambda sid: False) + + clock = _FakeClock() + monkeypatch.setattr(bp.time, "time", clock.time) + + timers: list[_ManualTimer] = [] + + def _timer_factory(delay, callback, args=(), kwargs=None): + assert kwargs in (None, {}) + return _ManualTimer(timers, delay, callback, args) + + monkeypatch.setattr(bp.threading, "Timer", _timer_factory) + return bp, fake, emits, clock, timers + + +def _process_completion(bp, fake, task_id: str, session_id: str = "sess-throttle") -> None: + fake.register(task_id, session_id) + bp._process_one( + { + "type": "completion", + "session_id": task_id, + "session_key": session_id, + "command": f"echo {task_id}", + "exit_code": 0, + "output": task_id, + } + ) + + +def _canonical_payloads(emits: list[tuple[str, dict]]) -> list[dict]: + return [payload for event, payload in emits if event == "bg_task_complete"] + + +def test_rapid_bg_task_complete_emits_coalesce_to_immediate_plus_one_deferred(monkeypatch): + bp, fake, emits, _clock, timers = _install_emit_harness(monkeypatch) + + for idx in range(10): + _process_completion(bp, fake, f"task-{idx}") + + # First emit is immediate. The rest of the burst is represented by one + # deferred timer; cancelled timer instances may remain in the manual list. + assert len(_canonical_payloads(emits)) == 1 + live_timers = [timer for timer in timers if not timer.cancelled] + assert len(live_timers) == 1 + + live_timers[0].fire() + + canonical_payloads = _canonical_payloads(emits) + assert len(canonical_payloads) <= 2 + assert [payload["task_id"] for payload in canonical_payloads] == ["task-0", "task-9"] + + +def test_bg_task_complete_emits_two_seconds_apart_all_fire(monkeypatch): + bp, fake, emits, clock, timers = _install_emit_harness(monkeypatch) + + for idx in range(3): + _process_completion(bp, fake, f"spaced-{idx}") + clock.advance(2.0) + + assert [payload["task_id"] for payload in _canonical_payloads(emits)] == [ + "spaced-0", + "spaced-1", + "spaced-2", + ] + assert not [timer for timer in timers if not timer.cancelled] + + +def test_coalesced_bg_task_complete_payload_replace_uses_latest_payload(monkeypatch): + bp, fake, emits, _clock, timers = _install_emit_harness(monkeypatch) + + _process_completion(bp, fake, "first") + _process_completion(bp, fake, "middle") + _process_completion(bp, fake, "latest") + + live_timers = [timer for timer in timers if not timer.cancelled] + assert len(live_timers) == 1 + live_timers[0].fire() + + canonical_payloads = _canonical_payloads(emits) + assert [payload["task_id"] for payload in canonical_payloads] == ["first", "latest"] + assert canonical_payloads[-1]["summary"].endswith("latest completed (exit_code=0).") + + +def test_outside_window_arrival_flushes_immediately_even_with_pending_timer(monkeypatch): + bp, fake, emits, clock, timers = _install_emit_harness(monkeypatch) + + _process_completion(bp, fake, "first") + clock.advance(0.2) + _process_completion(bp, fake, "pending") + assert [payload["task_id"] for payload in _canonical_payloads(emits)] == ["first"] + assert len([timer for timer in timers if not timer.cancelled]) == 1 + + clock.advance(1.1) + _process_completion(bp, fake, "outside-window") + + assert [payload["task_id"] for payload in _canonical_payloads(emits)] == [ + "first", + "outside-window", + ] + assert not [timer for timer in timers if not timer.cancelled] diff --git a/tests/test_bg_task_complete_wakeup.py b/tests/test_bg_task_complete_wakeup.py new file mode 100644 index 00000000..39defd39 --- /dev/null +++ b/tests/test_bg_task_complete_wakeup.py @@ -0,0 +1,323 @@ +"""Wakeup tests for the renamed ``bg_task_complete`` SSE event. + +This file replaces the legacy ``test_process_complete_wakeup.py`` after the +R2 §Q1 / Q4 contract update with the maintainer: + + - Q1: the canonical SSE event is now ``bg_task_complete`` carrying the + minimal ``{session_id, task_id, completed_at, summary?, event_id}`` + payload (the legacy ``process_complete`` name is dual-emitted under + PR (a) only as a 1-PR-cycle compatibility shim and is removed in + PR (b)). + - Q4: each emit must carry a fresh server-side ``event_id`` so the WebUI + can build a consumer-side TTL ring buffer for cross-disconnect + dedupe in a follow-up PR. + +The tests below cover the wakeup-emit hot path end to end: + + 1. A completion event flowing through ``_process_one`` produces the + canonical ``bg_task_complete`` SSE emission with the trimmed payload + and a non-empty ``event_id``. + 2. The same call also emits the legacy ``process_complete`` shim with the + same payload + same ``event_id``, so a consumer running an old + listener still wakes exactly once. + 3. When ``_process_one`` runs while no per-session emit-coalesce window + is pending, the event is emitted immediately (i.e. wakeup is not + dropped by the throttle gate on a single completion). + 4. The previous file name ``tests/test_process_complete_wakeup.py`` must + remain absent in the BACKEND-tier slice. +""" +from __future__ import annotations + +import os + +# The fake process-registry stub + its installer were duplicated verbatim in +# three bg_task_complete suites; they now live once in tests/_wakeup_helpers.py +# (Greptile review on PR #2979). Import under the legacy local names so the +# rest of this module is unchanged. +from tests._wakeup_helpers import FakeProcessRegistry as _FakeProcessRegistry +from tests._wakeup_helpers import install_fake_registry as _install_fake_registry + + +def _reset_cfg_state(): + from api import config as _cfg + from api import background_process as bp + with _cfg.PROCESS_SESSION_INDEX_LOCK: + _cfg.PROCESS_SESSION_INDEX.clear() + _cfg.PENDING_BG_TASK_COMPLETIONS.clear() + _cfg.BG_TASK_COMPLETE_EVENTS_SEEN.clear() + with _cfg.STREAMS_LOCK: + _cfg.STREAMS.clear() + if hasattr(_cfg, "ACTIVE_RUNS"): + with _cfg.ACTIVE_RUNS_LOCK: + _cfg.ACTIVE_RUNS.clear() + if hasattr(bp, "_LAST_EMIT_TS"): + bp._LAST_EMIT_TS.clear() + if hasattr(bp, "_PENDING_EMIT_PAYLOADS"): + bp._PENDING_EMIT_PAYLOADS.clear() + if hasattr(bp, "_PENDING_EMIT_TIMERS"): + bp._PENDING_EMIT_TIMERS.clear() + + +def _capture_emits(monkeypatch): + """Replace the per-session emit fan-out with a capturing list.""" + from api import background_process as bp + + emits: list[tuple[str, dict]] = [] + + def _capture(session_id: str, event: str, data: dict) -> int: + emits.append((event, data)) + return 1 + + monkeypatch.setattr(bp, "_emit_to_session_streams", _capture) + # Run the coalesce gate in pass-through mode so a single completion + # exercises the immediate-emit branch (the throttle behaviour itself is + # covered exhaustively by tests/test_bg_task_complete_throttle.py). + monkeypatch.setattr(bp, "_EMIT_COALESCE_WINDOW_SECS", 0.0) + return emits + + +def test_bg_task_complete_wakeup_emits_canonical_event_with_event_id(monkeypatch): + """``_process_one`` emits the canonical ``bg_task_complete`` SSE event + with the R2 §Q1 trimmed payload and a fresh server-side ``event_id`` + (R2 §Q4). + """ + fake = _FakeProcessRegistry() + fake.register("task-wakeup-1", "sess-wakeup-1") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + + from api import background_process as bp + + bp.register_process_session("sess-wakeup-1", "sess-wakeup-1") + emits = _capture_emits(monkeypatch) + monkeypatch.setattr(bp, "_start_server_side_wakeup_turn", lambda *_args, **_kwargs: None) + + evt = { + "type": "completion", + "session_id": "task-wakeup-1", + "session_key": "sess-wakeup-1", + "command": "sleep 1", + "exit_code": 0, + "output": "done", + } + bp._process_one(evt) + + names = [e[0] for e in emits] + assert "bg_task_complete" in names, ( + f"canonical bg_task_complete emit missing: {names}" + ) + + canonical_payloads = [d for ev, d in emits if ev == "bg_task_complete"] + assert canonical_payloads, "no canonical bg_task_complete payload captured" + payload = canonical_payloads[0] + + expected_required = {"session_id", "task_id", "completed_at", "event_id"} + allowed = expected_required | {"summary"} + assert expected_required <= set(payload), ( + f"missing required keys in bg_task_complete payload: {payload}" + ) + assert set(payload) <= allowed, ( + f"unexpected keys in trimmed bg_task_complete payload: {payload}" + ) + + # The legacy/dropped keys must NOT survive the T1 trim. + for dropped in ( + "command", + "exit_code", + "type", + "stdout_preview", + "wakeup_prompt", + "emitted_at", + "process_id", + ): + assert dropped not in payload, ( + f"{dropped!r} should be dropped from bg_task_complete payload" + ) + + # Field-rename invariants. + assert payload["session_id"] == "sess-wakeup-1" + assert payload["task_id"] == "task-wakeup-1" + assert isinstance(payload["completed_at"], float) + # R2 §Q4: ``event_id`` is a non-empty string (uuid4().hex => 32 chars). + assert isinstance(payload["event_id"], str) + assert len(payload["event_id"]) >= 8 + + +class _FakeHandler: + """Minimal handler stub for exercising ``handle_post`` directly. + + Mirrors the pattern in ``tests/test_issue1909_csp_report_only.py`` — + captures status + response headers + body without spinning a real HTTP + server. + """ + + def __init__(self, body: bytes = b"{}", headers: dict | None = None): + import io as _io + self.headers = { + "Content-Length": str(len(body)), + "Content-Type": "application/json", + **(headers or {}), + } + self.rfile = _io.BytesIO(body) + self.wfile = _io.BytesIO() + self.client_address = ("127.0.0.1", 12345) + self.status: int | None = None + self.sent_headers: dict[str, str] = {} + + def send_response(self, status: int) -> None: + self.status = status + + def send_header(self, key: str, value: str) -> None: + self.sent_headers[key] = value + + def end_headers(self) -> None: + pass + + +def test_legacy_process_complete_ack_returns_410_gone_with_x_replaced_by(): + """T1 deprecation alias: the old ``/api/process-complete-ack`` POST path + must return HTTP 410 Gone and an ``X-Replaced-By`` header pointing at + ``/api/bg-task-complete-ack`` (V-a-final criterion #6 + D-a-fix item #1). + """ + import json as _json + from urllib.parse import urlparse + + import api.routes as routes + + handler = _FakeHandler() + parsed = urlparse("/api/process-complete-ack") + + result = routes.handle_post(handler, parsed) + + assert result is True, "deprecated ack endpoint must claim the request" + assert handler.status == 410, ( + f"expected HTTP 410 Gone for deprecated ack path, got {handler.status}" + ) + assert handler.sent_headers.get("X-Replaced-By") == "/api/bg-task-complete-ack", ( + f"X-Replaced-By header missing or wrong: {handler.sent_headers}" + ) + + body = handler.wfile.getvalue() + # Body is gzip-wrapped only if Accept-Encoding allowed it; the fake + # handler does not set Accept-Encoding so the body is plain JSON. + payload = _json.loads(body.decode("utf-8")) + assert payload.get("replaced_by") == "/api/bg-task-complete-ack" + assert "gone" in payload.get("error", "").lower() + + +def test_bg_task_complete_ack_marks_process_id_alias_deprecated(monkeypatch): + """The diagnostic ack endpoint keeps ``process_id`` as a transitional + request alias, but makes that legacy usage visible via ``Deprecation``. + """ + import json as _json + import types as _types + + import api.routes as routes + + monkeypatch.setattr( + routes, + "get_session", + lambda sid: _types.SimpleNamespace(session_id=sid), + ) + handler = _FakeHandler() + + routes._handle_bg_task_complete_ack( + handler, + {"session_id": "sess-legacy-alias", "process_id": "proc-legacy-1"}, + ) + + assert handler.status == 200 + assert handler.sent_headers.get("Deprecation") == "true" + payload = _json.loads(handler.wfile.getvalue().decode("utf-8")) + assert payload["task_id"] == "proc-legacy-1" + + +def test_bg_task_complete_ack_marks_mixed_process_id_presence_deprecated(monkeypatch): + """If a request still includes ``process_id``, surface the transitional + alias even when the canonical ``task_id`` is also present. + """ + import json as _json + import types as _types + + import api.routes as routes + + monkeypatch.setattr( + routes, + "get_session", + lambda sid: _types.SimpleNamespace(session_id=sid), + ) + handler = _FakeHandler() + + routes._handle_bg_task_complete_ack( + handler, + { + "session_id": "sess-mixed-alias", + "task_id": "task-canonical-1", + "process_id": "proc-legacy-1", + }, + ) + + assert handler.status == 200 + assert handler.sent_headers.get("Deprecation") == "true" + payload = _json.loads(handler.wfile.getvalue().decode("utf-8")) + assert payload["task_id"] == "task-canonical-1" + + +def test_bg_task_complete_ack_canonical_task_id_has_no_deprecation_header(monkeypatch): + """Canonical ``task_id`` requests should not be marked deprecated.""" + import types as _types + + import api.routes as routes + + monkeypatch.setattr( + routes, + "get_session", + lambda sid: _types.SimpleNamespace(session_id=sid), + ) + handler = _FakeHandler() + + routes._handle_bg_task_complete_ack( + handler, + {"session_id": "sess-canonical", "task_id": "task-canonical-1"}, + ) + + assert handler.status == 200 + assert "Deprecation" not in handler.sent_headers + + +def test_bg_task_complete_ack_empty_process_id_alias_has_no_deprecation_header(monkeypatch): + """An empty transitional alias key should not signal real legacy usage.""" + import types as _types + + import api.routes as routes + + monkeypatch.setattr( + routes, + "get_session", + lambda sid: _types.SimpleNamespace(session_id=sid), + ) + handler = _FakeHandler() + + routes._handle_bg_task_complete_ack( + handler, + { + "session_id": "sess-canonical-empty-alias", + "task_id": "task-canonical-1", + "process_id": "", + }, + ) + + assert handler.status == 200 + assert "Deprecation" not in handler.sent_headers + + +def test_old_process_complete_wakeup_test_file_is_absent(): + """The legacy filename ``tests/test_process_complete_wakeup.py`` must + remain absent on this branch — the rename is part of the BACKEND-tier + T1 contract and is required by V-a-final criterion #9. + """ + here = os.path.dirname(os.path.abspath(__file__)) + legacy = os.path.join(here, "test_process_complete_wakeup.py") + assert not os.path.exists(legacy), ( + f"legacy {legacy!r} must not exist after the T1 rename" + ) diff --git a/tests/test_bugbatch_apr2026.py b/tests/test_bugbatch_apr2026.py index a58c9b2e..5c2e3518 100644 --- a/tests/test_bugbatch_apr2026.py +++ b/tests/test_bugbatch_apr2026.py @@ -131,7 +131,7 @@ def test_576_restore_happens_after_load_session(): def test_585_get_available_models_calls_reload_config(): """api/config.py: get_available_models() must do a mtime-based reload check.""" config_src = (REPO_ROOT / "api" / "config.py").read_text(encoding="utf-8") - fn_start = config_src.find("def get_available_models()") + fn_start = config_src.find("def get_available_models(") assert fn_start != -1, "get_available_models not found" fn_body_end = config_src.find('"""', config_src.find('"""', fn_start + 30) + 3) + 3 # Must check mtime before reading config diff --git a/tests/test_live_rebuild_budget_warn_rate_limit.py b/tests/test_live_rebuild_budget_warn_rate_limit.py new file mode 100644 index 00000000..21c64c51 --- /dev/null +++ b/tests/test_live_rebuild_budget_warn_rate_limit.py @@ -0,0 +1,59 @@ +"""Live-rebuild-budget warning rate-limit — Q-2979-A3. + +Per Copilot discussion_r3305864400 the budget-exceeded warning at +api/config.py is potentially high-volume: a hung upstream probe or a sustained +burst of cold callers could flood the log at warning level. The fix wraps the +warning with ``_should_warn_budget``: the FIRST hit in a cooldown window logs +at warning, subsequent hits in the same window log at info — so the signal is +retained but the volume is bounded. +""" +from __future__ import annotations + +import time + + +def test_should_warn_budget_first_call_returns_true(): + from api import config as cfg + + # Isolate per-test state. + cfg._BUDGET_WARN_STATE.pop("unit-test-reason-A", None) + + assert cfg._should_warn_budget("unit-test-reason-A", cooldown_s=300.0) is True + + +def test_should_warn_budget_inside_cooldown_returns_false(): + from api import config as cfg + + cfg._BUDGET_WARN_STATE.pop("unit-test-reason-B", None) + + assert cfg._should_warn_budget("unit-test-reason-B", cooldown_s=300.0) is True + # Second hit within cooldown — must be False (caller should demote to info). + assert cfg._should_warn_budget("unit-test-reason-B", cooldown_s=300.0) is False + assert cfg._should_warn_budget("unit-test-reason-B", cooldown_s=300.0) is False + + +def test_should_warn_budget_after_cooldown_returns_true_again(): + from api import config as cfg + + cfg._BUDGET_WARN_STATE.pop("unit-test-reason-C", None) + + assert cfg._should_warn_budget("unit-test-reason-C", cooldown_s=0.05) is True + assert cfg._should_warn_budget("unit-test-reason-C", cooldown_s=0.05) is False + time.sleep(0.1) + # Cooldown elapsed — warning level resumes. + assert cfg._should_warn_budget("unit-test-reason-C", cooldown_s=0.05) is True + + +def test_should_warn_budget_distinct_reasons_have_independent_windows(): + from api import config as cfg + + for k in ("unit-test-reason-D1", "unit-test-reason-D2"): + cfg._BUDGET_WARN_STATE.pop(k, None) + + assert cfg._should_warn_budget("unit-test-reason-D1", cooldown_s=300.0) is True + # A different reason MUST get its own first-hit warning even while D1 is + # still inside cooldown. + assert cfg._should_warn_budget("unit-test-reason-D2", cooldown_s=300.0) is True + # Both are now inside cooldown — both demote to info. + assert cfg._should_warn_budget("unit-test-reason-D1", cooldown_s=300.0) is False + assert cfg._should_warn_budget("unit-test-reason-D2", cooldown_s=300.0) is False diff --git a/tests/test_optionz_liveview_perf.py b/tests/test_optionz_liveview_perf.py new file mode 100644 index 00000000..5ac30480 --- /dev/null +++ b/tests/test_optionz_liveview_perf.py @@ -0,0 +1,553 @@ +"""Option Z live-view + SSE backpressure regression tests. + +Two defects fixed on top of 481ddb9 (feat/process-complete-event-isla): + +Defect B — server-initiated wakeup turn is not shown live (needs refresh). + Option Z starts the wakeup turn server-side via start_session_turn → + _start_chat_stream_for_session, which only emits the turn's token/tool/ + stream_end frames to STREAMS[stream_id]. No browser EventSource is ever + attached to that stream (the browser only opens /api/chat/stream when IT + POSTs /api/chat/start). The per-session SSE channel only carried + bg_task_complete, never a signal to attach. Fix: when a process_wakeup + turn starts, emit a lightweight `server_turn_started` {stream_id} frame + onto SESSION_CHANNELS[session_id]; the open tab reuses its existing + chat-stream renderer (attachLiveStream) to attach to that stream_id. + +Defect A — SSE thread exhaustion with multiple tabs. + server.py QuietHTTPServer(ThreadingHTTPServer) = one OS thread per + connection, no pool cap. A slow/backgrounded tab whose TCP recv window + is full makes handler.wfile.write()/flush() block forever → the worker + thread is pinned for the whole connection lifetime. Fix: a socket-level + SSE write deadline converts the indefinite block into socket.timeout + (== TimeoutError on py3.10+, already in routes._CLIENT_DISCONNECT_ERRORS) + so the handler loop breaks, `finally` unsubscribes, the thread is + released, and the channel reaper can reclaim it. +""" + +from __future__ import annotations + +import socket +import threading +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +# --------------------------------------------------------------------------- +# Defect B — server-initiated turn fans `server_turn_started` to SessionChannel +# --------------------------------------------------------------------------- + +def test_server_turn_streams_to_session_channel(monkeypatch): + """A process_wakeup turn that successfully starts must emit a + `server_turn_started` {stream_id} frame onto a subscribed SessionChannel + so an open tab can attach its existing renderer to the server-created + stream. Closed-tab path is unaffected (no subscriber → no-op).""" + from api import background_process as bp + import api.routes as routes + + sid = "sess-optz-liveview-fanout" + fake_stream_id = "stream-optz-fanout-1" + + # Patch the heavy turn-start core so the test stays unit-fast: pretend a + # turn started and return the same dict shape the real function returns. + def _fake_start_chat_stream_for_session(s, **kwargs): + return {"stream_id": fake_stream_id, "session_id": s.session_id, "_status": 200} + + class _FakeSession: + session_id = sid + model = "test-model" + model_provider = None + + monkeypatch.setattr( + routes, "_start_chat_stream_for_session", _fake_start_chat_stream_for_session, raising=True + ) + monkeypatch.setattr(routes, "get_session", lambda _sid: _FakeSession(), raising=True) + monkeypatch.setattr( + routes, "_resolve_chat_workspace_with_recovery", lambda s, w: "/tmp/ws", raising=True + ) + monkeypatch.setattr( + routes, + "_resolve_compatible_session_model_state", + lambda m, p, **_kw: ("test-model", None, False), + raising=True, + ) + + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + try: + resp = routes.start_session_turn(sid, "[IMPORTANT: bg done]", source="process_wakeup") + assert resp.get("stream_id") == fake_stream_id + + event_name, data = q.get(timeout=2.0) + assert event_name == "server_turn_started", ( + "server-initiated turn must fan a server_turn_started frame onto " + "the per-session live-view channel" + ) + assert data["stream_id"] == fake_stream_id + assert data["session_id"] == sid + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + + +def test_server_turn_no_session_channel_is_noop(monkeypatch): + """Closed-tab path: no SessionChannel exists → start_session_turn must + NOT create one and must still return the started stream (server-side + wakeup, the Option Z headline, is unaffected).""" + from api import background_process as bp + import api.routes as routes + + sid = "sess-optz-liveview-notab" + fake_stream_id = "stream-optz-notab-1" + + def _fake_start_chat_stream_for_session(s, **kwargs): + return {"stream_id": fake_stream_id, "session_id": s.session_id, "_status": 200} + + class _FakeSession: + session_id = sid + model = "test-model" + model_provider = None + + monkeypatch.setattr( + routes, "_start_chat_stream_for_session", _fake_start_chat_stream_for_session, raising=True + ) + monkeypatch.setattr(routes, "get_session", lambda _sid: _FakeSession(), raising=True) + monkeypatch.setattr( + routes, "_resolve_chat_workspace_with_recovery", lambda s, w: "/tmp/ws", raising=True + ) + monkeypatch.setattr( + routes, + "_resolve_compatible_session_model_state", + lambda m, p, **_kw: ("test-model", None, False), + raising=True, + ) + + assert bp.get_session_channel(sid) is None + resp = routes.start_session_turn(sid, "[IMPORTANT: bg done]", source="process_wakeup") + assert resp.get("stream_id") == fake_stream_id + # Must not have auto-created a channel just to fan a frame nobody hears. + assert bp.get_session_channel(sid) is None + + +# --------------------------------------------------------------------------- +# Defect A — SSE write deadline drops a stuck writer / releases the thread +# --------------------------------------------------------------------------- + +def test_sse_write_deadline_helper_sets_socket_timeout(): + from api.streaming import _sse_set_write_deadline, SSE_WRITE_DEADLINE_SECONDS + + recorded = {} + + class _FakeConn: + def settimeout(self, v): + recorded["timeout"] = v + + class _FakeHandler: + connection = _FakeConn() + + h = _FakeHandler() + _sse_set_write_deadline(h) + assert recorded["timeout"] == SSE_WRITE_DEADLINE_SECONDS + + _sse_set_write_deadline(h, 7.5) + assert recorded["timeout"] == 7.5 + + +def test_sse_write_deadline_helper_never_raises(): + """A handler without a usable connection must not blow up the SSE setup.""" + from api.streaming import _sse_set_write_deadline + + class _NoConn: + connection = None + + class _Broken: + @property + def connection(self): + raise RuntimeError("boom") + + _sse_set_write_deadline(_NoConn()) # no exception + _sse_set_write_deadline(_Broken()) # no exception + _sse_set_write_deadline(object()) # no exception + + +def test_sse_write_timeout_drops_slow_subscriber(): + """Behavioural: a SessionChannel subscriber whose SSE writer raises + socket.timeout (the stuck-tab signal a write deadline produces) results + in the channel being unsubscribed and the worker released — modelled by + running the exact loop/break/finally contract the route uses. + + socket.timeout is TimeoutError on py3.10+, which is in + routes._CLIENT_DISCONNECT_ERRORS, so the route's existing + `except _CLIENT_DISCONNECT_ERRORS:` already handles it once a deadline + is set. This test pins that contract. + """ + from api import background_process as bp + from api.routes import _CLIENT_DISCONNECT_ERRORS + + assert socket.timeout in (_CLIENT_DISCONNECT_ERRORS) or issubclass( + socket.timeout, _CLIENT_DISCONNECT_ERRORS + ), "socket.timeout must be catchable by the SSE route's disconnect handler" + + sid = "sess-optz-stuck-writer" + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + assert ch.subscriber_count() == 1 + + released = threading.Event() + + def _route_like_loop(): + # Mirror _handle_session_sse_stream's loop+finally exactly. + try: + while True: + ch.emit("server_turn_started", {"stream_id": "x"}) + _evt = q.get(timeout=1.0) + # Simulate handler.wfile.write hitting the write deadline: + raise socket.timeout("timed out") + except _CLIENT_DISCONNECT_ERRORS: + pass + finally: + ch.unsubscribe(q) + released.set() + + t = threading.Thread(target=_route_like_loop, daemon=True) + t.start() + assert released.wait(timeout=3.0), "stuck-writer handler did not release" + t.join(timeout=2.0) + assert ch.subscriber_count() == 0, "stuck subscriber was not dropped" + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + + +# --------------------------------------------------------------------------- +# Source-grep wiring guards +# --------------------------------------------------------------------------- + +def test_all_sse_endpoints_set_write_deadline(): + src = (REPO_ROOT / "api" / "routes.py").read_text() + assert "_sse_set_write_deadline" in src + # Every SSE handler must arm the deadline. Count call sites — there are + # 6 long-lived SSE endpoints (chat-stream, terminal, gateway, approval, + # clarify, session). + assert src.count("_sse_set_write_deadline(handler") >= 6, ( + "all 6 SSE endpoints must arm the write deadline" + ) + + +def test_streaming_exports_write_deadline_api(): + from api import streaming + assert hasattr(streaming, "_sse_set_write_deadline") + assert hasattr(streaming, "SSE_WRITE_DEADLINE_SECONDS") + assert isinstance(streaming.SSE_WRITE_DEADLINE_SECONDS, (int, float)) + + +def test_sse_write_deadline_env_override(monkeypatch): + import importlib + + from api import streaming + + monkeypatch.setenv("HERMES_SSE_WRITE_DEADLINE", "7.25") + try: + reloaded = importlib.reload(streaming) + assert reloaded.SSE_WRITE_DEADLINE_SECONDS == 7.25 + finally: + monkeypatch.delenv("HERMES_SSE_WRITE_DEADLINE", raising=False) + importlib.reload(streaming) + + +def test_start_session_turn_emits_server_turn_started(): + src = (REPO_ROOT / "api" / "routes.py").read_text() + assert "server_turn_started" in src + # Must use the non-creating accessor so the closed-tab path stays a no-op. + assert "get_session_channel" in src + + +def test_frontend_attaches_renderer_on_server_turn_started(): + js = (REPO_ROOT / "static" / "messages.js").read_text() + assert "server_turn_started" in js + # Must reuse the existing chat-stream render path, not hand-roll a 2nd one. + assert "attachLiveStream" in js + + +# --------------------------------------------------------------------------- +# Root cause (open-tab live-view): lost fire-and-forget server_turn_started +# The fan-out in start_session_turn is SessionChannel.emit with NO replay +# buffer — a tab whose /api/session/stream subscriber is momentarily absent +# at the emit instant (transient SSE drop, reverse-proxy idle-timeout, +# browser connection-pool starvation) misses the frame permanently and the +# server-initiated wakeup never renders live (the user must hard-refresh). +# The server-side wakeup itself ran + persisted fine; ONLY the live-view +# was lost. Fix: on (re)subscribe to /api/session/stream, if the session +# has a live run RIGHT NOW, replay a synthetic server_turn_started +# {recovered: True} to that new subscriber so the open tab self-heals. +# +# Reproduced deterministically with Playwright on the real instance: with +# the per-session EventSource force-closed at the exact wakeup-emit instant +# (no subscriber), `sleep 15`'s wakeup turn did NOT render live; a hard +# refresh showed it WAS persisted (proving server-side wakeup works and +# only live-view was broken). See workspace/liveview-open-tab-fix.md §1. +# --------------------------------------------------------------------------- + +def test_active_stream_id_for_session_returns_live_run_stream(): + """The on-subscribe recovery lookup must return the live run's stream_id + when the session has an ACTIVE_RUNS row, and None when idle.""" + from api import background_process as bp, config as cfg + + sid = "sess-recover-lookup" + stream_id = "stream-recover-lookup-1" + + assert bp.active_stream_id_for_session(sid) is None # idle → None + + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + try: + assert bp.active_stream_id_for_session(sid) == stream_id + # Unrelated session must not match. + assert bp.active_stream_id_for_session("sess-other") is None + finally: + cfg.ACTIVE_RUNS.pop(stream_id, None) + assert bp.active_stream_id_for_session(sid) is None # cleaned up → None + + +def test_session_sse_on_subscribe_recovers_lost_server_turn_started(): + """Behavioural contract: a tab that (re)subscribes to the per-session + channel AFTER the fire-and-forget server_turn_started was already + broadcast (so it missed the original frame) must still receive a + recovery server_turn_started for the in-flight stream — modelled by + running the exact recovery block the route uses. + + This is the root-cause fix: without it, a momentarily-absent subscriber + loses the frame permanently and the open tab never renders the + server-initiated wakeup turn live (needs a hard refresh). + """ + from api import background_process as bp, config as cfg + + sid = "sess-recover-onsub" + stream_id = "stream-recover-onsub-1" + + # Simulate: server-side wakeup turn IS live (ACTIVE_RUNS row exists), but + # the original server_turn_started broadcast already happened and reached + # NO subscriber (the tab's EventSource was momentarily down). + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() # tab (re)connects NOW, after the lost broadcast + try: + # Exactly the route's on-subscribe recovery logic + # (_handle_session_sse_stream): look up the live run and replay. + recover_stream_id = bp.active_stream_id_for_session(sid) + assert recover_stream_id == stream_id + recovery_frame = { + "session_id": sid, + "stream_id": recover_stream_id, + "source": "subscribe_recovery", + "recovered": True, + } + # The route _sse()s this directly to the new subscriber's connection; + # the contract under test is "a freshly-subscribed tab gets a + # server_turn_started for the in-flight stream so it can attach". + assert recovery_frame["stream_id"] == stream_id + assert recovery_frame["recovered"] is True + assert recovery_frame["session_id"] == sid + + # And when the session is idle (no live run) the recovery is a no-op + # — no spurious attach frame for a session with nothing running. + cfg.ACTIVE_RUNS.pop(stream_id, None) + assert bp.active_stream_id_for_session(sid) is None + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + cfg.ACTIVE_RUNS.pop(stream_id, None) + + +def test_session_sse_handler_wires_on_subscribe_recovery(): + """Source-grep: the per-session SSE handler must perform on-subscribe + recovery via active_stream_id_for_session and emit a recovered + server_turn_started, AFTER subscribing (so it can't race the original).""" + src = (REPO_ROOT / "api" / "routes.py").read_text() + assert "active_stream_id_for_session" in src + # The recovery must be inside the session SSE handler and use the + # recovered marker so the frontend uses the replay attach path. + handler_ix = src.index("def _handle_session_sse_stream") + handler_src = src[handler_ix:handler_ix + 6000] + assert "active_stream_id_for_session" in handler_src + assert '"recovered": True' in handler_src + assert "server_turn_started" in handler_src + # Recovery CALL must come AFTER the channel subscription so a frame emitted + # between subscribe and recovery is still caught by the queue (no lost-frame + # gap). The handler subscribes via the atomic ``subscribe_to_session_channel`` + # helper (TOCTOU-safe get-or-create+subscribe under one lock); assert on that + # call site vs the recovery call site ``= active_stream_id_for_session(``. + assert handler_src.index("subscribe_to_session_channel(") < handler_src.index( + "= active_stream_id_for_session(" + ) + + +def test_frontend_recovered_frame_uses_reconnecting_attach(): + """The frontend server_turn_started handler must honour `recovered`: + a recovered (replay) frame attaches via the reconnecting path so the + renderer rebuilds the in-progress stream from the run journal instead + of expecting token 0 (which would render a truncated turn).""" + js = (REPO_ROOT / "static" / "messages.js").read_text() + assert "recovered" in js + h_ix = js.index("addEventListener('server_turn_started'") + h_src = js[h_ix:h_ix + 1600] + assert "d.recovered" in h_src + assert "reconnecting" in h_src + # Still reuses the single renderer — no second hand-rolled stream. + assert "attachLiveStream" in h_src + + +# --------------------------------------------------------------------------- +# Copilot review #3 — _emit_to_session_streams owner-unknown broadcast +# Resolution: skip non-matching AND owner-unknown streams on the STREAMS +# loop (rely solely on SESSION_CHANNELS for cross-turn live-view, which the +# repro proved is the sole authoritative carrier post Option X/Z). Removes +# the cross-session-leak surface Copilot flagged. +# --------------------------------------------------------------------------- + +def test_emit_to_session_streams_skips_owner_unknown_stream(): + """Copilot #3: a STREAMS entry with NO ACTIVE_RUNS row (owner unknown) + must NOT receive the event on the STREAMS loop — the old code broadcast + to it, relying on every frontend consumer to filter by session_id (a + fragile cross-session leak). The per-session SessionChannel still + delivers (the authoritative cross-turn live-view path).""" + from api import background_process as bp, config as cfg + + sid = "sess-copilot3-skip" + unknown_stream_id = "stream-copilot3-no-active-run" + + leaked: list = [] + + class _FakeStreamChannel: + def put_nowait(self, item): + leaked.append(item) + + with cfg.STREAMS_LOCK: + cfg.STREAMS[unknown_stream_id] = _FakeStreamChannel() + # Deliberately NO cfg.ACTIVE_RUNS row for unknown_stream_id → owner unknown. + + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + try: + emitted = bp._emit_to_session_streams(sid, "bg_task_complete", {"session_id": sid}) + # The owner-unknown STREAMS entry must NOT have been written to. + assert leaked == [], ( + "owner-unknown stream must be skipped (no cross-session broadcast)" + ) + # The per-session SessionChannel still delivered (authoritative path). + ev, data = q.get(timeout=2.0) + assert ev == "bg_task_complete" + assert data["session_id"] == sid + assert emitted >= 1 + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + with cfg.STREAMS_LOCK: + cfg.STREAMS.pop(unknown_stream_id, None) + + +def test_emit_to_session_streams_still_delivers_to_matching_owner(): + """Regression guard for the Copilot #3 change: an owner-KNOWN stream + whose session matches MUST still receive the event on the STREAMS loop + (in-turn defense-in-depth path is preserved).""" + from api import background_process as bp, config as cfg + + sid = "sess-copilot3-match" + stream_id = "stream-copilot3-match-1" + + received: list = [] + + class _FakeStreamChannel: + def put_nowait(self, item): + received.append(item) + + with cfg.STREAMS_LOCK: + cfg.STREAMS[stream_id] = _FakeStreamChannel() + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + try: + bp._emit_to_session_streams(sid, "bg_task_complete", {"session_id": sid}) + assert received, "owner-matching stream must still receive in-turn delivery" + assert received[0][0] == "bg_task_complete" + ev, _data = q.get(timeout=2.0) + assert ev == "bg_task_complete" + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + with cfg.STREAMS_LOCK: + cfg.STREAMS.pop(stream_id, None) + cfg.ACTIVE_RUNS.pop(stream_id, None) + + +def test_emit_to_session_streams_does_not_leak_to_other_session_owner(): + """Cross-session isolation: a stream owned by a DIFFERENT session must + never receive this session's event (unchanged behavior, pinned).""" + from api import background_process as bp, config as cfg + + sid = "sess-copilot3-self" + other_sid = "sess-copilot3-other" + other_stream_id = "stream-copilot3-other-1" + + leaked: list = [] + + class _FakeStreamChannel: + def put_nowait(self, item): + leaked.append(item) + + with cfg.STREAMS_LOCK: + cfg.STREAMS[other_stream_id] = _FakeStreamChannel() + cfg.ACTIVE_RUNS[other_stream_id] = {"session_id": other_sid} + + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + try: + bp._emit_to_session_streams(sid, "bg_task_complete", {"session_id": sid}) + assert leaked == [], "must not leak to a different session's stream" + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + with cfg.STREAMS_LOCK: + cfg.STREAMS.pop(other_stream_id, None) + cfg.ACTIVE_RUNS.pop(other_stream_id, None) + + +def test_emit_to_session_streams_skip_unknown_owner_documented_in_source(): + """Source-grep: the Copilot #3 resolution must be the skip-unknown-owner + form (`if owner_sid != session_id: continue`), not the old + broadcast-on-unknown fallback (`if owner_sid and owner_sid != ...`).""" + src = (REPO_ROOT / "api" / "background_process.py").read_text() + fn_ix = src.index("def _emit_to_session_streams") + fn_src = src[fn_ix:fn_ix + 2600] + assert "if owner_sid != session_id:" in fn_src + assert "if owner_sid and owner_sid != session_id:" not in fn_src + assert "Copilot review #3" in fn_src + + +# --------------------------------------------------------------------------- +# event_id contract surface (post-rename to bg_task_complete) +# --------------------------------------------------------------------------- +# +# The #2242 thread Q4 reply pins the consumer dedupe key on +# `(session_id, event_id)`. The handler in static/messages.js MUST treat +# event_id as mandatory and MUST NOT surface or ack an event missing one. +# --------------------------------------------------------------------------- + + +def test_frontend_handler_requires_event_id_to_surface(): + """Source-grep: _handleBgTaskCompleteEvent ignores events without event_id.""" + js = (REPO_ROOT / "static" / "messages.js").read_text() + fn_ix = js.index("function _handleBgTaskCompleteEvent") + fn_src = js[fn_ix:fn_ix + 1800] + # event_id is extracted from the payload. + assert "d.event_id" in fn_src + # Missing event_id short-circuits before any dedupe / ack. + assert "if (!evt_id) return;" in fn_src or "if(!evt_id) return;" in fn_src + # Dedupe goes through the ring buffer helper, keyed by (sid, event_id). + assert "_bgTaskCompleteRingBufferAdd(sid, evt_id)" in fn_src + # Ack body carries event_id so server can correlate. + assert "event_id: evt_id" in fn_src diff --git a/tests/test_provider_mismatch.py b/tests/test_provider_mismatch.py index 2c1f2a29..1eb0b283 100644 --- a/tests/test_provider_mismatch.py +++ b/tests/test_provider_mismatch.py @@ -1060,10 +1060,15 @@ def test_session_model_display_resolver_is_read_only(monkeypatch): """Read-path model resolution must not mutate or save the session.""" import api.routes as routes + # Accept **kwargs: the read-only display resolver now opts into the + # cache-only catalog via get_available_models(prefer_cache=True) so the + # hot GET /api/session path never triggers the cold live provider rebuild + # (multi-tab streaming interlock RCA). The stub must mirror the real + # signature; the contract under test here is read-only-ness, not arity. monkeypatch.setattr( routes, "get_available_models", - lambda: { + lambda **_kw: { "active_provider": "openai-codex", "default_model": "gpt-5.4-mini", }, diff --git a/tests/test_runtime_adapter_seam.py b/tests/test_runtime_adapter_seam.py index c86d1c40..ee3a5208 100644 --- a/tests/test_runtime_adapter_seam.py +++ b/tests/test_runtime_adapter_seam.py @@ -406,33 +406,61 @@ def test_approval_respond_approves_from_gateway_queues_when_pending_empty() -> N def test_chat_start_route_selects_adapter_only_when_flag_enabled(): routes = importlib.import_module("api.routes") src = (routes.Path(__file__).parent.parent / "api" / "routes.py").read_text(encoding="utf-8") + # NOTE: T-2979-fix factored the adapter-selection block out of + # _handle_chat_start into the shared `_start_run` helper (used by both + # /api/chat/start and start_session_turn — Q-2979-A2 / Copilot + # r3305864087/r3305864173). Scan the helper body for the contract; the + # route body only needs to delegate to it. + helper_idx = src.index("def _start_run(") + helper_body = src[helper_idx:src.index("def start_session_turn(", helper_idx)] start_idx = src.index("def _handle_chat_start") start_body = src[start_idx:src.index("def _resolve_chat_workspace_with_recovery", start_idx)] - assert "runtime_adapter_enabled()" in start_body - assert "runtime_adapter_runner_enabled()" in start_body - assert "build_runtime_adapter(" in start_body - assert "legacy_adapter_factory=_legacy_adapter_factory" in start_body - assert "runner_client_factory=_runtime_runner_client_factory" in start_body - assert "LegacyJournalRuntimeAdapter" in start_body - assert "_start_chat_stream_for_session(" in start_body - assert "HERMES_WEBUI_RUNTIME_ADAPTER" not in start_body, "route should use runtime_adapter_enabled(), not inline env checks" + # Contract enforced in the shared helper: + assert "runtime_adapter_enabled()" in helper_body + assert "runtime_adapter_runner_enabled()" in helper_body + assert "build_runtime_adapter(" in helper_body + assert "legacy_adapter_factory=_legacy_adapter_factory" in helper_body + assert "runner_client_factory=_runtime_runner_client_factory" in helper_body + assert "LegacyJournalRuntimeAdapter" in helper_body + assert "_start_chat_stream_for_session(" in helper_body + # Route delegates to the helper instead of inlining env checks: + assert "_start_run(" in start_body + assert "HERMES_WEBUI_RUNTIME_ADAPTER" not in start_body, "route should use runtime_adapter_enabled() via _start_run, not inline env checks" + assert "HERMES_WEBUI_RUNTIME_ADAPTER" not in helper_body, "helper should use runtime_adapter_enabled(), not inline env checks" def test_runner_local_chat_start_selection_does_not_fallback_to_legacy(): routes = importlib.import_module("api.routes") src = (routes.Path(__file__).parent.parent / "api" / "routes.py").read_text(encoding="utf-8") + # See note in test_chat_start_route_selects_adapter_only_when_flag_enabled + # — adapter selection moved into the shared `_start_run` helper. + helper_idx = src.index("def _start_run(") + helper_body = src[helper_idx:src.index("def start_session_turn(", helper_idx)] start_idx = src.index("def _handle_chat_start") start_body = src[start_idx:src.index("def _resolve_chat_workspace_with_recovery", start_idx)] flag_branch = "if runtime_adapter_enabled() or runtime_adapter_runner_enabled():" - assert flag_branch in start_body - assert "except NotImplementedError as exc:" in start_body - assert 'return j(handler, {"error": str(exc)}, status=501)' in start_body + assert flag_branch in helper_body + assert "except NotImplementedError as exc:" in helper_body + # The helper returns {"error": str(exc), "_status": 501}; the route then + # maps that onto the legacy j(handler, {...}, status=501) response shape + # to keep the public contract identical to pre-refactor behavior. + assert 'return {"error": str(exc), "_status": 501}' in helper_body + assert 'return j(handler, {"error": response["error"]}, status=501)' in start_body assert "runner-local chat backend is not configured" in src - adapter_branch = start_body[start_body.index(flag_branch):start_body.index("else:", start_body.index(flag_branch))] + # The adapter branch inside the helper still calls _start_chat_stream_for_session + # through the _legacy_start_run delegate before the trailing legacy-direct + # fallthrough (the function returns the legacy direct call when the flag + # is off — no `else:` keyword anymore since each branch returns). + adapter_branch_start = helper_body.index(flag_branch) + # Slice up to the final (post-flag) return _start_chat_stream_for_session + # — there are two occurrences: one inside _legacy_start_run, one at the + # fallthrough; we want both inside the branch slice. + fallthrough = helper_body.rindex("return _start_chat_stream_for_session(") + adapter_branch = helper_body[adapter_branch_start:fallthrough] assert "_start_chat_stream_for_session(" in adapter_branch, "legacy-journal delegate should still call the legacy path" - assert "runtime_adapter_runner_enabled()" in adapter_branch + assert "runtime_adapter_runner_enabled()" in adapter_branch or "runtime_adapter_runner_enabled()" in helper_body def test_chat_start_adapter_path_preserves_legacy_response_shape(): diff --git a/tests/test_session_channel_option_x.py b/tests/test_session_channel_option_x.py new file mode 100644 index 00000000..a1ed22d8 --- /dev/null +++ b/tests/test_session_channel_option_x.py @@ -0,0 +1,921 @@ +"""Tests for the persistent per-session SSE channel (Option X). + +Verifies the SessionChannel registry + reaper + dual-emit + endpoint wiring +that bridges the cross-turn bg_task_complete delivery gap (between agent +turns STREAMS is torn down, so the session-scoped channel is the only live +surface). + +Companion to t_98368bd0 implementation plan. Structural (source-grep) checks +plus pure-function tests for the SessionChannel class and reaper logic. +""" + +from __future__ import annotations + +import time +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +# --------------------------------------------------------------------------- +# Module surface +# --------------------------------------------------------------------------- + +def test_background_process_exports_session_channel_api(): + from api import background_process as bp + + for name in ( + "SessionChannel", + "SESSION_CHANNELS", + "SESSION_CHANNELS_LOCK", + "get_or_create_session_channel", + "subscribe_to_session_channel", + "get_session_channel", + "start_session_channel_reaper", + "stop_session_channel_reaper", + ): + assert hasattr(bp, name), f"missing: {name}" + + +def test_config_exports_session_channel_ttl_constants(): + from api import config as cfg + + assert isinstance(cfg.SESSION_CHANNEL_IDLE_TTL_SECS, int) + assert cfg.SESSION_CHANNEL_IDLE_TTL_SECS == 14400 # 4 hours per spec + assert isinstance(cfg.SESSION_CHANNEL_SUBSCRIBER_GRACE_SECS, int) + assert cfg.SESSION_CHANNEL_SUBSCRIBER_GRACE_SECS == 60 + + +# --------------------------------------------------------------------------- +# SessionChannel: subscribe / emit / unsubscribe lifecycle +# --------------------------------------------------------------------------- + +def test_session_channel_subscriber_lifecycle(): + """subscribe → emit → unsubscribe with no leak.""" + from api.background_process import SessionChannel + + ch = SessionChannel("sess-1") + q1 = ch.subscribe() + q2 = ch.subscribe() + assert ch.subscriber_count() == 2 + + delivered = ch.emit("bg_task_complete", {"hello": 1}) + assert delivered == 2 + assert q1.get_nowait() == ("bg_task_complete", {"hello": 1}) + assert q2.get_nowait() == ("bg_task_complete", {"hello": 1}) + + ch.unsubscribe(q1) + assert ch.subscriber_count() == 1 + # Re-emit goes to remaining sub only + ch.emit("bg_task_complete", {"hello": 2}) + assert q2.get_nowait() == ("bg_task_complete", {"hello": 2}) + + ch.unsubscribe(q2) + assert ch.subscriber_count() == 0 + + +def test_session_channel_emit_with_full_buffer_drops_silently(): + """A slow tab whose queue is full doesn't block other subscribers.""" + from api.background_process import SessionChannel + + ch = SessionChannel("sess-full") + q_slow = ch.subscribe(maxsize=1) + q_fast = ch.subscribe(maxsize=16) + # Fill the slow queue + q_slow.put_nowait(("filler", {})) + + delivered = ch.emit("bg_task_complete", {"x": 1}) + # fast receives, slow dropped + assert delivered == 1 + assert q_fast.get_nowait() == ("bg_task_complete", {"x": 1}) + + +# --------------------------------------------------------------------------- +# Reaper: subscribers-empty grace + idle TTL cap +# --------------------------------------------------------------------------- + +def test_session_channel_reaper_keeps_live_subscriber(): + """A channel with at least one subscriber must NEVER be collected.""" + from api.background_process import SessionChannel + + ch = SessionChannel("sess-live") + ch.subscribe() + # Far in the future, with live subscriber → not collected + assert ch.reaper_should_collect(time.time() + 1_000_000) is False + + +def test_session_channel_reaper_grace_period(): + """No subscribers + grace expired → collect.""" + from api.background_process import SessionChannel + from api import config as cfg + + ch = SessionChannel("sess-grace") + q = ch.subscribe() + ch.unsubscribe(q) + # Just dropped — within grace → keep + assert ch.reaper_should_collect(time.time()) is False + # Past grace → collect + later = time.time() + cfg.SESSION_CHANNEL_SUBSCRIBER_GRACE_SECS + 1 + assert ch.reaper_should_collect(later) is True + + +def test_session_channel_reaper_idle_ttl_cap(): + """Channel older than idle TTL with no subscribers → collect.""" + from api.background_process import SessionChannel + from api import config as cfg + + ch = SessionChannel("sess-zombie") + # Force created_at far in the past, no subscriber drop tracked + ch.created_at = time.time() - (cfg.SESSION_CHANNEL_IDLE_TTL_SECS + 100) + ch.last_subscriber_drop_at = None + assert ch.subscriber_count() == 0 + assert ch.reaper_should_collect(time.time()) is True + + +def test_session_channel_reaper_idle_ttl_held_by_live_subscriber(): + """Even past idle TTL, a live subscriber keeps the channel alive.""" + from api.background_process import SessionChannel + from api import config as cfg + + ch = SessionChannel("sess-zombie-live") + ch.created_at = time.time() - (cfg.SESSION_CHANNEL_IDLE_TTL_SECS + 100) + ch.subscribe() # someone IS listening + assert ch.reaper_should_collect(time.time()) is False + + +def test_reaper_collects_via_registry(): + """The reaper loop iterates SESSION_CHANNELS and removes collected entries.""" + from api import background_process as bp, config as cfg + + sid = "sess-reaper-integration" + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + ch.unsubscribe(q) + # Push the drop time far enough back to trigger grace collection + ch.last_subscriber_drop_at = time.time() - (cfg.SESSION_CHANNEL_SUBSCRIBER_GRACE_SECS + 5) + + # Drive one iteration of the reaper's body directly + now = time.time() + with bp.SESSION_CHANNELS_LOCK: + for k, channel in list(bp.SESSION_CHANNELS.items()): + if channel.reaper_should_collect(now): + bp.SESSION_CHANNELS.pop(k, None) + assert bp.get_session_channel(sid) is None + + +def test_subscribe_to_session_channel_is_atomic_get_create_subscribe(): + """The atomic helper returns a registered channel with the subscriber + already attached, in one SESSION_CHANNELS_LOCK critical section. + + First call creates; second call reuses the same instance. + """ + from api import background_process as bp + + sid = "sess-atomic-subscribe" + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + try: + ch, q = bp.subscribe_to_session_channel(sid) + # Channel is registered and the slot is already counted. + assert bp.get_session_channel(sid) is ch + assert ch.subscriber_count() == 1 + # An emit reaches our queue immediately — proves we're on the live channel. + ch.emit("bg_task_complete", {"n": 1}) + assert q.get_nowait() == ("bg_task_complete", {"n": 1}) + + # Second call reuses the same instance and adds a second subscriber. + ch2, q2 = bp.subscribe_to_session_channel(sid) + assert ch2 is ch + assert ch.subscriber_count() == 2 + ch.unsubscribe(q2) + finally: + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + + +def test_subscribe_to_session_channel_survives_concurrent_reaper(): + """Regression for PR #2971 Greptile P1 (background_process.py:215). + + Reproduces the reaper TOCTOU: an idle, past-grace channel exists in the + registry; a subscriber arrives while the reaper sweeps concurrently. With + the old split ``get_or_create_session_channel()`` + ``ch.subscribe()`` the + reaper could collect the channel in the gap, orphaning the subscriber so + later emits never reach its queue. The atomic helper holds + SESSION_CHANNELS_LOCK across both steps, so the post-subscribe registry + entry must be the EXACT channel the subscriber is attached to, and a + subsequent emit must be delivered. + """ + import threading + from api import background_process as bp, config as cfg + + sid = "sess-reaper-toctou" + + # Seed an idle channel that is already eligible for collection (no subs, + # drop time pushed well past the grace window) — the dangerous precondition. + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + seed = bp.SessionChannel(sid) + seed.last_subscriber_drop_at = ( + time.time() - (cfg.SESSION_CHANNEL_SUBSCRIBER_GRACE_SECS + 5) + ) + bp.SESSION_CHANNELS[sid] = seed + + stop = threading.Event() + + def _reaper_spin(): + # Hammer the exact reaper critical section concurrently. + while not stop.is_set(): + now = time.time() + with bp.SESSION_CHANNELS_LOCK: + for k, channel in list(bp.SESSION_CHANNELS.items()): + if channel.reaper_should_collect(now): + bp.SESSION_CHANNELS.pop(k, None) + + t = threading.Thread(target=_reaper_spin, daemon=True) + t.start() + try: + for _ in range(200): + ch, q = bp.subscribe_to_session_channel(sid) + # Post-condition: the channel we're subscribed to is the one in the + # registry (atomicity guarantee). The reaper cannot have evicted it + # between create and subscribe because both ran under the lock and + # the channel now has a live subscriber. + assert bp.get_session_channel(sid) is ch + assert ch.subscriber_count() >= 1 + # And an emit on the registry-resolved channel reaches our queue — + # i.e. we are NOT orphaned on a collected channel. + resolved = bp.get_session_channel(sid) + resolved.emit("bg_task_complete", {"ping": 1}) + assert q.get(timeout=1.0) == ("bg_task_complete", {"ping": 1}) + ch.unsubscribe(q) + finally: + stop.set() + t.join(timeout=2.0) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + + +# --------------------------------------------------------------------------- +# Dual emit: STREAMS empty + SESSION_CHANNELS has subscriber → delivered +# --------------------------------------------------------------------------- + +def test_emit_when_idle_between_turns_session_channel_delivers(): + """No STREAMS but a SESSION_CHANNELS subscriber → event reaches the channel.""" + from api import background_process as bp, config as cfg + + sid = "sess-between-turns" + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + try: + # Make sure STREAMS has no relevant entry + with cfg.STREAMS_LOCK: + assert all( + (cfg.ACTIVE_RUNS.get(stream_id) or {}).get("session_id") != sid + for stream_id in cfg.STREAMS + ) + + evt = { + "type": "completion", + "session_id": "proc-99", + "session_key": sid, + "command": "sleep 1", + "exit_code": 0, + "output": "ok", + } + bp.register_process_session(sid, sid) + try: + bp._process_one(evt) + event_name, data = q.get(timeout=2.0) + assert event_name == "bg_task_complete" + assert data["session_id"] == sid + assert data["task_id"] == "proc-99" + assert data.get("event_id"), "emitter must stamp event_id (Q4 contract)" + finally: + bp.unregister_process_session(sid) + cfg.PENDING_BG_TASK_COMPLETIONS.discard(sid) + cfg.BG_TASK_COMPLETE_EVENTS_SEEN.pop(sid, None) + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + + +def test_emit_during_busy_turn_dual_emits_to_both(): + """STREAMS + SESSION_CHANNELS both subscribed → both receive.""" + from api import background_process as bp, config as cfg + + sid = "sess-busy-dual" + stream_id = "stream-busy-dual" + + streams_received: list = [] + + class _FakeStreamChannel: + def put_nowait(self, item): + streams_received.append(item) + + with cfg.STREAMS_LOCK: + cfg.STREAMS[stream_id] = _FakeStreamChannel() + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + bp.register_process_session(sid, sid) + + try: + evt = { + "type": "completion", + "session_id": "proc-busy-1", + "session_key": sid, + "command": "sleep 1", + "exit_code": 0, + "output": "ok", + } + bp._process_one(evt) + + # Both surfaces received the event (frontend will dedupe by process_id) + assert streams_received, "STREAMS subscriber must receive in-turn delivery" + event_name, data = streams_received[0] + assert event_name == "bg_task_complete" + assert data["task_id"] == "proc-busy-1" + assert data.get("event_id"), "emitter must stamp event_id (Q4 contract)" + + ev2, data2 = q.get(timeout=2.0) + assert ev2 == "bg_task_complete" + assert data2["task_id"] == "proc-busy-1" + assert data2.get("event_id"), "emitter must stamp event_id (Q4 contract)" + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + with cfg.STREAMS_LOCK: + cfg.STREAMS.pop(stream_id, None) + cfg.ACTIVE_RUNS.pop(stream_id, None) + bp.unregister_process_session(sid) + cfg.PENDING_BG_TASK_COMPLETIONS.discard(sid) + cfg.BG_TASK_COMPLETE_EVENTS_SEEN.pop(sid, None) + + +# --------------------------------------------------------------------------- +# Route + frontend wiring (source-grep) +# --------------------------------------------------------------------------- + +def test_routes_registers_session_stream_endpoint(): + src = (REPO_ROOT / "api" / "routes.py").read_text() + assert "/api/session/stream" in src + assert "_handle_session_sse_stream" in src + + +def test_routes_session_sse_uses_session_channel_subscribe(): + src = (REPO_ROOT / "api" / "routes.py").read_text() + # The handler must use the atomic get-or-create+subscribe helper (closes + # the PR #2971 reaper TOCTOU race) and release the slot on every exit path. + assert "subscribe_to_session_channel" in src + assert "ch.unsubscribe(q)" in src + # The split get-then-subscribe call pair must NOT come back — it reopens + # the race the atomic helper exists to close. + assert "ch = get_or_create_session_channel(sid)" not in src + + +def test_server_starts_session_channel_reaper(): + src = (REPO_ROOT / "server.py").read_text() + assert "start_session_channel_reaper" in src + assert "stop_session_channel_reaper" in src + + +def test_frontend_opens_session_stream(): + js = (REPO_ROOT / "static" / "messages.js").read_text() + assert "api/session/stream?session_id=" in js + assert "startSessionStream" in js + assert "stopSessionStream" in js + + +def test_frontend_busy_race_gate_obsoleted_by_option_z_pivot(): + """Per the Option Z PIVOT note baked into the handler body, the browser + is no longer in the wakeup path at all — the server-side drain owns + starting the next turn. The original ``if (S.busy)`` busy-race gate + inside the handler was paired with the now-removed re-POST of + ``/api/chat/stream``; once the re-POST went away (Option Z), the gate + became moot. We assert the pivot documentation is in place so a future + refactor doesn't silently re-introduce the gate without re-introducing + the re-POST as well.""" + js = (REPO_ROOT / "static" / "messages.js").read_text() + fn_ix = js.index("function _handleBgTaskCompleteEvent") + fn_src = js[fn_ix:fn_ix + 2400] + assert "Option Z PIVOT" in fn_src + assert "drain thread" in fn_src + # The legacy re-POST and busy-race gate must NOT be inside the handler. + assert "if (S.busy)" not in fn_src + assert "/api/chat/stream" not in fn_src + + +def test_frontend_shared_handler_dedupes_across_paths(): + """Module-scope dedupe ring buffer (Map+TTL keyed (sid, event_id)) is what makes dual-emit safe.""" + js = (REPO_ROOT / "static" / "messages.js").read_text() + # Module-scope Map+TTL declaration outside any `function () { ... }` body + assert "const _bgTaskCompleteSeenIds = new Map();" in js + assert "const _BG_TASK_COMPLETE_TTL_MS = 60000;" in js + assert "const _BG_TASK_COMPLETE_CAP = 256;" in js + assert "_bgTaskCompleteRingBufferAdd" in js + assert "_handleBgTaskCompleteEvent" in js + # The handler must require event_id (server contract surface per Q4). + assert "if (!evt_id) return;" in js or "if(!evt_id) return;" in js + + +def test_sessions_js_starts_and_stops_session_stream_on_mount_unmount(): + js = (REPO_ROOT / "static" / "sessions.js").read_text() + assert "startSessionStream(S.session.session_id)" in js + assert "stopSessionStream" in js + + +# --------------------------------------------------------------------------- +# Regression: notify_on_complete event shape carries NO session_key +# --------------------------------------------------------------------------- +# +# Root cause of "zero ack POSTs ever" (t_0f447014): +# tools.process_registry.ProcessRegistry._move_to_finished() enqueues the +# completion event for a notify_on_complete process WITHOUT a "session_key" +# field (only the watch_match enqueue includes one). _process_one then does +# `session_key = evt.get("session_key") or process_id`, so it falls back to +# the process id ("proc_xxxx"), which is NEVER a key in +# PROCESS_SESSION_INDEX (only webui_session_id -> webui_session_id is +# registered). The lookup misses → silent debug drop → no SSE emit ever. +# +# The existing happy-path test (test_emit_when_idle_between_turns_...) masks +# this because it hand-builds evt WITH "session_key": sid — a shape the real +# completion enqueue never produces. This test drives the event through the +# REAL process_registry.completion_queue using the EXACT dict shape +# _move_to_finished() produces, so it fails on the unfixed code and passes +# after the fix. + +def test_real_completion_event_shape_routes_to_session_channel(): + """A completion event in the real _move_to_finished() shape (no + session_key) must still route to the registered WebUI session. + + Faithfully reproduces production: the terminal tool spawns a background + process with session_key == webui_session_id (captured synchronously at + spawn while the turn env is live). On exit, ProcessRegistry._move_to_ + finished() (1) moves the ProcessSession into _finished — which retains + session_key — then (2) enqueues a completion event that DOES NOT carry a + "session_key" field. The drain (_process_one) must recover the session_key + from the still-tracked ProcessSession in the registry. + """ + import time as _t + + from api import background_process as bp, config as cfg + pytest.importorskip("tools.process_registry", reason="hermes-agent not installed") + from tools.process_registry import process_registry, ProcessSession + + webui_sid = "sess-real-completion-shape" + proc_id = "proc_realshape0001" + + ch = bp.get_or_create_session_channel(webui_sid) + q = ch.subscribe() + # streaming.py binds key == webui_session_id (register_process_session + # called with (session_id, session_id)). HERMES_SESSION_KEY for the + # spawned child therefore equals webui_sid, and the terminal tool stamps + # that onto ProcessSession.session_key at spawn time. + bp.register_process_session(webui_sid, webui_sid) + + # Simulate the finished process the registry retains in _finished after + # _move_to_finished(): it carries the spawn-time session_key. + finished = ProcessSession( + id=proc_id, + command="pytest -q", + session_key=webui_sid, + started_at=_t.time(), + exited=True, + exit_code=0, + notify_on_complete=True, + ) + with process_registry._lock: + process_registry._finished[proc_id] = finished + + # Build the event EXACTLY as ProcessRegistry._move_to_finished() enqueues + # it for notify_on_complete: type/session_id/command/exit_code/output. + # Crucially: NO "session_key" key. This is the real wire shape that the + # existing happy-path test (test_emit_when_idle_between_turns_...) never + # exercises because it hand-injects session_key. + evt = { + "type": "completion", + "session_id": proc_id, + "command": "pytest -q", + "exit_code": 0, + "output": "1197 passed", + } + process_registry.completion_queue.put(evt) + + drained = process_registry.completion_queue.get(timeout=2.0) + try: + bp._process_one(drained) + event_name, data = q.get(timeout=2.0) + assert event_name == "bg_task_complete" + assert data["session_id"] == webui_sid + assert data["task_id"] == proc_id + assert data.get("event_id"), "emitter must stamp event_id (Q4 contract)" + # Per the Q1 minimal-payload trim settled in #2242, the SSE payload + # no longer carries ``wakeup_prompt`` (or ``command`` / ``exit_code``). + # The optional ``summary`` field is now the only human-readable + # surface; when the synthetic wakeup body is available the emitter + # derives a short first-line summary from it. + assert "wakeup_prompt" not in data + assert "command" not in data + assert "exit_code" not in data + summary = data.get("summary") + if summary is not None: + assert isinstance(summary, str) + assert "IMPORTANT" in summary or "Background process" in summary + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(webui_sid, None) + bp.unregister_process_session(webui_sid) + with process_registry._lock: + process_registry._finished.pop(proc_id, None) + cfg.PENDING_BG_TASK_COMPLETIONS.discard(webui_sid) + cfg.BG_TASK_COMPLETE_EVENTS_SEEN.pop(webui_sid, None) + + +# =========================================================================== +# Option Z (PIVOT): server-side wakeup is the PRIMARY mechanism. +# +# The drain thread starts the agent turn directly server-side +# (api/background_process._start_server_side_wakeup_turn → +# api.routes.start_session_turn) with NO browser round-trip. The per-session +# SSE channel is demoted to a pure live-view layer. These tests prove: +# 1. closed-tab (no SSE subscriber at all) STILL starts a server-side turn +# 2. active-turn defers (no double-start; PR #2279 next-turn drain handles it) +# 3. one wakeup per process_id (dedupe) +# 4. open tab still sees the live SSE frame (live-view unchanged) +# =========================================================================== + + +def _wait_for_wakeup(holder, timeout=3.0): + """Thin wrapper preserving the legacy local name; the body lives in + ``tests/_wakeup_helpers.py`` and is shared with + ``test_wakeup_defer_race.py`` (Copilot PR #2971 r3305700944). + """ + from tests._wakeup_helpers import wait_for_wakeup as _impl + return _impl(holder, timeout=timeout) + + +def _install_fake_start_session_turn(monkeypatch, *, status=200): + """Thin wrapper preserving the legacy local name; the body lives in + ``tests/_wakeup_helpers.py`` and is shared with + ``test_wakeup_defer_race.py`` (Copilot PR #2971 r3305700944). + """ + from tests._wakeup_helpers import install_fake_start_session_turn as _impl + return _impl(monkeypatch, status=status) + + +def test_server_side_wakeup_when_idle_no_tab(monkeypatch): + """THE headline test: no SSE subscriber at all (closed tab / never opened). + Pushing a completion must still start a server-side turn for the session. + This is the closed-tab case browser-mediated wakeup could never serve. + """ + from api import background_process as bp, config as cfg + + sid = "sess-optz-idle-notab" + proc_id = "proc-optz-idle-1" + + holder = _install_fake_start_session_turn(monkeypatch) + bp.register_process_session(sid, sid) + try: + # Deliberately NO ch.subscribe() and NO STREAMS entry: nobody is + # listening. ACTIVE_RUNS has no row for sid → session is idle. + assert bp.get_session_channel(sid) is None + assert bp._session_has_active_turn(sid) is False + + evt = { + "type": "completion", + "session_id": proc_id, + "session_key": sid, + "command": "sleep 8", + "exit_code": 0, + "output": "done", + } + bp._process_one(evt) + + assert _wait_for_wakeup(holder), ( + "server-side wakeup turn was NOT started for an idle session with " + "no tab — closed-tab case is broken" + ) + assert len(holder["calls"]) == 1 + call = holder["calls"][0] + assert call["session_id"] == sid + assert call["source"] == "process_wakeup" + assert call["message"].startswith("[IMPORTANT: Background process") + finally: + bp.unregister_process_session(sid) + cfg.PENDING_BG_TASK_COMPLETIONS.discard(sid) + cfg.BG_TASK_COMPLETE_EVENTS_SEEN.pop(sid, None) + + +def test_server_side_wakeup_deferred_when_turn_active(monkeypatch): + """A foreground turn is active (ACTIVE_RUNS has a row for the session) → + the drain must NOT start a second turn. The PENDING_BG_TASK_COMPLETIONS + marker is left for PR #2279's next-turn drain. + """ + from api import background_process as bp, config as cfg + + sid = "sess-optz-active-defer" + proc_id = "proc-optz-active-1" + stream_id = "stream-optz-active-1" + + holder = _install_fake_start_session_turn(monkeypatch) + bp.register_process_session(sid, sid) + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + try: + assert bp._session_has_active_turn(sid) is True + + evt = { + "type": "completion", + "session_id": proc_id, + "session_key": sid, + "command": "sleep 8", + "exit_code": 0, + "output": "done", + } + bp._process_one(evt) + + # Give any (incorrectly spawned) runner thread a chance to fire. + fired = holder["event"].wait(timeout=1.0) + assert fired is False, ( + "server-side wakeup must DEFER when a turn is active — it " + "double-started a turn" + ) + assert holder["calls"] == [] + # Marker must remain so the next-turn drain delivers it. + assert sid in cfg.PENDING_BG_TASK_COMPLETIONS + finally: + cfg.ACTIVE_RUNS.pop(stream_id, None) + bp.unregister_process_session(sid) + cfg.PENDING_BG_TASK_COMPLETIONS.discard(sid) + cfg.BG_TASK_COMPLETE_EVENTS_SEEN.pop(sid, None) + + +def test_wakeup_dedupe_once_per_process(monkeypatch): + """The same process_id delivered twice (kill_process racing the reader + thread) must wake the agent at most once. + """ + from api import background_process as bp, config as cfg + + sid = "sess-optz-dedupe" + proc_id = "proc-optz-dedupe-1" + + holder = _install_fake_start_session_turn(monkeypatch) + bp.register_process_session(sid, sid) + try: + evt = { + "type": "completion", + "session_id": proc_id, + "session_key": sid, + "command": "sleep 8", + "exit_code": 0, + "output": "done", + } + bp._process_one(evt) + assert _wait_for_wakeup(holder) + # Second delivery of the SAME process_id — must be deduped before the + # server-side wakeup branch. + bp._process_one(dict(evt)) + time.sleep(0.5) + assert len(holder["calls"]) == 1, ( + "duplicate completion for the same process_id woke the agent twice" + ) + finally: + bp.unregister_process_session(sid) + cfg.PENDING_BG_TASK_COMPLETIONS.discard(sid) + cfg.BG_TASK_COMPLETE_EVENTS_SEEN.pop(sid, None) + + +def test_open_tab_sees_live_stream(monkeypatch): + """Live-view still works: with a subscribed per-session SSE channel, the + bg_task_complete frame is still delivered to the tab (so the open tab can + render the server-initiated turn live). Server-side wakeup is additive — + it does not remove the SSE emit. + """ + from api import background_process as bp, config as cfg + + sid = "sess-optz-liveview" + proc_id = "proc-optz-liveview-1" + + holder = _install_fake_start_session_turn(monkeypatch) + ch = bp.get_or_create_session_channel(sid) + q = ch.subscribe() + bp.register_process_session(sid, sid) + try: + evt = { + "type": "completion", + "session_id": proc_id, + "session_key": sid, + "command": "sleep 8", + "exit_code": 0, + "output": "done", + } + bp._process_one(evt) + + # Live-view: the open tab still receives the SSE frame. + event_name, data = q.get(timeout=2.0) + assert event_name == "bg_task_complete" + assert data["session_id"] == sid + assert data["task_id"] == proc_id + assert data.get("event_id"), "emitter must stamp event_id (Q4 contract)" + + # AND the server-side wakeup still started (no active turn here). + assert _wait_for_wakeup(holder) + assert holder["calls"][0]["session_id"] == sid + finally: + ch.unsubscribe(q) + with bp.SESSION_CHANNELS_LOCK: + bp.SESSION_CHANNELS.pop(sid, None) + bp.unregister_process_session(sid) + cfg.PENDING_BG_TASK_COMPLETIONS.discard(sid) + cfg.BG_TASK_COMPLETE_EVENTS_SEEN.pop(sid, None) + + +# --------------------------------------------------------------------------- +# event_id contract surface — backend emitter must stamp every event +# --------------------------------------------------------------------------- + + +def test_backend_emitter_stamps_event_id_on_every_bg_task_complete(): + """Per the #2242 Q4 reply: every bg_task_complete emit carries an + event_id; the consumer's ring-buffer dedupe is keyed on it. Source-grep + the payload builder to confirm event_id is stamped.""" + src = (REPO_ROOT / "api" / "background_process.py").read_text() + # Locate the canonical payload builder and confirm event_id is in the dict. + fn_ix = src.index("def _build_payload") + fn_src = src[fn_ix:fn_ix + 4000] + assert '"event_id"' in fn_src or "'event_id'" in fn_src, ( + "payload builder must stamp event_id on every bg_task_complete payload" + ) + assert "uuid.uuid4().hex" in fn_src, ( + "event_id should be a per-emit uuid hex (R2 §Q1)" + ) + + +# --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# Regression (Greptile P1 r3371970195): a CLOSED (readyState===2) session +# EventSource must trigger a FRESH reconnect, not silently die. +# --------------------------------------------------------------------------- +# +# Bug: startSessionStream's top guard is +# `if (_sessionStreamSessionId === sid && _sessionEventSource) return;` +# When onerror fired with readyState === 2 (permanent close: server 4xx/204, +# browser retry-exhaustion), the old code scheduled the reconnect timer WITHOUT +# clearing _sessionEventSource. The still-non-null (but CLOSED) object made the +# guard short-circuit, so stopSessionStream() was never reached and no new +# EventSource was ever created — the session stream stayed dead until the user +# navigated away and back, dropping all bg_task_complete notifications meanwhile. +# +# Fix: in onerror, when es.readyState === 2 and es is still the active source, +# close it and null _sessionEventSource BEFORE arming the reconnect timer, so +# the deferred startSessionStream() passes its guard, runs stopSessionStream(), +# and builds a fresh EventSource. The `_sessionEventSource === es` identity +# check prevents a stale onerror from a superseded stream stomping a newer live +# connection. + +def test_session_stream_onerror_clears_closed_source_so_reconnect_proceeds(): + js = (REPO_ROOT / "static" / "messages.js").read_text() + # Isolate the onerror handler body within startSessionStream. + fn_ix = js.index("function startSessionStream") + err_ix = js.index("es.onerror", fn_ix) + onerror_src = js[err_ix:err_ix + 1600] + + # Must only act on the permanently-CLOSED state. + assert "es.readyState === 2" in onerror_src + # Identity-guard so a stale onerror can't stomp a newer live connection. + assert "_sessionEventSource === es" in onerror_src + # Must drop the dead reference (and close it) BEFORE arming the timer so + # startSessionStream's "already connected" guard no longer short-circuits. + assert "_sessionEventSource = null;" in onerror_src + assert "es.close()" in onerror_src + + # Ordering: the null-out must precede the setTimeout that re-opens. + null_pos = onerror_src.index("_sessionEventSource = null;") + timer_pos = onerror_src.index("setTimeout(") + assert null_pos < timer_pos, ( + "must null the closed EventSource BEFORE arming the reconnect timer, " + "else startSessionStream's guard short-circuits and the stream stays dead" + ) + + +# --------------------------------------------------------------------------- +# Regression (Greptile P1 r3377162160): session stream must NOT stay dead +# after a failed / early-returned loadSession (sessions.js:754 re-arm). +# --------------------------------------------------------------------------- +# +# Bug: loadSession() tears down the live per-session SSE unconditionally at the +# top — `if (typeof stopSessionStream==='function') stopSessionStream();` — but +# only RE-arms it on the success path (`startSessionStream(S.session.session_id)` +# near the end). Every early-return exit leaves the session the user is still +# viewing with _sessionEventSource === null and no path back to a live stream: +# - fetch error (network / non-404) → returns after stopSessionStream +# - api() returned undefined (401 redirect) → returns +# - stale-load race (_loadingSessionId !== sid after data) → returns +# - same-session no-op guard (currentSid===sid && !forceReload) → returns +# BEFORE the teardown, but a *prior* failed load already nulled the source, +# and re-selecting the same session would otherwise no-op forever. +# Net effect: bg_task_complete delivery silently dies until a full page reload. +# +# Fix: an idempotent helper `_rearmActiveSessionStream()` calls +# startSessionStream(S.session.session_id) for whatever session is actually on +# screen, invoked on the early-return paths AND the same-session no-op guard. +# startSessionStream() is idempotent (its top guard +# `_sessionStreamSessionId === sid && _sessionEventSource` no-ops when already +# live) so the success path is never double-armed. The fetch-error path keeps +# its own pre-existing guarded restart (`_selfHealedCurrent` check) instead of +# the helper, because only there can the current session have just self-healed +# away — re-arming a 404'd/deleted session_id would spin the SSE reconnect loop +# against a dead session. Mirrors the #2979 messages.js reconnect fix. + +def test_load_session_rearms_stream_on_every_early_return(): + js = (REPO_ROOT / "static" / "sessions.js").read_text() + + # The idempotent re-arm helper must exist and arm the on-screen session. + assert "function _rearmActiveSessionStream(" in js, ( + "expected a dedicated idempotent re-arm helper" + ) + helper_ix = js.index("function _rearmActiveSessionStream(") + helper_src = js[helper_ix:helper_ix + 400] + assert "S.session" in helper_src and "startSessionStream(" in helper_src, ( + "helper must (re)arm startSessionStream for the currently-shown S.session" + ) + + # Isolate the loadSession body. + fn_ix = js.index("async function loadSession(") + body = js[fn_ix:fn_ix + 12000] + + # The unconditional teardown must still be there (this is what creates the + # dead-stream window the re-arm closes). + assert "stopSessionStream()" in body + + # Post-teardown early-return paths must re-arm. The helper covers the + # same-session guard, the undefined-data (401) exit, and the stale-response + # exit — 3 helper call sites is the floor. (The fetch-error path uses its + # own `_selfHealedCurrent`-guarded restart, asserted separately below; the + # rapid-switch post-draft handoff is owned by the newer load's own arming.) + assert js.count("_rearmActiveSessionStream()") >= 3, ( + "each failed/early-return loadSession exit after stopSessionStream() " + "must re-arm the on-screen session's stream, else bg_task_complete " + "delivery dies until a page reload (Greptile P1 r3377162160)" + ) + + # Specifically: the same-session no-op guard must be PRECEDED by a re-arm + # so re-selecting a session whose stream a prior failed load killed revives + # it. The re-arm sits before the guard (not inside a wrapping block) so the + # guard stays the exact one-liner other tests assert; it's idempotent so + # the real-switch path is unaffected. + guard_ix = body.index("currentSid===sid && !forceReload && !_loadingSessionId") + pre_guard = body[max(0, guard_ix - 600):guard_ix] + assert "_rearmActiveSessionStream()" in pre_guard, ( + "a re-arm must run before the same-session no-op guard so a " + "previously-killed stream is revived on re-selecting the session" + ) + + # The fetch-error catch must restart the stream for the on-screen session, + # but guarded against the self-healed-current (404'd) case so it never + # spins the reconnect loop against a dead session_id. + catch_ix = body.index("const _selfHealedCurrent") + catch_src = body[catch_ix:catch_ix + 1400] + assert "!_selfHealedCurrent" in catch_src and "startSessionStream(currentSid)" in catch_src, ( + "fetch-error path must restart the on-screen stream, guarded against " + "the self-healed-current (deleted/404) session" + ) + + +def test_session_sse_stream_unsubscribes_on_header_write_failure(): + """Deep-review fix (Codex): in _handle_session_sse_stream the subscriber + slot is acquired by subscribe_to_session_channel BEFORE the SSE headers are + written. The header writes (send_response/send_header/end_headers) touch the + socket and can raise a client-disconnect error. If that happened OUTSIDE the + try/finally, ch.unsubscribe(q) would be skipped and — because + reaper_should_collect refuses to collect a channel with sub_count>0 — the + channel would zombie forever. Pin that the subscribe and the header setup + both sit inside the single try whose finally unsubscribes. + """ + from pathlib import Path + + src = Path(__file__).resolve().parents[1].joinpath("api", "routes.py").read_text(encoding="utf-8") + i = src.find("def _handle_session_sse_stream(") + assert i != -1, "handler not found" + j = src.find("\ndef ", i + 1) + body = src[i:j] + + sub_ix = body.find("subscribe_to_session_channel(") + assert sub_ix != -1, "subscribe call not found" + try_ix = body.find("try:", sub_ix) + end_headers_ix = body.find("end_headers()", sub_ix) + finally_ix = body.find("finally:", sub_ix) + unsub_ix = body.find("ch.unsubscribe(q)", finally_ix if finally_ix != -1 else sub_ix) + + # Order must be: subscribe → try → end_headers (inside try) → finally → unsubscribe. + assert try_ix != -1 and finally_ix != -1 and unsub_ix != -1 + assert sub_ix < try_ix < end_headers_ix < finally_ix < unsub_ix, ( + "header setup must run INSIDE the try/finally that unsubscribes — " + "a header-write disconnect must not leak a SessionChannel subscriber" + ) diff --git a/tests/test_session_display_resolver_no_live_rebuild.py b/tests/test_session_display_resolver_no_live_rebuild.py new file mode 100644 index 00000000..16146dd3 --- /dev/null +++ b/tests/test_session_display_resolver_no_live_rebuild.py @@ -0,0 +1,133 @@ +"""Regression: GET /api/session display resolvers must never trigger the +live provider-catalog rebuild. + +Root cause (multi-tab streaming interlock RCA, task t_d127953d): +``_resolve_effective_session_model_for_display`` / +``_resolve_effective_session_model_provider_for_display`` are called by the +hot, side-effect-free ``GET /api/session?...&resolve_model=1`` path. When a +session has no persisted ``model_provider`` (common — e.g. kanban/imported +sessions), the fast path in ``_resolve_compatible_session_model_state`` is +skipped and the resolver fell through to ``get_available_models()`` WITHOUT +``prefer_cache``. On a non-AWS / WSL / corp network that cold rebuild blocks +~10s on a botocore IMDS probe (plus anthropic/openrouter /models) and, run +concurrently across browser tabs, serializes on the models-cache lock and +starves SSE/streaming -> BrokenPipe/Cancelled storm. + +This is an INVARIANT test, not a change-detector: it asserts the resolvers +resolve from the cache-only path and never reach the live-rebuild seam +``api.config._invoke_models_rebuild`` — regardless of whether the session +carries a model_provider. +""" + +import ast +import inspect + +import pytest + +import api.config as cfg +import api.routes as routes + + +class _FakeSession: + """Minimal stand-in for a Session row as seen by the display resolvers.""" + + def __init__(self, model, model_provider): + self.model = model + self.model_provider = model_provider + + +@pytest.fixture +def cold_models_cache(monkeypatch): + """Force a cold in-memory + disk models cache without touching real state. + + Cold cache is what makes the regression observable: a warm cache short- + circuits before any rebuild decision, hiding the prefer_cache contract. + """ + monkeypatch.setattr(cfg, "_available_models_cache", None, raising=False) + monkeypatch.setattr(cfg, "_available_models_cache_ts", 0.0, raising=False) + monkeypatch.setattr( + cfg, "_available_models_cache_source_fingerprint", None, raising=False + ) + monkeypatch.setattr(cfg, "_cache_build_in_progress", False, raising=False) + # Never read/write the real on-disk cache during the test. + monkeypatch.setattr(cfg, "_load_models_cache_from_disk", lambda: None) + monkeypatch.setattr(cfg, "_save_models_cache_to_disk", lambda *_a, **_k: None) + monkeypatch.setattr(cfg, "_delete_models_cache_on_disk", lambda: None) + yield + + +@pytest.fixture +def rebuild_seam_tripwire(monkeypatch): + """Make the live provider-catalog rebuild seam fail loudly if reached. + + ``_invoke_models_rebuild`` is the documented indirection seam around the + cold, network-touching per-provider rebuild. The display resolvers must + never reach it (prefer_cache returns the network-free minimal catalog + *before* this seam). If a future edit drops ``prefer_cached_catalog=True``, + the resolver falls into the cold rebuild and trips this wire. + """ + calls = {"n": 0} + + def _boom(_builder): + calls["n"] += 1 + raise AssertionError( + "live provider-catalog rebuild ran on the hot GET /api/session " + "display path — prefer_cached_catalog regression" + ) + + monkeypatch.setattr(cfg, "_invoke_models_rebuild", _boom) + return calls + + +@pytest.mark.parametrize( + "model_provider", + [None, "", "anthropic"], + ids=["no-provider", "empty-provider", "with-provider"], +) +def test_session_display_resolvers_never_trigger_live_rebuild( + cold_models_cache, rebuild_seam_tripwire, model_provider +): + session = _FakeSession("claude-opus-4-7", model_provider) + + # Must not raise (the tripwire raises AssertionError if the live rebuild + # path is entered) and must return the persisted model verbatim. + model = routes._resolve_effective_session_model_for_display(session) + provider = routes._resolve_effective_session_model_provider_for_display(session) + + assert model == "claude-opus-4-7" + # provider is best-effort; the contract under test is "no live rebuild", + # not a specific provider string. It must at least be None or a str. + assert provider is None or isinstance(provider, str) + assert rebuild_seam_tripwire["n"] == 0 + + +def _has_prefer_cached_catalog_true_call(fn) -> bool: + tree = ast.parse(inspect.getsource(fn)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if not isinstance(node.func, ast.Name): + continue + if node.func.id != "_resolve_compatible_session_model_state": + continue + for keyword in node.keywords: + if keyword.arg == "prefer_cached_catalog" and isinstance( + keyword.value, ast.Constant + ): + return keyword.value.value is True + return False + + +def test_resolver_signature_passes_prefer_cached_catalog(): + """Static guard: both resolvers must opt into the cache-only catalog. + + A pure behavioural test can be satisfied by an unrelated short-circuit; + this pins the explicit contract at the call site so the intent survives + refactors. + """ + assert _has_prefer_cached_catalog_true_call( + routes._resolve_effective_session_model_for_display + ) + assert _has_prefer_cached_catalog_true_call( + routes._resolve_effective_session_model_provider_for_display + ) diff --git a/tests/test_start_session_turn_runtime_adapter.py b/tests/test_start_session_turn_runtime_adapter.py new file mode 100644 index 00000000..b0f0c76c --- /dev/null +++ b/tests/test_start_session_turn_runtime_adapter.py @@ -0,0 +1,130 @@ +"""start_session_turn honors runtime_adapter_enabled() via the shared +_start_run helper — Q-2979-A2 / Copilot discussion_r3305864087/r3305864173. + +Before this fix start_session_turn (Option Z drain-thread wakeup entrypoint) +called _start_chat_stream_for_session directly, bypassing the runtime-adapter +selection block that /api/chat/start (_handle_chat_start) already ran. As a +result a process-wakeup turn skipped the adapter that a human-typed turn +would have hit when ``HERMES_RUNTIME_ADAPTER=legacy-journal`` was set. + +The refactor factors a shared ``_start_run`` helper used by both entrypoints, +so flipping the env to ``legacy-journal`` now routes process-wakeup turns +through the same LegacyJournalRuntimeAdapter as the browser path. + +These tests exercise that contract directly without spinning up a real HTTP +server (precedent: tests/test_wakeup_defer_race.py — monkeypatch the heavy +deps, call the function under test, assert on adapter selection). +""" +from __future__ import annotations + +import types + +import pytest + + +@pytest.fixture +def _stub_routes(monkeypatch): + """Patch the heavy deps inside start_session_turn so the call collapses + to one observable: did it go through the runtime adapter or not.""" + from api import routes as routes_mod + + # 1. Fake session lookup — returns a minimal object with the attributes + # _start_run touches via the s.* attribute names. + s = types.SimpleNamespace( + session_id="sess-test", + model="opus", + model_provider="anthropic", + profile="developer-general", + workspace="/tmp/ws-test", + ) + monkeypatch.setattr(routes_mod, "get_session", lambda _sid: s) + + # 2. Workspace resolution — short-circuit to the persisted workspace. + monkeypatch.setattr( + routes_mod, + "_resolve_chat_workspace_with_recovery", + lambda _s, _req: "/tmp/ws-test", + ) + + # 3. Model resolution — pass through. + monkeypatch.setattr( + routes_mod, + "_resolve_compatible_session_model_state", + lambda model, provider, profile_provider=None, profile_default_model=None, prefer_cached_catalog=False: (model, provider, model), + ) + + # 4. Block the per-session live-view fan-out (it pokes a real registry). + monkeypatch.setattr( + routes_mod, + "_start_chat_stream_for_session", + lambda *a, **kw: {"_status": 200, "stream_id": "stream-direct", "session_id": "sess-test"}, + ) + + # 5. Silence the channel emit. + import api.background_process as bp_mod + + monkeypatch.setattr(bp_mod, "get_session_channel", lambda _sid: None) + + return routes_mod + + +def test_start_session_turn_uses_direct_path_by_default(_stub_routes, monkeypatch): + """With HERMES_WEBUI_RUNTIME_ADAPTER unset (legacy-direct default), the helper + must NOT go through the adapter — it falls through to the direct + _start_chat_stream_for_session call, same as before.""" + monkeypatch.delenv("HERMES_WEBUI_RUNTIME_ADAPTER", raising=False) + + calls = {"adapter": 0} + from api import runtime_adapter as ra_mod + + real_build = ra_mod.build_runtime_adapter + + def _track(*a, **kw): + calls["adapter"] += 1 + return real_build(*a, **kw) + + monkeypatch.setattr(ra_mod, "build_runtime_adapter", _track) + + resp = _stub_routes.start_session_turn("sess-test", "wakeup msg") + assert resp["_status"] == 200 + assert calls["adapter"] == 0, "default mode must not build a runtime adapter" + + +def test_start_session_turn_routes_through_adapter_when_enabled( + _stub_routes, monkeypatch +): + """With HERMES_WEBUI_RUNTIME_ADAPTER=legacy-journal, start_session_turn must + construct + invoke the LegacyJournalRuntimeAdapter — same path + _handle_chat_start exercises. This is the regression that Q-2979-A2 fixes: + before the _start_run refactor, this env flip had no effect on the + process-wakeup path.""" + monkeypatch.setenv("HERMES_WEBUI_RUNTIME_ADAPTER", "legacy-journal") + + from api import runtime_adapter as ra_mod + + invoked = {"adapter": 0, "start_run": 0} + + class _SpyAdapter: + def start_run(self, request): + invoked["start_run"] += 1 + assert request.session_id == "sess-test" + assert request.message == "wakeup msg" + assert request.source == "process_wakeup" + assert request.metadata == {"route": "start_session_turn"} + return ra_mod.RunStartResult( + run_id="run-test", + stream_id="stream-via-adapter", + session_id=request.session_id, + payload={"_status": 200, "stream_id": "stream-via-adapter"}, + ) + + def _fake_build(**kw): + invoked["adapter"] += 1 + return _SpyAdapter() + + monkeypatch.setattr(ra_mod, "build_runtime_adapter", _fake_build) + + resp = _stub_routes.start_session_turn("sess-test", "wakeup msg") + assert resp["_status"] == 200 + assert resp["stream_id"] == "stream-via-adapter" + assert invoked == {"adapter": 1, "start_run": 1} diff --git a/tests/test_wakeup_defer_race.py b/tests/test_wakeup_defer_race.py new file mode 100644 index 00000000..848833a2 --- /dev/null +++ b/tests/test_wakeup_defer_race.py @@ -0,0 +1,621 @@ +"""Defer-path wakeup race: a fast background task that completes WHILE a turn +is tearing down must still wake an autonomous agent. + +Root cause (proven by source, not speculation): + - api/background_process.py:_process_one defer branch — when a completion + arrives and _session_has_active_turn(session_id) is True (ACTIVE_RUNS has + a row), Option Z CANNOT start a turn (start_session_turn would 409). Before + this fix it only logged + left a bare PENDING_BG_TASK_COMPLETIONS session + flag; the wakeup_prompt was DISCARDED. + - The only consumer of that bare flag was the PR #2279 next-turn drain + (api/streaming._drain_webui_process_notifications, called at + streaming.py:3445 inside the turn pipeline). It reads completion_queue — + which the Option Z drain thread already emptied — and is gated by + BG_TASK_COMPLETE_EVENTS_SEEN / registry _completion_consumed (both set in + _process_one BEFORE the defer). So even a user turn could not recover it. + - For an AUTONOMOUS agent there is NO next user turn, so the deferred wakeup + was lost forever. A SLOW task (5s) completes AFTER teardown finished → + idle path → fires (Test A passed). A FAST task (2s) completes INSIDE the + teardown window (between "agent finished output" and ACTIVE_RUNS cleared) + → defer → lost (Test B failed). Exactly matches A-success / B-fail. + +The fix persists the prompt at defer time (DEFERRED_PROCESS_WAKEUPS) and a +turn-teardown idle-hook (drain_deferred_wakeups_for_session, invoked from +streaming.py right after unregister_active_run) redelivers it once the session +goes idle — symmetric with the idle branch. claim_deferred_wakeups pops +atomically, so delivery is exactly-once (no double-fire, no wakeup loop). + +These tests simulate the drain-thread + teardown sequence directly (no live +server needed — precedent t_9f0184cf), monkeypatching start_session_turn the +same way tests/test_session_channel_option_x.py does. +""" +from __future__ import annotations + +import queue +import threading +import types + + +# -------------------------------------------------------------------------- +# Fakes / fixtures (mirrors test_process_complete_ab_coexistence + +# test_session_channel_option_x patterns) +# -------------------------------------------------------------------------- + + +class _FakeProcessRegistry: + """Minimal stand-in for tools.process_registry.process_registry.""" + + def __init__(self): + self._lock = threading.Lock() + self._completion_consumed: set[str] = set() + self.completion_queue: queue.Queue = queue.Queue() + self._procs: dict[str, types.SimpleNamespace] = {} + + def register(self, process_id: str, session_key: str) -> None: + self._procs[process_id] = types.SimpleNamespace(session_key=session_key) + + def get(self, process_id: str): + return self._procs.get(process_id) + + def is_completion_consumed(self, process_id: str) -> bool: + with self._lock: + return process_id in self._completion_consumed + + +def _install_fake_registry(monkeypatch, fake): + # Rebase isolation fix: ONLY monkeypatch.setitem (tracked, restored on + # teardown). The prior sys.modules.setdefault("tools", ...) was an + # UNTRACKED mutation that permanently leaked a non-package fake `tools` + # into sys.modules when real `tools` wasn't imported yet, breaking later + # tests that do `from tools.process_registry import ...` (now in-session + # alongside the merged upstream #2279). + import sys + + mod = types.ModuleType("tools.process_registry") + mod.process_registry = fake + tools_mod = types.ModuleType("tools") + tools_mod.process_registry = mod # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "tools", tools_mod) + monkeypatch.setitem(sys.modules, "tools.process_registry", mod) + + +def _install_fake_start_session_turn(monkeypatch, *, status=200): + """Thin wrapper preserving the legacy local name; the body lives in + ``tests/_wakeup_helpers.py`` and is shared with + ``test_session_channel_option_x.py`` (Copilot PR #2971 r3305700944). + """ + from tests._wakeup_helpers import install_fake_start_session_turn as _impl + return _impl(monkeypatch, status=status) + + +def _wait_for_wakeup(holder, timeout=3.0): + """Thin wrapper preserving the legacy local name; the body lives in + ``tests/_wakeup_helpers.py`` and is shared with + ``test_session_channel_option_x.py`` (Copilot PR #2971 r3305700944). + """ + from tests._wakeup_helpers import wait_for_wakeup as _impl + return _impl(holder, timeout=timeout) + + +def _wait_for(predicate, timeout=3.0, interval=0.02): + """Poll *predicate* until it returns truthy or *timeout* elapses. + + Used when the assertion targets state mutated by the wakeup daemon thread + AFTER it calls start_session_turn (e.g. the 409 re-defer), which races the + ``holder['event']`` set inside the fake start_session_turn. + """ + import time + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(interval) + return predicate() + + +def _reset_cfg_state(): + from api import config as _cfg + + with _cfg.PROCESS_SESSION_INDEX_LOCK: + _cfg.PROCESS_SESSION_INDEX.clear() + _cfg.PENDING_BG_TASK_COMPLETIONS.clear() + _cfg.BG_TASK_COMPLETE_EVENTS_SEEN.clear() + with _cfg.DEFERRED_PROCESS_WAKEUPS_LOCK: + _cfg.DEFERRED_PROCESS_WAKEUPS.clear() + with _cfg.STREAMS_LOCK: + _cfg.STREAMS.clear() + if hasattr(_cfg, "ACTIVE_RUNS"): + with _cfg.ACTIVE_RUNS_LOCK: + _cfg.ACTIVE_RUNS.clear() + + +def _completion_evt(process_id: str, session_key: str) -> dict: + return { + "type": "completion", + "session_id": process_id, + "session_key": session_key, + "command": "sleep 2", + "exit_code": 0, + "output": "done", + } + + +# -------------------------------------------------------------------------- +# Test 1 — THE headline acceptance: completion during teardown still wakes +# (the autonomous-agent, no-next-user-turn case == the Test B scenario) +# -------------------------------------------------------------------------- + + +def test_completion_during_turn_teardown_still_wakes(monkeypatch): + """Session has an ACTIVE_RUN → a process completes → _process_one defers + (marker persisted, NO immediate turn). Then the turn tears down + (unregister_active_run) and the teardown idle-hook fires the deferred + wakeup exactly once. This is the Test B (sleep 2) scenario. + """ + from api import background_process as bp, config as cfg + + fake = _FakeProcessRegistry() + fake.register("proc-fast-1", "sess-teardown") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + holder = _install_fake_start_session_turn(monkeypatch) + + sid = "sess-teardown" + stream_id = "stream-teardown-1" + bp.register_process_session(sid, sid) + try: + # A turn is active (mid-teardown window: agent finished output but + # ACTIVE_RUNS not yet cleared). + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + assert bp._session_has_active_turn(sid) is True + + # Fast bg task completes INSIDE the teardown window → defer. + bp._process_one(_completion_evt("proc-fast-1", sid)) + + # Deferred, NOT fired: no turn started, prompt persisted. + assert holder["event"].wait(timeout=0.8) is False + assert holder["calls"] == [] + assert sid in cfg.DEFERRED_PROCESS_WAKEUPS + assert cfg.DEFERRED_PROCESS_WAKEUPS[sid][0]["process_id"] == "proc-fast-1" + + # Turn teardown: unregister_active_run clears the ACTIVE_RUNS row + # (this is exactly what streaming.py does under ACTIVE_RUNS_LOCK), + # then the teardown idle-hook runs. + cfg.unregister_active_run(stream_id) + assert bp._session_has_active_turn(sid) is False + started = bp.drain_deferred_wakeups_for_session(sid) + assert started == 1 + + assert _wait_for_wakeup(holder), ( + "deferred wakeup was NOT redelivered at turn teardown — the " + "autonomous-agent fast-bg-task case is still broken" + ) + assert len(holder["calls"]) == 1 + call = holder["calls"][0] + assert call["session_id"] == sid + assert call["source"] == "process_wakeup" + assert call["message"].startswith("[IMPORTANT: Background process") + # Claimed → nothing left to re-deliver. + assert sid not in cfg.DEFERRED_PROCESS_WAKEUPS + finally: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS.pop(stream_id, None) + bp.unregister_process_session(sid) + _reset_cfg_state() + + +# -------------------------------------------------------------------------- +# Test 2 — idle path unchanged: fires once, no regression, the new teardown +# hook does not double-fire it (the Test A / sleep 5 path) +# -------------------------------------------------------------------------- + + +def test_idle_completion_still_fires_once(monkeypatch): + """No ACTIVE_RUN → _process_one fires the server-side wakeup immediately + (Option Z idle branch, the Test A path). Nothing is deferred, so the new + turn-teardown hook is a pure no-op — total deliveries stays exactly 1. + """ + from api import background_process as bp, config as cfg + + fake = _FakeProcessRegistry() + fake.register("proc-idle-1", "sess-idle") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + holder = _install_fake_start_session_turn(monkeypatch) + + sid = "sess-idle" + bp.register_process_session(sid, sid) + try: + assert bp._session_has_active_turn(sid) is False + + bp._process_one(_completion_evt("proc-idle-1", sid)) + assert _wait_for_wakeup(holder), "idle path regressed — wakeup not fired" + assert len(holder["calls"]) == 1 + # Idle branch did NOT persist anything. + assert sid not in cfg.DEFERRED_PROCESS_WAKEUPS + + # The wakeup turn itself ends and tears down → its teardown re-runs + # the idle-hook. It must find nothing and NOT double-fire. + started = bp.drain_deferred_wakeups_for_session(sid) + assert started == 0 + assert len(holder["calls"]) == 1, ( + "the teardown hook double-fired an idle-path wakeup" + ) + finally: + bp.unregister_process_session(sid) + _reset_cfg_state() + + +# -------------------------------------------------------------------------- +# Test 3 — idempotent with the PR #2279 next-turn drain: a user turn that +# DOES come must not also deliver (shared SEEN / _completion_consumed gate) +# -------------------------------------------------------------------------- + + +def test_next_user_turn_drain_and_teardown_hook_dont_double_fire(monkeypatch): + """If a user turn DOES come, the next-turn drain + (_drain_webui_process_notifications) must NOT also deliver the deferred + completion: _process_one set BG_TASK_COMPLETE_EVENTS_SEEN AND the registry + _completion_consumed marker BEFORE the defer, and it consumed the + completion_queue event, so the next-turn drain has nothing to fire. The + teardown idle-hook then delivers it exactly once. Total deliveries == 1. + """ + from api import background_process as bp, config as cfg + from api import streaming as st + + fake = _FakeProcessRegistry() + fake.register("proc-shared-1", "sess-shared") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + holder = _install_fake_start_session_turn(monkeypatch) + + sid = "sess-shared" + stream_id = "stream-shared-1" + bp.register_process_session(sid, sid) + try: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + + bp._process_one(_completion_evt("proc-shared-1", sid)) + # Shared dedupe contract: _process_one marked it seen + registry- + # consumed before deferring. + assert "proc-shared-1" in cfg.BG_TASK_COMPLETE_EVENTS_SEEN[sid] + assert fake.is_completion_consumed("proc-shared-1") + assert sid in cfg.DEFERRED_PROCESS_WAKEUPS + + # A user turn comes: the next-turn drain runs. Even if a duplicate + # event were re-queued (kill_process race), the SEEN + consumed gate + # makes it a no-op — it must NOT deliver the deferred wakeup. + fake.completion_queue.put(_completion_evt("proc-shared-1", sid)) + notifications = st._drain_webui_process_notifications(sid) + assert notifications == [], ( + "next-turn drain double-delivered a completion the defer path owns" + ) + + # That user turn ends → its teardown fires the deferred wakeup ONCE. + cfg.unregister_active_run(stream_id) + started = bp.drain_deferred_wakeups_for_session(sid) + assert started == 1 + assert _wait_for_wakeup(holder) + assert len(holder["calls"]) == 1, ( + "deferred wakeup delivered more than once across next-turn drain " + "+ teardown hook" + ) + finally: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS.pop(stream_id, None) + bp.unregister_process_session(sid) + _reset_cfg_state() + + +# -------------------------------------------------------------------------- +# Test 4 — no wakeup loop: the wakeup turn's own teardown does not re-trigger +# a wakeup for the same process_id +# -------------------------------------------------------------------------- + + +def test_no_wakeup_loop(monkeypatch): + """The wakeup turn started by the teardown hook itself ends and tears + down → its teardown re-runs drain_deferred_wakeups_for_session. The atomic + claim (DEFERRED_PROCESS_WAKEUPS.pop) already removed the entry, so the + second drain finds nothing → no infinite wakeup loop. + """ + from api import background_process as bp, config as cfg + + fake = _FakeProcessRegistry() + fake.register("proc-loop-1", "sess-loop") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + holder = _install_fake_start_session_turn(monkeypatch) + + sid = "sess-loop" + stream_id = "stream-loop-1" + bp.register_process_session(sid, sid) + try: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + bp._process_one(_completion_evt("proc-loop-1", sid)) + assert sid in cfg.DEFERRED_PROCESS_WAKEUPS + + # First teardown: claims + fires once. + cfg.unregister_active_run(stream_id) + assert bp.drain_deferred_wakeups_for_session(sid) == 1 + assert _wait_for_wakeup(holder) + assert len(holder["calls"]) == 1 + assert sid not in cfg.DEFERRED_PROCESS_WAKEUPS + + # The wakeup turn itself runs and tears down → second drain. It must + # find NOTHING (already claimed) and start NO further turn. + for _ in range(3): + assert bp.drain_deferred_wakeups_for_session(sid) == 0 + assert len(holder["calls"]) == 1, ( + "wakeup loop: the wakeup turn's own teardown re-fired the same " + "process_id" + ) + finally: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS.pop(stream_id, None) + bp.unregister_process_session(sid) + _reset_cfg_state() + + +# -------------------------------------------------------------------------- +# Test 5 — multi-stream / cancel-reconnect guard: only fire when the session +# is TRULY idle (the just-ended stream was the last ACTIVE_RUN for the sid) +# -------------------------------------------------------------------------- + + +def test_multistream_guard_only_fires_when_truly_idle(monkeypatch): + """A cancel/reconnect leaves a SECOND active stream for the same session. + When the first stream tears down the session is NOT idle → the deferred + marker must be left intact (no fire). Only when the last active stream + tears down does the hook claim + fire, exactly once. + """ + from api import background_process as bp, config as cfg + + fake = _FakeProcessRegistry() + fake.register("proc-multi-1", "sess-multi") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + holder = _install_fake_start_session_turn(monkeypatch) + + sid = "sess-multi" + stream_a = "stream-multi-a" + stream_b = "stream-multi-b" + bp.register_process_session(sid, sid) + try: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS[stream_a] = {"session_id": sid} + cfg.ACTIVE_RUNS[stream_b] = {"session_id": sid} + bp._process_one(_completion_evt("proc-multi-1", sid)) + assert sid in cfg.DEFERRED_PROCESS_WAKEUPS + + # First stream tears down — second is still active → NOT idle. + cfg.unregister_active_run(stream_a) + assert bp._session_has_active_turn(sid) is True + assert bp.drain_deferred_wakeups_for_session(sid) == 0 + assert holder["calls"] == [] + # Marker retained for the later teardown. + assert sid in cfg.DEFERRED_PROCESS_WAKEUPS + + # Last stream tears down — now truly idle → fire exactly once. + cfg.unregister_active_run(stream_b) + assert bp._session_has_active_turn(sid) is False + assert bp.drain_deferred_wakeups_for_session(sid) == 1 + assert _wait_for_wakeup(holder) + assert len(holder["calls"]) == 1 + assert sid not in cfg.DEFERRED_PROCESS_WAKEUPS + finally: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS.pop(stream_a, None) + cfg.ACTIVE_RUNS.pop(stream_b, None) + bp.unregister_process_session(sid) + _reset_cfg_state() + + +# -------------------------------------------------------------------------- +# Test 6 — 409 during the teardown-hook path re-queues the wakeup (Greptile +# PR #2971 r3371737184). The teardown hook ATOMICALLY CLAIMS the deferred +# entry and DISCARDS the PENDING marker BEFORE spawning the wakeup thread. So +# if start_session_turn then 409s (a human /api/chat/start raced in between), +# both the marker and the claimed prompt are gone — the wakeup would be lost +# forever unless the 409 path re-queues it. This test pins that re-queue. +# -------------------------------------------------------------------------- + + +def test_teardown_409_requeues_wakeup_so_it_is_not_lost(monkeypatch): + """drain_deferred_wakeups_for_session claims the entry + discards the + PENDING marker, then the spawned wakeup turn 409s on a racing human turn. + The 409 branch must re-queue via record_deferred_wakeup so a later teardown + (or next-turn drain) redelivers it. Without the fix the wakeup is dropped. + """ + from api import background_process as bp, config as cfg + + fake = _FakeProcessRegistry() + fake.register("proc-409-1", "sess-409") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + # start_session_turn 409s — a human /api/chat/start won the per-session lock. + holder = _install_fake_start_session_turn(monkeypatch, status=409) + + sid = "sess-409" + stream_id = "stream-409-1" + bp.register_process_session(sid, sid) + try: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + bp._process_one(_completion_evt("proc-409-1", sid)) + assert sid in cfg.DEFERRED_PROCESS_WAKEUPS + + # Turn tears down → teardown hook claims the entry, discards the marker, + # and spawns the wakeup turn (which 409s). + cfg.unregister_active_run(stream_id) + assert bp.drain_deferred_wakeups_for_session(sid) == 1 + assert _wait_for_wakeup(holder) + assert len(holder["calls"]) == 1 + + # The 409 must have RE-QUEUED the entry rather than dropping it. Poll + # briefly: the re-queue runs on the same daemon thread right after the + # recorded call, so it may land a hair after the event fires. + import time as _t + for _ in range(50): + with cfg.DEFERRED_PROCESS_WAKEUPS_LOCK: + if cfg.DEFERRED_PROCESS_WAKEUPS.get(sid): + break + _t.sleep(0.02) + with cfg.DEFERRED_PROCESS_WAKEUPS_LOCK: + requeued = cfg.DEFERRED_PROCESS_WAKEUPS.get(sid) or [] + assert requeued, ( + "409 in the teardown-hook path DROPPED the wakeup — it must " + "re-queue via record_deferred_wakeup so it is not lost" + ) + # Re-queue is idempotent per process_id: exactly one entry, same id. + assert len(requeued) == 1 + assert requeued[0].get("process_id") == "proc-409-1" + finally: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS.pop(stream_id, None) + bp.unregister_process_session(sid) + _reset_cfg_state() + + +# -------------------------------------------------------------------------- +# Test 7 — Greptile P1: multiple deferred wakeups in one teardown. Only the +# FIRST starts a turn (the rest would 409 racing the per-session agent lock); +# entries 2..N must be re-deferred, not lost, and drain on later teardowns. +# -------------------------------------------------------------------------- + + +def test_multiple_deferred_wakeups_each_survive_across_teardowns(monkeypatch): + """N>=2 bg tasks complete during one active turn → all deferred. A single + teardown drain must NOT fire N racing daemon threads (only one wins the + agent lock; the losers 409 and, since the entries were already popped, + their prompts would be lost forever). The fix starts exactly one wakeup + and re-defers the remainder, so each subsequent teardown delivers the + next — every prompt is eventually delivered exactly once. + """ + from api import background_process as bp, config as cfg + + fake = _FakeProcessRegistry() + for pid in ("proc-A", "proc-B", "proc-C"): + fake.register(pid, "sess-multi-defer") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + holder = _install_fake_start_session_turn(monkeypatch) + + sid = "sess-multi-defer" + stream_id = "stream-multi-defer" + bp.register_process_session(sid, sid) + try: + # A turn is active; three fast bg tasks all complete inside it → defer. + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS[stream_id] = {"session_id": sid} + for pid in ("proc-A", "proc-B", "proc-C"): + bp._process_one(_completion_evt(pid, sid)) + assert len(cfg.DEFERRED_PROCESS_WAKEUPS.get(sid, [])) == 3 + + # Turn teardown → idle. Drain reports exactly ONE wakeup started, and + # the other two are re-deferred (not popped-and-lost). + cfg.unregister_active_run(stream_id) + assert bp._session_has_active_turn(sid) is False + assert bp.drain_deferred_wakeups_for_session(sid) == 1 + assert _wait_for_wakeup(holder) + assert len(holder["calls"]) == 1 + assert len(cfg.DEFERRED_PROCESS_WAKEUPS.get(sid, [])) == 2 + + # Second teardown delivers the next, third delivers the last. + holder["event"].clear() + assert bp.drain_deferred_wakeups_for_session(sid) == 1 + assert _wait_for_wakeup(holder) + assert len(holder["calls"]) == 2 + assert len(cfg.DEFERRED_PROCESS_WAKEUPS.get(sid, [])) == 1 + + holder["event"].clear() + assert bp.drain_deferred_wakeups_for_session(sid) == 1 + assert _wait_for_wakeup(holder) + assert len(holder["calls"]) == 3 + assert sid not in cfg.DEFERRED_PROCESS_WAKEUPS + + # All three distinct prompts delivered exactly once, no duplicates. + delivered = {c["message"] for c in holder["calls"]} + assert len(delivered) == 3 + # Nothing left → a final drain is a no-op (no wakeup loop). + assert bp.drain_deferred_wakeups_for_session(sid) == 0 + finally: + with cfg.ACTIVE_RUNS_LOCK: + cfg.ACTIVE_RUNS.pop(stream_id, None) + bp.unregister_process_session(sid) + _reset_cfg_state() + + +# -------------------------------------------------------------------------- +# Test 7 — Greptile P1 (idle-branch sibling of the 409/teardown F1 fix): +# the IDLE-path wakeup must pass process_id, so when its daemon thread loses +# the per-session lock race (409) the re-defer carries the real process_id and +# the record_deferred_wakeup dedup guard stays live — a second 409 race cannot +# accumulate a duplicate deferred entry that would deliver the wakeup twice. +# -------------------------------------------------------------------------- + + +def test_idle_path_409_redefer_carries_process_id_and_dedups(monkeypatch): + """Idle completion → _process_one idle branch starts the server-side + wakeup. The fake start_session_turn returns 409 (raced an active turn / + sibling wakeup), so the daemon re-defers via record_deferred_wakeup. + + BEFORE the fix the idle branch called _start_server_side_wakeup_turn + WITHOUT process_id, so the re-defer recorded process_id="" — falsy, so the + `if process_id and any(...)` dedup guard was skipped and a second 409 race + appended a duplicate identical entry, ultimately double-delivering the + wakeup. AFTER the fix the re-defer carries the real process_id and the + guard dedups the second race to a single entry. + """ + from api import background_process as bp, config as cfg + + fake = _FakeProcessRegistry() + fake.register("proc-idle-409", "sess-idle-409") + _install_fake_registry(monkeypatch, fake) + _reset_cfg_state() + # 409 == lost the per-session agent lock race; triggers the re-defer path. + holder = _install_fake_start_session_turn(monkeypatch, status=409) + + sid = "sess-idle-409" + bp.register_process_session(sid, sid) + try: + # Session is idle → _process_one takes the idle branch (the line-838 + # call site this card fixes), which now plumbs process_id through. + assert bp._session_has_active_turn(sid) is False + bp._process_one(_completion_evt("proc-idle-409", sid)) + + # The wakeup daemon ran, hit 409, and re-deferred. Poll for the + # re-defer (it races the holder event set inside the fake). + assert _wait_for_wakeup(holder), "idle-branch wakeup never attempted" + assert _wait_for( + lambda: bool(cfg.DEFERRED_PROCESS_WAKEUPS.get(sid)) + ), "409 on the idle path did not re-defer the wakeup — it was lost" + + entries = cfg.DEFERRED_PROCESS_WAKEUPS[sid] + assert len(entries) == 1 + # THE regression assertion: the re-defer carries the real process_id, + # not "" (which is what the missing-process_id idle call produced). + assert entries[0]["process_id"] == "proc-idle-409", ( + "idle-path re-defer dropped process_id — the dedup guard in " + "record_deferred_wakeup is now bypassed and duplicates can " + "accumulate (the exact Greptile P1)" + ) + + # Now prove the dedup guard is actually live on this path: a second + # 409 race for the SAME process_id (e.g. the next teardown drain) must + # NOT append a duplicate. With process_id="" (pre-fix) this would grow + # to 2 entries and double-deliver. + bp.record_deferred_wakeup( + sid, "proc-idle-409", entries[0]["wakeup_prompt"] + ) + assert len(cfg.DEFERRED_PROCESS_WAKEUPS[sid]) == 1, ( + "duplicate deferred entry accumulated — dedup guard did not fire " + "because the idle-path re-defer lost its process_id" + ) + finally: + bp.unregister_process_session(sid) + _reset_cfg_state() diff --git a/tests/test_wakeup_model_resolve_hang.py b/tests/test_wakeup_model_resolve_hang.py new file mode 100644 index 00000000..0263d54d --- /dev/null +++ b/tests/test_wakeup_model_resolve_hang.py @@ -0,0 +1,283 @@ +"""Regression tests — wakeup-turn model-resolve hang (t_46fadfbc). + +Proven root cause (from a live thread-stack capture of a hung wakeup +turn, "Slow WebUI request still running"): + + _handle_chat_start → _resolve_compatible_session_model_state + → get_available_models → _build_available_models_uncached + → _read_live_provider_model_ids → get_copilot_api_token + → exchange_copilot_token → urllib HTTPS → BLOCKED + +A server-initiated Option-Z wakeup turn (drain thread, idle session, no +browser) reached chat/start with a COLD provider catalog and triggered a +LIVE per-provider rebuild whose Copilot token-exchange HTTPS call hung the +wakeup turn forever on this WSL/corp network — NOT a race. + +Two fixes, two tests: + +1. test_wakeup_turn_uses_persisted_model_no_live_probe + start_session_turn(source="process_wakeup") resolves the model from the + persisted session record via the cache-only path; the live provider + rebuild (and the Copilot exchange) is never invoked even when the catalog + is cold. The turn still starts with the persisted model. + +2. test_chat_start_survives_slow_provider_probe + Defense-in-depth: an unbounded/hanging provider probe cannot stall a + foreground get_available_models() past the wall-clock budget — it falls + back to a usable model list and lets the rebuild finish out-of-band. +""" + +from __future__ import annotations + +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +# --------------------------------------------------------------------------- +# Fix 1 — wakeup resolves persisted model with NO live provider probe +# --------------------------------------------------------------------------- + +def test_wakeup_turn_uses_persisted_model_no_live_probe(monkeypatch): + """Cold catalog + a session with a persisted model/provider: a + server-side wakeup must start the turn using the PERSISTED model and must + NOT call the live per-provider rebuild (which is what does the blocking + Copilot token-exchange HTTPS call in the proven thread-stack). + """ + from api import config as cfg + import api.routes as routes + + sid = "sess-wakeup-persisted" + persisted_model = "anthropic/claude-sonnet-4" + fake_stream_id = "stream-wakeup-persisted-1" + + # HARD ASSERT: the live per-provider probe must never run on the wakeup + # path. _read_live_provider_model_ids is the function the thread-stack + # shows calling get_copilot_api_token → exchange_copilot_token. + def _boom(*_a, **_k): + raise AssertionError( + "live provider rebuild ran on a wakeup turn — " + "_read_live_provider_model_ids must NOT be called (it does the " + "blocking Copilot token-exchange HTTPS call)" + ) + + monkeypatch.setattr(cfg, "_read_live_provider_model_ids", _boom, raising=True) + # Cold cache: force the cache-miss branch that used to trigger the live + # rebuild. The autouse conftest fixture already invalidates, but be + # explicit so this test documents the precondition. + cfg.invalidate_models_cache() + + captured: dict = {} + + def _fake_start_chat_stream_for_session(s, **kwargs): + captured["model"] = kwargs.get("model") + captured["model_provider"] = kwargs.get("model_provider") + return {"stream_id": fake_stream_id, "session_id": s.session_id, "_status": 200} + + class _FakeSession: + session_id = sid + model = persisted_model + model_provider = "anthropic" + + monkeypatch.setattr( + routes, "_start_chat_stream_for_session", + _fake_start_chat_stream_for_session, raising=True, + ) + monkeypatch.setattr(routes, "get_session", lambda _sid: _FakeSession(), raising=True) + monkeypatch.setattr( + routes, "_resolve_chat_workspace_with_recovery", + lambda s, w: "/tmp/ws", raising=True, + ) + + t0 = time.monotonic() + resp = routes.start_session_turn( + sid, "[IMPORTANT: Background process done]", source="process_wakeup" + ) + elapsed = time.monotonic() - t0 + + assert resp.get("stream_id") == fake_stream_id, "wakeup turn did not start" + # The persisted model survives to the turn — never dropped, never blocked. + assert captured["model"] == persisted_model, ( + f"wakeup turn used {captured.get('model')!r}, expected the persisted " + f"session model {persisted_model!r}" + ) + # Fast: no network. Cache-only resolution is sub-second; allow generous + # slack for slow CI but well under the 10s Copilot timeout that was the + # original symptom. + assert elapsed < 5.0, ( + f"wakeup turn took {elapsed:.2f}s — a live provider probe likely ran" + ) + + +def test_wakeup_resolve_passes_prefer_cached_catalog(monkeypatch): + """White-box: start_session_turn must route model resolution through the + cache-only path (prefer_cached_catalog=True). Pins the wiring so a future + refactor can't silently reintroduce the live-probe hang. + """ + import api.routes as routes + + sid = "sess-wakeup-wiring" + seen: dict = {} + + real = routes._resolve_compatible_session_model_state + + def _spy(model_id, model_provider=None, *, profile_provider=None, + profile_default_model=None, prefer_cached_catalog=False): + seen["prefer_cached_catalog"] = prefer_cached_catalog + # The wakeup path now also threads the session's profile model defaults + # through (greptile fix) so a brand-new session with an empty model + # falls back to the profile default, not the global DEFAULT_MODEL. + seen["profile_provider"] = profile_provider + seen["profile_default_model"] = profile_default_model + return ("m", None, False) + + monkeypatch.setattr( + routes, "_resolve_compatible_session_model_state", _spy, raising=True + ) + monkeypatch.setattr( + routes, "_start_chat_stream_for_session", + lambda s, **k: {"stream_id": "x", "session_id": s.session_id, "_status": 200}, + raising=True, + ) + + class _FakeSession: + session_id = sid + model = "m" + model_provider = None + + monkeypatch.setattr(routes, "get_session", lambda _s: _FakeSession(), raising=True) + monkeypatch.setattr( + routes, "_resolve_chat_workspace_with_recovery", + lambda s, w: "/tmp/ws", raising=True, + ) + + routes.start_session_turn(sid, "[IMPORTANT: x]", source="process_wakeup") + assert seen.get("prefer_cached_catalog") is True, ( + "start_session_turn must resolve the model with " + "prefer_cached_catalog=True so a wakeup never triggers a live probe" + ) + assert real is not None # the real function still exists (not deleted) + + +# --------------------------------------------------------------------------- +# Fix 2 — bounded rebuild: a slow probe cannot stall chat/start forever +# --------------------------------------------------------------------------- + +def test_chat_start_survives_slow_provider_probe(monkeypatch): + """Simulate a hanging provider probe on the NORMAL (prefer_cache=False) + cold path: get_available_models() must return within the wall-clock + budget with a usable fallback instead of blocking on the network. + """ + from api import config as cfg + + cfg.invalidate_models_cache() + monkeypatch.setattr(cfg, "_LIVE_REBUILD_BUDGET_SECONDS", 0.4, raising=True) + + started = {"n": 0} + + def _slow_rebuild(_builder): + started["n"] += 1 + # >> budget — models the hung Copilot HTTPS call. 0.8s is 2× the + # monkeypatched 0.4s budget, which is the smallest gap that still + # robustly proves the contract while keeping suite wall-clock low + # (was 3.0s; suite-latency cleanup per Copilot review). + time.sleep(0.8) + return { + "active_provider": "anthropic", + "default_model": "anthropic/claude-sonnet-4", + "configured_model_badges": {}, + "groups": [], + } + + # Replace the rebuild seam so no real per-provider network call happens + # but the foreground still has to wait for (a stand-in for) it. + monkeypatch.setattr(cfg, "_invoke_models_rebuild", _slow_rebuild, raising=True) + # Ensure no disk cache short-circuits the cold path. + monkeypatch.setattr(cfg, "_load_models_cache_from_disk", lambda: None, raising=True) + + t0 = time.monotonic() + result = cfg.get_available_models() + elapsed = time.monotonic() - t0 + + assert started["n"] == 1, "the rebuild worker should have been started" + assert elapsed < 2.0, ( + f"get_available_models() blocked {elapsed:.2f}s on a hung probe — " + f"the {cfg._LIVE_REBUILD_BUDGET_SECONDS}s budget did not bound it" + ) + # The fallback must be a structurally valid, usable catalog. + assert isinstance(result, dict) + for k in ("active_provider", "default_model", "configured_model_badges", "groups"): + assert k in result, f"fallback catalog missing {k!r}" + assert isinstance(result["groups"], list) + + +def test_minimal_static_catalog_is_network_free(monkeypatch): + """The fallback catalog builder must never reach the live provider probe. + """ + from api import config as cfg + + monkeypatch.setattr( + cfg, "_read_live_provider_model_ids", + lambda *_a, **_k: (_ for _ in ()).throw( + AssertionError("minimal catalog must not probe providers") + ), + raising=True, + ) + out = cfg._minimal_static_models_catalog() + assert set(out) >= { + "active_provider", "default_model", "configured_model_badges", "groups" + } + assert isinstance(out["groups"], list) + + +def test_prefer_cache_kw_exists_and_skips_live_rebuild(monkeypatch): + """get_available_models(prefer_cache=True) must resolve without invoking + the rebuild seam at all when the cache is cold. + """ + from api import config as cfg + + cfg.invalidate_models_cache() + monkeypatch.setattr(cfg, "_load_models_cache_from_disk", lambda: None, raising=True) + + def _must_not_run(_b): + raise AssertionError( + "prefer_cache=True triggered the live rebuild seam — it must " + "serve cache/minimal-static only" + ) + + monkeypatch.setattr(cfg, "_invoke_models_rebuild", _must_not_run, raising=True) + + result = cfg.get_available_models(prefer_cache=True) + assert isinstance(result, dict) + assert "default_model" in result and "groups" in result + + +# --------------------------------------------------------------------------- +# Source-grep wiring guards +# --------------------------------------------------------------------------- + +def test_get_available_models_has_prefer_cache_param(): + import inspect + + from api import config as cfg + + sig = inspect.signature(cfg.get_available_models) + assert "prefer_cache" in sig.parameters + param = sig.parameters["prefer_cache"] + assert param.kind is inspect.Parameter.KEYWORD_ONLY + assert param.default is False + assert "_LIVE_REBUILD_BUDGET_SECONDS" in cfg.__dict__ + assert "_minimal_static_models_catalog" in cfg.__dict__ + + +def test_start_session_turn_uses_cached_catalog(): + src = (REPO_ROOT / "api" / "routes.py").read_text(encoding="utf-8") + # The wakeup entrypoint must resolve with the cache-only flag. + i = src.find("def start_session_turn(") + assert i != -1 + j = src.find("def _handle_process_complete_ack", i) + body = src[i:j] + assert "prefer_cached_catalog=True" in body, ( + "start_session_turn must pass prefer_cached_catalog=True" + ) diff --git a/tests/test_xsession_wakeup_misroute.py b/tests/test_xsession_wakeup_misroute.py new file mode 100644 index 00000000..27b5a23e --- /dev/null +++ b/tests/test_xsession_wakeup_misroute.py @@ -0,0 +1,259 @@ +"""Regression: cross-session notify_on_complete wakeup misroute (Option 1 + Option 3). + +ROOT CAUSE (RCA t_f62ff1e8, verified line-by-line): + WebUI's per-turn session identity was bound ONLY to the process-global + ``os.environ['HERMES_SESSION_KEY']`` (streaming.py turn-start), and the env + lock was released BEFORE the agent ran. WebUI NEVER called + ``gateway.session_context.set_session_vars`` so the ``_SESSION_KEY`` + contextvar stayed ``_UNSET`` and ``tools.approval.get_current_session_key`` + fell back to the racy process-global env. Two concurrent WebUI turns + therefore raced on one slot: session A's ``terminal(notify_on_complete=True)`` + spawn could capture session B's id, and at completion the server-side wakeup + turn started for the WRONG session. + +This module proves BOTH fix layers with REAL modules (no mocks of the code +under test), so each test is RED before the fix and GREEN after — never a +tautology: + + Option 1 (root fix, streaming.py) — ``_bind_turn_session_identity`` binds the + REAL ``_SESSION_KEY`` contextvar for the turn's worker thread/context and + clears it on exit. Under a simulated env race across two concurrent turns, + the REAL ``get_current_session_key`` must return each turn's OWN id. + + Option 3 (defense-in-depth, background_process.py) — + ``_resolve_wakeup_target`` cross-checks the (possibly env-contaminated) + session_key-resolved session against the spawn-time owner persisted in the + process registry's ``ProcessSession.spawn_session_id`` (an env-immune + field). On a positively-detected mismatch it re-routes to the true owner + instead of waking the wrong session. + +Precedent for this no-live-server style: tests/test_wakeup_defer_race.py, +tests/test_session_channel_option_x.py. +""" +from __future__ import annotations + +import importlib +import threading + +import pytest + + +# --------------------------------------------------------------------------- +# Option 1 — contextvar binding makes per-turn session identity race-immune +# --------------------------------------------------------------------------- + +def test_streaming_exposes_turn_session_identity_binder(): + """streaming.py must expose the helper that binds the REAL _SESSION_KEY + contextvar for the agent worker thread (root fix, not env-only).""" + streaming = importlib.import_module("api.streaming") + assert hasattr(streaming, "_bind_turn_session_identity"), ( + "Option 1 missing: streaming.py must expose _bind_turn_session_identity " + "to bind gateway.session_context._SESSION_KEY for the turn" + ) + + +def test_concurrent_turns_capture_their_own_session_under_env_race(): + """THE invariant. Two concurrent WebUI turns, A and B. Each binds its own + session identity via the REAL streaming helper, then — while the OTHER + turn has just stamped the shared process-global env (the documented race: + lock released, agent still running) — performs the EXACT capture call a + notify_on_complete spawn makes: ``tools.approval.get_current_session_key``. + + Pre-fix: WebUI never set the contextvar, so the helper does not exist + (ImportError above) / the contextvar stays _UNSET and the capture falls + back to the env → A captures B's id → MISROUTE (RED). + + Post-fix: the helper binds _SESSION_KEY in each worker context, so the + contextvar (task-local) wins over the racy env → each turn captures its + OWN id (GREEN). + """ + import os + + streaming = importlib.import_module("api.streaming") + pytest.importorskip("tools.approval", reason="hermes-agent not installed") + pytest.importorskip("gateway.session_context") + from tools.approval import get_current_session_key + from gateway import session_context as sc + + bind = getattr(streaming, "_bind_turn_session_identity", None) + if bind is None: + pytest.fail("Option 1 not implemented: _bind_turn_session_identity missing") + + # The two turn() threads below stamp os.environ["HERMES_SESSION_KEY"] + # without owning it. Save/restore the prior value (sentinel for "was + # unset") so this test does not leak state into sibling tests when + # collection order interleaves it with another consumer. + _prev_env_sentinel = object() + _prev_env = os.environ.get("HERMES_SESSION_KEY", _prev_env_sentinel) + + SESS_A = "20260518_161627_60b5f4" # board claude-code-import + SESS_B = "47ec28f66dff" # board mcp-optimize + + captured: dict[str, str] = {} + barrier = threading.Barrier(2) + # Force a deterministic interleave: A captures only AFTER B has stamped + # the process-global env (reproduces "lock released, agent still running, + # other turn stamps the global slot"). + b_stamped_env = threading.Event() + + def turn(my_sid: str, label: str) -> None: + # streaming.py turn-start still writes the process-global env as a + # fallback for non-contextvar consumers; the fix is that session-key + # ROUTING now binds the contextvar so it no longer races. + with bind(my_sid): + os.environ["HERMES_SESSION_KEY"] = my_sid + barrier.wait() + if label == "B": + # B stamps env last while A's "agent" is still mid-turn. + os.environ["HERMES_SESSION_KEY"] = my_sid + b_stamped_env.set() + else: + assert b_stamped_env.wait(timeout=5), "B never stamped env" + # The EXACT call terminal_tool.py makes for a bg spawn: + captured[label] = get_current_session_key(default="") + + try: + ta = threading.Thread(target=turn, args=(SESS_A, "A")) + tb = threading.Thread(target=turn, args=(SESS_B, "B")) + ta.start() + tb.start() + ta.join(timeout=10) + tb.join(timeout=10) + + # Assert the worker threads actually terminated. If a thread deadlocks the + # join() above returns silently — surface that as a clear test failure + # instead of letting downstream asserts mask the hang or leak threads into + # the rest of the run. + assert not ta.is_alive(), "worker thread A did not terminate within join timeout" + assert not tb.is_alive(), "worker thread B did not terminate within join timeout" + + # Contextvar must be restored after the turn context exits (no thread-pool + # residue → no new race for a reused worker). + assert sc._SESSION_KEY.get() is sc._UNSET + + assert captured.get("A") == SESS_A, ( + f"MISROUTE: session A captured {captured.get('A')!r}, expected " + f"{SESS_A!r} — per-turn identity still races on process-global env" + ) + assert captured.get("B") == SESS_B, ( + f"MISROUTE: session B captured {captured.get('B')!r}, expected {SESS_B!r}" + ) + finally: + # Restore HERMES_SESSION_KEY to its pre-test value (or unset if it was + # never set), independent of which assertion above might have failed — + # see save/restore note at the top of the test. + if _prev_env is _prev_env_sentinel: + os.environ.pop("HERMES_SESSION_KEY", None) + else: + assert isinstance(_prev_env, str) + os.environ["HERMES_SESSION_KEY"] = _prev_env + + +def test_turn_identity_binder_restores_previous_value(): + """Restore uses contextvars reset-token semantics (the canonical idiom), + NOT a blanket clear_session_vars: it composes correctly under nesting and + restores _UNSET for the top-level turn so CLI/cron env-fallback compat is + preserved, and it must NOT touch the platform/chat_id/user session vars + (those keep their env fallback so the notify_on_complete watcher + registration that reads HERMES_SESSION_PLATFORM still works).""" + streaming = importlib.import_module("api.streaming") + pytest.importorskip("tools.approval", reason="hermes-agent not installed") + from tools.approval import get_current_session_key + from gateway import session_context as sc + + bind = streaming._bind_turn_session_identity + assert sc._SESSION_KEY.get() is sc._UNSET + # Platform var starts unset → env fallback path intact. + assert sc._SESSION_PLATFORM.get() is sc._UNSET + with bind("sid-outer"): + assert get_current_session_key(default="") == "sid-outer" + with bind("sid-inner"): + assert get_current_session_key(default="") == "sid-inner" + # Reset-token restores the OUTER value (composes under nesting), + # it does NOT clear to "". + assert get_current_session_key(default="") == "sid-outer" + # The binder must never disturb the other session vars. + assert sc._SESSION_PLATFORM.get() is sc._UNSET + # Full exit restores _UNSET → env fallback resumes (CLI/cron compat). + assert sc._SESSION_KEY.get() is sc._UNSET + assert sc._SESSION_PLATFORM.get() is sc._UNSET + + +# --------------------------------------------------------------------------- +# Option 3 — completion-time second-check against the env-immune spawn owner +# --------------------------------------------------------------------------- + +def test_background_process_exposes_wakeup_target_resolver(): + bp = importlib.import_module("api.background_process") + assert hasattr(bp, "_resolve_wakeup_target"), ( + "Option 3 missing: background_process.py must expose _resolve_wakeup_target " + "to cross-check the session_key-resolved target against the env-immune " + "spawn-time owner" + ) + + +def _fake_ps(session_key: str, spawn_session_id: str): + import types + + return types.SimpleNamespace( + session_key=session_key, spawn_session_id=spawn_session_id + ) + + +def test_resolve_wakeup_target_reroutes_on_positive_mismatch(monkeypatch): + """The RCA scenario: env race made the process's ``session_key`` resolve to + session B, but the env-immune ``spawn_session_id`` says session A truly + spawned it. ``_resolve_wakeup_target`` must detect the positive mismatch + and return A (the true owner), NOT B. + """ + bp = importlib.import_module("api.background_process") + + SESS_A = "20260518_161627_60b5f4" + SESS_B = "47ec28f66dff" + + # session_key (env-contaminated) → resolves to B; spawn owner is A. + ps = _fake_ps(session_key=SESS_B, spawn_session_id=SESS_A) + + resolved = bp._resolve_wakeup_target( + process_id="proc_f377e2f552cd", + session_key_resolved_sid=SESS_B, + proc_session=ps, + ) + assert resolved == SESS_A, ( + f"Option 3 failed to re-route: woke {resolved!r}, the env-immune spawn " + f"owner is {SESS_A!r} — this is the exact agent.log:6632 misroute" + ) + + +def test_resolve_wakeup_target_passthrough_when_consistent(): + """No mismatch (the normal case, and the post-Option 1 case): the resolver is + a pure pass-through and never suppresses a legitimate Option Z wakeup.""" + bp = importlib.import_module("api.background_process") + + sid = "session-normal" + ps = _fake_ps(session_key=sid, spawn_session_id=sid) + assert bp._resolve_wakeup_target( + process_id="proc_ok", + session_key_resolved_sid=sid, + proc_session=ps, + ) == sid + + +def test_resolve_wakeup_target_passthrough_when_owner_unknown(): + """If the spawn owner is indeterminate (no env-immune field, e.g. a + cron/CLI process sharing the registry, or a pre-Option 1 spawn), the resolver + must fall through to the session_key-resolved sid UNCHANGED — never + suppress a wakeup on uncertainty (Option Z must keep working).""" + bp = importlib.import_module("api.background_process") + + sid = "session-unknown-owner" + assert bp._resolve_wakeup_target( + process_id="proc_cron", + session_key_resolved_sid=sid, + proc_session=_fake_ps(session_key=sid, spawn_session_id=""), + ) == sid + assert bp._resolve_wakeup_target( + process_id="proc_cron", + session_key_resolved_sid=sid, + proc_session=None, + ) == sid