Release v0.51.329 — Release KS (#3814 O(n) index map + #3815 startup recovery skip) (#3840)

#3814: O(n) index-count map for /api/sessions. #3815: skip backup-less sidecars on startup recovery. Full suite 8275, Codex SAFE, Opus SHIP, CI 11/11. Co-authored-by: ai-ag2026 <ai-ag2026@users.noreply.github.com>

CHANGELOG.md

@@ -3,6 +3,12 @@
 
 ## [Unreleased]
 
+## [v0.51.329] — 2026-06-08 — Release KS (session-list + startup latency)
+
+### Fixed
+- **`/api/sessions` no longer does redundant `_index.json` parses per legacy sidecar row.** The stale-metadata refresh reuses the already-parsed index (O(n) instead of O(n²) for installs with many pre-`message_count` sidecars). (#3814, @ai-ag2026)
+- **Startup no longer reads every session's full JSON when there is nothing to recover.** Recovery now only reads sidecars that have a `.json.bak` backup; the reported `scanned` count is unchanged. (#3815, @ai-ag2026)
+
 ## [v0.51.328] — 2026-06-08 — Release KR (preserve full compaction summaries)
 
 ### Added

api/models.py

@@ -513,24 +513,41 @@ def _read_metadata_json_prefix(path, max_prefix_bytes=65536):
 
 def _lookup_index_message_count(session_id):
     """Return the indexed message count without loading the full session file."""
-    try:
-        entries = json.loads(SESSION_INDEX_FILE.read_text(encoding='utf-8'))
-    except Exception:
-        return None
+    return _index_message_count_map().get(str(session_id))
+
+
+def _index_message_count_map(entries=None) -> dict[str, int]:
+    """Return indexed message counts keyed by session id.
+
+    ``load_metadata_only()`` is called in loops for stale lineage/sidebar rows.
+    Reading and parsing ``_index.json`` once per row turns /api/sessions into an
+    accidental O(n²) poll for old sidecars that predate persisted
+    ``message_count``. Accepting already-loaded index rows lets callers reuse
+    the index they just parsed.
+    """
+    if entries is None:
+        try:
+            entries = json.loads(SESSION_INDEX_FILE.read_text(encoding='utf-8'))
+        except Exception:
+            return {}
     if not isinstance(entries, list):
-        return None
+        return {}
+    counts: dict[str, int] = {}
     for entry in entries:
-        if entry.get('session_id') != session_id:
+        if not isinstance(entry, dict):
+            continue
+        sid = str(entry.get('session_id') or '')
+        if not sid:
             continue
         count = entry.get('message_count')
-        if isinstance(count, int) and count >= 0:
-            return count
-        try:
-            count = int(count)
-        except (TypeError, ValueError):
-            return None
-        return count if count >= 0 else None
-    return None
+        if not isinstance(count, int):
+            try:
+                count = int(count)
+            except (TypeError, ValueError):
+                continue
+        if count >= 0:
+            counts[sid] = count
+    return counts
 
 
 def _parse_nonnegative_int(value):
@@ -791,7 +808,7 @@ class Session:
         return session
 
     @classmethod
-    def load_metadata_only(cls, sid):
+    def load_metadata_only(cls, sid, *, index_message_counts=None):
         """Load only the compact metadata fields, skipping the messages array.
 
         Session JSON files have metadata fields (session_id, title, model, etc.)
@@ -820,7 +837,10 @@ class Session:
             sidecar_message_count = _parse_nonnegative_int(parsed.get('message_count'))
             index_message_count = None
             if sidecar_message_count is None:
-                index_message_count = _lookup_index_message_count(sid)
+                if index_message_counts is not None:
+                    index_message_count = index_message_counts.get(str(sid))
+                else:
+                    index_message_count = _lookup_index_message_count(sid)
             # Modern sidecars carry an accurate message_count, so it is the
             # source of truth and we skip the per-row _index.json read in the
             # common case. The sidebar index is only a cache (it can lag behind
@@ -2728,7 +2748,11 @@ def _stale_snapshot_metadata_refresh_ids(sessions: list[dict]) -> set[str]:
     return refresh_ids
 
 
-def _refresh_index_rows_from_sidecar_metadata(sessions: list[dict]) -> list[dict]:
+def _refresh_index_rows_from_sidecar_metadata(
+    sessions: list[dict],
+    *,
+    index_message_counts: dict[str, int] | None = None,
+) -> list[dict]:
     """Overlay fuller sidecar metadata onto stale sidebar index rows.
 
     ``_index.json`` is a cache and can lag behind the canonical session sidecar
@@ -2749,7 +2773,10 @@ def _refresh_index_rows_from_sidecar_metadata(sessions: list[dict]) -> list[dict
         if not sid:
             out.append(session)
             continue
-        sidecar = Session.load_metadata_only(sid)
+        sidecar = Session.load_metadata_only(
+            sid,
+            index_message_counts=index_message_counts,
+        )
         if not sidecar:
             out.append(session)
             continue
@@ -3051,7 +3078,11 @@ def all_sessions(diag=None):
                         active_stream_ids=active_stream_ids,
                     )
             _diag_stage(diag, "all_sessions.refresh_sidecar_metadata")
-            refreshed_index_rows = _refresh_index_rows_from_sidecar_metadata(list(index_map.values()))
+            index_message_counts = _index_message_count_map(index)
+            refreshed_index_rows = _refresh_index_rows_from_sidecar_metadata(
+                list(index_map.values()),
+                index_message_counts=index_message_counts,
+            )
             index_map = {
                 row['session_id']: row
                 for row in refreshed_index_rows

api/session_recovery.py

@@ -588,18 +588,20 @@ def recover_all_sessions_on_startup(
     """
     if not session_dir.exists():
         return {"scanned": 0, "restored": 0, "orphaned_backups": 0, "details": []}
-    scanned = 0
     restored = 0
     details: list[dict] = []
     live_paths = [path for path in sorted(session_dir.glob('*.json')) if not path.name.startswith('_')]
     orphan_paths = _orphaned_backup_live_paths(session_dir, state_db_path=state_db_path)
-    for path in [*live_paths, *orphan_paths]:
-        # Skip non-session JSON files in the same dir:
-        # - ``_index.json`` is a top-level list of session metadata
-        # - any future non-session JSON marked with the ``_`` convention is
-        #   skipped automatically (project convention for system files in
-        #   directories that otherwise hold user data)
-        scanned += 1
+    # Only sessions with a backup can be restored through this startup path.
+    # Older code called recover_session() for every live sidecar, and
+    # inspect_session_recovery_status() read the complete JSON file before even
+    # checking whether <sid>.json.bak existed. Large WebUI installs therefore
+    # parsed the entire session corpus on every boot even when there was
+    # nothing to recover. Keep the public scanned count compatible, but limit
+    # expensive reads to actual recovery candidates.
+    recovery_paths = [path for path in live_paths if path.with_suffix('.json.bak').exists()]
+    scanned = len(live_paths) + len(orphan_paths)
+    for path in [*recovery_paths, *orphan_paths]:
         try:
             result = recover_session(path)
         except Exception as exc:

tests/test_metadata_save_wipe_1558.py

@@ -348,6 +348,37 @@ def test_recover_all_sessions_on_startup_is_idempotent_no_op_on_clean_state(temp
     assert live_before == live_after
 
 
+def test_recover_all_sessions_on_startup_does_not_read_live_files_without_backup(temp_session_dir, monkeypatch):
+    """Clean live sidecars without .bak are not recovery candidates at startup."""
+    clean_sid = _make_session_on_disk(temp_session_dir, sid="clean_no_bak", n_msgs=500)
+    backed_sid = _make_session_on_disk(temp_session_dir, sid="backed_candidate", n_msgs=4)
+    clean_path = temp_session_dir / f"{clean_sid}.json"
+    backed_path = temp_session_dir / f"{backed_sid}.json"
+    backed_path.with_suffix('.json.bak').write_text(
+        backed_path.read_text(encoding="utf-8"),
+        encoding="utf-8",
+    )
+
+    import api.session_recovery as sr
+
+    real_msg_count = sr._msg_count
+    msg_count_paths = []
+
+    def tracking_msg_count(path):
+        msg_count_paths.append(path)
+        return real_msg_count(path)
+
+    monkeypatch.setattr(sr, "_msg_count", tracking_msg_count)
+
+    result = sr.recover_all_sessions_on_startup(temp_session_dir)
+
+    assert result["restored"] == 0
+    assert result["scanned"] == 2
+    assert clean_path not in msg_count_paths
+    assert backed_path in msg_count_paths
+    assert backed_path.with_suffix('.json.bak') in msg_count_paths
+
+
 def test_recover_all_sessions_on_startup_skips_non_session_index_json(temp_session_dir):
     """Regression for v0.50.284 startup: ``_index.json`` is a top-level list
     (not a dict), and the recovery scanner globs ``*.json``. Without the

tests/test_session_index.py

@@ -944,6 +944,67 @@ def test_load_metadata_only_skips_index_read_when_sidecar_has_message_count(monk
     assert meta.compact()["message_count"] == 1
 
 
+
+
+def test_all_sessions_reuses_loaded_index_counts_for_legacy_sidecar_refresh(monkeypatch):
+    """Refreshing multiple legacy lineage rows must not parse _index.json per row."""
+    index_file = models.SESSION_INDEX_FILE
+    rows = []
+    for sid, count in (("legacy_lineage_a", 3), ("legacy_lineage_b", 4)):
+        payload = {
+            "session_id": sid,
+            "title": sid,
+            "workspace": "/tmp",
+            "model": "test",
+            "created_at": 100.0,
+            "updated_at": 200.0,
+            "pinned": False,
+            "archived": False,
+            "parent_session_id": "lineage_parent",
+            "messages": [{"role": "user", "content": "legacy"}],
+            "tool_calls": [],
+        }
+        # Deliberately bypass Session.save(): pre-fix legacy sidecars do not have
+        # a persisted message_count field in their metadata prefix.
+        (models.SESSION_DIR / f"{sid}.json").write_text(
+            json.dumps(payload, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+        rows.append({
+            "session_id": sid,
+            "title": sid,
+            "workspace": "/tmp",
+            "model": "test",
+            "created_at": 100.0,
+            "updated_at": 100.0,
+            "last_message_at": 100.0,
+            "message_count": count,
+            "pinned": False,
+            "archived": False,
+            "parent_session_id": "lineage_parent",
+        })
+    _write_index_file(index_file, rows)
+
+    original_read_text = Path.read_text
+    index_reads = 0
+
+    def _counting_read_text(self, *args, **kwargs):
+        nonlocal index_reads
+        if self == index_file:
+            index_reads += 1
+        return original_read_text(self, *args, **kwargs)
+
+    monkeypatch.setattr(Path, "read_text", _counting_read_text)
+    monkeypatch.setattr(models, "_enrich_sidebar_lineage_metadata", lambda _sessions: None)
+
+    result = models.all_sessions()
+
+    counts = {row["session_id"]: row["message_count"] for row in result}
+    assert counts["legacy_lineage_a"] == 3
+    assert counts["legacy_lineage_b"] == 4
+    assert index_reads == 1
+
+
 def test_session_save_does_not_persist_metadata_message_count_hint():
     s = Session(
         session_id="sess_private_hint",