Release v0.51.305 — Release JU (stage-p2b — dormant unified-SessionDB adapter) (#3759)

* refactor(sessions): add dormant JSON-backed SessionDB adapter (#3720, #3383)

First, lowest-risk slice of the unified-session-db migration: a SessionDB-shaped
adapter over the existing WebUI JSON store, behind a dormant experimental flag
(experimental.unified_session_db, default false). No runtime call site is rewired
— is_unified_session_db_enabled() has no live callers, so persistence behavior is
unchanged until a later migration PR opts in. Includes the adapter, the dormant
config flag + _apply_config_defaults wiring, an architecture doc, and adapter tests.

Also adds docs/architecture/ to the .gitignore docs allowlist (the docs/* rule
excludes subdirectories; the new architecture doc and its presence test would
otherwise be silently dropped on a clean checkout).

Co-authored-by: rodboev <[email protected]>

* docs(changelog): stamp v0.51.305 — Release JU (stage-p2b #3720)

---------

Co-authored-by: nesquena-hermes <[email protected]>

.gitignore

@@ -44,6 +44,8 @@ docs/*
 !docs/ui-ux/**
 !docs/rfcs/
 !docs/rfcs/**
+!docs/architecture/
+!docs/architecture/**
 
 # Local-only AI assistant context — never committed even under docs/.
 docs/AGENTS.md

CHANGELOG.md

@@ -3,6 +3,11 @@
 
 ## [Unreleased]
 
+## [v0.51.305] — 2026-06-06 — Release JU (stage-p2b — dormant unified-SessionDB adapter groundwork)
+
+### Changed
+- **Added the first, dormant slice of the unified-SessionDB migration (no runtime behavior change).** A new `SessionDB`-shaped adapter over the existing WebUI JSON session store lands behind an experimental flag (`experimental.unified_session_db`, default `false`). No runtime session call site is rewired — `is_unified_session_db_enabled()` has no live callers — so WebUI persistence is byte-for-byte unchanged until a later migration PR deliberately opts in. This is intentionally the lowest-risk first step toward expressing WebUI session access behind a stable adapter contract before any CLI/WebUI storage unification. Ships the adapter, the dormant config flag, an architecture doc, and adapter tests. (#3720 advances #3383, @rodboev)
+
 ## [v0.51.304] — 2026-06-06 — Release JT (stage-p2a — un-held terminal reaper fix + opt-in Docker GPU image)
 
 ### Fixed

api/config.py

@@ -320,6 +320,22 @@ def _get_config_path() -> Path:
 
 _WEBUI_SESSION_SAVE_MODES = {"deferred", "eager"}
 _DEFAULT_WEBUI_SESSION_SAVE_MODE = "deferred"
+_DEFAULT_EXPERIMENTAL_CONFIG = {
+    # Dormant first slice for the unified SessionDB migration. Runtime WebUI
+    # session call sites must continue using the existing JSON paths unless a
+    # later PR deliberately enables and wires this flag.
+    "unified_session_db": False,
+}
+
+
+def _apply_config_defaults(config_data: dict) -> None:
+    """Populate documented default-only config keys in-place."""
+    experimental = config_data.get("experimental")
+    if not isinstance(experimental, dict):
+        experimental = {}
+        config_data["experimental"] = experimental
+    for key, value in _DEFAULT_EXPERIMENTAL_CONFIG.items():
+        experimental.setdefault(key, value)
 
 
 def get_config() -> dict:
@@ -367,6 +383,19 @@ def get_webui_session_save_mode(config_data: dict | None = None) -> str:
     return _DEFAULT_WEBUI_SESSION_SAVE_MODE
 
 
+def is_unified_session_db_enabled(config_data: dict | None = None) -> bool:
+    """Return the dormant unified-session-db feature flag.
+
+    The default is intentionally false so adding the JSON adapter cannot change
+    runtime persistence until a later migration PR switches call sites.
+    """
+    active_cfg = config_data if isinstance(config_data, dict) else cfg
+    experimental = active_cfg.get("experimental", {}) if isinstance(active_cfg, dict) else {}
+    if not isinstance(experimental, dict):
+        return False
+    return experimental.get("unified_session_db") is True
+
+
 def reload_config() -> None:
     """Reload config.yaml from the active profile's directory."""
     global _cfg_mtime, _cfg_path, _cfg_fingerprint
@@ -391,6 +420,7 @@ def reload_config() -> None:
                         _cfg_mtime = 0.0
         except Exception:
             logger.debug("Failed to load yaml config from %s", config_path)
+        _apply_config_defaults(_cfg_cache)
         _cfg_fingerprint = _fingerprint_config(_cfg_cache)
         # Bust the models cache so the next request sees fresh config values.
         # Only delete the disk cache when config has actually changed -- not on

api/webui_session_db.py

@@ -0,0 +1,250 @@
+"""Dormant JSON-backed SessionDB-shaped adapter for WebUI sessions.
+
+This module intentionally does not replace existing WebUI runtime call sites.
+It provides a small compatibility surface over the current JSON sidecars so the
+unified SessionDB contract can be tested without changing persistence behavior.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+import threading
+from pathlib import Path
+from typing import Any
+
+import api.models as models
+
+
+_METADATA_FIELDS = frozenset(
+    {
+        "title",
+        "workspace",
+        "model",
+        "model_provider",
+        "created_at",
+        "updated_at",
+        "pinned",
+        "archived",
+        "project_id",
+        "profile",
+        "input_tokens",
+        "output_tokens",
+        "estimated_cost",
+        "cache_read_tokens",
+        "cache_write_tokens",
+        "personality",
+        "active_stream_id",
+        "pending_user_message",
+        "pending_attachments",
+        "pending_started_at",
+        "compression_anchor_visible_idx",
+        "compression_anchor_message_key",
+        "compression_anchor_summary",
+        "pre_compression_snapshot",
+        "context_engine",
+        "compression_anchor_engine",
+        "compression_anchor_mode",
+        "compression_anchor_details",
+        "context_engine_state",
+        "context_length",
+        "threshold_tokens",
+        "last_prompt_tokens",
+        "truncation_watermark",
+        "gateway_routing",
+        "gateway_routing_history",
+        "llm_title_generated",
+        "manual_title",
+        "parent_session_id",
+        "worktree_path",
+        "worktree_branch",
+        "worktree_repo_root",
+        "worktree_created_at",
+        "is_cli_session",
+        "source_tag",
+        "raw_source",
+        "session_source",
+        "source_label",
+        "read_only",
+        "enabled_toolsets",
+        "composer_draft",
+    }
+)
+
+_UNSAFE_FIELDS = frozenset({"session_id", "messages", "tool_calls", "message_count"})
+
+
+class WebUIJsonSessionDB:
+    """Small SessionDB-like facade over existing WebUI session JSON files."""
+
+    def __init__(self, session_dir: Path | str | None = None):
+        self._session_dir = Path(session_dir).expanduser().resolve() if session_dir else None
+
+    @property
+    def session_dir(self) -> Path:
+        return self._session_dir or models.SESSION_DIR
+
+    def list_sessions(self) -> list[dict[str, Any]]:
+        """Return compact metadata for persisted WebUI JSON sessions.
+
+        Reads are direct JSON loads and never call ``Session.load()``, because
+        that path may self-heal and write repaired transcripts.
+        """
+        rows: list[dict[str, Any]] = []
+        if not self.session_dir.exists():
+            return rows
+        for path in self.session_dir.glob("*.json"):
+            if path.name.startswith("_"):
+                continue
+            data = self._read_path(path)
+            if not isinstance(data, dict):
+                continue
+            sid = str(data.get("session_id") or path.stem)
+            if not models.is_safe_session_id(sid):
+                continue
+            rows.append(self._metadata_row(sid, data))
+        rows.sort(key=lambda row: (bool(row.get("pinned")), self._sort_timestamp(row)), reverse=True)
+        return rows
+
+    def read_session(self, sid: str) -> dict[str, Any] | None:
+        """Return the full JSON session payload for ``sid`` without mutation."""
+        path = self._path_for_sid(sid)
+        if path is None or not path.exists():
+            return None
+        data = self._read_path(path)
+        if not isinstance(data, dict):
+            return None
+        return copy.deepcopy(data)
+
+    def update_metadata(self, sid: str, fields: dict[str, Any]) -> dict[str, Any]:
+        """Persist allowlisted metadata fields while preserving messages.
+
+        This dormant adapter method is for migration experiments and tests only.
+        Runtime wiring must add Session lock/cache/index parity before using it
+        from live WebUI routes.
+        """
+        if not isinstance(fields, dict):
+            raise TypeError("fields must be a dict")
+        unsafe = sorted((set(fields) & _UNSAFE_FIELDS) | (set(fields) - _METADATA_FIELDS))
+        if unsafe:
+            raise ValueError(f"Unsafe session metadata fields: {', '.join(unsafe)}")
+
+        path = self._existing_path_for_sid(sid)
+        data = self._read_writable_session(path)
+        data.update(copy.deepcopy(fields))
+        data["message_count"] = len(data["messages"])
+        self._atomic_write(path, data)
+        return self._metadata_row(str(data.get("session_id") or sid), data)
+
+    def archive(self, sid: str, archived: bool = True) -> dict[str, Any]:
+        """Set the archived metadata flag without touching transcript messages."""
+        return self.update_metadata(sid, {"archived": bool(archived)})
+
+    def write_session(self, session: dict[str, Any]) -> dict[str, Any]:
+        """Write a full session payload for tests and migration experiments."""
+        if not isinstance(session, dict):
+            raise TypeError("session must be a dict")
+        sid = session.get("session_id")
+        path = self._path_for_sid(sid)
+        if path is None:
+            raise ValueError(f"Unsafe session_id {sid!r}")
+        messages = session.get("messages")
+        if not isinstance(messages, list):
+            raise ValueError("session payload must include a messages list")
+        payload = copy.deepcopy(session)
+        payload["message_count"] = len(messages)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        self._atomic_write(path, payload)
+        return copy.deepcopy(payload)
+
+    def _path_for_sid(self, sid: str) -> Path | None:
+        if not models.is_safe_session_id(sid):
+            return None
+        return self.session_dir / f"{sid}.json"
+
+    def _existing_path_for_sid(self, sid: str) -> Path:
+        path = self._path_for_sid(sid)
+        if path is None:
+            raise ValueError(f"Unsafe session_id {sid!r}")
+        if not path.exists():
+            raise KeyError(sid)
+        return path
+
+    def _read_writable_session(self, path: Path) -> dict[str, Any]:
+        data = self._read_path(path)
+        if not isinstance(data, dict):
+            raise ValueError(f"Malformed session JSON: {path.name}")
+        sid = data.get("session_id")
+        if not models.is_safe_session_id(sid):
+            raise ValueError(f"Unsafe session_id {sid!r}")
+        if not isinstance(data.get("messages"), list):
+            raise ValueError(f"Refusing to write metadata-only session stub: {sid!r}")
+        return data
+
+    @staticmethod
+    def _read_path(path: Path) -> dict[str, Any] | None:
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError, UnicodeDecodeError):
+            return None
+
+    @staticmethod
+    def _metadata_row(sid: str, data: dict[str, Any]) -> dict[str, Any]:
+        messages = data.get("messages")
+        message_count = data.get("message_count")
+        if not isinstance(message_count, int):
+            message_count = len(messages) if isinstance(messages, list) else 0
+        row = {field: copy.deepcopy(data.get(field)) for field in _METADATA_FIELDS if field in data}
+        row["session_id"] = sid
+        row["message_count"] = message_count
+        row["last_message_at"] = data.get("last_message_at") or data.get("updated_at") or data.get("created_at")
+        return row
+
+    @staticmethod
+    def _sort_timestamp(row: dict[str, Any]) -> float:
+        for key in ("last_message_at", "updated_at", "created_at"):
+            value = row.get(key)
+            if value is None or value == "":
+                continue
+            try:
+                return float(value)
+            except (TypeError, ValueError):
+                continue
+        return 0.0
+
+    @staticmethod
+    def _atomic_write(path: Path, data: dict[str, Any]) -> None:
+        payload = json.dumps(data, ensure_ascii=False, indent=2)
+        tmp = path.with_suffix(f".tmp.{os.getpid()}.{threading.current_thread().ident}")
+        try:
+            with open(tmp, "w", encoding="utf-8") as handle:
+                handle.write(payload)
+                handle.flush()
+                os.fsync(handle.fileno())
+            os.replace(tmp, path)
+        finally:
+            try:
+                tmp.unlink(missing_ok=True)
+            except OSError:
+                pass
+
+
+def list_sessions() -> list[dict[str, Any]]:
+    return WebUIJsonSessionDB().list_sessions()
+
+
+def read_session(sid: str) -> dict[str, Any] | None:
+    return WebUIJsonSessionDB().read_session(sid)
+
+
+def update_metadata(sid: str, fields: dict[str, Any]) -> dict[str, Any]:
+    return WebUIJsonSessionDB().update_metadata(sid, fields)
+
+
+def archive(sid: str, archived: bool = True) -> dict[str, Any]:
+    return WebUIJsonSessionDB().archive(sid, archived)
+
+
+def write_session(session: dict[str, Any]) -> dict[str, Any]:
+    return WebUIJsonSessionDB().write_session(session)

docs/architecture/unified-session-db.md

@@ -0,0 +1,98 @@
+# Unified SessionDB Adapter Spike
+
+WebUI currently persists conversations as JSON files under the WebUI session
+directory, while the CLI uses its own session database. The first safe slice of
+unification is a dormant adapter that presents a small SessionDB-shaped API over
+the existing WebUI JSON files without changing runtime call sites or file
+format.
+
+## Adapter Contract
+
+`api.webui_session_db.WebUIJsonSessionDB` exposes:
+
+- `list_sessions()` returns compact metadata rows for persisted WebUI JSON
+  sessions.
+- `read_session(sid)` returns a full session JSON payload or `None`.
+- `update_metadata(sid, fields)` writes only allowlisted metadata fields and
+  rejects unsafe keys such as `session_id`, `messages`, `tool_calls`, and
+  `message_count`.
+- `archive(sid, archived=True)` is a convenience metadata update for the
+  archived flag.
+- `write_session(session)` exists for tests and migration experiments that need
+  to materialize a complete JSON payload.
+
+Read operations must not call `Session.load()` or `all_sessions()`, because
+those paths can repair indexes or transcripts. Metadata writes must load the
+complete JSON payload, verify that a real `messages` list is present, update only
+safe fields, recompute `message_count`, and atomically replace the file. The
+adapter must never write a metadata-only stub that could drop transcript
+messages.
+
+## Why JSON-Backed And Dormant
+
+The selected first slice is infrastructure only. Keeping the adapter backed by
+the current JSON sidecars validates the API shape while preserving all current
+WebUI behavior, backups, and import paths. The feature flag defaults to:
+
+```yaml
+experimental:
+  unified_session_db: false
+```
+
+No UI exposes this flag, and no runtime session route switches to the adapter in
+this slice.
+
+## Runtime Wiring Preconditions
+
+Before any route uses this adapter for live metadata changes, a follow-up PR must
+prove parity with the existing `Session.save()` path:
+
+- take the same per-session mutation locks used by streaming and session routes,
+  so metadata writes cannot replace a newer transcript with a stale copy;
+- refresh or invalidate the in-memory `Session` cache and `_index.json`, so
+  sidebar rows and later `Session.save()` calls cannot overwrite adapter changes;
+- match `Session.compact()` sidebar semantics for pending first turns,
+  `has_pending_user_message`, `pending_started_at`, and real non-tool
+  `last_message_at` ordering.
+
+Until those invariants are implemented, `update_metadata()` and `archive()` are
+test/migration helpers, not runtime persistence replacements.
+
+## Planned Migration Sequence
+
+1. Land the dormant JSON adapter and contract tests.
+2. Add parity tests that compare adapter reads with existing WebUI sidebar and
+   session payloads.
+3. Introduce an opt-in dual-read or shadow-read mode for development builds.
+4. Add a migration path that can write unified SessionDB records without
+   deleting or rewriting JSON sidecars.
+5. Switch selected call sites behind the flag only after parity and rollback
+   behavior are proven.
+6. Make the unified store authoritative in a later release after import,
+   archive, pin, profile, project, and recovery semantics match WebUI JSON.
+
+## Authoritative Fields And Open Questions
+
+The JSON sidecar remains authoritative for `messages`, `tool_calls`, metadata
+display fields, profile/project ownership, archive and pin state, token/cost
+totals, pending stream recovery fields, worktree metadata, and composer draft
+state during this spike.
+
+Open questions for later slices:
+
+- Whether `updated_at` should reflect metadata-only changes such as archive and
+  pin operations or only transcript changes.
+- How to resolve conflicts when CLI and WebUI update titles, archive state, or
+  project/profile ownership concurrently.
+- Whether imported CLI sessions remain read-only projections or become editable
+  unified records.
+- How unified records should preserve WebUI recovery safeguards such as backup
+  creation before transcript shrinkage.
+- Which store owns sidebar ordering once JSON and SessionDB records coexist.
+
+## Out Of Scope
+
+This spike does not switch runtime WebUI call sites, migrate existing session
+files, expose a UI setting, alter CLI storage, change session import behavior, or
+remove any JSON sidecars. It is a contract and safety test bed for future
+migration work.

tests/test_webui_session_db_adapter.py

@@ -0,0 +1,162 @@
+import json
+from pathlib import Path
+
+import pytest
+
+import api.config as config
+import api.models as models
+import api.webui_session_db as session_db
+from api.webui_session_db import WebUIJsonSessionDB
+
+
+@pytest.fixture
+def session_dir(tmp_path, monkeypatch):
+    path = tmp_path / "sessions"
+    path.mkdir()
+    monkeypatch.setattr(models, "SESSION_DIR", path)
+    return path
+
+
+def _write_json_session(session_dir, sid="session_1", **overrides):
+    payload = {
+        "session_id": sid,
+        "title": "Adapter Session",
+        "workspace": str(session_dir.parent),
+        "model": "gpt-test",
+        "model_provider": "openai",
+        "created_at": 100.0,
+        "updated_at": 200.0,
+        "pinned": False,
+        "archived": False,
+        "profile": "default",
+        "messages": [
+            {"role": "user", "content": "hello"},
+            {"role": "assistant", "content": "world"},
+        ],
+        "tool_calls": [{"id": "tool-1", "name": "demo"}],
+    }
+    payload.update(overrides)
+    payload["message_count"] = len(payload["messages"])
+    path = session_dir / f"{sid}.json"
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    return payload, path
+
+
+def test_list_and_read_existing_json_sessions(session_dir):
+    payload, _path = _write_json_session(session_dir)
+    db = WebUIJsonSessionDB()
+
+    rows = db.list_sessions()
+    loaded = db.read_session(payload["session_id"])
+
+    assert [row["session_id"] for row in rows] == [payload["session_id"]]
+    assert rows[0]["title"] == payload["title"]
+    assert rows[0]["message_count"] == 2
+    assert loaded == payload
+
+
+def test_metadata_update_survives_reload_and_preserves_messages(session_dir):
+    payload, path = _write_json_session(session_dir)
+    db = WebUIJsonSessionDB()
+
+    db.update_metadata(payload["session_id"], {"title": "Renamed", "pinned": True})
+    reloaded = json.loads(path.read_text(encoding="utf-8"))
+
+    assert reloaded["title"] == "Renamed"
+    assert reloaded["pinned"] is True
+    assert reloaded["messages"] == payload["messages"]
+    assert reloaded["tool_calls"] == payload["tool_calls"]
+    assert reloaded["message_count"] == len(payload["messages"])
+
+
+def test_metadata_update_rejects_unsafe_fields(session_dir):
+    payload, path = _write_json_session(session_dir)
+    before = path.read_text(encoding="utf-8")
+    db = WebUIJsonSessionDB()
+
+    with pytest.raises(ValueError):
+        db.update_metadata(payload["session_id"], {"messages": []})
+    with pytest.raises(ValueError):
+        db.update_metadata(payload["session_id"], {"unknown_field": "unsafe"})
+
+    assert path.read_text(encoding="utf-8") == before
+
+
+def test_metadata_update_refuses_metadata_only_stub(session_dir):
+    sid = "stub_session"
+    path = session_dir / f"{sid}.json"
+    path.write_text(
+        json.dumps({"session_id": sid, "title": "Stub"}, indent=2),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="metadata-only"):
+        WebUIJsonSessionDB().update_metadata(sid, {"title": "Nope"})
+
+
+def test_archive_unarchive_round_trip(session_dir):
+    payload, path = _write_json_session(session_dir)
+    db = WebUIJsonSessionDB()
+
+    archived = db.archive(payload["session_id"])
+    unarchived = db.archive(payload["session_id"], archived=False)
+    reloaded = json.loads(path.read_text(encoding="utf-8"))
+
+    assert archived["archived"] is True
+    assert unarchived["archived"] is False
+    assert reloaded["archived"] is False
+    assert reloaded["messages"] == payload["messages"]
+
+
+def test_read_only_operations_do_not_mutate_files(session_dir):
+    payload, path = _write_json_session(session_dir)
+    before_text = path.read_text(encoding="utf-8")
+    before_stat = path.stat()
+    db = WebUIJsonSessionDB()
+
+    assert db.list_sessions()
+    assert db.read_session(payload["session_id"]) == payload
+
+    after_stat = path.stat()
+    assert path.read_text(encoding="utf-8") == before_text
+    assert after_stat.st_mtime_ns == before_stat.st_mtime_ns
+    assert after_stat.st_size == before_stat.st_size
+
+
+def test_sort_timestamp_falls_back_past_missing_values():
+    assert WebUIJsonSessionDB._sort_timestamp({"last_message_at": None, "updated_at": 25.0}) == 25.0
+    assert WebUIJsonSessionDB._sort_timestamp({"last_message_at": "", "created_at": "15.5"}) == 15.5
+
+
+def test_module_level_write_session_wrapper(session_dir):
+    payload, _path = _write_json_session(session_dir, sid="wrapper_session")
+    written = session_db.write_session(payload)
+
+    assert written == payload
+    assert session_db.read_session(payload["session_id"]) == payload
+
+
+def test_unified_session_db_flag_default_remains_false(monkeypatch, tmp_path):
+    cfg_path = tmp_path / "missing-config.yaml"
+    monkeypatch.setattr(config, "_get_config_path", lambda: cfg_path)
+
+    config.reload_config()
+
+    assert config.get_config()["experimental"]["unified_session_db"] is False
+    assert config.is_unified_session_db_enabled() is False
+    assert config.is_unified_session_db_enabled({"experimental": {"unified_session_db": True}}) is True
+
+
+def test_adapter_docs_pin_runtime_wiring_preconditions():
+    doc = (Path(__file__).resolve().parents[1] / "docs" / "architecture" / "unified-session-db.md").read_text(
+        encoding="utf-8"
+    )
+
+    assert "Runtime Wiring Preconditions" in doc
+    assert "per-session mutation locks" in doc
+    assert "in-memory `Session` cache and `_index.json`" in doc
+    assert "pending first turns" in doc
+    assert "test/migration helpers, not runtime persistence replacements" in doc
+    assert "Runtime wiring must add Session lock/cache/index parity" in (
+        session_db.WebUIJsonSessionDB.update_metadata.__doc__ or ""
+    )