fix(models): delegate generic provider catalogs to Hermes CLI
This commit is contained in:
Michael Lam
@@ -1963,6 +1963,65 @@ def _get_label_for_model(model_id: str, existing_groups: list) -> str:
|
||||
)
|
||||
|
||||
|
||||
def _read_live_provider_model_ids(provider_id: str) -> list[str]:
|
||||
"""Return live model IDs from Hermes CLI for a provider, or [] on failure.
|
||||
|
||||
WebUI's static ``_PROVIDER_MODELS`` table is only a fallback. The agent CLI
|
||||
owns the provider registry and catalog-discovery logic, so ordinary picker
|
||||
groups should ask ``hermes_cli.models.provider_model_ids()`` first (#1240).
|
||||
Provider aliases are tried as a secondary lookup because WebUI keeps a few
|
||||
display-facing IDs (for example ``google`` / ``x-ai``) that Hermes CLI may
|
||||
normalize internally.
|
||||
"""
|
||||
pid = str(provider_id or "").strip()
|
||||
if not pid:
|
||||
return []
|
||||
try:
|
||||
from hermes_cli.models import provider_model_ids as _provider_model_ids
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
candidates = [pid]
|
||||
try:
|
||||
alias = _resolve_provider_alias(pid)
|
||||
except Exception:
|
||||
alias = ""
|
||||
if alias and alias not in candidates:
|
||||
candidates.append(alias)
|
||||
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
try:
|
||||
live_ids = _provider_model_ids(candidate) or []
|
||||
except Exception:
|
||||
logger.debug("Failed to load %s models from hermes_cli", candidate)
|
||||
continue
|
||||
result: list[str] = []
|
||||
for mid in live_ids:
|
||||
mid_s = str(mid or "").strip()
|
||||
if mid_s and mid_s not in seen:
|
||||
seen.add(mid_s)
|
||||
result.append(mid_s)
|
||||
if result:
|
||||
return result
|
||||
return []
|
||||
|
||||
|
||||
def _models_from_live_provider_ids(provider_id: str, live_ids: list[str]) -> list[dict]:
|
||||
"""Convert Hermes CLI model ids into WebUI picker model entries."""
|
||||
formatter = _format_ollama_label if provider_id in ("ollama", "ollama-cloud") else None
|
||||
models: list[dict] = []
|
||||
seen: set[str] = set()
|
||||
for mid in live_ids:
|
||||
mid_s = str(mid or "").strip()
|
||||
if not mid_s or mid_s in seen:
|
||||
continue
|
||||
seen.add(mid_s)
|
||||
label = formatter(mid_s) if formatter else _get_label_for_model(mid_s, [])
|
||||
models.append({"id": mid_s, "label": label})
|
||||
return models
|
||||
|
||||
|
||||
def _read_visible_codex_cache_model_ids() -> list[str]:
|
||||
"""Return visible model slugs from Codex's local models_cache.json.
|
||||
|
||||
@@ -2912,18 +2971,33 @@ def get_available_models() -> dict:
|
||||
group_entry["extra_models"] = extras
|
||||
groups.append(group_entry)
|
||||
elif pid in _PROVIDER_MODELS or pid in cfg.get("providers", {}):
|
||||
raw_models = copy.deepcopy(_PROVIDER_MODELS.get(pid, []))
|
||||
detected_models = auto_detected_models_by_provider.get(pid, [])
|
||||
if detected_models and not raw_models:
|
||||
raw_models = copy.deepcopy(detected_models)
|
||||
|
||||
provider_cfg = cfg.get("providers", {}).get(pid, {})
|
||||
raw_models = []
|
||||
|
||||
# User-configured model allowlists are explicit local
|
||||
# source-of-truth and should still beat auto-discovery.
|
||||
# Otherwise, ask Hermes CLI first so WebUI tracks the same
|
||||
# live catalog as the agent/CLI picker; WebUI's static
|
||||
# _PROVIDER_MODELS table is now a fallback only (#1240).
|
||||
if isinstance(provider_cfg, dict) and "models" in provider_cfg:
|
||||
cfg_models = provider_cfg["models"]
|
||||
if isinstance(cfg_models, dict):
|
||||
raw_models = [{"id": k, "label": k} for k in cfg_models.keys()]
|
||||
elif isinstance(cfg_models, list):
|
||||
raw_models = [{"id": k, "label": k} for k in cfg_models]
|
||||
|
||||
if not raw_models:
|
||||
raw_models = _models_from_live_provider_ids(
|
||||
pid,
|
||||
_read_live_provider_model_ids(pid),
|
||||
)
|
||||
|
||||
if not raw_models:
|
||||
raw_models = copy.deepcopy(_PROVIDER_MODELS.get(pid, []))
|
||||
|
||||
detected_models = auto_detected_models_by_provider.get(pid, [])
|
||||
if detected_models and not raw_models:
|
||||
raw_models = copy.deepcopy(detected_models)
|
||||
models = _apply_provider_prefix(raw_models, pid, active_provider)
|
||||
groups.append(
|
||||
{
|
||||
|
||||
162
tests/test_issue1240_generic_cli_catalog_sync.py
Normal file
162
tests/test_issue1240_generic_cli_catalog_sync.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""Regression tests for #1240 — WebUI model catalog should delegate to Hermes CLI.
|
||||
|
||||
The WebUI picker should not freeze ordinary providers to its static
|
||||
``_PROVIDER_MODELS`` snapshot when Hermes CLI can return a fresher provider
|
||||
catalog. Static lists remain a fallback only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
|
||||
import api.config as config
|
||||
|
||||
|
||||
_PROVIDER_ENV_VARS = (
|
||||
"ANTHROPIC_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENROUTER_API_KEY",
|
||||
"GOOGLE_API_KEY",
|
||||
"GEMINI_API_KEY",
|
||||
"GLM_API_KEY",
|
||||
"KIMI_API_KEY",
|
||||
"DEEPSEEK_API_KEY",
|
||||
"OPENCODE_ZEN_API_KEY",
|
||||
"OPENCODE_GO_API_KEY",
|
||||
"MINIMAX_API_KEY",
|
||||
"MINIMAX_CN_API_KEY",
|
||||
"XAI_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"OLLAMA_CLOUD_API_KEY",
|
||||
"OLLAMA_API_KEY",
|
||||
"NOUS_API_KEY",
|
||||
"NVIDIA_API_KEY",
|
||||
)
|
||||
|
||||
|
||||
def _scrub_provider_env(monkeypatch):
|
||||
for name in _PROVIDER_ENV_VARS:
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
|
||||
def _install_fake_hermes_cli(monkeypatch, *, provider_id: str, live_ids, raise_on_lookup: bool = False):
|
||||
"""Install a hermes_cli stub that reports one authenticated provider."""
|
||||
fake_pkg = types.ModuleType("hermes_cli")
|
||||
fake_pkg.__path__ = []
|
||||
|
||||
fake_models = types.ModuleType("hermes_cli.models")
|
||||
fake_models.list_available_providers = lambda: [
|
||||
{"id": provider_id, "authenticated": True}
|
||||
]
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def provider_model_ids(pid):
|
||||
calls.append(pid)
|
||||
if raise_on_lookup:
|
||||
raise RuntimeError("simulated provider_model_ids failure")
|
||||
return list(live_ids) if pid == provider_id else []
|
||||
|
||||
fake_models.provider_model_ids = provider_model_ids
|
||||
|
||||
fake_auth = types.ModuleType("hermes_cli.auth")
|
||||
|
||||
def get_auth_status(pid):
|
||||
if pid == provider_id:
|
||||
return {"logged_in": True, "key_source": ""}
|
||||
return {"logged_in": False, "key_source": ""}
|
||||
|
||||
fake_auth.get_auth_status = get_auth_status
|
||||
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli", fake_pkg)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli.models", fake_models)
|
||||
monkeypatch.setitem(sys.modules, "hermes_cli.auth", fake_auth)
|
||||
monkeypatch.delitem(sys.modules, "agent.credential_pool", raising=False)
|
||||
monkeypatch.delitem(sys.modules, "agent", raising=False)
|
||||
config.invalidate_models_cache()
|
||||
return calls
|
||||
|
||||
|
||||
def _configure(monkeypatch, tmp_path, *, provider: str, default: str = ""):
|
||||
monkeypatch.setattr(config, "_get_config_path", lambda: tmp_path / "missing-config.yaml")
|
||||
monkeypatch.setattr(config, "_models_cache_path", tmp_path / "models_cache.json")
|
||||
monkeypatch.setattr(
|
||||
config,
|
||||
"cfg",
|
||||
{
|
||||
"model": {"provider": provider, "default": default},
|
||||
"providers": {},
|
||||
"fallback_providers": [],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(config, "_cfg_mtime", 0.0)
|
||||
config.invalidate_models_cache()
|
||||
|
||||
|
||||
def _provider_group(result: dict, provider_id: str) -> dict:
|
||||
return next(g for g in result["groups"] if g.get("provider_id") == provider_id)
|
||||
|
||||
|
||||
def _ids(group: dict) -> list[str]:
|
||||
return [m.get("id") for m in group.get("models", [])]
|
||||
|
||||
|
||||
def test_generic_provider_uses_hermes_cli_catalog_before_static_snapshot(monkeypatch, tmp_path):
|
||||
"""A normal provider should show fresh CLI-discovered models.
|
||||
|
||||
``claude-sonnet-5.0`` is intentionally absent from WebUI's static Anthropic
|
||||
list. Before this fix the group came entirely from ``_PROVIDER_MODELS`` and
|
||||
this model was invisible even though Hermes CLI knew about it.
|
||||
"""
|
||||
_scrub_provider_env(monkeypatch)
|
||||
calls = _install_fake_hermes_cli(
|
||||
monkeypatch,
|
||||
provider_id="anthropic",
|
||||
live_ids=["claude-opus-4.7", "claude-sonnet-5.0"],
|
||||
)
|
||||
_configure(monkeypatch, tmp_path, provider="anthropic", default="claude-opus-4.7")
|
||||
|
||||
result = config.get_available_models()
|
||||
group = _provider_group(result, "anthropic")
|
||||
|
||||
assert calls == ["anthropic"]
|
||||
assert _ids(group) == ["claude-opus-4.7", "claude-sonnet-5.0"]
|
||||
assert group["models"][1]["label"] == "Claude Sonnet 5.0"
|
||||
|
||||
|
||||
def test_generic_provider_keeps_static_catalog_as_cli_failure_fallback(monkeypatch, tmp_path):
|
||||
_scrub_provider_env(monkeypatch)
|
||||
calls = _install_fake_hermes_cli(
|
||||
monkeypatch,
|
||||
provider_id="anthropic",
|
||||
live_ids=[],
|
||||
raise_on_lookup=True,
|
||||
)
|
||||
_configure(monkeypatch, tmp_path, provider="anthropic", default="claude-opus-4.7")
|
||||
|
||||
result = config.get_available_models()
|
||||
group = _provider_group(result, "anthropic")
|
||||
|
||||
assert calls == ["anthropic"]
|
||||
assert "claude-opus-4.7" in _ids(group)
|
||||
assert "claude-sonnet-4.6" in _ids(group)
|
||||
|
||||
|
||||
def test_generic_provider_prefixes_live_ids_when_not_active_provider(monkeypatch, tmp_path):
|
||||
"""Provider-qualified live IDs must route through the selected provider."""
|
||||
_scrub_provider_env(monkeypatch)
|
||||
calls = _install_fake_hermes_cli(
|
||||
monkeypatch,
|
||||
provider_id="anthropic",
|
||||
live_ids=["claude-sonnet-5.0"],
|
||||
)
|
||||
# Anthropic is authenticated via Hermes CLI, but OpenAI is the active
|
||||
# default. The Anthropic row still has to be pickable/routable.
|
||||
_configure(monkeypatch, tmp_path, provider="openai", default="gpt-5.5")
|
||||
|
||||
result = config.get_available_models()
|
||||
group = _provider_group(result, "anthropic")
|
||||
|
||||
assert "anthropic" in calls
|
||||
assert _ids(group) == ["@anthropic:claude-sonnet-5.0"]
|
||||
@@ -116,8 +116,9 @@ class TestConfigYamlModelsLoading:
|
||||
)
|
||||
break
|
||||
|
||||
def test_provider_in_provider_models_but_no_cfg_override_unchanged(self):
|
||||
"""When no models key in cfg.providers, hardcoded _PROVIDER_MODELS still used."""
|
||||
def test_provider_in_provider_models_but_no_cfg_override_uses_static_fallback(self, monkeypatch):
|
||||
"""When Hermes CLI has no live catalog, _PROVIDER_MODELS remains fallback."""
|
||||
monkeypatch.setattr(_cfg, "_read_live_provider_model_ids", lambda _pid: [])
|
||||
cfg = {
|
||||
"model": {"provider": "anthropic"},
|
||||
"providers": {
|
||||
@@ -132,10 +133,9 @@ class TestConfigYamlModelsLoading:
|
||||
for g in result["groups"]:
|
||||
if g["provider"] == "Anthropic":
|
||||
returned_ids = {m["id"] for m in g["models"]}
|
||||
# Should still have the hardcoded models
|
||||
overlap = raw_ids & returned_ids
|
||||
assert overlap, (
|
||||
f"No _PROVIDER_MODELS models found in Anthropic group. "
|
||||
f"No _PROVIDER_MODELS fallback models found in Anthropic group. "
|
||||
f"Expected subset of {raw_ids}, got {returned_ids}"
|
||||
)
|
||||
break
|
||||
|
||||
@@ -234,18 +234,16 @@ def test_no_duplicate_when_default_model_is_prefixed():
|
||||
_cfg.cfg.update(old_cfg)
|
||||
|
||||
|
||||
def test_default_provider_models_not_prefixed():
|
||||
def test_default_provider_models_not_prefixed(monkeypatch):
|
||||
"""The active provider's models remain bare (no @prefix added)."""
|
||||
import api.config as _cfg
|
||||
raw_anthropic_ids = {m['id'] for m in _cfg._PROVIDER_MODELS.get('anthropic', [])}
|
||||
monkeypatch.setattr(_cfg, "_read_live_provider_model_ids", lambda pid: ["claude-sonnet-5.0"] if pid == "anthropic" else [])
|
||||
result = _available_models_with_provider('anthropic')
|
||||
groups = {g['provider']: g['models'] for g in result['groups']}
|
||||
if 'Anthropic' in groups:
|
||||
returned_ids = {m['id'] for m in groups['Anthropic']}
|
||||
for bare_id in raw_anthropic_ids:
|
||||
assert bare_id in returned_ids, (
|
||||
f"_PROVIDER_MODELS entry '{bare_id}' is missing from the Anthropic group"
|
||||
)
|
||||
assert "claude-sonnet-5.0" in returned_ids
|
||||
assert not any(mid.startswith('@anthropic:') for mid in returned_ids), returned_ids
|
||||
|
||||
|
||||
# ── get_available_models(): phantom "Custom" group regression ─────────────
|
||||
@@ -437,8 +435,10 @@ def test_custom_endpoint_uses_model_config_api_key_for_model_discovery(monkeypat
|
||||
return False
|
||||
|
||||
def _fake_urlopen(req, timeout=10):
|
||||
captured['auth'] = req.get_header('Authorization')
|
||||
captured['ua'] = req.get_header('User-agent')
|
||||
url = getattr(req, 'full_url', '')
|
||||
if 'example.test' in url:
|
||||
captured['auth'] = req.get_header('Authorization')
|
||||
captured['ua'] = req.get_header('User-agent')
|
||||
return _Resp()
|
||||
|
||||
monkeypatch.setattr('urllib.request.urlopen', _fake_urlopen)
|
||||
|
||||
Reference in New Issue
Block a user