#3327 fix(reasoning): normalize custom-provider model ids for fallback heuristics Co-authored-by: Carry00 <Carry00@users.noreply.github.com> #3334 docs(rfc): mark run-adapter Slice 4f shipped, define Slice 4g gate Co-authored-by: Michaelyklam <Michaelyklam@users.noreply.github.com> #3341 fix(profiles): show enabled vs compatible skill counts Co-authored-by: b3nw <b3nw@users.noreply.github.com>
This commit is contained in:
@@ -2094,6 +2094,61 @@ def _strip_provider_hint_for_reasoning(model_id: str) -> str:
|
|||||||
return model
|
return model
|
||||||
|
|
||||||
|
|
||||||
|
def _reasoning_name_candidates(model_id: str) -> list[str]:
|
||||||
|
"""Return normalized model-name candidates for heuristic capability checks."""
|
||||||
|
bare = str(model_id or "").strip().lower().rsplit("/", 1)[-1]
|
||||||
|
if not bare:
|
||||||
|
return []
|
||||||
|
|
||||||
|
candidates: list[str] = []
|
||||||
|
|
||||||
|
def _add(value: str) -> None:
|
||||||
|
candidate = str(value or "").strip().lower()
|
||||||
|
if candidate and candidate not in candidates:
|
||||||
|
candidates.append(candidate)
|
||||||
|
|
||||||
|
_add(bare)
|
||||||
|
|
||||||
|
dot_parts = [part for part in bare.split(".") if part]
|
||||||
|
if len(dot_parts) > 1:
|
||||||
|
# Try progressively stripping dot-separated vendor namespaces so inputs like
|
||||||
|
# "moonshotai.kimi-k2.5" and "vendor.deepseek.v3.2" both surface the real
|
||||||
|
# model family rather than treating every dot as part of the provider slug.
|
||||||
|
for index in range(1, len(dot_parts)):
|
||||||
|
suffix = ".".join(dot_parts[index:])
|
||||||
|
if any(ch.isalpha() for ch in suffix):
|
||||||
|
_add(suffix)
|
||||||
|
|
||||||
|
for candidate in list(candidates):
|
||||||
|
normalized = re.sub(r"[^a-z0-9]+", "-", candidate).strip("-")
|
||||||
|
_add(normalized)
|
||||||
|
|
||||||
|
return candidates
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_supports_reasoning(candidate: str) -> bool:
|
||||||
|
normalized = re.sub(r"[^a-z0-9]+", "-", str(candidate or "").strip().lower()).strip("-")
|
||||||
|
if not normalized:
|
||||||
|
return False
|
||||||
|
|
||||||
|
tokens = [token for token in normalized.split("-") if token]
|
||||||
|
token_set = set(tokens)
|
||||||
|
|
||||||
|
if "thinking" in token_set or "reasoning" in token_set:
|
||||||
|
return True
|
||||||
|
if normalized in {"o1", "o3", "o4"} or normalized.startswith(("o1-", "o3-", "o4-")):
|
||||||
|
return True
|
||||||
|
if normalized.startswith(("kimi-k2", "kimi-thinking", "claude-3", "claude-4")):
|
||||||
|
return True
|
||||||
|
if normalized.startswith("qwen3") or "qwen3" in token_set:
|
||||||
|
return True
|
||||||
|
if normalized.startswith(("deepseek-v3", "deepseek-v4", "deepseek-r1", "deepseek-r2")):
|
||||||
|
return True
|
||||||
|
if len(tokens) >= 2 and tokens[0] == "deepseek" and tokens[1] in {"v3", "v4", "r1", "r2"}:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _heuristic_reasoning_efforts(model_id: str, provider_id: str) -> list[str]:
|
def _heuristic_reasoning_efforts(model_id: str, provider_id: str) -> list[str]:
|
||||||
"""Fallback when hermes_cli is unavailable."""
|
"""Fallback when hermes_cli is unavailable."""
|
||||||
model = _strip_provider_hint_for_reasoning(model_id).lower()
|
model = _strip_provider_hint_for_reasoning(model_id).lower()
|
||||||
@@ -2123,31 +2178,11 @@ def _heuristic_reasoning_efforts(model_id: str, provider_id: str) -> list[str]:
|
|||||||
)
|
)
|
||||||
if any(model.startswith(prefix) for prefix in prefixes):
|
if any(model.startswith(prefix) for prefix in prefixes):
|
||||||
return list(VALID_REASONING_EFFORTS)
|
return list(VALID_REASONING_EFFORTS)
|
||||||
# Custom API aggregators (e.g. New API, One API) use non-standard model naming:
|
# Named custom providers often rewrite model ids with dots, underscores, or
|
||||||
# bare names like "deepseek-v4-flash" or dot-separated "moonshotai.kimi-k2.5"
|
# extra vendor namespaces. Normalize those shapes before applying family-level
|
||||||
# rather than the OpenRouter-style "vendor/model" that the prefix list targets.
|
# reasoning heuristics so "deepseek.v3.2", "deepseek_v4_flash", and
|
||||||
# Strip a dot-vendor prefix (e.g. "moonshotai.kimi-k2.5" → "kimi-k2.5") and
|
# "vendor.deepseek.v3.2" are treated consistently.
|
||||||
# check both the original bare name and the stripped suffix.
|
if any(_candidate_supports_reasoning(candidate) for candidate in _reasoning_name_candidates(bare)):
|
||||||
bare_after_dot = bare.split(".", 1)[-1] if "." in bare else bare
|
|
||||||
thinking_bare_prefixes = (
|
|
||||||
"deepseek-v4",
|
|
||||||
"deepseek-r1",
|
|
||||||
"deepseek-r2",
|
|
||||||
"kimi-k2",
|
|
||||||
"kimi-thinking",
|
|
||||||
"qwen3",
|
|
||||||
"claude-3",
|
|
||||||
"claude-4",
|
|
||||||
"o1-",
|
|
||||||
"o3-",
|
|
||||||
"o4-",
|
|
||||||
)
|
|
||||||
if any(
|
|
||||||
bare.startswith(p) or bare_after_dot.startswith(p)
|
|
||||||
for p in thinking_bare_prefixes
|
|
||||||
):
|
|
||||||
return list(VALID_REASONING_EFFORTS)
|
|
||||||
if "thinking" in bare or "reasoning" in bare:
|
|
||||||
return list(VALID_REASONING_EFFORTS)
|
return list(VALID_REASONING_EFFORTS)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|||||||
@@ -891,6 +891,7 @@ def switch_profile(name: str, *, process_wide: bool = True) -> dict:
|
|||||||
raise ValueError(f"Profile '{name}' does not exist.")
|
raise ValueError(f"Profile '{name}' does not exist.")
|
||||||
|
|
||||||
with _profile_lock:
|
with _profile_lock:
|
||||||
|
_SKILLS_STATS_CACHE.clear()
|
||||||
if process_wide:
|
if process_wide:
|
||||||
global _active_profile
|
global _active_profile
|
||||||
_active_profile = name
|
_active_profile = name
|
||||||
@@ -989,6 +990,77 @@ def switch_profile(name: str, *, process_wide: bool = True) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_SKILLS_STATS_CACHE: dict[Path, tuple[int, int, float]] = {}
|
||||||
|
_SKILLS_STATS_CACHE_TTL = 8.0 # seconds
|
||||||
|
|
||||||
|
|
||||||
|
def _get_profile_skills_stats(profile_dir: Path) -> tuple[int, int]:
|
||||||
|
"""Calculate (enabled_count, compatible_count) for a profile directory."""
|
||||||
|
import time
|
||||||
|
profile_dir = Path(profile_dir).resolve()
|
||||||
|
now = time.time()
|
||||||
|
if profile_dir in _SKILLS_STATS_CACHE:
|
||||||
|
enabled, compat, expiry = _SKILLS_STATS_CACHE[profile_dir]
|
||||||
|
if now < expiry:
|
||||||
|
return enabled, compat
|
||||||
|
|
||||||
|
skills_dir = profile_dir / "skills"
|
||||||
|
if not skills_dir.is_dir():
|
||||||
|
res = (0, 0)
|
||||||
|
_SKILLS_STATS_CACHE[profile_dir] = (res[0], res[1], now + _SKILLS_STATS_CACHE_TTL)
|
||||||
|
return res
|
||||||
|
|
||||||
|
disabled = set()
|
||||||
|
config_path = profile_dir / "config.yaml"
|
||||||
|
if config_path.exists():
|
||||||
|
try:
|
||||||
|
import yaml as _yaml
|
||||||
|
cfg = _yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
||||||
|
if isinstance(cfg, dict):
|
||||||
|
skills_cfg = cfg.get("skills")
|
||||||
|
if isinstance(skills_cfg, dict):
|
||||||
|
# Align with get_disabled_skill_names(platform="webui") behavior:
|
||||||
|
platform_disabled = (skills_cfg.get("platform_disabled") or {}).get("webui")
|
||||||
|
if platform_disabled is not None:
|
||||||
|
disabled_val = platform_disabled
|
||||||
|
else:
|
||||||
|
disabled_val = skills_cfg.get("disabled")
|
||||||
|
|
||||||
|
if disabled_val is not None:
|
||||||
|
if isinstance(disabled_val, str):
|
||||||
|
disabled_val = [disabled_val]
|
||||||
|
disabled = {str(v).strip() for v in disabled_val if str(v).strip()}
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
from agent.skill_utils import iter_skill_index_files, parse_frontmatter, skill_matches_platform
|
||||||
|
|
||||||
|
seen_names = set()
|
||||||
|
enabled_count = 0
|
||||||
|
compatible_count = 0
|
||||||
|
|
||||||
|
for skill_md in iter_skill_index_files(skills_dir, "SKILL.md"):
|
||||||
|
try:
|
||||||
|
content = skill_md.read_text(encoding="utf-8")[:4000]
|
||||||
|
frontmatter, _ = parse_frontmatter(content)
|
||||||
|
if not skill_matches_platform(frontmatter):
|
||||||
|
continue
|
||||||
|
name = frontmatter.get("name", skill_md.parent.name)[:64]
|
||||||
|
if name in seen_names:
|
||||||
|
continue
|
||||||
|
seen_names.add(name)
|
||||||
|
|
||||||
|
compatible_count += 1
|
||||||
|
if name not in disabled:
|
||||||
|
enabled_count += 1
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
res = (enabled_count, compatible_count)
|
||||||
|
_SKILLS_STATS_CACHE[profile_dir] = (res[0], res[1], now + _SKILLS_STATS_CACHE_TTL)
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
def list_profiles_api() -> list:
|
def list_profiles_api() -> list:
|
||||||
"""List all profiles with metadata, serialized for JSON response."""
|
"""List all profiles with metadata, serialized for JSON response."""
|
||||||
try:
|
try:
|
||||||
@@ -1001,6 +1073,7 @@ def list_profiles_api() -> list:
|
|||||||
active = get_active_profile_name()
|
active = get_active_profile_name()
|
||||||
result = []
|
result = []
|
||||||
for p in infos:
|
for p in infos:
|
||||||
|
enabled_count, total_count = _get_profile_skills_stats(p.path)
|
||||||
result.append({
|
result.append({
|
||||||
'name': p.name,
|
'name': p.name,
|
||||||
'path': str(p.path),
|
'path': str(p.path),
|
||||||
@@ -1010,13 +1083,16 @@ def list_profiles_api() -> list:
|
|||||||
'model': p.model,
|
'model': p.model,
|
||||||
'provider': p.provider,
|
'provider': p.provider,
|
||||||
'has_env': p.has_env,
|
'has_env': p.has_env,
|
||||||
'skill_count': p.skill_count,
|
'skill_count': enabled_count,
|
||||||
|
'enabled_skills': enabled_count,
|
||||||
|
'total_skills': total_count,
|
||||||
})
|
})
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _default_profile_dict() -> dict:
|
def _default_profile_dict() -> dict:
|
||||||
"""Fallback profile dict when hermes_cli is not importable."""
|
"""Fallback profile dict when hermes_cli is not importable."""
|
||||||
|
enabled_count, compatible_count = _get_profile_skills_stats(_DEFAULT_HERMES_HOME)
|
||||||
return {
|
return {
|
||||||
'name': 'default',
|
'name': 'default',
|
||||||
'path': str(_DEFAULT_HERMES_HOME),
|
'path': str(_DEFAULT_HERMES_HOME),
|
||||||
@@ -1026,7 +1102,9 @@ def _default_profile_dict() -> dict:
|
|||||||
'model': None,
|
'model': None,
|
||||||
'provider': None,
|
'provider': None,
|
||||||
'has_env': (_DEFAULT_HERMES_HOME / '.env').exists(),
|
'has_env': (_DEFAULT_HERMES_HOME / '.env').exists(),
|
||||||
'skill_count': 0,
|
'skill_count': enabled_count,
|
||||||
|
'enabled_skills': enabled_count,
|
||||||
|
'total_skills': compatible_count,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1437,6 +1515,7 @@ def create_profile_api(name: str, clone_from: str = None,
|
|||||||
|
|
||||||
# Invalidate cached root-profile-name lookup; create_profile may have added
|
# Invalidate cached root-profile-name lookup; create_profile may have added
|
||||||
# a new profile that flips is_default semantics on the agent side (#1612).
|
# a new profile that flips is_default semantics on the agent side (#1612).
|
||||||
|
_SKILLS_STATS_CACHE.clear()
|
||||||
_invalidate_root_profile_cache()
|
_invalidate_root_profile_cache()
|
||||||
|
|
||||||
# Find and return the newly created profile info.
|
# Find and return the newly created profile info.
|
||||||
@@ -1456,6 +1535,8 @@ def create_profile_api(name: str, clone_from: str = None,
|
|||||||
'provider': None,
|
'provider': None,
|
||||||
'has_env': (profile_path / '.env').exists(),
|
'has_env': (profile_path / '.env').exists(),
|
||||||
'skill_count': 0,
|
'skill_count': 0,
|
||||||
|
'enabled_skills': 0,
|
||||||
|
'total_skills': 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -1488,5 +1569,6 @@ def delete_profile_api(name: str) -> dict:
|
|||||||
raise ValueError(f"Profile '{name}' does not exist.")
|
raise ValueError(f"Profile '{name}' does not exist.")
|
||||||
|
|
||||||
# Drop cached root-profile-name lookup — list_profiles_api() shape changed.
|
# Drop cached root-profile-name lookup — list_profiles_api() shape changed.
|
||||||
|
_SKILLS_STATS_CACHE.clear()
|
||||||
_invalidate_root_profile_cache()
|
_invalidate_root_profile_cache()
|
||||||
return {'ok': True, 'name': name}
|
return {'ok': True, 'name': name}
|
||||||
|
|||||||
@@ -956,28 +956,34 @@ Non-goals for Slice 4e:
|
|||||||
|
|
||||||
#### Slice 4f: Supervised local runner client backend gate
|
#### Slice 4f: Supervised local runner client backend gate
|
||||||
|
|
||||||
Status as of 2026-05-28: client transport proposed in #3073 behind
|
Status as of 2026-05-31: shipped in v0.51.188 via #3073 / #3274. The client
|
||||||
`HERMES_WEBUI_RUNNER_BASE_URL`; it should be described as under review until a
|
transport is now implemented behind `HERMES_WEBUI_RUNNER_BASE_URL` and remains
|
||||||
release PR actually ships it.
|
default-off. With no endpoint configured, `runner-local` still returns the
|
||||||
`runner-local` still remains default-off and returns the bounded not-configured
|
bounded not-configured path and the live in-process `_run_agent_streaming` path
|
||||||
path unless that endpoint is explicitly configured. When configured, WebUI uses a
|
is unchanged. When configured, WebUI uses a JSON HTTP client boundary for start /
|
||||||
JSON HTTP client boundary for start / observe / status / controls and bridges
|
observe / status / controls and bridges observed runner events through the
|
||||||
observed runner events through the existing SSE stream route rather than adding
|
existing SSE stream route rather than adding main-process runner-owned maps.
|
||||||
main-process runner-owned maps.
|
|
||||||
|
The release added two security hardening checks while absorbing #3073:
|
||||||
|
`HttpRunnerClient` rejects non-`http(s)` base URL schemes and uses an opener that
|
||||||
|
does not follow redirects, so a misconfigured or compromised runner cannot leak a
|
||||||
|
Bearer token to a redirected host. The release gate reported full pytest passing
|
||||||
|
and independent default-off/inert-path review.
|
||||||
|
|
||||||
This bridge is intentionally a WebUI consumer transport seam: the configured
|
This bridge is intentionally a WebUI consumer transport seam: the configured
|
||||||
runner must emit events that are already compatible with the browser SSE event
|
runner must emit events that are already compatible with the browser SSE event
|
||||||
names/payloads, or a later runner-owned normalization layer must translate
|
names/payloads, or a later runner-owned normalization layer must translate
|
||||||
Hermes runtime families such as `token.delta`, `tool.started`, and `done` before
|
Hermes runtime families such as `token.delta`, `tool.started`, and `done` before
|
||||||
they reach this route.
|
they reach this route.
|
||||||
|
|
||||||
After the route-selection harness ships, the next reviewable step is not to make
|
After the configured runner-client boundary ships, the next reviewable step is
|
||||||
`runner-local` the default. It is to define the first concrete supervised/local
|
not to make `runner-local` the default. It is to define the first supervised
|
||||||
runner client backend that can replace the bounded 501 path under the existing
|
runner process harness that can actually own `AIAgent` execution behind that
|
||||||
feature flag and prove execution ownership has moved out of the main WebUI
|
client boundary and prove restart/reattach with a real local runner, not just a
|
||||||
request process.
|
configured external endpoint or fake-runner fixture.
|
||||||
|
|
||||||
This slice is a contract gate before backend code lands. The goal is to pin the
|
This slice was the client-boundary implementation gate. The goal was to pin the
|
||||||
minimum runner client behavior so the implementation cannot become a renamed
|
minimum runner client behavior so the implementation could not become a renamed
|
||||||
`STREAMS` / `CANCEL_FLAGS` / cached `AIAgent` surrogate inside `api/routes.py`.
|
`STREAMS` / `CANCEL_FLAGS` / cached `AIAgent` surrogate inside `api/routes.py`.
|
||||||
|
|
||||||
Scope:
|
Scope:
|
||||||
@@ -1027,6 +1033,59 @@ Non-goals for Slice 4f:
|
|||||||
- no permanent WebUI-owned active-run discovery cache that duplicates runner or
|
- no permanent WebUI-owned active-run discovery cache that duplicates runner or
|
||||||
future Hermes Runtime API responsibility.
|
future Hermes Runtime API responsibility.
|
||||||
|
|
||||||
|
#### Slice 4g: Supervised local runner process harness gate
|
||||||
|
|
||||||
|
After #3073 / #3274, WebUI has an explicit configured-runner HTTP client and SSE
|
||||||
|
consumer bridge, but it still does not ship the supervised runner process itself.
|
||||||
|
The next gate should define the smallest local runner harness that can own
|
||||||
|
`AIAgent` execution outside the main WebUI request process while being consumed
|
||||||
|
through the already-shipped `runner-local` client boundary.
|
||||||
|
|
||||||
|
Scope:
|
||||||
|
|
||||||
|
- define the local runner process lifecycle: spawn/start, health check, run
|
||||||
|
ownership, graceful shutdown, crash classification, and cleanup;
|
||||||
|
- keep WebUI as a client of `HERMES_WEBUI_RUNNER_BASE_URL`, not the owner of
|
||||||
|
process-local runner execution state;
|
||||||
|
- persist run/session lookup, ordered events, terminal state, and active controls
|
||||||
|
in runner-owned or journal-backed state that a restarted WebUI can discover;
|
||||||
|
- carry explicit profile, workspace, attachments, provider/model, toolset,
|
||||||
|
source, and metadata payloads into the runner without WebUI process-global
|
||||||
|
environment mutation;
|
||||||
|
- prove cancel as the first live control for active runner-owned runs, with
|
||||||
|
approval, clarify, goal, and queue either mapped to explicit runner
|
||||||
|
capabilities or returned as bounded unsupported/conflict `ControlResult`
|
||||||
|
values.
|
||||||
|
|
||||||
|
Acceptance tests for Slice 4g:
|
||||||
|
|
||||||
|
1. **Process ownership moved.** A local runner process, not `hermes-webui`, owns
|
||||||
|
`AIAgent` construction/reuse and active run execution for `runner-local` runs.
|
||||||
|
2. **Restart/reattach with a real runner.** Start a non-trivial `runner-local`
|
||||||
|
run, restart only `hermes-webui`, reload the session, rediscover the active or
|
||||||
|
terminal runner-owned run, replay/catch up from cursor without duplicate
|
||||||
|
transcript/tool/reasoning state, and preserve cancel if still active.
|
||||||
|
3. **No runtime-surrogate globals in WebUI.** The main WebUI server still does not
|
||||||
|
gain new module-level maps for runner-owned streams, cancel flags,
|
||||||
|
approval/clarify callbacks, cached agents, child process run registries, goal
|
||||||
|
state, or queue schedulers.
|
||||||
|
4. **Default-off and reversible.** Unset `HERMES_WEBUI_RUNNER_BASE_URL` or switch
|
||||||
|
the adapter mode back to legacy and the existing in-process path remains
|
||||||
|
available without session or journal migration.
|
||||||
|
5. **Runner health and failure are observable.** A missing, unhealthy, or crashed
|
||||||
|
runner returns bounded diagnostics and terminal/interrupted state rather than
|
||||||
|
silently falling back to WebUI-owned execution for a runner-selected run.
|
||||||
|
|
||||||
|
Non-goals for Slice 4g:
|
||||||
|
|
||||||
|
- no default-on runner mode;
|
||||||
|
- no removal of `legacy-direct` or `legacy-journal`;
|
||||||
|
- no server-side queue scheduler just for adapter symmetry;
|
||||||
|
- no broad WebUI product-surface migration;
|
||||||
|
- no claim that this is the canonical Hermes Agent Runtime API; if Hermes Agent
|
||||||
|
later ships `/v1/runs`, this local runner remains a replaceable backend behind
|
||||||
|
the same adapter/client boundary.
|
||||||
|
|
||||||
## First Meaningful Success Criteria
|
## First Meaningful Success Criteria
|
||||||
|
|
||||||
The first meaningful milestones are deliberately split.
|
The first meaningful milestones are deliberately split.
|
||||||
|
|||||||
@@ -5035,7 +5035,7 @@ async function loadProfilesPanel() {
|
|||||||
const meta = [];
|
const meta = [];
|
||||||
if (p.model) meta.push(p.model.split('/').pop());
|
if (p.model) meta.push(p.model.split('/').pop());
|
||||||
if (p.provider) meta.push(p.provider);
|
if (p.provider) meta.push(p.provider);
|
||||||
if (p.skill_count) meta.push(t('profile_skill_count', p.skill_count));
|
if (p.total_skills && p.total_skills > 0) meta.push(t('profile_skill_count', p.total_skills).replace(String(p.total_skills), `${p.enabled_skills} / ${p.total_skills}`));
|
||||||
const gwDot = p.gateway_running
|
const gwDot = p.gateway_running
|
||||||
? `<span class="profile-opt-badge running" title="${esc(t('profile_gateway_running'))}"></span>`
|
? `<span class="profile-opt-badge running" title="${esc(t('profile_gateway_running'))}"></span>`
|
||||||
: `<span class="profile-opt-badge stopped" title="${esc(t('profile_gateway_stopped'))}"></span>`;
|
: `<span class="profile-opt-badge stopped" title="${esc(t('profile_gateway_stopped'))}"></span>`;
|
||||||
@@ -5109,7 +5109,7 @@ function _renderProfileDetail(p, activeName){
|
|||||||
if (p.provider) rows.push(`<div class="detail-row"><div class="detail-row-label">Provider</div><div class="detail-row-value">${esc(p.provider)}</div></div>`);
|
if (p.provider) rows.push(`<div class="detail-row"><div class="detail-row-label">Provider</div><div class="detail-row-value">${esc(p.provider)}</div></div>`);
|
||||||
if (p.base_url) rows.push(`<div class="detail-row"><div class="detail-row-label">Base URL</div><div class="detail-row-value"><code>${esc(p.base_url)}</code></div></div>`);
|
if (p.base_url) rows.push(`<div class="detail-row"><div class="detail-row-label">Base URL</div><div class="detail-row-value"><code>${esc(p.base_url)}</code></div></div>`);
|
||||||
rows.push(`<div class="detail-row"><div class="detail-row-label">API key</div><div class="detail-row-value">${p.has_env ? esc(t('profile_api_keys_configured')) : '<span style="color:var(--muted)">Not configured</span>'}</div></div>`);
|
rows.push(`<div class="detail-row"><div class="detail-row-label">API key</div><div class="detail-row-value">${p.has_env ? esc(t('profile_api_keys_configured')) : '<span style="color:var(--muted)">Not configured</span>'}</div></div>`);
|
||||||
if (typeof p.skill_count === 'number') rows.push(`<div class="detail-row"><div class="detail-row-label">Skills</div><div class="detail-row-value">${esc(t('profile_skill_count', p.skill_count))}</div></div>`);
|
if (p.total_skills && p.total_skills > 0) rows.push(`<div class="detail-row"><div class="detail-row-label">Skills</div><div class="detail-row-value">${esc(t('profile_skill_count', p.total_skills).replace(String(p.total_skills), `${p.enabled_skills} / ${p.total_skills}`))}</div></div>`);
|
||||||
if (p.default_workspace) rows.push(`<div class="detail-row"><div class="detail-row-label">Default space</div><div class="detail-row-value"><code>${esc(p.default_workspace)}</code></div></div>`);
|
if (p.default_workspace) rows.push(`<div class="detail-row"><div class="detail-row-label">Default space</div><div class="detail-row-value"><code>${esc(p.default_workspace)}</code></div></div>`);
|
||||||
body.innerHTML = `
|
body.innerHTML = `
|
||||||
<div class="main-view-content">
|
<div class="main-view-content">
|
||||||
@@ -5199,7 +5199,7 @@ function renderProfileDropdown(data) {
|
|||||||
opt.className = 'profile-opt' + (p.name === active ? ' active' : '');
|
opt.className = 'profile-opt' + (p.name === active ? ' active' : '');
|
||||||
const meta = [];
|
const meta = [];
|
||||||
if (p.model) meta.push(p.model.split('/').pop());
|
if (p.model) meta.push(p.model.split('/').pop());
|
||||||
if (p.skill_count) meta.push(t('profile_skill_count', p.skill_count));
|
if (p.total_skills && p.total_skills > 0) meta.push(t('profile_skill_count', p.total_skills).replace(String(p.total_skills), `${p.enabled_skills} / ${p.total_skills}`));
|
||||||
const gwDot = `<span class="profile-opt-badge ${p.gateway_running ? 'running' : 'stopped'}"></span>`;
|
const gwDot = `<span class="profile-opt-badge ${p.gateway_running ? 'running' : 'stopped'}"></span>`;
|
||||||
const checkmark = p.name === active ? ' <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="var(--link)" stroke-width="3" style="vertical-align:-1px"><polyline points="20 6 9 17 4 12"/></svg>' : '';
|
const checkmark = p.name === active ? ' <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="var(--link)" stroke-width="3" style="vertical-align:-1px"><polyline points="20 6 9 17 4 12"/></svg>' : '';
|
||||||
const defaultBadge = p.is_default ? ` <span style="opacity:.5;font-weight:400">${esc(t('profile_default_label'))}</span>` : '';
|
const defaultBadge = p.is_default ? ` <span style="opacity:.5;font-weight:400">${esc(t('profile_default_label'))}</span>` : '';
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ these combinations, hiding the reasoning effort selector in the UI even though
|
|||||||
the underlying models fully support thinking/reasoning.
|
the underlying models fully support thinking/reasoning.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
import api.config as cfg
|
import api.config as cfg
|
||||||
|
|
||||||
|
|
||||||
@@ -33,6 +35,26 @@ def test_deepseek_r1_bare_name_custom_provider():
|
|||||||
assert set(efforts) >= {"low", "medium", "high"}
|
assert set(efforts) >= {"low", "medium", "high"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"model_id",
|
||||||
|
[
|
||||||
|
"deepseek.v3.2",
|
||||||
|
"deepseek_v3_2",
|
||||||
|
"vendor.deepseek.v3.2",
|
||||||
|
"deepseek.v4-flash",
|
||||||
|
"deepseek_v4_flash",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_deepseek_separator_variants_custom_provider(model_id):
|
||||||
|
efforts = cfg.resolve_model_reasoning_efforts(
|
||||||
|
model_id,
|
||||||
|
provider_id="custom:newapi",
|
||||||
|
)
|
||||||
|
assert set(efforts) >= {"low", "medium", "high"}, (
|
||||||
|
f"{model_id} via custom provider should expose reasoning efforts"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# ── dot-separated model names (vendor.model) ─────────────────────────────────
|
# ── dot-separated model names (vendor.model) ─────────────────────────────────
|
||||||
|
|
||||||
def test_kimi_dot_separated_custom_provider():
|
def test_kimi_dot_separated_custom_provider():
|
||||||
@@ -91,6 +113,20 @@ def test_plain_llm_dot_separated_custom_provider_no_reasoning():
|
|||||||
) == []
|
) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"model_id",
|
||||||
|
[
|
||||||
|
"thinkinghub.llama-3.1-70b",
|
||||||
|
"reasoninghub.llama-3.1-70b",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_vendor_prefix_keyword_does_not_trigger_reasoning(model_id):
|
||||||
|
assert cfg.resolve_model_reasoning_efforts(
|
||||||
|
model_id,
|
||||||
|
provider_id="custom:newapi",
|
||||||
|
) == []
|
||||||
|
|
||||||
|
|
||||||
# ── slash-prefixed names must still work (no regression) ─────────────────────
|
# ── slash-prefixed names must still work (no regression) ─────────────────────
|
||||||
|
|
||||||
def test_deepseek_slash_prefix_still_works():
|
def test_deepseek_slash_prefix_still_works():
|
||||||
|
|||||||
142
tests/test_profile_skills_stats.py
Normal file
142
tests/test_profile_skills_stats.py
Normal file
@@ -0,0 +1,142 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
import yaml
|
||||||
|
from api import profiles
|
||||||
|
from tests.conftest import requires_agent_modules
|
||||||
|
|
||||||
|
|
||||||
|
def _write_skill(root: Path, name: str, platforms=None):
|
||||||
|
skill_dir = root / "skills" / name
|
||||||
|
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
platforms_line = f"platforms: {platforms}\n" if platforms else ""
|
||||||
|
(skill_dir / "SKILL.md").write_text(
|
||||||
|
f"---\nname: {name}\ndescription: {name} skill\n{platforms_line}---\n\n# {name}\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _write_config(home: Path, disabled):
|
||||||
|
home.mkdir(parents=True, exist_ok=True)
|
||||||
|
(home / "config.yaml").write_text(
|
||||||
|
yaml.safe_dump({"skills": {"disabled": list(disabled)}}, sort_keys=False),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
@requires_agent_modules
|
||||||
|
def test_get_profile_skills_stats(tmp_path):
|
||||||
|
# Setup skills directory with:
|
||||||
|
# 1 compatible & enabled skill ("alpha")
|
||||||
|
# 1 compatible & disabled skill ("beta")
|
||||||
|
# 1 incompatible skill ("gamma" - macos only on linux test run)
|
||||||
|
profile_home = tmp_path / "auditor"
|
||||||
|
_write_skill(profile_home, "alpha")
|
||||||
|
_write_skill(profile_home, "beta")
|
||||||
|
_write_skill(profile_home, "gamma", platforms=["macos"])
|
||||||
|
_write_config(profile_home, ["beta"])
|
||||||
|
|
||||||
|
# Explicitly clear the stats cache to ensure we compute fresh
|
||||||
|
profiles._SKILLS_STATS_CACHE.clear()
|
||||||
|
|
||||||
|
enabled, compatible = profiles._get_profile_skills_stats(profile_home)
|
||||||
|
assert enabled == 1
|
||||||
|
assert compatible == 2
|
||||||
|
|
||||||
|
@requires_agent_modules
|
||||||
|
def test_list_profiles_api_contains_formatted_skills(monkeypatch, tmp_path):
|
||||||
|
class FakeProfile:
|
||||||
|
def __init__(self, name, path):
|
||||||
|
self.name = name
|
||||||
|
self.path = Path(path)
|
||||||
|
self.is_default = name == "default"
|
||||||
|
self.gateway_running = False
|
||||||
|
self.model = "gpt-4"
|
||||||
|
self.provider = "openai"
|
||||||
|
self.has_env = False
|
||||||
|
self.skill_count = 3 # Raw on-disk count from hermes_cli
|
||||||
|
|
||||||
|
p_default = tmp_path / "default"
|
||||||
|
p_fintech = tmp_path / "profiles" / "fintech"
|
||||||
|
|
||||||
|
_write_skill(p_default, "a1")
|
||||||
|
_write_skill(p_default, "a2")
|
||||||
|
_write_config(p_default, ["a2"])
|
||||||
|
|
||||||
|
_write_skill(p_fintech, "f1")
|
||||||
|
_write_skill(p_fintech, "f2")
|
||||||
|
_write_skill(p_fintech, "f3")
|
||||||
|
_write_config(p_fintech, ["f2", "f3"])
|
||||||
|
|
||||||
|
fake_infos = [
|
||||||
|
FakeProfile("default", p_default),
|
||||||
|
FakeProfile("fintech", p_fintech)
|
||||||
|
]
|
||||||
|
|
||||||
|
monkeypatch.setattr(profiles, "get_active_profile_name", lambda: "default")
|
||||||
|
|
||||||
|
# We patch the upstream list_profiles call to return our FakeProfile list
|
||||||
|
try:
|
||||||
|
import hermes_cli.profiles as cli_p
|
||||||
|
monkeypatch.setattr(cli_p, "list_profiles", lambda: fake_infos)
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
profiles._SKILLS_STATS_CACHE.clear()
|
||||||
|
|
||||||
|
results = profiles.list_profiles_api()
|
||||||
|
by_name = {p["name"]: p for p in results}
|
||||||
|
|
||||||
|
assert "default" in by_name
|
||||||
|
assert "fintech" in by_name
|
||||||
|
|
||||||
|
# backward-compatible skill_count as an integer:
|
||||||
|
assert by_name["default"]["skill_count"] == 1
|
||||||
|
assert by_name["fintech"]["skill_count"] == 1
|
||||||
|
|
||||||
|
# new enabled_skills and total_skills integer fields:
|
||||||
|
assert by_name["default"]["enabled_skills"] == 1
|
||||||
|
assert by_name["default"]["total_skills"] == 2
|
||||||
|
assert by_name["fintech"]["enabled_skills"] == 1
|
||||||
|
assert by_name["fintech"]["total_skills"] == 3
|
||||||
|
|
||||||
|
@requires_agent_modules
|
||||||
|
def test_no_skills_dir(tmp_path):
|
||||||
|
"""Profile with no skills/ directory should return (0, 0)."""
|
||||||
|
profiles._SKILLS_STATS_CACHE.clear()
|
||||||
|
enabled, compat = profiles._get_profile_skills_stats(tmp_path)
|
||||||
|
assert enabled == 0 and compat == 0
|
||||||
|
|
||||||
|
@requires_agent_modules
|
||||||
|
def test_corrupt_config(tmp_path):
|
||||||
|
"""Corrupt config.yaml should not crash — disabled set stays empty."""
|
||||||
|
profiles._SKILLS_STATS_CACHE.clear()
|
||||||
|
_write_skill(tmp_path, "a")
|
||||||
|
(tmp_path / "config.yaml").write_text("not: [valid: yaml: {{", encoding="utf-8")
|
||||||
|
enabled, compat = profiles._get_profile_skills_stats(tmp_path)
|
||||||
|
assert compat == 1 and enabled == 1 # no disabled parsing, all enabled
|
||||||
|
|
||||||
|
@requires_agent_modules
|
||||||
|
def test_platform_disabled_webui(tmp_path):
|
||||||
|
"""platform_disabled.webui list should be used when present."""
|
||||||
|
profiles._SKILLS_STATS_CACHE.clear()
|
||||||
|
_write_skill(tmp_path, "web-only")
|
||||||
|
cfg = {"skills": {"platform_disabled": {"webui": ["web-only"]}, "disabled": []}}
|
||||||
|
(tmp_path / "config.yaml").write_text(yaml.safe_dump(cfg), encoding="utf-8")
|
||||||
|
enabled, compat = profiles._get_profile_skills_stats(tmp_path)
|
||||||
|
assert compat == 1 and enabled == 0
|
||||||
|
|
||||||
|
@requires_agent_modules
|
||||||
|
def test_skills_stats_cache(tmp_path):
|
||||||
|
"""Verify that caching works and has short TTL behavior."""
|
||||||
|
profiles._SKILLS_STATS_CACHE.clear()
|
||||||
|
|
||||||
|
_write_skill(tmp_path, "alpha")
|
||||||
|
enabled, compat = profiles._get_profile_skills_stats(tmp_path)
|
||||||
|
assert enabled == 1 and compat == 1
|
||||||
|
|
||||||
|
# Add a skill but since cache is active (TTL 8s), we should still get old values
|
||||||
|
_write_skill(tmp_path, "beta")
|
||||||
|
enabled, compat = profiles._get_profile_skills_stats(tmp_path)
|
||||||
|
assert enabled == 1 and compat == 1
|
||||||
|
|
||||||
|
# Force clear or override the mock TTL, or clear cache manually to see changes
|
||||||
|
profiles._SKILLS_STATS_CACHE.clear()
|
||||||
|
enabled, compat = profiles._get_profile_skills_stats(tmp_path)
|
||||||
|
assert enabled == 2 and compat == 2
|
||||||
@@ -574,12 +574,14 @@ def test_rfc_defines_slice4f_supervised_local_runner_client_gate():
|
|||||||
rfc = (routes.Path(__file__).parent.parent / "docs" / "rfcs" / "hermes-run-adapter-contract.md").read_text(encoding="utf-8")
|
rfc = (routes.Path(__file__).parent.parent / "docs" / "rfcs" / "hermes-run-adapter-contract.md").read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert "#### Slice 4f: Supervised local runner client backend gate" in rfc
|
assert "#### Slice 4f: Supervised local runner client backend gate" in rfc
|
||||||
assert "Status as of 2026-05-28: client transport proposed in #3073 behind" in rfc
|
assert "Status as of 2026-05-31: shipped in v0.51.188 via #3073 / #3274" in rfc
|
||||||
assert "it should be described as under review until a\nrelease PR actually ships it" in rfc
|
assert "The client\ntransport is now implemented behind `HERMES_WEBUI_RUNNER_BASE_URL`" in rfc
|
||||||
assert "replace the bounded 501 path under the existing\nfeature flag" in rfc
|
assert "`HttpRunnerClient` rejects non-`http(s)` base URL schemes" in rfc
|
||||||
assert "durable runner-owned run id plus session-to-run lookup" in rfc
|
assert "uses an opener that\ndoes not follow redirects" in rfc
|
||||||
assert "the configured\nrunner must emit events that are already compatible with the browser SSE event\nnames/payloads" in rfc
|
assert "the configured\nrunner must emit events that are already compatible with the browser SSE event\nnames/payloads" in rfc
|
||||||
assert "a later runner-owned normalization layer must translate\nHermes runtime families such as `token.delta`, `tool.started`, and `done`" in rfc
|
assert "a later runner-owned normalization layer must translate\nHermes runtime families such as `token.delta`, `tool.started`, and `done`" in rfc
|
||||||
|
assert "After the configured runner-client boundary ships" in rfc
|
||||||
|
assert "configured external endpoint or fake-runner fixture" in rfc
|
||||||
assert "cancel as the first required live control" in rfc
|
assert "cancel as the first required live control" in rfc
|
||||||
assert "501 path replaced only when configured" in rfc
|
assert "501 path replaced only when configured" in rfc
|
||||||
assert "Restart/reattach proves ownership moved" in rfc
|
assert "Restart/reattach proves ownership moved" in rfc
|
||||||
@@ -588,6 +590,24 @@ def test_rfc_defines_slice4f_supervised_local_runner_client_gate():
|
|||||||
assert "Unsupported runner controls return safe\n `unsupported`, `not-active`, or `conflict` results" in rfc
|
assert "Unsupported runner controls return safe\n `unsupported`, `not-active`, or `conflict` results" in rfc
|
||||||
assert "no permanent WebUI-owned active-run discovery cache" in rfc
|
assert "no permanent WebUI-owned active-run discovery cache" in rfc
|
||||||
|
|
||||||
|
|
||||||
|
def test_rfc_defines_slice4g_supervised_local_runner_process_gate():
|
||||||
|
routes = importlib.import_module("api.routes")
|
||||||
|
rfc = (routes.Path(__file__).parent.parent / "docs" / "rfcs" / "hermes-run-adapter-contract.md").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "#### Slice 4g: Supervised local runner process harness gate" in rfc
|
||||||
|
assert "After #3073 / #3274, WebUI has an explicit configured-runner HTTP client" in rfc
|
||||||
|
assert "still does not ship the supervised runner process itself" in rfc
|
||||||
|
assert "own\n`AIAgent` execution outside the main WebUI request process" in rfc
|
||||||
|
assert "keep WebUI as a client of `HERMES_WEBUI_RUNNER_BASE_URL`" in rfc
|
||||||
|
assert "without WebUI process-global\n environment mutation" in rfc
|
||||||
|
assert "Process ownership moved" in rfc
|
||||||
|
assert "Restart/reattach with a real runner" in rfc
|
||||||
|
assert "No runtime-surrogate globals in WebUI" in rfc
|
||||||
|
assert "Default-off and reversible" in rfc
|
||||||
|
assert "Runner health and failure are observable" in rfc
|
||||||
|
assert "no claim that this is the canonical Hermes Agent Runtime API" in rfc
|
||||||
|
|
||||||
def test_runtime_runner_client_factory_stays_bounded_until_endpoint_configured(monkeypatch):
|
def test_runtime_runner_client_factory_stays_bounded_until_endpoint_configured(monkeypatch):
|
||||||
routes = importlib.import_module("api.routes")
|
routes = importlib.import_module("api.routes")
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user