fix: show TPS in assistant message headers
This commit is contained in:
Michael Lam
@@ -1,17 +1,17 @@
|
||||
"""
|
||||
Hermes Web UI -- Streaming performance metering.
|
||||
|
||||
Tracks Tokens Per Second (TPS) across all active WebUI sessions, and the
|
||||
HIGH/LOW TPS values observed over the past 60 minutes. Metering data is
|
||||
emitted via SSE events so the header label can update live during a stream.
|
||||
Tracks Tokens Per Second (TPS) across active WebUI streams. Metering data is
|
||||
emitted via SSE events so a streaming assistant message can update its own
|
||||
header while the turn is running.
|
||||
|
||||
Architecture
|
||||
────────────
|
||||
Each streaming session is tracked independently. TPS per session is:
|
||||
Each streaming session is tracked independently. TPS per stream is:
|
||||
|
||||
session_tps = total_tokens / (last_token_ts - first_token_ts)
|
||||
stream_tps = total_stream_deltas / (last_delta_ts - first_delta_ts)
|
||||
|
||||
The global tps is the average of all currently active sessions' TPS values.
|
||||
The global tps is the average of all currently active streams' TPS values.
|
||||
This correctly represents the system's real-time capacity regardless of how
|
||||
many sessions are running or how long each has been streaming.
|
||||
|
||||
@@ -19,8 +19,8 @@ For HIGH/LOW tracking, every stats snapshot records the current global tps
|
||||
(only when > 0 — idle periods are skipped) into a rolling 60-minute history.
|
||||
The max/min of that history gives the peak throughput observed over the past hour.
|
||||
|
||||
The ticker in streaming.py calls get_interval() — it returns 1.0 when sessions
|
||||
are actively receiving tokens so the header updates at 1 Hz, and 10.0 when idle
|
||||
The ticker in streaming.py calls get_interval() — it returns 1.0 when streams
|
||||
are actively receiving output deltas so message headers update at 1 Hz, and 10.0 when idle
|
||||
so the ticker exits and no idle readings are emitted.
|
||||
|
||||
Usage from api/streaming.py
|
||||
@@ -28,15 +28,17 @@ Usage from api/streaming.py
|
||||
from api.metering import meter
|
||||
|
||||
meter().begin_session(stream_id) # stream starts
|
||||
meter().record_token(stream_id, running_output) # per output token
|
||||
meter().record_reasoning(stream_id, running_reasoning_len) # per reasoning token
|
||||
meter().record_token(stream_id, running_output_deltas)
|
||||
meter().record_reasoning(stream_id, running_reasoning_deltas)
|
||||
|
||||
The SSE `metering` event payload:
|
||||
{
|
||||
"tps": 47.3, # average TPS across active sessions (real-time)
|
||||
"high": 52.1, # highest average TPS observed in the past 60 minutes
|
||||
"low": 31.4, # lowest average TPS (excl. readings < 1 tps, to ignore idle)
|
||||
"active": 1, # sessions currently streaming
|
||||
"tps": 47.3, # omitted/null until a real reading exists
|
||||
"tps_available": true, # frontend must hide TPS when false
|
||||
"estimated": false, # never show byte/character-size estimates
|
||||
"high": 52.1,
|
||||
"low": 31.4,
|
||||
"active": 1,
|
||||
}
|
||||
"""
|
||||
|
||||
@@ -60,9 +62,9 @@ class _SessionMeter:
|
||||
def total_tokens(self) -> int:
|
||||
return self.output_tokens + self.reasoning_tokens
|
||||
|
||||
def tps(self) -> float:
|
||||
def tps(self) -> float | None:
|
||||
if self.first_token_ts == 0.0 or self.last_token_ts <= self.first_token_ts:
|
||||
return 0.0
|
||||
return None
|
||||
return self.total_tokens() / (self.last_token_ts - self.first_token_ts)
|
||||
|
||||
|
||||
@@ -148,12 +150,15 @@ class GlobalMeter:
|
||||
if not self._sessions:
|
||||
self._window_start = now
|
||||
|
||||
# Compute global tps: average of per-session TPS values
|
||||
# Compute global tps: average only streams with a real reading. The
|
||||
# UI hides TPS entirely when this is unavailable instead of showing
|
||||
# placeholder/estimated values.
|
||||
active = [s for s in self._sessions.values() if s.first_token_ts > 0]
|
||||
if active:
|
||||
global_tps = sum(s.tps() for s in active) / len(active)
|
||||
active_tps = [v for s in active for v in [s.tps()] if v is not None and v > 0]
|
||||
if active_tps:
|
||||
global_tps = sum(active_tps) / len(active_tps)
|
||||
else:
|
||||
global_tps = 0.0
|
||||
global_tps = None
|
||||
|
||||
# Prune readings older than 1 hour
|
||||
cutoff = now - _HOUR_SECS
|
||||
@@ -162,7 +167,7 @@ class GlobalMeter:
|
||||
# Only record this snapshot for HIGH/LOW if there is active work.
|
||||
# This prevents idle periods from flooding the history and keeps
|
||||
# HIGH/LOW meaningful for the past hour of actual throughput.
|
||||
if global_tps > 0:
|
||||
if global_tps is not None and global_tps > 0:
|
||||
self._readings.append((now, global_tps))
|
||||
|
||||
# HIGH/LOW from the past hour (skip near-zero idle readings)
|
||||
@@ -171,9 +176,11 @@ class GlobalMeter:
|
||||
low = min(active_readings) if active_readings else 0.0
|
||||
|
||||
return {
|
||||
'tps': round(global_tps, 1),
|
||||
'high': round(high, 1),
|
||||
'low': round(low, 1),
|
||||
'tps': round(global_tps, 1) if global_tps is not None else None,
|
||||
'tps_available': global_tps is not None,
|
||||
'estimated': False,
|
||||
'high': round(high, 1) if high else None,
|
||||
'low': round(low, 1) if low else None,
|
||||
'active': len(self._sessions),
|
||||
}
|
||||
|
||||
|
||||
@@ -1620,9 +1620,11 @@ def _run_agent_streaming(
|
||||
_reasoning_text = '' # accumulates reasoning/thinking trace for persistence
|
||||
_live_tool_calls = [] # tool progress fallback when final messages omit tool IDs
|
||||
|
||||
# Throttle: emit metering events at most every 100 ms so the TPS label
|
||||
# feels live during fast token streams without flooding the SSE channel.
|
||||
# Throttle: emit metering events at most every 100 ms so the per-message
|
||||
# TPS label feels live during fast token streams without flooding SSE.
|
||||
_metering_last_emit = [time.monotonic() - 1] # fire immediately on first token
|
||||
_metering_output_deltas = [0]
|
||||
_metering_reasoning_deltas = [0]
|
||||
|
||||
def _emit_metering():
|
||||
now = time.monotonic()
|
||||
@@ -1631,6 +1633,8 @@ def _run_agent_streaming(
|
||||
_metering_last_emit[0] = now
|
||||
stats = meter().get_stats()
|
||||
stats['session_id'] = stream_id
|
||||
stats.setdefault('tps_available', False)
|
||||
stats.setdefault('estimated', False)
|
||||
put('metering', stats)
|
||||
|
||||
def on_token(text):
|
||||
@@ -1642,8 +1646,11 @@ def _run_agent_streaming(
|
||||
if stream_id in STREAM_PARTIAL_TEXT:
|
||||
STREAM_PARTIAL_TEXT[stream_id] += str(text)
|
||||
put('token', {'text': text})
|
||||
# Update global throughput meter
|
||||
meter().record_token(stream_id, len(STREAM_PARTIAL_TEXT[stream_id]))
|
||||
# Update live throughput from stream delta callbacks, not from
|
||||
# byte/character length. If a backend cannot provide live deltas,
|
||||
# the frontend hides TPS rather than showing an estimate.
|
||||
_metering_output_deltas[0] += 1
|
||||
meter().record_token(stream_id, _metering_output_deltas[0])
|
||||
_emit_metering()
|
||||
|
||||
def on_reasoning(text):
|
||||
@@ -1655,8 +1662,9 @@ def _run_agent_streaming(
|
||||
if stream_id in STREAM_REASONING_TEXT:
|
||||
STREAM_REASONING_TEXT[stream_id] += str(text)
|
||||
put('reasoning', {'text': str(text)})
|
||||
# Track reasoning tokens in the meter so TPS reflects all AI output
|
||||
meter().record_reasoning(stream_id, len(_reasoning_text))
|
||||
# Track reasoning deltas in the meter so live TPS reflects all AI output.
|
||||
_metering_reasoning_deltas[0] += 1
|
||||
meter().record_reasoning(stream_id, _metering_reasoning_deltas[0])
|
||||
_emit_metering()
|
||||
|
||||
# Pre-initialise the activity counter here so on_tool (which
|
||||
@@ -1690,7 +1698,8 @@ def _run_agent_streaming(
|
||||
if stream_id in STREAM_REASONING_TEXT:
|
||||
STREAM_REASONING_TEXT[stream_id] += str(reason_text)
|
||||
put('reasoning', {'text': str(reason_text)})
|
||||
meter().record_reasoning(stream_id, len(_reasoning_text))
|
||||
_metering_reasoning_deltas[0] += 1
|
||||
meter().record_reasoning(stream_id, _metering_reasoning_deltas[0])
|
||||
_emit_metering()
|
||||
return
|
||||
|
||||
@@ -2459,10 +2468,15 @@ def _run_agent_streaming(
|
||||
_turn_duration_seconds = max(0.0, time.time() - float(_turn_started_at))
|
||||
except Exception:
|
||||
_turn_duration_seconds = 0.0
|
||||
_turn_tps = None
|
||||
if output_tokens and _turn_duration_seconds > 0:
|
||||
_turn_tps = round(float(output_tokens) / _turn_duration_seconds, 1)
|
||||
if s.messages:
|
||||
for _dm in reversed(s.messages):
|
||||
if isinstance(_dm, dict) and _dm.get('role') == 'assistant':
|
||||
_dm['_turnDuration'] = round(_turn_duration_seconds, 3)
|
||||
if _turn_tps is not None:
|
||||
_dm['_turnTps'] = _turn_tps
|
||||
break
|
||||
# Persist context window data on the session so the context-ring
|
||||
# indicator survives a page reload (#1318). Must run BEFORE
|
||||
@@ -2517,6 +2531,8 @@ def _run_agent_streaming(
|
||||
'estimated_cost': estimated_cost,
|
||||
'duration_seconds': round(_turn_duration_seconds, 3),
|
||||
}
|
||||
if _turn_tps is not None:
|
||||
usage['tps'] = _turn_tps
|
||||
# Include context window data from the agent's compressor for the UI indicator.
|
||||
# The session-level persistence happens above (before s.save()) so the values
|
||||
# survive a page reload; this block only populates the live SSE usage payload.
|
||||
@@ -2567,9 +2583,11 @@ def _run_agent_streaming(
|
||||
logger.debug("Failed to drain pending steer for session %s", session_id)
|
||||
raw_session = s.compact() | {'messages': s.messages, 'tool_calls': tool_calls}
|
||||
put('done', {'session': redact_session_data(raw_session), 'usage': usage})
|
||||
# Emit metering stats for the header TPS label
|
||||
# Emit one last metering packet for the live message-header TPS label.
|
||||
meter_stats = meter().get_stats()
|
||||
meter_stats['session_id'] = session_id
|
||||
meter_stats.setdefault('tps_available', False)
|
||||
meter_stats.setdefault('estimated', False)
|
||||
put('metering', meter_stats)
|
||||
if _should_bg_title and _u0 and _a0:
|
||||
threading.Thread(
|
||||
|
||||
BIN
docs/pr-media/issue-1617/after.png
Normal file
BIN
docs/pr-media/issue-1617/after.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 41 KiB |
BIN
docs/pr-media/issue-1617/before.png
Normal file
BIN
docs/pr-media/issue-1617/before.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 40 KiB |
@@ -903,6 +903,9 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
|
||||
if(typeof d.usage.duration_seconds==='number'){
|
||||
lastAsst._turnDuration=d.usage.duration_seconds;
|
||||
}
|
||||
if(typeof d.usage.tps==='number'&&d.usage.tps>0){
|
||||
lastAsst._turnTps=d.usage.tps;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(d.session.tool_calls&&d.session.tool_calls.length){
|
||||
@@ -984,16 +987,14 @@ function attachLiveStream(activeSid, streamId, uploaded=[], options={}){
|
||||
});
|
||||
|
||||
source.addEventListener('metering',e=>{
|
||||
// TPS + HIGH/LOW stats for the header chip — emitted at 1 Hz during a stream,
|
||||
// silenced entirely when no sessions are active (ticker exits when idle).
|
||||
try{
|
||||
const d=JSON.parse(e.data||'{}');
|
||||
const el=$('tpsStat');
|
||||
if(!el) return;
|
||||
const tps=typeof d.tps==='number'?d.tps.toFixed(1):'0.0';
|
||||
const high=typeof d.high==='number' && d.high>=0?d.high.toFixed(1)+' high':'—';
|
||||
const low=typeof d.low==='number' && d.low>=0?d.low.toFixed(1)+' low':'';
|
||||
el.textContent=`${tps} t/s · ${high}${low?' · '+low:''}`;
|
||||
if((d.session_id||activeSid)!==activeSid) return;
|
||||
if(d.estimated===true||d.tps_available!==true||typeof d.tps!=='number'||d.tps<=0){
|
||||
if(typeof _setLiveAssistantTps==='function') _setLiveAssistantTps(null);
|
||||
return;
|
||||
}
|
||||
if(typeof _setLiveAssistantTps==='function') _setLiveAssistantTps(d.tps);
|
||||
}catch(_){}
|
||||
});
|
||||
|
||||
|
||||
@@ -2665,6 +2665,20 @@ main.main.showing-profiles > #mainProfiles{display:flex;}
|
||||
.msg-role { font-size: 11px; font-weight: 500; margin-bottom: 6px; opacity: .8; letter-spacing: 0; }
|
||||
.msg-role:hover { opacity: 1; }
|
||||
.role-icon { width: 20px; height: 20px; font-size: 9px; }
|
||||
.msg-tps-inline {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
margin-left: 2px;
|
||||
padding: 1px 6px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 999px;
|
||||
color: var(--muted);
|
||||
background: var(--surface);
|
||||
font-size: 10.5px;
|
||||
font-weight: 500;
|
||||
font-variant-numeric: tabular-nums;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.msg-time { opacity: .65; font-size: 10px; }
|
||||
.msg-role:hover .msg-time { opacity: 1; }
|
||||
|
||||
|
||||
37
static/ui.js
37
static/ui.js
@@ -3179,15 +3179,40 @@ function _messageHasReasoningPayload(m){
|
||||
if(Array.isArray(m.content)) return m.content.some(p=>p&&(p.type==='thinking'||p.type==='reasoning'));
|
||||
return /<think>[\s\S]*?<\/think>|<\|channel>thought\n[\s\S]*?<channel\|>|<\|turn\|>thinking\n[\s\S]*?<turn\|>/.test(String(m.content||''));
|
||||
}
|
||||
function _assistantRoleHtml(tsTitle=''){
|
||||
const _bn=window._botName||'Hermes';
|
||||
return `<div class="msg-role assistant" ${tsTitle?`title="${esc(tsTitle)}"`:''}><div class="role-icon assistant">${esc(_bn.charAt(0).toUpperCase())}</div><span style="font-size:12px">${esc(_bn)}</span></div>`;
|
||||
function _formatTurnTps(value){
|
||||
const n=Number(value);
|
||||
if(!Number.isFinite(n)||n<=0) return '';
|
||||
const fixed=n>=100?Math.round(n).toLocaleString():n>=10?n.toFixed(1):n.toFixed(1);
|
||||
return `${fixed} t/s`;
|
||||
}
|
||||
function _createAssistantTurn(tsTitle=''){
|
||||
function _assistantRoleHtml(tsTitle='', tpsText=''){
|
||||
const _bn=window._botName||'Hermes';
|
||||
const tps=tpsText?`<span class="msg-tps-inline" title="Tokens per second">${esc(tpsText)}</span>`:'';
|
||||
return `<div class="msg-role assistant" ${tsTitle?`title="${esc(tsTitle)}"`:''}><div class="role-icon assistant">${esc(_bn.charAt(0).toUpperCase())}</div><span style="font-size:12px">${esc(_bn)}</span>${tps}</div>`;
|
||||
}
|
||||
function _setAssistantTurnTps(turn, tpsText=''){
|
||||
if(!turn) return;
|
||||
const role=turn.querySelector('.msg-role.assistant');
|
||||
if(!role) return;
|
||||
let chip=role.querySelector('.msg-tps-inline');
|
||||
const text=String(tpsText||'').trim();
|
||||
if(!text){if(chip) chip.remove();return;}
|
||||
if(!chip){
|
||||
chip=document.createElement('span');
|
||||
chip.className='msg-tps-inline';
|
||||
chip.title='Tokens per second';
|
||||
role.appendChild(chip);
|
||||
}
|
||||
chip.textContent=text;
|
||||
}
|
||||
function _setLiveAssistantTps(value){
|
||||
_setAssistantTurnTps($('liveAssistantTurn'), _formatTurnTps(value));
|
||||
}
|
||||
function _createAssistantTurn(tsTitle='', tpsText=''){
|
||||
const row=document.createElement('div');
|
||||
row.className='msg-row assistant-turn';
|
||||
row.dataset.role='assistant';
|
||||
row.innerHTML=`${_assistantRoleHtml(tsTitle)}<div class="assistant-turn-blocks"></div>`;
|
||||
row.innerHTML=`${_assistantRoleHtml(tsTitle, tpsText)}<div class="assistant-turn-blocks"></div>`;
|
||||
return row;
|
||||
}
|
||||
function _assistantTurnBlocks(turn){
|
||||
@@ -3837,7 +3862,7 @@ function renderMessages(){
|
||||
}
|
||||
|
||||
if(!currentAssistantTurn){
|
||||
currentAssistantTurn=_createAssistantTurn(tsTitle);
|
||||
currentAssistantTurn=_createAssistantTurn(tsTitle, _formatTurnTps(m._turnTps));
|
||||
inner.appendChild(currentAssistantTurn);
|
||||
}
|
||||
const seg=document.createElement('div');
|
||||
|
||||
59
tests/test_issue1617_tps_message_header.py
Normal file
59
tests/test_issue1617_tps_message_header.py
Normal file
@@ -0,0 +1,59 @@
|
||||
"""Regression coverage for issue #1617: TPS belongs on message headers.
|
||||
|
||||
Product decision:
|
||||
- show live TPS in the assistant message header while streaming when real TPS is available;
|
||||
- persist/show the final TPS at the end of the turn;
|
||||
- do not show placeholder or estimated TPS when unavailable.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
STREAMING_PY = (REPO / "api" / "streaming.py").read_text(encoding="utf-8")
|
||||
MESSAGES_JS = (REPO / "static" / "messages.js").read_text(encoding="utf-8")
|
||||
UI_JS = (REPO / "static" / "ui.js").read_text(encoding="utf-8")
|
||||
CSS = (REPO / "static" / "style.css").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_tps_renders_in_message_header_not_global_titlebar():
|
||||
assert "msg-tps-inline" in UI_JS, "assistant message headers need a TPS chip hook"
|
||||
assert "msg-tps-inline" in CSS, "TPS header chip needs an explicit CSS hook"
|
||||
assert "_assistantRoleHtml(tsTitle='', tpsText='')" in UI_JS, (
|
||||
"assistant role/header rendering should accept the per-message TPS text"
|
||||
)
|
||||
assert "_formatTurnTps" in UI_JS, "TPS formatting should be centralized"
|
||||
assert "_turnTps" in UI_JS, "settled assistant messages should render final TPS from message metadata"
|
||||
assert "tpsStat" not in MESSAGES_JS, "live TPS must not target the removed/global titlebar chip"
|
||||
|
||||
|
||||
def test_live_metering_updates_only_real_tps_and_never_placeholders():
|
||||
listener_start = MESSAGES_JS.find("source.addEventListener('metering'")
|
||||
assert listener_start != -1, "messages.js should listen for metering SSE events"
|
||||
listener_end = MESSAGES_JS.find("source.addEventListener('apperror'", listener_start)
|
||||
assert listener_end != -1, "apperror listener should follow metering listener"
|
||||
listener = MESSAGES_JS[listener_start:listener_end]
|
||||
assert "_setLiveAssistantTps" in listener, "live metering should update the live assistant header"
|
||||
assert "tps_available" in listener and "estimated" in listener, (
|
||||
"live TPS display must check availability and reject estimated readings"
|
||||
)
|
||||
assert "0.0 t/s" not in listener, "unavailable TPS should render nothing, not a 0.0 placeholder"
|
||||
assert "'—'" not in listener and '"—"' not in listener, "unavailable TPS should render nothing, not a dash"
|
||||
assert "high" not in listener.lower() and "low" not in listener.lower(), (
|
||||
"message-header TPS should not carry global HIGH/LOW titlebar semantics"
|
||||
)
|
||||
|
||||
|
||||
def test_done_payload_persists_final_tps_when_exact_usage_available():
|
||||
assert "usage['tps']" in STREAMING_PY, "done usage payload should include final exact TPS when available"
|
||||
assert "output_tokens" in STREAMING_PY and "duration_seconds" in STREAMING_PY, (
|
||||
"final TPS should be based on exact completion tokens over measured turn duration"
|
||||
)
|
||||
assert "d.usage.tps" in MESSAGES_JS, "done handler should read final TPS from the usage payload"
|
||||
assert "lastAsst._turnTps" in MESSAGES_JS, "done handler should persist final TPS on the last assistant message"
|
||||
|
||||
|
||||
def test_backend_marks_streaming_metering_availability_explicitly():
|
||||
assert "tps_available" in STREAMING_PY, "metering SSE payloads must explicitly say whether TPS is displayable"
|
||||
assert "estimated" in STREAMING_PY, "metering SSE payloads must explicitly distinguish estimated readings"
|
||||
assert "record_token(stream_id, len(STREAM_PARTIAL_TEXT[stream_id]))" not in STREAMING_PY, (
|
||||
"live TPS must not be derived from streamed character count / byte-size estimates"
|
||||
)
|
||||
@@ -38,7 +38,7 @@ def test_streaming_persists_context_fields_on_session_before_save():
|
||||
# Save call follows shortly after
|
||||
save_call = src.find("\n s.save()", block_start)
|
||||
assert save_call != -1, "s.save() not found after the post-merge marker"
|
||||
assert save_call - block_start < 3000, (
|
||||
assert save_call - block_start < 3400, (
|
||||
"s.save() should be close to the post-merge marker — block expanded unexpectedly. "
|
||||
"If you've added a new pre-save mutation block here, bump this limit."
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user