Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

One definition of tok/s: produced tokens over decode time, from the engine's own figures

Niko1221fccfd343d764
diff --git a/apps/host/server/agent/agent-bridge.mjs b/apps/host/server/agent/agent-bridge.mjs
index 988fe2069..5ba219612 100644
--- a/apps/host/server/agent/agent-bridge.mjs
+++ b/apps/host/server/agent/agent-bridge.mjs
@@ -28,6 +28,7 @@ import { ChildProjector } from './children.mjs';
import { nearestBudget } from './llm-shape.mjs';
import { DeliverableTracker, ActivityGroups } from './deliverables.mjs';
import { SkillsUsed, skillInfoFrom } from './skills-used.mjs';
+import { turnEngineSpeed } from './engine-speed.mjs';
import { TurnGuard, guardNote, GUARD_NOTES } from './turn-guard.mjs';
import { PlanStore, planPrompt, parsePlan, parsePoints, stepObjective, stepLine, firstSentence, stepSummary, planAsPrompt, editedSteps, planView, researchPrompt, researchWritePrompt, RESEARCH_NUDGE, parseSources } from './plan.mjs';
@@ -228,6 +229,13 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
return next;
}
+ /** The local engine's own speed for this turn's requests (null when it cannot be read: the stream's estimate stays). */
+ async function engineSpeedOf(route, projector) {
+ const inst = localInstances().find(i => i.id === route.instanceId);
+ if (!inst) return null;
+ return turnEngineSpeed({ baseUrl: inst.baseUrl, key: inst.key, since: projector.startedAt, attempts: projector.streams.size, timeoutMs: 800 });
+ }
+
/** Make sure a runtime exists that knows the route this turn needs. */
async function ensureRuntime(choice, signal) {
for (;;) {
@@ -429,7 +437,7 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
turn.finish(`The assistant stopped answering for ${mins > 1 ? mins + ' minutes' : 'a minute'}, so I ended this turn. Anything it already finished is kept. Send your message again to go on.`);
}, Math.min(15_000, Math.max(200, Math.floor(stallMs / 4))));
turn.watchdog.unref?.();
- function finishTurn(failure) {
+ async function finishTurn(failure) {
if (turn.finished) return;
turn.finished = true;
clearTimeout(turn.settleTimer);
@@ -450,7 +458,9 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
}
if (noteReason && !extra.some(f => f.type === 'plan' && f.event === 'proposed')) { const note = guardNote(noteReason); if (note) extra.push(note); }
for (const f of extra) emit(f);
- const frames = failure ? [{ type: 'error', message: plainAgentFailure(failure), details: failure, code: 'AGENT_STOPPED' }] : projector.finish({ model: route.name, contextMax: route.contextWindow });
+ // The answer speed of a turn on this computer is the engine's own (tokens it produced over its decode time, as the Monitor shows it).
+ const engine = !failure && route.kind === 'local' ? await engineSpeedOf(route, projector) : null;
+ const frames = failure ? [{ type: 'error', message: plainAgentFailure(failure), details: failure, code: 'AGENT_STOPPED' }] : projector.finish({ model: route.name, contextMax: route.contextWindow, engine });
for (const f of frames.slice(0, -1)) emit(f);
// Messages typed while the goal ran and not delivered between steps go back to the page now, to be sent as ordinary messages.
if (turn.goalRun && attached) {
diff --git a/apps/host/server/agent/engine-speed.mjs b/apps/host/server/agent/engine-speed.mjs
new file mode 100644
index 000000000..7a11bf404
--- /dev/null
+++ b/apps/host/server/agent/engine-speed.mjs
@@ -0,0 +1,41 @@
+/**
+ * The engine's own answer speed for one assistant turn: every token its requests produced (thinking, answer and tool calls; with
+ * MTP drafting only the tokens it kept) over the time they spent decoding. These are the numbers the Monitor and `/metrics` show,
+ * so the chat footer reads the same. Nothing here is a guess from the stream: it returns null when the engine does not say.
+ */
+
+/** Sum the engine's finished requests that began at or after `since` (ms). `attempts` is how many model requests the turn made:
+ * when the engine lists more (another chat ran at the same time), only the newest `attempts` are the turn's. */
+export function sumEngineRequests(requests, { since, attempts = Infinity } = {}) {
+ if (!Array.isArray(requests)) return null;
+ const mine = requests
+ .filter(r => Number.isFinite(r?.time) && r.time * 1000 >= since - 1500)
+ .sort((a, b) => a.time - b.time)
+ .slice(-Math.max(1, attempts));
+ let tokens = 0, decodeMs = 0, promptRead = 0, promptMs = 0;
+ for (const r of mine) {
+ const made = Number.isFinite(r.engine_generated) ? r.engine_generated : r.output_tokens;
+ if (!(made > 0) || !(r.decode_ms > 0)) continue;
+ tokens += made; decodeMs += r.decode_ms;
+ const read = Number.isFinite(r.prompt_read) ? r.prompt_read : (r.prompt_tokens ?? 0) - (r.reused ?? 0);
+ if (read > 0 && r.prompt_ms > 0) { promptRead += read; promptMs += r.prompt_ms; }
+ }
+ if (!(tokens > 0 && decodeMs > 0)) return null;
+ return {
+ tokens, decodeMs, tps: Number((tokens / (decodeMs / 1000)).toFixed(1)),
+ promptTps: promptRead > 0 && promptMs > 0 ? Number((promptRead / (promptMs / 1000)).toFixed(1)) : null,
+ };
+}
+
+/** Ask a local engine for its request history. Any failure is null: the caller keeps the stream's own estimate. */
+export async function turnEngineSpeed({ baseUrl, key, since, attempts, fetcher = fetch, timeoutMs = 1500 }) {
+ try {
+ const origin = new URL(baseUrl).origin;
+ const res = await fetcher(`${origin}/metrics?requests=all`, {
+ signal: AbortSignal.timeout(timeoutMs),
+ headers: key ? { authorization: `Bearer ${key}` } : undefined,
+ });
+ if (!res.ok) return null;
+ return sumEngineRequests((await res.json())?.requests, { since, attempts });
+ } catch { return null; }
+}
diff --git a/apps/host/server/agent/events.mjs b/apps/host/server/agent/events.mjs
index 83466bb41..a8b82de85 100644
--- a/apps/host/server/agent/events.mjs
+++ b/apps/host/server/agent/events.mjs
@@ -395,7 +395,7 @@ export class TurnProjector {
}
/** The terminal frames for a finished turn. */
- finish({ model, contextMax } = {}) {
+ finish({ model, contextMax, engine = null } = {}) {
const reason = this.ended ?? { kind: 'interrupted' };
const durationMs = Date.now() - this.startedAt;
const usage = this.usageSeen
@@ -403,8 +403,15 @@ export class TurnProjector {
: { tokensIn: null, tokensOut: null, usageSource: 'unavailable' };
const decoded = this.decode();
const generated = this.usageSeen && this.tokensOut > 0 ? this.tokensOut : decoded.tokens;
- const tps = decoded.seconds >= 0.3 && generated > 0 ? Number((generated / decoded.seconds).toFixed(1)) : null;
- const base = { durationMs, ttftMs: this.firstTokenAt ? this.firstTokenAt - this.startedAt : null, model, contextMax, tps, contextTokens: this.contextTokens };
+ // Answer speed, one definition everywhere: all the tokens the model produced over its decode time. The engine's own figures (the ones the
+ // Monitor shows) when the bridge could read them; otherwise the stream's own span. A very short reply has almost no span, so its decode time
+ // is then the time from its first chunk to the end of the turn (only when no tool ran, so no tool time is in it): the number is still shown.
+ let tps = null;
+ if (engine?.tps > 0) tps = engine.tps;
+ else if (generated > 0 && decoded.seconds >= 0.3) tps = Number((generated / decoded.seconds).toFixed(1));
+ else if (generated > 0 && this.tools.size === 0 && this.firstTokenAt && Date.now() - this.firstTokenAt >= 50) tps = Number((generated / ((Date.now() - this.firstTokenAt) / 1000)).toFixed(1));
+ const promptTps = engine?.promptTps > 0 ? engine.promptTps : null;
+ const base = { durationMs, ttftMs: this.firstTokenAt ? this.firstTokenAt - this.startedAt : null, model, contextMax, tps, ...(promptTps ? { promptTps } : {}), contextTokens: this.contextTokens };
switch (reason.kind) {
case 'completed':
return [{ type: 'done', ...base, ...usage, finishReason: 'stop' }];
diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjs
index fc3b1e861..9b4dd67bd 100644
--- a/apps/host/server/index.mjs
+++ b/apps/host/server/index.mjs
@@ -709,6 +709,7 @@ async function runTurn({ message, history = [], onFrame, signal, client, workspa
durationMs: Date.now() - started,
ttftMs: firstTokenMs ?? undefined,
cachedTokens: result.cachedTokens,
+ tps: result.tps,
usageSource: result.usageSource,
});
if(models.REAL){monitor.setRecordedContext({ ...result, requestId: entry.id, sessionId });monitor.idle();}
diff --git a/apps/host/server/model-providers.mjs b/apps/host/server/model-providers.mjs
index 2bb1c0f8f..30fa6fbd2 100644
--- a/apps/host/server/model-providers.mjs
+++ b/apps/host/server/model-providers.mjs
@@ -30,6 +30,16 @@ export const PROVIDER_CATALOG = [
const PROTOCOLS = ['openai-chat', 'openai-responses', 'anthropic'];
const fail = message => { throw new Error(message); };
const text = (value, max = 160) => typeof value === 'string' ? value.trim().slice(0, max) : '';
+/** The answer speed of a model API's reply: the tokens it reports over the time from its first streamed token to the end, never over the whole
+ * wait (that would put the time to the first token, often seconds of prompt reading, into the speed). null when it reported no tokens;
+ * with no usable first-token time (or under a quarter of a second) the whole time is the best there is. */
+export function answerSpeed(tokensOut, durationMs, firstTokenMs) {
+ if (!Number.isFinite(tokensOut) || tokensOut <= 0) return null;
+ const decodeMs = Number.isFinite(firstTokenMs) ? durationMs - firstTokenMs : NaN;
+ const ms = decodeMs >= 250 ? decodeMs : durationMs;
+ return Number((tokensOut / Math.max(0.001, ms / 1000)).toFixed(1));
+}
+
export function normalizeBaseUrl(value) {
let url; try { url = new URL(text(value, 2048)); } catch { fail('Enter a full http:// or https:// API address.'); }
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) fail('Use an HTTP or HTTPS base URL without credentials, a query, or a fragment.');
@@ -207,7 +217,7 @@ export function createModelProviders({ directory, changed = () => {}, fetchImpl
if (!textLength) fail('The model returned no text. Choose a model that supports text chat.');
} finally { await reader.cancel().catch(() => {}); reader.releaseLock(); }
const durationMs = Date.now() - started;
- const result = { tokensIn, tokensOut, usageSource: tokensIn !== null && tokensOut !== null ? 'reported' : 'unavailable', durationMs, ttftMs: firstToken, tps: tokensOut !== null ? tokensOut / Math.max(0.001, durationMs / 1000) : null };
+ const result = { tokensIn, tokensOut, usageSource: tokensIn !== null && tokensOut !== null ? 'reported' : 'unavailable', durationMs, ttftMs: firstToken, tps: answerSpeed(tokensOut, durationMs, firstToken) };
onFrame({ type: 'done', ...result }); return result;
}
/** Connections WITH their keys, for the host's own agent bridge only; never serialised to a client. */
diff --git a/apps/host/server/monitor.mjs b/apps/host/server/monitor.mjs
index 4619265c9..4c6733603 100644
--- a/apps/host/server/monitor.mjs
+++ b/apps/host/server/monitor.mjs
@@ -347,12 +347,14 @@ export function idle(label = 'Waiting for a request') {
hubRef?.send('monitor', snapshot());
}
-export function record({ tokensIn, tokensOut, durationMs = 0, ttftMs = null, cachedTokens = null }) {
+export function record({ tokensIn, tokensOut, durationMs = 0, ttftMs = null, cachedTokens = null, tps: answerTps = null }) {
if (Number.isFinite(tokensIn)) state.totals.tokensIn += tokensIn;
if (Number.isFinite(tokensOut)) state.totals.tokensOut += tokensOut;
state.totals.requests += 1;
- if (Number.isFinite(tokensOut) && durationMs > 0) {
- const tps = tokensOut / (durationMs / 1000);
+ // The average is of answer speeds (tokens over decode time). A request with no decode figure is left out: dividing by the whole request would
+ // put the prompt reading into it and read lower than the engine and the chat do.
+ if (Number.isFinite(answerTps) && answerTps > 0) {
+ const tps = answerTps;
const n = ++state.totals.usageRequests;
state.totals.avgTps = round((state.totals.avgTps * (n - 1) + tps) / n, 1);
}
diff --git a/apps/host/server/serving.mjs b/apps/host/server/serving.mjs
index efa7db28b..c98f3eb84 100644
--- a/apps/host/server/serving.mjs
+++ b/apps/host/server/serving.mjs
@@ -149,7 +149,8 @@ export function cancel(id) {
export function rateSample() {
const done = requests.filter((r) => r.state === 'done' && r.tokensOut > 0);
if (!done.length) return 0;
- const tps = done.map((r) => r.tokensOut / Math.max(0.001, r.durationMs / 1000));
+ const tps = done.filter((r) => r.tps > 0).map((r) => r.tps); // answer speeds (decode time), not tokens over the whole request
+ if (!tps.length) return 0;
return round(tps.reduce((a, b) => a + b, 0) / tps.length, 1);
}
diff --git a/apps/host/src/components/TelemetryPopover.tsx b/apps/host/src/components/TelemetryPopover.tsx
index a45c0ce06..43c7af7d9 100644
--- a/apps/host/src/components/TelemetryPopover.tsx
+++ b/apps/host/src/components/TelemetryPopover.tsx
@@ -50,7 +50,7 @@ export function ChatTelemetry({ session }: {
const approx = m.estimated ? tr('≈ ') : tr('');
const dur = (n: number) => elapsed(n);
return <div className="telemetry xs muted" aria-label={tr("Chat statistics")}>
- <Chip icon="gauge" label={tr("{0}{1} tok/s{2}", [approx, speed === null ? tr('—') : Math.round(speed), m.streaming ? tr('') : tr(' · last reply')])} live={around(m.streaming ? "{0}{1} tok/s" : "{0}{1} tok/s · last reply", approx, <Num value={speed} format={int}/>)} title={tr("Session statistics")}>{tr(m.estimated ? <p className="xs muted">{tr("Live estimate from response length and elapsed time. Final usage comes from the engine.")}</p> : null)}<Row label={tr("Thinking time")} value={<Num value={m.thinkMs} format={dur}/>}/><Row label={tr("Writing time")} value={<Num value={m.writeMs} format={dur}/>}/><Row label={tr("Tool time")} value={<Num value={m.toolMs} format={dur}/>}/><Row label={tr("Average time to first token")} value={m.ttft === null ? tr('Not reported') : <Num value={m.ttft} format={dur}/>}/><Row label={tr("Latest response speed")} value={speed === null ? tr('Not reported') : <>{<Num value={speed} format={int}/>}{tr(' tokens/s')}</>}/><Row label={tr("Turns / tool calls")} value={<><Num value={m.turns} format={int}/> / <Num value={m.steps} format={int}/></>}/></Chip>
+ <Chip icon="gauge" label={tr("{0}{1} tok/s{2}", [approx, speed === null ? tr('—') : Math.round(speed), m.streaming ? tr('') : tr(' · last reply')])} live={around(m.streaming ? "{0}{1} tok/s" : "{0}{1} tok/s · last reply", approx, <Num value={speed} format={int}/>)} title={tr("Session statistics")}>{tr(m.estimated ? <p className="xs muted">{tr("Live estimate from response length and elapsed time. Final usage comes from the engine.")}</p> : null)}<Row label={tr("Thinking time")} value={<Num value={m.thinkMs} format={dur}/>}/><Row label={tr("Writing time")} value={<Num value={m.writeMs} format={dur}/>}/><Row label={tr("Tool time")} value={<Num value={m.toolMs} format={dur}/>}/><Row label={tr("Average time to first token")} value={m.ttft === null ? tr('Not reported') : <Num value={m.ttft} format={dur}/>}/><Row label={tr("Latest response speed")} value={speed === null ? tr('Not reported') : <>{<Num value={speed} format={int}/>}{tr(' tokens/s')}</>}/><Row label={tr("Prompt reading")} value={m.promptSpeed === null ? tr('Not reported') : <>{<Num value={m.promptSpeed} format={int}/>}{tr(' tokens/s')}</>}/><Row label={tr("Turns / tool calls")} value={<><Num value={m.turns} format={int}/> / <Num value={m.steps} format={int}/></>}/></Chip>
<Chip icon="memory" label={tr("{0}{1} total tokens", [approx, compactTokens(m.total)])} live={around("{0}{1} total tokens", approx, <Num value={m.total} format={compactTokens}/>)} title={tr("Token usage")}><Row label={tr("Total in this chat")} value={<Num value={m.total} format={compactTokens}/>}/><Row label={tr("Input tokens")} value={<Num value={m.input} format={compactTokens}/>}/><Row label={tr("Output tokens")} value={<Num value={m.output} format={compactTokens}/>}/></Chip>
<Chip visual={<ContextRing percent={m.pct}/>} label={tr(m.pct === null ? '—' : `${Math.round(m.pct)}% of context`)} live={m.pct === null ? undefined : <><Num value={m.pct} format={int}/>{tr('% of context')}</>} title={tr("Context length")}><Row label={tr("Context used")} value={m.pct === null ? tr('Not reported') : <><Num value={m.pct} format={int}/>%</>}/><div className="context-meter" role="meter" aria-label={tr("Context window used")} aria-valuemin={0} aria-valuemax={100} aria-valuenow={m.pct ?? undefined}><span style={{ width: `${m.pct ?? 0}%` }}/></div><Row label={tr("Current context")} value={<Num value={m.used} format={compactTokens}/>}/><Row label={tr("Context limit")} value={<Num value={m.max} format={compactTokens}/>}/><Row label={tr("Space remaining")} value={m.used !== null && m.max !== null ? <Num value={Math.max(0, m.max - m.used)} format={compactTokens}/> : '—'}/></Chip>
</div>;
diff --git a/apps/host/src/lib/chat-metrics.ts b/apps/host/src/lib/chat-metrics.ts
index 03cdaf22f..5fcd1af17 100644
--- a/apps/host/src/lib/chat-metrics.ts
+++ b/apps/host/src/lib/chat-metrics.ts
@@ -16,12 +16,15 @@ export function chatMetrics(session:Session|null, monitor:MonitorState|null, con
const used=live?(liveUsed??finished):(finished??liveUsed);
const max=remote?(replies.at(-1)?.contextMax||null):configuredContext??(active?monitor!.context.maxTokens:latest?.contextMax??null);
const streaming=replies.find(m=>m.streaming);
- const speed=streaming?.tps??[...replies].reverse().find(m=>m.tps!==undefined)?.tps??null;
+ // Answer speed: everything the model generated over its decode time. While a reply streams it is that reply's; after it, the LAST reply's own
+ // (never an older reply's number under the words "last reply").
+ const lastReply=streaming??replies.at(-1);
+ const speed=lastReply?.tps??null,promptSpeed=lastReply?.promptTps??null;
const ttfts=replies.flatMap(m=>m.ttftMs===undefined?[]:[m.ttftMs]);
const hasUsage=session?.statsAvailable===true;
const llmTotal=replies.reduce((n,m)=>n+(m.durationMs??(m.streaming?Math.max(0,now-(Date.parse(m.at)||now)):0)),0),thinkTotal=replies.reduce((n,m)=>n+(m.reasoningMs??(m.streaming&&m.reasoningStartedAt?Math.max(0,now-m.reasoningStartedAt):0)),0);
return {streaming:!!streaming,writeMs:Math.max(0,llmTotal-thinkTotal),estimated:streaming?.usageSource==='estimated',hasReplies:replies.length>0,example:!!session?.showcase||replies.some(m=>m.usageSource==='demo'),
- speed:active&&monitor?.phase==='generating'?monitor.outputTps:speed,
+ speed:active&&['generating','thinking'].includes(monitor?.phase??'')&&monitor!.outputTps>0?monitor!.outputTps:speed,promptSpeed,
total:session?.statsAvailable===false?null:hasUsage?session.tokensIn+session.tokensOut+(streaming?.tokens??0)+(streaming?.inputTokens??0):replies.every(m=>m.inputTokens!==undefined&&m.tokens!==undefined)?replies.reduce((n,m)=>n+m.inputTokens!+m.tokens!,0):null,
input:hasUsage?session.tokensIn:null,output:hasUsage?session.tokensOut:null,
llmMs:replies.reduce((n,m)=>n+(m.durationMs??(m.streaming?Math.max(0,now-(Date.parse(m.at)||now)):0)),0),
diff --git a/apps/host/src/lib/store.ts b/apps/host/src/lib/store.ts
index 74ee2b48e..14b31d8c9 100644
--- a/apps/host/src/lib/store.ts
+++ b/apps/host/src/lib/store.ts
@@ -1731,6 +1731,7 @@ export async function sendMessage(text: string, options: { fromLoop?: boolean; g
...m,
tokens: typeof frame.tokensOut === 'number' ? frame.tokensOut : m.tokens,
tps: typeof frame.tps === 'number' ? frame.tps : m.tps,
+ promptTps: typeof frame.promptTps === 'number' ? frame.promptTps : m.promptTps,
}));
} else if (type === 'error') {
clearStatus(sessionId);
@@ -1752,7 +1753,8 @@ export async function sendMessage(text: string, options: { fromLoop?: boolean; g
inputTokens: typeof frame.tokensIn === 'number' ? frame.tokensIn : undefined,
contextUsed: typeof frame.contextTokens === 'number' ? frame.contextTokens : undefined,
usageSource: frame.usageSource === 'reported' || frame.usageSource === 'demo' ? frame.usageSource : 'unavailable',
- tps: typeof frame.tps === 'number' ? frame.tps : undefined,
+ tps: typeof frame.tps === 'number' ? frame.tps : m.tps,
+ promptTps: typeof frame.promptTps === 'number' ? frame.promptTps : m.promptTps,
}));
}
},
diff --git a/apps/host/src/lib/types.ts b/apps/host/src/lib/types.ts
index 1e8b053ef..b28df3490 100644
--- a/apps/host/src/lib/types.ts
+++ b/apps/host/src/lib/types.ts
@@ -355,6 +355,7 @@ export interface ChatMessage {
reasoningOpen?: boolean;
tokens?: number;
tps?: number;
+ promptTps?: number;
durationMs?: number;
error?: string;
tools?: ToolCall[];
diff --git a/apps/host/tests/answer-speed.test.mjs b/apps/host/tests/answer-speed.test.mjs
new file mode 100644
index 000000000..0f0a0644e
--- /dev/null
+++ b/apps/host/tests/answer-speed.test.mjs
@@ -0,0 +1,123 @@
+/** "Answer speed" is one number everywhere: every token the model produced over its decode time, never over a time that holds the prompt being read. */
+import { fileURLToPath } from 'node:url';
+import test, { mock } from 'node:test';
+import assert from 'node:assert/strict';
+import { build } from 'esbuild';
+import { sumEngineRequests, turnEngineSpeed } from '../server/agent/engine-speed.mjs';
+import { TurnProjector } from '../server/agent/events.mjs';
+import { answerSpeed } from '../server/model-providers.mjs';
+import * as monitor from '../server/monitor.mjs';
+import * as serving from '../server/serving.mjs';
+
+async function moduleAt(path) { const result = await build({ entryPoints: [fileURLToPath(new URL(path, import.meta.url))], bundle: true, write: false, platform: 'node', format: 'esm' }); return import(`data:text/javascript;base64,${Buffer.from(result.outputFiles[0].text).toString('base64')}`); }
+const { chatMetrics } = await moduleAt('../src/lib/chat-metrics.ts');
+
+// Two real requests from the owner PC (Gyro-S with MTP drafting, RTX 5070), as the engine's history lists them.
+const REAL = [
+ { time: 1000.0, prompt_tokens: 62, reused: 0, prompt_ms: 941, decode_ms: 711, engine_generated: 34, output_tokens: 34, drafts_offered: 34, drafts_accepted: 18 },
+ { time: 1010.0, prompt_tokens: 308, reused: 0, prompt_ms: 1212, decode_ms: 1108, engine_generated: 73, output_tokens: 73, drafts_offered: 68, drafts_accepted: 45 },
+];
+
+test('the engine speed of a turn: produced tokens over decode time, prompt reading kept apart', () => {
+ const one = sumEngineRequests(REAL, { since: 1009_000 });
+ assert.equal(one.tokens, 73);
+ assert.equal(one.tps, 65.9);
+ assert.equal(one.promptTps, 254.1);
+ const both = sumEngineRequests(REAL, { since: 999_000 });
+ assert.equal(both.tokens, 107);
+ assert.equal(both.tps, Number((107 / 1.819).toFixed(1))); // 58.8: the sum over the sum, not the mean of the two
+ // dividing by the whole time (reading + decoding) is the number users saw as "lower than before"
+ assert.ok(73 / ((1212 + 1108) / 1000) < 32);
+});
+
+test('drafted tokens are not output: the speed uses the engine count, whatever was offered', () => {
+ const r = sumEngineRequests([{ ...REAL[1], drafts_offered: 9999 }], { since: 1009_000 });
+ assert.equal(r.tps, 65.9);
+});
+
+test("another chat's requests on the same engine are not this turn's", () => {
+ const stranger = { time: 1011.0, prompt_tokens: 10, reused: 0, prompt_ms: 100, decode_ms: 1000, engine_generated: 10 };
+ const r = sumEngineRequests([REAL[1], stranger], { since: 1009_000, attempts: 1 });
+ assert.equal(r.tokens, 10); // the newest one only: the turn made one model request
+ assert.equal(sumEngineRequests([], { since: 0 }), null);
+ assert.equal(sumEngineRequests([{ time: 5, engine_generated: 5, decode_ms: 0 }], { since: 0 }), null);
+});
+
+test('reading the engine: its own /metrics with the key, null when it fails', async () => {
+ let seen;
+ const fetcher = async (url, init) => { seen = { url, auth: init.headers.authorization }; return { ok: true, json: async () => ({ requests: REAL }) }; };
+ const r = await turnEngineSpeed({ baseUrl: 'http://127.0.0.1:8080/v1', key: 'k', since: 1009_000, fetcher });
+ assert.equal(r.tps, 65.9);
+ assert.equal(seen.url, 'http://127.0.0.1:8080/metrics?requests=all');
+ assert.equal(seen.auth, 'Bearer k');
+ assert.equal(await turnEngineSpeed({ baseUrl: 'http://127.0.0.1:8080/v1', since: 0, fetcher: async () => { throw new Error('down'); } }), null);
+ assert.equal(await turnEngineSpeed({ baseUrl: 'http://127.0.0.1:8080/v1', since: 0, fetcher: async () => ({ ok: false }) }), null);
+});
+
+const chunk = (attemptId, c) => ({ type: 'assistant-stream', frame: { type: 'chunk', attemptId, chunk: c } });
+
+test('the finished agent turn shows the engine speed and the prompt reading speed', () => {
+ const p = new TurnProjector({ startedAt: Date.now() });
+ p.consume(chunk('a1', { type: 'text-delta', text: 'Done.' }));
+ p.consume(chunk('a1', { type: 'usage', usage: { inputTokens: 308, outputTokens: 73 } }));
+ p.ended = { kind: 'completed' };
+ const done = p.finish({ model: 'm', contextMax: 32768, engine: { tps: 65.9, promptTps: 254.1 } }).find(f => f.type === 'done');
+ assert.equal(done.tps, 65.9);
+ assert.equal(done.promptTps, 254.1);
+ assert.equal(done.tokensOut, 73);
+});
+
+test('a short reply still gets a speed (it used to come out empty under 0.3 s of streaming)', () => {
+ mock.timers.enable({ apis: ['Date'], now: 3_000_000 });
+ try {
+ const p = new TurnProjector({ startedAt: Date.now() });
+ mock.timers.tick(2000); // reading the prompt
+ p.consume(chunk('a1', { type: 'text-delta', text: 'Hi!' }));
+ p.consume(chunk('a1', { type: 'usage', usage: { inputTokens: 40, outputTokens: 3 } }));
+ mock.timers.tick(100);
+ p.ended = { kind: 'completed' };
+ const done = p.finish({}).find(f => f.type === 'done');
+ assert.ok(done.tps > 0 && done.tps <= 60, `${done.tps}`);
+ } finally { mock.timers.reset(); }
+});
+
+test('a model API reply is timed from its first token, not from the request', () => {
+ assert.equal(answerSpeed(100, 12_000, 2_000), 10); // 100 tokens in the 10 s after the first token
+ assert.ok(100 / 12 < answerSpeed(100, 12_000, 2_000)); // the whole-wait number is the lower one
+ assert.equal(answerSpeed(50, 1000, null), 50); // no first-token time: the whole time is all there is
+ assert.equal(answerSpeed(50, 1000, 900), 50); // under a quarter second left: not used
+ assert.equal(answerSpeed(null, 1000, 0), null);
+ assert.equal(answerSpeed(0, 1000, 0), null);
+});
+
+test('the monitor and the request list average answer speeds, not tokens over the whole request', () => {
+ monitor.record({ tokensIn: 308, tokensOut: 73, durationMs: 2320, tps: 65.9 });
+ monitor.record({ tokensIn: 62, tokensOut: 34, durationMs: 1652, tps: 47.8 });
+ assert.equal(monitor.snapshot().totals.avgTps, 56.9);
+ monitor.record({ tokensIn: 1, tokensOut: 5, durationMs: 100 }); // no decode figure: left out, not divided by the wall time
+ assert.equal(monitor.snapshot().totals.avgTps, 56.9);
+ const a = serving.open({ client: 't', endpoint: '/x', model: 'm' });
+ serving.mark(a.id, { tps: 65.9 });
+ serving.finish(a.id, { tokensOut: 73, durationMs: 2320 });
+ assert.equal(serving.rateSample(), 65.9);
+});
+
+const reply = (o) => ({ role: 'assistant', at: new Date().toISOString(), content: 'x', ...o });
+const chat = (messages) => ({ id: 's', messages, tokensIn: 0, tokensOut: 0, statsAvailable: true });
+
+test('the footer shows the last reply own speed, and the prompt reading speed next to it', () => {
+ const s = chat([reply({ tps: 61, inputTokens: 10, tokens: 5 }), reply({ tps: 66, promptTps: 254, inputTokens: 20, tokens: 7 })]);
+ const m = chatMetrics(s, null, 32768);
+ assert.equal(m.speed, 66);
+ assert.equal(m.promptSpeed, 254);
+ // an older reply's number is never shown under "last reply"
+ const m2 = chatMetrics(chat([reply({ tps: 61, inputTokens: 10, tokens: 5 }), reply({ inputTokens: 20, tokens: 7 })]), null, 32768);
+ assert.equal(m2.speed, null);
+});
+
+test('while it thinks the footer reads the engine like the monitor does', () => {
+ const s = chat([reply({ streaming: true, tps: 12 })]);
+ const monitorState = { phase: 'thinking', outputTps: 64.2, context: { sessionId: 's', usedTokens: 100, maxTokens: 32768 } };
+ assert.equal(chatMetrics(s, monitorState, 32768).speed, 64.2);
+ assert.equal(chatMetrics(s, { ...monitorState, outputTps: 0 }, 32768).speed, 12); // no reading yet: the stream's own estimate
+});
diff --git a/serve/server.py b/serve/server.py
index dc653e904..83a5fb515 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -4648,7 +4648,7 @@ class Service:
done = f"{pr[0]:,} of {pr[1]:,}" if pr and pr[1] else f"{s.get('prompt_tokens', 0):,}" # as read (#29)
print(f"[strata] reading the prompt: {done} tokens, {el:.0f} s so far", flush=True)
else:
- rate = s["generated"] / max(1e-6, now - s["first_token"])
+ rate = s["generated"] / max(RATE_MIN_SPAN_S, now - s["first_token"])
# #984: while a thinking budget applies, the thinking line shows it, not the request's max_tokens
limit = (f"budget {s['thinking_budget']} (max {s.get('max_tokens')})"
if s["phase"] == "thinking" and s.get("thinking_budget") else f"max {s.get('max_tokens')}")
@@ -5124,11 +5124,12 @@ class Service:
now = time.time()
el = now - st.get("started", now)
ft = st.get("first_token")
- rate = n / max(1e-6, now - ft) if ft else 0.0
+ rate = answer_speed(last, n, now - ft if ft else None)
hit_msg = f", expert cache {hit_rate*100:.1f}% hit" if hit_rate is not None else ""
if hit_msg and pcie_share:
hit_msg += f" (+{pcie_share*100:.1f}% of the routed experts over PCIe)"
- print(f"[strata] done: {n} tokens in {el:.0f} s ({rate:.1f} tok/s) "
+ rate_txt = f"{rate:.1f} tok/s" if rate is not None else "no speed"
+ print(f"[strata] done: {n} tokens in {el:.0f} s ({rate_txt}) "
f"({finish}, cancel={cancel.is_set()}){hit_msg}", flush=True)
if finish == "length" and parser.state in ("reasoning", "rcall"): # #530
print("[strata] the reply reached max tokens while still thinking, so it has no "
@@ -5218,6 +5219,20 @@ def add_carry(carry: dict, last: dict) -> dict:
return out
+def answer_speed(last: dict, generated: int, wall_s: float | None = None) -> float | None:
+ """The one definition of "answer speed": every token the model produced (thinking and answer, the tokens it kept - not
+ the MTP drafts it threw away) over the decode time, first generated token to last. The engine's DONE line holds both
+ numbers; /metrics (`decode_tok_s`), the timings (`predicted_per_second`), the Terminal monitor and the app all use them.
+ Only an engine that sent no DONE falls back to the server's own clock from the first token, and a clock younger than
+ RATE_MIN_SPAN_S says nothing (it printed 1000000 tok/s for a one-token reply). Prompt reading is never in it."""
+ produced, decode_ms = last.get("generated"), last.get("decode_ms")
+ if produced and decode_ms and decode_ms > 0:
+ return round(produced / (decode_ms / 1000), 1)
+ if generated and wall_s is not None and wall_s >= RATE_MIN_SPAN_S:
+ return round(generated / wall_s, 1)
+ return None
+
+
def request_timings(prompt_tokens: int, generated: int, last: dict) -> dict | None:
"""One request's `timings` in llama.cpp's names (what its clients show as speed), from the engine's own clock
(StrataEngine.last): prompt_n is what was read, cache_n what the conversation cache already held. None when the
diff --git a/serve/test_answer_speed.py b/serve/test_answer_speed.py
new file mode 100644
index 000000000..96548681b
--- /dev/null
+++ b/serve/test_answer_speed.py
@@ -0,0 +1,39 @@
+"""The one definition of answer speed in the server: produced tokens over the engine's decode time, never over a time that
+holds the prompt being read (no GPU or model pack).
+
+ python -m unittest serve.test_answer_speed -v
+"""
+import unittest
+
+from serve.server import answer_speed, request_timings
+
+
+class AnswerSpeed(unittest.TestCase):
+ # A real request from the owner PC (Gyro-S, MTP): 308 prompt tokens read in 1212 ms, 73 generated in 1108 ms.
+ REAL = {"generated": 73, "decode_ms": 1108.0, "prompt_ms": 1212.0, "drafts_accepted": 45, "drafts_offered": 68}
+
+ def test_the_engines_decode_time_is_the_denominator(self):
+ self.assertEqual(answer_speed(self.REAL, 73, 2.3), 65.9)
+
+ def test_it_matches_the_timings_and_metrics_figure(self):
+ self.assertEqual(answer_speed(self.REAL, 73), request_timings(308, 73, self.REAL)["predicted_per_second"])
+
+ def test_prompt_reading_time_is_not_in_it(self):
+ wall = (1212.0 + 1108.0) / 1000
+ self.assertLess(round(73 / wall, 1), 40) # the number a wall-clock division would show
+ self.assertEqual(answer_speed(self.REAL, 73, wall), 65.9)
+
+ def test_drafts_are_not_counted(self):
+ # 68 drafted, 45 accepted: the engine's count (73) is what the model kept; the speed never uses the draft figures.
+ self.assertEqual(answer_speed({**self.REAL, "drafts_offered": 500}, 73), 65.9)
+
+ def test_a_one_token_reply_has_no_absurd_rate(self):
+ self.assertIsNone(answer_speed({}, 1, 1e-6))
+ self.assertIsNone(answer_speed({}, 1, None))
+
+ def test_without_a_done_line_the_server_clock_from_the_first_token_is_used(self):
+ self.assertEqual(answer_speed({}, 50, 2.0), 25.0)
+
+
+if __name__ == "__main__":
+ unittest.main()