Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
One definition of tok/s: produced tokens over decode time, from the engine's own figures
Niko1221fccfd343d764
diff --git a/apps/host/server/agent/agent-bridge.mjs b/apps/host/server/agent/agent-bridge.mjsindex 988fe2069..5ba219612 100644--- a/apps/host/server/agent/agent-bridge.mjs+++ b/apps/host/server/agent/agent-bridge.mjs@@ -28,6 +28,7 @@ import { ChildProjector } from './children.mjs';import { nearestBudget } from './llm-shape.mjs';import { DeliverableTracker, ActivityGroups } from './deliverables.mjs';import { SkillsUsed, skillInfoFrom } from './skills-used.mjs';+import { turnEngineSpeed } from './engine-speed.mjs';import { TurnGuard, guardNote, GUARD_NOTES } from './turn-guard.mjs';import { PlanStore, planPrompt, parsePlan, parsePoints, stepObjective, stepLine, firstSentence, stepSummary, planAsPrompt, editedSteps, planView, researchPrompt, researchWritePrompt, RESEARCH_NUDGE, parseSources } from './plan.mjs';@@ -228,6 +229,13 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =return next;}+ /** The local engine's own speed for this turn's requests (null when it cannot be read: the stream's estimate stays). */+ async function engineSpeedOf(route, projector) {+ const inst = localInstances().find(i => i.id === route.instanceId);+ if (!inst) return null;+ return turnEngineSpeed({ baseUrl: inst.baseUrl, key: inst.key, since: projector.startedAt, attempts: projector.streams.size, timeoutMs: 800 });+ }+/** Make sure a runtime exists that knows the route this turn needs. */async function ensureRuntime(choice, signal) {for (;;) {@@ -429,7 +437,7 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =turn.finish(`The assistant stopped answering for ${mins > 1 ? mins + ' minutes' : 'a minute'}, so I ended this turn. Anything it already finished is kept. Send your message again to go on.`);}, Math.min(15_000, Math.max(200, Math.floor(stallMs / 4))));turn.watchdog.unref?.();- function finishTurn(failure) {+ async function finishTurn(failure) {if (turn.finished) return;turn.finished = true;clearTimeout(turn.settleTimer);@@ -450,7 +458,9 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =}if (noteReason && !extra.some(f => f.type === 'plan' && f.event === 'proposed')) { const note = guardNote(noteReason); if (note) extra.push(note); }for (const f of extra) emit(f);- const frames = failure ? [{ type: 'error', message: plainAgentFailure(failure), details: failure, code: 'AGENT_STOPPED' }] : projector.finish({ model: route.name, contextMax: route.contextWindow });+ // The answer speed of a turn on this computer is the engine's own (tokens it produced over its decode time, as the Monitor shows it).+ const engine = !failure && route.kind === 'local' ? await engineSpeedOf(route, projector) : null;+ const frames = failure ? [{ type: 'error', message: plainAgentFailure(failure), details: failure, code: 'AGENT_STOPPED' }] : projector.finish({ model: route.name, contextMax: route.contextWindow, engine });for (const f of frames.slice(0, -1)) emit(f);// Messages typed while the goal ran and not delivered between steps go back to the page now, to be sent as ordinary messages.if (turn.goalRun && attached) {diff --git a/apps/host/server/agent/engine-speed.mjs b/apps/host/server/agent/engine-speed.mjsnew file mode 100644index 000000000..7a11bf404--- /dev/null+++ b/apps/host/server/agent/engine-speed.mjs@@ -0,0 +1,41 @@+/**+ * The engine's own answer speed for one assistant turn: every token its requests produced (thinking, answer and tool calls; with+ * MTP drafting only the tokens it kept) over the time they spent decoding. These are the numbers the Monitor and `/metrics` show,+ * so the chat footer reads the same. Nothing here is a guess from the stream: it returns null when the engine does not say.+ */++/** Sum the engine's finished requests that began at or after `since` (ms). `attempts` is how many model requests the turn made:+ * when the engine lists more (another chat ran at the same time), only the newest `attempts` are the turn's. */+export function sumEngineRequests(requests, { since, attempts = Infinity } = {}) {+ if (!Array.isArray(requests)) return null;+ const mine = requests+ .filter(r => Number.isFinite(r?.time) && r.time * 1000 >= since - 1500)+ .sort((a, b) => a.time - b.time)+ .slice(-Math.max(1, attempts));+ let tokens = 0, decodeMs = 0, promptRead = 0, promptMs = 0;+ for (const r of mine) {+ const made = Number.isFinite(r.engine_generated) ? r.engine_generated : r.output_tokens;+ if (!(made > 0) || !(r.decode_ms > 0)) continue;+ tokens += made; decodeMs += r.decode_ms;+ const read = Number.isFinite(r.prompt_read) ? r.prompt_read : (r.prompt_tokens ?? 0) - (r.reused ?? 0);+ if (read > 0 && r.prompt_ms > 0) { promptRead += read; promptMs += r.prompt_ms; }+ }+ if (!(tokens > 0 && decodeMs > 0)) return null;+ return {+ tokens, decodeMs, tps: Number((tokens / (decodeMs / 1000)).toFixed(1)),+ promptTps: promptRead > 0 && promptMs > 0 ? Number((promptRead / (promptMs / 1000)).toFixed(1)) : null,+ };+}++/** Ask a local engine for its request history. Any failure is null: the caller keeps the stream's own estimate. */+export async function turnEngineSpeed({ baseUrl, key, since, attempts, fetcher = fetch, timeoutMs = 1500 }) {+ try {+ const origin = new URL(baseUrl).origin;+ const res = await fetcher(`${origin}/metrics?requests=all`, {+ signal: AbortSignal.timeout(timeoutMs),+ headers: key ? { authorization: `Bearer ${key}` } : undefined,+ });+ if (!res.ok) return null;+ return sumEngineRequests((await res.json())?.requests, { since, attempts });+ } catch { return null; }+}diff --git a/apps/host/server/agent/events.mjs b/apps/host/server/agent/events.mjsindex 83466bb41..a8b82de85 100644--- a/apps/host/server/agent/events.mjs+++ b/apps/host/server/agent/events.mjs@@ -395,7 +395,7 @@ export class TurnProjector {}/** The terminal frames for a finished turn. */- finish({ model, contextMax } = {}) {+ finish({ model, contextMax, engine = null } = {}) {const reason = this.ended ?? { kind: 'interrupted' };const durationMs = Date.now() - this.startedAt;const usage = this.usageSeen@@ -403,8 +403,15 @@ export class TurnProjector {: { tokensIn: null, tokensOut: null, usageSource: 'unavailable' };const decoded = this.decode();const generated = this.usageSeen && this.tokensOut > 0 ? this.tokensOut : decoded.tokens;- const tps = decoded.seconds >= 0.3 && generated > 0 ? Number((generated / decoded.seconds).toFixed(1)) : null;- const base = { durationMs, ttftMs: this.firstTokenAt ? this.firstTokenAt - this.startedAt : null, model, contextMax, tps, contextTokens: this.contextTokens };+ // Answer speed, one definition everywhere: all the tokens the model produced over its decode time. The engine's own figures (the ones the+ // Monitor shows) when the bridge could read them; otherwise the stream's own span. A very short reply has almost no span, so its decode time+ // is then the time from its first chunk to the end of the turn (only when no tool ran, so no tool time is in it): the number is still shown.+ let tps = null;+ if (engine?.tps > 0) tps = engine.tps;+ else if (generated > 0 && decoded.seconds >= 0.3) tps = Number((generated / decoded.seconds).toFixed(1));+ else if (generated > 0 && this.tools.size === 0 && this.firstTokenAt && Date.now() - this.firstTokenAt >= 50) tps = Number((generated / ((Date.now() - this.firstTokenAt) / 1000)).toFixed(1));+ const promptTps = engine?.promptTps > 0 ? engine.promptTps : null;+ const base = { durationMs, ttftMs: this.firstTokenAt ? this.firstTokenAt - this.startedAt : null, model, contextMax, tps, ...(promptTps ? { promptTps } : {}), contextTokens: this.contextTokens };switch (reason.kind) {case 'completed':return [{ type: 'done', ...base, ...usage, finishReason: 'stop' }];diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjsindex fc3b1e861..9b4dd67bd 100644--- a/apps/host/server/index.mjs+++ b/apps/host/server/index.mjs@@ -709,6 +709,7 @@ async function runTurn({ message, history = [], onFrame, signal, client, workspadurationMs: Date.now() - started,ttftMs: firstTokenMs ?? undefined,cachedTokens: result.cachedTokens,+ tps: result.tps,usageSource: result.usageSource,});if(models.REAL){monitor.setRecordedContext({ ...result, requestId: entry.id, sessionId });monitor.idle();}diff --git a/apps/host/server/model-providers.mjs b/apps/host/server/model-providers.mjsindex 2bb1c0f8f..30fa6fbd2 100644--- a/apps/host/server/model-providers.mjs+++ b/apps/host/server/model-providers.mjs@@ -30,6 +30,16 @@ export const PROVIDER_CATALOG = [const PROTOCOLS = ['openai-chat', 'openai-responses', 'anthropic'];const fail = message => { throw new Error(message); };const text = (value, max = 160) => typeof value === 'string' ? value.trim().slice(0, max) : '';+/** The answer speed of a model API's reply: the tokens it reports over the time from its first streamed token to the end, never over the whole+ * wait (that would put the time to the first token, often seconds of prompt reading, into the speed). null when it reported no tokens;+ * with no usable first-token time (or under a quarter of a second) the whole time is the best there is. */+export function answerSpeed(tokensOut, durationMs, firstTokenMs) {+ if (!Number.isFinite(tokensOut) || tokensOut <= 0) return null;+ const decodeMs = Number.isFinite(firstTokenMs) ? durationMs - firstTokenMs : NaN;+ const ms = decodeMs >= 250 ? decodeMs : durationMs;+ return Number((tokensOut / Math.max(0.001, ms / 1000)).toFixed(1));+}+export function normalizeBaseUrl(value) {let url; try { url = new URL(text(value, 2048)); } catch { fail('Enter a full http:// or https:// API address.'); }if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.search || url.hash) fail('Use an HTTP or HTTPS base URL without credentials, a query, or a fragment.');@@ -207,7 +217,7 @@ export function createModelProviders({ directory, changed = () => {}, fetchImplif (!textLength) fail('The model returned no text. Choose a model that supports text chat.');} finally { await reader.cancel().catch(() => {}); reader.releaseLock(); }const durationMs = Date.now() - started;- const result = { tokensIn, tokensOut, usageSource: tokensIn !== null && tokensOut !== null ? 'reported' : 'unavailable', durationMs, ttftMs: firstToken, tps: tokensOut !== null ? tokensOut / Math.max(0.001, durationMs / 1000) : null };+ const result = { tokensIn, tokensOut, usageSource: tokensIn !== null && tokensOut !== null ? 'reported' : 'unavailable', durationMs, ttftMs: firstToken, tps: answerSpeed(tokensOut, durationMs, firstToken) };onFrame({ type: 'done', ...result }); return result;}/** Connections WITH their keys, for the host's own agent bridge only; never serialised to a client. */diff --git a/apps/host/server/monitor.mjs b/apps/host/server/monitor.mjsindex 4619265c9..4c6733603 100644--- a/apps/host/server/monitor.mjs+++ b/apps/host/server/monitor.mjs@@ -347,12 +347,14 @@ export function idle(label = 'Waiting for a request') {hubRef?.send('monitor', snapshot());}-export function record({ tokensIn, tokensOut, durationMs = 0, ttftMs = null, cachedTokens = null }) {+export function record({ tokensIn, tokensOut, durationMs = 0, ttftMs = null, cachedTokens = null, tps: answerTps = null }) {if (Number.isFinite(tokensIn)) state.totals.tokensIn += tokensIn;if (Number.isFinite(tokensOut)) state.totals.tokensOut += tokensOut;state.totals.requests += 1;- if (Number.isFinite(tokensOut) && durationMs > 0) {- const tps = tokensOut / (durationMs / 1000);+ // The average is of answer speeds (tokens over decode time). A request with no decode figure is left out: dividing by the whole request would+ // put the prompt reading into it and read lower than the engine and the chat do.+ if (Number.isFinite(answerTps) && answerTps > 0) {+ const tps = answerTps;const n = ++state.totals.usageRequests;state.totals.avgTps = round((state.totals.avgTps * (n - 1) + tps) / n, 1);}diff --git a/apps/host/server/serving.mjs b/apps/host/server/serving.mjsindex efa7db28b..c98f3eb84 100644--- a/apps/host/server/serving.mjs+++ b/apps/host/server/serving.mjs@@ -149,7 +149,8 @@ export function cancel(id) {export function rateSample() {const done = requests.filter((r) => r.state === 'done' && r.tokensOut > 0);if (!done.length) return 0;- const tps = done.map((r) => r.tokensOut / Math.max(0.001, r.durationMs / 1000));+ const tps = done.filter((r) => r.tps > 0).map((r) => r.tps); // answer speeds (decode time), not tokens over the whole request+ if (!tps.length) return 0;return round(tps.reduce((a, b) => a + b, 0) / tps.length, 1);}diff --git a/apps/host/src/components/TelemetryPopover.tsx b/apps/host/src/components/TelemetryPopover.tsxindex a45c0ce06..43c7af7d9 100644--- a/apps/host/src/components/TelemetryPopover.tsx+++ b/apps/host/src/components/TelemetryPopover.tsx@@ -50,7 +50,7 @@ export function ChatTelemetry({ session }: {const approx = m.estimated ? tr('≈ ') : tr('');const dur = (n: number) => elapsed(n);return <div className="telemetry xs muted" aria-label={tr("Chat statistics")}>- <Chip icon="gauge" label={tr("{0}{1} tok/s{2}", [approx, speed === null ? tr('—') : Math.round(speed), m.streaming ? tr('') : tr(' · last reply')])} live={around(m.streaming ? "{0}{1} tok/s" : "{0}{1} tok/s · last reply", approx, <Num value={speed} format={int}/>)} title={tr("Session statistics")}>{tr(m.estimated ? <p className="xs muted">{tr("Live estimate from response length and elapsed time. Final usage comes from the engine.")}</p> : null)}<Row label={tr("Thinking time")} value={<Num value={m.thinkMs} format={dur}/>}/><Row label={tr("Writing time")} value={<Num value={m.writeMs} format={dur}/>}/><Row label={tr("Tool time")} value={<Num value={m.toolMs} format={dur}/>}/><Row label={tr("Average time to first token")} value={m.ttft === null ? tr('Not reported') : <Num value={m.ttft} format={dur}/>}/><Row label={tr("Latest response speed")} value={speed === null ? tr('Not reported') : <>{<Num value={speed} format={int}/>}{tr(' tokens/s')}</>}/><Row label={tr("Turns / tool calls")} value={<><Num value={m.turns} format={int}/> / <Num value={m.steps} format={int}/></>}/></Chip>+ <Chip icon="gauge" label={tr("{0}{1} tok/s{2}", [approx, speed === null ? tr('—') : Math.round(speed), m.streaming ? tr('') : tr(' · last reply')])} live={around(m.streaming ? "{0}{1} tok/s" : "{0}{1} tok/s · last reply", approx, <Num value={speed} format={int}/>)} title={tr("Session statistics")}>{tr(m.estimated ? <p className="xs muted">{tr("Live estimate from response length and elapsed time. Final usage comes from the engine.")}</p> : null)}<Row label={tr("Thinking time")} value={<Num value={m.thinkMs} format={dur}/>}/><Row label={tr("Writing time")} value={<Num value={m.writeMs} format={dur}/>}/><Row label={tr("Tool time")} value={<Num value={m.toolMs} format={dur}/>}/><Row label={tr("Average time to first token")} value={m.ttft === null ? tr('Not reported') : <Num value={m.ttft} format={dur}/>}/><Row label={tr("Latest response speed")} value={speed === null ? tr('Not reported') : <>{<Num value={speed} format={int}/>}{tr(' tokens/s')}</>}/><Row label={tr("Prompt reading")} value={m.promptSpeed === null ? tr('Not reported') : <>{<Num value={m.promptSpeed} format={int}/>}{tr(' tokens/s')}</>}/><Row label={tr("Turns / tool calls")} value={<><Num value={m.turns} format={int}/> / <Num value={m.steps} format={int}/></>}/></Chip><Chip icon="memory" label={tr("{0}{1} total tokens", [approx, compactTokens(m.total)])} live={around("{0}{1} total tokens", approx, <Num value={m.total} format={compactTokens}/>)} title={tr("Token usage")}><Row label={tr("Total in this chat")} value={<Num value={m.total} format={compactTokens}/>}/><Row label={tr("Input tokens")} value={<Num value={m.input} format={compactTokens}/>}/><Row label={tr("Output tokens")} value={<Num value={m.output} format={compactTokens}/>}/></Chip><Chip visual={<ContextRing percent={m.pct}/>} label={tr(m.pct === null ? '—' : `${Math.round(m.pct)}% of context`)} live={m.pct === null ? undefined : <><Num value={m.pct} format={int}/>{tr('% of context')}</>} title={tr("Context length")}><Row label={tr("Context used")} value={m.pct === null ? tr('Not reported') : <><Num value={m.pct} format={int}/>%</>}/><div className="context-meter" role="meter" aria-label={tr("Context window used")} aria-valuemin={0} aria-valuemax={100} aria-valuenow={m.pct ?? undefined}><span style={{ width: `${m.pct ?? 0}%` }}/></div><Row label={tr("Current context")} value={<Num value={m.used} format={compactTokens}/>}/><Row label={tr("Context limit")} value={<Num value={m.max} format={compactTokens}/>}/><Row label={tr("Space remaining")} value={m.used !== null && m.max !== null ? <Num value={Math.max(0, m.max - m.used)} format={compactTokens}/> : '—'}/></Chip></div>;diff --git a/apps/host/src/lib/chat-metrics.ts b/apps/host/src/lib/chat-metrics.tsindex 03cdaf22f..5fcd1af17 100644--- a/apps/host/src/lib/chat-metrics.ts+++ b/apps/host/src/lib/chat-metrics.ts@@ -16,12 +16,15 @@ export function chatMetrics(session:Session|null, monitor:MonitorState|null, conconst used=live?(liveUsed??finished):(finished??liveUsed);const max=remote?(replies.at(-1)?.contextMax||null):configuredContext??(active?monitor!.context.maxTokens:latest?.contextMax??null);const streaming=replies.find(m=>m.streaming);- const speed=streaming?.tps??[...replies].reverse().find(m=>m.tps!==undefined)?.tps??null;+ // Answer speed: everything the model generated over its decode time. While a reply streams it is that reply's; after it, the LAST reply's own+ // (never an older reply's number under the words "last reply").+ const lastReply=streaming??replies.at(-1);+ const speed=lastReply?.tps??null,promptSpeed=lastReply?.promptTps??null;const ttfts=replies.flatMap(m=>m.ttftMs===undefined?[]:[m.ttftMs]);const hasUsage=session?.statsAvailable===true;const llmTotal=replies.reduce((n,m)=>n+(m.durationMs??(m.streaming?Math.max(0,now-(Date.parse(m.at)||now)):0)),0),thinkTotal=replies.reduce((n,m)=>n+(m.reasoningMs??(m.streaming&&m.reasoningStartedAt?Math.max(0,now-m.reasoningStartedAt):0)),0);return {streaming:!!streaming,writeMs:Math.max(0,llmTotal-thinkTotal),estimated:streaming?.usageSource==='estimated',hasReplies:replies.length>0,example:!!session?.showcase||replies.some(m=>m.usageSource==='demo'),- speed:active&&monitor?.phase==='generating'?monitor.outputTps:speed,+ speed:active&&['generating','thinking'].includes(monitor?.phase??'')&&monitor!.outputTps>0?monitor!.outputTps:speed,promptSpeed,total:session?.statsAvailable===false?null:hasUsage?session.tokensIn+session.tokensOut+(streaming?.tokens??0)+(streaming?.inputTokens??0):replies.every(m=>m.inputTokens!==undefined&&m.tokens!==undefined)?replies.reduce((n,m)=>n+m.inputTokens!+m.tokens!,0):null,input:hasUsage?session.tokensIn:null,output:hasUsage?session.tokensOut:null,llmMs:replies.reduce((n,m)=>n+(m.durationMs??(m.streaming?Math.max(0,now-(Date.parse(m.at)||now)):0)),0),diff --git a/apps/host/src/lib/store.ts b/apps/host/src/lib/store.tsindex 74ee2b48e..14b31d8c9 100644--- a/apps/host/src/lib/store.ts+++ b/apps/host/src/lib/store.ts@@ -1731,6 +1731,7 @@ export async function sendMessage(text: string, options: { fromLoop?: boolean; g...m,tokens: typeof frame.tokensOut === 'number' ? frame.tokensOut : m.tokens,tps: typeof frame.tps === 'number' ? frame.tps : m.tps,+ promptTps: typeof frame.promptTps === 'number' ? frame.promptTps : m.promptTps,}));} else if (type === 'error') {clearStatus(sessionId);@@ -1752,7 +1753,8 @@ export async function sendMessage(text: string, options: { fromLoop?: boolean; ginputTokens: typeof frame.tokensIn === 'number' ? frame.tokensIn : undefined,contextUsed: typeof frame.contextTokens === 'number' ? frame.contextTokens : undefined,usageSource: frame.usageSource === 'reported' || frame.usageSource === 'demo' ? frame.usageSource : 'unavailable',- tps: typeof frame.tps === 'number' ? frame.tps : undefined,+ tps: typeof frame.tps === 'number' ? frame.tps : m.tps,+ promptTps: typeof frame.promptTps === 'number' ? frame.promptTps : m.promptTps,}));}},diff --git a/apps/host/src/lib/types.ts b/apps/host/src/lib/types.tsindex 1e8b053ef..b28df3490 100644--- a/apps/host/src/lib/types.ts+++ b/apps/host/src/lib/types.ts@@ -355,6 +355,7 @@ export interface ChatMessage {reasoningOpen?: boolean;tokens?: number;tps?: number;+ promptTps?: number;durationMs?: number;error?: string;tools?: ToolCall[];diff --git a/apps/host/tests/answer-speed.test.mjs b/apps/host/tests/answer-speed.test.mjsnew file mode 100644index 000000000..0f0a0644e--- /dev/null+++ b/apps/host/tests/answer-speed.test.mjs@@ -0,0 +1,123 @@+/** "Answer speed" is one number everywhere: every token the model produced over its decode time, never over a time that holds the prompt being read. */+import { fileURLToPath } from 'node:url';+import test, { mock } from 'node:test';+import assert from 'node:assert/strict';+import { build } from 'esbuild';+import { sumEngineRequests, turnEngineSpeed } from '../server/agent/engine-speed.mjs';+import { TurnProjector } from '../server/agent/events.mjs';+import { answerSpeed } from '../server/model-providers.mjs';+import * as monitor from '../server/monitor.mjs';+import * as serving from '../server/serving.mjs';++async function moduleAt(path) { const result = await build({ entryPoints: [fileURLToPath(new URL(path, import.meta.url))], bundle: true, write: false, platform: 'node', format: 'esm' }); return import(`data:text/javascript;base64,${Buffer.from(result.outputFiles[0].text).toString('base64')}`); }+const { chatMetrics } = await moduleAt('../src/lib/chat-metrics.ts');++// Two real requests from the owner PC (Gyro-S with MTP drafting, RTX 5070), as the engine's history lists them.+const REAL = [+ { time: 1000.0, prompt_tokens: 62, reused: 0, prompt_ms: 941, decode_ms: 711, engine_generated: 34, output_tokens: 34, drafts_offered: 34, drafts_accepted: 18 },+ { time: 1010.0, prompt_tokens: 308, reused: 0, prompt_ms: 1212, decode_ms: 1108, engine_generated: 73, output_tokens: 73, drafts_offered: 68, drafts_accepted: 45 },+];++test('the engine speed of a turn: produced tokens over decode time, prompt reading kept apart', () => {+ const one = sumEngineRequests(REAL, { since: 1009_000 });+ assert.equal(one.tokens, 73);+ assert.equal(one.tps, 65.9);+ assert.equal(one.promptTps, 254.1);+ const both = sumEngineRequests(REAL, { since: 999_000 });+ assert.equal(both.tokens, 107);+ assert.equal(both.tps, Number((107 / 1.819).toFixed(1))); // 58.8: the sum over the sum, not the mean of the two+ // dividing by the whole time (reading + decoding) is the number users saw as "lower than before"+ assert.ok(73 / ((1212 + 1108) / 1000) < 32);+});++test('drafted tokens are not output: the speed uses the engine count, whatever was offered', () => {+ const r = sumEngineRequests([{ ...REAL[1], drafts_offered: 9999 }], { since: 1009_000 });+ assert.equal(r.tps, 65.9);+});++test("another chat's requests on the same engine are not this turn's", () => {+ const stranger = { time: 1011.0, prompt_tokens: 10, reused: 0, prompt_ms: 100, decode_ms: 1000, engine_generated: 10 };+ const r = sumEngineRequests([REAL[1], stranger], { since: 1009_000, attempts: 1 });+ assert.equal(r.tokens, 10); // the newest one only: the turn made one model request+ assert.equal(sumEngineRequests([], { since: 0 }), null);+ assert.equal(sumEngineRequests([{ time: 5, engine_generated: 5, decode_ms: 0 }], { since: 0 }), null);+});++test('reading the engine: its own /metrics with the key, null when it fails', async () => {+ let seen;+ const fetcher = async (url, init) => { seen = { url, auth: init.headers.authorization }; return { ok: true, json: async () => ({ requests: REAL }) }; };+ const r = await turnEngineSpeed({ baseUrl: 'http://127.0.0.1:8080/v1', key: 'k', since: 1009_000, fetcher });+ assert.equal(r.tps, 65.9);+ assert.equal(seen.url, 'http://127.0.0.1:8080/metrics?requests=all');+ assert.equal(seen.auth, 'Bearer k');+ assert.equal(await turnEngineSpeed({ baseUrl: 'http://127.0.0.1:8080/v1', since: 0, fetcher: async () => { throw new Error('down'); } }), null);+ assert.equal(await turnEngineSpeed({ baseUrl: 'http://127.0.0.1:8080/v1', since: 0, fetcher: async () => ({ ok: false }) }), null);+});++const chunk = (attemptId, c) => ({ type: 'assistant-stream', frame: { type: 'chunk', attemptId, chunk: c } });++test('the finished agent turn shows the engine speed and the prompt reading speed', () => {+ const p = new TurnProjector({ startedAt: Date.now() });+ p.consume(chunk('a1', { type: 'text-delta', text: 'Done.' }));+ p.consume(chunk('a1', { type: 'usage', usage: { inputTokens: 308, outputTokens: 73 } }));+ p.ended = { kind: 'completed' };+ const done = p.finish({ model: 'm', contextMax: 32768, engine: { tps: 65.9, promptTps: 254.1 } }).find(f => f.type === 'done');+ assert.equal(done.tps, 65.9);+ assert.equal(done.promptTps, 254.1);+ assert.equal(done.tokensOut, 73);+});++test('a short reply still gets a speed (it used to come out empty under 0.3 s of streaming)', () => {+ mock.timers.enable({ apis: ['Date'], now: 3_000_000 });+ try {+ const p = new TurnProjector({ startedAt: Date.now() });+ mock.timers.tick(2000); // reading the prompt+ p.consume(chunk('a1', { type: 'text-delta', text: 'Hi!' }));+ p.consume(chunk('a1', { type: 'usage', usage: { inputTokens: 40, outputTokens: 3 } }));+ mock.timers.tick(100);+ p.ended = { kind: 'completed' };+ const done = p.finish({}).find(f => f.type === 'done');+ assert.ok(done.tps > 0 && done.tps <= 60, `${done.tps}`);+ } finally { mock.timers.reset(); }+});++test('a model API reply is timed from its first token, not from the request', () => {+ assert.equal(answerSpeed(100, 12_000, 2_000), 10); // 100 tokens in the 10 s after the first token+ assert.ok(100 / 12 < answerSpeed(100, 12_000, 2_000)); // the whole-wait number is the lower one+ assert.equal(answerSpeed(50, 1000, null), 50); // no first-token time: the whole time is all there is+ assert.equal(answerSpeed(50, 1000, 900), 50); // under a quarter second left: not used+ assert.equal(answerSpeed(null, 1000, 0), null);+ assert.equal(answerSpeed(0, 1000, 0), null);+});++test('the monitor and the request list average answer speeds, not tokens over the whole request', () => {+ monitor.record({ tokensIn: 308, tokensOut: 73, durationMs: 2320, tps: 65.9 });+ monitor.record({ tokensIn: 62, tokensOut: 34, durationMs: 1652, tps: 47.8 });+ assert.equal(monitor.snapshot().totals.avgTps, 56.9);+ monitor.record({ tokensIn: 1, tokensOut: 5, durationMs: 100 }); // no decode figure: left out, not divided by the wall time+ assert.equal(monitor.snapshot().totals.avgTps, 56.9);+ const a = serving.open({ client: 't', endpoint: '/x', model: 'm' });+ serving.mark(a.id, { tps: 65.9 });+ serving.finish(a.id, { tokensOut: 73, durationMs: 2320 });+ assert.equal(serving.rateSample(), 65.9);+});++const reply = (o) => ({ role: 'assistant', at: new Date().toISOString(), content: 'x', ...o });+const chat = (messages) => ({ id: 's', messages, tokensIn: 0, tokensOut: 0, statsAvailable: true });++test('the footer shows the last reply own speed, and the prompt reading speed next to it', () => {+ const s = chat([reply({ tps: 61, inputTokens: 10, tokens: 5 }), reply({ tps: 66, promptTps: 254, inputTokens: 20, tokens: 7 })]);+ const m = chatMetrics(s, null, 32768);+ assert.equal(m.speed, 66);+ assert.equal(m.promptSpeed, 254);+ // an older reply's number is never shown under "last reply"+ const m2 = chatMetrics(chat([reply({ tps: 61, inputTokens: 10, tokens: 5 }), reply({ inputTokens: 20, tokens: 7 })]), null, 32768);+ assert.equal(m2.speed, null);+});++test('while it thinks the footer reads the engine like the monitor does', () => {+ const s = chat([reply({ streaming: true, tps: 12 })]);+ const monitorState = { phase: 'thinking', outputTps: 64.2, context: { sessionId: 's', usedTokens: 100, maxTokens: 32768 } };+ assert.equal(chatMetrics(s, monitorState, 32768).speed, 64.2);+ assert.equal(chatMetrics(s, { ...monitorState, outputTps: 0 }, 32768).speed, 12); // no reading yet: the stream's own estimate+});diff --git a/serve/server.py b/serve/server.pyindex dc653e904..83a5fb515 100644--- a/serve/server.py+++ b/serve/server.py@@ -4648,7 +4648,7 @@ class Service:done = f"{pr[0]:,} of {pr[1]:,}" if pr and pr[1] else f"{s.get('prompt_tokens', 0):,}" # as read (#29)print(f"[strata] reading the prompt: {done} tokens, {el:.0f} s so far", flush=True)else:- rate = s["generated"] / max(1e-6, now - s["first_token"])+ rate = s["generated"] / max(RATE_MIN_SPAN_S, now - s["first_token"])# #984: while a thinking budget applies, the thinking line shows it, not the request's max_tokenslimit = (f"budget {s['thinking_budget']} (max {s.get('max_tokens')})"if s["phase"] == "thinking" and s.get("thinking_budget") else f"max {s.get('max_tokens')}")@@ -5124,11 +5124,12 @@ class Service:now = time.time()el = now - st.get("started", now)ft = st.get("first_token")- rate = n / max(1e-6, now - ft) if ft else 0.0+ rate = answer_speed(last, n, now - ft if ft else None)hit_msg = f", expert cache {hit_rate*100:.1f}% hit" if hit_rate is not None else ""if hit_msg and pcie_share:hit_msg += f" (+{pcie_share*100:.1f}% of the routed experts over PCIe)"- print(f"[strata] done: {n} tokens in {el:.0f} s ({rate:.1f} tok/s) "+ rate_txt = f"{rate:.1f} tok/s" if rate is not None else "no speed"+ print(f"[strata] done: {n} tokens in {el:.0f} s ({rate_txt}) "f"({finish}, cancel={cancel.is_set()}){hit_msg}", flush=True)if finish == "length" and parser.state in ("reasoning", "rcall"): # #530print("[strata] the reply reached max tokens while still thinking, so it has no "@@ -5218,6 +5219,20 @@ def add_carry(carry: dict, last: dict) -> dict:return out+def answer_speed(last: dict, generated: int, wall_s: float | None = None) -> float | None:+ """The one definition of "answer speed": every token the model produced (thinking and answer, the tokens it kept - not+ the MTP drafts it threw away) over the decode time, first generated token to last. The engine's DONE line holds both+ numbers; /metrics (`decode_tok_s`), the timings (`predicted_per_second`), the Terminal monitor and the app all use them.+ Only an engine that sent no DONE falls back to the server's own clock from the first token, and a clock younger than+ RATE_MIN_SPAN_S says nothing (it printed 1000000 tok/s for a one-token reply). Prompt reading is never in it."""+ produced, decode_ms = last.get("generated"), last.get("decode_ms")+ if produced and decode_ms and decode_ms > 0:+ return round(produced / (decode_ms / 1000), 1)+ if generated and wall_s is not None and wall_s >= RATE_MIN_SPAN_S:+ return round(generated / wall_s, 1)+ return None++def request_timings(prompt_tokens: int, generated: int, last: dict) -> dict | None:"""One request's `timings` in llama.cpp's names (what its clients show as speed), from the engine's own clock(StrataEngine.last): prompt_n is what was read, cache_n what the conversation cache already held. None when thediff --git a/serve/test_answer_speed.py b/serve/test_answer_speed.pynew file mode 100644index 000000000..96548681b--- /dev/null+++ b/serve/test_answer_speed.py@@ -0,0 +1,39 @@+"""The one definition of answer speed in the server: produced tokens over the engine's decode time, never over a time that+holds the prompt being read (no GPU or model pack).++ python -m unittest serve.test_answer_speed -v+"""+import unittest++from serve.server import answer_speed, request_timings+++class AnswerSpeed(unittest.TestCase):+ # A real request from the owner PC (Gyro-S, MTP): 308 prompt tokens read in 1212 ms, 73 generated in 1108 ms.+ REAL = {"generated": 73, "decode_ms": 1108.0, "prompt_ms": 1212.0, "drafts_accepted": 45, "drafts_offered": 68}++ def test_the_engines_decode_time_is_the_denominator(self):+ self.assertEqual(answer_speed(self.REAL, 73, 2.3), 65.9)++ def test_it_matches_the_timings_and_metrics_figure(self):+ self.assertEqual(answer_speed(self.REAL, 73), request_timings(308, 73, self.REAL)["predicted_per_second"])++ def test_prompt_reading_time_is_not_in_it(self):+ wall = (1212.0 + 1108.0) / 1000+ self.assertLess(round(73 / wall, 1), 40) # the number a wall-clock division would show+ self.assertEqual(answer_speed(self.REAL, 73, wall), 65.9)++ def test_drafts_are_not_counted(self):+ # 68 drafted, 45 accepted: the engine's count (73) is what the model kept; the speed never uses the draft figures.+ self.assertEqual(answer_speed({**self.REAL, "drafts_offered": 500}, 73), 65.9)++ def test_a_one_token_reply_has_no_absurd_rate(self):+ self.assertIsNone(answer_speed({}, 1, 1e-6))+ self.assertIsNone(answer_speed({}, 1, None))++ def test_without_a_done_line_the_server_clock_from_the_first_token_is_used(self):+ self.assertEqual(answer_speed({}, 50, 2.0), 25.0)+++if __name__ == "__main__":+ unittest.main()