Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Chat footer speed: tool-call writing time counts as decoding time, so tok/s is output tokens over decode time

Niko1221f523a94227f0
diff --git a/apps/host/server/agent/events.mjs b/apps/host/server/agent/events.mjs
index cbf158238..83466bb41 100644
--- a/apps/host/server/agent/events.mjs
+++ b/apps/host/server/agent/events.mjs
@@ -236,7 +236,7 @@ export class TurnProjector {
return { type: 'agent-tool', tool: entry.card };
}
- /* The speed shown while a reply streams, and after it: everything the model generated (reasoning included) over the time its
+ /* The speed shown while a reply streams, and after it: everything the model generated (reasoning and tool calls included) over the time its
requests spent producing it, not over the whole turn (tool runs and the wait for the prompt are not decoding). One stream per model
request; tokens are one per streamed chunk, or characters / 4 when a server batches. */
#produced(attemptId, text, out) {
@@ -284,6 +284,8 @@ export class TurnProjector {
const call = this.composing.get(c.id) ?? { name: c.name ?? '', head: '', bytes: 0 };
if (c.name) call.name = c.name;
const piece = String(c.argumentsDelta ?? '');
+ // Writing a tool call (a whole file, say) is decoding too: its tokens are in the engine's output count, so its time belongs in the speed.
+ this.#produced(f.attemptId, piece, out);
call.bytes += piece.length;
if (call.head.length < 4000) call.head += piece;
this.composing.set(c.id, call);
diff --git a/apps/host/tests/chat-speed.test.mjs b/apps/host/tests/chat-speed.test.mjs
new file mode 100644
index 000000000..6af6d0709
--- /dev/null
+++ b/apps/host/tests/chat-speed.test.mjs
@@ -0,0 +1,43 @@
+/** The chat footer's "tok/s · last reply" is the answer-writing speed: the engine's output tokens over the time spent decoding them. */
+import test, { mock } from 'node:test';
+import assert from 'node:assert/strict';
+import { TurnProjector } from '../server/agent/events.mjs';
+
+const chunk = (attemptId, c) => ({ type: 'assistant-stream', frame: { type: 'chunk', attemptId, chunk: c } });
+
+/** One model request: `pieces` chunks `gap` ms apart of the given kind, then the engine's usage; tool runs and the wait for the prompt take their own time. */
+test('writing a big tool call counts as decoding time: 600 tokens over 10 s is 60 tok/s, not hundreds', () => {
+ mock.timers.enable({ apis: ['Date'], now: 1_000_000 });
+ try {
+ const p = new TurnProjector({ startedAt: Date.now() });
+ // the model reads a 30 s prompt first: no chunk yet
+ mock.timers.tick(30_000);
+ // it writes a file as a tool call: 100 chunks, 100 ms apart, 600 output tokens in the engine's count
+ for (let i = 0; i < 100; i++) {
+ p.consume(chunk('a1', { type: 'tool-call-delta', id: 'call-1', name: 'write', argumentsDelta: 'x'.repeat(24) }));
+ mock.timers.tick(100);
+ }
+ p.consume(chunk('a1', { type: 'usage', usage: { inputTokens: 5000, outputTokens: 600 } }));
+ // a short closing sentence
+ p.consume(chunk('a2', { type: 'text-delta', text: 'Done.' }));
+ p.consume({ type: 'event', event: { seq: 9, time: 9, type: 'turn/end', data: { reason: { kind: 'completed' } } } });
+ p.ended = { kind: 'completed' };
+ const done = p.finish({ model: 'm', contextMax: 32768 }).find(f => f.type === 'done');
+ assert.ok(done.tps >= 55 && done.tps <= 65, `${done.tps} tok/s`);
+ assert.equal(done.tokensOut, 600, "the engine's own count");
+ } finally { mock.timers.reset(); }
+});
+
+test('text and reasoning keep their speed, and tool-run time is not decoding time', () => {
+ mock.timers.enable({ apis: ['Date'], now: 2_000_000 });
+ try {
+ const p = new TurnProjector({ startedAt: Date.now() });
+ for (let i = 0; i < 50; i++) { p.consume(chunk('a1', { type: 'reasoning-delta', text: 'think ' })); mock.timers.tick(20); }
+ for (let i = 0; i < 50; i++) { p.consume(chunk('a1', { type: 'text-delta', text: 'word ' })); mock.timers.tick(20); }
+ p.consume(chunk('a1', { type: 'usage', usage: { inputTokens: 100, outputTokens: 100 } }));
+ mock.timers.tick(60_000); // a slow command runs: not decoding
+ p.ended = { kind: 'completed' };
+ const done = p.finish({}).find(f => f.type === 'done');
+ assert.ok(done.tps >= 45 && done.tps <= 55, `${done.tps} tok/s`);
+ } finally { mock.timers.reset(); }
+});