Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

helpers wait for their turn visibly; batch-mtp policy off; helper cap max(slots,4)

Niko122131556cd1a0a8
diff --git a/apps/host/server/agent/children.mjs b/apps/host/server/agent/children.mjs
index c88a120b0..162cbb997 100644
--- a/apps/host/server/agent/children.mjs
+++ b/apps/host/server/agent/children.mjs
@@ -60,6 +60,7 @@ export class ChildProjector {
const at = this.now();
const done = this.status === 'completed' || this.status === 'failed' || this.status === 'cancelled';
let description = this.task;
+ if (this.status === 'queued') description = this.task ? `Waiting for its turn. ${this.task}` : 'Waiting for its turn.';
if (this.status === 'working' && this.activity) description = this.activity;
if (this.status === 'completed') description = this.closing ? clip(this.closing, 140) : 'Finished.';
if (this.status === 'failed') description = this.failure ?? 'Stopped because something went wrong.';
@@ -105,8 +106,9 @@ export class ChildProjector {
switch (e.type) {
case 'step/start':
case 'turn/start':
- // A helper that was finished and got a new message works again.
- if (this.status !== 'working') { this.endedAt = null; this.failure = null; set('working'); }
+ // A request was sent. Whether the engine has room for it yet is told by its first word: until then the helper is waiting for its turn.
+ // A helper that had finished and got a new message starts over as waiting.
+ if (this.finished) { this.endedAt = null; this.failure = null; set('queued'); }
break;
case 'assistant/message': {
const text = (d.message?.content ?? []).filter(p => p.type === 'text').map(p => p.text).join('').trim();
diff --git a/apps/host/server/agent/profile.mjs b/apps/host/server/agent/profile.mjs
index 4abd090b3..b0c361850 100644
--- a/apps/host/server/agent/profile.mjs
+++ b/apps/host/server/agent/profile.mjs
@@ -41,6 +41,12 @@ export const MODE_PRESETS = Object.freeze({
export const DEFAULT_MODE = 'manual';
/** Helper agents running at once when no local engine sets the limit (the Harness's own default). */
export const DEFAULT_API_HELPERS = 8;
+/**
+ * Helpers a local chat may have at once: at least the engine's request slots, and never fewer than this. With one slot the engine
+ * serves its requests one after the other (its queue is unbounded), so the extra helpers show as waiting for their turn instead
+ * of being refused.
+ */
+export const MIN_LOCAL_HELPERS = 4;
/** The preset for a chat's stored mode. Anything unknown (older values such as `auto`) gets the strictest ask-first mode. */
export const modeOf = value => (Object.hasOwn(MODE_PRESETS, value) ? value : DEFAULT_MODE);
@@ -97,7 +103,7 @@ export function buildProfile({ local = [], connections = [], gate = true }) {
const env = {};
const routes = [];
const mcpEnv = {};
- // Helper agents run at most as many at once as the engine has request slots (a local model); API models take the Harness's own limit.
+ // Helper agents: at least as many as the engine has request slots (a local model); API models take the Harness's own limit.
let slots = 0;
local.forEach((inst, index) => {
@@ -154,7 +160,7 @@ export function buildProfile({ local = [], connections = [], gate = true }) {
{ id: 'llm-pi-ai', name: '@deepseek-ai/dsh-llm-pi-ai', config: { providers } },
// The chat modes. Replacing the table also drops the Harness's own three presets: a chat is always in one of ours.
{ id: 'permission', config: { presets: Object.fromEntries(Object.values(MODE_PRESETS).map(({ key, sandbox, approval, name, description }) => [key, { sandbox, approval, name, description }])), defaultPreset: MODE_PRESETS[DEFAULT_MODE].key } },
- { id: 'subagent', config: { maxActiveSubagents: slots > 0 ? slots : DEFAULT_API_HELPERS } },
+ { id: 'subagent', config: { maxActiveSubagents: slots > 0 ? Math.max(slots, MIN_LOCAL_HELPERS) : DEFAULT_API_HELPERS } },
// Manual asks before a change runs, so the person sees one Approve/Deny card and nothing has happened yet.
...(gate ? [{ insert: [{ id: 'strata-gate', name: GATE_PLUGIN }] }] : []),
...(mcp.length ? [{ insert: mcp }] : []),
diff --git a/apps/host/server/native-parallel.mjs b/apps/host/server/native-parallel.mjs
index 0275e51c4..2543112cb 100644
--- a/apps/host/server/native-parallel.mjs
+++ b/apps/host/server/native-parallel.mjs
@@ -30,6 +30,13 @@ export const MAX_SLOTS = 8;
export const PARALLEL_POLICY = 'fit';
/** Share of ALL the graphics memory the slots' sessions may take together under the 'fit' policy. */
export const SLOT_SHARE = 0.2;
+/**
+ * `--batch-mtp` lets each slot verify draft tokens. Measured on a 12 GB RTX 3060 (docs/BATCHING.md says it pays on larger
+ * cards): a lone chat 35.6 tok/s with it against 37.8 without, 2 chats 37.2 against 33.8 together, 4 chats 42.5 against 38.4 -
+ * a small gain for several chats, a small loss for one, and 400 MiB less expert cache. Off until a card shows it pays; Expert
+ * mode adds it by hand (`--batch-mtp` in the engine arguments).
+ */
+export const BATCH_MTP_POLICY = 'off';
const argValue = (args, flag) => { const at = args.indexOf(flag); return at >= 0 ? args[at + 1] : undefined; };
@@ -75,7 +82,7 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requ
}
const cards = (hardware?.gpus ?? []).filter(g => facts.gpus.includes(g.index)).map(g => Number(g.vram_gb)).filter(n => Number.isFinite(n) && n > 0);
// Without a hardware read the engine's own fitting is all there is: ask for the default and let it say what it can run.
- if (!cards.length) return { slots: requested, batchMtp: facts.mtp && facts.gpus.length === 1, explicit: false, requested, note: null, slotGb: slot };
+ if (!cards.length) return { slots: requested, batchMtp: BATCH_MTP_POLICY === 'on' && facts.mtp && facts.gpus.length === 1, explicit: false, requested, note: null, slotGb: slot };
const total = cards.reduce((a, b) => a + b, 0);
let slots = requested;
if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > SLOT_SHARE * total) slots -= 1;
@@ -84,7 +91,7 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requ
? `Your graphics card has room for ${slots} requests at once with this context size, so Strata set that. A smaller context size leaves room for more.`
: 'Your graphics card has no spare room for several requests at once with this context size, so Strata set it to one at a time. A smaller context size leaves room for more.')
: null;
- return { slots, batchMtp: slots > 1 && facts.mtp && !facts.batchMtp && facts.gpus.length === 1, explicit: false, requested, note, slotGb: slot };
+ return { slots, batchMtp: BATCH_MTP_POLICY === 'on' && slots > 1 && facts.mtp && !facts.batchMtp && facts.gpus.length === 1, explicit: false, requested, note, slotGb: slot };
}
/**
diff --git a/apps/host/tests/agent-children.test.mjs b/apps/host/tests/agent-children.test.mjs
index 7dc21a97e..687807eb7 100644
--- a/apps/host/tests/agent-children.test.mjs
+++ b/apps/host/tests/agent-children.test.mjs
@@ -27,9 +27,12 @@ test('a helper row follows its helper: queued, working, completed with its closi
let t = 0;
const cp = new ChildProjector({ id: 'c1', name: 'Check the tests', task: 'Run the unit tests and report', now: () => (t += 100) });
assert.equal(cp.agent().status, 'queued');
- assert.equal(cp.agent().description, 'Run the unit tests and report');
+ assert.equal(cp.agent().description, 'Waiting for its turn. Run the unit tests and report');
cp.consume(ev('step/start', { turn: 1, step: 1 }));
+ assert.equal(cp.agent().status, 'queued', 'a request that was sent is still waiting until the engine says its first word');
+ cp.consume({ type: 'assistant-stream', frame: { type: 'chunk', chunk: { type: 'text-delta', text: 'Running' } } });
assert.equal(cp.agent().status, 'working');
+ assert.equal(cp.agent().description, 'Run the unit tests and report');
cp.consume({ type: 'assistant-stream', frame: { type: 'chunk', chunk: { type: 'usage', usage: { inputTokens: 100, outputTokens: 20 } } } });
assert.equal(cp.agent().tokens, 120);
cp.consume(ev('assistant/message', { message: { content: [{ type: 'text', text: 'All 12 tests pass.' }] } }));
@@ -72,8 +75,8 @@ test('goal wording: plain, never a code', () => {
// ------------------------------------------------------------------------------------------------- real Harness
-async function rig(t, { slots = 4 } = {}) {
- const worker = await startScriptedWorker({});
+async function rig(t, { slots = 4, serial = false } = {}) {
+ const worker = await startScriptedWorker({ serial });
const dir = mkdtempSync(join(tmpdir(), 'strata-kids-'));
const ws = join(dir, 'ws'); mkdirSync(ws, { recursive: true });
const local = [{ id: 'inst-1', label: 'Scripted', baseUrl: worker.url, modelId: 'scripted-model', context: 32768, slots }];
@@ -118,10 +121,10 @@ test('two helpers run at the same time as two rows, the run ends when both have
assert.equal(r.worker.requests.filter(q => q.directive === 'slow').length, 2);
});
-test('helpers never outnumber the engine slots', { skip }, async (t) => {
+test('helpers are limited to the Harness cap (4, or the slot count if larger); the fifth is refused with a message the model reads', { skip }, async (t) => {
const r = await rig(t, { slots: 2 });
- const { frames } = await r.run('few', 'SCRIPT:spawn 3 slow 30');
- assert.equal(rows(frames).size, 2, 'the third was refused by the Harness limit, which matches the slot count');
+ const { frames } = await r.run('few', 'SCRIPT:spawn 5 slow 30');
+ assert.equal(rows(frames).size, 4);
});
test('Manual: a helper that wants to change a file reaches the person as the same single Approve card, and the answer is honoured', { skip }, async (t) => {
@@ -196,3 +199,22 @@ test('/goal: sets a goal, runs its rounds in the chat, says how it ended, refuse
assert.equal(r.worker.requests.length, before, 'no further round starts after Stop');
assert.match(text(await r.bridge.goalCommand({ chatId: 'g2', line: '', workspacePath: r.ws, mode: 'bypass', onFrame: f => frames.push(f) }).then(() => frames.filter(f => f.type === 'delta').slice(-1))), /paused/);
});
+
+test('one request slot: the second helper shows as waiting for its turn, then works, and the run still ends cleanly', { skip }, async (t) => {
+ const r = await rig(t, { slots: 1, serial: true });
+ const { frames } = await r.run('queue', 'SCRIPT:spawn 2 slow 25');
+ const byId = rows(frames);
+ assert.equal(byId.size, 2, 'both helpers exist (they are not refused)');
+ const histories = [...byId.values()];
+ // walk the frames in order: at some moment one helper was working while the other was waiting for its turn
+ const now = new Map(); let overlap = null;
+ for (const f of frames.filter(x => x.type === 'agent')) {
+ now.set(f.agent.id, f.agent);
+ const all = [...now.values()];
+ const waiting = all.find(a => a.status === 'queued' && a.description.startsWith('Waiting for its turn.'));
+ if (waiting && all.some(a => a.status === 'working')) overlap ??= waiting;
+ }
+ assert.ok(overlap, 'one helper visibly waited for its turn while the other worked');
+ for (const h of histories) assert.equal(h.at(-1).status, 'completed');
+ assert.equal(frames.at(-1).type, 'done');
+});
diff --git a/apps/host/tests/agent-tools.test.mjs b/apps/host/tests/agent-tools.test.mjs
index 4e5dafd76..4236aaf41 100644
--- a/apps/host/tests/agent-tools.test.mjs
+++ b/apps/host/tests/agent-tools.test.mjs
@@ -55,11 +55,11 @@ test('the preset: our list is the Harness standard list minus exactly what the a
});
test('the profile loads the gate by path, sizes helper agents to the engine slots and keeps no secret', () => {
- const p = buildProfile({ local: [{ id: 'a', baseUrl: 'http://127.0.0.1:1/v1', modelId: 'm', context: 32768, slots: 3, key: 'sekret-key-123' }] });
+ const p = buildProfile({ local: [{ id: 'a', baseUrl: 'http://127.0.0.1:1/v1', modelId: 'm', context: 32768, slots: 2, key: 'sekret-key-123' }] });
const entries = JSON.parse(p.patch.split('\n').slice(1).join('\n'));
assert.deepEqual(entries.find(e => e.insert)?.insert, [{ id: 'strata-gate', name: GATE_PLUGIN }]);
assert.ok(existsSync(GATE_PLUGIN));
- assert.equal(entries.find(e => e.id === 'subagent').config.maxActiveSubagents, 3);
+ assert.equal(entries.find(e => e.id === 'subagent').config.maxActiveSubagents, 4, 'at least the 4 helpers; more only with more slots');
assert.equal(entries.find(e => e.id === 'preset-standard').config.plugins.some(x => x.id === 'tool-schedule'), false);
assert.ok(!p.patch.includes('sekret-key-123'));
const api = buildProfile({ local: [], connections: [{ id: 'c1', name: 'C', baseUrl: 'https://x/v1', protocol: 'openai-chat', models: [{ id: 'm' }] }] });
diff --git a/apps/host/tests/helpers/scripted-worker.mjs b/apps/host/tests/helpers/scripted-worker.mjs
index 076780705..9515fad54 100644
--- a/apps/host/tests/helpers/scripted-worker.mjs
+++ b/apps/host/tests/helpers/scripted-worker.mjs
@@ -32,7 +32,10 @@ import { createHash } from 'node:crypto';
const delay = ms => new Promise(r => setTimeout(r, ms));
-export async function startScriptedWorker({ port = 0, chunkDelayMs = 40, model = 'scripted-model' } = {}) {
+export async function startScriptedWorker({ port = 0, chunkDelayMs = 40, model = 'scripted-model', serial = false } = {}) {
+ // `serial` stands in for an engine with one request slot: the next request waits until the one before it has finished.
+ let tail = Promise.resolve();
+ const turn = () => { let release; const mine = new Promise(r => { release = r; }); const wait = tail; tail = tail.then(() => mine); return wait.then(() => release); };
const requests = [];
let aborted = 0;
const server = createServer(async (req, res) => {
@@ -42,6 +45,8 @@ export async function startScriptedWorker({ port = 0, chunkDelayMs = 40, model =
if (req.method === 'GET' && url.pathname === '/v1/models') return json(200, { object: 'list', data: [{ id: model, object: 'model' }] });
if (req.method === 'GET' && url.pathname === '/__requests') return json(200, { requests, aborted });
if (req.method !== 'POST' || !url.pathname.endsWith('/chat/completions')) return json(404, { error: 'no route' });
+ const release = serial ? await turn() : () => {};
+ res.on('close', release);
let raw = '';
for await (const c of req) raw += c;
const body = JSON.parse(raw);
diff --git a/apps/host/tests/parallel-default.test.mjs b/apps/host/tests/parallel-default.test.mjs
index 5aada2972..39a3bf34c 100644
--- a/apps/host/tests/parallel-default.test.mjs
+++ b/apps/host/tests/parallel-default.test.mjs
@@ -9,7 +9,7 @@ import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'nod
import { join, basename } from 'node:path';
import { tmpdir } from 'node:os';
import { createNativeControl } from '../server/native-control.mjs';
-import { chooseParallel, applyParallel, withoutParallel, slotGb, DEFAULT_SLOTS, PARALLEL_POLICY, FALLBACK_NOTE } from '../server/native-parallel.mjs';
+import { chooseParallel, applyParallel, withoutParallel, slotGb, DEFAULT_SLOTS, PARALLEL_POLICY, BATCH_MTP_POLICY, FALLBACK_NOTE } from '../server/native-parallel.mjs';
import { buildProfile } from '../server/agent/profile.mjs';
const hw = (gb, n = 1) => ({ gpus: Array.from({ length: n }, (_, i) => ({ index: i, vram_gb: gb })) });
@@ -38,14 +38,17 @@ test('sizing: 4 where the card has room, fewer where the slots would crowd out t
assert.doesNotMatch(small.note + tight.note, /--|batch|VRAM|GiB|tok/i, 'no engine words');
});
-test('batch-mtp is added where it applies: one card with a draft layer, not on a split, not when already there or one slot', () => {
+test('batch-mtp is off by default (measured: it does not pay on a 12 GB card); where the policy is on it applies to one card with a draft layer only', () => {
const mtp = c => chooseParallel({ config: c, hardware: hw(24) });
- assert.equal(mtp(cfg(32768)).batchMtp, true);
- assert.equal(mtp(cfg(32768, { gpu: [0, 1] })).batchMtp, false, 'measured slower on a split with all experts resident');
- assert.equal(mtp({ args: ['--max-context', '32768'], gpu: 0 }).batchMtp, false, 'no draft layer, nothing to batch');
- assert.equal(mtp(cfg(32768, { args: ['--max-context', '32768', '--mtp', '/m', '--batch-mtp'] })).batchMtp, false);
+ assert.equal(BATCH_MTP_POLICY, 'off');
+ assert.equal(mtp(cfg(32768)).batchMtp, false);
const applied = applyParallel(cfg(32768), mtp(cfg(32768)));
- assert.equal(applied.cfg.parallel, 4); assert.ok(applied.cfg.args.includes('--batch-mtp'));
+ assert.equal(applied.cfg.parallel, 4); assert.ok(!applied.cfg.args.includes('--batch-mtp'));
+ // the rule itself, for the day the policy is turned on
+ if (BATCH_MTP_POLICY === 'on') {
+ assert.equal(mtp(cfg(32768)).batchMtp, true);
+ assert.equal(mtp(cfg(32768, { gpu: [0, 1] })).batchMtp, false);
+ }
});
test("a number the person chose is kept exactly: Expert mode's value, the saved settings, an explicit --batch", () => {
@@ -64,10 +67,11 @@ test('the retry configuration has no slots and no batch-mtp', () => {
assert.equal(next.parallel, undefined); assert.ok(!next.args.includes('--batch-mtp'));
});
-test('helper agents never outnumber the engine slots', () => {
- const p = buildProfile({ local: [{ id: 'a', baseUrl: 'http://127.0.0.1:1/v1', modelId: 'm', slots: 4 }] });
- assert.equal(JSON.parse(p.patch.split('\n').slice(1).join('\n')).find(e => e.id === 'subagent').config.maxActiveSubagents, 4);
- assert.notEqual(p.signature, buildProfile({ local: [{ id: 'a', baseUrl: 'http://127.0.0.1:1/v1', modelId: 'm', slots: 2 }] }).signature, 'a changed slot count restarts the agent when idle');
+test('helper agents: at least as many as the engine slots, and extra ones wait for their turn instead of being refused', () => {
+ const profile = slots => buildProfile({ local: [{ id: 'a', baseUrl: 'http://127.0.0.1:1/v1', modelId: 'm', slots }] });
+ const cap = slots => JSON.parse(profile(slots).patch.split('\n').slice(1).join('\n')).find(e => e.id === 'subagent').config.maxActiveSubagents;
+ assert.deepEqual([cap(1), cap(2), cap(4), cap(6)], [4, 4, 4, 6]);
+ assert.notEqual(profile(6).signature, profile(2).signature, 'a changed helper limit restarts the agent when idle');
});
// ---------------------------------------------------------------------------------------- the installed app's load
@@ -94,7 +98,7 @@ function fixture({ failStarts = 0, config = {} } = {}) {
}
const finish = async (native, id) => { for (let i = 0; i < 500; i++) { const job = native.state().jobs.find(j => j.id === id); if (job.state !== 'running') return job; await new Promise(r => setTimeout(r, 10)); } throw new Error('job did not finish'); };
-test('the installed app loads with 4 slots and batch-mtp by default', async () => {
+test('the installed app loads with 4 request slots by default', async () => {
const f = fixture();
try {
await f.native.refresh(true);
@@ -102,7 +106,7 @@ test('the installed app loads with 4 slots and batch-mtp by default', async () =
assert.equal(job.state, 'done', job.error);
assert.equal(f.saved.length, 1);
assert.equal(f.saved[0].parallel, 4);
- assert.ok(f.saved[0].args.includes('--batch-mtp'));
+ assert.ok(!f.saved[0].args.includes('--batch-mtp'));
assert.equal(f.native.state().instances[0].parallel, 4, 'the agent reads the slot count from here');
} finally { await f.native.close(); f.cleanup(); }
});