Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Agent watchdog: the engine's prompt-read progress counts as activity; local routes get a 60 min stream-idle timeout (#3623)

Niko1221425bca71998a
diff --git a/apps/host/server/agent/agent-bridge.mjs b/apps/host/server/agent/agent-bridge.mjs
index 5ba219612..d9ef42942 100644
--- a/apps/host/server/agent/agent-bridge.mjs
+++ b/apps/host/server/agent/agent-bridge.mjs
@@ -115,7 +115,7 @@ const deferred = () => { let resolve; const promise = new Promise(r => { resolve
* @param {string} [options.nodePath] the Node runtime that runs the Harness
* @param {(line:string)=>void} [options.log]
*/
-export function createAgentBridge({ dataDir, componentsDir = null, bundledDir = null, gate = true, localInstances, connections, tools = () => undefined, skillsDir = () => null, onGoal = () => {}, onPlan = () => {}, onQueue = () => {}, proxyBase = null, searchMock = null, expertSampling = () => null, stallMs = 4 * 60_000, stallToolMs = 20 * 60_000, env = process.env, nodePath = process.execPath, log = () => {}, startTimeoutMs = 120_000, sessionReadyMs = 60_000, sessionRetryMs = 1_500 }) {
+export function createAgentBridge({ dataDir, componentsDir = null, bundledDir = null, gate = true, localInstances, connections, tools = () => undefined, skillsDir = () => null, onGoal = () => {}, onPlan = () => {}, onQueue = () => {}, proxyBase = null, searchMock = null, expertSampling = () => null, engineProgress = () => null, engineGraceMs = 30 * 60_000, stallMs = 4 * 60_000, stallToolMs = 20 * 60_000, env = process.env, nodePath = process.execPath, log = () => {}, startTimeoutMs = 120_000, sessionReadyMs = 60_000, sessionRetryMs = 1_500 }) {
const root = join(dataDir, 'agent');
const home = join(root, 'home');
const sessionsFile = join(root, 'sessions.json');
@@ -415,7 +415,7 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
abortFromRuntime: reason => finishTurn(reason), finish: failure => finishTurn(failure),
// Helper agents (subagents) the Harness started for this chat, and the state of the run around them.
children: new Map(), subagentCalls: new Map(), parentOpen: false, stopping: false, settleTimer: null, mode: appliedMode,
- plan: null, advancing: false, goalRun: false, lastActivity: Date.now(), watchdog: null,
+ plan: null, advancing: false, goalRun: false, lastActivity: Date.now(), heard: Date.now(), engineMark: null, watchdog: null,
research: researchTopic ? { phase: 'gather', nudged: false, waitingStart: false } : null,
};
c.turn = turn;
@@ -428,6 +428,13 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
if (turn.finished || turn.stopping) return;
const toolRunning = [...projector.tools.values()].some(x => x.card?.state === 'running' || x.card?.state === 'proposed') || pending.size > 0 && pendingFor(chatId).length > 0;
const limit = toolRunning ? stallToolMs : stallMs;
+ // A local engine reading a long prompt sends the Harness nothing for minutes (#3623). Its /metrics progress counts as activity,
+ // for at most engineGraceMs after the Harness last spoke: progress for another chat cannot keep a lost turn alive forever.
+ if (turn.route?.kind === 'local' && Date.now() - turn.heard < engineGraceMs) {
+ let mark = null;
+ try { mark = engineProgress(turn.route.instanceId); } catch { /* no progress known: the Harness alone decides */ }
+ if (mark != null && mark !== turn.engineMark) { turn.engineMark = mark; turn.lastActivity = Date.now(); return; }
+ }
if (Date.now() - turn.lastActivity < limit) return;
turn.stalledFor = Math.round((Date.now() - turn.lastActivity) / 1000);
turn.stopping = true;
@@ -501,6 +508,7 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
function handle(item) {
if (turn.finished) return;
turn.lastActivity = Date.now();
+ turn.heard = turn.lastActivity;
if (item.type === 'event') {
const e = item.event;
if (e.seq <= turn.lastSeq) return;
@@ -538,7 +546,7 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
emitRow(child, true);
const fold = item => {
if (turn.finished) return;
- turn.lastActivity = Date.now();
+ turn.lastActivity = turn.heard = Date.now();
// A helper that finishes before its throttled "working" row went out would show queued -> completed and never "working": send the held row first.
if (item.type === 'event' && item.event.type === 'turn/end' && child.rowPending) emitRow(child, true);
const { frames, row } = cp.consume(item);
diff --git a/apps/host/server/agent/engine-progress.mjs b/apps/host/server/agent/engine-progress.mjs
new file mode 100644
index 000000000..01f88f131
--- /dev/null
+++ b/apps/host/server/agent/engine-progress.mjs
@@ -0,0 +1,11 @@
+/**
+ * How far a local engine has got, as a short text that changes whenever it makes progress. The agent's stall watchdog
+ * compares it with the last one it saw: while a long prompt is read the Harness hears nothing for minutes, but the
+ * engine's /metrics (polled every 2 s) shows `live.prompt_read` moving (#3623). Null when the engine is idle or says nothing.
+ */
+export function engineProgressMark(instance) {
+ const live = instance?.metrics?.live;
+ if (!live || live.state === 'idle' || live.state === 'unloaded') return null;
+ const slots = Array.isArray(live.slots) ? live.slots.map(s => `${s?.state}:${s?.prompt_read ?? ''}:${s?.generated ?? ''}`).join(',') : '';
+ return `${live.state}|${live.prompt_read ?? ''}|${live.generated ?? ''}|${slots}`;
+}
diff --git a/apps/host/server/agent/profile.mjs b/apps/host/server/agent/profile.mjs
index 2637b87fa..31c6bca29 100644
--- a/apps/host/server/agent/profile.mjs
+++ b/apps/host/server/agent/profile.mjs
@@ -202,6 +202,9 @@ export function buildProfile({ local = [], connections = [], gate = true, tools
api: 'openai-completions',
baseURL: inst.baseUrl,
apiKeyEnv: keyEnv,
+ // A long prompt read sends only SSE comments, which pi-ai's OpenAI client drops, so its 300 s stream-idle timer would cut the
+ // request five minutes in (#3623). 60 minutes is a backstop: serve ends a hung engine and the bridge's watchdog ends a lost turn.
+ streamIdleTimeoutMs: 60 * 60_000,
// The Strata server reads the thinking level from `chat_template_kwargs` (enable_thinking, reasoning_effort) or from a top-level
// reasoning_effort; a top-level `enable_thinking` (pi-ai's `qwen` format) is ignored, so the model went on thinking with the menu on Off.
compat: THINKING_COMPAT,
diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjs
index f8d638ad7..faa7d8ffc 100644
--- a/apps/host/server/index.mjs
+++ b/apps/host/server/index.mjs
@@ -1,4 +1,5 @@
#!/usr/bin/env node
+import { engineProgressMark } from './agent/engine-progress.mjs';
import { createCloudflare } from './cloudflare.mjs';
import { createConnectorInstaller } from './cloudflared-install.mjs';
import { findRunningConnector } from './cloudflared-find.mjs';
@@ -174,6 +175,8 @@ const agent = createAgentBridge({
// The person's Response settings (thinking budget, Expert sampling) reach the agent's model through this host's own forwarding endpoint.
proxyBase: `http://127.0.0.1:${PORT}/api/internal/agent-llm`,
expertSampling: () => samplingFromExpert(store.get().expert),
+ // How far a local engine has got (the 2 s /metrics poll), for the agent's stall watchdog: it moves while a long prompt is read.
+ engineProgress: instanceId => engineProgressMark(native.readyInstances().find(i => i.id === instanceId)),
onPlan: (chatId, event, plan) => hub.send('plan', { chatId, event, plan }),
onQueue: (chatId, queue) => hub.send('queue', { chatId, queue }),
dataDir: store.DATA_DIR,
diff --git a/apps/host/tests/agent-watchdog.test.mjs b/apps/host/tests/agent-watchdog.test.mjs
index 29c2055a3..ce8a8d6e6 100644
--- a/apps/host/tests/agent-watchdog.test.mjs
+++ b/apps/host/tests/agent-watchdog.test.mjs
@@ -26,3 +26,19 @@ test('a silent model ends the turn with a plain message, and the chat works agai
await bridge.runTurn({ chatId: 'w1', message: 'SCRIPT:echo fine', workspacePath: ws, mode: 'bypass', onFrame: f => again.push(f) });
assert.match(again.filter(f => f.type === 'delta').map(f => f.text).join(''), /fine/);
});
+
+test('engine progress keeps a long prompt read alive past the stall limit, then the normal limit applies (#3623)', { skip }, async (t) => {
+ const worker = await startScriptedWorker({});
+ const dir = mkdtempSync(join(tmpdir(), 'strata-wd-')); const ws = join(dir, 'ws'); mkdirSync(ws, { recursive: true });
+ const started = Date.now();
+ const bridge = createAgentBridge({ dataDir: dir, localInstances: () => [{ id: 'i', label: 'S', baseUrl: worker.url, modelId: 'scripted-model', context: 32768, slots: 2 }], connections: () => [], env: process.env, startTimeoutMs: 90_000, stallMs: 2500, stallToolMs: 6000, engineGraceMs: 60_000,
+ // The engine moves for 7 s (a prompt being read), then goes quiet.
+ engineProgress: () => (Date.now() - started < 7_000 ? `reading:${Date.now()}` : null) });
+ t.after(async () => { await bridge.close(); await worker.close(); try { rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); } catch { /* best effort */ } });
+ const out = [];
+ const t0 = Date.now();
+ await bridge.runTurn({ chatId: 'w2', message: 'SCRIPT:hang', workspacePath: ws, mode: 'bypass', onFrame: f => out.push(f) });
+ assert.ok(Date.now() - started > 7_000, 'the 2.5 s stall limit did not fire while the engine was reading');
+ assert.ok(Date.now() - t0 < 40_000, 'and it still ended by itself once the engine went quiet');
+ assert.match(out.find(f => f.type === 'error').message, /stopped answering/);
+});
diff --git a/apps/host/tests/engine-progress.test.mjs b/apps/host/tests/engine-progress.test.mjs
new file mode 100644
index 000000000..b5ef3eb3a
--- /dev/null
+++ b/apps/host/tests/engine-progress.test.mjs
@@ -0,0 +1,23 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { engineProgressMark } from '../server/agent/engine-progress.mjs';
+import { readFileSync } from 'node:fs';
+
+test('the mark moves with the prompt read and with generation, and is null when the engine is idle or silent', () => {
+ const at = live => engineProgressMark({ metrics: { live } });
+ assert.equal(engineProgressMark(null), null);
+ assert.equal(engineProgressMark({ metrics: null }), null);
+ assert.equal(at({ state: 'idle' }), null);
+ assert.equal(at({ state: 'unloaded' }), null);
+ const a = at({ state: 'reading', prompt_read: 4096, prompt_total: 90000 });
+ const b = at({ state: 'reading', prompt_read: 8192, prompt_total: 90000 });
+ assert.notEqual(a, b);
+ assert.equal(a, at({ state: 'reading', prompt_read: 4096, prompt_total: 90000 }), 'no progress, same mark');
+ assert.notEqual(at({ state: 'generating', generated: 1 }), at({ state: 'generating', generated: 2 }));
+ assert.notEqual(at({ state: 'reading', slots: [{ state: 'reading', prompt_read: 1 }] }), at({ state: 'reading', slots: [{ state: 'reading', prompt_read: 2 }] }));
+});
+
+test('the host wires it in and the local profile lifts the 300 s stream-idle timer', () => {
+ assert.match(readFileSync(new URL('../server/index.mjs', import.meta.url), 'utf8'), /engineProgress: instanceId => engineProgressMark\(/);
+ assert.match(readFileSync(new URL('../server/agent/profile.mjs', import.meta.url), 'utf8'), /streamIdleTimeoutMs: 60 \* 60_000/);
+});