Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: recover TUI startups interrupted by server restarts

Adam Eivyb842a4d269e9
diff --git a/docs/features/cos-agent-runner.md b/docs/features/cos-agent-runner.md
index 59bd626d9f..ae95003efb 100644
--- a/docs/features/cos-agent-runner.md
+++ b/docs/features/cos-agent-runner.md
@@ -41,6 +41,24 @@ A separate `portos-cos` PM2 process that:
- A disconnect does **not** demote. In runner mode the runner owns every agent process, so while it is down new tasks are **held** as `pending` (logged once, not per task) and resume on reconnect. Demoting would spawn them as children of `portos-server` — the orphaning this app exists to prevent.
- Agents already spawned directly keep completing through the direct path across a promotion; reconciliation only adopts agents this server does not already own.
+### TUI startup across a server restart
+
+The runner retains a submission receipt beside each opted-in TUI's live PTY.
+It records the receipt only when the task's submit Enter is written successfully;
+startup-dialog answers, a paste alone, and writes queued in the server do not count.
+If the server restarts before submission, its prompt-delivery timers disappear.
+Recovery registers the surviving terminal's completion owner, then asks the runner
+to stop it only if the receipt is still absent. Normal exit handling records a
+retryable startup failure and releases the task for a fresh attempt. Recovery never
+replays a potentially unsent paste into the old composer.
+
+The runner checks the receipt and claims cancellation synchronously, so a task
+submitted after the recovery listing survives, and a canceled startup cannot
+accept a late submission. Submitted tasks continue across restarts. Older servers
+and runners without this protocol report an unknown receipt and retain their
+existing recovery behavior; a legacy terminal already stuck at startup may still
+need relaunching.
+
## Unattended recovery gates
Nobody is watching a CoS TUI session, so nothing types into it when the model stops early. Six gates in `server/services/agentTuiSpawning.js` watch the PTY stream and act on one shared 5s poll. Five of them prefer a nudge into the live session over a kill, because the TUI still holds the whole conversation; only the retry stall ends the run, since a request the provider never answers will not answer the next one either. No gate reaps a run on the clock alone — the wall-clock ceiling was removed deliberately, after it killed agents 30 seconds past a merged PR.
diff --git a/server/cos-runner/index.js b/server/cos-runner/index.js
index 6a60a058ca..af31955d41 100644
--- a/server/cos-runner/index.js
+++ b/server/cos-runner/index.js
@@ -215,6 +215,8 @@ app.get('/agents', async (req, res) => {
command: agent.command || null,
workspacePath: agent.workspacePath || null,
securityScan: agent.securityScan || null,
+ // null means an older caller did not opt into submission tracking.
+ promptSubmitted: agent.promptSubmitted ?? null,
});
}
res.json(agents);
@@ -244,6 +246,7 @@ app.post('/spawn-tui', lifecycle.spawnRoute(async (req, res) => {
cols = 80,
rows = 24,
doneSentinelPath = null,
+ trackPromptSubmission = false,
} = body.data;
// Name the offending field. A single opaque "missing or invalid fields" 400
@@ -348,6 +351,9 @@ app.post('/spawn-tui', lifecycle.spawnRoute(async (req, res) => {
command,
doneSentinelPath,
completedBySentinel: false,
+ // Kept beside the live PTY, so a portos-server restart cannot erase the
+ // distinction between a running task and an empty startup composer.
+ promptSubmitted: trackPromptSubmission ? false : null,
doneWatcher: null,
// node-pty reports pid 0 for ConPTY on Windows, so a pid probe is not a
// liveness signal. onExit is the runner's authority; GET /agents reads this.
@@ -791,12 +797,21 @@ app.post('/terminate/:agentId', async (req, res) => {
return res.status(404).json({ error: 'Agent not found or not running' });
}
+ if (req.body?.onlyIfPromptUnsubmitted === true) {
+ if (agent.kind !== 'tui' || agent.promptSubmitted !== false) {
+ return res.json({ success: true, agentId, terminated: false });
+ }
+ // Claim cancellation synchronously with the receipt check, before scanner
+ // settlement yields. Late server writes must not start this canceled task.
+ agent.startupInterrupted = true;
+ }
+
console.log(`🔪 Terminating agent ${agentId}`);
await agent.stopSecurityScan?.();
killProcessTree(agent.process, 'SIGTERM');
armForceKill(agentId, agent);
- res.json({ success: true, agentId });
+ res.json({ success: true, agentId, terminated: true });
});
/**
@@ -946,10 +961,17 @@ io.on('connection', (socket) => {
});
console.log(`🔌 Client connected: ${socket.id}`);
- socket.on('tui:input', ({ sessionId, data }) => {
+ socket.on('tui:input', ({ sessionId, data, promptSubmission }) => {
try {
const agent = [...activeAgents.values()].find(candidate => candidate.sessionId === sessionId);
- if (agent?.kind === 'tui' && typeof data === 'string') agent.process.write(data);
+ if (agent?.kind === 'tui' && !agent.startupInterrupted && typeof data === 'string') {
+ agent.process.write(data);
+ // The write and receipt are synchronous in the runner. A queued socket
+ // write in the server is not evidence that the Enter reached the PTY.
+ if (promptSubmission === true && data === '\r' && agent.promptSubmitted !== null) {
+ agent.promptSubmitted = true;
+ }
+ }
} catch (err) {
console.error(`❌ TUI input relay failed: ${err.message}`);
}
diff --git a/server/cos-runner/requestSchemas.js b/server/cos-runner/requestSchemas.js
index a0592f8b9f..e5dac3e309 100644
--- a/server/cos-runner/requestSchemas.js
+++ b/server/cos-runner/requestSchemas.js
@@ -37,6 +37,7 @@ export const spawnTuiBodySchema = z.object({
cols: z.number().int().min(1).optional(),
rows: z.number().int().min(1).optional(),
doneSentinelPath: optionalString,
+ trackPromptSubmission: z.boolean().optional(),
});
export const spawnBodySchema = z.object({
diff --git a/server/cos-runner/tuiSentinel.test.js b/server/cos-runner/tuiSentinel.test.js
index d8161df1b3..6b067729d6 100644
--- a/server/cos-runner/tuiSentinel.test.js
+++ b/server/cos-runner/tuiSentinel.test.js
@@ -1,4 +1,4 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { EventEmitter } from 'node:events';
import { readFileSync } from 'node:fs';
import { join, basename } from 'node:path';
@@ -13,6 +13,7 @@ import { createTuiExitHandler } from './tuiExit.js';
import { armForceKill } from './forceKill.js';
import { createRunnerShutdown, registerRunnerShutdownSignals } from './shutdown.js';
import * as requestSchemas from './requestSchemas.js';
+import { usableAgentPid, runnerAgentLivenessFields } from '../lib/runnerAgentLiveness.js';
// index.js binds a server at module load, so (like shutdown.test.js) it is
// executed in a VM with its imports replaced by injected fakes. Nothing listens,
@@ -24,9 +25,9 @@ const SENTINEL_PATH = '/example/workspace/.agent-done-agent-1';
// `sentinel` overrides the fake access object; `watch` captures the handler the
// runner registers so a test can fire the "sentinel appeared" event itself.
-async function spawnTui(sentinel = {}) {
+async function spawnTui(sentinel = {}, { trackPromptSubmission = true } = {}) {
const routes = new Map();
- const app = { use: vi.fn(), get: vi.fn(), post: (path, handler) => routes.set(path, handler) };
+ const app = { use: vi.fn(), get: (path, handler) => routes.set(path, handler), post: (path, handler) => routes.set(path, handler) };
const server = Object.assign(new EventEmitter(), {
listen: vi.fn(), close: vi.fn(cb => cb?.()), closeAllConnections: vi.fn(),
});
@@ -47,6 +48,7 @@ async function spawnTui(sentinel = {}) {
...sentinel,
};
const state = { agents: {}, stats: { spawned: 0, completed: 0, failed: 0 } };
+ const killProcessTree = vi.fn();
runInNewContext(source, {
assertDatabaseAdmission: vi.fn(), maintenance,
withBackupAssetPublication: vi.fn(), persistRunnerCompletion: vi.fn(),
@@ -61,7 +63,8 @@ async function spawnTui(sentinel = {}) {
...requestSchemas,
createRunnerShutdown, registerRunnerShutdownSignals, createTuiExitHandler, createHttpDrain,
createSecurityScanManager, securityScanRequestSchema,
- armForceKillShared: armForceKill, killProcessTree: vi.fn(),
+ armForceKillShared: armForceKill, killProcessTree,
+ usableAgentPid, runnerAgentLivenessFields, getProcessStats: vi.fn(async () => ({})),
spawn: vi.fn(), pty: { spawn: vi.fn(() => child) },
createAgentSentinelAccess: vi.fn(() => access),
parseSentinelPayload, SENTINEL_COMPLETION_MARKER,
@@ -71,12 +74,25 @@ async function spawnTui(sentinel = {}) {
});
const res = { statusCode: 200, status(code) { this.statusCode = code; return this; }, json: vi.fn() };
await routes.get('/spawn-tui')({ body: {
- agentId: 'agent-1', taskId: 'task-1', command: 'example-cli', doneSentinelPath: SENTINEL_PATH,
+ agentId: 'agent-1', taskId: 'task-1', command: 'example-cli', doneSentinelPath: SENTINEL_PATH, trackPromptSubmission,
} }, res);
expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ success: true }));
// The runner hands its handler to watch(); firing it returns the tracked work.
const fireSentinel = () => access.watch.mock.calls.at(-1)[0]();
- return { access, child, stopWatching, fireSentinel };
+ const socket = new EventEmitter();
+ socket.use = vi.fn();
+ io.emit('connection', socket);
+ const listing = async () => {
+ const response = { json: vi.fn() };
+ await routes.get('/agents')({}, response);
+ return response.json.mock.calls[0][0][0];
+ };
+ const stopStartup = async () => {
+ const response = { json: vi.fn() };
+ await routes.get('/terminate/:agentId')({ params: { agentId: 'agent-1' }, body: { onlyIfPromptUnsubmitted: true } }, response);
+ return response.json.mock.calls[0][0];
+ };
+ return { access, child, stopWatching, fireSentinel, socket, listing, stopStartup, killProcessTree };
}
beforeEach(() => {
@@ -84,6 +100,43 @@ beforeEach(() => {
vi.spyOn(console, 'error').mockImplementation(() => {});
});
+describe('cos-runner TUI startup recovery receipt', () => {
+ beforeEach(() => { vi.useFakeTimers(); });
+ afterEach(() => { vi.clearAllTimers(); vi.useRealTimers(); });
+
+ it('counts only a successful task-submit Enter, preserving it across a server reconnect', async () => {
+ const run = await spawnTui();
+ const input = data => run.socket.emit('tui:input', { sessionId: 'agent-1', ...data });
+ input({ data: '\r' }); // startup dialog, not the task
+ input({ data: '\x1b[200~task body\x1b[201~' });
+ expect((await run.listing()).promptSubmitted).toBe(false);
+ run.child.write.mockImplementationOnce(() => { throw new Error('PTY write failed'); });
+ input({ data: '\r', promptSubmission: true });
+ expect((await run.listing()).promptSubmitted).toBe(false);
+ input({ data: '\r', promptSubmission: true });
+ expect((await run.listing()).promptSubmitted).toBe(true);
+ // A stale GET /agents from the old server must not authorize this kill.
+ expect(await run.stopStartup()).toMatchObject({ terminated: false });
+ expect(run.killProcessTree).not.toHaveBeenCalled();
+ });
+
+ it('stops an unsubmitted composer and refuses a late submission after the cancellation decision', async () => {
+ const run = await spawnTui();
+ expect(await run.stopStartup()).toMatchObject({ terminated: true });
+ expect(run.killProcessTree).toHaveBeenCalledWith(run.child, 'SIGTERM');
+ run.socket.emit('tui:input', { sessionId: 'agent-1', data: '\r', promptSubmission: true });
+ expect(run.child.write).not.toHaveBeenCalled();
+ expect((await run.listing()).promptSubmitted).toBe(false);
+ });
+
+ it('preserves terminals spawned by an older server without the tracking protocol', async () => {
+ const run = await spawnTui({}, { trackPromptSubmission: false });
+ expect((await run.listing()).promptSubmitted).toBeNull();
+ expect(await run.stopStartup()).toMatchObject({ terminated: false });
+ expect(run.killProcessTree).not.toHaveBeenCalled();
+ });
+});
+
describe('cos-runner TUI completion sentinel (#3202)', () => {
it('completes the agent and kills the PTY once the sentinel validates', async () => {
const run = await spawnTui();
diff --git a/server/services/agentLifecycle.js b/server/services/agentLifecycle.js
index 73257c1755..79a6d6d645 100644
--- a/server/services/agentLifecycle.js
+++ b/server/services/agentLifecycle.js
@@ -590,7 +590,7 @@ async function handleAdmittedAgentCompletion(agentId, exitCode, success, duratio
// the persisted record stores an ISO string; toEpochMs handles both.
const runStartedAt = toEpochMs(agent.startedAt);
let effectiveSuccess = success;
- if (!effectiveSuccess && task?.id) {
+ if (!effectiveSuccess && task?.id && !agent.startupErrorAnalysis) {
const commitFound = workspacePath && await committedDuringRun(workspacePath, runStartedAt);
if (commitFound) {
emitLog('warn', `Agent ${agentId} reported failure (exit ${exitCode}) but work completed - commit found for task ${task.id}`, { agentId, taskId: task.id, exitCode });
@@ -611,7 +611,7 @@ async function handleAdmittedAgentCompletion(agentId, exitCode, success, duratio
}
// Analyze failure if applicable
- const errorAnalysis = effectiveSuccess ? null : analyzeAgentFailure(outputBuffer, task, model);
+ const errorAnalysis = effectiveSuccess ? null : agent.startupErrorAnalysis || analyzeAgentFailure(outputBuffer, task, model);
// The gate for finalizeAgent's PR-claim verification (#3358): a PortOS-owned
// PR is created by `runAgentCompletionCleanup` below, i.e. AFTER finalize, so
diff --git a/server/services/agentLifecycle.spawnViaRunner.test.js b/server/services/agentLifecycle.spawnViaRunner.test.js
index ac2a56f231..026187c3c4 100644
--- a/server/services/agentLifecycle.spawnViaRunner.test.js
+++ b/server/services/agentLifecycle.spawnViaRunner.test.js
@@ -109,6 +109,9 @@ vi.mock('./agentErrorAnalysis.js', () => ({
}));
vi.mock('./appActivity.js', () => ({ releaseAppReviewMarker: vi.fn().mockResolvedValue(undefined) }));
vi.mock('./instanceIdentity.js', () => ({ ensureInstanceId: vi.fn().mockResolvedValue('instance-1') }));
+vi.mock('../lib/gitCommitProbe.js', async importOriginal => ({
+ ...(await importOriginal()), committedDuringRun: vi.fn(async () => true),
+}));
vi.mock('./toolStateMachine.js', () => ({
createToolExecution: vi.fn(() => ({ id: 'exec-1' })),
startExecution: vi.fn(),
@@ -121,7 +124,8 @@ vi.mock('./executionLanes.js', () => ({
release: vi.fn(),
}));
-import { spawnViaRunner } from './agentLifecycle.js';
+import { handleAgentCompletion, spawnViaRunner } from './agentLifecycle.js';
+import { committedDuringRun } from '../lib/gitCommitProbe.js';
import { spawnAgentViaRunner, RUNNER_SPAWN_REFUSED, RUNNER_SPAWN_AMBIGUOUS } from './cosRunnerClient.js';
import { completeAgent, updateAgent } from './cosAgentLifecycle.js';
import { completeAgentRun } from './agentRunTracking.js';
@@ -133,6 +137,25 @@ import { runnerAgents } from './agentState.js';
const REJECTION = 'Command not allowed: grok. Permitted commands: claude, codex';
+it('finalizes an interrupted TUI startup as retryable failure, without crediting a shared-checkout commit', async () => {
+ vi.clearAllMocks();
+ const startupErrorAnalysis = {
+ category: 'startup-failure', origin: 'runner', actionable: false,
+ message: 'Server restarted before the TUI task prompt was submitted',
+ };
+ runnerAgents.set('agent-interrupted', {
+ task: { id: 'task-interrupted', taskType: 'user', metadata: {} },
+ runId: 'run-interrupted', workspacePath: '/tmp/ws', startedAt: Date.now(),
+ startupErrorAnalysis, recoveredFromRunner: true,
+ });
+ await handleAgentCompletion('agent-interrupted', 143, false, 1000);
+ expect(finalizeAgent).toHaveBeenCalledWith(expect.objectContaining({
+ agentId: 'agent-interrupted', success: false, errorAnalysis: startupErrorAnalysis,
+ }));
+ expect(committedDuringRun).not.toHaveBeenCalled();
+ expect(runnerAgents.has('agent-interrupted')).toBe(false);
+});
+
// The runner ANSWERED and declined. The shape mirrors what `spawnAgentViaRunner`
// stamps on a non-2xx (pinned in cosRunnerClient.test.js); a bare Error would
// classify as AMBIGUOUS, which is the whole point of the split (#4615).
diff --git a/server/services/agentRunnerSync.js b/server/services/agentRunnerSync.js
index a377b619e6..b0c91d00d4 100644
--- a/server/services/agentRunnerSync.js
+++ b/server/services/agentRunnerSync.js
@@ -12,7 +12,7 @@ import { maintenance } from '../lib/maintenanceAdmission.js';
* or `agentManagement.js` (enforced by `agentImportCycles.test.js`).
*/
-import { connectTuiSessionViaRunner, getActiveAgentsFromRunner } from './cosRunnerClient.js';
+import { connectTuiSessionViaRunner, getActiveAgentsFromRunner, terminateAgentViaRunner } from './cosRunnerClient.js';
import { runnerEntryShieldsRunningRecord } from '../lib/runnerAgentLiveness.js';
import { isInternalTaskId } from '../lib/taskParser.js';
import { isAgentOwnedLocally, runnerAgents } from './agentState.js';
@@ -20,6 +20,33 @@ import { AGENT_RECORD_UNREADABLE, isLiveAgentRecord, readAgentRecordOrUnreadable
import { appendRunEvent } from './agentRunEventLog.js';
import * as shellService from './shell.js';
+// Recovery cannot replay a paste safely: it may already be sitting unsent in
+// the composer. Stop only a PTY whose runner positively knows no task submit
+// reached it; normal completion owns task retry and workspace cleanup.
+const startupErrorAnalysis = {
+ category: 'startup-failure',
+ origin: 'runner',
+ actionable: false,
+ message: 'Server restarted before the TUI task prompt was submitted',
+ suggestedFix: 'Retry the task in a fresh terminal; the recovered startup terminal was stopped without replaying its prompt.',
+};
+
+async function stopInterruptedStartup(agentId) {
+ try {
+ const result = await terminateAgentViaRunner(agentId, { onlyIfPromptUnsubmitted: true });
+ // Submission can land between GET /agents and the guarded stop. The runner
+ // decides atomically; preserve the now-working run and its normal verdict.
+ if (result.terminated === false) {
+ const owned = runnerAgents.get(agentId);
+ if (owned) delete owned.startupErrorAnalysis;
+ }
+ } catch (err) {
+ // Keep ownership and retry on the next recovery sweep. An RPC failure is
+ // not proof of exit and must not finalize a still-live PTY.
+ console.error(`❌ Failed to stop interrupted TUI startup ${agentId}: ${err.message}`);
+ }
+}
+
/**
* Sync running agents from the runner (recovery after server restart).
* This allows us to receive completion events for agents spawned before restart.
@@ -76,6 +103,10 @@ export async function syncRunnerAgents() {
// sweep would treat that local map as ownership. Live rows still sync.
if (!(await runnerEntryShieldsRunningRecord(agent))) continue;
maintenance.recoverOwned('agent', agent.id);
+ if (runnerAgents.get(agent.id)?.startupErrorAnalysis) {
+ await stopInterruptedStartup(agent.id);
+ continue;
+ }
// Only sync if this process isn't already driving it
if (!isAgentOwnedLocally(agent.id)) {
const task = taskMap.get(agent.taskId);
@@ -118,7 +149,8 @@ export async function syncRunnerAgents() {
workspacePath: persisted?.metadata?.workspacePath || agent.workspacePath || null,
recoveredFromRunner: true,
hasStartedWorking: true,
- startedAt: agent.startedAt
+ startedAt: agent.startedAt,
+ ...(agent.kind === 'tui' && agent.promptSubmitted === false ? { startupErrorAnalysis } : {}),
});
if (!recoveredRunId) {
console.warn(`⚠️ Recovered agent ${agent.id} has no run id on record — its run stays open and unbilled`);
@@ -173,6 +205,10 @@ export async function syncRunnerAgents() {
}
console.log(`🔄 Recovered agent ${agent.id} (task: ${agent.taskId})`);
syncedCount++;
+ if (runnerAgents.get(agent.id)?.startupErrorAnalysis) {
+ console.warn(`⚠️ Stopping recovered TUI ${agent.id}: ${startupErrorAnalysis.message}`);
+ await stopInterruptedStartup(agent.id);
+ }
}
}
diff --git a/server/services/agentRunnerSync.test.js b/server/services/agentRunnerSync.test.js
index 79102f326e..0fc0a45e01 100644
--- a/server/services/agentRunnerSync.test.js
+++ b/server/services/agentRunnerSync.test.js
@@ -10,6 +10,7 @@ const recoveredPty = {
vi.mock('./cosRunnerClient.js', () => ({
getActiveAgentsFromRunner: vi.fn(),
+ terminateAgentViaRunner: vi.fn(async () => ({ success: true, terminated: true })),
connectTuiSessionViaRunner: vi.fn(() => ({
sessionId: 'tui-session-1',
pid: 1234,
@@ -45,7 +46,7 @@ vi.mock('./cosAgentLifecycle.js', async (importOriginal) => ({
const { appendRunEvent } = vi.hoisted(() => ({ appendRunEvent: vi.fn(async () => ({ appended: true })) }));
vi.mock('./agentRunEventLog.js', () => ({ appendRunEvent }));
-import { connectTuiSessionViaRunner, getActiveAgentsFromRunner } from './cosRunnerClient.js';
+import { connectTuiSessionViaRunner, getActiveAgentsFromRunner, terminateAgentViaRunner } from './cosRunnerClient.js';
import * as shellService from './shell.js';
import { AGENT_RECORD_UNREADABLE, readAgentRecordOrUnreadable } from './cosAgentLifecycle.js';
import { activeAgents, runnerAgents } from './agentState.js';
@@ -256,4 +257,50 @@ describe('syncRunnerAgents — reconnect boundary (#4540)', () => {
expect(appendRunEvent.mock.calls.map(([e]) => e.kind)).toEqual(['run.runner-recovered']);
});
+
+ // A server restart loses the controller's paste/Enter timers, while the
+ // runner keeps the empty composer alive. Recovery must await real exit and
+ // use ordinary completion/retry, rather than replaying a possibly pasted task.
+ it('stops an unsubmitted startup only after registering its completion owner and relay', async () => {
+ getActiveAgentsFromRunner.mockResolvedValue([{ ...survivingTui, promptSubmitted: false }]);
+ terminateAgentViaRunner.mockImplementationOnce(async () => {
+ expect(runnerAgents.get('agent-1')).toMatchObject({
+ startupErrorAnalysis: { category: 'startup-failure', actionable: false },
+ });
+ expect(shellService.registerExternalSession).toHaveBeenCalled();
+ expect(recoveredPty.onExit).toHaveBeenCalled();
+ return { success: true, terminated: true };
+ });
+
+ await expect(syncRunnerAgents()).resolves.toBe(1);
+ expect(terminateAgentViaRunner).toHaveBeenCalledWith('agent-1', { onlyIfPromptUnsubmitted: true });
+ expect(runnerAgents.has('agent-1')).toBe(true);
+ expect(recoveredPty.write).not.toHaveBeenCalled();
+ });
+
+ it.each([true, null, undefined])('preserves a submitted or legacy TUI (receipt=%s)', async promptSubmitted => {
+ getActiveAgentsFromRunner.mockResolvedValue([{ ...survivingTui, promptSubmitted }]);
+ await syncRunnerAgents();
+ expect(terminateAgentViaRunner).not.toHaveBeenCalled();
+ expect(runnerAgents.get('agent-1').startupErrorAnalysis).toBeUndefined();
+ });
+
+ it('preserves a task submitted after the recovery listing, using the runner refusal', async () => {
+ getActiveAgentsFromRunner.mockResolvedValue([{ ...survivingTui, promptSubmitted: false }]);
+ terminateAgentViaRunner.mockResolvedValueOnce({ success: true, terminated: false });
+ await syncRunnerAgents();
+ expect(runnerAgents.get('agent-1').startupErrorAnalysis).toBeUndefined();
+ await syncRunnerAgents();
+ expect(terminateAgentViaRunner).toHaveBeenCalledTimes(1);
+ });
+
+ it('retains ownership and retries a failed startup stop on the next sweep', async () => {
+ getActiveAgentsFromRunner.mockResolvedValue([{ ...survivingTui, promptSubmitted: false }]);
+ terminateAgentViaRunner.mockRejectedValueOnce(new Error('runner connection reset'));
+ await syncRunnerAgents();
+ expect(runnerAgents.has('agent-1')).toBe(true);
+ await syncRunnerAgents();
+ expect(terminateAgentViaRunner).toHaveBeenCalledTimes(2);
+ expect(shellService.registerExternalSession).toHaveBeenCalledTimes(1);
+ });
});
diff --git a/server/services/agentTuiSpawning.js b/server/services/agentTuiSpawning.js
index ac95bea60a..15db8bd410 100644
--- a/server/services/agentTuiSpawning.js
+++ b/server/services/agentTuiSpawning.js
@@ -191,6 +191,7 @@ export async function createAgentTuiSession({
providerAuth: cliProviderAuthDescriptor(provider),
doneSentinelPath,
securityScanGrant,
+ trackPromptSubmission: true,
onData,
onExit: (event) => {
exited = true;
@@ -389,7 +390,9 @@ export async function spawnTuiAgent({
mergeGateIsOwed,
spooler,
session: {
- write: (sessionId, keys) => shellService.writeToSession(sessionId, keys),
+ write: (sessionId, keys, options) => options
+ ? shellService.writeToSession(sessionId, keys, options)
+ : shellService.writeToSession(sessionId, keys),
paste: (sessionId, text, options) => shellService.pasteToSession(sessionId, text, options),
isAlive: (sessionId) => !!shellService.getSession(sessionId),
kill: (sessionId) => shellService.killSession(sessionId),
diff --git a/server/services/agentTuiSpawning/sessionController.js b/server/services/agentTuiSpawning/sessionController.js
index adb72459fa..5c23175875 100644
--- a/server/services/agentTuiSpawning/sessionController.js
+++ b/server/services/agentTuiSpawning/sessionController.js
@@ -266,7 +266,7 @@ function createPasteRetryController({
}
markPromptSubmitted();
submitEnterTimer = scheduleSubmitEnters(
- () => write(SUBMIT_KEY),
+ () => write(SUBMIT_KEY, { promptSubmission: true }),
() => isFinalized()
);
};
@@ -1675,7 +1675,9 @@ export function createTuiSessionController({
pasteController = createPasteRetryController({
agentId,
sessionLabel,
- write: (keys) => session.write(sessionId, keys),
+ write: (keys, options) => options
+ ? session.write(sessionId, keys, options)
+ : session.write(sessionId, keys),
paste: (text, options) => (sessionId ? session.paste(sessionId, text, options) : false),
hasLiveChild: () => session.hasLiveChild(pid),
launchShape,
diff --git a/server/services/agentTuiSpawning/sessionController.test.js b/server/services/agentTuiSpawning/sessionController.test.js
index ed4b5ebd38..d3c1d0e579 100644
--- a/server/services/agentTuiSpawning/sessionController.test.js
+++ b/server/services/agentTuiSpawning/sessionController.test.js
@@ -251,6 +251,7 @@ describe('TUI session controller — teardown owns its own machinery (#8021)', (
await vi.advanceTimersByTimeAsync(300);
await controller.handleData('A sufficiently long prompt for this controller test');
await vi.advanceTimersByTimeAsync(300);
+ expect(write).toHaveBeenCalledWith('session-abcdef12', '\r', { promptSubmission: true });
const banner = "\n⏺ You've hit your session limit · resets 6:00 PM";
await controller.handleData(banner);
diff --git a/server/services/cosRunnerClient.js b/server/services/cosRunnerClient.js
index 2335bfd7f8..687d0cbb44 100644
--- a/server/services/cosRunnerClient.js
+++ b/server/services/cosRunnerClient.js
@@ -80,7 +80,10 @@ const createTuiProxy = (sessionId, pid, state) => {
pid,
onData: (handler) => subscribe(state.dataHandlers, handler),
onExit: (handler) => subscribe(state.exitHandlers, handler),
- write: (data) => emitControl('tui:input', { data }),
+ write: (data, options) => emitControl('tui:input', {
+ data,
+ ...(options?.promptSubmission ? { promptSubmission: true } : {}),
+ }),
resize: (cols, rows) => emitControl('tui:resize', { cols, rows }),
kill: (signal = 'SIGTERM') => emitControl('tui:kill', { signal }),
},
@@ -481,9 +484,13 @@ export async function getActiveAgentsFromRunner() {
/**
* Terminate an agent via the runner (graceful SIGTERM with SIGKILL fallback)
*/
-export async function terminateAgentViaRunner(agentId) {
+export async function terminateAgentViaRunner(agentId, { onlyIfPromptUnsubmitted = false } = {}) {
const response = await fetchWithTimeout(`${COS_RUNNER_URL}/terminate/${agentId}`, {
- method: 'POST'
+ method: 'POST',
+ ...(onlyIfPromptUnsubmitted ? {
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ onlyIfPromptUnsubmitted: true }),
+ } : {}),
}, 30000);
if (!response.ok) {
const error = await readRunnerJson(response);
diff --git a/server/services/cosRunnerClient.test.js b/server/services/cosRunnerClient.test.js
index 6b53a457a8..f59f6a4d26 100644
--- a/server/services/cosRunnerClient.test.js
+++ b/server/services/cosRunnerClient.test.js
@@ -481,6 +481,7 @@ describe('cosRunnerClient', () => {
capturedSocketListeners['tui:output']({ sessionId: session.sessionId, data: 'working' });
session.ptyProcess.write('hello');
+ session.ptyProcess.write('\r', { promptSubmission: true });
session.ptyProcess.resize(120, 40);
capturedSocketListeners['tui:exit']({
sessionId: session.sessionId,
@@ -499,6 +500,9 @@ describe('cosRunnerClient', () => {
sessionId: 'agent-tui-1',
data: 'hello',
});
+ expect(capturedSocket.emit).toHaveBeenCalledWith('tui:input', {
+ sessionId: 'agent-tui-1', data: '\r', promptSubmission: true,
+ });
expect(capturedSocket.emit).toHaveBeenCalledWith('tui:resize', {
sessionId: 'agent-tui-1',
cols: 120,
@@ -548,6 +552,16 @@ describe('cosRunnerClient', () => {
// terminateAgentViaRunner
// ===========================================================================
describe('terminateAgentViaRunner', () => {
+ it('requests an atomic unsubmitted-prompt guard for startup recovery', async () => {
+ fetchWithTimeout.mockResolvedValue(mockResponse(true, { terminated: false }));
+ await expect(terminateAgentViaRunner('agent-123', { onlyIfPromptUnsubmitted: true }))
+ .resolves.toEqual({ terminated: false });
+ expect(fetchWithTimeout).toHaveBeenCalledWith(
+ expect.stringContaining('/terminate/agent-123'),
+ { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{"onlyIfPromptUnsubmitted":true}' },
+ 30000,
+ );
+ });
it('should POST terminate and return result', async () => {
fetchWithTimeout.mockResolvedValue(mockResponse(true, { terminated: true }));
const result = await terminateAgentViaRunner('agent-123');
diff --git a/server/services/shell.js b/server/services/shell.js
index ec72a6975e..0f0b5f222f 100644
--- a/server/services/shell.js
+++ b/server/services/shell.js
@@ -693,12 +693,15 @@ export function getSessionProcess(sessionId) {
/**
* Write input to a shell session
*/
-export function writeToSession(sessionId, data) {
+export function writeToSession(sessionId, data, options) {
const session = shellSessions.get(sessionId);
if (session) {
session.lastInputAt = Date.now();
session.inputRevision = (session.inputRevision || 0) + 1;
- session.pty.write(data);
+ // Runner proxies carry submission evidence alongside the Enter. Native
+ // node-pty sessions retain their ordinary one-argument write contract.
+ if (session.external && options?.promptSubmission) session.pty.write(data, options);
+ else session.pty.write(data);
return true;
}
return false;
diff --git a/server/services/shell.test.js b/server/services/shell.test.js
index 5d827983ac..29d1d3ebd0 100644
--- a/server/services/shell.test.js
+++ b/server/services/shell.test.js
@@ -867,6 +867,8 @@ describe('registerExternalSession / unregisterExternalSession', () => {
const id = shell.registerExternalSession('run-rw', pty, {});
expect(shell.writeToSession(id, 'yes\n')).toBe(true);
expect(pty.write).toHaveBeenCalledWith('yes\n');
+ shell.writeToSession(id, '\r', { promptSubmission: true });
+ expect(pty.write).toHaveBeenCalledWith('\r', { promptSubmission: true });
expect(shell.resizeSession(id, 120, 40)).toBe(true);
expect(pty.resize).toHaveBeenCalledWith(120, 40);
});