Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
import { isPrivateSecurityTask } from '../lib/privateSecurityPolicy.js';
/**
* Agent CLI Spawning
*
* Handles building spawn configurations, stream-json parsing, tool input
* summarization, and Claude settings env injection for agent processes.
*/
import { join } from 'path';
import { readFile } from 'fs/promises';
import { existsSync } from 'fs';
import { spawn } from '../lib/childProcess.js';
import { homedir } from 'os';
import { cosEvents, emitLog } from './cosEvents.js';
// The DEFINING module, not a barrel (#3450) — see the note in
// `agentManagement.js`. This module is a LEAF that `agentLifecycle.js` imports,
// which puts it inside the facade's closure, so the facade is out of reach here.
import { updateAgent, completeAgent, createAgentOutputBatcher, appendAgentOutputLines } from './cosAgentLifecycle.js';
import { release } from './executionLanes.js';
import { completeExecution, errorExecution } from './toolStateMachine.js';
import { analyzeAgentFailure } from './agentErrorAnalysis.js';
import { completeAgentRun } from './agentRunTracking.js';
import { appendRunEvent } from './agentRunEventLog.js';
import { finalizeAgent, releaseAgentLane } from './agentFinalization.js';
import { runSpawnerCompletionCleanup } from './agentCompletionCleanup.js';
import { finalizeAgentRunCommon } from './agentRunFinalize.js';
import { activeAgents, pausedAgents, consumePausedAgentExit, registerSpawnedAgent, unregisterSpawnedAgent } from './agentState.js';
import { PATHS, writeFileGuarded } from '../lib/fileUtils.js';
import { createCodexStderrFormatter } from '../lib/codexCliOutput.js';
import { isKnownCliStderrNoise } from '../lib/cliStderrNoise.js';
import { createStreamingAnsiStripper } from '../lib/ansiStrip.js';
import { PROVIDER_TYPES } from '../lib/aiToolkit/constants.js';
import { createImmediateFallbackSignalDetector } from '../lib/aiToolkit/errorDetection.js';
import { prepareCliPrompt } from '../lib/cliProviderArgs.js';
// buildCliSpawnConfig's per-vendor argv construction dispatches through the
// PROVIDER_VENDORS registry (#3618) instead of a hand-rolled per-vendor
// if-chain — see providerVendors.js for the vendor rows.
import { buildVendorSpawnConfig } from '../lib/providerVendors.js';
import { resolveCliModel, providerSuppliesGithubToken, isOllamaClaudeProvider } from '../lib/providerModels.js';
import { resolveForgeTokenEnv } from './forgeAuth.js';
import { createTranscriptWriteReporter } from './agentTuiSpawning/outputSpooler.js';
import { resolveAgentCliCwd } from '../lib/spawnCwd.js';
import { prepareCliSpawn, killProcessTree, guardChildStdin, deliverChildStdin } from '../lib/bufferedSpawn.js';
import { buildCliChildEnv } from '../lib/cliChildEnv.js';
import { applyCredentialBootstrap, needsProcessGroup, trackDetachedGroup } from '../lib/credentialBootstrap.js';
import { resolvePrOwnership } from '../lib/slashdoInvocation.js';
import { doneSentinelPath } from '../lib/agentSentinel.js';
import { isHostShuttingDown, HOST_SHUTDOWN_REASON } from '../lib/hostShutdown.js';
import { ensureOllamaAgentContext } from './ollamaAgentContext.js';
import { isOllamaBackedProvider } from './providers.js';
import { isPublicReviewRestrictedProfile } from '../lib/agentExecutionProfiles.js';
import { createStreamJsonParser, summarizeToolInput, safeParse } from '../lib/streamJsonParser.js';
const _AGENTS_DIR = PATHS.cosAgents;
/**
* Build spawn command and arguments for a CLI provider.
* Returns { command, args, stdinMode } based on provider type.
*
* `settingsEnv` is the `~/.claude/settings.json` env block (from
* `getClaudeSettingsEnv()`). It MUST be folded into the Bedrock-mapping env
* because that is how a Bedrock host commonly supplies `CLAUDE_CODE_USE_BEDROCK`
* to the spawned child (see the spawn env below at the `claudeSettingsEnv`
* merge) — without it, a settings-only Bedrock box would map against an env
* missing the flag and still emit a bare, Bedrock-invalid `--model`.
*
* Per-vendor argv construction is dispatched through the PROVIDER_VENDORS
* registry (#3618) — see providerVendors.js for each vendor's `spawnArgs` row
* and the file header on the (preserved, not silently "fixed") asymmetries vs
* `buildCliArgs` (e.g. codex never forwards `provider.args` here, and there is
* no gemini-cli row so a legacy gemini-cli provider falls through to claude's
* default, exactly as before this registry existed).
*/
export function buildCliSpawnConfig(provider, model, settingsEnv = {}, {
systemPromptFile = null,
effort = null,
maxConcurrentThreads = null,
safetyProfile = null,
} = {}) {
// Configured-default sentinels (Codex / Antigravity / Grok Build) → null so
// the CLI uses its own default without a --model flag.
const effectiveModel = resolveCliModel(model);
return buildVendorSpawnConfig(provider, {
effectiveModel,
effort,
maxConcurrentThreads,
systemPromptFile,
settingsEnv,
safetyProfile,
});
}
/**
* Check if a provider is a Claude CLI provider that needs settings.json env injection.
*/
export const isClaudeCliProvider = (provider) =>
provider?.type === PROVIDER_TYPES.CLI && (provider.id === 'claude-code' || provider.id === 'claude-code-bedrock');
/**
* Check if a provider is a TUI-backed agent provider (Claude Code, Codex,
* Antigravity, etc. that run in a PTY). Used by callers that need to branch
* between headless CLI/API runs and TUI shell sessions.
*/
export { isTuiProvider } from '../lib/providerTypes.js';
/**
* Read env vars from ~/.claude/settings.json to inject into Claude CLI spawns.
* Ensures user's Bedrock/provider config (CLAUDE_CODE_USE_BEDROCK, AWS_PROFILE, etc.)
* is present in spawned agent environments even if PM2 was started without them.
*/
let _claudeSettingsEnvCache = null;
let _claudeSettingsEnvCacheTime = 0;
const SETTINGS_CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes
export async function getClaudeSettingsEnv() {
if (_claudeSettingsEnvCache !== null && (Date.now() - _claudeSettingsEnvCacheTime) < SETTINGS_CACHE_TTL_MS) return _claudeSettingsEnvCache;
try {
const settingsPath = join(homedir(), '.claude', 'settings.json');
if (existsSync(settingsPath)) {
const raw = await readFile(settingsPath, 'utf-8');
const settings = JSON.parse(raw);
_claudeSettingsEnvCache = settings.env || {};
} else {
_claudeSettingsEnvCache = {};
}
} catch (err) {
console.warn(`⚠️ Failed to read claude settings: ${err.message}`);
_claudeSettingsEnvCache = {};
}
_claudeSettingsEnvCacheTime = Date.now();
return _claudeSettingsEnvCache;
}
/**
* What went wrong with a direct-CLI run, or `null` when there is nothing to
* analyze — a successful run, or a private security assessment whose transcript
* must not be read (its findings stay in the local assessment archive).
*
* The raw stream buffer is preferred over the display buffer: it still carries
* the full stream-json payload, error details included. A fallback signal the
* run already raised (a usage-limit hit, say) outranks a keyword sweep.
*/
function resolveCliErrorAnalysis({ finalSuccess, task, model, rawStreamBuffer, outputBuffer, immediateFallbackAnalysis }) {
if (finalSuccess || isPrivateSecurityTask(task)) return null;
return immediateFallbackAnalysis || analyzeAgentFailure(rawStreamBuffer || outputBuffer, task, model);
}
/**
* Spawn agent directly (fallback when runner not available).
* Completion cleanup is not injected: `runSpawnerCompletionCleanup` is imported at top level, since
* nothing in its static closure reaches this module or agentLifecycle.js.
*/
export async function spawnDirectly({
agentId,
task,
prompt,
workspacePath,
model,
provider,
runId,
cliConfig,
agentDir,
executionId,
laneName,
prOpenedBy,
safetyProfile = null,
}) {
const fullCommand = `${cliConfig.command} ${cliConfig.args.join(' ')} <<< "${(task.description || '').substring(0, 100)}..."`;
const ROOT_DIR = PATHS.root;
// CD no-worktree tasks get an isolated scratch cwd so native AGENTS.md
// discovery cannot reach the PortOS repo tree (#4650). Everyone else keeps
// workspacePath, falling back to the repo root when it was omitted.
const cwd = resolveAgentCliCwd({ workspacePath, fallbackRoot: ROOT_DIR, task, agentId });
// Direct CLI agents bypass runner.js, so their Ollama-backed harnesses need
// the same daemon context preparation as runner and TUI launches. This is
// deliberately before spawn: a per-request window cannot reach a CLI that
// talks to Ollama on its own.
const ollamaContext = isOllamaBackedProvider(provider)
? await ensureOllamaAgentContext(provider, { model })
: null;
if (ollamaContext?.warning) {
emitLog('warn', `Agent ${agentId} Ollama context preparation: ${ollamaContext.warning}`, {
agentId,
taskId: task.id,
});
}
// Two independent async env lookups, resolved together: Claude's
// ~/.claude/settings.json Bedrock config (CLAUDE_CODE_USE_BEDROCK, AWS_PROFILE,
// etc., present even if PM2 lacks them) and the repo-owner-pinned GH_TOKEN
// (so the agent's own `gh pr create` auths as the right account — see
// resolveForgeTokenEnv; `{}` when there's no owner match). Skip the token probe
// entirely when the provider supplies its own GH_TOKEN/GITHUB_TOKEN so its
// explicit credential wins (gh prefers GH_TOKEN, so injecting one would shadow a
// provider GITHUB_TOKEN).
// A third rides alongside: the loopback PortOS session token this agent's own
// `curl` snippets need when the install has an instance password set (see
// agentApiAuth.js). It is `{}` when auth is off, and never resolved for a
// public-content stage.
const [claudeSettingsEnv, forgeTokenEnv, agentApiEnv] = isPublicReviewRestrictedProfile(safetyProfile)
? [{}, {}, {}]
: await Promise.all([
isClaudeCliProvider(provider) ? getClaudeSettingsEnv() : Promise.resolve({}),
providerSuppliesGithubToken(provider) ? Promise.resolve({}) : resolveForgeTokenEnv(cwd),
// Deferred, not a static import: `agentApiAuth.js` pulls the auth/session
// subtree, and this module is reached by enough of the suite that the eager
// edge added ~190 static module instantiations — over the budget
// `lib/importScoping.test.js` holds. The sibling TUI and runner spawn sites
// import it normally; almost nothing reaches those, so they cost nothing.
import('./agentApiAuth.js').then(({ resolveAgentApiEnv }) => resolveAgentApiEnv({ safetyProfile })),
]);
// Shared composition (provider.envVars + OpenCode models map + PWD pin +
// CLAUDECODE strip) — see buildCliChildEnv. forgeTokenEnv/claudeSettingsEnv go
// in `before` so they sit UNDER provider.envVars and an explicit provider
// GH_TOKEN override still wins. `guard: true` prepends the pm2 shim onto the
// final PATH so a `--dangerously-skip-permissions` agent can't `pm2 kill` the
// shared daemon.
const childEnv = buildCliChildEnv({
before: { ...forgeTokenEnv, ...claudeSettingsEnv, ...agentApiEnv },
provider,
model,
cwd,
guard: true,
safetyProfile,
});
// Resolve a bare npm-installed CLI (a .cmd/.bat shim on Windows) to its real
// path and wrap a shim as `cmd.exe /c <path>` so spawn() under shell:false
// can launch it — without this Windows can't find e.g. `opencode.cmd` from the
// bare name → spawn ENOENT (errno -4058) → startup-failure. Mirrors the
// working "Run Prompt" path (server/services/runner.js); resolved against
// childEnv so a provider PATH override is honored. See issue #2243.
// Deliver the prompt per provider convention: antigravity as the --print
// VALUE and kimi as the --prompt VALUE (no stdin); grok's `--prompt-file
// /dev/stdin` via stdin (POSIX) / temp file (Windows); every other provider via
// stdin (writePromptToStdin=true).
const { args: deliveredArgs, useStdin: writePromptToStdin, cleanup: cleanupPromptFile } = prepareCliPrompt(cliConfig.command, cliConfig.args, prompt, { cwd });
// A credential-bootstrap-configured provider spawns its bootstrap CLI in
// front of the harness instead of the harness directly — see
// credentialBootstrap.js. Applied AFTER prepareCliPrompt, which still keys
// prompt-delivery convention off the harness's own command — and never under
// a public-review `safetyProfile`, whose enforced recipe is the sandbox.
const { command: bootstrappedCommand, args: bootstrappedArgs, wrapped } = applyCredentialBootstrap(provider, cliConfig.command, deliveredArgs, { safetyProfile });
// A bootstrap-wrapped child is the WRAPPER supervising the harness, so every
// stop/timeout/cancel for this agent must signal the whole process group —
// here, and in agentManagement's pause/terminate/force-kill, which reads the
// flag back off the activeAgents entry below (#7496). The private-security
// sandbox wrap below is likewise a supervising parent, so it keeps the flag.
const processGroup = needsProcessGroup(wrapped);
const preparedSpawn = prepareCliSpawn(bootstrappedCommand, bootstrappedArgs, childEnv);
const isolatedSpawn = isPrivateSecurityTask(task)
? await import('../lib/privateSecuritySandbox.js')
.then(({ preparePrivateSecuritySpawn }) => preparePrivateSecuritySpawn({ ...preparedSpawn, env: childEnv, cwd, provider }))
.catch(async () => {
// No child exists yet, so no error/close listener can finish this run.
// Keep sandbox paths out of shared diagnostics and fail closed locally.
const message = 'Private assessment could not prepare its isolated local harness. Verify the CLI and macOS sandbox, then retry.';
cleanupPromptFile();
releaseAgentLane({ agentId, success: false, exitCode: 1, executionId, laneName, errorExecutionMessage: message });
await finalizeAgent({
agentId, task, runId, providerId: provider.id, success: false, exitCode: 1,
duration: 0, outputBuffer: '', workspacePath, prExpected: false,
error: message, completionReason: 'spawn-error',
errorAnalysis: { category: 'actionable', error: message, suggestedFix: message },
});
cosEvents.emit('agent:error', { agentId, taskId: task.id, error: message });
return null;
})
: { ...preparedSpawn, env: childEnv };
if (!isolatedSpawn) return null;
const { command: spawnCommand, args: spawnArgs } = isolatedSpawn;
const claudeProcess = spawn(spawnCommand, spawnArgs, {
cwd,
shell: false,
stdio: ['pipe', 'pipe', 'pipe'],
env: isolatedSpawn.env,
// Never `unref()`ed — this stays an ordinary awaited child, just one in
// its own signalable process group. See needsProcessGroup.
detached: processGroup
});
// Remember the detached group so the graceful-shutdown sweep can reach it:
// detaching moved this child out of the server's own process group, and a
// shutdown driven by a signal to THAT group would otherwise orphan it (#7496).
trackDetachedGroup(claudeProcess, processGroup);
// Every child listener is registered HERE, in the same tick as spawn(), into
// forwarding shims that buffer until the real handler bodies exist further
// down (#5791). Those bodies close over state built by the async setup that
// follows — and that setup yields for real state-file I/O (`await
// updateAgent(..., { pid })`), not just a microtask. A CLI that starts and
// exits immediately (bad flag, instant auth refusal) emits its entire life
// cycle inside that window: without the shims its stdout/stderr is lost and
// its `close` lands on nothing, leaving the run record, the execution lane
// and the `activeAgents` entry non-terminal until the orphan reaper notices.
// Buffering rather than hoisting the handler bodies keeps the change local.
//
// spawn() can also hand back a handle with no pid or stdio when command
// lookup fails, hence the optional chaining on the stream registrations.
let pendingSpawnError = null;
let handleSpawnError = null;
// One ordered queue for both streams: buffering them separately would lose
// the relative order of interleaved stdout/stderr chunks in the transcript.
const pendingChildOutput = [];
let pendingClose = null;
let handleStdout = null;
let handleStderr = null;
let handleClose = null;
claudeProcess.on('error', (err) => {
if (handleSpawnError) void handleSpawnError(err);
else pendingSpawnError = err;
});
claudeProcess.stdout?.on('data', (data) => {
if (handleStdout) handleStdout(data);
else pendingChildOutput.push({ stream: 'stdout', data });
});
claudeProcess.stderr?.on('data', (data) => {
if (handleStderr) handleStderr(data);
else pendingChildOutput.push({ stream: 'stderr', data });
});
claudeProcess.on('close', (code) => {
if (handleClose) void handleClose(code);
else pendingClose = { code };
});
// Same reasoning for the stdin pipe: a child that exits before reading it
// emits EPIPE, and an unlistened stream 'error' out here would crash the
// server. The 'error'/'exit' handlers below settle the run with the real cause.
guardChildStdin(claudeProcess);
const spawnedPid = claudeProcess.pid;
if (spawnedPid != null) {
registerSpawnedAgent(spawnedPid, {
fullCommand,
agentId,
taskId: task.id,
model,
workspacePath: cwd,
prompt: (task.description || '').substring(0, 500)
});
deliverChildStdin(claudeProcess, writePromptToStdin ? prompt : null, `agent ${agentId}`);
activeAgents.set(agentId, {
process: claudeProcess,
// Read by agentManagement's pause/terminate/force-kill paths, which hold
// only this entry and cannot re-derive the bootstrap wrap (#7496).
processGroup,
taskId: task.id,
startedAt: Date.now(),
runId,
pid: spawnedPid,
providerId: provider.id,
executionId,
laneName
});
// Store PID in persisted state for zombie detection only after spawn gave
// us a live process identity.
await updateAgent(agentId, { pid: spawnedPid });
}
let outputBuffer = '';
let outputRevision = 0;
let rawStreamBuffer = ''; // Raw stdout for stream-json (used for error analysis)
let hasStartedWorking = false;
// Deliberately NOT `hasStartedWorking`: that flag also flips on a 3s
// initialization timeout with no output behind it, so reusing it would file a
// `run.output` for a run that has produced nothing — the exact case (a
// provider that never spoke) the ledger is supposed to make visible.
let firstOutputRecorded = false;
/**
* Record the run's first observed output, once (#4540) — never per chunk. A
* per-chunk event would make the ledger a copy of the output it deliberately
* redacts and would exhaust the retention bound in minutes. What the one event
* buys is time-to-first-output: a run that stalled after speaking and a run
* that never spoke at all are indistinguishable in the mutable record.
*
* Called from BOTH streams. Several providers say everything they have to say
* on stderr (codex's whole progress feed is stderr), and that output lands in
* the same transcript — a stdout-only recorder would file those runs as silent.
* The explicit key keeps the append idempotent however the two callers race.
*/
const recordFirstOutput = (source, chars) => {
if (firstOutputRecorded) return;
firstOutputRecorded = true;
return appendRunEvent({
kind: 'run.output',
runId,
agentId,
taskId: task.id,
eventId: `output:${agentId}:${runId || 'no-run'}:first`,
data: { source, firstChunkChars: chars },
});
};
const outputFile = join(agentDir, 'output.txt');
const isStreamJson = cliConfig.streamFormat === 'stream-json';
const streamParser = isStreamJson ? createStreamJsonParser() : null;
const codexStderrFormatter = provider.id === 'codex' ? createCodexStderrFormatter(prompt) : null;
const transcriptWriteReporter = createTranscriptWriteReporter({ agentId });
// A headless CLI still colors its progress output: `opencode run` emits bare
// `\x1B[0m` resets around its status line. Those bytes reached output.txt and
// the live tail verbatim, and the browser drops only the ESC itself — so the
// card rendered `[stderr] [0m` and a working agent was indistinguishable from
// a wedged one. Strip per stream (each stripper is stateful, buffering an
// escape split across two `data` chunks) BEFORE the fallback-signal detector,
// which matches on provider text that color codes would otherwise break up.
// A stream-json stdout is NDJSON, which escapes an ESC into its six-character
// JSON form and never carries the raw byte — so the busiest stdout stream in
// the system skips the scan entirely.
const stripStdoutAnsi = isStreamJson ? (text) => text : createStreamingAnsiStripper();
const stripStderrAnsi = createStreamingAnsiStripper();
let immediateFallbackAnalysis = null;
const detectImmediateFallbackSignal = createImmediateFallbackSignalDetector();
let persistedOutputRevision = -1;
let persistedOutputLength = 0;
let transcriptFileWriteTail = Promise.resolve();
const persistTranscript = (bytes, revision) => {
// Serialize timer/manual drains. Only a successful write establishes the
// prefix that the next snapshot can extend; an append may fail partway.
const write = transcriptFileWriteTail.then(async () => {
const append = persistedOutputRevision === revision && persistedOutputLength <= bytes.length;
const delta = append ? bytes.slice(persistedOutputLength) : bytes;
if (append && !delta) return;
await writeFileGuarded(outputFile, delta, { flag: append ? 'a' : 'w' });
persistedOutputRevision = revision;
persistedOutputLength = bytes.length;
}).catch(err => {
// Rewrite the next snapshot after any failure: retrying an uncertain
// append could duplicate bytes, while skipping it could lose output.
persistedOutputRevision = -1;
maintenance.markResourceUnsettled('agent', agentId);
return transcriptWriteReporter.report('output.txt', err);
});
transcriptFileWriteTail = write;
return write;
};
// Debounced state-write batcher for streamed output. stdout/stderr `data`
// events fire per chunk (and stream-json yields many lines per chunk), so a
// per-line/per-chunk appendAgentOutput would round-trip a full state
// load+save each time. The batcher coalesces a ~250ms window; the close/error
// handlers `await outputBatcher.flush()` so the final transcript and state
// lines persist as one admitted batch before the agent is finalized.
// Findings stay in the local transcript/report, never the shared agent stream.
const outputBatcher = createAgentOutputBatcher(agentId, { flushBatch: batch => {
// Capture bytes at the same instant as this batch, before waiting for the
// cut. Chunks received during I/O belong to the next flush, not this pair.
const bytes = outputBuffer;
const revision = outputRevision;
return withBackupAssetPublication(() => Promise.all([
persistTranscript(bytes, revision),
isPrivateSecurityTask(task) ? undefined : appendAgentOutputLines(agentId, batch)
.catch(err => transcriptWriteReporter.report('state', err)),
]));
} });
if (ollamaContext?.warning) outputBatcher.push(ollamaContext.warning);
if (ollamaContext?.applied) outputBatcher.push(`🪟 Reloaded Ollama at a ${ollamaContext.contextLength}-token context window`);
// Expose a drain hook on the activeAgents entry so the user-terminate/kill
// paths (agentManagement.js) can flush pending batched output before they
// mark the agent complete — otherwise up to one debounce window of
// stdout/stderr could land after the terminal record. (The close handler
// still drains too; flush() is idempotent.)
const cliAgentEntry = activeAgents.get(agentId);
// Drain the serialized write chain (defined just below) before flushing the
// batcher so any enqueued-but-unrun stdout/stderr push lands first.
if (cliAgentEntry) cliAgentEntry.flushOutput = async () => {
await drainTranscriptWrites();
await outputBatcher.flush();
await transcriptFileWriteTail;
};
// Serialize stdout/stderr buffer updates before their shared debounced file
// and state flush, so interleaved setup awaits cannot reorder the transcript
// (#2384). Fallback
// detection stays OUTSIDE this chain — it runs synchronously in the raw
// listener before the enqueue, so a blocked earlier write can never delay
// killing the provider on a usage-limit signal. The close/error handlers
// `await drainTranscriptWrites()` before finalize so the tail lands first.
// Mirrors shell.js `hookQueue` and pipeline/issuesShared.js `queueSeriesIssuesWrite`.
let transcriptWriteTail = Promise.resolve();
const enqueueTranscriptWrite = (fn) => {
transcriptWriteTail = transcriptWriteTail.then(fn).catch((err) => {
// Runs in a child-process callback — a rejection here would escape as an
// unhandled rejection and crash Node. output.txt is best-effort; log+swallow.
console.error(`❌ agentCli transcript write failed for ${agentId}: ${err.message}`);
});
return transcriptWriteTail;
};
const drainTranscriptWrites = () => transcriptWriteTail;
const stopForImmediateFallbackSignal = (text) => {
if (immediateFallbackAnalysis || claudeProcess.killed) return;
const analysis = detectImmediateFallbackSignal(text);
if (!analysis) return;
immediateFallbackAnalysis = analysis;
emitLog('warn', `Agent ${agentId} detected provider fallback signal (${analysis.category}); stopping ${provider.name || provider.id}`, {
agentId,
taskId: task.id,
providerId: provider.id,
category: analysis.category
});
// killProcessTree so a Windows cmd.exe-wrapped shim's real child isn't orphaned (#2243).
killProcessTree(claudeProcess, 'SIGTERM', { processGroup });
};
// If no output after 3 seconds, transition from initializing to working to show progress
const initializationTimeout = setTimeout(async () => {
try {
if (!hasStartedWorking && activeAgents.has(agentId)) {
hasStartedWorking = true;
await updateAgent(agentId, { metadata: { phase: 'working' } });
emitLog('info', `Agent ${agentId} working (after initialization delay)...`, { agentId, phase: 'working' });
}
} catch (err) {
console.error(`❌ agentCliSpawning init timeout failed for ${agentId}: ${err.message}`);
}
}, 3000);
handleStdout = (data) => {
try {
const raw = data.toString();
const text = stripStdoutAnsi(raw);
// Detect fallback signals SYNCHRONOUSLY, before enqueuing any transcript
// mutation — a blocked earlier write must never delay killing the provider
// on a usage-limit signal (#2384).
stopForImmediateFallbackSignal(text);
// Serialize the transcript body so two `data` events can't interleave their
// awaits and reorder output.txt / the batched live tail.
enqueueTranscriptWrite(async () => {
// Raw length, not stripped: a chunk that was ONLY color codes is still
// proof the child is alive, and reporting 0 would read as "no output yet".
await recordFirstOutput('cli-stdout', raw.length);
if (!hasStartedWorking) {
hasStartedWorking = true;
await updateAgent(agentId, { metadata: { phase: 'working' } });
emitLog('info', `Agent ${agentId} working...`, { agentId, phase: 'working' });
}
if (streamParser) {
// Parse stream-json and emit extracted text lines (cap buffer at 512KB for error analysis)
rawStreamBuffer += text;
if (rawStreamBuffer.length > 512 * 1024) {
rawStreamBuffer = rawStreamBuffer.slice(-512 * 1024);
}
const lines = streamParser.processChunk(text);
for (const line of lines) outputBuffer += line + '\n';
outputBatcher.push(lines);
} else {
// Non-stream providers: emit stdout as-is once decolored. A chunk that
// was purely terminal control has nothing left to show. Unlike stderr
// below, whitespace is NOT dropped here — a blank line is legitimate
// formatting when it isn't wearing an `[stderr]` tag.
if (!text) return;
outputBuffer += text;
outputBatcher.push(text);
}
});
} catch (err) {
console.error(`❌ agentCli stdout handler failed: ${err.message}`);
}
};
handleStderr = (data) => {
try {
const raw = data.toString();
const text = stripStderrAnsi(raw);
// Synchronous fallback detection before the serialized write (see stdout).
stopForImmediateFallbackSignal(`[stderr] ${text}`);
enqueueTranscriptWrite(async () => {
await recordFirstOutput('cli-stderr', raw.length);
// Codex stderr: show thinking + tool names, skip config dump and command output
if (codexStderrFormatter) {
const lines = codexStderrFormatter.processChunk(text);
for (const line of lines) outputBuffer += line + '\n';
outputBatcher.push(lines);
return;
}
// A chunk that decolors down to whitespace was pure terminal control
// (`opencode run` emits a bare reset per progress redraw). Tagging it
// `[stderr]` would add one blank noise line to the tail per redraw.
const trimmed = text.trim();
if (!trimmed || isKnownCliStderrNoise(trimmed)) return;
outputBuffer += `[stderr] ${text}`;
outputBatcher.push(`[stderr] ${text}`);
});
} catch (err) {
console.error(`❌ agentCli stderr handler failed: ${err.message}`);
}
};
handleSpawnError = async (err) => {
// Runs outside the request lifecycle — an uncaught throw from the awaited
// completeAgent/completeAgentRun would crash the process, so wrap the body.
try {
clearTimeout(initializationTimeout);
cleanupPromptFile();
console.error(`❌ Agent ${agentId} spawn error: ${err.message}`);
outputBatcher.push(`❌ Agent ${agentId} spawn error: ${err.message}`);
// Release execution lane
if (laneName) {
release(agentId);
}
// Complete tool execution tracking with error
if (executionId) {
errorExecution(executionId, { message: err.message, category: 'spawn-error' });
completeExecution(executionId, { success: false });
}
const agentDataErr = activeAgents.get(agentId);
if (agentDataErr?.killTimer) {
clearTimeout(agentDataErr.killTimer);
agentDataErr.killTimer = null;
}
cosEvents.emit('agent:error', { agentId, error: err.message });
// Drain queued transcript writes before flushing the batcher + recording
// the run so outputBuffer reflects everything that streamed (#2384).
await drainTranscriptWrites();
await outputBatcher.flush();
await transcriptFileWriteTail;
await completeAgent(agentId, { success: false, error: err.message });
await completeAgentRun(runId, isPrivateSecurityTask(task) ? 'Private security assessment failed; inspect its local assessment archive.' : outputBuffer, 1, 0, { message: err.message, category: 'spawn-error' });
unregisterSpawnedAgent(claudeProcess.pid);
activeAgents.delete(agentId);
if (!claudeProcess.pid) await maintenance.finishResource('agent', agentId);
} catch (handlerErr) {
maintenance.markResourceUnsettled('agent', agentId);
console.error(`❌ Agent ${agentId} error handler failed: ${handlerErr.message}`);
activeAgents.delete(agentId);
}
};
/**
* The close handler threw — outside any request lifecycle, so nothing above
* will catch it and the process now SURVIVES rather than restarting. Leave
* nothing held: free a lane the throw beat, and persist a terminal record so
* the agent/run aren't stuck `running` with no live process to finish them.
*
* A paused agent was already persisted + lane-released by `markAgentPaused`, so
* it gets ONLY the in-memory cleanup — finalizing it as failed here would
* overwrite the paused state and break a later resume.
*
* Each persistence call is guarded on its own so a failure can't re-crash.
*/
const recoverFromCloseHandlerFailure = async (handlerErr, { code, laneReleased, outputBuffer }) => {
console.error(`❌ Agent ${agentId} close handler error: ${handlerErr.message}`);
if (pausedAgents.has(agentId)) {
consumePausedAgentExit(agentId);
unregisterSpawnedAgent(claudeProcess.pid);
activeAgents.delete(agentId);
return;
}
if (!laneReleased) {
try {
releaseAgentLane({
agentId,
success: false,
exitCode: code,
executionId,
laneName,
errorExecutionMessage: `Agent close handler error: ${handlerErr.message}`,
});
} catch (releaseErr) {
console.error(`❌ Agent ${agentId} lane release failed during recovery: ${releaseErr.message}`);
}
}
try {
await completeAgent(agentId, { success: false, error: `Close handler error: ${handlerErr.message}` });
} catch (completeErr) {
console.error(`❌ Agent ${agentId} completeAgent failed during recovery: ${completeErr.message}`);
}
try {
await completeAgentRun(runId, isPrivateSecurityTask(task) ? 'Private security assessment failed; inspect its local assessment archive.' : outputBuffer, 1, 0, { message: handlerErr.message, category: 'close-handler-error' });
} catch (runErr) {
console.error(`❌ Agent ${agentId} completeAgentRun failed during recovery: ${runErr.message}`);
}
unregisterSpawnedAgent(claudeProcess.pid);
activeAgents.delete(agentId);
};
handleClose = async (code) => {
// Runs outside the request lifecycle — a throw from outputBatcher.flush,
// analyzeAgentFailure, or finalizeAgent would re-escape this async handler
// as an unhandled rejection and crash the process. The inner try/finally
// only covers finalizeAgent's cleanup; this outer guard is the crash net.
// `laneReleased` lets the recovery path free a still-held execution lane
// (a throw before releaseAgentLane) without double-releasing one that ran.
let laneReleased = false;
try {
clearTimeout(initializationTimeout);
cleanupPromptFile();
const success = code === 0;
const agentData = activeAgents.get(agentId);
const duration = Date.now() - (agentData?.startedAt || Date.now());
// If terminateAgent scheduled a SIGKILL fallback, the process exited
// before it fired — clear it so we don't leak the timer.
if (agentData?.killTimer) {
clearTimeout(agentData.killTimer);
agentData.killTimer = null;
}
// This run's own sentinel (see doneSentinelName) — a worktree-less agent
// shares its workspace and must not read a sibling's signal as its own.
const sentinelPath = doneSentinelPath(cwd, agentId);
const completionSentinelPresent = !!sentinelPath && existsSync(sentinelPath);
// A mid-stream fallback signal (e.g. usage-limit hit) kills the CLI; if it
// races to exit 0, don't record success or the fallback/retry never fires.
// A completion sentinel written before host shutdown is also authoritative:
// TreeKill may surface its otherwise-completed child with a null exit code.
const completedBeforeHostShutdown = isHostShuttingDown() && completionSentinelPresent;
const rawSuccess = (success || completedBeforeHostShutdown) && !immediateFallbackAnalysis;
// Drain any queued transcript writes before reading/appending outputBuffer
// for finalization — a still-pending stdout/stderr write would otherwise
// land after the terminal record or clobber the final output.txt (#2384).
// No new `data` events fire after 'close', so a single drain is sufficient.
await drainTranscriptWrites();
// Flush remaining stream parser data (persists the tail of the transcript
// before any early-return, so a paused agent's output.txt is complete for
// the resume modal).
if (streamParser) {
const remaining = streamParser.flush();
for (const line of remaining) {
outputBuffer += line + '\n';
outputBatcher.push(line);
}
// Use the parsed final result for the output file if available
const finalResult = streamParser.getFinalResult();
if (finalResult) {
outputBuffer = finalResult;
outputRevision++;
}
}
if (codexStderrFormatter) {
for (const line of codexStderrFormatter.flush()) {
outputBuffer += line + '\n';
outputBatcher.push(line);
}
}
// Drain pending output to state before finalize so the transcript tail
// lands before the agent's terminal record (covers the paused early-return
// below too, since output.txt is written next).
await withBackupAssetPublication(async () => {
await outputBatcher.flush();
await transcriptFileWriteTail;
await writeFileGuarded(outputFile, outputBuffer).catch((err) => {
maintenance.markResourceUnsettled('agent', agentId);
return transcriptWriteReporter.report('output.txt', err);
});
});
// The teardown both in-process spawners share: paused early return, then
// the host-shutdown abandon gate, the user-termination consume, the
// final success/error derivation, and the lane release (done before the
// error-analysis + state-write chain, since lanes serialize related work).
// The transcript is already flushed and output.txt written above, so every
// early-out below still leaves a complete record. See agentRunFinalize.js.
const finalizeOutcome = finalizeAgentRunCommon({
agentId,
agentData,
success: rawSuccess,
error: null,
exitCode: code,
duration,
executionId,
laneName,
sentinelPresent: completionSentinelPresent,
});
if (finalizeOutcome.outcome === 'paused') { await maintenance.finishResource('agent', agentId); return; }
if (finalizeOutcome.outcome === 'abandoned') {
outputBatcher.push('🛑 PortOS restarted while this agent was running — the run was interrupted, not completed. Its worktree is preserved and the task will resume.');
emitLog('info', `Agent ${agentId} interrupted by a PortOS host restart — preserved for resume`, { agentId, phase: 'interrupted' });
await Promise.all([
outputBatcher.flush().catch(() => {}),
updateAgent(agentId, { metadata: { phase: 'interrupted', interruptedBy: HOST_SHUTDOWN_REASON } })
.catch(err => emitLog('warn', `Could not mark agent ${agentId} interrupted: ${err.message}`, { agentId })),
]);
// activeAgents entry left in place — the shutdown handler reads that map
// to name the agents in the host-shutdown marker.
return;
}
const { finalSuccess, finalError, terminatedByUser } = finalizeOutcome;
laneReleased = true;
const errorAnalysis = resolveCliErrorAnalysis({
finalSuccess, task, model, rawStreamBuffer, outputBuffer, immediateFallbackAnalysis,
});
// Every CLI agent that is a real coding harness drives its own push → PR →
// review → merge (#3733): a slashdo-capable Claude runs `/simplify` +
// `/do:pr`, codex/grok/agy/OpenCode run the plain `git`/`gh` equivalent from
// the same prompt (see buildCliCompletionSection in agentPromptBuilder.js).
// Resolved from the live provider descriptor — the same reading the TUI
// path takes up front — so PortOS doesn't double-fire push+PR creation; see
// `resolvePrOwnership` for why finalize's `prClaimExpected` and cleanup's
// `agentOpensOwnPr` are two predicates (#3358).
const prOwnership = resolvePrOwnership({
task,
persistedPrOpenedBy: prOpenedBy,
providerId: provider?.id,
providerCommand: provider?.command,
leanMode: isOllamaClaudeProvider(provider),
});
// Whether finalize's check ACTUALLY produced a forge answer (filled in from
// its return below) — not the same question as whether one was expected.
// Finalize substitutes `{ok:true}` for a user-terminated run and for a check
// that threw, and a throw from finalize skips the assignment entirely; in all
// three cases nothing was verified, so cleanup must ask rather than stand down.
let prClaimVerified = false;
let branchProvenEmpty = false;
// try/finally so a throw from finalizeAgent still runs the local cleanup
// (the shared completion dispatch, pid unregister, activeAgents delete).
// A PR-claim downgrade must reach cleanup, or a run that opened no PR is
// cleaned up as a success and loses its retry state (#3358).
let cleanupSuccess = finalSuccess;
try {
const finalized = await finalizeAgent({
agentId,
task,
runId: agentData?.runId || runId,
providerId: agentData?.providerId || provider.id,
success: finalSuccess,
exitCode: code,
duration,
outputBuffer,
finalSummary: streamParser?.getCompletionResult() ?? null,
errorAnalysis,
terminatedByUser,
error: finalError || undefined,
completionReason: terminatedByUser ? 'user-terminated' : undefined,
workspacePath: cwd,
prExpected: prOwnership.prClaimExpected,
// The run window the commit criterion is evaluated against (#3637).
startedAt: agentData?.startedAt ?? null,
});
if (finalized && typeof finalized.success === 'boolean') cleanupSuccess = finalized.success;
if (finalized?.prClaimVerified === true) prClaimVerified = true;
if (finalized?.branchProvenEmpty === true) branchProvenEmpty = true;
} finally {
// Pipeline progression → worktree cleanup with the PR disposition →
// retry-hold release, in the one owner both in-process spawners share.
// Caught so a throw there cannot skip the in-memory teardown below — this
// is a child 'close' handler, outside any request lifecycle.
await runSpawnerCompletionCleanup({
agentId,
task,
success: cleanupSuccess,
prOwnership,
prClaimVerified,
branchProvenEmpty,
outputBuffer,
}).catch(err => {
maintenance.markResourceUnsettled('agent', agentId);
console.error(`❌ CLI completion cleanup failed for ${agentId}: ${err.message}`);
});
unregisterSpawnedAgent(agentData?.pid || claudeProcess.pid);
activeAgents.delete(agentId);
await import('./agentRepoStateVerification.js').then(module => module.resumeAgentCompletionRetirement(agentId))
.catch(error => emitLog('warn', `Completion retirement deferred for ${agentId}: ${error.message}`, { agentId }));
}
maintenance.finishResource('agent', agentId);
} catch (handlerErr) {
maintenance.markResourceUnsettled('agent', agentId);
await recoverFromCloseHandlerFailure(handlerErr, { code, laneReleased, outputBuffer });
}
};
// Replay whatever the child emitted while the async setup above was yielding
// (#5791). Output first, so the transcript is complete before a buffered
// terminal event finalizes the run — and it is enqueued onto
// `transcriptWriteTail` here, before either terminal handler's
// `drainTranscriptWrites()` captures that tail.
for (const { stream, data } of pendingChildOutput) {
if (stream === 'stderr') handleStderr(data);
else handleStdout(data);
}
pendingChildOutput.length = 0;
// A failed spawn emits 'error' AND then 'close'; the error path already
// finalizes the run, so replay exactly one terminal event, never both.
if (pendingSpawnError) void handleSpawnError(pendingSpawnError);
else if (pendingClose) void handleClose(pendingClose.code);
return agentId;
}