A desktop coding agent with free access to frontier models.
TypeScript
/**
* Blue/green rollover for the managed OpenCode engine.
*
* A config change the engine can only read at instance build time (provider
* blocks, plugins, agents, permissions) normally forces a dispose, which
* aborts every run in flight. Deferring the dispose until the engine idles
* avoids the abort but leaves config hostage to long sessions.
*
* This pool takes the third option when a reload lands on a busy engine:
* spawn a standby with the new config, point every new request at it, and
* keep the old engine alive only until its running sessions finish. Sessions
* live in OpenCode's shared SQLite, so only the live run is process-bound.
*
* Steady state stays one engine. During a drain there are two, and never more
* than two long-lived ones: a config change arriving mid-drain coalesces into
* a single pending rollover (latest wins) instead of stacking processes.
*
* Always on for managed engines. Attached engines have no pool, so their
* callers keep the defer-while-busy behavior instead.
*/
import { createHash, randomUUID } from "node:crypto";
import { readFile } from "node:fs/promises";
import {
buildEngineAuthProbeHeader,
registerEngineInstance,
removeEngineInstance,
updateEngineInstanceRole,
} from "./engine-registry.js";
import { createManagedOpencodeServer, type ManagedOpencodeServer } from "./managed-opencode.js";
import { appendRuntimeRestartRecord } from "./runtime-restart-log.js";
import { loopbackFetch } from "./server-fetch.js";
import type { ServerConfig, WorkspaceInfo } from "./types.js";
export type EnginePoolLogger = {
log: (level: "info" | "warn" | "error", message: string, attributes?: Record<string, unknown>) => void;
};
/** Everything needed to spawn a replacement engine identical to the first one. */
export type EngineSpawnTemplate = {
bin?: string;
cwd: string;
runtimeConfigPath: string;
env: Record<string, string | undefined>;
/** Ports a standby must avoid (the OmniRush.ai server, and the live engine). */
reservedPorts: () => number[];
/** Readiness budget for a standby spawn. Defaults to the managed-engine default. */
spawnTimeoutMs?: number;
};
export type EnginePoolStandby = {
workspace: WorkspaceInfo;
generationId: string;
baseUrl: string;
username: string;
password: string;
};
export type EnginePoolHooks = {
/** Today's in-place dispose. Used when the engine is idle. */
reloadInPlace: (
config: ServerConfig,
workspace: WorkspaceInfo,
options?: { awaitPostRefreshSync?: boolean },
) => Promise<void>;
/** Whether the given engine has non-idle sessions. */
engineBusy: (config: ServerConfig, workspace: WorkspaceInfo) => Promise<boolean>;
/** Re-register runtime MCPs and reconcile cloud MCP against a fresh engine. */
postRefreshSync: (config: ServerConfig, workspace: WorkspaceInfo) => Promise<void>;
/**
* Prepare a healthy standby before it becomes primary (managed provider
* credentials). A spawned engine receives no provider keys in its
* environment; without this the first sync after the flip re-delivers them,
* reports a credential change, and forces yet another standby.
*/
prepareStandby?: (config: ServerConfig, standby: EnginePoolStandby) => Promise<void>;
/** Rebuild the engine-visible runtime config file (workspace-independent). */
writeRuntimeConfigFile: (config: ServerConfig) => Promise<{ path: string }>;
registerTrusted: (config: ServerConfig, generation: { baseUrl: string; identity: string; isAlive: () => boolean }) => void;
clearTrusted: (config: ServerConfig, identity: string) => void;
spawn?: (template: EngineSpawnTemplate) => Promise<ManagedOpencodeServer>;
now?: () => number;
schedule?: (operation: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
waitForHealthy?: (handle: ManagedOpencodeServer) => Promise<void>;
/**
* One liveness probe of a running engine: "ok" when it answered at all,
* "refused" when nothing listens on its port, "failed" for anything else
* (timeout, reset). Defaults to GET /global/health.
*/
probeHealth?: (handle: ManagedOpencodeServer) => Promise<"ok" | "refused" | "failed">;
/**
* Called right before the watchdog closes a primary it confirmed dead, so
* the runs it was carrying can be queued for task recovery.
*/
onPrimaryLost?: (config: ServerConfig) => void;
/** Called once when the pool shuts down, before its engines are retired. */
onDisposed?: () => void;
/**
* Called when an engine is closed while another serves (a drained
* generation retired, a dead primary replaced), before it closes: whatever
* still reads the closed engine (the session uploader's turn observers) moves to
* `replacement`, which reads the same sessions from the shared database.
*/
onEngineReplaced?: (closedBaseUrl: string, replacement: EnginePoolConnection) => void;
logger?: EnginePoolLogger;
};
export type RolloverReason = string;
export type RolloverOutcome =
| { action: "skipped"; reason: "unchanged" }
| { action: "reloaded_in_place" }
| { action: "coalesced" }
| { action: "rolled_over"; generationId: string; drainingSessions: number };
type GenerationStatus = "starting" | "primary" | "draining" | "dead";
type Generation = {
id: string;
handle: ManagedOpencodeServer;
status: GenerationStatus;
spawnedAt: number;
fingerprint: string;
registryId: string | null;
trustedIdentity: string | null;
drainTimer: ReturnType<typeof setInterval> | null;
drainDeadline: number | null;
drainActivityWatch: AbortController | null;
/** Proxied requests in flight to this engine (a drained engine closes only once they answered, or a grace period passed). */
requestsInFlight: number;
/** Re-runs the drain check (set while draining). */
drainCheck: (() => void) | null;
};
export type EnginePoolSnapshot = {
generations: Array<{
role: GenerationStatus;
pid: number | null;
port: number | null;
spawnedAt: number;
drainRemainingMs: number | null;
}>;
};
export type EnginePoolConnection = {
generationId: string;
role: "primary" | "draining";
baseUrl: string;
username: string;
password: string;
};
export type EnginePoolRoute = {
target: EnginePoolConnection;
fallback: EnginePoolConnection | null;
};
export type EngineEventProxyLease = {
signal: AbortSignal;
release: () => void;
};
export type EnginePoolProcess = {
pid: number | null;
isAlive: () => boolean;
};
function positiveIntFromEnv(name: string, fallback: number): number {
const raw = Number(process.env[name]);
return Number.isFinite(raw) && raw > 0 ? raw : fallback;
}
/** Same, but an explicit 0 is meaningful (it disables the throttle). */
function nonNegativeIntFromEnv(name: string, fallback: number): number {
const value = process.env[name]?.trim();
if (!value) return fallback;
const raw = Number(value);
return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
}
/**
* How long a draining engine may go without any owned-session activity before
* its sessions are aborted. Activity on the engine event stream pushes the
* deadline out, so this bounds inactivity, not total drain time.
*/
function drainTimeoutMs(): number {
return positiveIntFromEnv("OMNIRUSH_ENGINE_DRAIN_TIMEOUT_MS", 15 * 60_000);
}
/**
* How long a drained engine stays up for proxied requests still in flight to
* it (a session create waiting on the instance's cold boot) before it closes
* anyway.
*/
function drainRequestGraceMs(): number {
return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_DRAIN_REQUEST_GRACE_MS", 60_000);
}
/**
* How long a session an engine was just sent a prompt for counts as running
* there before the engine reports it (the engine accepts a prompt before its
* run starts, which may wait on the folder's instance booting).
*/
function promptStartGraceMs(): number {
return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_PROMPT_START_GRACE_MS", 60_000);
}
/** Delay before the drain activity watch reconnects to a lost engine event stream. */
function drainActivityReconnectMs(): number {
return positiveIntFromEnv("OMNIRUSH_ENGINE_DRAIN_ACTIVITY_RECONNECT_MS", 1_000);
}
/** Floor between automatic spawns, so a burst of triggers cannot thrash. 0 disables it. */
function minSpawnIntervalMs(): number {
return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS", 30_000);
}
/**
* Spacing between the watchdog's liveness probes of a primary that stopped
* answering proxied requests. Three probes spread over this interval must all
* fail before a live engine process is closed.
*/
function recoveryProbeIntervalMs(): number {
return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS", 10_000);
}
const RECOVERY_PROBE_ATTEMPTS = 3;
function drainPollIntervalMs(): number {
return positiveIntFromEnv("OMNIRUSH_ENGINE_DRAIN_POLL_MS", 5_000);
}
/** Grace given to aborted sessions to unwind before the engine is closed. */
function abortSettleMs(): number {
return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_ABORT_SETTLE_MS", 5_000);
}
function portOf(url: string): number {
try {
return Number(new URL(url).port) || 0;
} catch {
return 0;
}
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
export function isEngineConnectionFailure(error: unknown): boolean {
// Node/undici occasionally drops the transport cause. This classifier is
// only used for managed OpenCode engine traffic, never external egress.
if (error instanceof TypeError && error.message === "fetch failed" && error.cause === undefined) return true;
const visited = new Set<object>();
let current: unknown = error;
for (let depth = 0; depth < 6 && current !== null && current !== undefined; depth += 1) {
if (typeof current === "string") {
return /ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_SOCKET|other side closed|socket hang up|Unable to connect|Headers Timeout Error/i.test(current);
}
if (!isRecord(current) || visited.has(current)) return false;
visited.add(current);
const code = current.code;
// UND_ERR_SOCKET ("other side closed"): the engine closed the connection before it answered.
if (code === "ECONNREFUSED" || code === "ECONNRESET" || code === "ETIMEDOUT" || code === "UND_ERR_CONNECT_TIMEOUT" || code === "UND_ERR_HEADERS_TIMEOUT" || code === "UND_ERR_SOCKET") return true;
const message = current.message;
if (typeof message === "string" && /ECONNREFUSED|ECONNRESET|ETIMEDOUT|UND_ERR_CONNECT_TIMEOUT|UND_ERR_HEADERS_TIMEOUT|UND_ERR_SOCKET|other side closed|socket hang up|Unable to connect|Headers Timeout Error/i.test(message)) return true;
current = current.cause;
}
return false;
}
function isConnectionRefused(error: unknown): boolean {
let current: unknown = error;
for (let depth = 0; depth < 6 && isRecord(current); depth += 1) {
if (current.code === "ECONNREFUSED") return true;
current = current.cause;
}
return false;
}
export function isConnectionRefusedClassError(error: unknown): boolean {
return isEngineConnectionFailure(error);
}
function sameOrigin(left: string, right: string): boolean {
try {
return new URL(left).origin === new URL(right).origin;
} catch {
return left === right;
}
}
function isRoutableGeneration(
generation: Generation,
): generation is Generation & { status: "primary" | "draining" } {
return generation.status === "primary" || generation.status === "draining";
}
function normalizeProxyPath(proxyPath: string): string {
const raw = proxyPath.trim() || "/";
const withoutPrefix = raw.startsWith("/opencode") ? raw.slice("/opencode".length) : raw;
const normalized = (withoutPrefix || "/").replace(/\/+$/, "");
return normalized || "/";
}
function decodePathPart(value: string | undefined): string | null {
if (!value) return null;
try {
return decodeURIComponent(value);
} catch {
return value;
}
}
function sessionIdFromPath(proxyPath: string): string | null {
const match = normalizeProxyPath(proxyPath).match(/^\/(?:api\/)?session\/([^/]+)(?:\/|$)/);
const sessionId = decodePathPart(match?.[1]);
return sessionId === "status" ? null : sessionId;
}
function requestIdFromPath(proxyPath: string): string | null {
const normalized = normalizeProxyPath(proxyPath);
const legacy = normalized.match(/^\/(?:permission|question)\/([^/]+)\/(?:reply|reject)$/);
if (legacy) return decodePathPart(legacy[1]);
const scoped = normalized.match(/^\/api\/session\/[^/]+\/(?:permission|question)\/([^/]+)\/(?:reply|reject)$/);
return decodePathPart(scoped?.[1]);
}
function isLegacyPromptReply(proxyPath: string): boolean {
return /^\/(?:permission|question)\/[^/]+\/(?:reply|reject)$/.test(normalizeProxyPath(proxyPath));
}
function isPromptishSessionRequest(method: string, proxyPath: string): boolean {
if (method.toUpperCase() !== "POST") return false;
const normalized = normalizeProxyPath(proxyPath);
return /^\/(?:api\/)?session\/[^/]+\/(?:message|prompt(?:_async)?|command|shell)(?:\/|$)/.test(normalized);
}
function firstString(record: Record<string, unknown>, keys: string[]): string | null {
for (const key of keys) {
const value = record[key];
if (typeof value === "string" && value.trim()) return value.trim();
}
return null;
}
function requestRecords(payload: unknown): Record<string, unknown>[] {
if (Array.isArray(payload)) return payload.filter(isRecord);
if (!isRecord(payload)) return [];
for (const key of ["items", "permissions", "questions", "requests"]) {
const value = payload[key];
if (Array.isArray(value)) return value.filter(isRecord);
}
return [];
}
function eventIdentifiers(payload: unknown): { sessionIds: Set<string>; requestIds: Set<string> } {
const sessionIds = new Set<string>();
const requestIds = new Set<string>();
const visit = (value: unknown, depth: number): void => {
if (depth > 6 || !isRecord(value)) return;
for (const [key, item] of Object.entries(value)) {
if (typeof item === "string") {
if (["sessionID", "sessionId", "session_id"].includes(key)) sessionIds.add(item);
if (["requestID", "requestId", "permissionID", "questionID"].includes(key)) requestIds.add(item);
continue;
}
if (Array.isArray(item)) {
for (const child of item) visit(child, depth + 1);
} else {
visit(item, depth + 1);
}
}
};
visit(payload, 0);
return { sessionIds, requestIds };
}
/**
* Cap on the text one unterminated SSE frame may buffer before the stream is
* treated as malformed. Sized well above any legitimate single engine event
* (large tool outputs included) so real frames are never truncated; only a
* stream that stops terminating frames hits it.
*/
export const ENGINE_SSE_FRAME_MAX_CHARS = 4 * 1024 * 1024;
/**
* Incremental SSE frame splitter shared by the client event fan-in and the
* drain activity watch. It bounds the memory a single unterminated frame can
* hold: completed frames are always returned, and `overflow` turns true once
* the pending remainder exceeds the cap so the caller can drop that
* connection instead of buffering it forever.
*/
export class BoundedSseFrameBuffer {
private readonly decoder = new TextDecoder();
private buffered = "";
constructor(private readonly maxFrameChars = ENGINE_SSE_FRAME_MAX_CHARS) {}
push(chunk: Uint8Array): { frames: string[]; overflow: boolean } {
this.buffered += this.decoder.decode(chunk, { stream: true });
const frames: string[] = [];
while (true) {
const delimiter = this.buffered.match(/\r?\n\r?\n/);
if (!delimiter || delimiter.index === undefined) break;
frames.push(this.buffered.slice(0, delimiter.index));
this.buffered = this.buffered.slice(delimiter.index + delimiter[0].length);
}
return { frames, overflow: this.buffered.length > this.maxFrameChars };
}
}
/** Decode one SSE frame's `data:` payload; null when the frame carries none. */
function sseFramePayload(frame: string): unknown {
const data = frame
.split(/\r?\n/)
.filter((line) => line.startsWith("data:"))
.map((line) => line.slice("data:".length).trim())
.join("\n");
if (!data) return null;
try {
return JSON.parse(data);
} catch {
return null;
}
}
/**
* Everything a spawned engine reads at build time. Comparing this is what
* lets a repeated no-op reload skip spawning a replacement.
*/
export async function computeEngineConfigFingerprint(template: EngineSpawnTemplate): Promise<string> {
const content = await readFile(template.runtimeConfigPath, "utf8").catch(() => "");
return createHash("sha256")
.update(content)
.update("\u0000")
.update(template.bin?.trim() ?? "")
.update("\u0000")
.update(template.env.OPENCODE_MODELS_URL ?? "")
.digest("hex");
}
/**
* The chats behind busy sessions: each session's root (following parents up, at most `maxDepth` steps; a
* longer chain throws, so the caller names no chat). Exported for tests.
*/
export async function rootChats(busy: string[], parentOf: (sessionId: string) => Promise<string | null>, maxDepth = 16): Promise<string[]> {
const roots = await Promise.all([...new Set(busy)].map(async (sessionId) => {
let current = sessionId;
for (let step = 0; ; step += 1) {
const parent = await parentOf(current);
if (!parent) return current;
if (step >= maxDepth) throw new Error("session chain too deep");
current = parent;
}
}));
return [...new Set(roots)];
}
export class EnginePool {
private readonly config: ServerConfig;
private readonly template: EngineSpawnTemplate;
private readonly hooks: EnginePoolHooks;
private generations: Generation[] = [];
private inFlight: Promise<RolloverOutcome> | null = null;
private pendingRollover: {
reason: RolloverReason;
workspace: WorkspaceInfo;
manual: boolean;
awaitPostRefreshSync: boolean;
forceStandby: boolean;
retireDrainingIf?: (baseUrl: string) => boolean;
} | null = null;
private lastSpawnAt = 0;
private consecutiveConnectionFailures = 0;
private lastRecoveryAt = Number.NEGATIVE_INFINITY;
private recoveryTimer: ReturnType<typeof setTimeout> | null = null;
private recoveryWorkspace: WorkspaceInfo | null = null;
private recoveryInFlight: Promise<void> | null = null;
private recoveryOrphan: ManagedOpencodeServer | null = null;
private disposed = false;
private readonly sessionOwnership = new Map<string, string>();
private readonly pinnedRequests = new Map<string, string>();
private readonly activeSessionsByGeneration = new Map<string, Set<string>>();
private readonly eventProxyControllers = new Set<AbortController>();
private readonly drainWaiters = new Set<() => void>();
/**
* Prompts sent to an engine whose session it may not report busy yet (the
* engine answers a prompt before its run starts), by session: the engine
* they went to and when. Such a session drains like a running one.
*/
private readonly startingPrompts = new Map<string, { generationId: string; at: number }>();
constructor(input: { config: ServerConfig; template: EngineSpawnTemplate; hooks: EnginePoolHooks }) {
this.config = input.config;
this.template = input.template;
this.hooks = input.hooks;
}
/**
* Register the engine spawned during startup as the first generation, so a
* later rollover knows what it is replacing.
*/
adoptPrimary(input: {
handle: ManagedOpencodeServer;
fingerprint: string;
registryId: string | null;
trustedIdentity: string | null;
}): void {
this.generations.push({
id: randomUUID(),
handle: input.handle,
status: "primary",
spawnedAt: Date.now(),
fingerprint: input.fingerprint,
registryId: input.registryId,
trustedIdentity: input.trustedIdentity,
drainTimer: null,
drainDeadline: null,
drainActivityWatch: null,
requestsInFlight: 0,
drainCheck: null,
});
this.lastSpawnAt = Date.now();
}
primaryUrl(): string | null {
return this.generations.find((entry) => entry.status === "primary")?.handle.url ?? null;
}
primaryProcess(): EnginePoolProcess | null {
const primary = this.generations.find((entry) => entry.status === "primary") ?? null;
return primary ? { pid: primary.handle.pid ?? null, isAlive: primary.handle.isAlive } : null;
}
reportRequestSuccess(baseUrl: string): void {
if (this.isPrimaryEndpoint(baseUrl)) this.consecutiveConnectionFailures = 0;
}
reportRequestFailure(baseUrl: string, error: unknown, workspace: WorkspaceInfo): void {
if (!this.isPrimaryEndpoint(baseUrl) || !isEngineConnectionFailure(error)) return;
this.consecutiveConnectionFailures += 1;
if (this.consecutiveConnectionFailures < 3) return;
const now = this.now();
if (now - this.lastRecoveryAt < minSpawnIntervalMs()) return;
const failures = this.consecutiveConnectionFailures;
this.consecutiveConnectionFailures = 0;
this.lastRecoveryAt = now;
this.recoveryWorkspace = workspace;
this.hooks.logger?.log("warn", "Managed engine stopped answering; checking whether it is still alive.", {
"engine.recovery.consecutive_failures": failures,
"engine.recovery.last_error": error instanceof Error ? error.message : String(error),
});
void this.recoverDeadPrimary(workspace).catch(() => undefined);
}
/**
* A proxied request to `baseUrl` is going out: a draining engine is not
* closed under it (for at most OMNIRUSH_ENGINE_DRAIN_REQUEST_GRACE_MS).
* Call the returned release once the engine answered or the request failed.
*/
beginRequest(baseUrl: string, request?: { method: string; proxyPath: string }): () => void {
const generation = this.generationForUrl(baseUrl);
if (!generation) return () => undefined;
const sessionId = request && isPromptishSessionRequest(request.method, request.proxyPath) ? sessionIdFromPath(request.proxyPath) : null;
if (sessionId) this.noteStartingPrompt(generation, sessionId);
generation.requestsInFlight += 1;
let released = false;
return () => {
if (released) return;
released = true;
generation.requestsInFlight = Math.max(0, generation.requestsInFlight - 1);
if (generation.requestsInFlight === 0 && generation.status === "draining") generation.drainCheck?.();
};
}
/**
* After a proxied request to `failedBaseUrl` failed to connect: the route
* to retry it on once a rollover moved the engine it went to (that engine
* is no longer serving, or a new primary took over while it was being
* replaced). Waits for a rollover in flight to flip, at most `timeoutMs`.
* Null when there is nothing to retry on: the engine it went to is still
* the one serving.
*/
async rerouteAfterFailure(method: string, proxyPath: string, failedBaseUrl: string, timeoutMs: number): Promise<EnginePoolRoute | null> {
const deadline = this.now() + timeoutMs;
while (!this.disposed) {
const route = this.routeRequest(method, proxyPath);
if (route && !sameOrigin(route.target.baseUrl, failedBaseUrl)) return route;
// The engine it went to still serves: only a rollover (or a recovery) in flight can replace it.
if ((!this.inFlight && !this.recoveryInFlight) || this.now() >= deadline) return null;
await new Promise((resolve) => {
const timer = setTimeout(resolve, 100);
timer.unref?.();
});
}
return null;
}
connections(): EnginePoolConnection[] {
return this.generations
.filter(isRoutableGeneration)
.sort((left, right) => left.status === right.status ? 0 : left.status === "primary" ? -1 : 1)
.map((entry) => this.connectionFor(entry));
}
routeRequest(method: string, proxyPath: string): EnginePoolRoute | null {
const primary = this.generations.find(
(entry): entry is Generation & { status: "primary" } => entry.status === "primary",
) ?? null;
if (!primary) return null;
const primaryConnection = this.connectionFor(primary);
const sessionId = sessionIdFromPath(proxyPath);
if (sessionId) {
const owner = this.generationForId(this.sessionOwnership.get(sessionId));
if (owner) {
const active = this.activeSessionsByGeneration.get(owner.id)?.has(sessionId) === true;
if (isPromptishSessionRequest(method, proxyPath) && !active) {
this.sessionOwnership.delete(sessionId);
} else {
return { target: this.connectionFor(owner), fallback: null };
}
}
}
const requestId = requestIdFromPath(proxyPath);
if (requestId) {
const owner = this.generationForId(this.pinnedRequests.get(requestId));
if (owner) {
const alternate = isLegacyPromptReply(proxyPath)
? this.generations.find((entry) => isRoutableGeneration(entry) && entry.id !== owner.id) ?? null
: null;
return {
target: this.connectionFor(owner),
fallback: alternate && isRoutableGeneration(alternate) ? this.connectionFor(alternate) : null,
};
}
}
const draining = this.generations.find(
(entry): entry is Generation & { status: "draining" } => entry.status === "draining",
) ?? null;
const fallback = draining && isLegacyPromptReply(proxyPath) ? this.connectionFor(draining) : null;
return { target: primaryConnection, fallback };
}
openEventProxy(clientSignal?: AbortSignal): EngineEventProxyLease {
const controller = new AbortController();
this.eventProxyControllers.add(controller);
const signal = clientSignal
? AbortSignal.any([clientSignal, controller.signal])
: controller.signal;
let released = false;
return {
signal,
release: () => {
if (released) return;
released = true;
this.eventProxyControllers.delete(controller);
},
};
}
shouldForwardEvent(generationId: string, payload: unknown): boolean {
const generation = this.generationForId(generationId);
if (!generation) return false;
const identifiers = eventIdentifiers(payload);
if (generation.status === "draining") {
const ownedSession = [...identifiers.sessionIds].some((id) => this.sessionOwnership.get(id) === generation.id);
if (ownedSession) {
for (const requestId of identifiers.requestIds) this.pinnedRequests.set(requestId, generation.id);
}
return ownedSession || [...identifiers.requestIds].some((id) => this.pinnedRequests.get(id) === generation.id);
}
return ![...identifiers.sessionIds].some((id) => {
const owner = this.sessionOwnership.get(id);
return owner !== undefined && owner !== generation.id;
}) && ![...identifiers.requestIds].some((id) => {
const owner = this.pinnedRequests.get(id);
return owner !== undefined && owner !== generation.id;
});
}
observePendingRequests(generationId: string, payload: unknown): string[] {
const generation = this.generationForId(generationId);
if (!generation) return [];
const requestIds: string[] = [];
for (const record of requestRecords(payload)) {
const sessionId = firstString(record, ["sessionID", "sessionId", "session_id"]);
const owner = this.generationForId(sessionId ? this.sessionOwnership.get(sessionId) : undefined) ?? generation;
const requestId = firstString(record, ["id", "requestID", "requestId", "permissionID", "questionID"]);
if (requestId) {
this.pinnedRequests.set(requestId, owner.id);
requestIds.push(requestId);
}
if (sessionId && !this.sessionOwnership.has(sessionId)) this.sessionOwnership.set(sessionId, generation.id);
}
return requestIds;
}
snapshot(): EnginePoolSnapshot {
const now = Date.now();
return {
generations: this.generations
.filter((entry) => entry.status !== "dead")
.map((entry) => ({
role: entry.status,
pid: entry.handle.pid ?? null,
port: portOf(entry.handle.url) || null,
spawnedAt: entry.spawnedAt,
drainRemainingMs: entry.drainDeadline === null ? null : Math.max(0, entry.drainDeadline - now),
})),
};
}
/**
* Non-idle sessions on every routable engine, for the desktop's restart
* guard. `unknown` counts the probes that could not be read; callers treat
* those as busy. Only the given directories are probed, so an instance the
* engine never booted is not booted just to learn that it is idle.
*/
async activeSessions(directories: string[]): Promise<{ sessions: string[]; unknown: number }> {
const sessions = new Set<string>();
let unknown = 0;
const targets = this.generations.filter(isRoutableGeneration);
await Promise.all(targets.flatMap((generation) => directories.map(async (directory) => {
if (!generation.handle.isAlive()) return;
try {
const url = new URL("/session/status", generation.handle.url);
url.searchParams.set("directory", directory);
const response = await loopbackFetch(url.toString(), {
headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
signal: AbortSignal.timeout(5_000),
});
if (!response.ok) throw new Error(`status ${response.status}`);
const payload: unknown = await response.json();
if (!isRecord(payload)) throw new Error("unreadable status");
for (const [sessionId, status] of Object.entries(payload)) {
if (isRecord(status) && status.type !== "idle") sessions.add(sessionId);
}
} catch {
unknown += 1;
}
})));
for (const owned of this.activeSessionsByGeneration.values()) {
for (const sessionId of owned) sessions.add(sessionId);
}
return { sessions: [...sessions], unknown };
}
/** Each session's parent (null: a top-level chat), as the engine reported it; it never changes. */
private readonly sessionParents = new Map<string, string | null>();
/**
* The running top-level chats (sub-agent sessions left out), for computer use's "Allow for this chat".
* `unknown` counts engines or sessions that could not be read; callers then name no chat.
*/
async activeChats(directories: string[]): Promise<{ sessions: string[]; unknown: number }> {
const chats = new Set<string>();
let unknown = 0;
const targets = this.generations.filter(isRoutableGeneration);
await Promise.all(targets.flatMap((generation) => directories.map(async (directory) => {
if (!generation.handle.isAlive()) return;
const headers = { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) };
try {
const url = new URL("/session/status", generation.handle.url);
url.searchParams.set("directory", directory);
const response = await loopbackFetch(url.toString(), { headers, signal: AbortSignal.timeout(5_000) });
if (!response.ok) throw new Error(`status ${response.status}`);
const payload: unknown = await response.json();
if (!isRecord(payload)) throw new Error("unreadable status");
const busy = Object.entries(payload).filter(([, status]) => isRecord(status) && status.type !== "idle").map(([sessionId]) => sessionId);
// Each busy session counts as the chat at the root of its parent chain (a background sub-agent whose
// chat is idle still counts as that chat). Parents not known yet are read (at most 64 per pass; more:
// unknown, so no chat is named).
let reads = 0;
const parentOf = async (sessionId: string): Promise<string | null> => {
if (this.sessionParents.has(sessionId)) return this.sessionParents.get(sessionId) ?? null;
if (++reads > 64) throw new Error("too many sessions to read");
const info = new URL(`/session/${encodeURIComponent(sessionId)}`, generation.handle.url);
info.searchParams.set("directory", directory);
const answer = await loopbackFetch(info.toString(), { headers, signal: AbortSignal.timeout(5_000) });
if (!answer.ok) throw new Error(`session ${answer.status}`);
const session: unknown = await answer.json();
if (!isRecord(session)) throw new Error("unreadable session");
const parent = typeof session.parentID === "string" && session.parentID ? session.parentID : null;
this.sessionParents.set(sessionId, parent);
if (this.sessionParents.size > 4096) this.sessionParents.delete(this.sessionParents.keys().next().value!);
return parent;
};
for (const chat of await rootChats(busy, parentOf)) chats.add(chat);
} catch {
unknown += 1;
}
})));
// Prompts on their way (the engine may not report them busy yet): chats the person prompted.
for (const owned of this.activeSessionsByGeneration.values()) {
for (const sessionId of owned) {
// A prompt on its way: its chat (the root of a known chain, else the session itself). A chain too
// long or a loop: unknown, like rootChats.
let root = sessionId;
let steps = 0;
while (this.sessionParents.get(root) && steps <= 16) { root = this.sessionParents.get(root)!; steps += 1; }
if (steps > 16) unknown += 1;
else chats.add(root);
}
}
return { sessions: [...chats], unknown };
}
/** Whether `baseUrl` is the engine new work goes to (not a draining one). */
isPrimaryUrl(baseUrl: string): boolean {
return this.isPrimaryEndpoint(baseUrl);
}
/**
* Close draining engines now instead of letting their runs finish: their
* running turns are aborted first. For a change the old engine must not keep
* serving under (the provider lock); `shouldRetire` picks the engines by URL.
*/
async retireDraining(shouldRetire: (baseUrl: string) => boolean, reason: string): Promise<number> {
const targets = this.generations.filter(
(entry): entry is Generation & { status: "draining" } => entry.status === "draining" && shouldRetire(entry.handle.url),
);
for (const generation of targets) {
const sessions = new Set([
...(this.activeSessionsByGeneration.get(generation.id) ?? []),
...await this.nonIdleSessionIds(generation).catch(() => [] as string[]),
]);
for (const sessionId of sessions) {
await this.abortSession(generation, sessionId, { source: "engine_pool.retire_draining", reason }).catch(() => undefined);
}
if (sessions.size > 0) await new Promise((resolve) => setTimeout(resolve, abortSettleMs()));
await this.retire(generation, "forced");
}
return targets.length;
}
/** Provider sync treats an active drain as busy so it defers instead of parking on waitForDrain(). */
hasDrainingGeneration(): boolean {
return this.generations.some((entry) => entry.status === "draining");
}
/**
* Bring the engine onto current config. Idle engines reload in place; busy
* ones roll over to a standby. Serialized: concurrent requests collapse into
* one pending rollover so a burst never stacks processes.
*/
async requestRollover(input: {
reason: RolloverReason;
workspace: WorkspaceInfo;
manual?: boolean;
awaitPostRefreshSync?: boolean;
/** Apply a config change through a healthy standby even when the primary is idle. */
forceStandby?: boolean;
/** Draining engines this matches are closed (runs aborted) instead of drained. */
retireDrainingIf?: (baseUrl: string) => boolean;
}): Promise<RolloverOutcome> {
if (this.disposed) return { action: "skipped", reason: "unchanged" };
const request = {
reason: input.reason,
workspace: input.workspace,
manual: input.manual === true,
awaitPostRefreshSync: input.awaitPostRefreshSync !== false,
forceStandby: input.forceStandby === true,
...(input.retireDrainingIf ? { retireDrainingIf: input.retireDrainingIf } : {}),
};
if (this.inFlight) {
// Latest wins: a manual request keeps its manual flag so it still
// bypasses the fingerprint guard when it runs.
this.pendingRollover = {
...request,
manual: request.manual || this.pendingRollover?.manual === true,
awaitPostRefreshSync: request.awaitPostRefreshSync || this.pendingRollover?.awaitPostRefreshSync === true,
forceStandby: request.forceStandby || this.pendingRollover?.forceStandby === true,
...((request.retireDrainingIf ?? this.pendingRollover?.retireDrainingIf)
? { retireDrainingIf: request.retireDrainingIf ?? this.pendingRollover?.retireDrainingIf }
: {}),
};
this.hooks.logger?.log("info", "Engine rollover coalesced into the in-flight request.", {
"engine.rollover.reason": request.reason,
});
return { action: "coalesced" };
}
const run = this.runRollover(request);
this.inFlight = run;
try {
return await run;
} finally {
this.inFlight = null;
const next = this.pendingRollover;
this.pendingRollover = null;
if (next && !this.disposed) {
void this.requestRollover(next).catch(() => undefined);
}
}
}
private async runRollover(request: {
reason: RolloverReason;
workspace: WorkspaceInfo;
manual: boolean;
awaitPostRefreshSync: boolean;
forceStandby: boolean;
retireDrainingIf?: (baseUrl: string) => boolean;
}): Promise<RolloverOutcome> {
const { workspace, reason, manual, awaitPostRefreshSync, forceStandby, retireDrainingIf } = request;
if (retireDrainingIf) await this.retireDraining(retireDrainingIf, `engine rollover: ${reason}`);
// The standby reads config from disk at spawn, so make sure the file is
// current before deciding anything.
await this.hooks.writeRuntimeConfigFile(this.config).catch(() => undefined);
const fingerprint = await this.currentFingerprint();
const primary = this.generations.find((entry) => entry.status === "primary") ?? null;
if (!manual && primary && primary.fingerprint === fingerprint) {
// Nothing the engine reads at build time changed. Skipping here is what
// keeps a repeating no-op sync from spawning an engine every pass.
return { action: "skipped", reason: "unchanged" };
}
const busy = forceStandby
? true
// Unknown activity counts as busy: a busy engine rolls over to a
// standby, while an in-place dispose would abort whatever it runs.
: await this.hooks.engineBusy(this.config, workspace).catch(() => true);
if (!busy) {
await this.hooks.reloadInPlace(this.config, workspace, { awaitPostRefreshSync });
if (primary) primary.fingerprint = fingerprint;
this.hooks.logger?.log("info", "Engine reloaded in place (idle).", {
"engine.rollover.reason": reason,
});
return { action: "reloaded_in_place" };
}
const draining = this.generations.find((entry) => entry.status === "draining");
if (draining) {
if (forceStandby) {
this.hooks.logger?.log("info", "Engine standby rollover waiting for the active drain.", {
"engine.rollover.reason": reason,
});
await this.waitForDrain();
if (this.disposed) return { action: "skipped", reason: "unchanged" };
return this.runRollover(request);
}
// Cap: one primary plus one draining. Park this change; it lands when
// the current drain finishes.
this.pendingRollover = { reason, workspace, manual, awaitPostRefreshSync, forceStandby, ...(retireDrainingIf ? { retireDrainingIf } : {}) };
this.hooks.logger?.log("info", "Engine rollover parked behind an active drain.", {
"engine.rollover.reason": reason,
});
return { action: "coalesced" };
}
const sinceLastSpawn = Date.now() - this.lastSpawnAt;
if (!manual && !forceStandby && sinceLastSpawn < minSpawnIntervalMs()) {
this.pendingRollover = { reason, workspace, manual, awaitPostRefreshSync, forceStandby, ...(retireDrainingIf ? { retireDrainingIf } : {}) };
this.hooks.logger?.log("info", "Engine rollover throttled by the minimum spawn interval.", {
"engine.rollover.reason": reason,
"engine.rollover.since_last_spawn_ms": sinceLastSpawn,
});
return { action: "coalesced" };
}
const outcome = await this.rollOver({ reason, workspace, fingerprint, primary });
if (retireDrainingIf && outcome.action === "rolled_over") await this.retireDraining(retireDrainingIf, `engine rollover: ${reason}`);
return outcome;
}
private async rollOver(input: {
reason: RolloverReason;
workspace: WorkspaceInfo;
fingerprint: string;
primary: Generation | null;
}): Promise<RolloverOutcome> {
const { reason, workspace, fingerprint, primary } = input;
this.hooks.logger?.log("info", "Engine rollover requested.", { "engine.rollover.reason": reason });
let handle: ManagedOpencodeServer;
this.lastSpawnAt = Date.now();
try {
handle = await this.spawn();
} catch (error) {
this.hooks.logger?.log("error", "Engine rollover standby failed to start; the live engine is untouched.", {
"engine.rollover.reason": reason,
"engine.rollover.failure": error instanceof Error ? error.message : String(error),
});
throw error;
}
const generation: Generation = {
id: randomUUID(),
handle,
status: "starting",
spawnedAt: Date.now(),
fingerprint,
registryId: null,
trustedIdentity: null,
drainTimer: null,
drainDeadline: null,
drainActivityWatch: null,
requestsInFlight: 0,
drainCheck: null,
};
this.generations.push(generation);
try {
await this.waitForHealthy(handle);
} catch (error) {
// Never flip onto an engine that has not answered: close the partial
// standby and leave the primary serving.
this.generations = this.generations.filter((entry) => entry.id !== generation.id);
await handle.close().catch(() => undefined);
this.hooks.logger?.log("error", "Engine rollover standby never became healthy; the live engine is untouched.", {
"engine.rollover.reason": reason,
"engine.rollover.failure": error instanceof Error ? error.message : String(error),
});
throw error;
}
if (this.hooks.prepareStandby) {
try {
await this.hooks.prepareStandby(this.config, {
workspace,
generationId: generation.id,
baseUrl: handle.url,
username: handle.username,
password: handle.password,
});
} catch (error) {
// The standby is healthy and the primary is still serving; a failed
// seed only means the next sync pass delivers credentials instead.
this.hooks.logger?.log("warn", "Engine standby preparation failed; flipping anyway.", {
"engine.rollover.reason": reason,
"engine.rollover.failure": error instanceof Error ? error.message : String(error),
});
}
}
if (handle.pid) {
generation.registryId = randomUUID();
await registerEngineInstance(this.config, {
id: generation.registryId,
pid: handle.pid,
port: portOf(handle.url),
url: handle.url,
startedAt: generation.spawnedAt,
role: "starting",
serverRunId: generation.id,
ownerPid: process.pid,
authProbe: buildEngineAuthProbeHeader(handle.username, handle.password),
bin: this.template.bin?.trim() || "opencode",
}).catch(() => undefined);
}
// A prompt the primary accepted but has not started yet drains with it too.
const drainingSessions = this.withStartingPrompts(primary, await this.nonIdleSessionIds(primary));
const pinnedRequestIds = await this.pendingRequestIds(primary);
this.flip(generation, primary, drainingSessions, pinnedRequestIds);
this.hooks.logger?.log("info", "Engine rollover flipped to the new generation.", {
"engine.rollover.reason": reason,
"engine.rollover.pid": handle.pid ?? null,
"engine.rollover.draining_sessions": drainingSessions.length,
});
// Best-effort and off the critical path: the new engine already has disk
// config; this pushes the runtime-DB MCPs a fresh instance cannot see.
this.detachPostRefreshSync(workspace);
if (primary) this.startDrainMonitor(primary, workspace);
return { action: "rolled_over", generationId: generation.id, drainingSessions: drainingSessions.length };
}
/**
* Point the server at the new generation. Synchronous on purpose: every
* request resolves its engine from `config`, so there must be no await
* between demoting the old engine and promoting the new one.
*/
private flip(
next: Generation,
previous: Generation | null,
drainingSessions: string[],
pinnedRequestIds: string[],
): void {
next.status = "primary";
if (previous) previous.status = "draining";
if (previous) {
const active = new Set(drainingSessions);
this.activeSessionsByGeneration.set(previous.id, active);
for (const sessionId of active) this.sessionOwnership.set(sessionId, previous.id);
for (const requestId of pinnedRequestIds) this.pinnedRequests.set(requestId, previous.id);
}
this.config.opencodeBaseUrl = next.handle.url;
this.config.opencodeUsername = next.handle.username;
this.config.opencodePassword = next.handle.password;
for (const entry of this.config.workspaces) {
if (entry.workspaceType === "remote") continue;
entry.baseUrl = next.handle.url;
entry.opencodeUsername = next.handle.username;
entry.opencodePassword = next.handle.password;
entry.directory = entry.path;
}
if (previous?.trustedIdentity) {
try {
this.hooks.clearTrusted(this.config, previous.trustedIdentity);
} catch {
// The registry is advisory; a stale entry must not block the flip.
}
}
next.trustedIdentity = [next.handle.pid ?? "unknown", randomUUID()].join(":");
try {
this.hooks.registerTrusted(this.config, {
baseUrl: next.handle.url,
identity: next.trustedIdentity,
isAlive: next.handle.isAlive,
});
} catch {
// Same: diagnostics eligibility only.
}
if (next.registryId) void updateEngineInstanceRole(this.config, next.registryId, "primary").catch(() => undefined);
if (previous?.registryId) {
void updateEngineInstanceRole(this.config, previous.registryId, "draining").catch(() => undefined);
}
this.abortEventProxies();
}
/**
* Watch a draining engine and close it once its runs finish. The grace
* window bounds inactivity, not total drain time: engine events for an
* owned session push the deadline out, so a run that is actively making
* progress is never aborted mid-flight, while a non-idle session that stops
* reporting anything for the whole window is aborted rather than kept alive
* forever.
*/
private startDrainMonitor(generation: Generation, workspace: WorkspaceInfo): void {
generation.drainDeadline = Date.now() + drainTimeoutMs();
this.watchDrainActivity(generation);
// When the drained engine was first found idle with requests still in flight to it.
let idleWithRequestsSince: number | null = null;
const tick = async (): Promise<void> => {
if (generation.status !== "draining") return;
const remaining = this.withStartingPrompts(generation, await this.nonIdleSessionIds(generation));
this.updateActiveSessions(generation, remaining);
if (remaining.length === 0) {
// A request it still serves (a session create waiting on a cold
// instance) would fail mid-flight: it closes once they answered.
if (generation.requestsInFlight > 0) {
idleWithRequestsSince ??= this.now();
if (this.now() - idleWithRequestsSince < drainRequestGraceMs()) return;
}
await this.retire(generation, "idle");
return;
}
idleWithRequestsSince = null;
if (generation.drainDeadline !== null && Date.now() >= generation.drainDeadline) {
this.hooks.logger?.log("warn", "Engine drain saw no session activity for the grace period; aborting the remaining sessions.", {
"engine.drain.sessions": remaining.join(","),
"engine.drain.session_count": remaining.length,
});
for (const sessionId of remaining) {
await this.abortSession(generation, sessionId).catch(() => undefined);
}
await new Promise((resolve) => setTimeout(resolve, abortSettleMs()));
await this.retire(generation, "forced");
}
};
const timer = setInterval(() => void tick().catch(() => undefined), drainPollIntervalMs());
timer.unref?.();
generation.drainTimer = timer;
generation.drainCheck = () => void tick().catch(() => undefined);
// Sessions can finish between the flip and the first poll.
void tick().catch(() => undefined);
}
/**
* Hold one global event-stream subscription on the draining engine so
* owned-session activity keeps extending the drain deadline. The client
* event fan-in only exists while a client is attached; the pool owns this
* watch so a background run with no observer still counts as active.
*/
private watchDrainActivity(generation: Generation): void {
const controller = new AbortController();
generation.drainActivityWatch = controller;
void this.runDrainActivityWatch(generation, controller.signal).catch(() => undefined);
}
private async runDrainActivityWatch(
generation: Generation,
signal: AbortSignal,
): Promise<void> {
while (!signal.aborted && generation.status === "draining") {
try {
const url = new URL("/global/event", generation.handle.url);
const response = await loopbackFetch(url.toString(), {
headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
signal,
});
if (response.ok && response.body) {
await this.consumeDrainActivityEvents(generation, response.body, signal);
}
} catch {
// Losing the stream only pauses activity credit. The poll loop still
// owns the abort decision, and an unreachable engine reports no
// sessions to drain in the first place.
}
if (signal.aborted || generation.status !== "draining") return;
await new Promise<void>((resolve) => {
const timer = setTimeout(resolve, drainActivityReconnectMs());
timer.unref?.();
});
}
}
private async consumeDrainActivityEvents(
generation: Generation,
body: ReadableStream<Uint8Array>,
signal: AbortSignal,
): Promise<void> {
const reader = body.getReader();
const frameBuffer = new BoundedSseFrameBuffer();
try {
while (!signal.aborted && generation.status === "draining") {
const chunk = await reader.read();
if (chunk.done) return;
const parsed = frameBuffer.push(chunk.value);
for (const frame of parsed.frames) {
this.noteDrainActivity(generation, sseFramePayload(frame));
}
// A frame that never terminates would buffer without bound; drop the
// stream and let the watch loop reconnect.
if (parsed.overflow) return;
}
} finally {
await reader.cancel().catch(() => undefined);
reader.releaseLock();
}
}
/** An engine event naming an owned session is proof of progress: push the deadline out. */
private noteDrainActivity(generation: Generation, payload: unknown): void {
if (payload === null || generation.status !== "draining" || generation.drainDeadline === null) return;
const owned = this.activeSessionsByGeneration.get(generation.id);
if (!owned || owned.size === 0) return;
const { sessionIds } = eventIdentifiers(payload);
let hasOwnedSession = false;
for (const sessionId of sessionIds) {
if (owned.has(sessionId)) {
hasOwnedSession = true;
break;
}
}
if (!hasOwnedSession) return;
const extended = Date.now() + drainTimeoutMs();
if (extended > generation.drainDeadline) generation.drainDeadline = extended;
}
private async retire(generation: Generation, cause: "idle" | "forced" | "shutdown"): Promise<void> {
if (generation.status === "dead") return;
generation.status = "dead";
if (generation.drainTimer) {
clearInterval(generation.drainTimer);
generation.drainTimer = null;
}
generation.drainActivityWatch?.abort();
generation.drainActivityWatch = null;
generation.drainDeadline = null;
generation.drainCheck = null;
this.activeSessionsByGeneration.delete(generation.id);
if (cause !== "shutdown") this.notifyEngineReplaced(generation);
for (const [sessionId, generationId] of this.sessionOwnership) {
if (generationId === generation.id) this.sessionOwnership.delete(sessionId);
}
for (const [requestId, generationId] of this.pinnedRequests) {
if (generationId === generation.id) this.pinnedRequests.delete(requestId);
}
if (generation.trustedIdentity) {
try {
this.hooks.clearTrusted(this.config, generation.trustedIdentity);
} catch {
// Advisory only.
}
generation.trustedIdentity = null;
}
let closeError: unknown;
try {
await generation.handle.close();
} catch (error) {
closeError = error;
}
if (generation.registryId) {
await removeEngineInstance(this.config, generation.registryId).catch(() => undefined);
generation.registryId = null;
}
this.generations = this.generations.filter((entry) => entry.id !== generation.id);
for (const resolve of this.drainWaiters) resolve();
this.drainWaiters.clear();
this.hooks.logger?.log("info", "Drained engine closed.", { "engine.drain.cause": cause });
const next = this.pendingRollover;
if (cause !== "shutdown" && next && !this.inFlight && !this.disposed) {
this.pendingRollover = null;
void this.requestRollover(next).catch(() => undefined);
}
if (closeError !== undefined && cause === "shutdown") throw closeError;
}
/** Close every engine this pool owns. Draining generations go first. */
async disposeAll(): Promise<void> {
this.disposed = true;
this.hooks.onDisposed?.();
this.pendingRollover = null;
if (this.recoveryTimer) clearTimeout(this.recoveryTimer);
this.recoveryTimer = null;
this.abortEventProxies();
const errors: unknown[] = [];
const recoveryOrphan = this.recoveryOrphan;
this.recoveryOrphan = null;
if (recoveryOrphan) {
try {
await recoveryOrphan.close();
} catch (error) {
errors.push(error);
}
}
const ordered = [...this.generations].sort((left, right) => {
if (left.status === right.status) return 0;
return left.status === "draining" ? -1 : 1;
});
for (const generation of ordered) {
try {
await this.retire(generation, "shutdown");
} catch (error) {
errors.push(error);
}
}
this.generations = [];
if (errors.length === 1) throw errors[0];
if (errors.length > 1) throw new AggregateError(errors, "Failed to close managed engines");
}
private async currentFingerprint(): Promise<string> {
return computeEngineConfigFingerprint(this.template);
}
private waitForDrain(): Promise<void> {
if (!this.generations.some((entry) => entry.status === "draining")) return Promise.resolve();
return new Promise((resolve) => {
this.drainWaiters.add(resolve);
});
}
private async recoverDeadPrimary(workspace: WorkspaceInfo): Promise<void> {
if (this.disposed) return;
if (this.recoveryInFlight) return this.recoveryInFlight;
const recovery = this.runDeadPrimaryRecovery(workspace);
this.recoveryInFlight = recovery;
try {
await recovery;
} finally {
this.recoveryInFlight = null;
}
}
private async runDeadPrimaryRecovery(workspace: WorkspaceInfo): Promise<void> {
const previous = this.generations.find((entry) => entry.status === "primary") ?? null;
let replacement: ManagedOpencodeServer | null = null;
try {
if (this.recoveryOrphan) {
await this.recoveryOrphan.close();
if (this.recoveryOrphan.isAlive()) throw new Error("Failed replacement remained alive after close");
this.recoveryOrphan = null;
}
if (previous) {
const liveness = await this.confirmPrimaryDead(previous);
if (!liveness.dead) {
// A slow or overloaded engine still carries every live run; closing
// it here would abort them all. Only a dead one is replaced.
this.consecutiveConnectionFailures = 0;
this.hooks.logger?.log("warn", "Managed engine recovery skipped: the engine still answers its health check.", {
"engine.recovery.reason": liveness.reason,
});
return;
}
this.hooks.logger?.log("warn", "Managed engine recovery started.", {
"engine.recovery.reason": liveness.reason,
"engine.pid": previous.handle.pid ?? null,
});
appendRuntimeRestartRecord({
kind: "engine",
action: "restarted",
reason: `engine_unreachable:${liveness.reason}`,
source: "engine-pool-watchdog",
});
try {
this.hooks.onPrimaryLost?.(this.config);
} catch {
// Recovery bookkeeping must never block replacing a dead engine.
}
if (previous.trustedIdentity) {
try {
this.hooks.clearTrusted(this.config, previous.trustedIdentity);
} catch {
// Advisory only; process teardown must still happen.
}
}
await previous.handle.close();
if (previous.handle.isAlive()) throw new Error("Managed OmniRush process remained alive after close");
previous.status = "dead";
if (previous.registryId) await removeEngineInstance(this.config, previous.registryId).catch(() => undefined);
this.generations = this.generations.filter((entry) => entry.id !== previous.id);
}
const handle = await this.spawn();
replacement = handle;
await this.waitForHealthy(handle);
const generation: Generation = {
id: randomUUID(),
handle,
status: "starting",
spawnedAt: this.now(),
fingerprint: await this.currentFingerprint(),
registryId: null,
trustedIdentity: null,
drainTimer: null,
drainDeadline: null,
drainActivityWatch: null,
requestsInFlight: 0,
drainCheck: null,
};
this.generations.push(generation);
if (handle.pid) {
generation.registryId = randomUUID();
await registerEngineInstance(this.config, {
id: generation.registryId,
pid: handle.pid,
port: portOf(handle.url),
url: handle.url,
startedAt: generation.spawnedAt,
role: "starting",
serverRunId: generation.id,
ownerPid: process.pid,
authProbe: buildEngineAuthProbeHeader(handle.username, handle.password),
bin: this.template.bin?.trim() || "opencode",
}).catch(() => undefined);
}
this.flip(generation, null, [], []);
replacement = null;
if (previous) this.notifyEngineReplaced(previous);
this.consecutiveConnectionFailures = 0;
this.recoveryWorkspace = null;
if (this.recoveryTimer) clearTimeout(this.recoveryTimer);
this.recoveryTimer = null;
this.detachPostRefreshSync(workspace);
} catch (error) {
if (replacement) {
try {
await replacement.close();
if (replacement.isAlive()) this.recoveryOrphan = replacement;
} catch {
this.recoveryOrphan = replacement;
}
}
this.hooks.logger?.log("error", "Managed engine recovery failed; retry scheduled.", {
"engine.recovery.failure": error instanceof Error ? error.message : String(error),
});
this.scheduleRecovery(workspace);
throw error;
}
}
/**
* Decide whether a primary that failed several proxied requests is really
* gone. A process that exited or a port nobody listens on is dead at once;
* timeouts and resets are only trusted after every spaced probe fails.
*/
private async confirmPrimaryDead(generation: Generation): Promise<{ dead: boolean; reason: string }> {
for (let attempt = 1; attempt <= RECOVERY_PROBE_ATTEMPTS; attempt += 1) {
if (!generation.handle.isAlive()) return { dead: true, reason: "process_exited" };
const verdict = await this.probeHealth(generation.handle);
if (verdict === "ok") return { dead: false, reason: "health_ok" };
if (verdict === "refused") return { dead: true, reason: "connection_refused" };
if (attempt < RECOVERY_PROBE_ATTEMPTS) {
await new Promise((resolve) => {
const timer = setTimeout(resolve, recoveryProbeIntervalMs());
timer.unref?.();
});
}
}
return { dead: true, reason: "health_probe_failed" };
}
private async probeHealth(handle: ManagedOpencodeServer): Promise<"ok" | "refused" | "failed"> {
if (this.hooks.probeHealth) return this.hooks.probeHealth(handle);
try {
await loopbackFetch(new URL("/global/health", handle.url).toString(), {
headers: { Authorization: buildEngineAuthProbeHeader(handle.username, handle.password) },
signal: AbortSignal.timeout(5_000),
});
return "ok";
} catch (error) {
return isConnectionRefused(error) ? "refused" : "failed";
}
}
private scheduleRecovery(workspace: WorkspaceInfo): void {
if (this.disposed || this.recoveryTimer) return;
this.recoveryWorkspace = workspace;
const delayMs = Math.max(1, minSpawnIntervalMs() - (this.now() - this.lastRecoveryAt));
const schedule = this.hooks.schedule ?? ((operation: () => void, delay: number) => setTimeout(operation, delay));
this.recoveryTimer = schedule(() => {
this.recoveryTimer = null;
const pendingWorkspace = this.recoveryWorkspace;
if (!pendingWorkspace || this.disposed) return;
this.lastRecoveryAt = this.now();
void this.recoverDeadPrimary(pendingWorkspace).catch(() => undefined);
}, delayMs);
this.recoveryTimer.unref?.();
}
private spawn(): Promise<ManagedOpencodeServer> {
if (this.hooks.spawn) return this.hooks.spawn(this.template);
return createManagedOpencodeServer({
bin: this.template.bin,
cwd: this.template.cwd,
excludedPorts: this.template.reservedPorts(),
...(this.template.spawnTimeoutMs ? { timeoutMs: this.template.spawnTimeoutMs } : {}),
env: {
...this.template.env,
OPENCODE_CONFIG: this.template.runtimeConfigPath,
},
});
}
private detachPostRefreshSync(workspace: WorkspaceInfo): void {
void this.hooks.postRefreshSync(this.config, workspace).catch((error) => {
this.hooks.logger?.log("error", "Engine post-refresh MCP sync failed.", {
"workspace.id": workspace.id,
"engine.post_refresh.failure": error instanceof Error ? error.message : String(error),
});
});
}
private now(): number {
return this.hooks.now?.() ?? Date.now();
}
private noteStartingPrompt(generation: Generation, sessionId: string): void {
const now = this.now();
for (const [id, entry] of this.startingPrompts) {
if (now - entry.at >= promptStartGraceMs()) this.startingPrompts.delete(id);
}
this.startingPrompts.set(sessionId, { generationId: generation.id, at: now });
}
/**
* `nonIdle` (the sessions `generation` reports running) plus those it was
* sent a prompt for within the start grace period and has not reported
* running yet. A session seen running is tracked by its status from then on.
*/
private withStartingPrompts(generation: Generation | null, nonIdle: string[]): string[] {
if (!generation) return nonIdle;
const sessions = new Set(nonIdle);
const now = this.now();
for (const [sessionId, entry] of this.startingPrompts) {
if (entry.generationId !== generation.id) continue;
if (sessions.has(sessionId) || now - entry.at >= promptStartGraceMs()) {
this.startingPrompts.delete(sessionId);
continue;
}
sessions.add(sessionId);
}
return [...sessions];
}
/** Tells the pool's owner that `closed` is going away and which engine serves its sessions now. */
private notifyEngineReplaced(closed: Generation): void {
const primary = this.generations.find(
(entry): entry is Generation & { status: "primary" } => entry.status === "primary" && entry.id !== closed.id,
);
if (!primary || !this.hooks.onEngineReplaced) return;
try {
this.hooks.onEngineReplaced(closed.handle.url, this.connectionFor(primary));
} catch {
// Advisory: closing the engine never waits on its readers.
}
}
private generationForUrl(baseUrl: string): Generation | null {
return this.generations.find((entry) => entry.status !== "dead" && sameOrigin(entry.handle.url, baseUrl)) ?? null;
}
private isPrimaryEndpoint(baseUrl: string): boolean {
try {
const primary = this.primaryUrl();
return primary !== null && new URL(primary).origin === new URL(baseUrl).origin;
} catch {
return false;
}
}
private async waitForHealthy(handle: ManagedOpencodeServer, attempts = 10): Promise<void> {
if (this.hooks.waitForHealthy) return this.hooks.waitForHealthy(handle);
const authorization = buildEngineAuthProbeHeader(handle.username, handle.password);
for (let attempt = 0; attempt < attempts; attempt += 1) {
try {
const response = await loopbackFetch(new URL("/global/health", handle.url).toString(), {
headers: { Authorization: authorization },
signal: AbortSignal.timeout(2_000),
});
if (response.ok) return;
} catch {
// Not up yet.
}
await new Promise((resolve) => setTimeout(resolve, 500));
}
throw new Error("Standby OmniRush engine did not report healthy");
}
private async nonIdleSessionIds(generation: Generation | null): Promise<string[]> {
if (!generation || generation.status === "dead" || !generation.handle.isAlive()) return [];
const sessionIds = new Set<string>();
await Promise.all(this.engineProbeDirectories().map(async (directory) => {
try {
const url = new URL("/session/status", generation.handle.url);
if (directory !== null) url.searchParams.set("directory", directory);
const response = await loopbackFetch(url.toString(), {
headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
signal: AbortSignal.timeout(5_000),
});
if (!response.ok) return;
const payload: unknown = await response.json();
if (!isRecord(payload)) return;
for (const [sessionId, status] of Object.entries(payload)) {
if (isRecord(status) && status.type !== "idle") sessionIds.add(sessionId);
}
} catch {
// Unknown activity never keeps a drain open forever; the grace timeout
// still bounds it, and an unreachable engine has nothing left to drain.
}
}));
return [...sessionIds];
}
private engineProbeDirectories(): Array<string | null> {
const directories = new Set(
this.config.workspaces
.filter((workspace) => workspace.workspaceType !== "remote")
.map((workspace) => workspace.path),
);
return directories.size > 0 ? [...directories] : [null];
}
private async pendingRequestIds(generation: Generation | null): Promise<string[]> {
if (!generation || generation.status === "dead" || !generation.handle.isAlive()) return [];
const requestIds = new Set<string>();
await Promise.all(this.engineProbeDirectories().flatMap((directory) =>
["/permission", "/question", "/api/permission/request", "/api/question/request"].map(async (path) => {
try {
const url = new URL(path, generation.handle.url);
if (directory !== null) url.searchParams.set("directory", directory);
const response = await loopbackFetch(url.toString(), {
headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
signal: AbortSignal.timeout(2_000),
});
if (!response.ok) return;
const payload: unknown = await response.json();
for (const requestId of this.observePendingRequests(generation.id, payload)) {
requestIds.add(requestId);
}
} catch {
// The legacy and v2 surfaces vary by bundled engine version. A missing
// list never blocks the rollover; path routing still handles scoped ids.
}
}),
));
return [...requestIds];
}
private updateActiveSessions(generation: Generation, sessionIds: string[]): void {
const next = new Set(sessionIds);
this.activeSessionsByGeneration.set(generation.id, next);
for (const [sessionId, generationId] of this.sessionOwnership) {
if (generationId === generation.id && !next.has(sessionId)) this.sessionOwnership.delete(sessionId);
}
for (const sessionId of next) this.sessionOwnership.set(sessionId, generation.id);
}
private connectionFor(generation: Generation & { status: "primary" | "draining" }): EnginePoolConnection {
return {
generationId: generation.id,
role: generation.status,
baseUrl: generation.handle.url,
username: generation.handle.username,
password: generation.handle.password,
};
}
private generationForId(generationId: string | undefined): (Generation & { status: "primary" | "draining" }) | null {
if (!generationId) return null;
const generation = this.generations.find((entry) => entry.id === generationId) ?? null;
return generation && isRoutableGeneration(generation) ? generation : null;
}
private abortEventProxies(): void {
const controllers = [...this.eventProxyControllers];
this.eventProxyControllers.clear();
for (const controller of controllers) controller.abort(new Error("OmniRush engine generation changed"));
}
private async abortSession(
generation: Generation,
sessionId: string,
cause: { source: string; reason: string } = {
source: "engine_pool.drain_timeout",
reason: "draining engine saw no session activity for the grace period",
},
): Promise<void> {
this.hooks.logger?.log("info", "Aborting OmniRush session from engine pool.", {
"abort.source": cause.source,
"abort.initiator": "system",
"abort.reason": cause.reason,
"session.id": sessionId,
"engine.generation_id": generation.id,
});
try {
await loopbackFetch(new URL(`/session/${encodeURIComponent(sessionId)}/abort`, generation.handle.url).toString(), {
method: "POST",
headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
signal: AbortSignal.timeout(5_000),
});
this.hooks.logger?.log("info", "OmniRush session abort from engine pool completed.", {
"abort.source": cause.source,
"abort.initiator": "system",
"session.id": sessionId,
"engine.generation_id": generation.id,
});
} catch (error) {
this.hooks.logger?.log("error", "OmniRush session abort from engine pool failed.", {
"abort.source": cause.source,
"abort.initiator": "system",
"session.id": sessionId,
"engine.generation_id": generation.id,
"error.message": error instanceof Error ? error.message : String(error),
});
throw error;
}
}
}
const poolByConfig = new WeakMap<ServerConfig, EnginePool>();
export function setEnginePoolForConfig(config: ServerConfig, pool: EnginePool): void {
poolByConfig.set(config, pool);
}
export function enginePoolForConfig(config: ServerConfig): EnginePool | null {
return poolByConfig.get(config) ?? null;
}
export function clearEnginePoolForConfig(config: ServerConfig): void {
poolByConfig.delete(config);
}