Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: close unlocked engines on relock and check the routed engine

ogsamratd1bbe9ab4dff
diff --git a/apps/server/src/engine-pool.test.ts b/apps/server/src/engine-pool.test.ts
index ee3f81b1..86c97538 100644
--- a/apps/server/src/engine-pool.test.ts
+++ b/apps/server/src/engine-pool.test.ts
@@ -1007,6 +1007,32 @@ describe("engine pool", () => {
expect(lines).toContain(`${oldPort} SIGTERM`);
});
+ test("a rollover that retires matching draining engines aborts their running sessions and closes them at once", async () => {
+ setEnv("OMNIRUSH_ENGINE_ABORT_SETTLE_MS", "50");
+ const fixture = await createFixture();
+ const { pool, primary } = await createPool(fixture);
+ const oldPort = portOf(primary.url);
+ await fixture.setBusy(oldPort, ["ses_live"]);
+ await fixture.setRuntimeConfig(JSON.stringify({ generation: 2 }));
+
+ const outcome = await pool.requestRollover({
+ reason: "config_changed",
+ workspace: fixture.workspace,
+ manual: true,
+ forceStandby: true,
+ retireDrainingIf: (url) => portOf(url) === oldPort,
+ });
+
+ expect(outcome.action).toBe("rolled_over");
+ expect(primary.isAlive()).toBe(false);
+ expect(pool.snapshot().generations.map((entry) => entry.role)).toEqual(["primary"]);
+ expect(pool.isPrimaryUrl(pool.primaryUrl()!)).toBe(true);
+ expect(pool.isPrimaryUrl(primary.url)).toBe(false);
+ const lines = await fixture.logLines();
+ expect(lines).toContain(`${oldPort} POST /session/ses_live/abort`);
+ expect(lines).toContain(`${oldPort} SIGTERM`);
+ });
+
test("uses one global activity stream across many workspaces and never aborts an active session", async () => {
setEnv("OMNIRUSH_ENGINE_DRAIN_TIMEOUT_MS", "300");
setEnv("OMNIRUSH_ENGINE_ABORT_SETTLE_MS", "100");
diff --git a/apps/server/src/engine-pool.ts b/apps/server/src/engine-pool.ts
index 2b0bfc9e..08669422 100644
--- a/apps/server/src/engine-pool.ts
+++ b/apps/server/src/engine-pool.ts
@@ -452,6 +452,7 @@ export class EnginePool {
manual: boolean;
awaitPostRefreshSync: boolean;
forceStandby: boolean;
+ retireDrainingIf?: (baseUrl: string) => boolean;
} | null = null;
private lastSpawnAt = 0;
private consecutiveConnectionFailures = 0;
@@ -730,6 +731,34 @@ export class EnginePool {
return { sessions: [...sessions], unknown };
}
+ /** Whether `baseUrl` is the engine new work goes to (not a draining one). */
+ isPrimaryUrl(baseUrl: string): boolean {
+ return this.isPrimaryEndpoint(baseUrl);
+ }
+
+ /**
+ * Close draining engines now instead of letting their runs finish: their
+ * running turns are aborted first. For a change the old engine must not keep
+ * serving under (the provider lock); `shouldRetire` picks the engines by URL.
+ */
+ async retireDraining(shouldRetire: (baseUrl: string) => boolean, reason: string): Promise<number> {
+ const targets = this.generations.filter(
+ (entry): entry is Generation & { status: "draining" } => entry.status === "draining" && shouldRetire(entry.handle.url),
+ );
+ for (const generation of targets) {
+ const sessions = new Set([
+ ...(this.activeSessionsByGeneration.get(generation.id) ?? []),
+ ...await this.nonIdleSessionIds(generation).catch(() => [] as string[]),
+ ]);
+ for (const sessionId of sessions) {
+ await this.abortSession(generation, sessionId, { source: "engine_pool.retire_draining", reason }).catch(() => undefined);
+ }
+ if (sessions.size > 0) await new Promise((resolve) => setTimeout(resolve, abortSettleMs()));
+ await this.retire(generation, "forced");
+ }
+ return targets.length;
+ }
+
/** Provider sync treats an active drain as busy so it defers instead of parking on waitForDrain(). */
hasDrainingGeneration(): boolean {
return this.generations.some((entry) => entry.status === "draining");
@@ -747,6 +776,8 @@ export class EnginePool {
awaitPostRefreshSync?: boolean;
/** Apply a config change through a healthy standby even when the primary is idle. */
forceStandby?: boolean;
+ /** Draining engines this matches are closed (runs aborted) instead of drained. */
+ retireDrainingIf?: (baseUrl: string) => boolean;
}): Promise<RolloverOutcome> {
if (this.disposed) return { action: "skipped", reason: "unchanged" };
const request = {
@@ -755,6 +786,7 @@ export class EnginePool {
manual: input.manual === true,
awaitPostRefreshSync: input.awaitPostRefreshSync !== false,
forceStandby: input.forceStandby === true,
+ ...(input.retireDrainingIf ? { retireDrainingIf: input.retireDrainingIf } : {}),
};
if (this.inFlight) {
// Latest wins: a manual request keeps its manual flag so it still
@@ -764,6 +796,9 @@ export class EnginePool {
manual: request.manual || this.pendingRollover?.manual === true,
awaitPostRefreshSync: request.awaitPostRefreshSync || this.pendingRollover?.awaitPostRefreshSync === true,
forceStandby: request.forceStandby || this.pendingRollover?.forceStandby === true,
+ ...((request.retireDrainingIf ?? this.pendingRollover?.retireDrainingIf)
+ ? { retireDrainingIf: request.retireDrainingIf ?? this.pendingRollover?.retireDrainingIf }
+ : {}),
};
this.hooks.logger?.log("info", "Engine rollover coalesced into the in-flight request.", {
"engine.rollover.reason": request.reason,
@@ -790,8 +825,10 @@ export class EnginePool {
manual: boolean;
awaitPostRefreshSync: boolean;
forceStandby: boolean;
+ retireDrainingIf?: (baseUrl: string) => boolean;
}): Promise<RolloverOutcome> {
- const { workspace, reason, manual, awaitPostRefreshSync, forceStandby } = request;
+ const { workspace, reason, manual, awaitPostRefreshSync, forceStandby, retireDrainingIf } = request;
+ if (retireDrainingIf) await this.retireDraining(retireDrainingIf, `engine rollover: ${reason}`);
// The standby reads config from disk at spawn, so make sure the file is
// current before deciding anything.
await this.hooks.writeRuntimeConfigFile(this.config).catch(() => undefined);
@@ -830,7 +867,7 @@ export class EnginePool {
}
// Cap: one primary plus one draining. Park this change; it lands when
// the current drain finishes.
- this.pendingRollover = { reason, workspace, manual, awaitPostRefreshSync, forceStandby };
+ this.pendingRollover = { reason, workspace, manual, awaitPostRefreshSync, forceStandby, ...(retireDrainingIf ? { retireDrainingIf } : {}) };
this.hooks.logger?.log("info", "Engine rollover parked behind an active drain.", {
"engine.rollover.reason": reason,
});
@@ -839,7 +876,7 @@ export class EnginePool {
const sinceLastSpawn = Date.now() - this.lastSpawnAt;
if (!manual && !forceStandby && sinceLastSpawn < minSpawnIntervalMs()) {
- this.pendingRollover = { reason, workspace, manual, awaitPostRefreshSync, forceStandby };
+ this.pendingRollover = { reason, workspace, manual, awaitPostRefreshSync, forceStandby, ...(retireDrainingIf ? { retireDrainingIf } : {}) };
this.hooks.logger?.log("info", "Engine rollover throttled by the minimum spawn interval.", {
"engine.rollover.reason": reason,
"engine.rollover.since_last_spawn_ms": sinceLastSpawn,
@@ -847,7 +884,9 @@ export class EnginePool {
return { action: "coalesced" };
}
- return await this.rollOver({ reason, workspace, fingerprint, primary });
+ const outcome = await this.rollOver({ reason, workspace, fingerprint, primary });
+ if (retireDrainingIf && outcome.action === "rolled_over") await this.retireDraining(retireDrainingIf, `engine rollover: ${reason}`);
+ return outcome;
}
private async rollOver(input: {
@@ -1596,11 +1635,18 @@ export class EnginePool {
for (const controller of controllers) controller.abort(new Error("OmniRush engine generation changed"));
}
- private async abortSession(generation: Generation, sessionId: string): Promise<void> {
+ private async abortSession(
+ generation: Generation,
+ sessionId: string,
+ cause: { source: string; reason: string } = {
+ source: "engine_pool.drain_timeout",
+ reason: "draining engine saw no session activity for the grace period",
+ },
+ ): Promise<void> {
this.hooks.logger?.log("info", "Aborting OmniRush session from engine pool.", {
- "abort.source": "engine_pool.drain_timeout",
+ "abort.source": cause.source,
"abort.initiator": "system",
- "abort.reason": "draining engine saw no session activity for the grace period",
+ "abort.reason": cause.reason,
"session.id": sessionId,
"engine.generation_id": generation.id,
});
@@ -1611,14 +1657,14 @@ export class EnginePool {
signal: AbortSignal.timeout(5_000),
});
this.hooks.logger?.log("info", "OmniRush session abort from engine pool completed.", {
- "abort.source": "engine_pool.drain_timeout",
+ "abort.source": cause.source,
"abort.initiator": "system",
"session.id": sessionId,
"engine.generation_id": generation.id,
});
} catch (error) {
this.hooks.logger?.log("error", "OmniRush session abort from engine pool failed.", {
- "abort.source": "engine_pool.drain_timeout",
+ "abort.source": cause.source,
"abort.initiator": "system",
"session.id": sessionId,
"engine.generation_id": generation.id,
diff --git a/apps/server/src/provider-lock.test.ts b/apps/server/src/provider-lock.test.ts
index 17fad472..f976eafd 100644
--- a/apps/server/src/provider-lock.test.ts
+++ b/apps/server/src/provider-lock.test.ts
@@ -15,6 +15,7 @@ import {
assertProxiedEngineRequestWithinProviderLock,
currentProviderLockPolicy,
engineRequestProviderLockDenial,
+ engineStartedUnlocked,
lockedEngineEnv,
providerAllowed,
providerLockConfigViolations,
@@ -244,6 +245,10 @@ describe("provider lock: refusals", () => {
["POST", "/session/ses_1/command", { command: "c", model: "openai/gpt-x" }],
["POST", "/api/session/ses_1/prompt", { model: { providerID: "opencode", modelID: "zen" } }],
["POST", "/session", { model: { providerID: "lpr_org1", id: "m" } }],
+ ["POST", "/SESSION/ses_1/prompt_async", { model: { providerID: "openai", modelID: "gpt-x" } }],
+ ["PUT", "/Auth/openai", { type: "api", key: "sk-fake" }],
+ ["PATCH", "/API/Config", { model: "openai/gpt-x" }],
+ ["POST", "/session/ses_1/prompt_async", { parts: [{ type: "subtask", prompt: "p", model: { providerID: "openai", modelID: "gpt-x" } }] }],
];
for (const [method, path, body] of refused) expect([method, path, engineRequestProviderLockDenial(method, path, body)]).toEqual([method, path, "Only OmniRush models are available."]);
const allowed: Array<[string, string, Record<string, unknown>]> = [
@@ -310,16 +315,33 @@ describe("provider lock: the app's own provider and engines started unlocked", (
noteEngineStarted("http://127.0.0.1:4001");
resetProviderLock();
noteEngineStarted("http://127.0.0.1:4002");
- for (const [method, path] of [["POST", "/session"], ["POST", "/session/ses_1/prompt_async"], ["POST", "/opencode/session/ses_1/message"], ["POST", "/api/session/ses_1/command"], ["POST", "/session/ses_1/shell"], ["POST", "/session/ses_1/summarize"]] as const) {
+ for (const [method, path] of [["POST", "/session"], ["POST", "/session/ses_1/prompt_async"], ["POST", "/opencode/session/ses_1/message"], ["POST", "/api/session/ses_1/command"], ["POST", "/session/ses_1/shell"], ["POST", "/session/ses_1/summarize"], ["POST", "/session/ses_1/revert"], ["POST", "/session/ses_1/unrevert"], ["POST", "/Session/ses_1/Prompt_Async"]] as const) {
expect(() => assertEngineStartedWithinLock("http://127.0.0.1:4001/", method, path)).toThrow("Only OmniRush models are available. Restart the app to continue.");
expect(() => assertEngineStartedWithinLock("http://127.0.0.1:4002", method, path)).not.toThrow();
}
expect(() => assertEngineStartedWithinLock("http://127.0.0.1:4001", "GET", "/session/ses_1/message")).not.toThrow();
expect(() => assertEngineStartedWithinLock("http://127.0.0.1:4001", "POST", "/session/ses_1/abort")).not.toThrow();
expect(() => assertEngineStartedWithinLock("http://127.0.0.1:4999", "POST", "/session")).not.toThrow();
+ expect(engineStartedUnlocked("http://127.0.0.1:4001/x")).toBe(true);
+ expect(engineStartedUnlocked("http://127.0.0.1:4002")).toBe(false);
applyProviderLockAnswer(UNLOCKED);
expect(() => assertEngineStartedWithinLock("http://127.0.0.1:4001", "POST", "/session")).not.toThrow();
});
+
+ test("while a locked engine takes over, a draining unlocked one answers retry with 503", () => {
+ applyProviderLockAnswer(UNLOCKED);
+ noteEngineStarted("http://127.0.0.1:4001");
+ resetProviderLock();
+ const error = (() => {
+ try {
+ assertEngineStartedWithinLock("http://127.0.0.1:4001", "POST", "/session/ses_1/prompt_async", currentProviderLockPolicy(), { switching: true });
+ } catch (caught) {
+ return caught;
+ }
+ return null;
+ })();
+ expect(error).toMatchObject({ status: 503, code: "provider_locked", message: "Switching to OmniRush models, retry in a moment." });
+ });
});
describe("provider lock: engine route paths", () => {
diff --git a/apps/server/src/provider-lock.ts b/apps/server/src/provider-lock.ts
index 93cfc735..93299d39 100644
--- a/apps/server/src/provider-lock.ts
+++ b/apps/server/src/provider-lock.ts
@@ -233,17 +233,19 @@ export function engineRequestProviderLockDenial(
if (!policy.locked) return null;
const verb = method.toUpperCase();
if (["GET", "HEAD", "OPTIONS", "DELETE"].includes(verb)) return null;
- const path = enginePath.replace(/^\/api(?=\/)/, "");
- const auth = path.match(/^\/auth\/([^/]+)\/?$/);
+ const path = enginePath.replace(/^\/api(?=\/)/i, "");
+ const auth = path.match(/^\/auth\/([^/]+)\/?$/i);
if (auth) return providerAllowed(decodeURIComponent(auth[1] ?? ""), policy) ? null : policy.message;
- const oauth = path.match(/^\/provider\/([^/]+)\/oauth(?:\/|$)/);
+ const oauth = path.match(/^\/provider\/([^/]+)\/oauth(?:\/|$)/i);
if (oauth) return providerAllowed(decodeURIComponent(oauth[1] ?? ""), policy) ? null : policy.message;
- if (/^\/(?:global\/)?config\/?$/.test(path)) {
+ if (/^\/(?:global\/)?config\/?$/i.test(path)) {
if (configAddsPlugins(body)) return PLUGINS_LOCKED_MESSAGE;
return providerLockConfigViolations(body, policy).length ? policy.message : null;
}
- if (/^\/session(?:\/|$)/.test(path)) {
- for (const value of [body.model, body.providerID === undefined ? undefined : { providerID: body.providerID }]) {
+ if (/^\/session(?:\/|$)/i.test(path)) {
+ // The prompt's model, and each part's own model (a sub-task part names one).
+ const parts = Array.isArray(body.parts) ? body.parts.filter(isRecord).map((part) => part.model) : [];
+ for (const value of [body.model, body.providerID === undefined ? undefined : { providerID: body.providerID }, ...parts]) {
const provider = modelProviderOf(value);
if (provider !== undefined && provider !== null && !providerAllowed(provider, policy)) return policy.message;
}
@@ -315,6 +317,11 @@ function originOf(url: string): string {
const engineStarts = new Map<string, boolean>();
/** An engine started at `url` under `policy` (managed-opencode.ts, managed-opencode-v2.ts). */
+/** An engine this app started before the lock applied (its prompts are refused, its runs closed). */
+export function engineStartedUnlocked(url: string): boolean {
+ return engineStarts.get(originOf(url)) === false;
+}
+
export function noteEngineStarted(url: string, policy: ProviderLockPolicy = current): void {
engineStarts.set(originOf(url), policy.locked);
}
@@ -327,8 +334,8 @@ export function resetEngineStarts(): void {
/** A request that makes the engine do model work: a new chat, a prompt, a command, a shell turn, a summary. */
export function isModelWorkRequest(method: string, enginePath: string): boolean {
if (method.toUpperCase() !== "POST") return false;
- const path = enginePath.replace(/^\/opencode2?(?=\/|$)/, "").replace(/^\/api(?=\/)/, "").replace(/\/+$/, "");
- return path === "/session" || /^\/session\/[^/]+\/(?:prompt|prompt_async|message|command|shell|summarize|init)$/.test(path);
+ const path = enginePath.replace(/^\/opencode2?(?=\/|$)/i, "").replace(/^\/api(?=\/)/i, "").replace(/\/+$/, "");
+ return /^\/session$/i.test(path) || /^\/session\/[^/]+\/(?:prompt|prompt_async|message|command|shell|summarize|init|revert|unrevert)$/i.test(path);
}
/**
@@ -338,9 +345,14 @@ export function isModelWorkRequest(method: string, enginePath: string): boolean
* providers): 403 provider_locked, asking for a restart. Engines the app
* did not start, and every other request, pass.
*/
-export function assertEngineStartedWithinLock(engineUrl: string | null | undefined, method: string, enginePath: string, policy: ProviderLockPolicy = current): void {
+export const PROVIDER_SWITCHING_MESSAGE = "Switching to OmniRush models, retry in a moment.";
+
+export function assertEngineStartedWithinLock(engineUrl: string | null | undefined, method: string, enginePath: string, policy: ProviderLockPolicy = current, options: { switching?: boolean } = {}): void {
if (!policy.locked || !engineUrl || !isModelWorkRequest(method, engineRoutePath(enginePath))) return;
- if (engineStarts.get(originOf(engineUrl)) === false) throw new ApiError(403, PROVIDER_LOCKED_CODE, `${policy.message} Restart the app to continue.`);
+ if (engineStarts.get(originOf(engineUrl)) !== false) return;
+ // A newer engine under the lock is already serving (this one drains): the request can be sent again.
+ if (options.switching) throw new ApiError(503, PROVIDER_LOCKED_CODE, PROVIDER_SWITCHING_MESSAGE);
+ throw new ApiError(403, PROVIDER_LOCKED_CODE, `${policy.message} Restart the app to continue.`);
}
/** The policy for the app's UI (GET /omnirush/provider-lock). */
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index a2de4102..9b3f0956 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -194,6 +194,7 @@ import {
assertEngineStartedWithinLock,
assertProxiedEngineRequestWithinProviderLock,
currentProviderLockPolicy,
+ engineStartedUnlocked,
parseJsoncContent,
providerLockStatus,
watchProviderLock,
@@ -1592,6 +1593,7 @@ export async function startServer(config: ServerConfig): Promise<ServeResult> {
if (!connection) {
return finalize(jsonResponse({ error: "engine_v2_preview_not_running" }, 503));
}
+ assertEngineStartedWithinLock(connection.url, request.method, mount.restPath);
proxyService = "opencode";
proxyBaseUrl = connection.url;
await engineV2Preview.ensureWorkspaceReady(workspace.path);
@@ -2434,6 +2436,12 @@ export async function proxyOpencodeRequest(input: {
const route = pool?.routeRequest(method, decodeEngineRoutePath(proxyPath)) ?? null;
const baseUrl = route?.target.baseUrl ??
(workspace ? resolveWorkspaceOpencodeConnection(input.config, workspace).baseUrl?.trim() ?? "" : "");
+ // The engine the request actually goes to (a draining one owns its sessions), and its fallback.
+ for (const target of [baseUrl, route?.fallback?.baseUrl]) {
+ if (!target) continue;
+ const switching = pool !== null && !pool.isPrimaryUrl(target) && !engineStartedUnlocked(pool.primaryUrl() ?? "");
+ assertEngineStartedWithinLock(target, method, proxyPath, currentProviderLockPolicy(), { switching });
+ }
if (!baseUrl) {
throw new ApiError(400, "opencode_unconfigured", "OmniRush base URL is missing for this workspace");
}
@@ -3187,7 +3195,14 @@ async function applyBoxEngineConfig(config: ServerConfig, reason: string, option
if (!pool || !workspace) return { status: "unmanaged" };
// `restart`: the engine's environment changes too (the provider lock), so a new engine process, never
// an in-place reload. If it cannot start, the old engine's prompts are refused (assertEngineStartedWithinLock).
- const outcome = await pool.requestRollover({ reason, workspace, manual: true, ...(options.restart ? { forceStandby: true } : {}) });
+ // Under the lock, an engine started without it is closed, its running turns aborted, rather than drained.
+ const outcome = await pool.requestRollover({
+ reason,
+ workspace,
+ manual: true,
+ ...(options.restart ? { forceStandby: true } : {}),
+ ...(options.restart && currentProviderLockPolicy().locked ? { retireDrainingIf: engineStartedUnlocked } : {}),
+ });
return { status: outcome.action === "coalesced" ? "deferred" : outcome.action === "skipped" ? "failed" : "applied" };
} catch {
return { status: "failed" };