Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
import { existsSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { Worker } from "node:worker_threads";
import { sanitizeErrorMessage } from "../../utils/errorSanitization.ts";
import { notifyCompressionFailOpen } from "./failOpenNotifier.ts";
import type { CompressionResult } from "./types.ts";
import type { StackedCompressionStep } from "./strategySelector.ts";
import type {
CompressionWorkerJob,
CompressionWorkerMessage,
CompressionWorkerOptions,
} from "./compressionWorkerProtocol.ts";
function positiveInteger(value: string | undefined, fallback: number): number {
const parsed = Number(value);
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
}
/** Relative path (from an install root) to the compression worker. */
const WORKER_JS_REL = join("open-sse", "services", "compression", "compressionWorker.js");
const WORKER_TS_REL = join("open-sse", "services", "compression", "compressionWorker.ts");
const MAX_WALK_UP = 8;
/**
* Walk up from each anchor directory (≤ MAX_WALK_UP levels) and return the first
* ancestor that actually contains `relPath`, or null. Pure + exported for tests.
*
* This deliberately avoids `import.meta.url`/`__dirname` (both dead in the standalone
* bundle) — see the LLMLingua worker comments in llmlingua/worker.ts.
*/
export function firstAncestorWith(anchors: string[], relPath: string): string | null {
for (const anchor of anchors) {
if (!anchor) continue;
let dir = resolve(anchor);
for (let i = 0; i <= MAX_WALK_UP; i++) {
if (existsSync(join(dir, relPath))) return dir;
const parent = dirname(dir);
if (parent === dir) break;
dir = parent;
}
}
return null;
}
/**
* Runtime install-root anchors that SURVIVE the standalone bundle:
* - `process.cwd()` — `dist/server.js` runs `process.chdir(__dirname)` → the dist root.
* - `dirname(process.argv[1])` — the entry script (server.js / bin), walked up.
*/
function runtimeAnchors(): string[] {
const anchors = [process.cwd()];
const argv1 = process.argv[1];
if (typeof argv1 === "string" && argv1) anchors.push(dirname(argv1));
return anchors;
}
/**
* Resolve the worker entry file across dev and prod WITHOUT `import.meta.url`.
*
* Prod: the worker is likely a .js file under the install root
* Dev: the same relative path resolves to the `.ts` source under the project
* root (cwd) and runs via the default Node.js loader.
*
* First existing candidate wins. Exported for tests.
*/
export function resolveWorkerFile(): string {
const anchors = runtimeAnchors();
// The tracked .js file is only a Turbopack build-time placeholder in the source tree.
// A colocated bundle has its own ESM scope, written by colocate-standalone.mjs.
const jsRoot = firstAncestorWith(anchors, WORKER_JS_REL);
if (
jsRoot &&
(!existsSync(join(jsRoot, WORKER_TS_REL)) ||
existsSync(join(dirname(join(jsRoot, WORKER_JS_REL)), "package.json")))
)
return join(jsRoot, WORKER_JS_REL);
// Dev: the .ts source.
const tsRoot = firstAncestorWith(anchors, WORKER_TS_REL);
if (tsRoot) return join(tsRoot, WORKER_TS_REL);
// Nothing found — return a cwd-relative .js path; the spawn will fail-open.
return join(process.cwd(), WORKER_JS_REL);
}
function unchanged(body: Record<string, unknown>): CompressionResult {
return { body, compressed: false, stats: null };
}
/**
* #13145: why a worker fault happened decides what the caller may do about it.
*
* `retryInProcess: false` marks a fault whose work is *provably expensive* — a dispatch
* timeout means the worker already spent its whole budget without finishing, so re-running
* the same CPU-bound pipeline on the main event loop would stall every other in-flight
* request. Those degrade to the uncompressed body, as before, but are now reported instead
* of being swallowed. Every other fault (thread error, exit, engine throw) fails fast
* without doing the work, so retrying in-process is cheap and restores compression.
*/
export class CompressionWorkerError extends Error {
readonly retryInProcess: boolean;
constructor(message: string, retryInProcess: boolean) {
super(message);
this.name = "CompressionWorkerError";
this.retryInProcess = retryInProcess;
}
}
/** Sanitized, single-line error text for fail-open log details. */
function errorText(error: unknown): string {
return sanitizeErrorMessage(error instanceof Error ? error.message : error);
}
/**
* Only known path/module/configuration errors are structural. Unknown failures
* (including ERR_WORKER_INIT_FAILED) can be transient resource exhaustion and
* must be retried on the next wave rather than disabling compression forever.
*/
function isStructuralSpawnFailure(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException | undefined)?.code;
return (
code === "MODULE_NOT_FOUND" ||
code === "ERR_MODULE_NOT_FOUND" ||
code === "ERR_WORKER_PATH" ||
code === "ERR_INVALID_ARG_TYPE" ||
code === "ERR_INVALID_ARG_VALUE"
);
}
function isWorkerOutOfMemory(error: unknown): boolean {
return (error as NodeJS.ErrnoException | undefined)?.code === "ERR_WORKER_OUT_OF_MEMORY";
}
interface PendingJob extends CompressionWorkerJob {
originalBody: Record<string, unknown>;
resolve: (result: CompressionResult) => void;
// #13145: a worker failure must be reportable to the caller. Without a reject path the
// pool could only degrade to `unchanged(...)`, which silently disabled compression for
// the whole request while every layer above still believed the plan had been applied.
reject: (error: Error) => void;
onEngineStep?: (step: StackedCompressionStep) => void;
}
interface PoolWorker {
worker: Worker;
job: PendingJob | null;
timeout: NodeJS.Timeout | null;
idle: NodeJS.Timeout | null;
}
export class CompressionWorkerPool {
private readonly queue: PendingJob[] = [];
private readonly workers = new Set<PoolWorker>();
private nextId = 1;
private readonly size: number;
private readonly timeoutMs: number;
private readonly idleMs: number;
private readonly maxOldGenerationSizeMb: number;
private readonly spawnWorker: () => Worker;
/**
* Set when spawn() throws synchronously (e.g. Turbopack's moduleContext
* MODULE_NOT_FOUND in the standalone build). A pool that cannot create a
* single worker is structurally broken — every subsequent run() fail-opens
* immediately instead of pushing jobs into a queue that can never drain
* (unbounded main-isolate heap leak, one full request body per job).
*/
private broken = false;
constructor({
size = positiveInteger(process.env.OMNI_COMPRESSION_WORKERS, 2),
timeoutMs = positiveInteger(process.env.OMNI_COMPRESSION_WORKER_TIMEOUT_MS, 120_000),
idleMs = positiveInteger(process.env.OMNI_COMPRESSION_WORKER_IDLE_MS, 60_000),
maxOldGenerationSizeMb = positiveInteger(process.env.OMNI_COMPRESSION_WORKER_MAX_OLD_MB, 1024),
workerFactory,
}: {
size?: number;
timeoutMs?: number;
idleMs?: number;
maxOldGenerationSizeMb?: number;
/** Test seam: replaces `new Worker(resolveWorkerFile(), { resourceLimits })`. */
workerFactory?: () => Worker;
} = {}) {
this.size = Math.max(1, Math.floor(size));
this.timeoutMs = Math.max(1, Math.floor(timeoutMs));
this.idleMs = Math.max(1, Math.floor(idleMs));
this.maxOldGenerationSizeMb = Math.max(128, Math.floor(maxOldGenerationSizeMb));
this.spawnWorker =
workerFactory ??
(() =>
new Worker(resolveWorkerFile(), {
resourceLimits: {
maxOldGenerationSizeMb: this.maxOldGenerationSizeMb,
},
}));
}
run(
body: Record<string, unknown>,
mode: CompressionWorkerJob["mode"],
options?: CompressionWorkerOptions,
onEngineStep?: (step: StackedCompressionStep) => void
): Promise<CompressionResult> {
if (this.broken) {
// Without this the pool fails open silently forever after the one startup
// warn — exactly the invisibility that let issue #2 leak for hours.
const reason = "compression pool broken (worker spawn failed)";
notifyCompressionFailOpen(reason);
return Promise.reject(new CompressionWorkerError(reason, true));
}
return new Promise((resolve, reject) => {
this.queue.push({
id: this.nextId++,
body,
mode,
options,
originalBody: body,
resolve,
reject,
onEngineStep,
});
this.dispatch();
});
}
async close(): Promise<void> {
for (const job of this.queue.splice(0)) job.resolve(unchanged(job.originalBody));
for (const slot of this.workers) {
const job = slot.job;
if (job) job.resolve(unchanged(job.originalBody));
slot.job = null;
}
await Promise.all([...this.workers].map((slot) => this.remove(slot)));
}
private spawn(): PoolWorker {
const slot: PoolWorker = {
worker: this.spawnWorker(),
job: null,
timeout: null,
idle: null,
};
this.workers.add(slot);
slot.worker.on("message", (message: CompressionWorkerMessage) =>
this.handleMessage(slot, message)
);
slot.worker.on("error", (error) =>
this.fail(
slot,
`worker error: ${errorText(error)}`,
// A worker that exhausted its private heap already proved this payload is
// unsafe to retry on the gateway's main isolate. Fail open instead of
// recreating the same host-OOM condition in-process.
!isWorkerOutOfMemory(error)
)
);
slot.worker.on("exit", (code) => {
if (this.workers.has(slot)) this.fail(slot, `worker exit code ${code}`);
});
return slot;
}
private spawnOrFailOpen(): PoolWorker | null {
try {
return this.spawn();
} catch (error) {
// A synchronous spawn failure (bundler module-context miss, bad worker
// path, …) must never leave the queue stranded: nothing else would ever
// call dispatch() again, so the jobs — and their full request bodies —
// would be retained for the lifetime of the process.
const structural = isStructuralSpawnFailure(error);
// A structural failure is only pool-wide when no worker was ever created.
// Existing workers remain usable even if an attempt to add capacity fails.
const stranded = this.workers.size === 0;
this.broken = structural && stranded;
const reason = `worker spawn failed — pool failing open${this.broken ? " permanently" : " for this wave"}: ${errorText(error)}`;
notifyCompressionFailOpen(reason);
// Only a queue with no worker left to drain it is stranded. While any worker
// survives it is busy by construction (dispatch() only spawns once no idle
// slot exists), and its finish()/abort() re-enters dispatch() — so failing
// the backlog open here would needlessly drop compression for jobs a healthy
// worker is about to pick up. A spawn throw is a fast, pre-dispatch fault, so
// reject as retryable and let the caller preserve compression in-process.
if (stranded) {
for (const job of this.queue.splice(0)) {
job.reject(new CompressionWorkerError(reason, true));
}
}
return null;
}
}
private dispatch(): void {
while (this.queue.length) {
let slot = [...this.workers].find((candidate) => !candidate.job);
if (!slot && this.workers.size < this.size) slot = this.spawnOrFailOpen() ?? undefined;
if (!slot) return;
if (slot.idle) clearTimeout(slot.idle);
const job = this.queue.shift();
if (!job) return;
slot.job = job;
slot.timeout = setTimeout(
() => this.fail(slot!, `worker job timeout after ${this.timeoutMs}ms`, false),
this.timeoutMs
);
slot.timeout.unref();
// `reject` must be stripped alongside the other non-cloneable fields: postMessage
// uses structured clone, and leaking any function into the wire job throws
// DataCloneError before the worker ever sees it.
const {
originalBody: _body,
resolve: _resolve,
reject: _reject,
onEngineStep: _step,
...wireJob
} = job;
try {
slot.worker.postMessage(wireJob);
} catch (error) {
// A non-cloneable payload (DataCloneError) must not strand the slot until its
// timeout. The work never reached a worker, so retrying in-process is cheap.
this.fail(slot, `worker postMessage failed: ${errorText(error)}`);
return;
}
}
}
private handleMessage(slot: PoolWorker, message: CompressionWorkerMessage): void {
const job = slot.job;
if (!job || job.id !== message.id) return;
if (message.type === "step") {
try {
job.onEngineStep?.(message.step);
} catch {
// Telemetry is best-effort.
}
return;
}
if (message.type === "result") {
this.finish(slot, message.result);
return;
}
// #13145: the worker reported a thrown engine error. Surface it instead of quietly
// handing back the uncompressed body — the caller falls back to in-process compression.
this.abort(
slot,
new CompressionWorkerError(`compression worker error: ${message.error}`, true)
);
}
private finish(slot: PoolWorker, result: CompressionResult): void {
const job = slot.job;
if (!job) return;
if (slot.timeout) clearTimeout(slot.timeout);
slot.timeout = null;
slot.job = null;
job.resolve(result);
// Idle eviction MUST terminate. Dropping the slot from the set only releases our
// reference - the thread, its MessagePort and its private heap outlive the pool
// for the whole process lifetime, invisible to process.memoryUsage(). (#12812)
slot.idle = setTimeout(() => void this.remove(slot), this.idleMs);
slot.idle.unref();
this.dispatch();
}
private fail(
slot: PoolWorker,
reason = "compression worker failed or timed out",
retryInProcess = true
): void {
this.abort(slot, new CompressionWorkerError(reason, retryInProcess));
}
/** #13145: release a slot and report the failure to the caller so it can fall back to
* in-process compression. Previously this resolved with the uncompressed body, which
* turned every worker fault into a silent, unlogged no-op — the exact zero-log
* condition that let issue #2 run for hours. The caller decides via
* `retryInProcess` whether recovery is cheap; the notifier keeps every fault
* visible either way. */
private abort(slot: PoolWorker, error: CompressionWorkerError): void {
notifyCompressionFailOpen(`compression worker job failed open (${error.message})`);
const job = slot.job;
if (slot.timeout) clearTimeout(slot.timeout);
slot.timeout = null;
slot.job = null;
if (job) job.reject(error);
void this.remove(slot).finally(() => this.dispatch());
}
/** Drop a slot and release its OS thread. Removal always terminates: a pooled worker
* has no other owner, so skipping terminate() strands the thread permanently. */
private async remove(slot: PoolWorker): Promise<void> {
if (!this.workers.delete(slot)) return;
if (slot.timeout) clearTimeout(slot.timeout);
if (slot.idle) clearTimeout(slot.idle);
await slot.worker.terminate().catch(() => undefined);
}
}
let pool: CompressionWorkerPool | null = null;
let workerFactoryOverride: (() => Worker) | undefined;
export function runCompressionInWorker(
body: Record<string, unknown>,
mode: CompressionWorkerJob["mode"],
options?: CompressionWorkerOptions,
onEngineStep?: (step: StackedCompressionStep) => void
): Promise<CompressionResult> {
pool ??= new CompressionWorkerPool({ workerFactory: workerFactoryOverride });
return pool.run(body, mode, options, onEngineStep);
}
/** Test seam: force the lazily created shared pool to use a factory that throws
* synchronously (Turbopack moduleContext MODULE_NOT_FOUND) or serves fake workers. */
export async function __setCompressionWorkerFactoryForTests(
factory: (() => Worker) | null
): Promise<void> {
await closeCompressionWorkerPoolForTests();
workerFactoryOverride = factory ?? undefined;
}
export async function closeCompressionWorkerPoolForTests(): Promise<void> {
const active = pool;
pool = null;
await active?.close();
}