Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

Fase 8 · Bloco C — resiliência runtime (chaos + heap-growth + k6 soak) (#3854)

Diego Rodrigues de Sa e Souzad3146a17515f
diff --git a/.github/workflows/nightly-resilience.yml b/.github/workflows/nightly-resilience.yml
new file mode 100644
index 0000000000..9af222d94d
--- /dev/null
+++ b/.github/workflows/nightly-resilience.yml
@@ -0,0 +1,70 @@
+name: Nightly Resilience
+on:
+ schedule:
+ - cron: "41 4 * * *"
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ heap:
+ name: Heap-growth gate
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ - uses: actions/setup-node@v6
+ with:
+ node-version: "24"
+ cache: npm
+ - run: npm ci
+ - run: npm run test:heap
+
+ chaos:
+ name: Resilience chaos (fault injection)
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ - uses: actions/setup-node@v6
+ with:
+ node-version: "24"
+ cache: npm
+ - run: npm ci
+ - run: npm run test:chaos
+
+ k6-soak:
+ name: k6 load/soak
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ - uses: actions/setup-node@v6
+ with:
+ node-version: "24"
+ cache: npm
+ - run: npm ci
+ - name: Build CLI bundle
+ env:
+ JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
+ run: npm run build:cli
+ - name: Start OmniRoute (background)
+ env:
+ JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation
+ PORT: "20128"
+ run: |
+ node dist/server.js > server.log 2>&1 &
+ echo $! > server.pid
+ for i in $(seq 1 30); do
+ if curl -sf http://localhost:20128/api/monitoring/health >/dev/null; then echo "server up"; break; fi
+ sleep 2
+ done
+ - name: Install k6
+ uses: grafana/setup-k6-action@v1
+ - name: Run k6 soak
+ run: k6 run tests/load/k6-soak.js
+ env:
+ BASE_URL: http://localhost:20128
+ SOAK_DURATION: "3m"
+ SOAK_VUS: "10"
+ - name: Stop server
+ if: always()
+ run: kill "$(cat server.pid)" || true
diff --git a/docs/architecture/RESILIENCE_GUIDE.md b/docs/architecture/RESILIENCE_GUIDE.md
index cd9f2c71df..92e2ba0779 100644
--- a/docs/architecture/RESILIENCE_GUIDE.md
+++ b/docs/architecture/RESILIENCE_GUIDE.md
@@ -144,6 +144,22 @@ Provider-specific stealth (JA3/JA4, CCH, obfuscation) is separately documented
---
+## Resilience testing (Fase 8 · Bloco C)
+
+Além dos unit tests da lógica de resiliência, três testes exercitam o runtime sob
+estresse/falha real (todos integração/nightly — nenhum bloqueia PR):
+
+| Teste | O quê | Rodar |
+|---|---|---|
+| Chaos | Fake-upstream node injeta latência/reset/timeout/503 reais; valida que o circuit breaker abre/recupera e `checkFallbackError` classifica 503 como fallback recuperável. | `RUN_CHAOS_INT=1 npm run test:chaos` |
+| Heap-growth | ~500 streams por `createSSEStream` sob `--expose-gc`; falha se o heap crescer além do teto (guarda OOM #3069). | `npm run test:heap` |
+| k6 soak | Carga sustentada contra `/api/monitoring/health`; thresholds p95/erro. | `k6 run tests/load/k6-soak.js` (nightly) |
+
+Orquestrados por `.github/workflows/nightly-resilience.yml` (cron + dispatch). No
+`test:integration` default, chaos e heap se auto-skipam (sem `RUN_CHAOS_INT`/`--expose-gc`).
+
+---
+
## See Also
- [Architecture Guide](./ARCHITECTURE.md) — System architecture and internals
diff --git a/package.json b/package.json
index 9df6c9ef93..784f2d00b6 100644
--- a/package.json
+++ b/package.json
@@ -156,6 +156,8 @@
"backfill-aggregation": "node --import tsx src/scripts/backfillAggregation.ts",
"env:sync": "node scripts/dev/sync-env.mjs",
"test:integration": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/*.test.ts",
+ "test:heap": "node --expose-gc --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit tests/integration/heap-growth.test.ts",
+ "test:chaos": "cross-env RUN_CHAOS_INT=1 node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/resilience-chaos.test.ts",
"test:e2e": "node scripts/dev/run-playwright-tests.mjs test tests/e2e/*.spec.ts",
"test:protocols:e2e": "node scripts/dev/run-protocol-clients-tests.mjs",
"test:vitest": "vitest run --config vitest.mcp.config.ts",
diff --git a/tests/helpers/faultyUpstream.ts b/tests/helpers/faultyUpstream.ts
new file mode 100644
index 0000000000..3eee4f1c19
--- /dev/null
+++ b/tests/helpers/faultyUpstream.ts
@@ -0,0 +1,72 @@
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+
+export type FaultMode =
+ | { kind: "ok"; body?: string }
+ | { kind: "status"; code: number; body?: string }
+ | { kind: "latency"; ms: number; body?: string }
+ | { kind: "reset" }
+ | { kind: "timeout" }
+ | { kind: "slowDrip"; chunkMs: number; body?: string };
+
+export interface FaultyUpstream {
+ readonly url: string;
+ setMode(mode: FaultMode): void;
+ stop(): Promise<void>;
+}
+
+export async function startFaultyUpstream(initial: FaultMode = { kind: "ok" }): Promise<FaultyUpstream> {
+ let mode: FaultMode = initial;
+
+ const server = http.createServer((req, res) => {
+ const m = mode;
+ if (m.kind === "reset") {
+ req.socket.destroy();
+ return;
+ }
+ if (m.kind === "timeout") {
+ return; // never respond; caller relies on AbortSignal/timeout
+ }
+ if (m.kind === "latency") {
+ setTimeout(() => {
+ res.writeHead(200, { "content-type": "text/plain" });
+ res.end(m.body ?? "ok");
+ }, m.ms);
+ return;
+ }
+ if (m.kind === "slowDrip") {
+ const body = m.body ?? "drip-drip-drip";
+ res.writeHead(200, { "content-type": "text/plain" });
+ let i = 0;
+ const timer = setInterval(() => {
+ if (i >= body.length) {
+ clearInterval(timer);
+ res.end();
+ return;
+ }
+ res.write(body[i++]);
+ }, m.chunkMs);
+ req.on("close", () => clearInterval(timer));
+ return;
+ }
+ const code = m.kind === "status" ? m.code : 200;
+ res.writeHead(code, { "content-type": "text/plain" });
+ res.end(m.body ?? (m.kind === "status" ? "error" : "ok"));
+ });
+
+ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+ const { port } = server.address() as AddressInfo;
+
+ return {
+ url: `http://127.0.0.1:${port}/`,
+ setMode(next: FaultMode) {
+ mode = next;
+ },
+ stop() {
+ return new Promise<void>((resolve) => {
+ server.closeAllConnections?.();
+ server.close(() => resolve());
+ });
+ },
+ };
+}
diff --git a/tests/integration/heap-growth.test.ts b/tests/integration/heap-growth.test.ts
new file mode 100644
index 0000000000..d541a9e9a2
--- /dev/null
+++ b/tests/integration/heap-growth.test.ts
@@ -0,0 +1,78 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+// DB setup required by createSSEStream (via trackPendingRequest / appendRequestLog)
+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-heap-growth-"));
+process.env.DATA_DIR = TEST_DATA_DIR;
+const core = await import("../../src/lib/db/core.ts");
+
+const { createSSEStream } = await import("../../open-sse/utils/stream.ts");
+
+test.after(() => {
+ core.resetDbInstance();
+ if (fs.existsSync(TEST_DATA_DIR)) {
+ fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
+ }
+});
+
+const HAS_GC = typeof (globalThis as { gc?: () => void }).gc === "function";
+const enc = new TextEncoder();
+
+function sseChunks(n: number): Uint8Array[] {
+ const parts: Uint8Array[] = [];
+ for (let i = 0; i < n; i++) {
+ parts.push(
+ enc.encode(
+ `data: {"id":"chatcmpl","object":"chat.completion.chunk","created":1,"model":"gpt-4o","choices":[{"delta":{"content":"x"},"index":0}]}\n\n`
+ )
+ );
+ }
+ parts.push(enc.encode(`data: [DONE]\n\n`));
+ return parts;
+}
+
+async function runOneStream(i: number): Promise<void> {
+ const ts = createSSEStream({
+ mode: "passthrough",
+ clientResponseFormat: "openai",
+ connectionId: `heap-${i}`,
+ }) as TransformStream<Uint8Array, Uint8Array>;
+ const writer = ts.writable.getWriter();
+ const reader = ts.readable.getReader();
+ const chunks = sseChunks(10);
+ const pump = (async () => {
+ for (const c of chunks) await writer.write(c);
+ await writer.close();
+ })();
+ for (;;) {
+ const { done } = await reader.read();
+ if (done) break;
+ }
+ await pump;
+}
+
+async function gcSettle(): Promise<void> {
+ const gc = (globalThis as { gc?: () => void }).gc!;
+ gc();
+ await new Promise((r) => setTimeout(r, 50));
+ gc();
+}
+
+test("SSE pipeline does not leak heap across 500 streams", { skip: !HAS_GC }, async () => {
+ for (let i = 0; i < 50; i++) await runOneStream(i); // warmup
+ await gcSettle();
+ const baseline = process.memoryUsage().heapUsed;
+
+ for (let i = 0; i < 500; i++) await runOneStream(i);
+ await gcSettle();
+ const after = process.memoryUsage().heapUsed;
+
+ const growthMB = (after - baseline) / 1024 / 1024;
+ console.log(`[heap] growth=${growthMB.toFixed(2)}MB after 500 streams`);
+ // Ceiling is 20MB — goal is detecting linear growth (leaks), not noise.
+ // If growth is stably near this boundary, raise to 30MB with justification.
+ assert.ok(growthMB < 20, `heap grew ${growthMB.toFixed(1)}MB after 500 streams (ceiling 20MB)`);
+});
diff --git a/tests/integration/resilience-chaos.test.ts b/tests/integration/resilience-chaos.test.ts
new file mode 100644
index 0000000000..bd16fdb993
--- /dev/null
+++ b/tests/integration/resilience-chaos.test.ts
@@ -0,0 +1,63 @@
+import { test, before, after, afterEach } from "node:test";
+import assert from "node:assert/strict";
+import {
+ getCircuitBreaker,
+ resetAllCircuitBreakers,
+ STATE,
+} from "../../src/shared/utils/circuitBreaker.ts";
+import { checkFallbackError } from "../../open-sse/services/accountFallback.ts";
+import { startFaultyUpstream, type FaultyUpstream } from "../helpers/faultyUpstream.ts";
+
+const RUN = process.env.RUN_CHAOS_INT === "1";
+
+let up: FaultyUpstream;
+before(async () => {
+ up = await startFaultyUpstream({ kind: "ok" });
+});
+after(async () => {
+ await up?.stop();
+});
+afterEach(() => {
+ resetAllCircuitBreakers();
+});
+
+test("breaker OPENs after real timeouts, then refreshes to HALF_OPEN", { skip: !RUN }, async () => {
+ up.setMode({ kind: "timeout" });
+ const breaker = getCircuitBreaker("chaos-timeout", {
+ failureThreshold: 3,
+ resetTimeout: 300,
+ isFailure: () => true,
+ });
+ for (let i = 0; i < 3; i++) {
+ await assert.rejects(() =>
+ breaker.execute(() => fetch(up.url, { signal: AbortSignal.timeout(120) }))
+ );
+ }
+ assert.equal(breaker.canExecute(), false);
+ assert.equal(breaker.getStatus().state, STATE.OPEN);
+ await new Promise((r) => setTimeout(r, 330));
+ assert.equal(breaker.getStatus().state, STATE.HALF_OPEN);
+ assert.equal(breaker.canExecute(), true);
+});
+
+test("breaker trips on a real connection reset", { skip: !RUN }, async () => {
+ up.setMode({ kind: "reset" });
+ const breaker = getCircuitBreaker("chaos-reset", {
+ failureThreshold: 2,
+ resetTimeout: 300,
+ isFailure: () => true,
+ });
+ for (let i = 0; i < 2; i++) {
+ await assert.rejects(() => breaker.execute(() => fetch(up.url)));
+ }
+ assert.equal(breaker.getStatus().state, STATE.OPEN);
+});
+
+test("checkFallbackError classifies a real 503 response as recoverable fallback", { skip: !RUN }, async () => {
+ up.setMode({ kind: "status", code: 503, body: "service unavailable" });
+ const res = await fetch(up.url);
+ const text = await res.text();
+ const decision = checkFallbackError(res.status, text, 0, null, "chaos-provider", res.headers);
+ assert.equal(decision.shouldFallback, true);
+ assert.ok(decision.cooldownMs > 0, "503 should yield a positive cooldown");
+});
diff --git a/tests/load/k6-soak.js b/tests/load/k6-soak.js
new file mode 100644
index 0000000000..3ae72c0d03
--- /dev/null
+++ b/tests/load/k6-soak.js
@@ -0,0 +1,19 @@
+import http from "k6/http";
+import { check } from "k6";
+
+// Soak contra um endpoint leve (sem custo de LLM) para medir liveness/leak do
+// servidor sob carga sustentada. Configurável por env: BASE_URL, SOAK_VUS, SOAK_DURATION.
+export const options = {
+ stages: [{ duration: __ENV.SOAK_DURATION || "3m", target: Number(__ENV.SOAK_VUS || 10) }],
+ thresholds: {
+ http_req_failed: ["rate<0.01"],
+ http_req_duration: ["p(95)<500"],
+ },
+};
+
+const BASE_URL = __ENV.BASE_URL || "http://localhost:20128";
+
+export default function () {
+ const res = http.get(`${BASE_URL}/api/monitoring/health`);
+ check(res, { "status is 200": (r) => r.status === 200 });
+}
diff --git a/tests/unit/faulty-upstream.test.ts b/tests/unit/faulty-upstream.test.ts
new file mode 100644
index 0000000000..729ee86d43
--- /dev/null
+++ b/tests/unit/faulty-upstream.test.ts
@@ -0,0 +1,48 @@
+import { test, after } from "node:test";
+import assert from "node:assert/strict";
+import { startFaultyUpstream } from "../helpers/faultyUpstream.ts";
+
+test("ok mode returns 200 with body", async () => {
+ const up = await startFaultyUpstream({ kind: "ok", body: "hello" });
+ after(() => up.stop());
+ const res = await fetch(up.url);
+ assert.equal(res.status, 200);
+ assert.equal(await res.text(), "hello");
+ await up.stop();
+});
+
+test("status mode returns the given HTTP error code", async () => {
+ const up = await startFaultyUpstream({ kind: "status", code: 503, body: "down" });
+ const res = await fetch(up.url);
+ assert.equal(res.status, 503);
+ assert.equal(await res.text(), "down");
+ await up.stop();
+});
+
+test("latency mode delays the response", async () => {
+ const up = await startFaultyUpstream({ kind: "latency", ms: 120, body: "slow" });
+ const t0 = Date.now();
+ await (await fetch(up.url)).text();
+ assert.ok(Date.now() - t0 >= 100, "should be delayed ~120ms");
+ await up.stop();
+});
+
+test("reset mode destroys the socket (fetch rejects)", async () => {
+ const up = await startFaultyUpstream({ kind: "reset" });
+ await assert.rejects(() => fetch(up.url).then((r) => r.text()));
+ await up.stop();
+});
+
+test("timeout mode never responds (AbortSignal aborts)", async () => {
+ const up = await startFaultyUpstream({ kind: "timeout" });
+ await assert.rejects(() => fetch(up.url, { signal: AbortSignal.timeout(120) }));
+ await up.stop();
+});
+
+test("setMode switches behavior on a running server", async () => {
+ const up = await startFaultyUpstream({ kind: "ok", body: "a" });
+ up.setMode({ kind: "status", code: 429, body: "rl" });
+ const res = await fetch(up.url);
+ assert.equal(res.status, 429);
+ await up.stop();
+});