Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
Fase 8 · Bloco C — resiliência runtime (chaos + heap-growth + k6 soak) (#3854)
Diego Rodrigues de Sa e Souzad3146a17515f
diff --git a/.github/workflows/nightly-resilience.yml b/.github/workflows/nightly-resilience.ymlnew file mode 100644index 0000000000..9af222d94d--- /dev/null+++ b/.github/workflows/nightly-resilience.yml@@ -0,0 +1,70 @@+name: Nightly Resilience+on:+ schedule:+ - cron: "41 4 * * *"+ workflow_dispatch:++permissions:+ contents: read++jobs:+ heap:+ name: Heap-growth gate+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v6+ - uses: actions/setup-node@v6+ with:+ node-version: "24"+ cache: npm+ - run: npm ci+ - run: npm run test:heap++ chaos:+ name: Resilience chaos (fault injection)+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v6+ - uses: actions/setup-node@v6+ with:+ node-version: "24"+ cache: npm+ - run: npm ci+ - run: npm run test:chaos++ k6-soak:+ name: k6 load/soak+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v6+ - uses: actions/setup-node@v6+ with:+ node-version: "24"+ cache: npm+ - run: npm ci+ - name: Build CLI bundle+ env:+ JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation+ run: npm run build:cli+ - name: Start OmniRoute (background)+ env:+ JWT_SECRET: ci-build-secret-with-sufficient-length-for-validation+ PORT: "20128"+ run: |+ node dist/server.js > server.log 2>&1 &+ echo $! > server.pid+ for i in $(seq 1 30); do+ if curl -sf http://localhost:20128/api/monitoring/health >/dev/null; then echo "server up"; break; fi+ sleep 2+ done+ - name: Install k6+ uses: grafana/setup-k6-action@v1+ - name: Run k6 soak+ run: k6 run tests/load/k6-soak.js+ env:+ BASE_URL: http://localhost:20128+ SOAK_DURATION: "3m"+ SOAK_VUS: "10"+ - name: Stop server+ if: always()+ run: kill "$(cat server.pid)" || truediff --git a/docs/architecture/RESILIENCE_GUIDE.md b/docs/architecture/RESILIENCE_GUIDE.mdindex cd9f2c71df..92e2ba0779 100644--- a/docs/architecture/RESILIENCE_GUIDE.md+++ b/docs/architecture/RESILIENCE_GUIDE.md@@ -144,6 +144,22 @@ Provider-specific stealth (JA3/JA4, CCH, obfuscation) is separately documented---+## Resilience testing (Fase 8 · Bloco C)++Além dos unit tests da lógica de resiliência, três testes exercitam o runtime sob+estresse/falha real (todos integração/nightly — nenhum bloqueia PR):++| Teste | O quê | Rodar |+|---|---|---|+| Chaos | Fake-upstream node injeta latência/reset/timeout/503 reais; valida que o circuit breaker abre/recupera e `checkFallbackError` classifica 503 como fallback recuperável. | `RUN_CHAOS_INT=1 npm run test:chaos` |+| Heap-growth | ~500 streams por `createSSEStream` sob `--expose-gc`; falha se o heap crescer além do teto (guarda OOM #3069). | `npm run test:heap` |+| k6 soak | Carga sustentada contra `/api/monitoring/health`; thresholds p95/erro. | `k6 run tests/load/k6-soak.js` (nightly) |++Orquestrados por `.github/workflows/nightly-resilience.yml` (cron + dispatch). No+`test:integration` default, chaos e heap se auto-skipam (sem `RUN_CHAOS_INT`/`--expose-gc`).++---+## See Also- [Architecture Guide](./ARCHITECTURE.md) — System architecture and internalsdiff --git a/package.json b/package.jsonindex 9df6c9ef93..784f2d00b6 100644--- a/package.json+++ b/package.json@@ -156,6 +156,8 @@"backfill-aggregation": "node --import tsx src/scripts/backfillAggregation.ts","env:sync": "node scripts/dev/sync-env.mjs","test:integration": "cross-env DISABLE_SQLITE_AUTO_BACKUP=true node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/*.test.ts",+ "test:heap": "node --expose-gc --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit tests/integration/heap-growth.test.ts",+ "test:chaos": "cross-env RUN_CHAOS_INT=1 node --import tsx --import ./open-sse/utils/setupPolyfill.ts --test --test-force-exit --test-concurrency=1 tests/integration/resilience-chaos.test.ts","test:e2e": "node scripts/dev/run-playwright-tests.mjs test tests/e2e/*.spec.ts","test:protocols:e2e": "node scripts/dev/run-protocol-clients-tests.mjs","test:vitest": "vitest run --config vitest.mcp.config.ts",diff --git a/tests/helpers/faultyUpstream.ts b/tests/helpers/faultyUpstream.tsnew file mode 100644index 0000000000..3eee4f1c19--- /dev/null+++ b/tests/helpers/faultyUpstream.ts@@ -0,0 +1,72 @@+import http from "node:http";+import type { AddressInfo } from "node:net";++export type FaultMode =+ | { kind: "ok"; body?: string }+ | { kind: "status"; code: number; body?: string }+ | { kind: "latency"; ms: number; body?: string }+ | { kind: "reset" }+ | { kind: "timeout" }+ | { kind: "slowDrip"; chunkMs: number; body?: string };++export interface FaultyUpstream {+ readonly url: string;+ setMode(mode: FaultMode): void;+ stop(): Promise<void>;+}++export async function startFaultyUpstream(initial: FaultMode = { kind: "ok" }): Promise<FaultyUpstream> {+ let mode: FaultMode = initial;++ const server = http.createServer((req, res) => {+ const m = mode;+ if (m.kind === "reset") {+ req.socket.destroy();+ return;+ }+ if (m.kind === "timeout") {+ return; // never respond; caller relies on AbortSignal/timeout+ }+ if (m.kind === "latency") {+ setTimeout(() => {+ res.writeHead(200, { "content-type": "text/plain" });+ res.end(m.body ?? "ok");+ }, m.ms);+ return;+ }+ if (m.kind === "slowDrip") {+ const body = m.body ?? "drip-drip-drip";+ res.writeHead(200, { "content-type": "text/plain" });+ let i = 0;+ const timer = setInterval(() => {+ if (i >= body.length) {+ clearInterval(timer);+ res.end();+ return;+ }+ res.write(body[i++]);+ }, m.chunkMs);+ req.on("close", () => clearInterval(timer));+ return;+ }+ const code = m.kind === "status" ? m.code : 200;+ res.writeHead(code, { "content-type": "text/plain" });+ res.end(m.body ?? (m.kind === "status" ? "error" : "ok"));+ });++ await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));+ const { port } = server.address() as AddressInfo;++ return {+ url: `http://127.0.0.1:${port}/`,+ setMode(next: FaultMode) {+ mode = next;+ },+ stop() {+ return new Promise<void>((resolve) => {+ server.closeAllConnections?.();+ server.close(() => resolve());+ });+ },+ };+}diff --git a/tests/integration/heap-growth.test.ts b/tests/integration/heap-growth.test.tsnew file mode 100644index 0000000000..d541a9e9a2--- /dev/null+++ b/tests/integration/heap-growth.test.ts@@ -0,0 +1,78 @@+import { test } from "node:test";+import assert from "node:assert/strict";+import fs from "node:fs";+import os from "node:os";+import path from "node:path";++// DB setup required by createSSEStream (via trackPendingRequest / appendRequestLog)+const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-heap-growth-"));+process.env.DATA_DIR = TEST_DATA_DIR;+const core = await import("../../src/lib/db/core.ts");++const { createSSEStream } = await import("../../open-sse/utils/stream.ts");++test.after(() => {+ core.resetDbInstance();+ if (fs.existsSync(TEST_DATA_DIR)) {+ fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });+ }+});++const HAS_GC = typeof (globalThis as { gc?: () => void }).gc === "function";+const enc = new TextEncoder();++function sseChunks(n: number): Uint8Array[] {+ const parts: Uint8Array[] = [];+ for (let i = 0; i < n; i++) {+ parts.push(+ enc.encode(+ `data: {"id":"chatcmpl","object":"chat.completion.chunk","created":1,"model":"gpt-4o","choices":[{"delta":{"content":"x"},"index":0}]}\n\n`+ )+ );+ }+ parts.push(enc.encode(`data: [DONE]\n\n`));+ return parts;+}++async function runOneStream(i: number): Promise<void> {+ const ts = createSSEStream({+ mode: "passthrough",+ clientResponseFormat: "openai",+ connectionId: `heap-${i}`,+ }) as TransformStream<Uint8Array, Uint8Array>;+ const writer = ts.writable.getWriter();+ const reader = ts.readable.getReader();+ const chunks = sseChunks(10);+ const pump = (async () => {+ for (const c of chunks) await writer.write(c);+ await writer.close();+ })();+ for (;;) {+ const { done } = await reader.read();+ if (done) break;+ }+ await pump;+}++async function gcSettle(): Promise<void> {+ const gc = (globalThis as { gc?: () => void }).gc!;+ gc();+ await new Promise((r) => setTimeout(r, 50));+ gc();+}++test("SSE pipeline does not leak heap across 500 streams", { skip: !HAS_GC }, async () => {+ for (let i = 0; i < 50; i++) await runOneStream(i); // warmup+ await gcSettle();+ const baseline = process.memoryUsage().heapUsed;++ for (let i = 0; i < 500; i++) await runOneStream(i);+ await gcSettle();+ const after = process.memoryUsage().heapUsed;++ const growthMB = (after - baseline) / 1024 / 1024;+ console.log(`[heap] growth=${growthMB.toFixed(2)}MB after 500 streams`);+ // Ceiling is 20MB — goal is detecting linear growth (leaks), not noise.+ // If growth is stably near this boundary, raise to 30MB with justification.+ assert.ok(growthMB < 20, `heap grew ${growthMB.toFixed(1)}MB after 500 streams (ceiling 20MB)`);+});diff --git a/tests/integration/resilience-chaos.test.ts b/tests/integration/resilience-chaos.test.tsnew file mode 100644index 0000000000..bd16fdb993--- /dev/null+++ b/tests/integration/resilience-chaos.test.ts@@ -0,0 +1,63 @@+import { test, before, after, afterEach } from "node:test";+import assert from "node:assert/strict";+import {+ getCircuitBreaker,+ resetAllCircuitBreakers,+ STATE,+} from "../../src/shared/utils/circuitBreaker.ts";+import { checkFallbackError } from "../../open-sse/services/accountFallback.ts";+import { startFaultyUpstream, type FaultyUpstream } from "../helpers/faultyUpstream.ts";++const RUN = process.env.RUN_CHAOS_INT === "1";++let up: FaultyUpstream;+before(async () => {+ up = await startFaultyUpstream({ kind: "ok" });+});+after(async () => {+ await up?.stop();+});+afterEach(() => {+ resetAllCircuitBreakers();+});++test("breaker OPENs after real timeouts, then refreshes to HALF_OPEN", { skip: !RUN }, async () => {+ up.setMode({ kind: "timeout" });+ const breaker = getCircuitBreaker("chaos-timeout", {+ failureThreshold: 3,+ resetTimeout: 300,+ isFailure: () => true,+ });+ for (let i = 0; i < 3; i++) {+ await assert.rejects(() =>+ breaker.execute(() => fetch(up.url, { signal: AbortSignal.timeout(120) }))+ );+ }+ assert.equal(breaker.canExecute(), false);+ assert.equal(breaker.getStatus().state, STATE.OPEN);+ await new Promise((r) => setTimeout(r, 330));+ assert.equal(breaker.getStatus().state, STATE.HALF_OPEN);+ assert.equal(breaker.canExecute(), true);+});++test("breaker trips on a real connection reset", { skip: !RUN }, async () => {+ up.setMode({ kind: "reset" });+ const breaker = getCircuitBreaker("chaos-reset", {+ failureThreshold: 2,+ resetTimeout: 300,+ isFailure: () => true,+ });+ for (let i = 0; i < 2; i++) {+ await assert.rejects(() => breaker.execute(() => fetch(up.url)));+ }+ assert.equal(breaker.getStatus().state, STATE.OPEN);+});++test("checkFallbackError classifies a real 503 response as recoverable fallback", { skip: !RUN }, async () => {+ up.setMode({ kind: "status", code: 503, body: "service unavailable" });+ const res = await fetch(up.url);+ const text = await res.text();+ const decision = checkFallbackError(res.status, text, 0, null, "chaos-provider", res.headers);+ assert.equal(decision.shouldFallback, true);+ assert.ok(decision.cooldownMs > 0, "503 should yield a positive cooldown");+});diff --git a/tests/load/k6-soak.js b/tests/load/k6-soak.jsnew file mode 100644index 0000000000..3ae72c0d03--- /dev/null+++ b/tests/load/k6-soak.js@@ -0,0 +1,19 @@+import http from "k6/http";+import { check } from "k6";++// Soak contra um endpoint leve (sem custo de LLM) para medir liveness/leak do+// servidor sob carga sustentada. Configurável por env: BASE_URL, SOAK_VUS, SOAK_DURATION.+export const options = {+ stages: [{ duration: __ENV.SOAK_DURATION || "3m", target: Number(__ENV.SOAK_VUS || 10) }],+ thresholds: {+ http_req_failed: ["rate<0.01"],+ http_req_duration: ["p(95)<500"],+ },+};++const BASE_URL = __ENV.BASE_URL || "http://localhost:20128";++export default function () {+ const res = http.get(`${BASE_URL}/api/monitoring/health`);+ check(res, { "status is 200": (r) => r.status === 200 });+}diff --git a/tests/unit/faulty-upstream.test.ts b/tests/unit/faulty-upstream.test.tsnew file mode 100644index 0000000000..729ee86d43--- /dev/null+++ b/tests/unit/faulty-upstream.test.ts@@ -0,0 +1,48 @@+import { test, after } from "node:test";+import assert from "node:assert/strict";+import { startFaultyUpstream } from "../helpers/faultyUpstream.ts";++test("ok mode returns 200 with body", async () => {+ const up = await startFaultyUpstream({ kind: "ok", body: "hello" });+ after(() => up.stop());+ const res = await fetch(up.url);+ assert.equal(res.status, 200);+ assert.equal(await res.text(), "hello");+ await up.stop();+});++test("status mode returns the given HTTP error code", async () => {+ const up = await startFaultyUpstream({ kind: "status", code: 503, body: "down" });+ const res = await fetch(up.url);+ assert.equal(res.status, 503);+ assert.equal(await res.text(), "down");+ await up.stop();+});++test("latency mode delays the response", async () => {+ const up = await startFaultyUpstream({ kind: "latency", ms: 120, body: "slow" });+ const t0 = Date.now();+ await (await fetch(up.url)).text();+ assert.ok(Date.now() - t0 >= 100, "should be delayed ~120ms");+ await up.stop();+});++test("reset mode destroys the socket (fetch rejects)", async () => {+ const up = await startFaultyUpstream({ kind: "reset" });+ await assert.rejects(() => fetch(up.url).then((r) => r.text()));+ await up.stop();+});++test("timeout mode never responds (AbortSignal aborts)", async () => {+ const up = await startFaultyUpstream({ kind: "timeout" });+ await assert.rejects(() => fetch(up.url, { signal: AbortSignal.timeout(120) }));+ await up.stop();+});++test("setMode switches behavior on a running server", async () => {+ const up = await startFaultyUpstream({ kind: "ok", body: "a" });+ up.setMode({ kind: "status", code: 429, body: "rl" });+ const res = await fetch(up.url);+ assert.equal(res.status, 429);+ await up.stop();+});