Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

18.4 KB
Raw Blame
/**
 * How long does `/track` take when the Redis CACHE is unreachable?
 *
 * This harness reproduces that in-repo so a fix can be justified with a
 * before/after on the same box, and re-run by anyone.
 *
 * It never touches the shared Redis server: the API under test is pointed at a
 * local TCP proxy which is then stopped (`refuse`, docker-stop shaped) or
 * frozen (`blackhole`, docker-pause shaped). The operator's stack keeps its own
 * connection to:6379 throughout.
 *
 * Dotenv -e../../.env -- bun e2e/redis-fail-fast.ts
 */

import { spawn } from 'node:child_process';
import net from 'node:net';

const UPSTREAM_HOST = '127.0.0.1';
const UPSTREAM_PORT = 6379;
const PROXY_PORT = Number(process.env.M18_PROXY_PORT ?? 6399);
/** Private Redis database + Kafka topic: this harness must not perturb :6379/0. */
const PROXY_REDIS_DB = 3;
const API_PORT = Number(process.env.M18_API_PORT ?? 3399);
const BOOT_API_PORT = API_PORT + 1;
const KAFKA_TOPIC = 'events-m18003-failfast';

const SECRET_CLIENT_ID = 'e2e1e2e1-0000-4000-8000-000000000002';
const CLIENT_SECRET = 'e2e-saturation-secret';
const CORS_ORIGIN = 'https://e2e.test';
const UA =
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36';

/** How long the client waits before recording "no response". */
const CLIENT_TIMEOUT_MS = Number(process.env.M18_CLIENT_TIMEOUT_MS ?? 30_000);
const BOOT_TIMEOUT_MS = 60_000;
const WARMUP_REQUESTS = 6;
const BASELINE_REQUESTS = Number(process.env.M18_BASELINE_REQUESTS ?? 20);
const OUTAGE_PROBE_REQUESTS = Number(process.env.M18_OUTAGE_PROBES ?? 10);
const BLIP_DURATION_MS = 1000;
const BLIP_CONCURRENCY = 5;
const BLIP_TOTAL_MS = 6000;
/** One device for the whole blip, so a session split shows up as a second id. */
const BLIP_CLIENT_IP = '10.18.3.200';
const BLACKHOLE_PROBES = 5;
const SUSTAINED_OUTAGE_MS = Number(process.env.M18_OUTAGE_MS ?? 20_000);
/**
 * Phases A-C only. The recovery window (C) is the one number that has to be
 * compared across several runs to be worth anything, and re-running the blip,
 * the blackhole and the second boot to get it costs minutes per sample.
 */
const QUICK_RUN = process.env.M18_QUICK === '1';

type ProxyMode = 'pass' | 'refuse' | 'blackhole';

// ── The proxy ───────────────────────────────────────────────────────────────

class RedisFaultProxy {
  private mode: ProxyMode = 'pass';
  private server: net.Server | null = null;
  private readonly sockets = new Set<net.Socket>();

  async start(): Promise<void> {
    await this.listen();
  }

  private listen(): Promise<void> {
    return new Promise((resolve, reject) => {
      const server = net.createServer((client) => this.onConnection(client));
      server.on('error', reject);
      server.listen(PROXY_PORT, UPSTREAM_HOST, () => {
        this.server = server;
        resolve();
      });
    });
  }

  private onConnection(client: net.Socket) {
    this.sockets.add(client);
    client.on('error', () => client.destroy());
    client.on('close', () => this.sockets.delete(client));

    // Frozen dependency: the handshake completes, nothing ever answers.
    if (this.mode === 'blackhole') {
      client.resume();
      return;
    }

    const upstream = net.connect(UPSTREAM_PORT, UPSTREAM_HOST);
    this.sockets.add(upstream);
    upstream.on('error', () => {
      upstream.destroy();
      client.destroy();
    });
    upstream.on('close', () => {
      this.sockets.delete(upstream);
      client.destroy();
    });
    client.on('data', (chunk) => {
      if (this.mode === 'pass') {
        upstream.write(chunk);
      }
    });
    upstream.on('data', (chunk) => {
      if (this.mode === 'pass') {
        client.write(chunk);
      }
    });
    client.on('close', () => upstream.destroy());
  }

  /** docker stop: the port stops answering and open sockets die. */
  async refuse(): Promise<void> {
    this.mode = 'refuse';
    // Sockets first: `server.close()` does not call back until every open
    // connection has gone, so closing before destroying deadlocks.
    this.destroyAll();
    await this.closeServer();
  }

  /** docker pause: sockets stay open, bytes stop moving. */
  async blackhole(): Promise<void> {
    if (!this.server) {
      await this.listen();
    }
    this.mode = 'blackhole';
  }

  async pass(): Promise<void> {
    if (!this.server) {
      await this.listen();
    }
    // Sockets opened while frozen have no upstream; drop them so ioredis
    // rebuilds a real one.
    if (this.mode === 'blackhole') {
      this.destroyAll();
    }
    this.mode = 'pass';
  }

  private destroyAll() {
    for (const socket of this.sockets) {
      socket.destroy();
    }
    this.sockets.clear();
  }

  private closeServer(): Promise<void> {
    const server = this.server;
    this.server = null;
    if (!server) {
      return Promise.resolve();
    }
    return new Promise((resolve) => server.close(() => resolve()));
  }

  async stop(): Promise<void> {
    this.destroyAll();
    await this.closeServer();
  }
}

// ── Probes ──────────────────────────────────────────────────────────────────

const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

type Shape = 'warm-secret' | 'cold-unknown' | 'browser-origin';

let deviceCounter = 0;
let payloadCounter = 0;

function headersFor(shape: Shape): Record<string, string> {
  const base: Record<string, string> = {
    'content-type': 'application/json',
    'user-agent': UA,
    'x-client-ip': `10.18.3.${(deviceCounter++ % 250) + 1}`,
  };
  if (shape === 'warm-secret') {
    return {
      ...base,
      'openpanel-client-id': SECRET_CLIENT_ID,
      'openpanel-client-secret': CLIENT_SECRET,
    };
  }
  if (shape === 'browser-origin') {
    return {
      ...base,
      'openpanel-client-id': SECRET_CLIENT_ID,
      origin: CORS_ORIGIN,
    };
  }
  return {
    ...base,
    'openpanel-client-id': crypto.randomUUID(),
    'openpanel-client-secret': CLIENT_SECRET,
  };
}

interface Probe {
  status: number | null;
  ms: number;
  error?: string;
  /** ms since the process started, so a phase can locate its failure window. */
  sentAt: number;
  /** `/track`'s answer, when it gave one — the blip's real cost is a session split. */
  sessionId?: string;
}

/**
 * A client that gives up after `CLIENT_TIMEOUT_MS`. An explicit controller
 * rather than `AbortSignal.timeout`, which does not interrupt an in-flight
 * Bun fetch here.
 */
async function timedFetch(url: string, init: RequestInit = {}): Promise<Probe> {
  const controller = new AbortController();
  const giveUp = setTimeout(() => controller.abort(), CLIENT_TIMEOUT_MS);
  const startedAt = performance.now();
  try {
    const res = await fetch(url, {
      ...init,
      signal: controller.signal,
      // No keep-alive: a socket left behind by an abandoned request must not
      // put the next probe behind it and inflate its latency.
      headers: {
        ...(init.headers as Record<string, string>),
        connection: 'close',
      },
    });
    const text = await res.text();
    let sessionId: string | undefined;
    try {
      sessionId = (JSON.parse(text) as { sessionId?: string }).sessionId;
    } catch {
      sessionId = undefined;
    }
    return {
      status: res.status,
      ms: performance.now() - startedAt,
      sentAt: startedAt,
      sessionId,
    };
  } catch (error) {
    return {
      status: null,
      ms: performance.now() - startedAt,
      sentAt: startedAt,
      error: (error as Error).name,
    };
  } finally {
    clearTimeout(giveUp);
  }
}

function trackOnce(
  shape: Shape,
  port = API_PORT,
  fixedIp?: string
): Promise<Probe> {
  const headers = headersFor(shape);
  if (fixedIp) {
    headers['x-client-ip'] = fixedIp;
  }
  return timedFetch(`http://127.0.0.1:${port}/track`, {
    method: 'POST',
    headers,
    body: JSON.stringify({
      type: 'track',
      payload: {
        name: 'm18003_failfast',
        // Unique per request: an identical payload from the same ip+origin is
        // collapsed by the dedupe lock into `200 "Duplicate event"`, which
        // would make the browser-shaped phases measure the lock and nothing
        // past it.
        properties: {
          __path: 'https://e2e.test/m18003',
          seq: payloadCounter++,
        },
      },
    }),
  });
}

function probeUrl(path: string, port = API_PORT): Promise<Probe> {
  return timedFetch(`http://127.0.0.1:${port}${path}`);
}

function summarize(label: string, probes: Probe[]): string {
  const statuses = new Map<string, number>();
  for (const p of probes) {
    const key = p.status === null ? `no-answer(${p.error})` : String(p.status);
    statuses.set(key, (statuses.get(key) ?? 0) + 1);
  }
  const sorted = probes.map((p) => p.ms).sort((a, b) => a - b);
  const at = (q: number) =>
    sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))] ?? 0;
  return `${label.padEnd(34)} n=${probes.length} ${[...statuses]
    .map(([k, v]) => `${k}×${v}`)
    .join(' ')
    .padEnd(
      22
    )} p50=${at(0.5).toFixed(1)}ms p95=${at(0.95).toFixed(1)}ms max=${(sorted.at(-1) ?? 0).toFixed(1)}ms`;
}

// ── The API under test ──────────────────────────────────────────────────────

function spawnApi(port: number) {
  const child = spawn('bun', ['src/main.ts'], {
    cwd: new URL('..', import.meta.url).pathname,
    env: {
      ...process.env,
      ROLE: 'api',
      API_PORT: String(port),
      REDIS_URL: `redis://${UPSTREAM_HOST}:${PROXY_PORT}/${PROXY_REDIS_DB}`,
      KAFKA_EVENTS_TOPIC: KAFKA_TOPIC,
      LOG_LEVEL: process.env.M18_LOG_LEVEL ?? 'warn',
    },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  const lines: string[] = [];
  child.stdout?.on('data', (d) => lines.push(String(d)));
  child.stderr?.on('data', (d) => lines.push(String(d)));
  return { child, lines };
}

async function waitForLive(port: number, timeoutMs = BOOT_TIMEOUT_MS) {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    const res = await fetch(`http://127.0.0.1:${port}/healthz/live`).catch(
      () => null
    );
    if (res?.ok) {
      return true;
    }
    await sleep(200);
  }
  return false;
}

async function main() {
  const proxy = new RedisFaultProxy();
  await proxy.start();
  console.log(
    `proxy 127.0.0.1:${PROXY_PORT} → ${UPSTREAM_HOST}:${UPSTREAM_PORT} (db ${PROXY_REDIS_DB}), api :${API_PORT}, topic ${KAFKA_TOPIC}`
  );

  const { child, lines } = spawnApi(API_PORT);
  const cleanup = async () => {
    child.kill('SIGKILL');
    await proxy.stop();
  };

  try {
    if (!(await waitForLive(API_PORT))) {
      console.log(lines.join(''));
      throw new Error('api did not become live');
    }

    // ── warm-up: the L1 caches the criterion turns on ──────────────────────
    for (let i = 0; i < WARMUP_REQUESTS; i++) {
      await trackOnce('warm-secret');
      await trackOnce('browser-origin');
    }

    console.log('\n── A. healthy baseline ──');
    const baseline: Probe[] = [];
    for (let i = 0; i < BASELINE_REQUESTS; i++) {
      baseline.push(await trackOnce('warm-secret'));
    }
    console.log(summarize('warm-secret /track', baseline));
    const baselineBrowser: Probe[] = [];
    for (let i = 0; i < BASELINE_REQUESTS; i++) {
      baselineBrowser.push(await trackOnce('browser-origin'));
    }
    console.log(summarize('browser-origin /track', baselineBrowser));

    // ── B. sustained outage ───────────────────────────────────────────────
    console.log(
      `\n── B. sustained outage (${SUSTAINED_OUTAGE_MS / 1000}s, refuse) ──`
    );
    const outageStart = Date.now();
    await proxy.refuse();
    await sleep(500);

    const warmDuringOutage: Probe[] = [];
    for (let i = 0; i < OUTAGE_PROBE_REQUESTS; i++) {
      warmDuringOutage.push(await trackOnce('warm-secret'));
    }
    console.log(summarize('warm-secret /track', warmDuringOutage));

    const coldDuringOutage: Probe[] = [];
    for (let i = 0; i < OUTAGE_PROBE_REQUESTS; i++) {
      coldDuringOutage.push(await trackOnce('cold-unknown'));
    }
    console.log(summarize('cold-unknown /track', coldDuringOutage));

    const browserDuringOutage: Probe[] = [];
    for (let i = 0; i < OUTAGE_PROBE_REQUESTS; i++) {
      browserDuringOutage.push(await trackOnce('browser-origin'));
    }
    console.log(summarize('browser-origin /track', browserDuringOutage));

    for (const path of ['/healthz/live', '/healthz/ready', '/healthcheck']) {
      const p = await probeUrl(path);
      console.log(
        `${path.padEnd(34)} status=${p.status ?? `no-answer(${p.error})`} ${p.ms.toFixed(1)}ms`
      );
    }

    const remaining = SUSTAINED_OUTAGE_MS - (Date.now() - outageStart);
    if (remaining > 0) {
      await sleep(remaining);
    }

    // ── C. recovery ───────────────────────────────────────────────────────
    console.log('\n── C. recovery ──');
    await proxy.pass();
    const recoveredAt = Date.now();
    // Browser-shaped on purpose: it is the shape that NEEDS Redis (the dedupe
    // lock), so it measures when the client is actually usable again. The
    // warm-secret shape answers 200 by degrading and would report 0 ms.
    let recovery: Probe = { status: null, ms: 0, sentAt: 0 };
    while (Date.now() - recoveredAt < 15_000) {
      recovery = await trackOnce('browser-origin');
      if (recovery.status === 200) {
        break;
      }
    }
    console.log(
      `first browser-origin 200 after restore: ${Date.now() - recoveredAt}ms (status ${recovery.status})`
    );
    const afterRestore: Probe[] = [];
    for (let i = 0; i < BASELINE_REQUESTS; i++) {
      afterRestore.push(await trackOnce('browser-origin'));
    }
    console.log(summarize('browser-origin /track', afterRestore));

    if (QUICK_RUN) {
      return;
    }

    // ── D. the blip ───────────────────────────────────────────────────────
    // The cost side of fail-fast: a reconnect blip the offline queue used to
    // absorb. Both shapes, because they degrade differently — the warm-secret
    // one only loses its session lookup, the browser one loses its dedupe lock.
    console.log(`\n── D. ${BLIP_DURATION_MS}ms blip under load ──`);
    for (const shape of ['warm-secret', 'browser-origin'] as const) {
      // A control pass first: the same load, same duration, no fault. Without
      // it a session id that changed for some unrelated reason would be read
      // as a cost of the blip.
      for (const withFault of [false, true]) {
        const probes: Probe[] = [];
        const runEnd = Date.now() + BLIP_TOTAL_MS;
        const workers = Array.from({ length: BLIP_CONCURRENCY }, async () => {
          while (Date.now() < runEnd) {
            probes.push(await trackOnce(shape, API_PORT, BLIP_CLIENT_IP));
          }
        });
        await sleep(2000);
        if (withFault) {
          await proxy.refuse();
          await sleep(BLIP_DURATION_MS);
          await proxy.pass();
        }
        await Promise.all(workers);

        const label = withFault ? 'blip' : 'control';
        console.log(summarize(`${shape}, ${label}`, probes));
        const failures = probes.filter((p) => p.status !== 200);
        const failureWindowMs =
          failures.length === 0
            ? 0
            : (failures.at(-1)?.sentAt ?? 0) +
              (failures.at(-1)?.ms ?? 0) -
              (failures[0]?.sentAt ?? 0);
        const sessionIds = new Set(
          probes.filter((p) => p.sessionId).map((p) => p.sessionId)
        );
        console.log(
          `${''.padEnd(34)} failures=${failures.length} over ${failureWindowMs.toFixed(0)}ms of wall clock, distinct sessionIds=${sessionIds.size}`
        );
        await sleep(1000);
      }
    }

    // ── E. blackhole ──────────────────────────────────────────────────────
    console.log('\n── E. blackhole (frozen dependency) ──');
    await sleep(1500);
    await proxy.blackhole();
    await sleep(300);
    const frozen: Probe[] = [];
    for (let i = 0; i < BLACKHOLE_PROBES; i++) {
      frozen.push(await trackOnce('warm-secret'));
    }
    console.log(summarize('warm-secret /track', frozen));
    const frozenBrowser: Probe[] = [];
    for (let i = 0; i < BLACKHOLE_PROBES; i++) {
      frozenBrowser.push(await trackOnce('browser-origin'));
    }
    console.log(summarize('browser-origin /track', frozenBrowser));
    await proxy.pass();
    await sleep(1500);

    // ── F. boot with Redis already down ───────────────────────────────────
    console.log('\n── F. boot with Redis down ──');
    await proxy.refuse();
    const boot = spawnApi(BOOT_API_PORT);
    const bootedAt = Date.now();
    const live = await waitForLive(BOOT_API_PORT, 30_000);
    console.log(
      `booted with redis unreachable: ${live} (${Date.now() - bootedAt}ms)`
    );
    if (live) {
      for (const path of ['/healthz/live', '/healthz/ready', '/healthcheck']) {
        const p = await probeUrl(path, BOOT_API_PORT);
        console.log(
          `  ${path.padEnd(32)} status=${p.status ?? `no-answer(${p.error})`} ${p.ms.toFixed(1)}ms`
        );
      }
      console.log(
        `  ${summarize('cold /track (never had redis)', [await trackOnce('warm-secret', BOOT_API_PORT)])}`
      );
      await proxy.pass();
      await sleep(2000);
      const after = await trackOnce('warm-secret', BOOT_API_PORT);
      console.log(
        `  /track once redis returns: status=${after.status} ${after.ms.toFixed(1)}ms`
      );
      for (const path of ['/healthz/ready', '/healthcheck']) {
        const p = await probeUrl(path, BOOT_API_PORT);
        console.log(
          `  ${path.padEnd(32)} status=${p.status ?? `no-answer(${p.error})`} ${p.ms.toFixed(1)}ms`
        );
      }
    } else {
      console.log(boot.lines.join('').slice(-4000));
    }
    boot.child.kill('SIGKILL');
  } finally {
    await cleanup();
  }
}

await main();
process.exit(0);