Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

15.9 KB
Raw Blame
/**
 * Ingest throughput harness: N devices × M events through `POST /track`
 * (API → Kafka → worker → Redis buffers → ClickHouse), timed from the first
 * request until every business event is in ClickHouse and every buffer is
 * empty. The run that produced the "100,000 events in 27 s" figure in #560
 * is `--devices 5000 --per-device 20 --concurrency 100`.
 *
 * Besides the wall-clock marks it reports what the run cost Redis: the delta
 * of `INFO commandstats` and `INFO cpu` over the run, so Redis CPU can be
 * attributed to commands and compared across machines as CPU-seconds per
 * event rather than as a percentage of a core.
 *
 * Refuses to start unless the pipeline is idle (no Kafka lag, every partition
 * committed, all three buffers empty), so the marks measure this run alone.
 *
 * Usage (from apps/api, against a running api + worker):
 *   dotenv -e ../../.env -- bun e2e/ingest-load.ts [options]
 *   --devices 5000 --per-device 20 --concurrency 100
 *   --properties 2        custom properties per event (one of them an array)
 *   --identified          send a profileId per device (identified traffic)
 *   --project acme-shop   project id, for the ClickHouse counts
 *   --client / --secret   write client; defaults to the seeded project's
 *   --label run           prefix for the output files in --out
 *   --out <dir>           default: a fresh private directory under $TMPDIR
 * Env: API_URL (or E2E_API_URL), REDIS_URL, CLICKHOUSE_URL, and the Kafka
 * topic and group from the api's own config loader.
 */

import {
  appendFileSync,
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { parseArgs } from 'node:util';
import { createKafkaAdmin, sampleConsumerGroupLag } from '@openpanel/core';
import { getRedisCache } from '@openpanel/redis';
import { loadConfig } from '../src/config/env';

const SAMPLE_INTERVAL_MS = 1000;
const DEFAULT_TIMEOUT_S = 1800;
const ARRAY_PROPERTY_LENGTH = 3;
const OCTET_RANGE = 256;
const TOP_COMMANDS = 14;
/** A reset keep-alive socket is retried like an SDK would; a 4xx/5xx is not. */
const SEND_ATTEMPTS = 3;
const BUFFER_KEYS = {
  event: 'event_buffer:queue',
  session: 'session-buffer',
  profile: 'profile-buffer',
} as const;
const USER_AGENT =
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/128.0 Safari/537.36';

const { values } = parseArgs({
  options: {
    devices: { type: 'string', default: '5000' },
    'per-device': { type: 'string', default: '20' },
    concurrency: { type: 'string', default: '100' },
    properties: { type: 'string', default: '2' },
    identified: { type: 'boolean', default: false },
    project: { type: 'string', default: 'acme-shop' },
    client: { type: 'string' },
    secret: { type: 'string' },
    label: { type: 'string', default: 'run' },
    out: { type: 'string' },
    timeout: { type: 'string', default: String(DEFAULT_TIMEOUT_S) },
  },
});

function integerOption(name: string, raw: string, min: number): number {
  const value = Number(raw);
  if (!Number.isInteger(value) || value < min) {
    throw new Error(`--${name} must be an integer >= ${min}, got "${raw}"`);
  }
  return value;
}

const devices = integerOption('devices', values.devices, 1);
const perDevice = integerOption('per-device', values['per-device'], 1);
const concurrency = integerOption('concurrency', values.concurrency, 1);
const propertyCount = integerOption('properties', values.properties, 0);
const timeoutS = integerOption('timeout', values.timeout, 1);
const total = devices * perDevice;
const projectId = values.project;
const runId = `${values.label}_${Date.now().toString(36)}`;
// A fixed name under /tmp could be pre-created by another local user, who would
// then decide where the samples are appended.
const outDir =
  values.out ?? mkdtempSync(join(tmpdir(), 'openpanel-ingest-load-'));
mkdirSync(outDir, { recursive: true });
const outPrefix = resolve(outDir, runId);
console.log(`Writing ${outPrefix}.samples and ${outPrefix}.json`);

const apiUrl = process.env.E2E_API_URL ?? process.env.API_URL;
if (!apiUrl) {
  throw new Error('Set API_URL (or E2E_API_URL) to the api you are loading.');
}
const clickhouseUrl = process.env.CLICKHOUSE_URL;
if (!clickhouseUrl) {
  throw new Error('Set CLICKHOUSE_URL.');
}

interface SeedManifest {
  projects: { id: string; clientId: string; clientSecret: string }[];
}

function resolveClient(): { id: string; secret: string } {
  if (values.client && values.secret) {
    return { id: values.client, secret: values.secret };
  }
  const manifestPath = resolve(import.meta.dir, '../../../.seed.json');
  if (!existsSync(manifestPath)) {
    throw new Error(
      'No --client/--secret and no .seed.json: run `bun run seed` or pass a write client.'
    );
  }
  const manifest = JSON.parse(
    readFileSync(manifestPath, 'utf8')
  ) as SeedManifest;
  const project = manifest.projects.find((p) => p.id === projectId);
  if (!project) {
    throw new Error(`Project ${projectId} is not in .seed.json.`);
  }
  return { id: project.clientId, secret: project.clientSecret };
}

const client = resolveClient();
const kafkaConfig = loadConfig().core;
const admin = createKafkaAdmin(kafkaConfig);
const redis = getRedisCache();

const log = (line: string) => {
  console.log(line);
  appendFileSync(`${outPrefix}.samples`, `${line}\n`);
};

async function clickhouse(query: string): Promise<string> {
  const url = new URL(clickhouseUrl as string);
  const database = url.pathname.slice(1);
  url.pathname = '/';
  if (database) {
    url.searchParams.set('database', database);
  }
  const response = await fetch(url, { method: 'POST', body: query });
  const text = await response.text();
  if (!response.ok) {
    throw new Error(text);
  }
  return text.trim();
}

const runFilter = `project_id = '${projectId}' AND properties['run'] = '${runId}'`;
const businessFilter = `${runFilter} AND name NOT IN ('session_start', 'session_end')`;

const countBusinessEvents = async () =>
  Number(
    await clickhouse(`SELECT count() FROM events WHERE ${businessFilter}`)
  );

async function kafkaState() {
  const [lag, groups] = await Promise.all([
    sampleConsumerGroupLag(
      admin,
      kafkaConfig.kafka.eventsTopic,
      kafkaConfig.kafka.consumerGroup
    ),
    admin.describeGroups([kafkaConfig.kafka.consumerGroup]),
  ]);
  const group = groups.groups[0];
  return {
    lag: lag.totalLag,
    state: group?.state ?? 'Unknown',
    members: group?.members.length ?? 0,
  };
}

async function bufferSizes() {
  const [event, session, profile] = await Promise.all([
    redis.llen(BUFFER_KEYS.event),
    redis.llen(BUFFER_KEYS.session),
    redis.llen(BUFFER_KEYS.profile),
  ]);
  return { event, session, profile };
}

type CommandStats = Map<string, { calls: number; usec: number }>;

async function redisCommandStats(): Promise<CommandStats> {
  const stats: CommandStats = new Map();
  const info = await redis.info('commandstats');
  for (const line of info.split('\n')) {
    const match = line.match(/^cmdstat_([^:]+):calls=(\d+),usec=(\d+)/);
    if (match) {
      stats.set(match[1] as string, {
        calls: Number(match[2]),
        usec: Number(match[3]),
      });
    }
  }
  return stats;
}

/** Main thread and the snapshot fork separately: RDB saves are the fork. */
async function redisCpuSeconds(): Promise<{ main: number; children: number }> {
  const info = await redis.info('cpu');
  const read = (key: string) =>
    Number(info.match(new RegExp(`${key}:([\\d.]+)`))?.[1] ?? 0);
  return {
    main: read('used_cpu_sys') + read('used_cpu_user'),
    children: read('used_cpu_sys_children') + read('used_cpu_user_children'),
  };
}

function reportRedisCost(before: CommandStats, after: CommandStats) {
  const rows: { command: string; calls: number; usec: number }[] = [];
  for (const [command, stat] of after) {
    const previous = before.get(command) ?? { calls: 0, usec: 0 };
    const calls = stat.calls - previous.calls;
    const usec = stat.usec - previous.usec;
    if (calls > 0) {
      rows.push({ command, calls, usec });
    }
  }
  rows.sort((a, b) => b.usec - a.usec);
  const totalUsec = rows.reduce((sum, row) => sum + row.usec, 0);
  log('Redis commands over the run (server-wide), by time spent:');
  log(
    '  command            calls   calls/event   usec/call   share of Redis time'
  );
  for (const row of rows.slice(0, TOP_COMMANDS)) {
    log(
      `  ${row.command.padEnd(16)} ${String(row.calls).padStart(8)} ${(row.calls / total).toFixed(2).padStart(13)} ${(row.usec / row.calls).toFixed(1).padStart(11)} ${((100 * row.usec) / totalUsec).toFixed(1).padStart(10)}%`
    );
  }
  log(
    `  total command time ${(totalUsec / 1e6).toFixed(1)} s = ${((1000 * totalUsec) / 1e6 / total).toFixed(3)} ms per business event`
  );
}

/** A distinct public-looking IP per device, so each gets its own device id. */
function ipFor(device: number) {
  const octets = [
    Math.floor(device / OCTET_RANGE ** 2) % OCTET_RANGE,
    Math.floor(device / OCTET_RANGE) % OCTET_RANGE,
    device % OCTET_RANGE,
  ];
  return `81.${octets.join('.')}`;
}

function customProperties(device: number, step: number) {
  const properties: Record<string, unknown> = {};
  for (let i = 1; i <= propertyCount; i++) {
    properties[`prop_${i}`] =
      i === propertyCount
        ? Array.from(
            { length: ARRAY_PROPERTY_LENGTH },
            (_, j) => `tag_${(step + j) % 7}`
          )
        : `v${(device * 31 + step * i) % 1000}`;
  }
  return properties;
}

async function assertIdle() {
  const [kafka, buffers] = await Promise.all([kafkaState(), bufferSizes()]);
  log(
    `preflight run=${runId} kafka=${JSON.stringify(kafka)} buffers=${JSON.stringify(buffers)}`
  );
  const idle =
    kafka.state === 'Stable' &&
    kafka.members > 0 &&
    kafka.lag === 0 &&
    buffers.event === 0 &&
    buffers.session === 0 &&
    buffers.profile === 0;
  if (!idle) {
    log('PREFLIGHT FAILED: pipeline is not idle, or no worker is consuming');
    process.exit(2);
  }
}

await assertIdle();

// Round-robin: event k of every device, then event k+1, so every session is
// open for the whole run and the worker sees the realistic interleaving.
const jobs: { device: number; step: number }[] = [];
for (let step = 0; step < perDevice; step++) {
  for (let device = 0; device < devices; device++) {
    jobs.push({ device, step });
  }
}

let accepted = 0;
let failed = 0;
let retried = 0;
let nextJob = 0;

async function send(job: { device: number; step: number }): Promise<Response> {
  const body = {
    type: 'track',
    payload: {
      name: job.step % 2 === 0 ? 'screen_view' : `load_${values.label}`,
      properties: {
        __path: `https://shop.example/p/${job.step}`,
        run: runId,
        ...customProperties(job.device, job.step),
      },
      ...(values.identified ? { profileId: `user_${job.device}` } : {}),
    },
  };
  let lastError: unknown;
  for (let attempt = 1; attempt <= SEND_ATTEMPTS; attempt++) {
    try {
      return await fetch(`${apiUrl}/track`, {
        method: 'POST',
        headers: {
          'content-type': 'application/json',
          'openpanel-client-id': client.id,
          'openpanel-client-secret': client.secret,
          // The run id makes every run's device ids its own, so two runs never
          // extend the same session.
          'user-agent': `${USER_AGENT} run/${runId}`,
          'x-client-ip': ipFor(job.device + 1),
        },
        body: JSON.stringify(body),
      });
    } catch (error) {
      lastError = error;
      retried++;
      if (retried <= 5) {
        log(`retrying after ${(error as Error).message}`);
      }
    }
  }
  throw lastError;
}

async function sender() {
  while (nextJob < jobs.length) {
    const job = jobs[nextJob++] as { device: number; step: number };
    const response = await send(job);
    await response.arrayBuffer();
    if (response.ok) {
      accepted++;
    } else {
      failed++;
      if (failed <= 5) {
        log(`HTTP ${response.status}`);
      }
    }
  }
}

const statsBefore = await redisCommandStats();
const cpuBefore = await redisCpuSeconds();
const startedAt = performance.now();
const elapsed = () => (performance.now() - startedAt) / 1000;
const marks = {
  sendDone: 0,
  kafkaDrained: 0,
  eventsInClickhouse: 0,
  buffersDrained: 0,
};
const peaks = {
  kafkaLag: 0,
  eventBuffer: 0,
  sessionBuffer: 0,
  profileBuffer: 0,
};

/** Resolves true once everything landed, false on timeout. */
const monitor = (async (): Promise<boolean> => {
  while (true) {
    const [buffers, kafka, inClickhouse] = await Promise.all([
      bufferSizes(),
      kafkaState(),
      countBusinessEvents(),
    ]);
    const t = elapsed();
    peaks.kafkaLag = Math.max(peaks.kafkaLag, kafka.lag);
    peaks.eventBuffer = Math.max(peaks.eventBuffer, buffers.event);
    peaks.sessionBuffer = Math.max(peaks.sessionBuffer, buffers.session);
    peaks.profileBuffer = Math.max(peaks.profileBuffer, buffers.profile);
    log(
      `${t.toFixed(1)}s sent=${accepted} kafkaLag=${kafka.lag} evBuf=${buffers.event} sessBuf=${buffers.session} profBuf=${buffers.profile} chEvents=${inClickhouse}`
    );
    if (marks.sendDone) {
      if (!marks.kafkaDrained && kafka.lag === 0) {
        marks.kafkaDrained = t;
      }
      // A rejected request wrote no event, so the target is what was accepted.
      if (!marks.eventsInClickhouse && inClickhouse >= accepted) {
        marks.eventsInClickhouse = t;
      }
      const drained =
        buffers.event === 0 && buffers.session === 0 && buffers.profile === 0;
      if (marks.eventsInClickhouse && marks.kafkaDrained && drained) {
        marks.buffersDrained = t;
        return true;
      }
    }
    if (t > timeoutS) {
      log('TIMEOUT');
      return false;
    }
    await Bun.sleep(SAMPLE_INTERVAL_MS);
  }
})();

await Promise.all(Array.from({ length: concurrency }, sender));
marks.sendDone = elapsed();
log(
  `send done at ${marks.sendDone.toFixed(1)}s (${(accepted / marks.sendDone).toFixed(0)} accepted requests/s)`
);
const completed = await monitor;
// On a timeout the rates are over the time the run was given.
const runSeconds = completed ? marks.buffersDrained : elapsed();

const cpuAfter = await redisCpuSeconds();
reportRedisCost(statsBefore, await redisCommandStats());
const redisCpu = cpuAfter.main - cpuBefore.main;
const redisForkCpu = cpuAfter.children - cpuBefore.children;
log(
  `Redis CPU over the run: ${redisCpu.toFixed(1)} s = ${((1000 * redisCpu) / total).toFixed(3)} ms per business event (${((100 * redisCpu) / runSeconds).toFixed(0)}% of one core on average), plus ${redisForkCpu.toFixed(1)} s in snapshot forks`
);

const [eventCounts, sessionStarts, sessionCounts] = await Promise.all([
  clickhouse(
    `SELECT count(), uniqExact(id) FROM events WHERE ${businessFilter} FORMAT TSV`
  ),
  clickhouse(
    `SELECT count() FROM events WHERE ${runFilter} AND name = 'session_start' FORMAT TSV`
  ),
  clickhouse(
    `SELECT count(), countIf(net != 1) FROM (
       SELECT id, sum(sign) AS net FROM sessions
       WHERE project_id = '${projectId}'
         AND id IN (SELECT DISTINCT session_id FROM events WHERE ${businessFilter})
       GROUP BY id) FORMAT TSV`
  ),
]);

const summary = {
  run: runId,
  completed,
  shape: {
    devices,
    perDevice,
    concurrency,
    properties: propertyCount,
    identified: values.identified,
  },
  total,
  accepted,
  failed,
  retried,
  seconds: Object.fromEntries(
    Object.entries(marks).map(([key, value]) => [key, Number(value.toFixed(1))])
  ),
  businessEventsPerSecond: Number((accepted / runSeconds).toFixed(0)),
  peaks,
  redisCpuSeconds: Number(redisCpu.toFixed(2)),
  redisForkCpuSeconds: Number(redisForkCpu.toFixed(2)),
  verify: {
    eventsAndUniqueIds: eventCounts,
    sessionStarts,
    sessionsAndUncollapsed: sessionCounts,
    expected: `${total} ${total} / ${devices} / ${devices} 0`,
  },
};
log(JSON.stringify(summary));
await Bun.write(`${outPrefix}.json`, JSON.stringify(summary, null, 2));
await admin.disconnect();
redis.disconnect();
if (!completed || failed > 0) {
  process.exit(1);
}