Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

10.4 KB
Raw Blame
/* eslint-env node */

/**
 * Runs the load benchmark across the conditions CI tracks, and prints one row
 * per condition.
 *
 * Each condition is measured twice. A series with the cache off gives the cold
 * load, where every run is a first visit. A series with the cache kept gives the
 * two loads that follow: the second visit, which reads the files from the HTTP
 * cache but still compiles them, and the steady state after V8 has cached the
 * compiled code.
 *
 * Times are relative to the machine that runs them. A CI runner is slower than a
 * laptop, so compare a number against the same machine's history, not against a
 * number from somewhere else.
 *
 * Point this at a real Metabase, which is what CI does, or at `serve.js` to
 * measure a built tree without a backend. Set SESSION_COOKIE to load the page
 * signed in.
 */
import { spawn } from "node:child_process";
import fs from "node:fs";
import path from "node:path";

/** One reading, as `timings` in `measure.ts` builds it. */
interface Timings {
  ttfbMs: number;
  firstContentfulPaintMs: number;
  domContentLoadedMs: number;
  appMountedMs: number;
  largestContentfulPaintMs: number;
  pageReadyMs: number;
}

/** What one `measure.ts` series prints. */
interface Series {
  runs: number;
  locale: string;
  scripts: number;
  scriptKb: number;
  cssKb: number;
  totalKb: number;
  median: Timings;
  secondLoad: Timings | null;
  steady: Timings | null;
  everyRunMs: number[];
  calibrationMs: number;
}

interface Conditions {
  mbps: number;
  latency: number;
  throttle: number;
  warm: boolean;
  offset: number;
  /** Loads in this series. Defaults to the run count on the command line. */
  loads?: number;
}

const url = process.argv[2];
const runs = Number(process.argv[3] || 8);

if (!url) {
  console.error("usage: bun matrix.ts <url> [runs]");
  console.error("env: SESSION_COOKIE");
  process.exit(1);
}

// The cache-kept series spends its first run filling an empty cache, its second
// on the second visit, and the rest on the steady state.
if (runs < 3) {
  console.error("matrix.ts needs at least 3 runs per condition");
  process.exit(1);
}

/** Throughput in Mbps and added latency in ms. */
const NETWORKS = {
  fast: { mbps: 40, latency: 20 },
  slow: { mbps: 5, latency: 150 },
};

/**
 * CPU slowdown. Neither condition is 1, because 1 leaves the reading at the
 * mercy of whichever CPU the runner happened to draw, and a throttle absorbs
 * some of that spread rather than passing it straight through.
 */
const CPUS = { fast: 2, slow: 4 };

/**
 * A second visit happens once per browser profile, so one cache-kept series can
 * only ever produce one of them. Several short series in their own profiles give
 * that reading a median and a spread for about the loads one long series costs.
 */
const WARM_PROFILES = 3;
const WARM_LOADS = 3;

/**
 * A series takes a minute or two. A Chrome that stops answering leaves
 * `measure.ts` waiting on its debugging socket without a bound, so past this
 * the series is stopped and measured again.
 */
const SERIES_TIMEOUT_MS = 5 * 60 * 1000;
const RETRY_PORT_OFFSET = 1000;

async function measure(conditions: Conditions): Promise<Series> {
  try {
    return await measureOnce(conditions);
  } catch (error) {
    const reason = error instanceof Error ? error.message : String(error);
    console.error(`${reason}, measuring the series again`);
    // A new port, because the Chrome that hung may still hold the old one.
    return measureOnce({
      ...conditions,
      offset: conditions.offset + RETRY_PORT_OFFSET,
    });
  }
}

function measureOnce({
  mbps,
  latency,
  throttle,
  warm,
  offset,
  loads,
}: Conditions): Promise<Series> {
  return new Promise<Series>((resolve, reject) => {
    const child = spawn(
      process.execPath,
      [
        path.join(import.meta.dirname, "measure.ts"),
        url,
        String(loads ?? runs),
      ],
      {
        stdio: ["ignore", "pipe", "inherit"],
        // Its own process group, so stopping it also stops its Chrome.
        detached: true,
        env: {
          ...process.env,
          NETWORK_MBPS: String(mbps),
          NETWORK_LATENCY: String(latency),
          CPU_THROTTLE: String(throttle),
          // Each series gets its own debugging port, so a Chrome that is slow
          // to release the last one cannot be mistaken for the new one.
          PORT_OFFSET: String(offset),
          WARM: warm ? "1" : "",
        },
      },
    );

    const timeout = setTimeout(() => {
      if (child.pid) {
        process.kill(-child.pid, "SIGKILL");
      }
    }, SERIES_TIMEOUT_MS);

    let output = "";
    child.stdout.on("data", (chunk) => (output += chunk));
    child.on("exit", (code, signal) => {
      clearTimeout(timeout);
      if (code === 0) {
        resolve(JSON.parse(output));
      } else {
        reject(new Error(`measure.ts exited with ${code ?? signal}`));
      }
    });
  });
}

/**
 * How far the middle half of the cold runs spread, as a percent of the median.
 *
 * Cold is the number a reader watches, and on a shared runner it is the number
 * noise moves. Recording the spread beside it is what tells a real change from
 * a busy machine.
 */
/**
 * `measure.ts` leaves these empty for a series too short to hold them, which the
 * run count above rules out. Fail loudly rather than report a zero if it ever
 * stops filling one in.
 */
function required(timings: Timings | null, name: string): Timings {
  if (!timings) {
    throw new Error(`measure.ts reported no ${name}`);
  }
  return timings;
}

/**
 * The series whose reading is the median of the batch.
 *
 * Picking a series rather than averaging each reading keeps a state's readings
 * in order with each other, the same reason `measure.ts` reports one load per
 * state instead of a median per reading.
 */
function representativeSeries(
  batch: Series[],
  of: (series: Series) => Timings,
): Series {
  const sorted = [...batch].sort(
    (a, b) => of(a).domContentLoadedMs - of(b).domContentLoadedMs,
  );
  return sorted[Math.floor(sorted.length / 2)];
}

function readProc(file: string): string {
  try {
    return fs.readFileSync(file, "utf8");
  } catch {
    return "";
  }
}

const cpuModel =
  readProc("/proc/cpuinfo").match(/^model name\s*:\s*(.+)$/m)?.[1] ?? "";

function spreadPercent(values: number[]) {
  const sorted = [...values].sort((a, b) => a - b);
  const at = (fraction: number) => sorted[Math.floor(sorted.length * fraction)];
  const middle = sorted[Math.floor(sorted.length / 2)];
  return Number((((at(0.75) - at(0.25)) / middle) * 100).toFixed(1));
}

(async () => {
  const rows = [];
  let offset = 0;

  // Object.keys widens to string, and these two objects are the only source of
  // condition names.
  for (const cpu of Object.keys(CPUS) as (keyof typeof CPUS)[]) {
    // Object.keys widens to string, as above.
    for (const network of Object.keys(NETWORKS) as (keyof typeof NETWORKS)[]) {
      const { mbps, latency } = NETWORKS[network];
      const throttle = CPUS[cpu];

      const cold = await measure({
        mbps,
        latency,
        throttle,
        warm: false,
        offset: offset++,
      });
      const warmBatch = [];
      for (let profile = 0; profile < WARM_PROFILES; profile++) {
        warmBatch.push(
          await measure({
            mbps,
            latency,
            throttle,
            warm: true,
            offset: offset++,
            loads: WARM_LOADS,
          }),
        );
      }

      // Each state picks its own series, because the second visit and the
      // steady state are different loads already.
      const warm = representativeSeries(warmBatch, (series) =>
        required(series.secondLoad, "secondLoad"),
      );
      const steadiest = representativeSeries(warmBatch, (series) =>
        required(series.steady, "steady"),
      );
      const calibrations = [cold, ...warmBatch]
        .map((series) => series.calibrationMs)
        .sort((a, b) => a - b);

      // Every cold reading in the row comes from `cold.median`, and every warm
      // reading from `warm.secondLoad`. Each is one load, so the readings in a
      // row stay in order with each other. A row assembled from a separate
      // median per reading describes a load that never happened.
      rows.push({
        network,
        networkMbps: mbps,
        latencyMs: latency,
        cpu,
        cpuThrottle: throttle,
        coldMs: cold.median.domContentLoadedMs,
        warmMs: required(warm.secondLoad, "secondLoad").domContentLoadedMs,
        steadyMs: required(steadiest.steady, "steady").domContentLoadedMs,
        coldSpreadPercent: spreadPercent(cold.everyRunMs),
        warmSpreadPercent: spreadPercent(
          warmBatch.map(
            (series) =>
              required(series.secondLoad, "secondLoad").domContentLoadedMs,
          ),
        ),
        steadySpreadPercent: spreadPercent(
          warmBatch.map(
            (series) => required(series.steady, "steady").domContentLoadedMs,
          ),
        ),
        // The cold load broken up, in the order a user meets it: bytes start
        // arriving, something is drawn, the shell commits, the page has its
        // data.
        coldTtfbMs: cold.median.ttfbMs,
        coldFirstPaintMs: cold.median.firstContentfulPaintMs,
        coldAppMountedMs: cold.median.appMountedMs,
        coldLargestPaintMs: cold.median.largestContentfulPaintMs,
        coldPageReadyMs: cold.median.pageReadyMs,
        warmPageReadyMs: required(warm.secondLoad, "secondLoad").pageReadyMs,
        // The same two readings for the visits that follow, so a returning
        // user's page is measured by when it draws rather than by
        // DOMContentLoaded alone.
        warmLargestPaintMs: required(warm.secondLoad, "secondLoad")
          .largestContentfulPaintMs,
        steadyLargestPaintMs: required(steadiest.steady, "steady")
          .largestContentfulPaintMs,
        steadyPageReadyMs: required(steadiest.steady, "steady").pageReadyMs,
        locale: cold.locale,
        scripts: cold.scripts,
        scriptKb: cold.scriptKb,
        cssKb: cold.cssKb,
        totalKb: cold.totalKb,
        runs: cold.runs,
        cpuModel,
        calibrationMs: calibrations[Math.floor(calibrations.length / 2)],
      });

      console.error(`measured ${cpu} cpu on a ${network} network`);
    }
  }

  // Written with a callback rather than console.log so the process cannot exit
  // with the JSON still buffered in a pipe.
  process.stdout.write(JSON.stringify(rows, null, 2) + "\n", () =>
    process.exit(0),
  );
})();