Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

19.7 KB
Raw Blame
import assert from "node:assert/strict";
import test from "node:test";
import { litFixture } from "./lit-fixture.ts";

const zero = { costUsd: 0, calls: 0, tokens: 0 };
const background = (sources: Record<string, number> = {}) => {
  const all = Object.fromEntries(
    ["cron", "loop", "swarm", "webhook", "monitor"].map((source) => [
      source,
      { ...zero, costUsd: sources[source] ?? 0 },
    ]),
  );
  return { ...zero, costUsd: Object.values(sources).reduce((a, b) => a + b, 0), sources: all };
};

const person = (principalId: string, costUsd: number, over: Record<string, any> = {}) => ({
  principalId,
  scopeId: `personal:${principalId}`,
  kind: "person",
  displayName: "",
  costUsd,
  calls: 2,
  tokens: 1000,
  input: 100,
  output: 100,
  cacheRead: 700,
  cacheWrite: 100,
  cacheHitRatio: 0.7,
  live: { costUsd, calls: 2, tokens: 1000 },
  background: background(),
  ...over,
});

const DATA = {
  window: { from: "2026-08-26", to: "2026-09-25", bucket: "day" },
  org: {
    costUsd: 16,
    calls: 8,
    tokens: 4000,
    input: 400,
    output: 400,
    cacheRead: 2800,
    cacheWrite: 400,
    cacheHitRatio: 0.7,
    live: { costUsd: 16, calls: 8, tokens: 4000 },
    background: background(),
  },
  series: [
    { day: "2026-09-23", costUsd: 6, calls: 4, live: { costUsd: 6 }, background: background() },
    {
      day: "2026-09-24",
      costUsd: 10,
      calls: 4,
      live: { costUsd: 10 },
      background: background(),
    },
  ],
  people: [person("alice", 5), person("bob", 10), person("carol", 1)],
  scopes: [person("", 0, { principalId: null, scopeId: "channel:C1", kind: "channel", displayName: "#general" })],
};

const EMPTY = {
  window: { from: "2026-08-26", to: "2026-09-25", bucket: "day" },
  org: {
    costUsd: 0,
    calls: 0,
    tokens: 0,
    input: 0,
    output: 0,
    cacheRead: 0,
    cacheWrite: 0,
    cacheHitRatio: null,
    live: { costUsd: 0, calls: 0, tokens: 0 },
    background: background(),
  },
  series: [],
  people: [],
  scopes: [],
};

function render(data: any, services: Record<string, unknown> = {}) {
  const f = litFixture();
  const calls: any = { shell: null, ranges: [], customs: [], opened: [], downloads: 0, modelDownloads: 0, api: 0 };
  const view = f.ui.spend.spend(f.root, data, {
    window: { from: "2026-09-01", to: "2026-10-01", preset: "30d", bucket: "day" },
    defaultShell: (opts: any) => {
      calls.shell = opts;
    },
    api: () => {
      calls.api += 1;
      return Promise.resolve({ ok: true, data: {} });
    },
    fmtPct: (n: number) => (n * 100).toFixed(1) + "%",
    fmtTokens: (n: number) => `${n} tokens`,
    shortName: String,
    plural: (n: number, one: string) => `${n} ${one}`,
    openUser: (principal: string) => calls.opened.push(principal),
    setRange: (range: string) => calls.ranges.push(range),
    setCustomRange: (from: string, to: string) => calls.customs.push([from, to]),
    downloadCsv: (breakdown?: string) => {
      if (breakdown === "model") calls.modelDownloads += 1;
      else calls.downloads += 1;
    },
    ...services,
  });
  const peopleRows = () =>
    [...f.root.querySelectorAll(".tablewrap")[0]!.querySelectorAll("tbody tr")].map((r) =>
      r.querySelector("td")!.textContent!.trim(),
    );
  return { f, calls, view, peopleRows };
}

test("spend view: shows the saved report freshness", () => {
  const { f } = render({ ...DATA, asOf: Date.UTC(2026, 8, 25, 12) });
  assert.match(f.root.textContent!, /Totals as of.*Updated every minute/);
  f.window.close();
});

test("spend view: KPI stats, stacked chart and range tabs describe the whole org", () => {
  const { f, calls } = render(DATA);
  assert.deepEqual(JSON.parse(JSON.stringify(calls.shell.stats)), [
    ["$16.00", "Total spend"],
    ["$16.00", "Live"],
    ["$0.00", "Background"],
    ["70.0%", "Cache hit rate"],
  ]);
  assert.deepEqual(JSON.parse(JSON.stringify(calls.shell.tabs.map((t: any) => [t.label, t.active]))), [
    ["7d", false],
    ["30d", true],
    ["90d", false],
    ["Custom", false],
  ]);
  assert.equal(f.root.querySelectorAll(".spend-column").length, 30, "zero-spend dates retain their space");
  assert.ok(f.root.textContent!.includes("$0.53"), "daily average includes zero-spend days");
  const bar = f.root.querySelectorAll<HTMLElement>(".spend-column")[29]!;
  bar.dispatchEvent(new f.window.Event("pointerenter"));
  assert.match(f.root.querySelector(".spend-chart-detail")!.textContent!, /Sep 24.*\$10.00 total.*Live \$10.00/);
  assert.equal(bar.querySelector<HTMLElement>(".spend-live")!.style.height, "100%");
  calls.shell.tabs[0].onClick();
  assert.deepEqual(calls.ranges, ["7d"]);
  f.window.close();
});

test("spend view: search narrows the per-person table without touching the org stats", () => {
  const { f, calls, peopleRows } = render(DATA);
  const before = calls.shell.stats;
  assert.deepEqual(peopleRows(), ["bob", "alice", "carol"]);
  calls.shell.search.onInput("ali");
  assert.deepEqual(peopleRows(), ["alice"]);
  assert.equal(calls.shell.stats, before, "the shellbar is not rebuilt and the totals stay org-wide");
  calls.shell.search.onInput("");
  assert.deepEqual(peopleRows(), ["bob", "alice", "carol"]);
  f.window.close();
});

test("spend view: a Total header click sorts ascending, a second click descending, with no refetch", () => {
  const { f, calls, peopleRows } = render(DATA);
  const totalHeader = () =>
    [...f.root.querySelectorAll(".tablewrap")[0]!.querySelectorAll("thead th button")].find((b) =>
      b.textContent!.includes("Total"),
    ) as HTMLElement;
  totalHeader().click();
  assert.deepEqual(peopleRows(), ["carol", "alice", "bob"]);
  totalHeader().click();
  assert.deepEqual(peopleRows(), ["bob", "alice", "carol"]);
  assert.equal(calls.api, 0, "sorting is client-side");
  f.window.close();
});

test("spend view: a row click opens the sorted row's principal, and Export CSV downloads", () => {
  const { f, calls } = render(DATA);
  const rows = f.root.querySelectorAll(".tablewrap")[0]!.querySelectorAll<HTMLTableRowElement>("tbody tr");
  rows[1]!.click();
  assert.deepEqual(calls.opened, ["alice"], "the click indexes the sorted rows on screen");
  calls.shell.actions[0].click();
  assert.equal(calls.downloads, 1);
  f.window.close();
});

test("spend view: shared-scope rows are labelled and never open a user page", () => {
  const { f } = render(DATA);
  const shared = f.root.querySelectorAll(".tablewrap")[1]!;
  assert.equal(shared.querySelector("tbody td")!.textContent!.trim(), "#general");
  assert.equal(shared.querySelectorAll("tr.openable").length, 0);
  f.window.close();
});

test("spend view: an empty window renders empty-state copy and n/a, not a confident 0.0%", () => {
  const { f, calls } = render(EMPTY);
  assert.deepEqual(JSON.parse(JSON.stringify(calls.shell.stats[3])), ["n/a", "Cache hit rate"]);
  assert.equal(f.root.querySelectorAll(".tablewrap").length, 0);
  const text = f.root.textContent!;
  assert.ok(text.includes("No spend recorded in this window."), text);
  assert.ok(text.includes("No model calls recorded in this window."), text);
  f.window.close();
});

test("spend view: a null per-row cache ratio reads n/a", () => {
  const { f } = render({ ...DATA, people: [person("alice", 0, { cacheHitRatio: null })], scopes: [] });
  const cells = [...f.root.querySelectorAll(".tablewrap")[0]!.querySelectorAll("tbody td")].map((c) =>
    c.textContent!.trim(),
  );
  assert.equal(cells.at(-1), "n/a");
  f.window.close();
});

test("spend chart: category stacks and keyboard details use exact amounts", () => {
  const { f } = render({
    ...DATA,
    series: [{ day: "2026-09-24", live: { costUsd: 3 }, background: background({ cron: 2, loop: 1, swarm: 0.5 }) }],
  });
  const bar = f.root.querySelectorAll<HTMLElement>(".spend-column")[29]!;
  bar.dispatchEvent(new f.window.Event("focus"));
  assert.match(
    f.root.querySelector(".spend-chart-detail")!.textContent!,
    /Live \$3.00.*Crons \$2.00.*Loops \$1.00.*Swarms \$0.50.*Webhooks \$0.00.*Monitors \$0.00/,
  );
  assert.equal(bar.querySelectorAll(".spend-segment").length, 6);
  assert.ok(!f.root.innerHTML.includes("NaN"));
  f.window.close();
});
test("spend chart: weekly bins preserve partial boundaries and zero spend is finite", () => {
  const { f } = render({
    ...DATA,
    window: { ...DATA.window, bucket: "week" },
    series: [{ day: "2026-08-24", live: { costUsd: 0 } }],
  });
  assert.equal(f.root.querySelectorAll(".spend-column").length, 5);
  assert.match(f.root.querySelector(".spend-column")!.getAttribute("aria-label")!, /Aug 26.*Aug 30/);
  assert.ok(!/NaN|Infinity/.test(f.root.innerHTML));
  f.window.close();
});

test("spend models: category breakdown, sorting, filtering and export", () => {
  const { f, calls } = render({
    ...DATA,
    models: [
      { ...person("", 10), model: "model-a", live: { costUsd: 3 }, background: background({ cron: 6, swarm: 1 }) },
      { ...person("", 6), model: null, live: { costUsd: 5 }, background: background({ webhook: 1 }) },
    ],
  });
  const names = () =>
    [...f.root.querySelectorAll(".spend-models tbody tr")].map((row) => row.querySelector("td")!.textContent!.trim());
  assert.deepEqual(names(), ["model-a", "Unknown model"]);
  const cells = [...f.root.querySelectorAll(".spend-models tbody tr:first-child td")].map((c) => c.textContent!.trim());
  assert.deepEqual(cells.slice(0, 4), ["model-a", "$10.00", "$3.00", "$7.00Crons $6.00 · Swarms $1.00"]);
  const backgroundHeader = [...f.root.querySelectorAll<HTMLElement>(".spend-models thead button")].find((b) =>
    b.textContent!.includes("Background"),
  )!;
  backgroundHeader.click();
  assert.deepEqual(names(), ["Unknown model", "model-a"]);
  calls.shell.search.onInput("model-a");
  assert.deepEqual(names(), ["model-a"]);
  calls.shell.search.onInput("unmatched");
  assert.match(f.root.querySelector(".spend-models")!.textContent!, /No models match/);
  [...f.root.querySelectorAll<HTMLElement>("button")].find((b) => b.textContent === "Export model CSV")!.click();
  assert.equal(calls.modelDownloads, 1);
  assert.equal(calls.downloads, 0);
  assert.deepEqual(calls.opened, []);
  f.window.close();
});

test("spend chart: model stacks use each bucket, retain gaps, and switch back to categories", () => {
  const data = {
    ...DATA,
    series: DATA.series.map((p, i) => ({
      ...p,
      models:
        i === 0
          ? [
              { model: "model-a", costUsd: 2 },
              { model: "model-b", costUsd: 4 },
            ]
          : [
              { model: "model-a", costUsd: 7 },
              { model: null, costUsd: 3 },
            ],
    })),
  };
  const { f, calls } = render(data);
  const bars = () => f.root.querySelectorAll<HTMLElement>(".spend-column");
  assert.match(f.root.querySelector(".spend-chart-plot")!.getAttribute("aria-label")!, /by model/);
  assert.match(bars()[28]!.getAttribute("aria-label")!, /model-a \$2.00.*model-b \$4.00.*Unknown model \$0.00/);
  assert.match(bars()[29]!.getAttribute("aria-label")!, /model-a \$7.00.*model-b \$0.00.*Unknown model \$3.00/);
  assert.deepEqual(
    [...bars()[29]!.querySelectorAll<HTMLElement>(".spend-segment")].map((s) => s.style.height),
    ["70%", "0%", "30%"],
  );
  assert.match(bars()[0]!.getAttribute("aria-label")!, /\$0.00 total/);
  bars()[29]!.dispatchEvent(new f.window.Event("focus"));
  assert.match(f.root.querySelector(".spend-chart-detail")!.textContent!, /model-a \$7.00/);
  calls.shell.search.onInput("alice");
  assert.match(f.root.querySelector(".spend-chart-plot")!.getAttribute("aria-label")!, /by model/);
  const toggle = (label: string) =>
    [...f.root.querySelectorAll<HTMLElement>(".spend-chart-switch button")].find(
      (b) => b.textContent!.trim() === label,
    )!;
  toggle("Category").click();
  assert.equal(toggle("Category").getAttribute("aria-pressed"), "true");
  assert.equal(bars()[29]!.querySelectorAll(".spend-live").length, 1);
  assert.match(bars()[29]!.getAttribute("aria-label")!, /Live \$10.00/);
  assert.doesNotMatch(f.root.querySelector(".spend-chart-detail")!.textContent!, /model-a/);
  toggle("Model").click();
  assert.equal(toggle("Model").getAttribute("aria-pressed"), "true");
  assert.equal(bars()[29]!.querySelectorAll(".spend-model").length, 3);
  assert.equal(calls.api, 0);
  f.window.close();
});

test("spend chart: weekly model stacks and zero totals remain finite", () => {
  const { f } = render({
    ...DATA,
    window: { ...DATA.window, bucket: "week" },
    series: [{ day: "2026-08-24", models: [{ model: null, costUsd: 0 }] }],
  });
  assert.equal(f.root.querySelectorAll(".spend-column").length, 5);
  assert.match(
    f.root.querySelector(".spend-column")!.getAttribute("aria-label")!,
    /Aug 26.*Aug 30.*Unknown model \$0.00/,
  );
  assert.ok(!/NaN|Infinity/.test(f.root.innerHTML));
  f.window.close();
});

for (const bucket of ["day", "week"]) {
  test(`spend chart: ${bucket} person toggle shows names, shared scopes and zero bins`, () => {
    const { f, calls, view } = render({
      ...DATA,
      window: { ...DATA.window, bucket },
      people: [person("alice", 3, { displayName: "Alice" }), person("bob", 2)],
      series: [
        {
          day: bucket === "day" ? "2026-09-24" : "2026-09-21",
          live: { costUsd: 10 },
          models: [{ model: "model-a", costUsd: 10 }],
          people: [
            { principalId: "alice", costUsd: 3 },
            { principalId: "bob", costUsd: 2 },
            { principalId: null, costUsd: 5 },
          ],
        },
      ],
    });
    const toggle = (label: string) =>
      [...f.root.querySelectorAll<HTMLButtonElement>(".spend-chart-switch button")].find(
        (b) => b.textContent!.trim() === label,
      )!;
    toggle("Person").click();
    assert.equal(toggle("Person").getAttribute("aria-pressed"), "true");
    const bars = () => f.root.querySelectorAll<HTMLElement>(".spend-column");
    assert.equal(bars().length, bucket === "day" ? 30 : 5);
    assert.match(
      bars()[0]!.getAttribute("aria-label")!,
      /\$0.00 total.*Alice \$0.00.*bob \$0.00.*Shared scopes \$0.00/,
    );
    const last = () => bars()[bars().length - 1]!;
    last().dispatchEvent(new f.window.Event("focus"));
    assert.match(
      f.root.querySelector(".spend-chart-detail")!.textContent!,
      /\$10.00 total.*Alice \$3.00.*bob \$2.00.*Shared scopes \$5.00/,
    );
    const colors = () => [...last().querySelectorAll<HTMLElement>(".spend-person")].map((s) => s.style.background);
    const before = colors();
    view.data.people[0].displayName = "Renamed Alice";
    calls.shell.search.onInput("unmatched");
    assert.deepEqual(colors(), before, "colors are stable across display-name changes and filtering");
    assert.deepEqual(
      [...last().querySelectorAll<HTMLElement>(".spend-segment")].map((s) => s.style.height),
      ["30%", "20%", "50%"],
    );
    for (const label of ["Category", "Model", "Person"]) {
      toggle(label).click();
      assert.equal(toggle(label).getAttribute("aria-pressed"), "true");
      assert.match(last().getAttribute("aria-label")!, /\$10.00 total/);
    }
    assert.ok(!/NaN|Infinity/.test(f.root.innerHTML));
    assert.equal(calls.api, 0);
    f.window.close();
  });
}

test("spend chart: person toggle is disabled for reports without person buckets", () => {
  const { f } = render(DATA);
  assert.equal(
    [...f.root.querySelectorAll<HTMLButtonElement>(".spend-chart-switch button")].find(
      (b) => b.textContent!.trim() === "Person",
    )!.disabled,
    true,
  );
  f.window.close();
});

test("spend view: Custom turns the current window into two editable UTC days, applied on Apply as a sorted half-open window", () => {
  const preset = render(DATA, { window: { from: "2026-09-24", to: "2026-10-01", preset: "7d", bucket: "day" } });
  assert.equal(preset.f.root.querySelector(".spend-custom"), null);
  assert.deepEqual(JSON.parse(JSON.stringify(preset.calls.shell.tabs.map((t: any) => t.active))), [
    true,
    false,
    false,
    false,
  ]);
  preset.calls.shell.tabs.at(-1).onClick();
  assert.deepEqual(preset.calls.customs, [["2026-09-24", "2026-10-01"]]);
  preset.f.window.close();

  const { f, calls } = render(DATA, { window: { from: "2026-08-28", to: "2026-09-03", preset: null, bucket: "day" } });
  assert.deepEqual(JSON.parse(JSON.stringify(calls.shell.tabs.map((t: any) => [t.label, t.active]))), [
    ["7d", false],
    ["30d", false],
    ["90d", false],
    ["Custom", true],
  ]);
  const [first, last] = [...f.root.querySelectorAll<HTMLInputElement>(".spend-custom input")];
  assert.deepEqual([first!.value, last!.value], ["2026-08-28", "2026-09-02"]);
  const applyButton = f.root.querySelector<HTMLButtonElement>(".spend-custom button")!;
  last!.value = "2026-08-01";
  last!.dispatchEvent(new f.window.Event("change", { bubbles: true }));
  assert.deepEqual(calls.customs, [], "editing a day applies nothing until Apply");
  applyButton.click();
  assert.deepEqual(calls.customs, [["2026-08-01", "2026-08-29"]]);
  first!.value = "";
  applyButton.click();
  assert.equal(calls.customs.length, 1, "a cleared day applies nothing");
  calls.shell.tabs[0].onClick();
  assert.deepEqual(calls.ranges, ["7d"]);
  assert.equal(calls.api, 0);
  f.window.close();
});

test("spend view: crons show each cron's spend under its title, scope and schedule, filter, and open the cron's runs", () => {
  const opened: string[][] = [];
  const downloads: string[] = [];
  const cron = (cronId: string, costUsd: number, scopeId: string, scopeName: string, info: unknown) => ({
    ...person("", costUsd, { principalId: null, scopeId, kind: "person" }),
    cronId,
    scopeName,
    info,
  });
  const digest = {
    id: "abc123",
    title: "Morning digest",
    owner: "alice",
    ownerScopeId: "channel:C1",
    schedule: { cron: "0 9 * * 1-5" },
    enabled: true,
  };
  const { f, calls } = render(
    {
      ...DATA,
      crons: [
        cron("gone1234567890", 1, "personal:alice", "Alice", null),
        cron("abc123", 7, "channel:C1", "#general", digest),
      ],
    },
    {
      cronName: (c: { id: string; title?: string }) => c.title || `Cron ${c.id.slice(0, 6)}`,
      shortSchedule: (schedule: { cron: string }) => schedule.cron,
      openCronRuns: (cronId: string, scope: string) => opened.push([cronId, scope]),
      downloadCsv: (breakdown?: string) => downloads.push(breakdown ?? "entity"),
    },
  );
  const rows = () => [...f.root.querySelectorAll<HTMLTableRowElement>(".spend-crons tbody tr")];
  const labels = () =>
    rows().map((row) => {
      const cell = row.querySelector("td")!;
      const where = cell.querySelector(".subline")!.textContent!;
      return [cell.textContent!.replace(where, "").trim(), where.trim()];
    });
  assert.deepEqual(labels(), [
    ["Morning digest", "#general · 0 9 * * 1-5"],
    ["Cron gone12", "Alice · deleted"],
  ]);
  rows()[1]!.click();
  rows()[0]!.click();
  assert.deepEqual(opened, [
    ["gone1234567890", "personal:alice"],
    ["abc123", "channel:C1"],
  ]);
  [...f.root.querySelectorAll<HTMLButtonElement>("button")].find((b) => b.textContent === "Export cron CSV")!.click();
  assert.deepEqual(downloads, ["cron"]);
  for (const query of ["digest", "abc123", "general", "channel:C1"]) {
    calls.shell.search.onInput(query);
    assert.deepEqual(labels(), [["Morning digest", "#general · 0 9 * * 1-5"]], query);
  }
  f.window.close();
});

test("spend view: background by source names where background spend came from, largest first", () => {
  const { f } = render({
    ...DATA,
    org: { ...DATA.org, background: background({ cron: 6, loop: 3, swarm: 1 }) },
  });
  const card = [...f.root.querySelectorAll(".tablewrap")].find((t) => t.textContent!.includes("Share of background"))!;
  const rows = [...card.querySelectorAll("tbody tr")].map((r) =>
    [...r.querySelectorAll("td")].slice(0, 3).map((c) => c.textContent!.trim()),
  );
  assert.deepEqual(rows, [
    ["Crons", "$6.00", "60.0%"],
    ["Loops", "$3.00", "30.0%"],
    ["Swarms", "$1.00", "10.0%"],
  ]);
  f.window.close();
});