Skip to content

A local-first multi-agent app development platform, built on PenguinHarness

91.8 KB
Raw Blame
/**
 * Error records, from where they are caught to the cost center that shows them.
 *
 * The log (ErrorsRepo):
 * - A summary counts every row and the unexpected ones; the top code groups by source, code and
 *   kind (none in range reads as nothing); dates and an Agent narrow every read.
 * - The recent list folds a day's repeats of one error into one row with its count and its
 *   first and last time, the day being the viewer's when the read carries an offset; a page
 *   and the row count are rows, the summary still counts records.
 * - A clear follows the caller's read: a member's leaves the unattributed rows, an admin's
 *   takes them; another Project's rows are out of reach either way.
 * - An ignore rule (a source · code, optionally narrowed by a message fragment) takes that class
 *   off every read's default side without deleting a row: the reads, the clear and the rule's own
 *   coverage count all take the same axis; rules are per Project and go with it.
 * - Past its cap the log evicts its oldest rows by count, never misled by id gaps.
 *
 * The recorder (ErrorRecorder):
 * - An HttpError is expected; any other failure is unexpected, an explicit kind wins over both,
 *   and a gateway close is filed by its own verdict.
 * - A non-Error is stringified and an overlong message truncated; the recorder never throws.
 * - The same error for the same Project persists once per short window, then resumes; source,
 *   code and Project each keep their own window; a full dedup table still records every
 *   distinct error and still collapses a burst.
 *
 * The stream watcher (LLM, tool and compaction failures in a Session's messages):
 * - An LLM failure the retry ladder carried is expected under its own code; an exhausted or
 *   fatal one is unexpected, worded from the abort reason, the request's own detail or the
 *   status text in that order; a user's Stop and a completion are not errors.
 * - A command tool's ordinary non-zero exit is information, not an error; a signal, a spawn
 *   failure, a timeout or a missing session manager are recorded; any other tool failure is
 *   expected under the tool's name, legacy spellings included.
 * - Parallel and nested (child Session) calls never swap names or reasons, and a child's
 *   failures are attributed to the child Agent and Session.
 * - An abandoned compaction is one unexpected row carrying its attempts and error.
 *
 * Through HTTP (onError):
 * - A business error is expected, an internal one unexpected with a 500, each attributed to
 *   the Project only when the requester has access to it (unauthenticated and non-member
 *   failures stay unattributed, and a throwing access check degrades to unattributed).
 * - A Skill directory another program is holding open (Windows EBUSY) answers as a named 409
 *   and is filed as expected, not as the internal failure the raw errno would look like.
 * - The usage endpoint shows a Project's own errors; unattributed ones only to the admin.
 * - The paged error route pages within the caller's tenant, narrows to one kind on request,
 *   and refuses a page it cannot serve; it and the dashboard fold by the viewer's day, and an
 *   offset no clock uses is a 400.
 * - Clearing takes exactly the filter or trailing window on screen, never an unbounded range;
 *   a member may not clear; an admin's clear takes the unattributed rows it showed.
 * - The owner silences a class (never a member, and never a non-member's Project), the panel then
 *   reports it under `ignored` and reads it back on request, and removing the rule restores it.
 * - Deleting a Project deletes its error records and nobody else's.
 */
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import fs from "node:fs/promises";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import {
  abortEvent,
  assistantText,
  compactionEnd,
  partialToolCallOutput,
  requestBegin,
  requestEnd,
  sessionMeta,
  skillsDir,
  toolCall,
  toolCallOutput,
  withOrigin,
} from "@adelie/core";
import type { OmniMessage, StopReason } from "@adelie/core";
import type {
  ProjectCreateResponse,
  UsageErrorIgnoreAdded,
  UsageErrorIgnores,
  UsageErrorsPage,
  UsageResponse,
} from "../src/api/types.js";
import { openDatabase } from "../src/db/database.js";
import { ErrorsRepo } from "../src/db/repos/errors.js";
import type { ErrorRecordInsert } from "../src/db/repos/errors.js";
import { HttpError } from "../src/http/errors.js";
import {
  DEDUP_KEYS_MAX,
  DEDUP_WINDOW_MS,
  ErrorRecorder,
  MESSAGE_MAX,
} from "../src/runtime/error-recorder.js";
import { messagingErrorKind } from "../src/runtime/messaging/error-kind.js";
import { MessagingConnectionClosedError } from "../src/runtime/messaging/qq-api.js";
import { StreamErrorWatcher } from "../src/runtime/stream-error-watcher.js";
import { apiClient, createTestApp, loginAdmin, provisionUser } from "./helpers.js";
import type { TestApp } from "./helpers.js";
import { wire } from "@adelie/core/kernel";

function row(date: string, o: Partial<ErrorRecordInsert> = {}): ErrorRecordInsert {
  return {
    ts: `${date}T10:00:00.000Z`,
    date,
    projectId: "p1",
    agentId: null,
    sessionId: null,
    source: "http",
    kind: "unexpected",
    code: "internal",
    status: 500,
    message: "boom",
    ...o,
  };
}

describe("errors-repo", () => {
  let db: DatabaseSync;
  let repo: ErrorsRepo;

  beforeEach(() => {
    db = openDatabase(":memory:");
    repo = wire(ErrorsRepo, { db: db });
  });
  afterEach(() => db.close());

  it("summary: total and unexpected count; expected ones are still recorded", () => {
    repo.insert(row("2026-07-06"));
    repo.insert(row("2026-07-06", { kind: "expected", code: "not_found", status: 404 }));
    repo.insert(row("2026-07-06", { kind: "expected", code: "bad_request", status: 400 }));
    expect(repo.summary("p1")).toEqual({ total: 3, unexpected: 1 });
  });

  it("top error code: grouped by source+code+kind, takes the highest count", () => {
    for (let i = 0; i < 3; i++) repo.insert(row("2026-07-06", { code: "internal" }));
    repo.insert(row("2026-07-06", { source: "session", code: "session_run_failed" }));
    repo.insert(row("2026-07-06", { kind: "expected", code: "not_found", status: 404 }));
    repo.insert(row("2026-07-06", { kind: "expected", code: "not_found", status: 404 }));

    expect(repo.topCode("p1")).toEqual({
      source: "http",
      code: "internal",
      kind: "unexpected",
      count: 3,
    });
    // No errors / no errors in range -> null (the frontend uses this to hide the metric).
    expect(repo.topCode("p-empty")).toBeNull();
    expect(repo.topCode("p1", { from: "2026-07-07" })).toBeNull();
  });

  it("date range and agent filters (HTTP / process errors have no agent_id)", () => {
    repo.insert(row("2026-07-05"));
    repo.insert(row("2026-07-06", { kind: "expected" }));
    repo.insert(row("2026-07-06", { agentId: "a1", source: "session" }));

    expect(repo.summary("p1")).toEqual({ total: 3, unexpected: 2 });
    expect(repo.summary("p1", { from: "2026-07-06" })).toEqual({ total: 2, unexpected: 1 });
    expect(repo.summary("p1", { agentId: "a1" })).toEqual({ total: 1, unexpected: 1 });
    expect(repo.topCode("p1", { agentId: "a1" })).toMatchObject({ source: "session", count: 1 });
    expect(repo.recent("p1", { agentId: "a1" })).toHaveLength(1);
  });

  it("recent folds a day's repeats into one row, with how many and the first and last time", () => {
    // A connection that drops and recovers all afternoon files the same record again and again;
    // the table reads it as one line saying how often. Anything that differs keeps its own row:
    // another message, another day, another kind (a record filed before a reclassification).
    const heartbeat = {
      source: "messaging",
      code: "messaging_connect_failed",
      kind: "expected",
      message: "gateway stopped acknowledging heartbeats",
    };
    const at = (ts: string, o: Partial<ErrorRecordInsert> = {}) =>
      repo.insert(row(ts.slice(0, 10), { ...heartbeat, ts, ...o }));
    at("2026-07-06T09:00:00.000Z");
    at("2026-07-06T12:30:00.000Z");
    at("2026-07-06T13:00:00.000Z", { message: "gateway connection closed (code 4009)" });
    at("2026-07-06T17:45:00.000Z");
    at("2026-07-06T18:00:00.000Z", { kind: "unexpected" });
    at("2026-07-07T08:00:00.000Z");

    const rows = repo
      .recent("p1")
      .map(({ ts, firstTs, count, kind, message }) => ({ ts, firstTs, count, kind, message }));
    expect(rows).toEqual([
      {
        ts: "2026-07-07T08:00:00.000Z",
        firstTs: "2026-07-07T08:00:00.000Z",
        count: 1,
        kind: "expected",
        message: heartbeat.message,
      },
      {
        ts: "2026-07-06T18:00:00.000Z",
        firstTs: "2026-07-06T18:00:00.000Z",
        count: 1,
        kind: "unexpected",
        message: heartbeat.message,
      },
      // Newest first by the row's latest record, so the fold sits where its last one happened.
      {
        ts: "2026-07-06T17:45:00.000Z",
        firstTs: "2026-07-06T09:00:00.000Z",
        count: 3,
        kind: "expected",
        message: heartbeat.message,
      },
      {
        ts: "2026-07-06T13:00:00.000Z",
        firstTs: "2026-07-06T13:00:00.000Z",
        count: 1,
        kind: "expected",
        message: "gateway connection closed (code 4009)",
      },
    ]);
    // A page is rows, and the pager counts rows; the summary still counts the records.
    expect(repo.recent("p1", {}, 2, 2).map((r) => r.ts)).toEqual([
      "2026-07-06T17:45:00.000Z",
      "2026-07-06T13:00:00.000Z",
    ]);
    expect(repo.rowCount("p1")).toBe(4);
    expect(repo.summary("p1")).toEqual({ total: 6, unexpected: 1 });
    expect(repo.topCode("p1")).toMatchObject({ code: "messaging_connect_failed", count: 5 });
    // Filters narrow the records before they fold.
    expect(repo.rowCount("p1", { kind: "unexpected" })).toBe(1);
    expect(repo.recent("p1", { from: "2026-07-06", to: "2026-07-06" })).toHaveLength(3);
  });

  it("the day a row folds by is the viewer's, from the offset the read carries", () => {
    // Two records an hour apart, either side of UTC midnight. The `date` column is the server's
    // day, which is what a read without an offset folds by.
    const late = { ts: "2026-07-06T23:30:00.000Z", message: "socket hang up" };
    const early = { ts: "2026-07-07T00:30:00.000Z", message: "socket hang up" };
    repo.insert(row("2026-07-06", late));
    repo.insert(row("2026-07-07", early));

    expect(repo.recent("p1").map((r) => r.count)).toEqual([1, 1]);
    expect(repo.rowCount("p1")).toBe(2);
    // UTC+8 and UTC-5: both land on one local day, so one row says twice.
    for (const offset of [480, -300]) {
      expect(repo.recent("p1", {}, 20, 0, offset).map((r) => r.count)).toEqual([2]);
      expect(repo.rowCount("p1", {}, offset)).toBe(1);
    }
    // UTC itself splits them again.
    expect(repo.rowCount("p1", {}, 0)).toBe(2);
  });

  it("deleteFiltered follows the caller's includeGlobal: a member's clear leaves unattributed rows, an admin's takes them", () => {
    // Unattributed rows appear in every Project's admin view and in no member's. The clear
    // reaches exactly what the caller's read reaches, so only a read that includes them
    // deletes them; another Project's rows are out of reach either way.
    repo.insert(row("2026-07-06", { projectId: null, source: "process", message: "crash" }));
    repo.insert(row("2026-07-06", { projectId: "p-other", message: "other tenant" }));
    repo.insert(row("2026-07-06", { message: "mine" }));
    const remaining = () =>
      db
        .prepare("SELECT message FROM error_records ORDER BY id")
        .all()
        .map((r) => r.message);

    expect(repo.deleteFiltered("p1")).toBe(1);
    expect(remaining()).toEqual(["crash", "other tenant"]);

    expect(repo.deleteFiltered("p1", { includeGlobal: true })).toBe(1);
    expect(remaining()).toEqual(["other tenant"]);
  });

  // —— Ignoring: rules that keep a class off the panel's default view, on read, without losing a row ——

  it("an ignore rule takes a whole class out of every read, while the records stay in the table", () => {
    // Three of the class to silence, two of another one that must keep showing.
    for (let i = 0; i < 3; i++) repo.insert(row("2026-07-06", { code: "internal" }));
    for (let i = 0; i < 2; i++)
      repo.insert(row("2026-07-06", { source: "messaging", code: "messaging_connect_failed" }));

    const rule = repo.addIgnoreRule("p1", {
      source: "http",
      code: "internal",
      createdBy: "owner",
      createdAt: "2026-07-06T10:00:00.000Z",
    });
    expect(rule).toMatchObject({ source: "http", code: "internal", messageLike: null, hidden: 3 });

    // Every read not asked for the ignored side leaves the class out …
    expect(repo.summary("p1")).toEqual({ total: 2, unexpected: 2 });
    expect(repo.topCode("p1")).toMatchObject({ source: "messaging", count: 2 });
    expect(repo.recent("p1").map((r) => r.source)).toEqual(["messaging"]);
    expect(repo.rowCount("p1")).toBe(1);
    // … and the other side reads exactly it, which is what the panel's "ignored" view shows.
    expect(repo.summary("p1", { ignored: true })).toEqual({ total: 3, unexpected: 3 });
    expect(repo.recent("p1", { ignored: true }).map((r) => r.source)).toEqual(["http"]);
    // Nothing was deleted: the rule changes which rows the reads select, not what is stored.
    expect(
      db.prepare("SELECT COUNT(*) AS n FROM error_records WHERE code = 'internal'").get(),
    ).toEqual({ n: 3 });
  });

  it("a fragment narrows a rule to the messages that contain it", () => {
    const at = (message: string) => repo.insert(row("2026-07-06", { code: "internal", message }));
    at("other side closed");
    at("other side closed (again)");
    at("disk full");
    repo.addIgnoreRule("p1", {
      source: "http",
      code: "internal",
      messageLike: "other side closed",
      createdBy: "owner",
      createdAt: "2026-07-06T10:00:00.000Z",
    });

    // A literal substring, not a pattern: the two sharing it go, the third stays.
    expect(repo.recent("p1").map((r) => r.message)).toEqual(["disk full"]);
    expect(repo.summary("p1")).toEqual({ total: 1, unexpected: 1 });
    expect(repo.summary("p1", { ignored: true })).toEqual({ total: 2, unexpected: 2 });
  });

  it("a blank fragment means the whole class, not a rule that swallows every record", () => {
    repo.insert(row("2026-07-06", { code: "internal", message: "boom" }));
    repo.insert(
      row("2026-07-06", { source: "session", code: "session_run_failed", message: "nope" }),
    );
    // instr(x, '') is > 0 for every row: keeping the empty string would make this one rule cover
    // every record in the table, so a blank fragment has to read as "no fragment".
    const rule = repo.addIgnoreRule("p1", {
      source: "http",
      code: "internal",
      messageLike: "   ",
      createdBy: "owner",
      createdAt: "2026-07-06T10:00:00.000Z",
    });
    expect(rule?.messageLike).toBeNull();
    expect(repo.summary("p1")).toEqual({ total: 1, unexpected: 1 });
    expect(repo.recent("p1").map((r) => r.source)).toEqual(["session"]);
  });

  it("one rule per class: the same (source, code, fragment) is refused, a different fragment is another rule", () => {
    const make = (messageLike: string | null) =>
      repo.addIgnoreRule("p1", {
        source: "http",
        code: "internal",
        messageLike,
        createdBy: "owner",
        createdAt: "2026-07-06T10:00:00.000Z",
      });

    expect(make(null)).not.toBeNull();
    // The same class is one rule, not two rows saying it twice.
    expect(make(null)).toBeNull();
    expect(make("other side closed")).not.toBeNull();
    // A blank fragment is the whole class, so it collides with the whole-class rule above.
    expect(make("  ")).toBeNull();
    expect(repo.ignoreRules("p1")).toHaveLength(2);
  });

  it("rules are per Project, and deleting a Project takes its rules with it", () => {
    repo.insert(row("2026-07-06", { projectId: "p1", code: "internal" }));
    repo.insert(row("2026-07-06", { projectId: "p2", code: "internal" }));
    const rule = (projectId: string) =>
      repo.addIgnoreRule(projectId, {
        source: "http",
        code: "internal",
        createdBy: "owner",
        createdAt: "2026-07-06T10:00:00.000Z",
      });
    rule("p1");
    rule("p2");

    // p2's rule silences p2's records only, so the same class in p1 reads untouched.
    expect(repo.summary("p1")).toEqual({ total: 0, unexpected: 0 });
    expect(repo.summary("p1", { ignored: true })).toEqual({ total: 1, unexpected: 1 });
    expect(repo.summary("p2", { ignored: true })).toEqual({ total: 1, unexpected: 1 });

    // A rule names one Project's errors and means nothing once they are gone.
    repo.deleteByProject("p1");
    expect(repo.ignoreRules("p1")).toEqual([]);
    expect(repo.ignoreRules("p2")).toHaveLength(1);
    expect(repo.summary("p2", { ignored: true })).toEqual({ total: 1, unexpected: 1 });
  });

  it("a clear on either side takes only that side's rows", () => {
    for (let i = 0; i < 2; i++)
      repo.insert(row("2026-07-06", { code: "internal", message: "silenced" }));
    repo.insert(
      row("2026-07-06", { source: "session", code: "session_run_failed", message: "shown" }),
    );
    repo.addIgnoreRule("p1", {
      source: "http",
      code: "internal",
      createdBy: "owner",
      createdAt: "2026-07-06T10:00:00.000Z",
    });

    // Default side: the rows the panel shows, and no other.
    expect(repo.deleteFiltered("p1")).toBe(1);
    expect(repo.summary("p1", { ignored: true })).toEqual({ total: 2, unexpected: 2 });

    // Standing on the ignored view, the same button is the one that may take the covered rows.
    expect(repo.deleteFiltered("p1", { ignored: true })).toBe(2);
    expect(db.prepare("SELECT COUNT(*) AS n FROM error_records").get()).toEqual({ n: 0 });
  });

  it("ignoreRuleMatches counts a class against a range, saved or not", () => {
    repo.insert(row("2026-07-05", { code: "internal" }));
    repo.insert(row("2026-07-06", { code: "internal" }));
    repo.insert(row("2026-07-06", { code: "internal", message: "other side closed" }));
    repo.insert(row("2026-07-07", { code: "internal" }));

    const wholeClass = { source: "http", code: "internal", messageLike: null };
    expect(repo.ignoreRuleMatches("p1", wholeClass)).toBe(4);
    // Against a caller's range, which is what a new rule reports having just hidden.
    expect(repo.ignoreRuleMatches("p1", wholeClass, { from: "2026-07-06", to: "2026-07-06" })).toBe(
      2,
    );
    expect(repo.ignoreRuleMatches("p1", wholeClass, { from: "2026-07-06", to: "2026-07-07" })).toBe(
      3,
    );
    // A fragment narrows it; the axis is never applied here — it counts what a rule will take off.
    expect(repo.ignoreRuleMatches("p1", { ...wholeClass, messageLike: "other side closed" })).toBe(
      1,
    );
  });

  // —— Row cap (the second line of defense against error storms; the first is ErrorRecorder's short-window dedup) ——

  const messages = () =>
    db
      .prepare("SELECT message FROM error_records ORDER BY id")
      .all()
      .map((r) => r.message as string);

  it("row cap: evicts the oldest rows by id (checked every pruneEvery inserts)", () => {
    const capped = wire(ErrorsRepo, { db: db, maxRows: 5, pruneEvery: 2 });
    for (let i = 0; i < 10; i++) capped.insert(row("2026-07-06", { message: `m${i}` }));
    // The 5 most recent rows within the cap are kept, older ones are evicted.
    expect(messages()).toEqual(["m5", "m6", "m7", "m8", "m9"]);
  });

  it("eviction counts rows: id gaps left by deleteByProject never misdelete valid data", () => {
    const capped = wire(ErrorsRepo, { db: db, maxRows: 3, pruneEvery: 1 });
    capped.insert(row("2026-07-06", { message: "keep-1" })); // id 1
    capped.insert(row("2026-07-06", { message: "keep-2" })); // id 2
    capped.insert(row("2026-07-06", { projectId: "p-gone", message: "gone" })); // id 3
    capped.deleteByProject("p-gone"); // id 3 becomes a gap: MAX(id) is now decoupled from the actual row count

    // 3 rows in the table = exactly at the cap; none should be deleted. An approximation (id <= MAX(id) - 3) would wrongly delete keep-1.
    capped.insert(row("2026-07-06", { message: "keep-3" })); // id 4
    expect(messages()).toEqual(["keep-1", "keep-2", "keep-3"]);

    // Once over the cap, the oldest row is evicted as usual (gaps don't affect the "oldest" determination).
    capped.insert(row("2026-07-06", { message: "keep-4" })); // id 5
    expect(messages()).toEqual(["keep-2", "keep-3", "keep-4"]);
  });
});

describe("error-recorder", () => {
  let db: DatabaseSync;
  let repo: ErrorsRepo;
  const now = () => new Date("2026-07-06T10:00:00");

  beforeEach(() => {
    db = openDatabase(":memory:");
    repo = wire(ErrorsRepo, { db: db });
  });
  afterEach(() => db.close());

  it("a non-HTTP failure is unexpected, with no status and the attribution it was given", () => {
    wire(ErrorRecorder, { errors: repo, clock: { now: now } }).record({
      source: "session",
      err: new Error("drive crashed"),
      ctx: { projectId: "p1", agentId: "a1", sessionId: "s1" },
      code: "session_run_failed",
    });
    expect(db.prepare("SELECT * FROM error_records").get()).toMatchObject({
      kind: "unexpected",
      code: "session_run_failed",
      status: null,
      project_id: "p1",
      agent_id: "a1",
      session_id: "s1",
      date: "2026-07-06",
    });
  });

  it("non-Error throwables and overlong messages: stringified and truncated to the cap", () => {
    const rec = wire(ErrorRecorder, { errors: repo, clock: { now: now } });
    rec.record({ source: "process", err: "a string error", code: "unhandled_rejection" });
    rec.record({ source: "usage", err: new Error("x".repeat(MESSAGE_MAX + 100)) });
    const rows = db.prepare("SELECT message FROM error_records ORDER BY id").all();
    expect(rows[0]!.message).toBe("a string error");
    expect((rows[1]!.message as string).length).toBe(MESSAGE_MAX);
  });

  it("the recorder itself never throws (hooked on onError it would recurse forever)", () => {
    const broken = {
      insert() {
        throw new Error("DB is closed");
      },
    } as unknown as ErrorsRepo;
    expect(() =>
      wire(ErrorRecorder, { errors: broken, clock: { now: () => new Date() } }).record({
        source: "http",
        err: new Error("x"),
      }),
    ).not.toThrow();
  });

  it("explicit kind wins over HttpError inference (sources self-report human need)", () => {
    const rec = wire(ErrorRecorder, { errors: repo, clock: { now: now } });
    rec.record({ source: "llm", err: "timed out", code: "llm_timeout", kind: "expected" });
    rec.record({ source: "llm", err: "auth failed", code: "llm_failed", kind: "unexpected" });
    const rows = db.prepare("SELECT kind, source, status FROM error_records ORDER BY id").all();
    expect(rows[0]).toMatchObject({ kind: "expected", source: "llm", status: null });
    expect(rows[1]).toMatchObject({ kind: "unexpected", source: "llm", status: null });
  });

  it("a gateway close is filed by its own verdict: the routine one leaves the count alone", () => {
    // The path the messaging bridge takes for a dropped connection: the connector's typed
    // close, classified by messagingErrorKind, persisted under `messaging_connect_failed`.
    // Both rows land — the log keeps everything — but only the one a person has to act on
    // reaches the unexpected count the cost center highlights.
    const rec = wire(ErrorRecorder, { errors: repo, clock: { now: now } });
    const closed = (message: string, code: number, recovers: boolean) =>
      new MessagingConnectionClosedError(message, code, recovers);
    for (const err of [
      closed("gateway connection closed (code 4009)", 4009, true),
      closed("gateway connection closed (code 4004)", 4004, false),
    ]) {
      rec.record({
        source: "messaging",
        err,
        code: "messaging_connect_failed",
        kind: messagingErrorKind(err, "messaging_connect_failed"),
        // Distinct Projects so the recorder's short-window dedup does not swallow the second.
        ctx: { projectId: err.recovers ? "p-routine" : "p-fault" },
      });
    }
    const rows = db.prepare("SELECT kind, code, message FROM error_records ORDER BY id").all();
    expect(rows[0]).toMatchObject({ kind: "expected", code: "messaging_connect_failed" });
    expect(rows[1]).toMatchObject({ kind: "unexpected", code: "messaging_connect_failed" });
    expect(repo.summary("p-routine")).toEqual({ total: 1, unexpected: 0 });
    expect(repo.summary("p-fault")).toEqual({ total: 1, unexpected: 1 });
  });

  // —— Short-window dedup (the first line of defense against error storms) ——

  const count = () =>
    db.prepare("SELECT COUNT(*) AS n FROM error_records").get()!.n as unknown as number;

  it("short-window dedup: same-kind errors persist once per window, then resume", () => {
    let t = Date.parse("2026-07-06T10:00:00Z");
    const rec = wire(ErrorRecorder, { errors: repo, clock: { now: () => new Date(t) } });
    const boom = () =>
      rec.record({
        source: "http",
        err: new HttpError(404, "not_found", "Not found."),
        ctx: { projectId: "p1" },
      });

    boom();
    expect(count()).toBe(1);

    t += DEDUP_WINDOW_MS - 1; // Still within the window: a burst of 404s from a scan discards straight away, no persist
    boom();
    boom();
    expect(count()).toBe(1);

    t += 1; // Outside the window: the same kind of error is recorded again (a sustained storm leaves exactly one entry per window, never suppressed forever)
    boom();
    expect(count()).toBe(2);
  });

  it("dedup never crosses source / code / Project (kinds don't suppress each other)", () => {
    const rec = wire(ErrorRecorder, { errors: repo, clock: { now: now } }); // time frozen: everything lands in the same window
    const err = new Error("boom");
    rec.record({ source: "http", err, ctx: { projectId: "p1" }, code: "c1" });
    rec.record({ source: "http", err, ctx: { projectId: "p1" }, code: "c1" }); // same kind: discarded
    rec.record({ source: "http", err, ctx: { projectId: "p1" }, code: "c2" }); // different code
    rec.record({ source: "http", err, ctx: { projectId: "p2" }, code: "c1" }); // different Project
    rec.record({ source: "session", err, ctx: { projectId: "p1" }, code: "c1" }); // different source
    rec.record({ source: "http", err, code: "c1" }); // unattributed (project_id is NULL): counts as its own kind
    expect(count()).toBe(5);
  });

  it("keeps recording and deduplicating after its table fills, expires and is wiped", () => {
    // The table is bounded: past DEDUP_KEYS_MAX keys it drops the expired ones, or all of them
    // when none has expired. Either way no distinct error may be lost and a burst of one error
    // must still collapse to one row.
    let t = Date.parse("2026-07-06T10:00:00Z");
    const rec = wire(ErrorRecorder, { errors: repo, clock: { now: () => new Date(t) } });
    const boom = (code: string) =>
      rec.record({ source: "http", err: "boom", ctx: { projectId: "p1" }, code });

    for (let i = 0; i < DEDUP_KEYS_MAX; i++) boom(`c${i}`); // one key per code: the table is full
    expect(count()).toBe(DEDUP_KEYS_MAX);

    t += DEDUP_WINDOW_MS; // every key has expired, so the next one cleans them out
    boom("after-window");
    boom("after-window");
    expect(count()).toBe(DEDUP_KEYS_MAX + 1);

    for (let i = 0; i < DEDUP_KEYS_MAX; i++) boom(`d${i}`); // one window, nothing expired: wiped
    expect(count()).toBe(2 * DEDUP_KEYS_MAX + 1);

    boom("tail");
    boom("tail");
    expect(count()).toBe(2 * DEDUP_KEYS_MAX + 2);
  });
});

describe("stream-error-watcher (LLM / Environment errors)", () => {
  let db: DatabaseSync;
  let repo: ErrorsRepo;
  const now = () => new Date("2026-07-06T10:00:00");
  const CTX = { projectId: "p1", agentId: "a1", sessionId: "s1" };

  beforeEach(() => {
    db = openDatabase(":memory:");
    repo = wire(ErrorsRepo, { db: db });
  });
  afterEach(() => db.close());

  const watcher = () =>
    new StreamErrorWatcher(wire(ErrorRecorder, { errors: repo, clock: { now: now } }), CTX);
  const rows = () =>
    db.prepare("SELECT * FROM error_records ORDER BY id").all() as Array<Record<string, unknown>>;

  /** A legacy stream's abort: prose reason only (cores from before the unified error pair). */
  function legacyAbort(reason: string): OmniMessage {
    const msg = abortEvent();
    delete (msg.payload as { error_code?: string }).error_code;
    (msg.payload as { reason?: string }).reason = reason;
    return msg;
  }

  /** Feeds a sequence of messages and finalizes (close: persists any still-pending failure), returning the persisted rows. */
  function feed(msgs: OmniMessage[]): Array<Record<string, unknown>> {
    const w = watcher();
    for (const m of msgs) w.observe(m);
    w.close();
    return rows();
  }

  /**
   * A sub-session's session_meta (its first message): origin = the child Session
   * id, and agentId is derived from the parent directory name in the `agent_state`
   * path (consistent with SessionManager.registerChildSession).
   */
  const childMeta = (sessionId: string, agentState: string) =>
    withOrigin(
      sessionMeta({
        session_id: sessionId,
        model_id: "m1",
        provider: "custom",
        model_context_window: 100000,
        system_prompt: "",
        agent_state: agentState,
        workspace: "/tmp/w",
        source: "user",
      }),
      sessionId,
    );

  // —— LLM ——

  it("an unrecovered LLM failed → unexpected (needs a human); message takes the abort reason", () => {
    // Nothing follows this failure but the abort, so the ladder did not carry it: the user
    // lost the turn and it belongs in front of an operator.
    const got = feed([
      requestBegin(),
      requestEnd("retryable"),
      legacyAbort("llm request failed after 5 retries: 400 unknown parameter"),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({
      source: "llm",
      kind: "unexpected",
      code: "llm_failed",
      message: "llm request failed after 5 retries: 400 unknown parameter",
      project_id: "p1",
      agent_id: "a1",
      session_id: "s1",
      status: null,
    });
  });

  it("a retried retryable → expected; an exhausted one → unexpected; messages from abort/fallback", () => {
    const got = feed([
      requestBegin(),
      requestEnd("retryable"), // First attempt fails → the engine retries (revealed by the next request_begin: no reason text yet)
      requestBegin(),
      requestEnd("retryable"),
      legacyAbort("llm request failed after 2 retries"),
    ]);
    expect(got).toHaveLength(2);
    expect(got[0]).toMatchObject({ source: "llm", kind: "expected", code: "llm_retried" });
    // No abort arrived for the carried failure: falls back to the generic status text.
    expect(got[0]!.message).toContain("reconnects and retries");
    expect(got[1]).toMatchObject({
      source: "llm",
      kind: "unexpected",
      code: "llm_failed",
      message: "llm request failed after 2 retries",
    });
  });

  it("request_end(fatal) gets its own llm_fatal code, out of the retried dedup bucket", () => {
    // A run-ending rejection needs a code of its own: dedup is (source, code, Project)
    // over a short window, and `llm_retried` fires on every blip the ladder absorbs —
    // sharing a bucket would let a real credential failure be dropped as a duplicate.
    const got = feed([
      requestBegin(),
      requestEnd("fatal", { errorMessage: "401 invalid x-api-key (invalid_api_key)" }),
      legacyAbort("llm request error: 401 invalid x-api-key (invalid_api_key)"),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({
      source: "llm",
      kind: "unexpected",
      code: "llm_fatal",
      message: "llm request error: 401 invalid x-api-key (invalid_api_key)",
    });
  });

  it("a fatal with no abort (the live protocol): close persists it with the same prose", () => {
    // Failures no longer emit an abort event; the pending record resolves at close, and
    // the message is composed from the request_end's own detail.
    const got = feed([
      requestBegin(),
      requestEnd("fatal", { errorMessage: "401 invalid x-api-key (invalid_api_key)" }),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({
      source: "llm",
      kind: "unexpected",
      code: "llm_fatal",
      message: "llm request error: 401 invalid x-api-key (invalid_api_key)",
    });
  });

  it("an exhausted ladder with no abort: the terminal request_end's attempt shapes the record", () => {
    const got = feed([
      requestBegin(),
      requestEnd("retryable", { errorMessage: "socket hang up", attempt: 1, retryInMs: 2000 }),
      requestBegin(),
      // The final failure plans no retry (no retry_in_ms) — the run ends on it.
      requestEnd("retryable", { errorMessage: "socket hang up", attempt: 2 }),
    ]);
    expect(got).toHaveLength(2);
    expect(got[0]).toMatchObject({ code: "llm_retried", kind: "expected" });
    expect(got[1]).toMatchObject({
      source: "llm",
      kind: "unexpected",
      code: "llm_failed",
      message: "llm request failed after 1 retry: socket hang up",
    });
  });

  it("a retryable the ladder carried → expected under its own code, not an operator incident", () => {
    // The same status covers "a gateway hiccup the user never saw" and "the run died on
    // it". A following request_begin proves another attempt happened — that one is
    // expected.
    const got = feed([
      requestBegin(),
      requestEnd("retryable", {
        errorMessage: "Upstream HTTP/2 stream failed (upstream_http2_stream_error)",
      }),
      requestBegin(),
      requestEnd("completed"),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({
      source: "llm",
      kind: "expected",
      code: "llm_retried",
      // No abort ever arrives on the retry path: the staged request_end's own detail is the
      // message of record.
      message: "Upstream HTTP/2 stream failed (upstream_http2_stream_error)",
    });
  });

  it("a recovered retryable does not dedup away a fatal that lands right after", () => {
    // The two share a 2s dedup window; separate codes keep the one failure that always
    // needs a human from being silenced by the one that never does.
    const got = feed([
      requestBegin(),
      requestEnd("retryable", { errorMessage: "Upstream HTTP/2 stream failed" }),
      requestBegin(), // The retry: resolves the failure above as recovered.
      requestEnd("fatal", { errorMessage: "401 invalid x-api-key" }),
      legacyAbort("llm request error: 401 invalid x-api-key"),
    ]);
    expect(got.map((r) => [r.code, r.kind])).toEqual([
      ["llm_retried", "expected"],
      ["llm_fatal", "unexpected"],
    ]);
  });

  it("interrupt during backoff with a staged detail: the detail wins over the status text", () => {
    // The interrupt message is distrusted (not the failure's reason), but the staged
    // request_end detail IS the failure's reason — prefer it over the generic status text.
    const got = feed([
      requestBegin(),
      // A backoff interrupt cuts a PLANNED retry short — the failure announced one.
      requestEnd("retryable", { errorMessage: "429 rate limited (slow down)", retryInMs: 4000 }),
      abortEvent("backoff_interrupted"),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]!.message).toBe("429 rate limited (slow down)");
  });

  it("aborted (user clicked Stop) is not an error: not recorded; neither is completed", () => {
    expect(
      feed([
        requestBegin(),
        requestEnd("completed"),
        requestBegin(),
        requestEnd("aborted"),
        abortEvent(),
      ]),
    ).toHaveLength(0);
  });

  it("interrupt during retry backoff: the failure still recorded, interrupt text distrusted", () => {
    const got = feed([requestBegin(), requestEnd("retryable"), abortEvent("backoff_interrupted")]);
    expect(got).toHaveLength(1);
    // No retry followed (the interrupt ended the run): the conservative branch records it
    // as the unrecovered class.
    expect(got[0]).toMatchObject({ code: "llm_failed", kind: "unexpected" });
    expect(got[0]!.message).toContain("did not recover");
    expect(got[0]!.message).not.toContain("aborted");
  });

  it("failure pends for its reason; unresolved at run end → close persists (status text)", () => {
    const w = watcher();
    w.observe(requestBegin());
    w.observe(requestEnd("retryable"));
    expect(rows()).toHaveLength(0); // Pending: waiting for the abort that immediately follows to supply the real reason
    w.close();
    const got = rows();
    expect(got).toHaveLength(1);
    // close() is not proof of a retry, so it takes the conservative branch: unrecovered.
    expect(got[0]).toMatchObject({ code: "llm_failed", kind: "unexpected" });
    expect(got[0]!.message).toBe("LLM request failed and the retries did not recover it.");
  });

  it("parent/child LLM failures pend separately by origin; abort reasons never cross over", () => {
    // The child fails fatal and the parent retryable-unrecovered: distinct codes, so the
    // short-window dedup (source, code, Project) suppresses neither.
    const got = feed([
      requestBegin(), // parent session initiates
      withOrigin(requestBegin(), "session-child"),
      withOrigin(requestEnd("fatal"), "session-child"),
      withOrigin(legacyAbort("llm request error: 401 invalid api key"), "session-child"),
      requestEnd("retryable"), // the parent session's failure only wraps up now
      legacyAbort("llm request error: 500 upstream"),
    ]);
    expect(got).toHaveLength(2);
    expect(got[0]).toMatchObject({
      code: "llm_fatal",
      message: "llm request error: 401 invalid api key",
    });
    expect(got[1]).toMatchObject({
      code: "llm_failed",
      message: "llm request error: 500 upstream", // not stolen by the sub-session's abort
    });
  });

  // —— Environment (tool execution) ——

  const call = (name: string, id: string) => toolCall({ name, arguments: "{}", toolCallId: id });

  it("a command tool's ordinary non-zero exit is not recorded: an exit status is information", () => {
    // Both command tools end in resultForExit, which maps ANY non-zero exit to `failed`, so
    // grep finding nothing (exit 1), `test -f` on a missing file, or a diff that differs would
    // all land in the cost center and bury the real errors. input_command is covered alongside
    // exec_command because it is how a backgrounded command is polled for its eventual exit —
    // dropping one but not the other would depend on where the command happened to finish.
    // Every other tool still records, whatever its output says.
    const got = feed([
      call("exec_command", "tc-1"),
      toolCallOutput({
        output: "grep: no match\n[exit code: 1]",
        toolCallId: "tc-1",
        stopReason: "fatal",
      }),
      call("input_command", "tc-2"),
      toolCallOutput({
        output: "make: *** [build] Error 2\n[exit code: 2]",
        toolCallId: "tc-2",
        stopReason: "fatal",
      }),
      call("write_file", "tc-3"),
      toolCallOutput({
        output: "EACCES\n[exit code: 1]",
        toolCallId: "tc-3",
        stopReason: "fatal",
      }),
    ]);
    expect(got.map((r) => r.code)).toEqual(["tool_fatal:write_file"]);
  });

  it("a command tool killed by a signal, or that never spawned, is still recorded", () => {
    // `failed` from these tools is not only "exited non-zero": an OOM kill or a segfault
    // (resultForExit's signal branch) and a spawn failure (nonexistent workdir, EMFILE, an
    // unresolvable shell) are config/environment faults — the recorder's "needs a human"
    // category — that no amount of Agent self-correction gets around. Only the exit marker
    // separates them, which is why the rule reads the note rather than the tool name.
    const got = feed([
      call("exec_command", "tc-1"),
      toolCallOutput({
        output: "cc1plus: out of memory\n[terminated by signal SIGKILL]",
        toolCallId: "tc-1",
        stopReason: "fatal",
      }),
      call("input_command", "tc-2"),
      toolCallOutput({
        output: "[spawn error: ENOENT: no such file or directory, posix_spawn '/bin/nope']",
        toolCallId: "tc-2",
        stopReason: "fatal",
      }),
    ]);
    expect(got.map((r) => r.code)).toEqual(["tool_fatal:exec_command", "tool_fatal:input_command"]);
  });

  it("a command tool's timeout and a missing session manager are recorded (neither is an exit)", () => {
    // Environment finalizes a tool timeout as stop_reason `failed` plus its own note — it never
    // emits stop_reason "timeout" for these tools — so a hung command surfaces as tool_failed
    // with no exit marker to drop it. A missing command session manager is a server
    // misconfiguration and produces no exit marker either.
    const got = feed([
      call("exec_command", "tc-1"),
      toolCallOutput({
        output: "still building…\n[tool timeout: exceeded 60000ms]",
        toolCallId: "tc-1",
        stopReason: "fatal",
      }),
      call("input_command", "tc-2"),
      toolCallOutput({
        output: "[input_command unavailable: no command session manager configured]",
        toolCallId: "tc-2",
        stopReason: "fatal",
      }),
    ]);
    expect(got.map((r) => r.code)).toEqual(["tool_fatal:exec_command", "tool_fatal:input_command"]);
  });

  it("an exit marker with no cached tool name is dropped, not filed under tool_fatal:unknown", () => {
    // Only the command tools ever write that marker, so a cache miss (the tool_call evicted, or
    // never seen) is still that noise — recording it nameless would defeat the exclusion.
    const got = feed([
      toolCallOutput({ output: "[exit code: 1]", toolCallId: "tc-1", stopReason: "fatal" }),
      toolCallOutput({ output: "boom", toolCallId: "tc-2", stopReason: "fatal" }),
    ]);
    expect(got.map((r) => [r.code, r.message])).toEqual([["tool_fatal:unknown", "boom"]]);
  });

  it("a tool fatal → environment + expected, code carries the tool name; legacy failed/timeout spellings still record", () => {
    const legacyOutput = (args: { output: string; toolCallId: string; stopReason: string }) =>
      toolCallOutput(args as Parameters<typeof toolCallOutput>[0]);
    const got = feed([
      call("write_file", "tc-1"),
      toolCallOutput({
        output: "EACCES: permission denied\n[tool error] write failed",
        toolCallId: "tc-1",
        stopReason: "fatal",
      }),
      // Traces written before the stop-reason unification spell tool failures failed/timeout.
      call("read_file", "tc-2"),
      legacyOutput({
        output: "[tool timeout: exceeded 30000ms]",
        toolCallId: "tc-2",
        stopReason: "timeout",
      }),
      call("edit_file", "tc-3"),
      legacyOutput({ output: "[tool error] boom", toolCallId: "tc-3", stopReason: "failed" }),
    ]);
    expect(got).toHaveLength(3);
    expect(got[0]).toMatchObject({
      source: "environment",
      kind: "expected", // error fed back to the model; the Agent adjusts on its own — no human needed
      code: "tool_fatal:write_file",
      project_id: "p1",
      agent_id: "a1",
      session_id: "s1",
    });
    expect(got[0]!.message).toContain("[tool error] write failed"); // the actual error text
    expect(got[1]).toMatchObject({ code: "tool_timeout:read_file", kind: "expected" });
    expect(got[2]).toMatchObject({ code: "tool_failed:edit_file", kind: "expected" });
  });

  it("tool aborted (denial / user interrupt) and completed are not recorded", () => {
    expect(
      feed([
        call("write_file", "tc-1"),
        toolCallOutput({
          output: "Tool call denied by user.",
          toolCallId: "tc-1",
          stopReason: "aborted",
        }),
        call("read_file", "tc-2"),
        toolCallOutput({ output: "ok", toolCallId: "tc-2", stopReason: "completed" }),
      ]),
    ).toHaveLength(0);
  });

  it("parallel tools: each tool_call_id maps to its own name despite out-of-order outputs", () => {
    const got = feed([
      call("write_file", "tc-1"),
      call("read_file", "tc-2"),
      call("write_file", "tc-3"),
      toolCallOutput({ output: "boom-2", toolCallId: "tc-2", stopReason: "fatal" }),
      toolCallOutput({ output: "ok", toolCallId: "tc-3", stopReason: "completed" }),
      toolCallOutput({ output: "boom-1", toolCallId: "tc-1", stopReason: "fatal" }),
    ]);
    expect(got.map((r) => r.code)).toEqual(["tool_fatal:read_file", "tool_fatal:write_file"]);
    expect(got.map((r) => r.message)).toEqual(["boom-2", "boom-1"]);
  });

  it("a child session's tool failure: no name mix-up with the parent's equal tool_call_id", () => {
    const got = feed([
      call("read_file", "tc-1"), // parent session
      withOrigin(call("write_file", "tc-1"), "session-child"), // sub-session happens to share the same id
      withOrigin(
        toolCallOutput({ output: "child boom", toolCallId: "tc-1", stopReason: "fatal" }),
        "session-child",
      ),
      toolCallOutput({ output: "parent boom", toolCallId: "tc-1", stopReason: "fatal" }),
    ]);
    expect(got).toHaveLength(2);
    expect(got[0]).toMatchObject({
      code: "tool_fatal:write_file", // the sub-session's tool name, not overwritten by the parent's tc-1
      message: "child boom",
      session_id: "s1", // this test didn't feed the sub-session's session_meta → attribution falls back to the parent ctx (see the "attribution" test cases below)
    });
    expect(got[1]).toMatchObject({ code: "tool_fatal:read_file", message: "parent boom" });
  });

  it("overlong tool output: message takes the tail (the reason is at the end)", () => {
    const got = feed([
      call("write_file", "tc-1"),
      toolCallOutput({
        output: `${"x".repeat(2000)}\n[tool error] boom`,
        toolCallId: "tc-1",
        stopReason: "fatal",
      }),
    ]);
    const message = got[0]!.message as string;
    expect(message.length).toBe(MESSAGE_MAX);
    expect(message.startsWith("…")).toBe(true);
    expect(message.endsWith("[tool error] boom")).toBe(true); // truncating from the head would cut off the reason entirely
  });

  it("irrelevant messages are a no-op: body text and streaming partial_*", () => {
    expect(
      feed([
        assistantText("normal output"),
        call("write_file", "tc-1"),
        partialToolCallOutput({ eventType: "stop", toolCallId: "tc-1", stopReason: "fatal" }),
      ]),
    ).toHaveLength(0);
  });

  // —— Attribution: an error is recorded against **the session that actually produced it** (a sub-session's failure must not be attributed to the parent Agent) ——

  it("a child session's LLM failure attributes to the child Agent / Session", () => {
    const got = feed([
      childMeta("session-child", "/data/agents/agent-child/agent_state"),
      withOrigin(requestBegin(), "session-child"),
      withOrigin(requestEnd("retryable"), "session-child"),
      withOrigin(legacyAbort("llm request error: 401 invalid api key"), "session-child"),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({
      source: "llm",
      code: "llm_failed",
      message: "llm request error: 401 invalid api key",
      agent_id: "agent-child", // derived from the agent_state path; not the parent's a1
      session_id: "session-child",
      project_id: "p1", // projectId always takes the parent's (a sub-session is always in the same Project)
    });
  });

  it("a child session's tool failure attributes to it (code still carries the tool name)", () => {
    const got = feed([
      childMeta("session-child", "/data/agents/agent-child/agent_state"),
      withOrigin(call("write_file", "tc-1"), "session-child"),
      withOrigin(
        toolCallOutput({ output: "child boom", toolCallId: "tc-1", stopReason: "fatal" }),
        "session-child",
      ),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({
      source: "environment",
      code: "tool_fatal:write_file",
      message: "child boom",
      agent_id: "agent-child",
      session_id: "session-child",
      project_id: "p1",
    });
  });

  it("parent/child interleaving: each attributes to its own; no-origin goes to the parent", () => {
    const got = feed([
      requestBegin(), // parent session initiates
      childMeta("session-child", "/data/agents/agent-child/agent_state"),
      withOrigin(requestBegin(), "session-child"),
      withOrigin(requestEnd("fatal"), "session-child"),
      withOrigin(legacyAbort("llm request error: 401 dead key"), "session-child"),
      withOrigin(call("write_file", "tc-9"), "session-child"),
      withOrigin(
        toolCallOutput({ output: "child tool boom", toolCallId: "tc-9", stopReason: "fatal" }),
        "session-child",
      ),
      call("read_file", "tc-9"), // parent session happens to share the same id
      toolCallOutput({ output: "parent tool boom", toolCallId: "tc-9", stopReason: "fatal" }),
      requestEnd("retryable"), // the parent session's LLM failure only wraps up now
      legacyAbort("llm request error: 500 upstream"),
    ]);
    // The sub-session's LLM / tool failures attribute to it, the parent's to the parent — the four entries never mix (each has a distinct code, so short-window dedup doesn't suppress any of them).
    expect(got.map((r) => [r.code, r.agent_id, r.session_id])).toEqual([
      ["llm_fatal", "agent-child", "session-child"],
      ["tool_fatal:write_file", "agent-child", "session-child"],
      ["tool_fatal:read_file", "a1", "s1"],
      ["llm_failed", "a1", "s1"],
    ]);
  });

  it("failure before session_meta arrives: falls back to the parent ctx, no crash", () => {
    const got = feed([
      withOrigin(requestEnd("retryable"), "session-child"), // the sub-session's meta hasn't arrived yet
      withOrigin(legacyAbort("llm request error: 500"), "session-child"),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({ code: "llm_failed", agent_id: "a1", session_id: "s1" });
  });

  it("malformed agent_state path (empty): not registered, falls back to the parent ctx", () => {
    const got = feed([
      childMeta("session-child", ""), // path.basename(path.dirname("")) === "." → caught by the defensive check
      withOrigin(requestEnd("retryable"), "session-child"),
      withOrigin(legacyAbort("llm request error: 500"), "session-child"),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({ code: "llm_failed", agent_id: "a1", session_id: "s1" });
  });

  // —— Compaction ——

  it("an abandoned compaction records one error row; the message carries attempts and the error", () => {
    // Issue #170: a compaction is an ordinary LLM request whose failures core retries under
    // the standard budget — a retryable end means the retries ran out, and the cost
    // center's row shows how many attempts were spent and what the last failure was.
    const got = feed([
      compactionEnd({
        reason: "context",
        mode: "summarize",
        status: "retryable",
        attempt: 5,
        errorMessage: "the response contained no usable summary",
      }),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({
      source: "compaction",
      kind: "unexpected",
      code: "compaction_failed",
      message:
        "summarize compaction failed after 5 attempts: the response contained no usable summary; trigger context, original context kept.",
      project_id: "p1",
      agent_id: "a1",
      session_id: "s1",
    });
  });

  it("compaction completed / aborted are not errors; fatal and an old-core failed still record", () => {
    const legacyEnd = (status: string) =>
      compactionEnd({ reason: "turns", mode: "summarize", status: status as StopReason });
    const got = feed([
      compactionEnd({ reason: "context", mode: "summarize", status: "completed", attempt: 1 }),
      compactionEnd({ reason: "manual", mode: "summarize", status: "aborted", attempt: 2 }),
      compactionEnd({ reason: "context", mode: "summarize", status: "fatal", attempt: 1 }),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({ code: "compaction_failed" });
    // An old core's compaction_end (retired `failed` spelling, no attempt/error fields)
    // still records — its own feed, since the short-window dedup shares the code above.
    const legacy = feed([legacyEnd("failed")]);
    expect(legacy).toHaveLength(2); // rows are cumulative: the fatal row above plus this one
    expect(legacy[1]).toMatchObject({
      code: "compaction_failed",
      message: "summarize compaction failed; trigger turns, original context kept.",
    });
  });

  it("a child session's failed compaction attributes to the child Agent/Session", () => {
    const got = feed([
      childMeta("session-child", "/data/agents/agent-child/agent_state"),
      withOrigin(
        compactionEnd({ reason: "context", mode: "summarize", status: "retryable", attempt: 6 }),
        "session-child",
      ),
    ]);
    expect(got).toHaveLength(1);
    expect(got[0]).toMatchObject({
      code: "compaction_failed",
      agent_id: "agent-child",
      session_id: "session-child",
    });
  });
});

describe("HTTP onError persistence (integration)", () => {
  let t: TestApp;
  let api: ReturnType<typeof apiClient>;
  let projectId: string;

  beforeEach(async () => {
    t = await createTestApp();
    const u = await provisionUser(t.app, "err_user");
    api = apiClient(t.app, u.cookie);
    const created = (await (
      await api.post("/api/projects", { projectId: "err_user-proj", name: "Error project" })
    ).json()) as ProjectCreateResponse;
    projectId = created.project.projectId;
  });
  afterEach(async () => {
    await t.cleanup();
  });

  const errorRows = () =>
    t.deps.db.prepare("SELECT * FROM error_records ORDER BY id").all() as Array<
      Record<string, unknown>
    >;

  it("business error (HttpError 404) → expected, with code / status / projectId", async () => {
    const res = await api.get(`/api/projects/${projectId}/agents/agent-nope/sessions`);
    expect(res.status).toBe(404);

    const rows = errorRows();
    expect(rows).toHaveLength(1);
    expect(rows[0]!.source).toBe("http");
    expect(rows[0]!.kind).toBe("expected");
    expect(rows[0]!.code).toBe("agent_not_found");
    expect(rows[0]!.status).toBe(404);
    // Taken from the route params, but only when the requester actually has access — see the "HTTP error attribution" test group below.
    expect(rows[0]!.project_id).toBe(projectId);
  });

  it("unexpected error (service layer throws a plain Error) → unexpected + 500", async () => {
    // handleError logs the stack trace: silence it in the test so it doesn't clutter output.
    const spy = vi.spyOn(console, "error").mockImplementation(() => {});
    t.deps.usageService.query = () => {
      throw new Error("query blew up");
    };
    const res = await api.get(`/api/projects/${projectId}/usage?groupBy=date`);
    expect(res.status).toBe(500);
    spy.mockRestore();

    const rows = errorRows();
    expect(rows).toHaveLength(1);
    expect(rows[0]!.kind).toBe("unexpected");
    expect(rows[0]!.code).toBe("internal");
    expect(rows[0]!.status).toBe(500);
    expect(rows[0]!.message).toBe("query blew up");
    expect(rows[0]!.project_id).toBe(projectId);
  });

  it("a Skill directory another program is holding open → 409 skill_dir_busy, recorded as expected", async () => {
    const agentId = "agent-busy";
    const dir = path.join(skillsDir(t.root, projectId, agentId), "busy-skill");
    await fs.mkdir(dir, { recursive: true });
    await fs.writeFile(path.join(dir, "SKILL.md"), "---\nname: busy-skill\n---\nBody\n");

    // Windows reports EBUSY ("resource busy or locked") on every removal while another program
    // still holds a handle inside the directory; Core retries and eventually gives up. The answer
    // must name what is wrong (409), not converge to a bare 500, and the error center must file it
    // where the user can act on it rather than as an internal bug.
    const realRm = fs.rm.bind(fs);
    const rm = vi.spyOn(fs, "rm").mockImplementation(async (target, options) => {
      if (path.resolve(String(target)) === path.resolve(dir)) {
        throw Object.assign(
          new Error(`EBUSY: resource busy or locked, rmdir '${String(target)}'`),
          { code: "EBUSY" },
        );
      }
      return realRm(target, options);
    });
    const res = await api.delete(`/api/projects/${projectId}/agents/${agentId}/skills/busy-skill`);
    rm.mockRestore();

    expect(res.status).toBe(409);
    expect(await res.json()).toMatchObject({ error: { code: "skill_dir_busy" } });

    const rows = errorRows();
    expect(rows).toHaveLength(1);
    expect(rows[0]!.source).toBe("http");
    expect(rows[0]!.kind).toBe("expected");
    expect(rows[0]!.code).toBe("skill_dir_busy");
    expect(rows[0]!.status).toBe(409);
    expect(rows[0]!.project_id).toBe(projectId);
  });

  it("errors exposed via the usage endpoint: summary / top code / recent", async () => {
    // An error in another Project owned by the same owner: attributed to that Project, not this one's view.
    const other = (await (
      await api.post("/api/projects", { projectId: "err_user-proj_2", name: "Another project" })
    ).json()) as ProjectCreateResponse;
    await api.get(`/api/projects/${other.project.projectId}/agents/agent-nope/sessions`); // 404
    await api.get(`/api/projects/${projectId}/agents/agent-nope/sessions`); // 404 → expected

    const res = await api.get(`/api/projects/${projectId}/usage?groupBy=date`);
    const body = (await res.json()) as UsageResponse;
    // The entry from another Project doesn't count in this view (only this Project's errors + admin-visible unattributed errors show here).
    expect(body.errors.total).toBe(1);
    expect(body.errors.unexpected).toBe(0);
    expect(body.errors.topCode).toEqual({
      source: "http",
      code: "agent_not_found",
      kind: "expected",
      count: 1,
    });
    expect(body.errors.recent[0]).toMatchObject({ source: "http", code: "agent_not_found" });
  });

  it("unattributed errors (login failure) are admin-only, hidden from members", async () => {
    // A login failure has no Project context → produces one unattributed error (project_id is NULL).
    const bad = await t.app.request("/api/auth/login", {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ userId: "err_user", password: "wrong-password" }),
    });
    expect(bad.status).toBe(401);
    expect(errorRows().filter((r) => r.project_id === null)).toHaveLength(1);

    // A regular member in their own Project: sees none of it.
    const plain = await provisionUser(t.app, "plain_user");
    expect(plain.user.isAdmin).toBe(false);
    const plainApi = apiClient(t.app, plain.cookie);
    const own = (await (
      await plainApi.post("/api/projects", { projectId: "plain_user-proj", name: "Member project" })
    ).json()) as ProjectCreateResponse;
    const plainBody = (await (
      await plainApi.get(`/api/projects/${own.project.projectId}/usage?groupBy=date`)
    ).json()) as UsageResponse;
    expect(plainBody.errors).toMatchObject({ total: 0, unexpected: 0, topCode: null, recent: [] });

    // The admin can see it — the category most in need of visibility isn't rendered invisible by the isolation.
    const adminApi = apiClient(t.app, (await loginAdmin(t.app)).cookie);
    const adminBody = (await (
      await adminApi.get(`/api/projects/default_project/usage?groupBy=date`)
    ).json()) as UsageResponse;
    expect(adminBody.errors.total).toBe(1);
    expect(adminBody.errors.topCode).toMatchObject({ source: "http", code: "invalid_credentials" });
    expect(adminBody.errors.recent[0]).toMatchObject({ code: "invalid_credentials" });
  });

  it("the paged error route pages inside the caller's own tenant, at every offset", async () => {
    // The point of the route is that paging back is not a way around the dashboard's isolation:
    // the rows a member can reach at offset N are the same set the summary counted, never
    // another tenant's and never the unattributed ones. Seeded directly so the interleaving is
    // exact — going through HTTP would collapse repeats into the recorder's dedup window.
    const repo = wire(ErrorsRepo, { db: t.deps.db });
    const seed = (owner: string | null, code: string) =>
      repo.insert({
        ts: "2026-07-27T00:00:00.000Z",
        date: "2026-07-27",
        projectId: owner,
        agentId: "a1",
        sessionId: "s1",
        source: "http",
        kind: "expected",
        code,
        status: 404,
        message: code,
      });
    const foreign = (await (
      await api.post("/api/projects", { projectId: "err_user-tenant_2", name: "Other tenant" })
    ).json()) as ProjectCreateResponse;
    // Interleaved, so a mistaken filter would show up inside the first page rather than past it.
    seed(projectId, "mine_0");
    seed(foreign.project.projectId, "theirs_0");
    seed(null, "unattributed_0");
    seed(projectId, "mine_1");
    seed(projectId, "mine_2");

    const pageOf = async (offset: number, limit: number) =>
      (await (
        await api.get(`/api/projects/${projectId}/usage/errors?offset=${offset}&limit=${limit}`)
      ).json()) as UsageErrorsPage;

    const first = await pageOf(0, 2);
    expect(first.total).toBe(3); // the filtered count, not the table's
    expect(first.items.map((e) => e.code)).toEqual(["mine_2", "mine_1"]); // newest first
    const second = await pageOf(2, 2);
    expect(second.total).toBe(3);
    expect(second.items.map((e) => e.code)).toEqual(["mine_0"]);
    expect((await pageOf(3, 2)).items).toEqual([]); // past the end is empty, not an error

    // The admin is the only one who reaches the unattributed rows — the same rule the dashboard
    // applies, so paging cannot be used to slip past it.
    const adminApi = apiClient(t.app, (await loginAdmin(t.app)).cookie);
    const adminPage = (await (
      await adminApi.get(`/api/projects/default_project/usage/errors?offset=0&limit=20`)
    ).json()) as UsageErrorsPage;
    expect(adminPage.items.map((e) => e.code)).toEqual(["unattributed_0"]);
  });

  it("the paged error route rejects a page it cannot serve rather than guessing", async () => {
    expect((await api.get(`/api/projects/${projectId}/usage/errors?offset=-1`)).status).toBe(400);
    expect((await api.get(`/api/projects/${projectId}/usage/errors?limit=0`)).status).toBe(400);
    expect((await api.get(`/api/projects/${projectId}/usage/errors?limit=1001`)).status).toBe(400);
    expect((await api.get(`/api/projects/${projectId}/usage/errors?from=2026-13-01`)).status).toBe(
      400,
    );
    // A `kind` outside the two categories is a bad request, not a silently empty page: the
    // cost-center badge reads `total` as "is anything waiting", and a typo answering 0 would
    // read as "nothing is".
    expect((await api.get(`/api/projects/${projectId}/usage/errors?kind=oops`)).status).toBe(400);
  });

  it("the paged error route narrows to one category on request, counting only that one", async () => {
    // What the cost-center badge asks: one row of `unexpected`, for the count and the newest
    // timestamp its dismissal is stamped against. Seeded directly, so the interleaving is exact.
    const repo = wire(ErrorsRepo, { db: t.deps.db });
    const seed = (kind: string, code: string, ts: string) =>
      repo.insert({
        ts,
        date: ts.slice(0, 10),
        projectId,
        agentId: "a1",
        sessionId: "s1",
        source: "http",
        kind,
        code,
        status: kind === "unexpected" ? 500 : 404,
        message: code,
      });
    seed("expected", "expected_0", "2026-07-27T00:00:00.000Z");
    seed("unexpected", "unexpected_0", "2026-07-27T01:00:00.000Z");
    seed("expected", "expected_1", "2026-07-27T02:00:00.000Z");
    seed("unexpected", "unexpected_1", "2026-07-27T03:00:00.000Z");

    const probe = (await (
      await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=1&kind=unexpected`)
    ).json()) as UsageErrorsPage;
    expect(probe.total).toBe(2); // only the unexpected ones are counted
    expect(probe.items.map((e) => e.code)).toEqual(["unexpected_1"]); // newest first, one row

    const expectedOnly = (await (
      await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20&kind=expected`)
    ).json()) as UsageErrorsPage;
    expect(expectedOnly.items.map((e) => e.code)).toEqual(["expected_1", "expected_0"]);

    // No `kind` still counts both, so the panel the badge leads to is unchanged.
    const all = (await (
      await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20`)
    ).json()) as UsageErrorsPage;
    expect(all.total).toBe(4);
  });

  it("the paged error route and the dashboard fold a day's repeats by the viewer's day", async () => {
    // Three of the same send failure: two on the evening of the 27th in UTC and one just after
    // UTC midnight — the same evening for a reader five hours behind.
    const repo = wire(ErrorsRepo, { db: t.deps.db });
    for (const ts of [
      "2026-07-27T20:00:00.000Z",
      "2026-07-27T22:00:00.000Z",
      "2026-07-28T01:00:00.000Z",
    ]) {
      repo.insert({
        ts,
        date: ts.slice(0, 10),
        projectId,
        agentId: "a1",
        sessionId: "s1",
        source: "messaging",
        kind: "expected",
        code: "messaging_send_failed",
        status: null,
        message: "wechat send failed: prepare failed",
      });
    }
    const pageFor = async (query: string) =>
      (await (
        await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20${query}`)
      ).json()) as UsageErrorsPage;

    const eastern = await pageFor("&utcOffsetMinutes=-300");
    expect(eastern.items).toHaveLength(1);
    expect(eastern.items[0]).toMatchObject({
      count: 3,
      ts: "2026-07-28T01:00:00.000Z",
      firstTs: "2026-07-27T20:00:00.000Z",
    });
    // Rows are what a pager pages through; the records behind them are still the total.
    expect(eastern.rows).toBe(1);
    expect(eastern.total).toBe(3);

    // Without an offset the day is the server's, the `date` column: two days, two rows.
    const server = await pageFor("");
    expect(server.items.map((e) => e.count)).toEqual([1, 2]);
    expect(server.rows).toBe(2);

    // The dashboard's first page folds the same way, and its summary keeps counting records.
    const dashboard = (await (
      await api.get(`/api/projects/${projectId}/usage?utcOffsetMinutes=-300`)
    ).json()) as UsageResponse;
    expect(dashboard.errors.recent.map((e) => e.count)).toEqual([3]);
    expect(dashboard.errors.rows).toBe(1);
    expect(dashboard.errors.total).toBe(3);

    // An offset no clock uses, or one that is not a whole number, is a 400 rather than a
    // quietly different day.
    for (const bad of ["900", "-841", "1.5", "east", "+60"]) {
      expect(
        (await api.get(`/api/projects/${projectId}/usage/errors?utcOffsetMinutes=${bad}`)).status,
      ).toBe(400);
    }
    expect((await api.get(`/api/projects/${projectId}/usage?utcOffsetMinutes=900`)).status).toBe(
      400,
    );
  });

  /** Seeds one row straight into the table, so a batch's dates and Agents are exact. */
  const seedRow = (o: Partial<ErrorRecordInsert> & { date: string; code: string }) =>
    wire(ErrorsRepo, { db: t.deps.db }).insert({
      ts: `${o.date}T00:00:00.000Z`,
      projectId,
      agentId: null,
      sessionId: null,
      source: "http",
      kind: "expected",
      status: 404,
      message: o.code,
      ...o,
    });

  it("clearing errors takes the filter on screen and nothing outside it", async () => {
    seedRow({ date: "2026-07-05", code: "before_range" });
    seedRow({ date: "2026-07-06", code: "in_range_a1", agentId: "a1" });
    seedRow({ date: "2026-07-06", code: "in_range_a2", agentId: "a2" });
    seedRow({ date: "2026-07-08", code: "after_range" });

    // Narrowed to one Agent inside the range: exactly the rows that filter reads.
    const narrow = await api.delete(
      `/api/projects/${projectId}/usage/errors?from=2026-07-06&to=2026-07-06&agentId=a1`,
    );
    expect(narrow.status).toBe(200);
    expect(await narrow.json()).toEqual({ deleted: 1 });
    expect(errorRows().map((r) => r.code)).toEqual(["before_range", "in_range_a2", "after_range"]);

    // The range on its own still spares the dates the reader was not looking at.
    const ranged = await api.delete(
      `/api/projects/${projectId}/usage/errors?from=2026-07-06&to=2026-07-07`,
    );
    expect(await ranged.json()).toEqual({ deleted: 1 });
    expect(errorRows().map((r) => r.code)).toEqual(["before_range", "after_range"]);

    // A missing bound is refused rather than read as unbounded: an open side is the Project's
    // whole history, which is wider than any filter a panel could have shown.
    for (const query of ["", "?from=2026-07-06", "?to=2026-07-06"]) {
      const refused = await api.delete(`/api/projects/${projectId}/usage/errors${query}`);
      expect(refused.status).toBe(400);
    }
    // Those refusals are themselves recorded by onError, so the seeded rows are what is
    // checked: nothing outside the two deletes above went.
    expect(
      errorRows()
        .map((r) => r.code)
        .filter((code) => code !== "bad_request"),
    ).toEqual(["before_range", "after_range"]);
  });

  it("a category-narrowed clear leaves the other category alone", async () => {
    const day = "2026-07-06";
    seedRow({ date: day, code: "boom", kind: "unexpected" });
    seedRow({ date: day, code: "missing", kind: "expected" });
    const kept = () =>
      errorRows()
        .map((r) => r.code)
        .filter((code) => code !== "bad_request");

    // What the panel's narrowed table reads — the same `kind` its clear carries, which is the
    // point: the button empties the table on screen, never the category filtered out of it.
    const narrowed = (await (
      await api.get(
        `/api/projects/${projectId}/usage/errors?offset=0&limit=20&from=${day}&to=${day}&kind=unexpected`,
      )
    ).json()) as UsageErrorsPage;
    expect(narrowed.items.map((e) => e.code)).toEqual(["boom"]);
    expect(narrowed.total).toBe(1);

    const cleared = await api.delete(
      `/api/projects/${projectId}/usage/errors?from=${day}&to=${day}&kind=unexpected`,
    );
    expect(await cleared.json()).toEqual({ deleted: 1 });
    expect(kept()).toEqual(["missing"]);

    // A category outside the closed set is refused rather than matched against nothing: a clear
    // that silently deleted none of them reads as "already empty" while they are all still there.
    const refused = await api.delete(
      `/api/projects/${projectId}/usage/errors?from=${day}&to=${day}&kind=oops`,
    );
    expect(refused.status).toBe(400);
    expect(kept()).toEqual(["missing"]);
  });

  it("a trailing-window clear deletes exactly the rows the window listed", async () => {
    const day = "2026-07-06";
    seedRow({ date: day, code: "before_window", ts: `${day}T09:59:59.999Z` });
    seedRow({ date: day, code: "at_from", ts: `${day}T10:00:00.000Z` });
    seedRow({ date: day, code: "at_to", ts: `${day}T11:00:00.000Z` });
    seedRow({ date: day, code: "after_window", ts: `${day}T11:00:00.001Z` });
    const window = `from=${day}&to=${day}&fromTs=${day}T10:00:00.000Z&toTs=${day}T11:00:00.000Z`;

    // What the panel lists for the window …
    const listed = (await (
      await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20&${window}`)
    ).json()) as UsageErrorsPage;
    expect(listed.items.map((e) => e.code)).toEqual(["at_to", "at_from"]);
    expect(listed.total).toBe(2);

    // … is what the clear of the same window takes: the same day's rows a millisecond
    // outside it stay.
    const cleared = await api.delete(`/api/projects/${projectId}/usage/errors?${window}`);
    expect(await cleared.json()).toEqual({ deleted: 2 });
    expect(errorRows().map((r) => r.code)).toEqual(["before_window", "after_window"]);

    // Half a window is refused, as the dashboard refuses it.
    const half = await api.delete(
      `/api/projects/${projectId}/usage/errors?fromTs=${day}T10:00:00.000Z`,
    );
    expect(half.status).toBe(400);
  });

  it("a clear stops at the boundary its confirmation counted", async () => {
    const day = "2026-07-06";
    seedRow({ date: day, code: "counted_a", ts: `${day}T10:00:00.000Z` });
    seedRow({ date: day, code: "counted_b", ts: `${day}T10:05:00.000Z` });

    // What the panel reads when the confirmation opens: the count, and the newest record's
    // instant, which is the boundary the delete carries.
    const counted = (await (
      await api.get(
        `/api/projects/${projectId}/usage/errors?offset=0&limit=1&from=${day}&to=${day}`,
      )
    ).json()) as UsageErrorsPage;
    expect(counted.total).toBe(2);
    const newest = counted.items[0];
    if (newest === undefined) throw new Error("expected the newest record on the first page");
    expect(newest.code).toBe("counted_b");

    // A record that lands while the dialog is open: the sentence counted two, so two go and
    // this one stays — without the boundary it would have gone unannounced.
    seedRow({ date: day, code: "arrived_later", ts: `${day}T10:06:00.000Z` });
    const cleared = await api.delete(
      `/api/projects/${projectId}/usage/errors?from=${day}&to=${day}&until=${newest.ts}`,
    );
    expect(await cleared.json()).toEqual({ deleted: 2 });
    expect(errorRows().map((r) => r.code)).toEqual(["arrived_later"]);

    // A boundary that is not an instant is refused rather than dropped: a clear that ignored
    // it would take the record above, which the confirmation said would stay.
    const refused = await api.delete(
      `/api/projects/${projectId}/usage/errors?from=${day}&to=${day}&until=yesterday`,
    );
    expect(refused.status).toBe(400);
    expect(
      errorRows()
        .map((r) => r.code)
        .filter((code) => code !== "bad_request"),
    ).toEqual(["arrived_later"]);
  });

  it("a member cannot clear the log, so it is no route to the unattributed rows", async () => {
    seedRow({ date: "2026-07-06", code: "project_row" });
    seedRow({ date: "2026-07-06", code: "unattributed_row", projectId: null, source: "process" });

    // The refusals below are recorded by app.onError like any other, so the seeded rows are
    // read back by code rather than by counting the table.
    const seeded = () =>
      errorRows()
        .map((r) => r.code as string)
        .filter((c) => c === "project_row" || c === "unattributed_row");

    const member = await provisionUser(t.app, "member_user");
    expect(member.user.isAdmin).toBe(false);
    const added = await api.post(`/api/projects/${projectId}/members`, { userId: "member_user" });
    expect(added.status).toBe(201);
    const memberApi = apiClient(t.app, member.cookie);

    // Reading the panel is a member's right; emptying it is the owner's, like deleting an Agent.
    expect((await memberApi.get(`/api/projects/${projectId}/usage/errors?offset=0`)).status).toBe(
      200,
    );
    const refused = await memberApi.delete(`/api/projects/${projectId}/usage/errors`);
    expect(refused.status).toBe(403);
    expect(((await refused.json()) as { error: { code: string } }).error.code).toBe(
      "owner_required",
    );
    expect(seeded()).toEqual(["project_row", "unattributed_row"]);

    // A non-member is not told the Project exists, let alone allowed to empty it.
    const outsider = await provisionUser(t.app, "outsider_user");
    const outsiderApi = apiClient(t.app, outsider.cookie);
    expect((await outsiderApi.delete(`/api/projects/${projectId}/usage/errors`)).status).toBe(404);
    expect(seeded()).toEqual(["project_row", "unattributed_row"]);
  });

  it("an admin's clear takes the unattributed rows an admin's read showed", async () => {
    // They show in every Project's admin view and in no member's, so the clear follows the
    // read: the one person who can see them anywhere empties them from the panel that showed
    // them, while a member's clear (whose read never lists them) never reaches them.
    const adminApi = apiClient(t.app, (await loginAdmin(t.app)).cookie);
    wire(ErrorsRepo, { db: t.deps.db }).insert({
      ts: "2026-07-06T00:00:00.000Z",
      date: "2026-07-06",
      projectId: null,
      agentId: null,
      sessionId: null,
      source: "process",
      kind: "unexpected",
      code: "uncaught_exception",
      status: null,
      message: "crash",
    });
    // The admin can see it from their own Project, which is exactly the view being cleared.
    const before = (await (
      await adminApi.get(`/api/projects/default_project/usage/errors?offset=0&limit=20`)
    ).json()) as UsageErrorsPage;
    expect(before.items.map((e) => e.code)).toEqual(["uncaught_exception"]);

    const cleared = await adminApi.delete(
      `/api/projects/default_project/usage/errors?from=2026-07-06&to=2026-07-06`,
    );
    expect(await cleared.json()).toEqual({ deleted: 1 });
    expect(errorRows().filter((r) => r.project_id === null)).toHaveLength(0);
  });

  it("deleting a Project deletes its error records and nobody else's", async () => {
    await api.get(`/api/projects/${projectId}/agents/agent-nope/sessions`);
    expect(errorRows().filter((r) => r.project_id === projectId)).toHaveLength(1);
    // An unattributed row (a failed login) belongs to no Project and must outlive this one.
    await t.app.request("/api/auth/login", {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({ userId: "err_user", password: "wrong-password" }),
    });

    const del = await api.delete(`/api/projects/${projectId}`);
    expect(del.status).toBe(204);
    expect(errorRows().map((r) => r.project_id)).toEqual([null]);
  });

  // —— Ignoring: the owner silences a class, the panel reads one side or the other, nothing is lost ——

  it("an ignore rule takes a class off the panel while its records stay in the table", async () => {
    // Seeded directly so the repeats are exact — going through HTTP would collapse them into the
    // recorder's dedup window.
    const repo = wire(ErrorsRepo, { db: t.deps.db });
    const seed = (code: string, message: string, ts: string) =>
      repo.insert({
        ts,
        date: ts.slice(0, 10),
        projectId,
        agentId: null,
        sessionId: null,
        source: "http",
        kind: "unexpected",
        code,
        status: 500,
        message,
      });
    seed("internal", "other side closed", "2026-07-27T01:00:00.000Z");
    seed("internal", "other side closed", "2026-07-27T02:00:00.000Z");
    seed("agent_not_found", "no such agent", "2026-07-27T03:00:00.000Z");

    const added = await api.post(`/api/projects/${projectId}/usage/errors/ignores`, {
      source: "http",
      code: "internal",
      note: "engine reconnects by design",
    });
    expect(added.status).toBe(201);
    const body = (await added.json()) as UsageErrorIgnoreAdded;
    expect(body.hidden).toBe(2);
    expect(body.rule).toMatchObject({
      source: "http",
      code: "internal",
      messageLike: null,
      hidden: 2,
    });

    // The dashboard drops it from what it shows and reports it once under `ignored`: a silenced
    // class is not invisible twice over.
    const dashboard = (await (
      await api.get(`/api/projects/${projectId}/usage?groupBy=date`)
    ).json()) as UsageResponse;
    expect(dashboard.errors).toMatchObject({ total: 1, ignored: 2 });
    expect(dashboard.errors.recent.map((e) => e.code)).toEqual(["agent_not_found"]);

    // The page follows the same axis, and asking for the ignored side reads exactly the covered rows.
    const shown = (await (
      await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20`)
    ).json()) as UsageErrorsPage;
    expect(shown.total).toBe(1);
    expect(shown.items.map((e) => e.code)).toEqual(["agent_not_found"]);
    const ignored = (await (
      await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20&ignored=1`)
    ).json()) as UsageErrorsPage;
    expect(ignored.total).toBe(2);
    expect(ignored.items.map((e) => e.code)).toEqual(["internal"]);

    // The records were never touched: the rule decides which rows the reads select, not what is stored.
    expect(errorRows().filter((r) => r.code === "internal")).toHaveLength(2);
  });

  it("a new rule's hidden figure counts the records it takes off the range it was made from", async () => {
    const repo = wire(ErrorsRepo, { db: t.deps.db });
    const seed = (ts: string, agentId: string) =>
      repo.insert({
        ts,
        date: ts.slice(0, 10),
        projectId,
        agentId,
        sessionId: null,
        source: "http",
        kind: "unexpected",
        code: "internal",
        status: 500,
        message: "boom",
      });
    seed("2026-07-27T01:00:00.000Z", "a1");
    seed("2026-07-27T02:00:00.000Z", "a2");
    seed("2026-07-29T01:00:00.000Z", "a1");

    // The range is the panel's own — the one its reads carry — so the number matches the table it
    // came from; the rule itself is permanent and covers all three. A body that names its own
    // author is ignored: the owner is the session's, so a rule can never be filed under someone else.
    const added = (await (
      await api.post(
        `/api/projects/${projectId}/usage/errors/ignores?from=2026-07-27&to=2026-07-27&agentId=a1`,
        { source: "http", code: "internal", createdBy: "someone_else" },
      )
    ).json()) as UsageErrorIgnoreAdded;
    expect(added.hidden).toBe(1);
    expect(added.rule.hidden).toBe(3);
    expect(added.rule).toMatchObject({ createdBy: "err_user", note: null });
  });

  it("ignoring the same class twice is refused, and unignoring puts the rows back", async () => {
    const repo = wire(ErrorsRepo, { db: t.deps.db });
    repo.insert({
      ts: "2026-07-27T01:00:00.000Z",
      date: "2026-07-27",
      projectId,
      agentId: null,
      sessionId: null,
      source: "http",
      kind: "unexpected",
      code: "internal",
      status: 500,
      message: "boom",
    });
    const write = { source: "http", code: "internal" };
    // The refusals below are recorded by app.onError like any other, so the seeded row is read
    // back by code rather than by counting the table.
    const visible = async () =>
      (
        (await (
          await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20`)
        ).json()) as UsageErrorsPage
      ).items
        .map((e) => e.code)
        .filter((code) => code === "internal");

    const first = await api.post(`/api/projects/${projectId}/usage/errors/ignores`, write);
    expect(first.status).toBe(201);
    const ruleId = ((await first.json()) as UsageErrorIgnoreAdded).rule.id;
    expect(await visible()).toEqual([]);

    // The same (source, code, fragment) is one rule: a second row covering the same records would
    // only say the same thing twice.
    const again = await api.post(`/api/projects/${projectId}/usage/errors/ignores`, write);
    expect(again.status).toBe(409);
    expect(((await again.json()) as { error: { code: string } }).error.code).toBe(
      "ignore_rule_exists",
    );

    const listed = (await (
      await api.get(`/api/projects/${projectId}/usage/errors/ignores`)
    ).json()) as UsageErrorIgnores;
    expect(listed.rules.map((r) => r.id)).toEqual([ruleId]);
    expect(listed.rules[0]!.hidden).toBe(1);

    const removed = await api.delete(`/api/projects/${projectId}/usage/errors/ignores/${ruleId}`);
    expect(await removed.json()).toEqual({ removed: true });
    // Nothing was ever deleted, so removing the rule is the whole of the undo.
    expect(await visible()).toEqual(["internal"]);

    // A rule this Project does not have is a 404, not a silent success.
    expect(
      (await api.delete(`/api/projects/${projectId}/usage/errors/ignores/${ruleId}`)).status,
    ).toBe(404);
    expect(
      (await api.delete(`/api/projects/${projectId}/usage/errors/ignores/not-a-number`)).status,
    ).toBe(400);
  });

  it("the default view's clear leaves the ignored rows; standing on the ignored view it takes them", async () => {
    const repo = wire(ErrorsRepo, { db: t.deps.db });
    const seed = (code: string, ts: string) =>
      repo.insert({
        ts,
        date: ts.slice(0, 10),
        projectId,
        agentId: null,
        sessionId: null,
        source: "http",
        kind: "unexpected",
        code,
        status: 500,
        message: code,
      });
    seed("internal", "2026-07-27T01:00:00.000Z");
    seed("internal", "2026-07-27T02:00:00.000Z");
    seed("agent_not_found", "2026-07-27T03:00:00.000Z");
    await api.post(`/api/projects/${projectId}/usage/errors/ignores`, {
      source: "http",
      code: "internal",
    });

    // The button empties the table on screen: the ignored rows are not on it, so they stay.
    const cleared = await api.delete(
      `/api/projects/${projectId}/usage/errors?from=2026-07-27&to=2026-07-27`,
    );
    expect(await cleared.json()).toEqual({ deleted: 1 });
    expect(errorRows().filter((r) => r.code === "internal")).toHaveLength(2);

    // Standing on the ignored view, the same button is the one that may take those rows.
    const clearedIgnored = await api.delete(
      `/api/projects/${projectId}/usage/errors?from=2026-07-27&to=2026-07-27&ignored=1`,
    );
    expect(await clearedIgnored.json()).toEqual({ deleted: 2 });
    expect(errorRows().filter((r) => r.code === "internal")).toHaveLength(0);
  });

  it("ignore management is the owner's, like the clear; a member still reads the panel and sees the count", async () => {
    seedRow({ date: "2026-07-27", code: "internal", kind: "unexpected" });
    const write = { source: "http", code: "internal" };
    const member = await provisionUser(t.app, "member_user");
    expect(member.user.isAdmin).toBe(false);
    await api.post(`/api/projects/${projectId}/members`, { userId: "member_user" });
    const memberApi = apiClient(t.app, member.cookie);
    const outsiderApi = apiClient(t.app, (await provisionUser(t.app, "outsider_user")).cookie);
    const rulesPath = `/api/projects/${projectId}/usage/errors/ignores`;

    // The owner silences the seeded class; the member then reads its effect as a count, without the
    // rule list. Read before the refusals below, whose own 403 / 404 are recorded like any error.
    expect((await api.post(rulesPath, write)).status).toBe(201);
    const memberView = (await (
      await memberApi.get(`/api/projects/${projectId}/usage?groupBy=date`)
    ).json()) as UsageResponse;
    expect(memberView.errors).toMatchObject({ total: 0, ignored: 1 });

    // Reading the panel is a member's right; managing what it hides is the owner's.
    expect((await memberApi.get(`/api/projects/${projectId}/usage/errors?offset=0`)).status).toBe(
      200,
    );
    expect((await memberApi.get(rulesPath)).status).toBe(403);
    expect((await memberApi.post(rulesPath, write)).status).toBe(403);
    expect((await memberApi.delete(`${rulesPath}/1`)).status).toBe(403);
    // A non-member is not told the Project exists, let alone allowed to silence its errors.
    expect((await outsiderApi.get(rulesPath)).status).toBe(404);
    expect((await outsiderApi.post(rulesPath, write)).status).toBe(404);
    expect((await outsiderApi.delete(`${rulesPath}/1`)).status).toBe(404);
  });
});

describe("HTTP error attribution (only when the requester actually has Project access)", () => {
  let t: TestApp;
  /** The built-in admin: unattributed errors are visible only to them. */
  let adminApi: ReturnType<typeof apiClient>;
  let adminProjectId: string;
  /** The victim Project's owner: a regular user, so their stats center only shows errors attributed to this Project. */
  let ownerApi: ReturnType<typeof apiClient>;
  let projectId: string;

  beforeEach(async () => {
    t = await createTestApp();
    const admin = await loginAdmin(t.app);
    adminApi = apiClient(t.app, admin.cookie);
    adminProjectId = (
      (await (
        await adminApi.post("/api/projects", { projectId: "admin_proj", name: "Admin project" })
      ).json()) as ProjectCreateResponse
    ).project.projectId;

    const owner = await provisionUser(t.app, "owner_user");
    expect(owner.user.isAdmin).toBe(false);
    ownerApi = apiClient(t.app, owner.cookie);
    projectId = (
      (await (
        await ownerApi.post("/api/projects", { projectId: "owner_user-victim", name: "Victim" })
      ).json()) as ProjectCreateResponse
    ).project.projectId;
  });
  afterEach(async () => {
    await t.cleanup();
  });

  const errorRows = () =>
    t.deps.db.prepare("SELECT * FROM error_records ORDER BY id").all() as Array<
      Record<string, unknown>
    >;
  /** The victim Project's stats center (owner's view: only this Project's errors). */
  const ownerErrors = async () =>
    (
      (await (
        await ownerApi.get(`/api/projects/${projectId}/usage?groupBy=date`)
      ).json()) as UsageResponse
    ).errors;
  /** The admin's stats center (this Project + unattributed errors). */
  const adminErrors = async () =>
    (
      (await (
        await adminApi.get(`/api/projects/${adminProjectId}/usage?groupBy=date`)
      ).json()) as UsageResponse
    ).errors;

  it("not logged in → 401: unattributed (anyone could flood another's error stats)", async () => {
    const res = await t.app.request(`/api/projects/${projectId}/usage?groupBy=date`);
    expect(res.status).toBe(401);

    // The requester isn't even logged in: this error must not be pinned to projectId.
    // Note: today this also **incidentally** relies on a Hono quirk — `c.req.param()`
    // resolves only against "the route the current handler belongs to"; the 401 is
    // thrown in the `/api/*` authMiddleware, whose route has no :projectId, so no
    // value is available there. The attribution guard removes the dependency on
    // that quirk: when not logged in, `c.var.user` is undefined at runtime → unattributed.
    const rows = errorRows();
    expect(rows).toHaveLength(1);
    expect(rows[0]).toMatchObject({ source: "http", code: "unauthorized", status: 401 });
    expect(rows[0]!.project_id).toBeNull();

    // The owner's stats center gains nothing; the trace of the unauthorized probe lands in the admin's view — right where it belongs.
    expect(await ownerErrors()).toMatchObject({ total: 0, unexpected: 0, topCode: null });
    const admin = await adminErrors();
    expect(admin.total).toBe(1);
    expect(admin.recent[0]).toMatchObject({ source: "http", code: "unauthorized" });
  });

  it("logged in but without access (non-member) → 404: likewise unattributed", async () => {
    const outsider = await provisionUser(t.app, "outsider");
    const res = await apiClient(t.app, outsider.cookie).get(
      `/api/projects/${projectId}/usage?groupBy=date`,
    );
    expect(res.status).toBe(404);

    const rows = errorRows();
    expect(rows).toHaveLength(1);
    expect(rows[0]).toMatchObject({ source: "http", code: "project_not_found", status: 404 });
    expect(rows[0]!.project_id).toBeNull();

    expect(await ownerErrors()).toMatchObject({ total: 0, unexpected: 0, topCode: null });
    expect((await adminErrors()).recent[0]).toMatchObject({ code: "project_not_found" });
  });

  it("business errors from an authorized member → attributed as usual", async () => {
    // owner: an invalid groupBy → 400.
    expect((await ownerApi.get(`/api/projects/${projectId}/usage?groupBy=bogus`)).status).toBe(400);

    // An authorized member: a 404 in the same Project → attributed the same way (the member branch of canAccess).
    const member = await provisionUser(t.app, "member_user");
    const added = await ownerApi.post(`/api/projects/${projectId}/members`, {
      userId: "member_user",
    });
    expect(added.status).toBe(201);
    const missing = await apiClient(t.app, member.cookie).get(
      `/api/projects/${projectId}/agents/agent-nope/sessions`,
    );
    expect(missing.status).toBe(404);

    expect(errorRows().map((r) => [r.code, r.project_id])).toEqual([
      ["bad_request", projectId],
      ["agent_not_found", projectId],
    ]);
    expect(await ownerErrors()).toMatchObject({ total: 2, unexpected: 0 });
  });

  it("the attribution check throws: onError survives, error lands unattributed", async () => {
    t.deps.access.canAccess = () => {
      throw new Error("access check blew up");
    };
    const res = await ownerApi.get(`/api/projects/${projectId}/usage?groupBy=bogus`);
    expect(res.status).toBe(400); // still the original business error: not turned into a 500, nor an empty response
    expect(await res.json()).toMatchObject({ error: { code: "bad_request" } });

    const rows = errorRows();
    expect(rows).toHaveLength(1);
    expect(rows[0]!.code).toBe("bad_request");
    expect(rows[0]!.project_id).toBeNull(); // a failed determination always falls back to unattributed
  });
});