A local-first multi-agent app development platform, built on PenguinHarness
/**
* Error records, from where they are caught to the cost center that shows them.
*
* The log (ErrorsRepo):
* - A summary counts every row and the unexpected ones; the top code groups by source, code and
* kind (none in range reads as nothing); dates and an Agent narrow every read.
* - The recent list folds a day's repeats of one error into one row with its count and its
* first and last time, the day being the viewer's when the read carries an offset; a page
* and the row count are rows, the summary still counts records.
* - A clear follows the caller's read: a member's leaves the unattributed rows, an admin's
* takes them; another Project's rows are out of reach either way.
* - An ignore rule (a source · code, optionally narrowed by a message fragment) takes that class
* off every read's default side without deleting a row: the reads, the clear and the rule's own
* coverage count all take the same axis; rules are per Project and go with it.
* - Past its cap the log evicts its oldest rows by count, never misled by id gaps.
*
* The recorder (ErrorRecorder):
* - An HttpError is expected; any other failure is unexpected, an explicit kind wins over both,
* and a gateway close is filed by its own verdict.
* - A non-Error is stringified and an overlong message truncated; the recorder never throws.
* - The same error for the same Project persists once per short window, then resumes; source,
* code and Project each keep their own window; a full dedup table still records every
* distinct error and still collapses a burst.
*
* The stream watcher (LLM, tool and compaction failures in a Session's messages):
* - An LLM failure the retry ladder carried is expected under its own code; an exhausted or
* fatal one is unexpected, worded from the abort reason, the request's own detail or the
* status text in that order; a user's Stop and a completion are not errors.
* - A command tool's ordinary non-zero exit is information, not an error; a signal, a spawn
* failure, a timeout or a missing session manager are recorded; any other tool failure is
* expected under the tool's name, legacy spellings included.
* - Parallel and nested (child Session) calls never swap names or reasons, and a child's
* failures are attributed to the child Agent and Session.
* - An abandoned compaction is one unexpected row carrying its attempts and error.
*
* Through HTTP (onError):
* - A business error is expected, an internal one unexpected with a 500, each attributed to
* the Project only when the requester has access to it (unauthenticated and non-member
* failures stay unattributed, and a throwing access check degrades to unattributed).
* - A Skill directory another program is holding open (Windows EBUSY) answers as a named 409
* and is filed as expected, not as the internal failure the raw errno would look like.
* - The usage endpoint shows a Project's own errors; unattributed ones only to the admin.
* - The paged error route pages within the caller's tenant, narrows to one kind on request,
* and refuses a page it cannot serve; it and the dashboard fold by the viewer's day, and an
* offset no clock uses is a 400.
* - Clearing takes exactly the filter or trailing window on screen, never an unbounded range;
* a member may not clear; an admin's clear takes the unattributed rows it showed.
* - The owner silences a class (never a member, and never a non-member's Project), the panel then
* reports it under `ignored` and reads it back on request, and removing the rule restores it.
* - Deleting a Project deletes its error records and nobody else's.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import fs from "node:fs/promises";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import {
abortEvent,
assistantText,
compactionEnd,
partialToolCallOutput,
requestBegin,
requestEnd,
sessionMeta,
skillsDir,
toolCall,
toolCallOutput,
withOrigin,
} from "@adelie/core";
import type { OmniMessage, StopReason } from "@adelie/core";
import type {
ProjectCreateResponse,
UsageErrorIgnoreAdded,
UsageErrorIgnores,
UsageErrorsPage,
UsageResponse,
} from "../src/api/types.js";
import { openDatabase } from "../src/db/database.js";
import { ErrorsRepo } from "../src/db/repos/errors.js";
import type { ErrorRecordInsert } from "../src/db/repos/errors.js";
import { HttpError } from "../src/http/errors.js";
import {
DEDUP_KEYS_MAX,
DEDUP_WINDOW_MS,
ErrorRecorder,
MESSAGE_MAX,
} from "../src/runtime/error-recorder.js";
import { messagingErrorKind } from "../src/runtime/messaging/error-kind.js";
import { MessagingConnectionClosedError } from "../src/runtime/messaging/qq-api.js";
import { StreamErrorWatcher } from "../src/runtime/stream-error-watcher.js";
import { apiClient, createTestApp, loginAdmin, provisionUser } from "./helpers.js";
import type { TestApp } from "./helpers.js";
import { wire } from "@adelie/core/kernel";
function row(date: string, o: Partial<ErrorRecordInsert> = {}): ErrorRecordInsert {
return {
ts: `${date}T10:00:00.000Z`,
date,
projectId: "p1",
agentId: null,
sessionId: null,
source: "http",
kind: "unexpected",
code: "internal",
status: 500,
message: "boom",
...o,
};
}
describe("errors-repo", () => {
let db: DatabaseSync;
let repo: ErrorsRepo;
beforeEach(() => {
db = openDatabase(":memory:");
repo = wire(ErrorsRepo, { db: db });
});
afterEach(() => db.close());
it("summary: total and unexpected count; expected ones are still recorded", () => {
repo.insert(row("2026-07-06"));
repo.insert(row("2026-07-06", { kind: "expected", code: "not_found", status: 404 }));
repo.insert(row("2026-07-06", { kind: "expected", code: "bad_request", status: 400 }));
expect(repo.summary("p1")).toEqual({ total: 3, unexpected: 1 });
});
it("top error code: grouped by source+code+kind, takes the highest count", () => {
for (let i = 0; i < 3; i++) repo.insert(row("2026-07-06", { code: "internal" }));
repo.insert(row("2026-07-06", { source: "session", code: "session_run_failed" }));
repo.insert(row("2026-07-06", { kind: "expected", code: "not_found", status: 404 }));
repo.insert(row("2026-07-06", { kind: "expected", code: "not_found", status: 404 }));
expect(repo.topCode("p1")).toEqual({
source: "http",
code: "internal",
kind: "unexpected",
count: 3,
});
// No errors / no errors in range -> null (the frontend uses this to hide the metric).
expect(repo.topCode("p-empty")).toBeNull();
expect(repo.topCode("p1", { from: "2026-07-07" })).toBeNull();
});
it("date range and agent filters (HTTP / process errors have no agent_id)", () => {
repo.insert(row("2026-07-05"));
repo.insert(row("2026-07-06", { kind: "expected" }));
repo.insert(row("2026-07-06", { agentId: "a1", source: "session" }));
expect(repo.summary("p1")).toEqual({ total: 3, unexpected: 2 });
expect(repo.summary("p1", { from: "2026-07-06" })).toEqual({ total: 2, unexpected: 1 });
expect(repo.summary("p1", { agentId: "a1" })).toEqual({ total: 1, unexpected: 1 });
expect(repo.topCode("p1", { agentId: "a1" })).toMatchObject({ source: "session", count: 1 });
expect(repo.recent("p1", { agentId: "a1" })).toHaveLength(1);
});
it("recent folds a day's repeats into one row, with how many and the first and last time", () => {
// A connection that drops and recovers all afternoon files the same record again and again;
// the table reads it as one line saying how often. Anything that differs keeps its own row:
// another message, another day, another kind (a record filed before a reclassification).
const heartbeat = {
source: "messaging",
code: "messaging_connect_failed",
kind: "expected",
message: "gateway stopped acknowledging heartbeats",
};
const at = (ts: string, o: Partial<ErrorRecordInsert> = {}) =>
repo.insert(row(ts.slice(0, 10), { ...heartbeat, ts, ...o }));
at("2026-07-06T09:00:00.000Z");
at("2026-07-06T12:30:00.000Z");
at("2026-07-06T13:00:00.000Z", { message: "gateway connection closed (code 4009)" });
at("2026-07-06T17:45:00.000Z");
at("2026-07-06T18:00:00.000Z", { kind: "unexpected" });
at("2026-07-07T08:00:00.000Z");
const rows = repo
.recent("p1")
.map(({ ts, firstTs, count, kind, message }) => ({ ts, firstTs, count, kind, message }));
expect(rows).toEqual([
{
ts: "2026-07-07T08:00:00.000Z",
firstTs: "2026-07-07T08:00:00.000Z",
count: 1,
kind: "expected",
message: heartbeat.message,
},
{
ts: "2026-07-06T18:00:00.000Z",
firstTs: "2026-07-06T18:00:00.000Z",
count: 1,
kind: "unexpected",
message: heartbeat.message,
},
// Newest first by the row's latest record, so the fold sits where its last one happened.
{
ts: "2026-07-06T17:45:00.000Z",
firstTs: "2026-07-06T09:00:00.000Z",
count: 3,
kind: "expected",
message: heartbeat.message,
},
{
ts: "2026-07-06T13:00:00.000Z",
firstTs: "2026-07-06T13:00:00.000Z",
count: 1,
kind: "expected",
message: "gateway connection closed (code 4009)",
},
]);
// A page is rows, and the pager counts rows; the summary still counts the records.
expect(repo.recent("p1", {}, 2, 2).map((r) => r.ts)).toEqual([
"2026-07-06T17:45:00.000Z",
"2026-07-06T13:00:00.000Z",
]);
expect(repo.rowCount("p1")).toBe(4);
expect(repo.summary("p1")).toEqual({ total: 6, unexpected: 1 });
expect(repo.topCode("p1")).toMatchObject({ code: "messaging_connect_failed", count: 5 });
// Filters narrow the records before they fold.
expect(repo.rowCount("p1", { kind: "unexpected" })).toBe(1);
expect(repo.recent("p1", { from: "2026-07-06", to: "2026-07-06" })).toHaveLength(3);
});
it("the day a row folds by is the viewer's, from the offset the read carries", () => {
// Two records an hour apart, either side of UTC midnight. The `date` column is the server's
// day, which is what a read without an offset folds by.
const late = { ts: "2026-07-06T23:30:00.000Z", message: "socket hang up" };
const early = { ts: "2026-07-07T00:30:00.000Z", message: "socket hang up" };
repo.insert(row("2026-07-06", late));
repo.insert(row("2026-07-07", early));
expect(repo.recent("p1").map((r) => r.count)).toEqual([1, 1]);
expect(repo.rowCount("p1")).toBe(2);
// UTC+8 and UTC-5: both land on one local day, so one row says twice.
for (const offset of [480, -300]) {
expect(repo.recent("p1", {}, 20, 0, offset).map((r) => r.count)).toEqual([2]);
expect(repo.rowCount("p1", {}, offset)).toBe(1);
}
// UTC itself splits them again.
expect(repo.rowCount("p1", {}, 0)).toBe(2);
});
it("deleteFiltered follows the caller's includeGlobal: a member's clear leaves unattributed rows, an admin's takes them", () => {
// Unattributed rows appear in every Project's admin view and in no member's. The clear
// reaches exactly what the caller's read reaches, so only a read that includes them
// deletes them; another Project's rows are out of reach either way.
repo.insert(row("2026-07-06", { projectId: null, source: "process", message: "crash" }));
repo.insert(row("2026-07-06", { projectId: "p-other", message: "other tenant" }));
repo.insert(row("2026-07-06", { message: "mine" }));
const remaining = () =>
db
.prepare("SELECT message FROM error_records ORDER BY id")
.all()
.map((r) => r.message);
expect(repo.deleteFiltered("p1")).toBe(1);
expect(remaining()).toEqual(["crash", "other tenant"]);
expect(repo.deleteFiltered("p1", { includeGlobal: true })).toBe(1);
expect(remaining()).toEqual(["other tenant"]);
});
// —— Ignoring: rules that keep a class off the panel's default view, on read, without losing a row ——
it("an ignore rule takes a whole class out of every read, while the records stay in the table", () => {
// Three of the class to silence, two of another one that must keep showing.
for (let i = 0; i < 3; i++) repo.insert(row("2026-07-06", { code: "internal" }));
for (let i = 0; i < 2; i++)
repo.insert(row("2026-07-06", { source: "messaging", code: "messaging_connect_failed" }));
const rule = repo.addIgnoreRule("p1", {
source: "http",
code: "internal",
createdBy: "owner",
createdAt: "2026-07-06T10:00:00.000Z",
});
expect(rule).toMatchObject({ source: "http", code: "internal", messageLike: null, hidden: 3 });
// Every read not asked for the ignored side leaves the class out …
expect(repo.summary("p1")).toEqual({ total: 2, unexpected: 2 });
expect(repo.topCode("p1")).toMatchObject({ source: "messaging", count: 2 });
expect(repo.recent("p1").map((r) => r.source)).toEqual(["messaging"]);
expect(repo.rowCount("p1")).toBe(1);
// … and the other side reads exactly it, which is what the panel's "ignored" view shows.
expect(repo.summary("p1", { ignored: true })).toEqual({ total: 3, unexpected: 3 });
expect(repo.recent("p1", { ignored: true }).map((r) => r.source)).toEqual(["http"]);
// Nothing was deleted: the rule changes which rows the reads select, not what is stored.
expect(
db.prepare("SELECT COUNT(*) AS n FROM error_records WHERE code = 'internal'").get(),
).toEqual({ n: 3 });
});
it("a fragment narrows a rule to the messages that contain it", () => {
const at = (message: string) => repo.insert(row("2026-07-06", { code: "internal", message }));
at("other side closed");
at("other side closed (again)");
at("disk full");
repo.addIgnoreRule("p1", {
source: "http",
code: "internal",
messageLike: "other side closed",
createdBy: "owner",
createdAt: "2026-07-06T10:00:00.000Z",
});
// A literal substring, not a pattern: the two sharing it go, the third stays.
expect(repo.recent("p1").map((r) => r.message)).toEqual(["disk full"]);
expect(repo.summary("p1")).toEqual({ total: 1, unexpected: 1 });
expect(repo.summary("p1", { ignored: true })).toEqual({ total: 2, unexpected: 2 });
});
it("a blank fragment means the whole class, not a rule that swallows every record", () => {
repo.insert(row("2026-07-06", { code: "internal", message: "boom" }));
repo.insert(
row("2026-07-06", { source: "session", code: "session_run_failed", message: "nope" }),
);
// instr(x, '') is > 0 for every row: keeping the empty string would make this one rule cover
// every record in the table, so a blank fragment has to read as "no fragment".
const rule = repo.addIgnoreRule("p1", {
source: "http",
code: "internal",
messageLike: " ",
createdBy: "owner",
createdAt: "2026-07-06T10:00:00.000Z",
});
expect(rule?.messageLike).toBeNull();
expect(repo.summary("p1")).toEqual({ total: 1, unexpected: 1 });
expect(repo.recent("p1").map((r) => r.source)).toEqual(["session"]);
});
it("one rule per class: the same (source, code, fragment) is refused, a different fragment is another rule", () => {
const make = (messageLike: string | null) =>
repo.addIgnoreRule("p1", {
source: "http",
code: "internal",
messageLike,
createdBy: "owner",
createdAt: "2026-07-06T10:00:00.000Z",
});
expect(make(null)).not.toBeNull();
// The same class is one rule, not two rows saying it twice.
expect(make(null)).toBeNull();
expect(make("other side closed")).not.toBeNull();
// A blank fragment is the whole class, so it collides with the whole-class rule above.
expect(make(" ")).toBeNull();
expect(repo.ignoreRules("p1")).toHaveLength(2);
});
it("rules are per Project, and deleting a Project takes its rules with it", () => {
repo.insert(row("2026-07-06", { projectId: "p1", code: "internal" }));
repo.insert(row("2026-07-06", { projectId: "p2", code: "internal" }));
const rule = (projectId: string) =>
repo.addIgnoreRule(projectId, {
source: "http",
code: "internal",
createdBy: "owner",
createdAt: "2026-07-06T10:00:00.000Z",
});
rule("p1");
rule("p2");
// p2's rule silences p2's records only, so the same class in p1 reads untouched.
expect(repo.summary("p1")).toEqual({ total: 0, unexpected: 0 });
expect(repo.summary("p1", { ignored: true })).toEqual({ total: 1, unexpected: 1 });
expect(repo.summary("p2", { ignored: true })).toEqual({ total: 1, unexpected: 1 });
// A rule names one Project's errors and means nothing once they are gone.
repo.deleteByProject("p1");
expect(repo.ignoreRules("p1")).toEqual([]);
expect(repo.ignoreRules("p2")).toHaveLength(1);
expect(repo.summary("p2", { ignored: true })).toEqual({ total: 1, unexpected: 1 });
});
it("a clear on either side takes only that side's rows", () => {
for (let i = 0; i < 2; i++)
repo.insert(row("2026-07-06", { code: "internal", message: "silenced" }));
repo.insert(
row("2026-07-06", { source: "session", code: "session_run_failed", message: "shown" }),
);
repo.addIgnoreRule("p1", {
source: "http",
code: "internal",
createdBy: "owner",
createdAt: "2026-07-06T10:00:00.000Z",
});
// Default side: the rows the panel shows, and no other.
expect(repo.deleteFiltered("p1")).toBe(1);
expect(repo.summary("p1", { ignored: true })).toEqual({ total: 2, unexpected: 2 });
// Standing on the ignored view, the same button is the one that may take the covered rows.
expect(repo.deleteFiltered("p1", { ignored: true })).toBe(2);
expect(db.prepare("SELECT COUNT(*) AS n FROM error_records").get()).toEqual({ n: 0 });
});
it("ignoreRuleMatches counts a class against a range, saved or not", () => {
repo.insert(row("2026-07-05", { code: "internal" }));
repo.insert(row("2026-07-06", { code: "internal" }));
repo.insert(row("2026-07-06", { code: "internal", message: "other side closed" }));
repo.insert(row("2026-07-07", { code: "internal" }));
const wholeClass = { source: "http", code: "internal", messageLike: null };
expect(repo.ignoreRuleMatches("p1", wholeClass)).toBe(4);
// Against a caller's range, which is what a new rule reports having just hidden.
expect(repo.ignoreRuleMatches("p1", wholeClass, { from: "2026-07-06", to: "2026-07-06" })).toBe(
2,
);
expect(repo.ignoreRuleMatches("p1", wholeClass, { from: "2026-07-06", to: "2026-07-07" })).toBe(
3,
);
// A fragment narrows it; the axis is never applied here — it counts what a rule will take off.
expect(repo.ignoreRuleMatches("p1", { ...wholeClass, messageLike: "other side closed" })).toBe(
1,
);
});
// —— Row cap (the second line of defense against error storms; the first is ErrorRecorder's short-window dedup) ——
const messages = () =>
db
.prepare("SELECT message FROM error_records ORDER BY id")
.all()
.map((r) => r.message as string);
it("row cap: evicts the oldest rows by id (checked every pruneEvery inserts)", () => {
const capped = wire(ErrorsRepo, { db: db, maxRows: 5, pruneEvery: 2 });
for (let i = 0; i < 10; i++) capped.insert(row("2026-07-06", { message: `m${i}` }));
// The 5 most recent rows within the cap are kept, older ones are evicted.
expect(messages()).toEqual(["m5", "m6", "m7", "m8", "m9"]);
});
it("eviction counts rows: id gaps left by deleteByProject never misdelete valid data", () => {
const capped = wire(ErrorsRepo, { db: db, maxRows: 3, pruneEvery: 1 });
capped.insert(row("2026-07-06", { message: "keep-1" })); // id 1
capped.insert(row("2026-07-06", { message: "keep-2" })); // id 2
capped.insert(row("2026-07-06", { projectId: "p-gone", message: "gone" })); // id 3
capped.deleteByProject("p-gone"); // id 3 becomes a gap: MAX(id) is now decoupled from the actual row count
// 3 rows in the table = exactly at the cap; none should be deleted. An approximation (id <= MAX(id) - 3) would wrongly delete keep-1.
capped.insert(row("2026-07-06", { message: "keep-3" })); // id 4
expect(messages()).toEqual(["keep-1", "keep-2", "keep-3"]);
// Once over the cap, the oldest row is evicted as usual (gaps don't affect the "oldest" determination).
capped.insert(row("2026-07-06", { message: "keep-4" })); // id 5
expect(messages()).toEqual(["keep-2", "keep-3", "keep-4"]);
});
});
describe("error-recorder", () => {
let db: DatabaseSync;
let repo: ErrorsRepo;
const now = () => new Date("2026-07-06T10:00:00");
beforeEach(() => {
db = openDatabase(":memory:");
repo = wire(ErrorsRepo, { db: db });
});
afterEach(() => db.close());
it("a non-HTTP failure is unexpected, with no status and the attribution it was given", () => {
wire(ErrorRecorder, { errors: repo, clock: { now: now } }).record({
source: "session",
err: new Error("drive crashed"),
ctx: { projectId: "p1", agentId: "a1", sessionId: "s1" },
code: "session_run_failed",
});
expect(db.prepare("SELECT * FROM error_records").get()).toMatchObject({
kind: "unexpected",
code: "session_run_failed",
status: null,
project_id: "p1",
agent_id: "a1",
session_id: "s1",
date: "2026-07-06",
});
});
it("non-Error throwables and overlong messages: stringified and truncated to the cap", () => {
const rec = wire(ErrorRecorder, { errors: repo, clock: { now: now } });
rec.record({ source: "process", err: "a string error", code: "unhandled_rejection" });
rec.record({ source: "usage", err: new Error("x".repeat(MESSAGE_MAX + 100)) });
const rows = db.prepare("SELECT message FROM error_records ORDER BY id").all();
expect(rows[0]!.message).toBe("a string error");
expect((rows[1]!.message as string).length).toBe(MESSAGE_MAX);
});
it("the recorder itself never throws (hooked on onError it would recurse forever)", () => {
const broken = {
insert() {
throw new Error("DB is closed");
},
} as unknown as ErrorsRepo;
expect(() =>
wire(ErrorRecorder, { errors: broken, clock: { now: () => new Date() } }).record({
source: "http",
err: new Error("x"),
}),
).not.toThrow();
});
it("explicit kind wins over HttpError inference (sources self-report human need)", () => {
const rec = wire(ErrorRecorder, { errors: repo, clock: { now: now } });
rec.record({ source: "llm", err: "timed out", code: "llm_timeout", kind: "expected" });
rec.record({ source: "llm", err: "auth failed", code: "llm_failed", kind: "unexpected" });
const rows = db.prepare("SELECT kind, source, status FROM error_records ORDER BY id").all();
expect(rows[0]).toMatchObject({ kind: "expected", source: "llm", status: null });
expect(rows[1]).toMatchObject({ kind: "unexpected", source: "llm", status: null });
});
it("a gateway close is filed by its own verdict: the routine one leaves the count alone", () => {
// The path the messaging bridge takes for a dropped connection: the connector's typed
// close, classified by messagingErrorKind, persisted under `messaging_connect_failed`.
// Both rows land — the log keeps everything — but only the one a person has to act on
// reaches the unexpected count the cost center highlights.
const rec = wire(ErrorRecorder, { errors: repo, clock: { now: now } });
const closed = (message: string, code: number, recovers: boolean) =>
new MessagingConnectionClosedError(message, code, recovers);
for (const err of [
closed("gateway connection closed (code 4009)", 4009, true),
closed("gateway connection closed (code 4004)", 4004, false),
]) {
rec.record({
source: "messaging",
err,
code: "messaging_connect_failed",
kind: messagingErrorKind(err, "messaging_connect_failed"),
// Distinct Projects so the recorder's short-window dedup does not swallow the second.
ctx: { projectId: err.recovers ? "p-routine" : "p-fault" },
});
}
const rows = db.prepare("SELECT kind, code, message FROM error_records ORDER BY id").all();
expect(rows[0]).toMatchObject({ kind: "expected", code: "messaging_connect_failed" });
expect(rows[1]).toMatchObject({ kind: "unexpected", code: "messaging_connect_failed" });
expect(repo.summary("p-routine")).toEqual({ total: 1, unexpected: 0 });
expect(repo.summary("p-fault")).toEqual({ total: 1, unexpected: 1 });
});
// —— Short-window dedup (the first line of defense against error storms) ——
const count = () =>
db.prepare("SELECT COUNT(*) AS n FROM error_records").get()!.n as unknown as number;
it("short-window dedup: same-kind errors persist once per window, then resume", () => {
let t = Date.parse("2026-07-06T10:00:00Z");
const rec = wire(ErrorRecorder, { errors: repo, clock: { now: () => new Date(t) } });
const boom = () =>
rec.record({
source: "http",
err: new HttpError(404, "not_found", "Not found."),
ctx: { projectId: "p1" },
});
boom();
expect(count()).toBe(1);
t += DEDUP_WINDOW_MS - 1; // Still within the window: a burst of 404s from a scan discards straight away, no persist
boom();
boom();
expect(count()).toBe(1);
t += 1; // Outside the window: the same kind of error is recorded again (a sustained storm leaves exactly one entry per window, never suppressed forever)
boom();
expect(count()).toBe(2);
});
it("dedup never crosses source / code / Project (kinds don't suppress each other)", () => {
const rec = wire(ErrorRecorder, { errors: repo, clock: { now: now } }); // time frozen: everything lands in the same window
const err = new Error("boom");
rec.record({ source: "http", err, ctx: { projectId: "p1" }, code: "c1" });
rec.record({ source: "http", err, ctx: { projectId: "p1" }, code: "c1" }); // same kind: discarded
rec.record({ source: "http", err, ctx: { projectId: "p1" }, code: "c2" }); // different code
rec.record({ source: "http", err, ctx: { projectId: "p2" }, code: "c1" }); // different Project
rec.record({ source: "session", err, ctx: { projectId: "p1" }, code: "c1" }); // different source
rec.record({ source: "http", err, code: "c1" }); // unattributed (project_id is NULL): counts as its own kind
expect(count()).toBe(5);
});
it("keeps recording and deduplicating after its table fills, expires and is wiped", () => {
// The table is bounded: past DEDUP_KEYS_MAX keys it drops the expired ones, or all of them
// when none has expired. Either way no distinct error may be lost and a burst of one error
// must still collapse to one row.
let t = Date.parse("2026-07-06T10:00:00Z");
const rec = wire(ErrorRecorder, { errors: repo, clock: { now: () => new Date(t) } });
const boom = (code: string) =>
rec.record({ source: "http", err: "boom", ctx: { projectId: "p1" }, code });
for (let i = 0; i < DEDUP_KEYS_MAX; i++) boom(`c${i}`); // one key per code: the table is full
expect(count()).toBe(DEDUP_KEYS_MAX);
t += DEDUP_WINDOW_MS; // every key has expired, so the next one cleans them out
boom("after-window");
boom("after-window");
expect(count()).toBe(DEDUP_KEYS_MAX + 1);
for (let i = 0; i < DEDUP_KEYS_MAX; i++) boom(`d${i}`); // one window, nothing expired: wiped
expect(count()).toBe(2 * DEDUP_KEYS_MAX + 1);
boom("tail");
boom("tail");
expect(count()).toBe(2 * DEDUP_KEYS_MAX + 2);
});
});
describe("stream-error-watcher (LLM / Environment errors)", () => {
let db: DatabaseSync;
let repo: ErrorsRepo;
const now = () => new Date("2026-07-06T10:00:00");
const CTX = { projectId: "p1", agentId: "a1", sessionId: "s1" };
beforeEach(() => {
db = openDatabase(":memory:");
repo = wire(ErrorsRepo, { db: db });
});
afterEach(() => db.close());
const watcher = () =>
new StreamErrorWatcher(wire(ErrorRecorder, { errors: repo, clock: { now: now } }), CTX);
const rows = () =>
db.prepare("SELECT * FROM error_records ORDER BY id").all() as Array<Record<string, unknown>>;
/** A legacy stream's abort: prose reason only (cores from before the unified error pair). */
function legacyAbort(reason: string): OmniMessage {
const msg = abortEvent();
delete (msg.payload as { error_code?: string }).error_code;
(msg.payload as { reason?: string }).reason = reason;
return msg;
}
/** Feeds a sequence of messages and finalizes (close: persists any still-pending failure), returning the persisted rows. */
function feed(msgs: OmniMessage[]): Array<Record<string, unknown>> {
const w = watcher();
for (const m of msgs) w.observe(m);
w.close();
return rows();
}
/**
* A sub-session's session_meta (its first message): origin = the child Session
* id, and agentId is derived from the parent directory name in the `agent_state`
* path (consistent with SessionManager.registerChildSession).
*/
const childMeta = (sessionId: string, agentState: string) =>
withOrigin(
sessionMeta({
session_id: sessionId,
model_id: "m1",
provider: "custom",
model_context_window: 100000,
system_prompt: "",
agent_state: agentState,
workspace: "/tmp/w",
source: "user",
}),
sessionId,
);
// —— LLM ——
it("an unrecovered LLM failed → unexpected (needs a human); message takes the abort reason", () => {
// Nothing follows this failure but the abort, so the ladder did not carry it: the user
// lost the turn and it belongs in front of an operator.
const got = feed([
requestBegin(),
requestEnd("retryable"),
legacyAbort("llm request failed after 5 retries: 400 unknown parameter"),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
source: "llm",
kind: "unexpected",
code: "llm_failed",
message: "llm request failed after 5 retries: 400 unknown parameter",
project_id: "p1",
agent_id: "a1",
session_id: "s1",
status: null,
});
});
it("a retried retryable → expected; an exhausted one → unexpected; messages from abort/fallback", () => {
const got = feed([
requestBegin(),
requestEnd("retryable"), // First attempt fails → the engine retries (revealed by the next request_begin: no reason text yet)
requestBegin(),
requestEnd("retryable"),
legacyAbort("llm request failed after 2 retries"),
]);
expect(got).toHaveLength(2);
expect(got[0]).toMatchObject({ source: "llm", kind: "expected", code: "llm_retried" });
// No abort arrived for the carried failure: falls back to the generic status text.
expect(got[0]!.message).toContain("reconnects and retries");
expect(got[1]).toMatchObject({
source: "llm",
kind: "unexpected",
code: "llm_failed",
message: "llm request failed after 2 retries",
});
});
it("request_end(fatal) gets its own llm_fatal code, out of the retried dedup bucket", () => {
// A run-ending rejection needs a code of its own: dedup is (source, code, Project)
// over a short window, and `llm_retried` fires on every blip the ladder absorbs —
// sharing a bucket would let a real credential failure be dropped as a duplicate.
const got = feed([
requestBegin(),
requestEnd("fatal", { errorMessage: "401 invalid x-api-key (invalid_api_key)" }),
legacyAbort("llm request error: 401 invalid x-api-key (invalid_api_key)"),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
source: "llm",
kind: "unexpected",
code: "llm_fatal",
message: "llm request error: 401 invalid x-api-key (invalid_api_key)",
});
});
it("a fatal with no abort (the live protocol): close persists it with the same prose", () => {
// Failures no longer emit an abort event; the pending record resolves at close, and
// the message is composed from the request_end's own detail.
const got = feed([
requestBegin(),
requestEnd("fatal", { errorMessage: "401 invalid x-api-key (invalid_api_key)" }),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
source: "llm",
kind: "unexpected",
code: "llm_fatal",
message: "llm request error: 401 invalid x-api-key (invalid_api_key)",
});
});
it("an exhausted ladder with no abort: the terminal request_end's attempt shapes the record", () => {
const got = feed([
requestBegin(),
requestEnd("retryable", { errorMessage: "socket hang up", attempt: 1, retryInMs: 2000 }),
requestBegin(),
// The final failure plans no retry (no retry_in_ms) — the run ends on it.
requestEnd("retryable", { errorMessage: "socket hang up", attempt: 2 }),
]);
expect(got).toHaveLength(2);
expect(got[0]).toMatchObject({ code: "llm_retried", kind: "expected" });
expect(got[1]).toMatchObject({
source: "llm",
kind: "unexpected",
code: "llm_failed",
message: "llm request failed after 1 retry: socket hang up",
});
});
it("a retryable the ladder carried → expected under its own code, not an operator incident", () => {
// The same status covers "a gateway hiccup the user never saw" and "the run died on
// it". A following request_begin proves another attempt happened — that one is
// expected.
const got = feed([
requestBegin(),
requestEnd("retryable", {
errorMessage: "Upstream HTTP/2 stream failed (upstream_http2_stream_error)",
}),
requestBegin(),
requestEnd("completed"),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
source: "llm",
kind: "expected",
code: "llm_retried",
// No abort ever arrives on the retry path: the staged request_end's own detail is the
// message of record.
message: "Upstream HTTP/2 stream failed (upstream_http2_stream_error)",
});
});
it("a recovered retryable does not dedup away a fatal that lands right after", () => {
// The two share a 2s dedup window; separate codes keep the one failure that always
// needs a human from being silenced by the one that never does.
const got = feed([
requestBegin(),
requestEnd("retryable", { errorMessage: "Upstream HTTP/2 stream failed" }),
requestBegin(), // The retry: resolves the failure above as recovered.
requestEnd("fatal", { errorMessage: "401 invalid x-api-key" }),
legacyAbort("llm request error: 401 invalid x-api-key"),
]);
expect(got.map((r) => [r.code, r.kind])).toEqual([
["llm_retried", "expected"],
["llm_fatal", "unexpected"],
]);
});
it("interrupt during backoff with a staged detail: the detail wins over the status text", () => {
// The interrupt message is distrusted (not the failure's reason), but the staged
// request_end detail IS the failure's reason — prefer it over the generic status text.
const got = feed([
requestBegin(),
// A backoff interrupt cuts a PLANNED retry short — the failure announced one.
requestEnd("retryable", { errorMessage: "429 rate limited (slow down)", retryInMs: 4000 }),
abortEvent("backoff_interrupted"),
]);
expect(got).toHaveLength(1);
expect(got[0]!.message).toBe("429 rate limited (slow down)");
});
it("aborted (user clicked Stop) is not an error: not recorded; neither is completed", () => {
expect(
feed([
requestBegin(),
requestEnd("completed"),
requestBegin(),
requestEnd("aborted"),
abortEvent(),
]),
).toHaveLength(0);
});
it("interrupt during retry backoff: the failure still recorded, interrupt text distrusted", () => {
const got = feed([requestBegin(), requestEnd("retryable"), abortEvent("backoff_interrupted")]);
expect(got).toHaveLength(1);
// No retry followed (the interrupt ended the run): the conservative branch records it
// as the unrecovered class.
expect(got[0]).toMatchObject({ code: "llm_failed", kind: "unexpected" });
expect(got[0]!.message).toContain("did not recover");
expect(got[0]!.message).not.toContain("aborted");
});
it("failure pends for its reason; unresolved at run end → close persists (status text)", () => {
const w = watcher();
w.observe(requestBegin());
w.observe(requestEnd("retryable"));
expect(rows()).toHaveLength(0); // Pending: waiting for the abort that immediately follows to supply the real reason
w.close();
const got = rows();
expect(got).toHaveLength(1);
// close() is not proof of a retry, so it takes the conservative branch: unrecovered.
expect(got[0]).toMatchObject({ code: "llm_failed", kind: "unexpected" });
expect(got[0]!.message).toBe("LLM request failed and the retries did not recover it.");
});
it("parent/child LLM failures pend separately by origin; abort reasons never cross over", () => {
// The child fails fatal and the parent retryable-unrecovered: distinct codes, so the
// short-window dedup (source, code, Project) suppresses neither.
const got = feed([
requestBegin(), // parent session initiates
withOrigin(requestBegin(), "session-child"),
withOrigin(requestEnd("fatal"), "session-child"),
withOrigin(legacyAbort("llm request error: 401 invalid api key"), "session-child"),
requestEnd("retryable"), // the parent session's failure only wraps up now
legacyAbort("llm request error: 500 upstream"),
]);
expect(got).toHaveLength(2);
expect(got[0]).toMatchObject({
code: "llm_fatal",
message: "llm request error: 401 invalid api key",
});
expect(got[1]).toMatchObject({
code: "llm_failed",
message: "llm request error: 500 upstream", // not stolen by the sub-session's abort
});
});
// —— Environment (tool execution) ——
const call = (name: string, id: string) => toolCall({ name, arguments: "{}", toolCallId: id });
it("a command tool's ordinary non-zero exit is not recorded: an exit status is information", () => {
// Both command tools end in resultForExit, which maps ANY non-zero exit to `failed`, so
// grep finding nothing (exit 1), `test -f` on a missing file, or a diff that differs would
// all land in the cost center and bury the real errors. input_command is covered alongside
// exec_command because it is how a backgrounded command is polled for its eventual exit —
// dropping one but not the other would depend on where the command happened to finish.
// Every other tool still records, whatever its output says.
const got = feed([
call("exec_command", "tc-1"),
toolCallOutput({
output: "grep: no match\n[exit code: 1]",
toolCallId: "tc-1",
stopReason: "fatal",
}),
call("input_command", "tc-2"),
toolCallOutput({
output: "make: *** [build] Error 2\n[exit code: 2]",
toolCallId: "tc-2",
stopReason: "fatal",
}),
call("write_file", "tc-3"),
toolCallOutput({
output: "EACCES\n[exit code: 1]",
toolCallId: "tc-3",
stopReason: "fatal",
}),
]);
expect(got.map((r) => r.code)).toEqual(["tool_fatal:write_file"]);
});
it("a command tool killed by a signal, or that never spawned, is still recorded", () => {
// `failed` from these tools is not only "exited non-zero": an OOM kill or a segfault
// (resultForExit's signal branch) and a spawn failure (nonexistent workdir, EMFILE, an
// unresolvable shell) are config/environment faults — the recorder's "needs a human"
// category — that no amount of Agent self-correction gets around. Only the exit marker
// separates them, which is why the rule reads the note rather than the tool name.
const got = feed([
call("exec_command", "tc-1"),
toolCallOutput({
output: "cc1plus: out of memory\n[terminated by signal SIGKILL]",
toolCallId: "tc-1",
stopReason: "fatal",
}),
call("input_command", "tc-2"),
toolCallOutput({
output: "[spawn error: ENOENT: no such file or directory, posix_spawn '/bin/nope']",
toolCallId: "tc-2",
stopReason: "fatal",
}),
]);
expect(got.map((r) => r.code)).toEqual(["tool_fatal:exec_command", "tool_fatal:input_command"]);
});
it("a command tool's timeout and a missing session manager are recorded (neither is an exit)", () => {
// Environment finalizes a tool timeout as stop_reason `failed` plus its own note — it never
// emits stop_reason "timeout" for these tools — so a hung command surfaces as tool_failed
// with no exit marker to drop it. A missing command session manager is a server
// misconfiguration and produces no exit marker either.
const got = feed([
call("exec_command", "tc-1"),
toolCallOutput({
output: "still building…\n[tool timeout: exceeded 60000ms]",
toolCallId: "tc-1",
stopReason: "fatal",
}),
call("input_command", "tc-2"),
toolCallOutput({
output: "[input_command unavailable: no command session manager configured]",
toolCallId: "tc-2",
stopReason: "fatal",
}),
]);
expect(got.map((r) => r.code)).toEqual(["tool_fatal:exec_command", "tool_fatal:input_command"]);
});
it("an exit marker with no cached tool name is dropped, not filed under tool_fatal:unknown", () => {
// Only the command tools ever write that marker, so a cache miss (the tool_call evicted, or
// never seen) is still that noise — recording it nameless would defeat the exclusion.
const got = feed([
toolCallOutput({ output: "[exit code: 1]", toolCallId: "tc-1", stopReason: "fatal" }),
toolCallOutput({ output: "boom", toolCallId: "tc-2", stopReason: "fatal" }),
]);
expect(got.map((r) => [r.code, r.message])).toEqual([["tool_fatal:unknown", "boom"]]);
});
it("a tool fatal → environment + expected, code carries the tool name; legacy failed/timeout spellings still record", () => {
const legacyOutput = (args: { output: string; toolCallId: string; stopReason: string }) =>
toolCallOutput(args as Parameters<typeof toolCallOutput>[0]);
const got = feed([
call("write_file", "tc-1"),
toolCallOutput({
output: "EACCES: permission denied\n[tool error] write failed",
toolCallId: "tc-1",
stopReason: "fatal",
}),
// Traces written before the stop-reason unification spell tool failures failed/timeout.
call("read_file", "tc-2"),
legacyOutput({
output: "[tool timeout: exceeded 30000ms]",
toolCallId: "tc-2",
stopReason: "timeout",
}),
call("edit_file", "tc-3"),
legacyOutput({ output: "[tool error] boom", toolCallId: "tc-3", stopReason: "failed" }),
]);
expect(got).toHaveLength(3);
expect(got[0]).toMatchObject({
source: "environment",
kind: "expected", // error fed back to the model; the Agent adjusts on its own — no human needed
code: "tool_fatal:write_file",
project_id: "p1",
agent_id: "a1",
session_id: "s1",
});
expect(got[0]!.message).toContain("[tool error] write failed"); // the actual error text
expect(got[1]).toMatchObject({ code: "tool_timeout:read_file", kind: "expected" });
expect(got[2]).toMatchObject({ code: "tool_failed:edit_file", kind: "expected" });
});
it("tool aborted (denial / user interrupt) and completed are not recorded", () => {
expect(
feed([
call("write_file", "tc-1"),
toolCallOutput({
output: "Tool call denied by user.",
toolCallId: "tc-1",
stopReason: "aborted",
}),
call("read_file", "tc-2"),
toolCallOutput({ output: "ok", toolCallId: "tc-2", stopReason: "completed" }),
]),
).toHaveLength(0);
});
it("parallel tools: each tool_call_id maps to its own name despite out-of-order outputs", () => {
const got = feed([
call("write_file", "tc-1"),
call("read_file", "tc-2"),
call("write_file", "tc-3"),
toolCallOutput({ output: "boom-2", toolCallId: "tc-2", stopReason: "fatal" }),
toolCallOutput({ output: "ok", toolCallId: "tc-3", stopReason: "completed" }),
toolCallOutput({ output: "boom-1", toolCallId: "tc-1", stopReason: "fatal" }),
]);
expect(got.map((r) => r.code)).toEqual(["tool_fatal:read_file", "tool_fatal:write_file"]);
expect(got.map((r) => r.message)).toEqual(["boom-2", "boom-1"]);
});
it("a child session's tool failure: no name mix-up with the parent's equal tool_call_id", () => {
const got = feed([
call("read_file", "tc-1"), // parent session
withOrigin(call("write_file", "tc-1"), "session-child"), // sub-session happens to share the same id
withOrigin(
toolCallOutput({ output: "child boom", toolCallId: "tc-1", stopReason: "fatal" }),
"session-child",
),
toolCallOutput({ output: "parent boom", toolCallId: "tc-1", stopReason: "fatal" }),
]);
expect(got).toHaveLength(2);
expect(got[0]).toMatchObject({
code: "tool_fatal:write_file", // the sub-session's tool name, not overwritten by the parent's tc-1
message: "child boom",
session_id: "s1", // this test didn't feed the sub-session's session_meta → attribution falls back to the parent ctx (see the "attribution" test cases below)
});
expect(got[1]).toMatchObject({ code: "tool_fatal:read_file", message: "parent boom" });
});
it("overlong tool output: message takes the tail (the reason is at the end)", () => {
const got = feed([
call("write_file", "tc-1"),
toolCallOutput({
output: `${"x".repeat(2000)}\n[tool error] boom`,
toolCallId: "tc-1",
stopReason: "fatal",
}),
]);
const message = got[0]!.message as string;
expect(message.length).toBe(MESSAGE_MAX);
expect(message.startsWith("…")).toBe(true);
expect(message.endsWith("[tool error] boom")).toBe(true); // truncating from the head would cut off the reason entirely
});
it("irrelevant messages are a no-op: body text and streaming partial_*", () => {
expect(
feed([
assistantText("normal output"),
call("write_file", "tc-1"),
partialToolCallOutput({ eventType: "stop", toolCallId: "tc-1", stopReason: "fatal" }),
]),
).toHaveLength(0);
});
// —— Attribution: an error is recorded against **the session that actually produced it** (a sub-session's failure must not be attributed to the parent Agent) ——
it("a child session's LLM failure attributes to the child Agent / Session", () => {
const got = feed([
childMeta("session-child", "/data/agents/agent-child/agent_state"),
withOrigin(requestBegin(), "session-child"),
withOrigin(requestEnd("retryable"), "session-child"),
withOrigin(legacyAbort("llm request error: 401 invalid api key"), "session-child"),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
source: "llm",
code: "llm_failed",
message: "llm request error: 401 invalid api key",
agent_id: "agent-child", // derived from the agent_state path; not the parent's a1
session_id: "session-child",
project_id: "p1", // projectId always takes the parent's (a sub-session is always in the same Project)
});
});
it("a child session's tool failure attributes to it (code still carries the tool name)", () => {
const got = feed([
childMeta("session-child", "/data/agents/agent-child/agent_state"),
withOrigin(call("write_file", "tc-1"), "session-child"),
withOrigin(
toolCallOutput({ output: "child boom", toolCallId: "tc-1", stopReason: "fatal" }),
"session-child",
),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
source: "environment",
code: "tool_fatal:write_file",
message: "child boom",
agent_id: "agent-child",
session_id: "session-child",
project_id: "p1",
});
});
it("parent/child interleaving: each attributes to its own; no-origin goes to the parent", () => {
const got = feed([
requestBegin(), // parent session initiates
childMeta("session-child", "/data/agents/agent-child/agent_state"),
withOrigin(requestBegin(), "session-child"),
withOrigin(requestEnd("fatal"), "session-child"),
withOrigin(legacyAbort("llm request error: 401 dead key"), "session-child"),
withOrigin(call("write_file", "tc-9"), "session-child"),
withOrigin(
toolCallOutput({ output: "child tool boom", toolCallId: "tc-9", stopReason: "fatal" }),
"session-child",
),
call("read_file", "tc-9"), // parent session happens to share the same id
toolCallOutput({ output: "parent tool boom", toolCallId: "tc-9", stopReason: "fatal" }),
requestEnd("retryable"), // the parent session's LLM failure only wraps up now
legacyAbort("llm request error: 500 upstream"),
]);
// The sub-session's LLM / tool failures attribute to it, the parent's to the parent — the four entries never mix (each has a distinct code, so short-window dedup doesn't suppress any of them).
expect(got.map((r) => [r.code, r.agent_id, r.session_id])).toEqual([
["llm_fatal", "agent-child", "session-child"],
["tool_fatal:write_file", "agent-child", "session-child"],
["tool_fatal:read_file", "a1", "s1"],
["llm_failed", "a1", "s1"],
]);
});
it("failure before session_meta arrives: falls back to the parent ctx, no crash", () => {
const got = feed([
withOrigin(requestEnd("retryable"), "session-child"), // the sub-session's meta hasn't arrived yet
withOrigin(legacyAbort("llm request error: 500"), "session-child"),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({ code: "llm_failed", agent_id: "a1", session_id: "s1" });
});
it("malformed agent_state path (empty): not registered, falls back to the parent ctx", () => {
const got = feed([
childMeta("session-child", ""), // path.basename(path.dirname("")) === "." → caught by the defensive check
withOrigin(requestEnd("retryable"), "session-child"),
withOrigin(legacyAbort("llm request error: 500"), "session-child"),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({ code: "llm_failed", agent_id: "a1", session_id: "s1" });
});
// —— Compaction ——
it("an abandoned compaction records one error row; the message carries attempts and the error", () => {
// Issue #170: a compaction is an ordinary LLM request whose failures core retries under
// the standard budget — a retryable end means the retries ran out, and the cost
// center's row shows how many attempts were spent and what the last failure was.
const got = feed([
compactionEnd({
reason: "context",
mode: "summarize",
status: "retryable",
attempt: 5,
errorMessage: "the response contained no usable summary",
}),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
source: "compaction",
kind: "unexpected",
code: "compaction_failed",
message:
"summarize compaction failed after 5 attempts: the response contained no usable summary; trigger context, original context kept.",
project_id: "p1",
agent_id: "a1",
session_id: "s1",
});
});
it("compaction completed / aborted are not errors; fatal and an old-core failed still record", () => {
const legacyEnd = (status: string) =>
compactionEnd({ reason: "turns", mode: "summarize", status: status as StopReason });
const got = feed([
compactionEnd({ reason: "context", mode: "summarize", status: "completed", attempt: 1 }),
compactionEnd({ reason: "manual", mode: "summarize", status: "aborted", attempt: 2 }),
compactionEnd({ reason: "context", mode: "summarize", status: "fatal", attempt: 1 }),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({ code: "compaction_failed" });
// An old core's compaction_end (retired `failed` spelling, no attempt/error fields)
// still records — its own feed, since the short-window dedup shares the code above.
const legacy = feed([legacyEnd("failed")]);
expect(legacy).toHaveLength(2); // rows are cumulative: the fatal row above plus this one
expect(legacy[1]).toMatchObject({
code: "compaction_failed",
message: "summarize compaction failed; trigger turns, original context kept.",
});
});
it("a child session's failed compaction attributes to the child Agent/Session", () => {
const got = feed([
childMeta("session-child", "/data/agents/agent-child/agent_state"),
withOrigin(
compactionEnd({ reason: "context", mode: "summarize", status: "retryable", attempt: 6 }),
"session-child",
),
]);
expect(got).toHaveLength(1);
expect(got[0]).toMatchObject({
code: "compaction_failed",
agent_id: "agent-child",
session_id: "session-child",
});
});
});
describe("HTTP onError persistence (integration)", () => {
let t: TestApp;
let api: ReturnType<typeof apiClient>;
let projectId: string;
beforeEach(async () => {
t = await createTestApp();
const u = await provisionUser(t.app, "err_user");
api = apiClient(t.app, u.cookie);
const created = (await (
await api.post("/api/projects", { projectId: "err_user-proj", name: "Error project" })
).json()) as ProjectCreateResponse;
projectId = created.project.projectId;
});
afterEach(async () => {
await t.cleanup();
});
const errorRows = () =>
t.deps.db.prepare("SELECT * FROM error_records ORDER BY id").all() as Array<
Record<string, unknown>
>;
it("business error (HttpError 404) → expected, with code / status / projectId", async () => {
const res = await api.get(`/api/projects/${projectId}/agents/agent-nope/sessions`);
expect(res.status).toBe(404);
const rows = errorRows();
expect(rows).toHaveLength(1);
expect(rows[0]!.source).toBe("http");
expect(rows[0]!.kind).toBe("expected");
expect(rows[0]!.code).toBe("agent_not_found");
expect(rows[0]!.status).toBe(404);
// Taken from the route params, but only when the requester actually has access — see the "HTTP error attribution" test group below.
expect(rows[0]!.project_id).toBe(projectId);
});
it("unexpected error (service layer throws a plain Error) → unexpected + 500", async () => {
// handleError logs the stack trace: silence it in the test so it doesn't clutter output.
const spy = vi.spyOn(console, "error").mockImplementation(() => {});
t.deps.usageService.query = () => {
throw new Error("query blew up");
};
const res = await api.get(`/api/projects/${projectId}/usage?groupBy=date`);
expect(res.status).toBe(500);
spy.mockRestore();
const rows = errorRows();
expect(rows).toHaveLength(1);
expect(rows[0]!.kind).toBe("unexpected");
expect(rows[0]!.code).toBe("internal");
expect(rows[0]!.status).toBe(500);
expect(rows[0]!.message).toBe("query blew up");
expect(rows[0]!.project_id).toBe(projectId);
});
it("a Skill directory another program is holding open → 409 skill_dir_busy, recorded as expected", async () => {
const agentId = "agent-busy";
const dir = path.join(skillsDir(t.root, projectId, agentId), "busy-skill");
await fs.mkdir(dir, { recursive: true });
await fs.writeFile(path.join(dir, "SKILL.md"), "---\nname: busy-skill\n---\nBody\n");
// Windows reports EBUSY ("resource busy or locked") on every removal while another program
// still holds a handle inside the directory; Core retries and eventually gives up. The answer
// must name what is wrong (409), not converge to a bare 500, and the error center must file it
// where the user can act on it rather than as an internal bug.
const realRm = fs.rm.bind(fs);
const rm = vi.spyOn(fs, "rm").mockImplementation(async (target, options) => {
if (path.resolve(String(target)) === path.resolve(dir)) {
throw Object.assign(
new Error(`EBUSY: resource busy or locked, rmdir '${String(target)}'`),
{ code: "EBUSY" },
);
}
return realRm(target, options);
});
const res = await api.delete(`/api/projects/${projectId}/agents/${agentId}/skills/busy-skill`);
rm.mockRestore();
expect(res.status).toBe(409);
expect(await res.json()).toMatchObject({ error: { code: "skill_dir_busy" } });
const rows = errorRows();
expect(rows).toHaveLength(1);
expect(rows[0]!.source).toBe("http");
expect(rows[0]!.kind).toBe("expected");
expect(rows[0]!.code).toBe("skill_dir_busy");
expect(rows[0]!.status).toBe(409);
expect(rows[0]!.project_id).toBe(projectId);
});
it("errors exposed via the usage endpoint: summary / top code / recent", async () => {
// An error in another Project owned by the same owner: attributed to that Project, not this one's view.
const other = (await (
await api.post("/api/projects", { projectId: "err_user-proj_2", name: "Another project" })
).json()) as ProjectCreateResponse;
await api.get(`/api/projects/${other.project.projectId}/agents/agent-nope/sessions`); // 404
await api.get(`/api/projects/${projectId}/agents/agent-nope/sessions`); // 404 → expected
const res = await api.get(`/api/projects/${projectId}/usage?groupBy=date`);
const body = (await res.json()) as UsageResponse;
// The entry from another Project doesn't count in this view (only this Project's errors + admin-visible unattributed errors show here).
expect(body.errors.total).toBe(1);
expect(body.errors.unexpected).toBe(0);
expect(body.errors.topCode).toEqual({
source: "http",
code: "agent_not_found",
kind: "expected",
count: 1,
});
expect(body.errors.recent[0]).toMatchObject({ source: "http", code: "agent_not_found" });
});
it("unattributed errors (login failure) are admin-only, hidden from members", async () => {
// A login failure has no Project context → produces one unattributed error (project_id is NULL).
const bad = await t.app.request("/api/auth/login", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ userId: "err_user", password: "wrong-password" }),
});
expect(bad.status).toBe(401);
expect(errorRows().filter((r) => r.project_id === null)).toHaveLength(1);
// A regular member in their own Project: sees none of it.
const plain = await provisionUser(t.app, "plain_user");
expect(plain.user.isAdmin).toBe(false);
const plainApi = apiClient(t.app, plain.cookie);
const own = (await (
await plainApi.post("/api/projects", { projectId: "plain_user-proj", name: "Member project" })
).json()) as ProjectCreateResponse;
const plainBody = (await (
await plainApi.get(`/api/projects/${own.project.projectId}/usage?groupBy=date`)
).json()) as UsageResponse;
expect(plainBody.errors).toMatchObject({ total: 0, unexpected: 0, topCode: null, recent: [] });
// The admin can see it — the category most in need of visibility isn't rendered invisible by the isolation.
const adminApi = apiClient(t.app, (await loginAdmin(t.app)).cookie);
const adminBody = (await (
await adminApi.get(`/api/projects/default_project/usage?groupBy=date`)
).json()) as UsageResponse;
expect(adminBody.errors.total).toBe(1);
expect(adminBody.errors.topCode).toMatchObject({ source: "http", code: "invalid_credentials" });
expect(adminBody.errors.recent[0]).toMatchObject({ code: "invalid_credentials" });
});
it("the paged error route pages inside the caller's own tenant, at every offset", async () => {
// The point of the route is that paging back is not a way around the dashboard's isolation:
// the rows a member can reach at offset N are the same set the summary counted, never
// another tenant's and never the unattributed ones. Seeded directly so the interleaving is
// exact — going through HTTP would collapse repeats into the recorder's dedup window.
const repo = wire(ErrorsRepo, { db: t.deps.db });
const seed = (owner: string | null, code: string) =>
repo.insert({
ts: "2026-07-27T00:00:00.000Z",
date: "2026-07-27",
projectId: owner,
agentId: "a1",
sessionId: "s1",
source: "http",
kind: "expected",
code,
status: 404,
message: code,
});
const foreign = (await (
await api.post("/api/projects", { projectId: "err_user-tenant_2", name: "Other tenant" })
).json()) as ProjectCreateResponse;
// Interleaved, so a mistaken filter would show up inside the first page rather than past it.
seed(projectId, "mine_0");
seed(foreign.project.projectId, "theirs_0");
seed(null, "unattributed_0");
seed(projectId, "mine_1");
seed(projectId, "mine_2");
const pageOf = async (offset: number, limit: number) =>
(await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=${offset}&limit=${limit}`)
).json()) as UsageErrorsPage;
const first = await pageOf(0, 2);
expect(first.total).toBe(3); // the filtered count, not the table's
expect(first.items.map((e) => e.code)).toEqual(["mine_2", "mine_1"]); // newest first
const second = await pageOf(2, 2);
expect(second.total).toBe(3);
expect(second.items.map((e) => e.code)).toEqual(["mine_0"]);
expect((await pageOf(3, 2)).items).toEqual([]); // past the end is empty, not an error
// The admin is the only one who reaches the unattributed rows — the same rule the dashboard
// applies, so paging cannot be used to slip past it.
const adminApi = apiClient(t.app, (await loginAdmin(t.app)).cookie);
const adminPage = (await (
await adminApi.get(`/api/projects/default_project/usage/errors?offset=0&limit=20`)
).json()) as UsageErrorsPage;
expect(adminPage.items.map((e) => e.code)).toEqual(["unattributed_0"]);
});
it("the paged error route rejects a page it cannot serve rather than guessing", async () => {
expect((await api.get(`/api/projects/${projectId}/usage/errors?offset=-1`)).status).toBe(400);
expect((await api.get(`/api/projects/${projectId}/usage/errors?limit=0`)).status).toBe(400);
expect((await api.get(`/api/projects/${projectId}/usage/errors?limit=1001`)).status).toBe(400);
expect((await api.get(`/api/projects/${projectId}/usage/errors?from=2026-13-01`)).status).toBe(
400,
);
// A `kind` outside the two categories is a bad request, not a silently empty page: the
// cost-center badge reads `total` as "is anything waiting", and a typo answering 0 would
// read as "nothing is".
expect((await api.get(`/api/projects/${projectId}/usage/errors?kind=oops`)).status).toBe(400);
});
it("the paged error route narrows to one category on request, counting only that one", async () => {
// What the cost-center badge asks: one row of `unexpected`, for the count and the newest
// timestamp its dismissal is stamped against. Seeded directly, so the interleaving is exact.
const repo = wire(ErrorsRepo, { db: t.deps.db });
const seed = (kind: string, code: string, ts: string) =>
repo.insert({
ts,
date: ts.slice(0, 10),
projectId,
agentId: "a1",
sessionId: "s1",
source: "http",
kind,
code,
status: kind === "unexpected" ? 500 : 404,
message: code,
});
seed("expected", "expected_0", "2026-07-27T00:00:00.000Z");
seed("unexpected", "unexpected_0", "2026-07-27T01:00:00.000Z");
seed("expected", "expected_1", "2026-07-27T02:00:00.000Z");
seed("unexpected", "unexpected_1", "2026-07-27T03:00:00.000Z");
const probe = (await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=1&kind=unexpected`)
).json()) as UsageErrorsPage;
expect(probe.total).toBe(2); // only the unexpected ones are counted
expect(probe.items.map((e) => e.code)).toEqual(["unexpected_1"]); // newest first, one row
const expectedOnly = (await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20&kind=expected`)
).json()) as UsageErrorsPage;
expect(expectedOnly.items.map((e) => e.code)).toEqual(["expected_1", "expected_0"]);
// No `kind` still counts both, so the panel the badge leads to is unchanged.
const all = (await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20`)
).json()) as UsageErrorsPage;
expect(all.total).toBe(4);
});
it("the paged error route and the dashboard fold a day's repeats by the viewer's day", async () => {
// Three of the same send failure: two on the evening of the 27th in UTC and one just after
// UTC midnight — the same evening for a reader five hours behind.
const repo = wire(ErrorsRepo, { db: t.deps.db });
for (const ts of [
"2026-07-27T20:00:00.000Z",
"2026-07-27T22:00:00.000Z",
"2026-07-28T01:00:00.000Z",
]) {
repo.insert({
ts,
date: ts.slice(0, 10),
projectId,
agentId: "a1",
sessionId: "s1",
source: "messaging",
kind: "expected",
code: "messaging_send_failed",
status: null,
message: "wechat send failed: prepare failed",
});
}
const pageFor = async (query: string) =>
(await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20${query}`)
).json()) as UsageErrorsPage;
const eastern = await pageFor("&utcOffsetMinutes=-300");
expect(eastern.items).toHaveLength(1);
expect(eastern.items[0]).toMatchObject({
count: 3,
ts: "2026-07-28T01:00:00.000Z",
firstTs: "2026-07-27T20:00:00.000Z",
});
// Rows are what a pager pages through; the records behind them are still the total.
expect(eastern.rows).toBe(1);
expect(eastern.total).toBe(3);
// Without an offset the day is the server's, the `date` column: two days, two rows.
const server = await pageFor("");
expect(server.items.map((e) => e.count)).toEqual([1, 2]);
expect(server.rows).toBe(2);
// The dashboard's first page folds the same way, and its summary keeps counting records.
const dashboard = (await (
await api.get(`/api/projects/${projectId}/usage?utcOffsetMinutes=-300`)
).json()) as UsageResponse;
expect(dashboard.errors.recent.map((e) => e.count)).toEqual([3]);
expect(dashboard.errors.rows).toBe(1);
expect(dashboard.errors.total).toBe(3);
// An offset no clock uses, or one that is not a whole number, is a 400 rather than a
// quietly different day.
for (const bad of ["900", "-841", "1.5", "east", "+60"]) {
expect(
(await api.get(`/api/projects/${projectId}/usage/errors?utcOffsetMinutes=${bad}`)).status,
).toBe(400);
}
expect((await api.get(`/api/projects/${projectId}/usage?utcOffsetMinutes=900`)).status).toBe(
400,
);
});
/** Seeds one row straight into the table, so a batch's dates and Agents are exact. */
const seedRow = (o: Partial<ErrorRecordInsert> & { date: string; code: string }) =>
wire(ErrorsRepo, { db: t.deps.db }).insert({
ts: `${o.date}T00:00:00.000Z`,
projectId,
agentId: null,
sessionId: null,
source: "http",
kind: "expected",
status: 404,
message: o.code,
...o,
});
it("clearing errors takes the filter on screen and nothing outside it", async () => {
seedRow({ date: "2026-07-05", code: "before_range" });
seedRow({ date: "2026-07-06", code: "in_range_a1", agentId: "a1" });
seedRow({ date: "2026-07-06", code: "in_range_a2", agentId: "a2" });
seedRow({ date: "2026-07-08", code: "after_range" });
// Narrowed to one Agent inside the range: exactly the rows that filter reads.
const narrow = await api.delete(
`/api/projects/${projectId}/usage/errors?from=2026-07-06&to=2026-07-06&agentId=a1`,
);
expect(narrow.status).toBe(200);
expect(await narrow.json()).toEqual({ deleted: 1 });
expect(errorRows().map((r) => r.code)).toEqual(["before_range", "in_range_a2", "after_range"]);
// The range on its own still spares the dates the reader was not looking at.
const ranged = await api.delete(
`/api/projects/${projectId}/usage/errors?from=2026-07-06&to=2026-07-07`,
);
expect(await ranged.json()).toEqual({ deleted: 1 });
expect(errorRows().map((r) => r.code)).toEqual(["before_range", "after_range"]);
// A missing bound is refused rather than read as unbounded: an open side is the Project's
// whole history, which is wider than any filter a panel could have shown.
for (const query of ["", "?from=2026-07-06", "?to=2026-07-06"]) {
const refused = await api.delete(`/api/projects/${projectId}/usage/errors${query}`);
expect(refused.status).toBe(400);
}
// Those refusals are themselves recorded by onError, so the seeded rows are what is
// checked: nothing outside the two deletes above went.
expect(
errorRows()
.map((r) => r.code)
.filter((code) => code !== "bad_request"),
).toEqual(["before_range", "after_range"]);
});
it("a category-narrowed clear leaves the other category alone", async () => {
const day = "2026-07-06";
seedRow({ date: day, code: "boom", kind: "unexpected" });
seedRow({ date: day, code: "missing", kind: "expected" });
const kept = () =>
errorRows()
.map((r) => r.code)
.filter((code) => code !== "bad_request");
// What the panel's narrowed table reads — the same `kind` its clear carries, which is the
// point: the button empties the table on screen, never the category filtered out of it.
const narrowed = (await (
await api.get(
`/api/projects/${projectId}/usage/errors?offset=0&limit=20&from=${day}&to=${day}&kind=unexpected`,
)
).json()) as UsageErrorsPage;
expect(narrowed.items.map((e) => e.code)).toEqual(["boom"]);
expect(narrowed.total).toBe(1);
const cleared = await api.delete(
`/api/projects/${projectId}/usage/errors?from=${day}&to=${day}&kind=unexpected`,
);
expect(await cleared.json()).toEqual({ deleted: 1 });
expect(kept()).toEqual(["missing"]);
// A category outside the closed set is refused rather than matched against nothing: a clear
// that silently deleted none of them reads as "already empty" while they are all still there.
const refused = await api.delete(
`/api/projects/${projectId}/usage/errors?from=${day}&to=${day}&kind=oops`,
);
expect(refused.status).toBe(400);
expect(kept()).toEqual(["missing"]);
});
it("a trailing-window clear deletes exactly the rows the window listed", async () => {
const day = "2026-07-06";
seedRow({ date: day, code: "before_window", ts: `${day}T09:59:59.999Z` });
seedRow({ date: day, code: "at_from", ts: `${day}T10:00:00.000Z` });
seedRow({ date: day, code: "at_to", ts: `${day}T11:00:00.000Z` });
seedRow({ date: day, code: "after_window", ts: `${day}T11:00:00.001Z` });
const window = `from=${day}&to=${day}&fromTs=${day}T10:00:00.000Z&toTs=${day}T11:00:00.000Z`;
// What the panel lists for the window …
const listed = (await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20&${window}`)
).json()) as UsageErrorsPage;
expect(listed.items.map((e) => e.code)).toEqual(["at_to", "at_from"]);
expect(listed.total).toBe(2);
// … is what the clear of the same window takes: the same day's rows a millisecond
// outside it stay.
const cleared = await api.delete(`/api/projects/${projectId}/usage/errors?${window}`);
expect(await cleared.json()).toEqual({ deleted: 2 });
expect(errorRows().map((r) => r.code)).toEqual(["before_window", "after_window"]);
// Half a window is refused, as the dashboard refuses it.
const half = await api.delete(
`/api/projects/${projectId}/usage/errors?fromTs=${day}T10:00:00.000Z`,
);
expect(half.status).toBe(400);
});
it("a clear stops at the boundary its confirmation counted", async () => {
const day = "2026-07-06";
seedRow({ date: day, code: "counted_a", ts: `${day}T10:00:00.000Z` });
seedRow({ date: day, code: "counted_b", ts: `${day}T10:05:00.000Z` });
// What the panel reads when the confirmation opens: the count, and the newest record's
// instant, which is the boundary the delete carries.
const counted = (await (
await api.get(
`/api/projects/${projectId}/usage/errors?offset=0&limit=1&from=${day}&to=${day}`,
)
).json()) as UsageErrorsPage;
expect(counted.total).toBe(2);
const newest = counted.items[0];
if (newest === undefined) throw new Error("expected the newest record on the first page");
expect(newest.code).toBe("counted_b");
// A record that lands while the dialog is open: the sentence counted two, so two go and
// this one stays — without the boundary it would have gone unannounced.
seedRow({ date: day, code: "arrived_later", ts: `${day}T10:06:00.000Z` });
const cleared = await api.delete(
`/api/projects/${projectId}/usage/errors?from=${day}&to=${day}&until=${newest.ts}`,
);
expect(await cleared.json()).toEqual({ deleted: 2 });
expect(errorRows().map((r) => r.code)).toEqual(["arrived_later"]);
// A boundary that is not an instant is refused rather than dropped: a clear that ignored
// it would take the record above, which the confirmation said would stay.
const refused = await api.delete(
`/api/projects/${projectId}/usage/errors?from=${day}&to=${day}&until=yesterday`,
);
expect(refused.status).toBe(400);
expect(
errorRows()
.map((r) => r.code)
.filter((code) => code !== "bad_request"),
).toEqual(["arrived_later"]);
});
it("a member cannot clear the log, so it is no route to the unattributed rows", async () => {
seedRow({ date: "2026-07-06", code: "project_row" });
seedRow({ date: "2026-07-06", code: "unattributed_row", projectId: null, source: "process" });
// The refusals below are recorded by app.onError like any other, so the seeded rows are
// read back by code rather than by counting the table.
const seeded = () =>
errorRows()
.map((r) => r.code as string)
.filter((c) => c === "project_row" || c === "unattributed_row");
const member = await provisionUser(t.app, "member_user");
expect(member.user.isAdmin).toBe(false);
const added = await api.post(`/api/projects/${projectId}/members`, { userId: "member_user" });
expect(added.status).toBe(201);
const memberApi = apiClient(t.app, member.cookie);
// Reading the panel is a member's right; emptying it is the owner's, like deleting an Agent.
expect((await memberApi.get(`/api/projects/${projectId}/usage/errors?offset=0`)).status).toBe(
200,
);
const refused = await memberApi.delete(`/api/projects/${projectId}/usage/errors`);
expect(refused.status).toBe(403);
expect(((await refused.json()) as { error: { code: string } }).error.code).toBe(
"owner_required",
);
expect(seeded()).toEqual(["project_row", "unattributed_row"]);
// A non-member is not told the Project exists, let alone allowed to empty it.
const outsider = await provisionUser(t.app, "outsider_user");
const outsiderApi = apiClient(t.app, outsider.cookie);
expect((await outsiderApi.delete(`/api/projects/${projectId}/usage/errors`)).status).toBe(404);
expect(seeded()).toEqual(["project_row", "unattributed_row"]);
});
it("an admin's clear takes the unattributed rows an admin's read showed", async () => {
// They show in every Project's admin view and in no member's, so the clear follows the
// read: the one person who can see them anywhere empties them from the panel that showed
// them, while a member's clear (whose read never lists them) never reaches them.
const adminApi = apiClient(t.app, (await loginAdmin(t.app)).cookie);
wire(ErrorsRepo, { db: t.deps.db }).insert({
ts: "2026-07-06T00:00:00.000Z",
date: "2026-07-06",
projectId: null,
agentId: null,
sessionId: null,
source: "process",
kind: "unexpected",
code: "uncaught_exception",
status: null,
message: "crash",
});
// The admin can see it from their own Project, which is exactly the view being cleared.
const before = (await (
await adminApi.get(`/api/projects/default_project/usage/errors?offset=0&limit=20`)
).json()) as UsageErrorsPage;
expect(before.items.map((e) => e.code)).toEqual(["uncaught_exception"]);
const cleared = await adminApi.delete(
`/api/projects/default_project/usage/errors?from=2026-07-06&to=2026-07-06`,
);
expect(await cleared.json()).toEqual({ deleted: 1 });
expect(errorRows().filter((r) => r.project_id === null)).toHaveLength(0);
});
it("deleting a Project deletes its error records and nobody else's", async () => {
await api.get(`/api/projects/${projectId}/agents/agent-nope/sessions`);
expect(errorRows().filter((r) => r.project_id === projectId)).toHaveLength(1);
// An unattributed row (a failed login) belongs to no Project and must outlive this one.
await t.app.request("/api/auth/login", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ userId: "err_user", password: "wrong-password" }),
});
const del = await api.delete(`/api/projects/${projectId}`);
expect(del.status).toBe(204);
expect(errorRows().map((r) => r.project_id)).toEqual([null]);
});
// —— Ignoring: the owner silences a class, the panel reads one side or the other, nothing is lost ——
it("an ignore rule takes a class off the panel while its records stay in the table", async () => {
// Seeded directly so the repeats are exact — going through HTTP would collapse them into the
// recorder's dedup window.
const repo = wire(ErrorsRepo, { db: t.deps.db });
const seed = (code: string, message: string, ts: string) =>
repo.insert({
ts,
date: ts.slice(0, 10),
projectId,
agentId: null,
sessionId: null,
source: "http",
kind: "unexpected",
code,
status: 500,
message,
});
seed("internal", "other side closed", "2026-07-27T01:00:00.000Z");
seed("internal", "other side closed", "2026-07-27T02:00:00.000Z");
seed("agent_not_found", "no such agent", "2026-07-27T03:00:00.000Z");
const added = await api.post(`/api/projects/${projectId}/usage/errors/ignores`, {
source: "http",
code: "internal",
note: "engine reconnects by design",
});
expect(added.status).toBe(201);
const body = (await added.json()) as UsageErrorIgnoreAdded;
expect(body.hidden).toBe(2);
expect(body.rule).toMatchObject({
source: "http",
code: "internal",
messageLike: null,
hidden: 2,
});
// The dashboard drops it from what it shows and reports it once under `ignored`: a silenced
// class is not invisible twice over.
const dashboard = (await (
await api.get(`/api/projects/${projectId}/usage?groupBy=date`)
).json()) as UsageResponse;
expect(dashboard.errors).toMatchObject({ total: 1, ignored: 2 });
expect(dashboard.errors.recent.map((e) => e.code)).toEqual(["agent_not_found"]);
// The page follows the same axis, and asking for the ignored side reads exactly the covered rows.
const shown = (await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20`)
).json()) as UsageErrorsPage;
expect(shown.total).toBe(1);
expect(shown.items.map((e) => e.code)).toEqual(["agent_not_found"]);
const ignored = (await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20&ignored=1`)
).json()) as UsageErrorsPage;
expect(ignored.total).toBe(2);
expect(ignored.items.map((e) => e.code)).toEqual(["internal"]);
// The records were never touched: the rule decides which rows the reads select, not what is stored.
expect(errorRows().filter((r) => r.code === "internal")).toHaveLength(2);
});
it("a new rule's hidden figure counts the records it takes off the range it was made from", async () => {
const repo = wire(ErrorsRepo, { db: t.deps.db });
const seed = (ts: string, agentId: string) =>
repo.insert({
ts,
date: ts.slice(0, 10),
projectId,
agentId,
sessionId: null,
source: "http",
kind: "unexpected",
code: "internal",
status: 500,
message: "boom",
});
seed("2026-07-27T01:00:00.000Z", "a1");
seed("2026-07-27T02:00:00.000Z", "a2");
seed("2026-07-29T01:00:00.000Z", "a1");
// The range is the panel's own — the one its reads carry — so the number matches the table it
// came from; the rule itself is permanent and covers all three. A body that names its own
// author is ignored: the owner is the session's, so a rule can never be filed under someone else.
const added = (await (
await api.post(
`/api/projects/${projectId}/usage/errors/ignores?from=2026-07-27&to=2026-07-27&agentId=a1`,
{ source: "http", code: "internal", createdBy: "someone_else" },
)
).json()) as UsageErrorIgnoreAdded;
expect(added.hidden).toBe(1);
expect(added.rule.hidden).toBe(3);
expect(added.rule).toMatchObject({ createdBy: "err_user", note: null });
});
it("ignoring the same class twice is refused, and unignoring puts the rows back", async () => {
const repo = wire(ErrorsRepo, { db: t.deps.db });
repo.insert({
ts: "2026-07-27T01:00:00.000Z",
date: "2026-07-27",
projectId,
agentId: null,
sessionId: null,
source: "http",
kind: "unexpected",
code: "internal",
status: 500,
message: "boom",
});
const write = { source: "http", code: "internal" };
// The refusals below are recorded by app.onError like any other, so the seeded row is read
// back by code rather than by counting the table.
const visible = async () =>
(
(await (
await api.get(`/api/projects/${projectId}/usage/errors?offset=0&limit=20`)
).json()) as UsageErrorsPage
).items
.map((e) => e.code)
.filter((code) => code === "internal");
const first = await api.post(`/api/projects/${projectId}/usage/errors/ignores`, write);
expect(first.status).toBe(201);
const ruleId = ((await first.json()) as UsageErrorIgnoreAdded).rule.id;
expect(await visible()).toEqual([]);
// The same (source, code, fragment) is one rule: a second row covering the same records would
// only say the same thing twice.
const again = await api.post(`/api/projects/${projectId}/usage/errors/ignores`, write);
expect(again.status).toBe(409);
expect(((await again.json()) as { error: { code: string } }).error.code).toBe(
"ignore_rule_exists",
);
const listed = (await (
await api.get(`/api/projects/${projectId}/usage/errors/ignores`)
).json()) as UsageErrorIgnores;
expect(listed.rules.map((r) => r.id)).toEqual([ruleId]);
expect(listed.rules[0]!.hidden).toBe(1);
const removed = await api.delete(`/api/projects/${projectId}/usage/errors/ignores/${ruleId}`);
expect(await removed.json()).toEqual({ removed: true });
// Nothing was ever deleted, so removing the rule is the whole of the undo.
expect(await visible()).toEqual(["internal"]);
// A rule this Project does not have is a 404, not a silent success.
expect(
(await api.delete(`/api/projects/${projectId}/usage/errors/ignores/${ruleId}`)).status,
).toBe(404);
expect(
(await api.delete(`/api/projects/${projectId}/usage/errors/ignores/not-a-number`)).status,
).toBe(400);
});
it("the default view's clear leaves the ignored rows; standing on the ignored view it takes them", async () => {
const repo = wire(ErrorsRepo, { db: t.deps.db });
const seed = (code: string, ts: string) =>
repo.insert({
ts,
date: ts.slice(0, 10),
projectId,
agentId: null,
sessionId: null,
source: "http",
kind: "unexpected",
code,
status: 500,
message: code,
});
seed("internal", "2026-07-27T01:00:00.000Z");
seed("internal", "2026-07-27T02:00:00.000Z");
seed("agent_not_found", "2026-07-27T03:00:00.000Z");
await api.post(`/api/projects/${projectId}/usage/errors/ignores`, {
source: "http",
code: "internal",
});
// The button empties the table on screen: the ignored rows are not on it, so they stay.
const cleared = await api.delete(
`/api/projects/${projectId}/usage/errors?from=2026-07-27&to=2026-07-27`,
);
expect(await cleared.json()).toEqual({ deleted: 1 });
expect(errorRows().filter((r) => r.code === "internal")).toHaveLength(2);
// Standing on the ignored view, the same button is the one that may take those rows.
const clearedIgnored = await api.delete(
`/api/projects/${projectId}/usage/errors?from=2026-07-27&to=2026-07-27&ignored=1`,
);
expect(await clearedIgnored.json()).toEqual({ deleted: 2 });
expect(errorRows().filter((r) => r.code === "internal")).toHaveLength(0);
});
it("ignore management is the owner's, like the clear; a member still reads the panel and sees the count", async () => {
seedRow({ date: "2026-07-27", code: "internal", kind: "unexpected" });
const write = { source: "http", code: "internal" };
const member = await provisionUser(t.app, "member_user");
expect(member.user.isAdmin).toBe(false);
await api.post(`/api/projects/${projectId}/members`, { userId: "member_user" });
const memberApi = apiClient(t.app, member.cookie);
const outsiderApi = apiClient(t.app, (await provisionUser(t.app, "outsider_user")).cookie);
const rulesPath = `/api/projects/${projectId}/usage/errors/ignores`;
// The owner silences the seeded class; the member then reads its effect as a count, without the
// rule list. Read before the refusals below, whose own 403 / 404 are recorded like any error.
expect((await api.post(rulesPath, write)).status).toBe(201);
const memberView = (await (
await memberApi.get(`/api/projects/${projectId}/usage?groupBy=date`)
).json()) as UsageResponse;
expect(memberView.errors).toMatchObject({ total: 0, ignored: 1 });
// Reading the panel is a member's right; managing what it hides is the owner's.
expect((await memberApi.get(`/api/projects/${projectId}/usage/errors?offset=0`)).status).toBe(
200,
);
expect((await memberApi.get(rulesPath)).status).toBe(403);
expect((await memberApi.post(rulesPath, write)).status).toBe(403);
expect((await memberApi.delete(`${rulesPath}/1`)).status).toBe(403);
// A non-member is not told the Project exists, let alone allowed to silence its errors.
expect((await outsiderApi.get(rulesPath)).status).toBe(404);
expect((await outsiderApi.post(rulesPath, write)).status).toBe(404);
expect((await outsiderApi.delete(`${rulesPath}/1`)).status).toBe(404);
});
});
describe("HTTP error attribution (only when the requester actually has Project access)", () => {
let t: TestApp;
/** The built-in admin: unattributed errors are visible only to them. */
let adminApi: ReturnType<typeof apiClient>;
let adminProjectId: string;
/** The victim Project's owner: a regular user, so their stats center only shows errors attributed to this Project. */
let ownerApi: ReturnType<typeof apiClient>;
let projectId: string;
beforeEach(async () => {
t = await createTestApp();
const admin = await loginAdmin(t.app);
adminApi = apiClient(t.app, admin.cookie);
adminProjectId = (
(await (
await adminApi.post("/api/projects", { projectId: "admin_proj", name: "Admin project" })
).json()) as ProjectCreateResponse
).project.projectId;
const owner = await provisionUser(t.app, "owner_user");
expect(owner.user.isAdmin).toBe(false);
ownerApi = apiClient(t.app, owner.cookie);
projectId = (
(await (
await ownerApi.post("/api/projects", { projectId: "owner_user-victim", name: "Victim" })
).json()) as ProjectCreateResponse
).project.projectId;
});
afterEach(async () => {
await t.cleanup();
});
const errorRows = () =>
t.deps.db.prepare("SELECT * FROM error_records ORDER BY id").all() as Array<
Record<string, unknown>
>;
/** The victim Project's stats center (owner's view: only this Project's errors). */
const ownerErrors = async () =>
(
(await (
await ownerApi.get(`/api/projects/${projectId}/usage?groupBy=date`)
).json()) as UsageResponse
).errors;
/** The admin's stats center (this Project + unattributed errors). */
const adminErrors = async () =>
(
(await (
await adminApi.get(`/api/projects/${adminProjectId}/usage?groupBy=date`)
).json()) as UsageResponse
).errors;
it("not logged in → 401: unattributed (anyone could flood another's error stats)", async () => {
const res = await t.app.request(`/api/projects/${projectId}/usage?groupBy=date`);
expect(res.status).toBe(401);
// The requester isn't even logged in: this error must not be pinned to projectId.
// Note: today this also **incidentally** relies on a Hono quirk — `c.req.param()`
// resolves only against "the route the current handler belongs to"; the 401 is
// thrown in the `/api/*` authMiddleware, whose route has no :projectId, so no
// value is available there. The attribution guard removes the dependency on
// that quirk: when not logged in, `c.var.user` is undefined at runtime → unattributed.
const rows = errorRows();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ source: "http", code: "unauthorized", status: 401 });
expect(rows[0]!.project_id).toBeNull();
// The owner's stats center gains nothing; the trace of the unauthorized probe lands in the admin's view — right where it belongs.
expect(await ownerErrors()).toMatchObject({ total: 0, unexpected: 0, topCode: null });
const admin = await adminErrors();
expect(admin.total).toBe(1);
expect(admin.recent[0]).toMatchObject({ source: "http", code: "unauthorized" });
});
it("logged in but without access (non-member) → 404: likewise unattributed", async () => {
const outsider = await provisionUser(t.app, "outsider");
const res = await apiClient(t.app, outsider.cookie).get(
`/api/projects/${projectId}/usage?groupBy=date`,
);
expect(res.status).toBe(404);
const rows = errorRows();
expect(rows).toHaveLength(1);
expect(rows[0]).toMatchObject({ source: "http", code: "project_not_found", status: 404 });
expect(rows[0]!.project_id).toBeNull();
expect(await ownerErrors()).toMatchObject({ total: 0, unexpected: 0, topCode: null });
expect((await adminErrors()).recent[0]).toMatchObject({ code: "project_not_found" });
});
it("business errors from an authorized member → attributed as usual", async () => {
// owner: an invalid groupBy → 400.
expect((await ownerApi.get(`/api/projects/${projectId}/usage?groupBy=bogus`)).status).toBe(400);
// An authorized member: a 404 in the same Project → attributed the same way (the member branch of canAccess).
const member = await provisionUser(t.app, "member_user");
const added = await ownerApi.post(`/api/projects/${projectId}/members`, {
userId: "member_user",
});
expect(added.status).toBe(201);
const missing = await apiClient(t.app, member.cookie).get(
`/api/projects/${projectId}/agents/agent-nope/sessions`,
);
expect(missing.status).toBe(404);
expect(errorRows().map((r) => [r.code, r.project_id])).toEqual([
["bad_request", projectId],
["agent_not_found", projectId],
]);
expect(await ownerErrors()).toMatchObject({ total: 2, unexpected: 0 });
});
it("the attribution check throws: onError survives, error lands unattributed", async () => {
t.deps.access.canAccess = () => {
throw new Error("access check blew up");
};
const res = await ownerApi.get(`/api/projects/${projectId}/usage?groupBy=bogus`);
expect(res.status).toBe(400); // still the original business error: not turned into a 500, nor an empty response
expect(await res.json()).toMatchObject({ error: { code: "bad_request" } });
const rows = errorRows();
expect(rows).toHaveLength(1);
expect(rows[0]!.code).toBe("bad_request");
expect(rows[0]!.project_id).toBeNull(); // a failed determination always falls back to unattributed
});
});