Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover terminal daily-summary provider errors

louis91e37b48ebb8
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index b2f66185f..701809b7a 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,10 +9,25 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 71 git-mined regressions. See
+The current app corpus contains 72 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-daily-summary-terminal-error` runs the actual summary orchestration and
+its internal helpers against synthetic native commands and event delivery.
+Thirteen outcomes cover terminal message/agent errors, original provider detail,
+partial-text precedence, retry hints, empty-result retry bounds and cleanup.
+The parent fails seven intended outcomes and preserves six; the reference and
+current orchestration pass all thirteen. This does not establish activity UI,
+recording, provider delivery, general runtime-start recovery or model performance.
+
+Run `bun test evals/coding-agent/calibrate-daily-summary-errors.test.js` with the
+desktop test dependencies installed. Ten controls include parent/reference,
+equivalent helper renaming, an unused correct module, lost error detail,
+ignored agent-end errors, retry-hint refusal, omitted cleanup, blanket refusal,
+and missing-source setup failure. Dependencies and fixtures are installed only
+when grading begins; dependency links do not establish agent isolation.
+
`app-onboarding-sdk-cold-start-assignment` runs the historical React page with
PostHog 1.359.1, the SDK pinned by that source revision. It observes eleven
cold-route, stale-identity, pinned-route, timeout, opt-out and unmount outcomes.
diff --git a/evals/coding-agent/calibrate-daily-summary-errors.test.js b/evals/coding-agent/calibrate-daily-summary-errors.test.js
new file mode 100644
index 000000000..2c79fd135
--- /dev/null
+++ b/evals/coding-agent/calibrate-daily-summary-errors.test.js
@@ -0,0 +1,51 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from "bun:test";
+import { execFileSync, spawnSync } from "node:child_process";
+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, symlinkSync, realpathSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { resolve, dirname, join } from "node:path";
+const repo = resolve(import.meta.dir, "../..");
+const fix = "ee91ade94484a349c98fd7ad9613f4a6018b50bf";
+const parent = "e58cc923243682bb8fb9d87fbf5f33d41f122a2f";
+const source = "apps/screenpipe-app-tauri/lib/daily-summary-pi.ts";
+const files = ["tsconfig.json", "lib/daily-summary-pi.ts", "lib/daily-summary-prompt.ts", "lib/model-metadata.ts", "lib/events/error-text.ts", "lib/utils/internal-session.ts"].map(p => "apps/screenpipe-app-tauri/" + p);
+const grader = "evals/coding-agent/graders/daily-summary-errors.test.js";
+const runtime = realpathSync(join(repo, "apps/screenpipe-app-tauri/node_modules"));
+const output = process.env.CALIBRATION_OUTPUT_DIR;
+function replace(text, from, to) { if (!text.includes(from)) throw Error("Control mutation no longer matches reference"); return text.replaceAll(from, to); }
+const controls = [
+ { name: "parent", ref: parent, pass: false },
+ { name: "reference", ref: fix, pass: true },
+ { name: "equivalent", ref: fix, pass: true, mutate: s => replace(replace(s, "terminalAssistantError", "providerFailure"), "terminalError && event.willRetry !== true", "event.willRetry !== true && terminalError") },
+ { name: "unused", ref: parent, pass: false, extra: true },
+ { name: "lost-detail", ref: fix, pass: false, mutate: s => replace(s, "new Error(terminalError)", 'new Error("AI failed")') },
+ { name: "ignored-agent-end", ref: fix, pass: false, mutate: s => replace(s, "const terminalError = finalAssistantError(envelope);", "const terminalError = null;") },
+ { name: "retry-hint", ref: fix, pass: false, mutate: s => replace(s, "terminalError && event.willRetry !== true", "terminalError") },
+ { name: "no-cleanup", ref: fix, pass: false, mutate: s => replace(s, " unregister();", " // omitted cleanup") },
+ { name: "blanket-refusal", ref: fix, pass: false, mutate: s => replace(s, " const sessionPrefix =", ' throw new Error("all requests refused");\n const sessionPrefix =') },
+ { name: "missing", ref: fix, pass: false, missing: true },
+];
+for (const control of controls) test(control.name, () => {
+ const root = mkdtempSync(join(tmpdir(), "daily-summary-calibration-"));
+ try {
+ for (const path of files) {
+ const dest = join(root, path); mkdirSync(dirname(dest), { recursive: true });
+ let text = execFileSync("git", ["show", control.ref + ":" + path], { cwd: repo, encoding: "utf8" });
+ if (path === source && control.mutate) text = control.mutate(text);
+ writeFileSync(dest, text);
+ }
+ if (control.extra) writeFileSync(join(root, source + ".unused"), execFileSync("git", ["show", fix + ":" + source], { cwd: repo }));
+ if (control.missing) rmSync(join(root, source));
+ mkdirSync(dirname(join(root, grader)), { recursive: true }); writeFileSync(join(root, grader), readFileSync(join(repo, grader)));
+ symlinkSync(runtime, join(root, "apps/screenpipe-app-tauri/node_modules"), "dir");
+ const result = spawnSync(process.execPath, ["--no-env-file", "test", grader], { cwd: root, encoding: "utf8", timeout: 30000 });
+ if (output) {
+ mkdirSync(output, { recursive: true }); writeFileSync(join(output, control.name + ".json"), JSON.stringify({ control: control.name, ref: control.ref, exit_code: result.status, signal: result.signal, stdout: result.stdout, stderr: result.stderr }, null, 2) + "\n");
+ }
+ expect(result.signal).toBeNull(); expect(result.error).toBeUndefined();
+ expect(result.status === 0).toBe(control.pass);
+ if (control.missing) { expect(result.stderr).toMatch(/0 pass/); expect(result.stderr).toMatch(/Cannot find module|ModuleNotFound/); }
+ else { expect(result.stderr).toMatch(/Ran 13 tests/); expect(result.stderr).not.toMatch(/Unhandled error|Cannot find module/); }
+ } finally { rmSync(root, { recursive: true, force: true }); }
+}, 45000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index fcb0d6122..05d4a9a6c 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -2910,6 +2910,49 @@
],
"command": "cargo test --offline --locked -p screenpipe-db --test eval_wal_read_cleanup -- --test-threads=1"
}
+ },
+ {
+ "id": "app-daily-summary-terminal-error",
+ "name": "Preserve terminal provider failures in daily summaries",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "ee91ade94484a349c98fd7ad9613f4a6018b50bf"
+ },
+ "base_ref": "e58cc923243682bb8fb9d87fbf5f33d41f122a2f",
+ "oracle_ref": "ee91ade94484a349c98fd7ad9613f4a6018b50bf",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/daily-summary-pi.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/daily-summary-pi.ts",
+ "apps/screenpipe-app-tauri/lib/events/error-text.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "provider-errors",
+ "quota",
+ "retry"
+ ],
+ "prompt": "Daily-summary generation can treat a terminal provider failure as a usable summary or lose the original allowance/rate-limit detail. Repair the actual runDailySummaryWithPi boundary. An assistant error in message_end without an explicit willRetry=true hint, or an assistant error in agent_end, must reject with the original provider detail, including errorMessage and legacy error fields. When detail is absent, return a non-empty failure explanation. A terminal error must beat partial or earlier assistant prose and must not trigger an empty-result reprompt. Preserve successful trimmed summary text, ignore error-like user/tool messages, and allow explicit retry hints to reach a later valid completion. Preserve one bounded empty-completion reprompt: it may recover valid text, but a second empty completion must reject as empty. Ordinary error events must retain provider detail. Unregister the event handler and stop the session when a started request completes or fails. Use synthetic native/event ports; do not call live providers or read personal activity. This task covers the summary orchestration boundary, not native recording, real provider delivery, the activity UI, model behavior or general runtime-start retry policy.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/daily-summary-errors.test.js",
+ "destination_path": "evals/coding-agent/graders/daily-summary-errors.test.js"
+ }
+ ],
+ "command": "bun --no-env-file test evals/coding-agent/graders/daily-summary-errors.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/daily-summary-errors.test.js b/evals/coding-agent/graders/daily-summary-errors.test.js
new file mode 100644
index 000000000..322f41489
--- /dev/null
+++ b/evals/coding-agent/graders/daily-summary-errors.test.js
@@ -0,0 +1,99 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeEach, afterEach, expect, mock, test } from "bun:test";
+import { existsSync } from "node:fs";
+import { posix } from "node:path";
+
+let state;
+function appPort(name, factory) {
+ mock.module(name, factory);
+ const stem = new URL("../../../apps/screenpipe-app-tauri/" + name.slice(2), import.meta.url).pathname;
+ for (const suffix of [".ts", ".tsx", ".js"]) {
+ if (existsSync(stem + suffix)) { mock.module(stem + suffix, factory); break; }
+ }
+}
+mock.module("@tauri-apps/api/path", () => ({ homeDir: async () => "/synthetic", join: async (...parts) => posix.join(...parts) }));
+appPort("@/lib/events/bus", () => ({
+ mountAgentEventBus: async () => {},
+ registerForeground: (_id, handler) => {
+ state.handler = handler;
+ return () => { state.unregistered++; state.handler = null; };
+ },
+}));
+appPort("@/lib/utils/tauri", () => ({ commands: {
+ piStart: async () => { state.starts++; return { status: "ok", data: { running: true } }; },
+ piPrompt: async () => {
+ const turn = state.prompts++;
+ // Native dispatch acknowledgement completes before the synthetic events.
+ setTimeout(() => { for (const event of state.turns[turn] ?? []) state.handler?.({ event }); }, 0);
+ return { status: "ok", data: null };
+ },
+ piStop: async () => { state.stops++; return { status: "ok", data: { running: false } }; },
+} }));
+globalThis.fetch = async () => { state.network++; throw Error("Live network forbidden"); };
+const { runDailySummaryWithPi } = await import("../../../apps/screenpipe-app-tauri/lib/daily-summary-pi.ts");
+const assistant = (text, extra = {}) => ({ role: "assistant", content: [{ type: "text", text }], ...extra });
+const end = (...messages) => ({ type: "agent_end", messages });
+const message = (value, extra = {}) => ({ type: "message_end", message: value, ...extra });
+const fail = (text, extra = {}) => assistant("", { stopReason: "error", errorMessage: text, ...extra });
+const quota = 'HTTP 429 {"error":{"code":"hosted_ai_allowance_exceeded"}}';
+const options = {
+ date: new Date("2026-08-18T12:00:00Z"), range: { start: "synthetic-start", end: "synthetic-end" },
+ preset: { id: "synthetic", provider: "screenpipe-cloud", url: "", model: "auto", prompt: "" },
+ userToken: "synthetic-token", prompt: "Summarize synthetic activity", systemPrompt: "Synthetic test instructions",
+};
+beforeEach(() => { state = { handler: null, turns: [], starts: 0, prompts: 0, stops: 0, unregistered: 0, network: 0 }; });
+afterEach(() => { expect(state.network).toBe(0); });
+async function run(turns) {
+ state.turns = turns;
+ return runDailySummaryWithPi(options).then(value => ({ ok: true, value }), error => ({ ok: false, error: error.message }));
+}
+function cleaned() {
+ expect(state.starts).toBe(1); expect(state.unregistered).toBe(1); expect(state.stops).toBeGreaterThanOrEqual(1);
+}
+for (const [name, event, expected] of [
+ ["message error preserves allowance code", message(fail(quota)), quota],
+ ["agent-end error preserves rate-limit code", end(fail("rate_limit_exceeded")), "rate_limit_exceeded"],
+ ["message legacy error field is preserved", message(fail(undefined, { error: "legacy-provider-refusal" })), "legacy-provider-refusal"],
+ ["agent-end legacy error field is preserved", end(fail(undefined, { error: "legacy-provider-refusal" })), "legacy-provider-refusal"],
+ ["missing provider text has an actionable fallback", message(fail("")), "AI request failed"],
+ ["terminal error beats partial assistant text", message(fail(quota, { content: [{ type: "text", text: "unfinished partial" }] })), quota],
+]) {
+ test(name, async () => {
+ const result = await run([[event, end(assistant("ordinary completion after failed turn"))]]);
+ expect(result.ok).toBe(false);
+ if (expected === "AI request failed") expect(result.error?.trim().length).toBeGreaterThan(0);
+ else expect(result.error).toContain(expected);
+ expect(state.prompts).toBe(1); cleaned();
+ });
+}
+test("agent-end terminal error beats earlier assistant prose", async () => {
+ const result = await run([[end(assistant("stale prose"), fail(quota))]]);
+ expect(result.ok).toBe(false); expect(result.error).toContain(quota); expect(state.prompts).toBe(1); cleaned();
+});
+test("explicit retry hint preserves later successful completion", async () => {
+ const result = await run([[message(fail(quota), { willRetry: true }), end(assistant(" recovered summary "))]]);
+ expect(result).toEqual({ ok: true, value: "recovered summary" }); expect(state.prompts).toBe(1); cleaned();
+});
+test("non-assistant error-like messages do not suppress a valid summary", async () => {
+ const result = await run([[message({ role: "toolResult", stopReason: "error", errorMessage: quota }), end({ role: "user", stopReason: "error", errorMessage: quota }, assistant("valid summary"))]]);
+ expect(result).toEqual({ ok: true, value: "valid summary" }); expect(state.prompts).toBe(1); cleaned();
+});
+test("successful assistant response is preserved and trimmed", async () => {
+ expect(await run([[end(assistant(" final summary "))]])).toEqual({ ok: true, value: "final summary" });
+ expect(state.prompts).toBe(1); cleaned();
+});
+test("one empty completion retry may recover valid text", async () => {
+ expect(await run([[end()], [end(assistant("recovered after tools"))]])).toEqual({ ok: true, value: "recovered after tools" });
+ expect(state.prompts).toBe(2); cleaned();
+});
+test("repeated empty completion fails after one bounded retry", async () => {
+ const result = await run([[end()], [end()]]);
+ expect(result.ok).toBe(false); expect(result.error).toMatch(/empty/i);
+ expect(state.prompts).toBe(2); cleaned();
+});
+test("ordinary error events still preserve provider detail", async () => {
+ const result = await run([[{ type: "error", errorMessage: quota }]]);
+ expect(result.ok).toBe(false); expect(result.error).toContain(quota);
+ expect(state.prompts).toBe(1); cleaned();
+});