YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover terminal daily-summary provider errors
louis91e37b48ebb8
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b2f66185f..701809b7a 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,10 +9,25 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 71 git-mined regressions. See+The current app corpus contains 72 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.+`app-daily-summary-terminal-error` runs the actual summary orchestration and+its internal helpers against synthetic native commands and event delivery.+Thirteen outcomes cover terminal message/agent errors, original provider detail,+partial-text precedence, retry hints, empty-result retry bounds and cleanup.+The parent fails seven intended outcomes and preserves six; the reference and+current orchestration pass all thirteen. This does not establish activity UI,+recording, provider delivery, general runtime-start recovery or model performance.++Run `bun test evals/coding-agent/calibrate-daily-summary-errors.test.js` with the+desktop test dependencies installed. Ten controls include parent/reference,+equivalent helper renaming, an unused correct module, lost error detail,+ignored agent-end errors, retry-hint refusal, omitted cleanup, blanket refusal,+and missing-source setup failure. Dependencies and fixtures are installed only+when grading begins; dependency links do not establish agent isolation.+`app-onboarding-sdk-cold-start-assignment` runs the historical React page withPostHog 1.359.1, the SDK pinned by that source revision. It observes elevencold-route, stale-identity, pinned-route, timeout, opt-out and unmount outcomes.diff --git a/evals/coding-agent/calibrate-daily-summary-errors.test.js b/evals/coding-agent/calibrate-daily-summary-errors.test.jsnew file mode 100644index 000000000..2c79fd135--- /dev/null+++ b/evals/coding-agent/calibrate-daily-summary-errors.test.js@@ -0,0 +1,51 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from "bun:test";+import { execFileSync, spawnSync } from "node:child_process";+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, symlinkSync, realpathSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { resolve, dirname, join } from "node:path";+const repo = resolve(import.meta.dir, "../..");+const fix = "ee91ade94484a349c98fd7ad9613f4a6018b50bf";+const parent = "e58cc923243682bb8fb9d87fbf5f33d41f122a2f";+const source = "apps/screenpipe-app-tauri/lib/daily-summary-pi.ts";+const files = ["tsconfig.json", "lib/daily-summary-pi.ts", "lib/daily-summary-prompt.ts", "lib/model-metadata.ts", "lib/events/error-text.ts", "lib/utils/internal-session.ts"].map(p => "apps/screenpipe-app-tauri/" + p);+const grader = "evals/coding-agent/graders/daily-summary-errors.test.js";+const runtime = realpathSync(join(repo, "apps/screenpipe-app-tauri/node_modules"));+const output = process.env.CALIBRATION_OUTPUT_DIR;+function replace(text, from, to) { if (!text.includes(from)) throw Error("Control mutation no longer matches reference"); return text.replaceAll(from, to); }+const controls = [+ { name: "parent", ref: parent, pass: false },+ { name: "reference", ref: fix, pass: true },+ { name: "equivalent", ref: fix, pass: true, mutate: s => replace(replace(s, "terminalAssistantError", "providerFailure"), "terminalError && event.willRetry !== true", "event.willRetry !== true && terminalError") },+ { name: "unused", ref: parent, pass: false, extra: true },+ { name: "lost-detail", ref: fix, pass: false, mutate: s => replace(s, "new Error(terminalError)", 'new Error("AI failed")') },+ { name: "ignored-agent-end", ref: fix, pass: false, mutate: s => replace(s, "const terminalError = finalAssistantError(envelope);", "const terminalError = null;") },+ { name: "retry-hint", ref: fix, pass: false, mutate: s => replace(s, "terminalError && event.willRetry !== true", "terminalError") },+ { name: "no-cleanup", ref: fix, pass: false, mutate: s => replace(s, " unregister();", " // omitted cleanup") },+ { name: "blanket-refusal", ref: fix, pass: false, mutate: s => replace(s, " const sessionPrefix =", ' throw new Error("all requests refused");\n const sessionPrefix =') },+ { name: "missing", ref: fix, pass: false, missing: true },+];+for (const control of controls) test(control.name, () => {+ const root = mkdtempSync(join(tmpdir(), "daily-summary-calibration-"));+ try {+ for (const path of files) {+ const dest = join(root, path); mkdirSync(dirname(dest), { recursive: true });+ let text = execFileSync("git", ["show", control.ref + ":" + path], { cwd: repo, encoding: "utf8" });+ if (path === source && control.mutate) text = control.mutate(text);+ writeFileSync(dest, text);+ }+ if (control.extra) writeFileSync(join(root, source + ".unused"), execFileSync("git", ["show", fix + ":" + source], { cwd: repo }));+ if (control.missing) rmSync(join(root, source));+ mkdirSync(dirname(join(root, grader)), { recursive: true }); writeFileSync(join(root, grader), readFileSync(join(repo, grader)));+ symlinkSync(runtime, join(root, "apps/screenpipe-app-tauri/node_modules"), "dir");+ const result = spawnSync(process.execPath, ["--no-env-file", "test", grader], { cwd: root, encoding: "utf8", timeout: 30000 });+ if (output) {+ mkdirSync(output, { recursive: true }); writeFileSync(join(output, control.name + ".json"), JSON.stringify({ control: control.name, ref: control.ref, exit_code: result.status, signal: result.signal, stdout: result.stdout, stderr: result.stderr }, null, 2) + "\n");+ }+ expect(result.signal).toBeNull(); expect(result.error).toBeUndefined();+ expect(result.status === 0).toBe(control.pass);+ if (control.missing) { expect(result.stderr).toMatch(/0 pass/); expect(result.stderr).toMatch(/Cannot find module|ModuleNotFound/); }+ else { expect(result.stderr).toMatch(/Ran 13 tests/); expect(result.stderr).not.toMatch(/Unhandled error|Cannot find module/); }+ } finally { rmSync(root, { recursive: true, force: true }); }+}, 45000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex fcb0d6122..05d4a9a6c 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -2910,6 +2910,49 @@],"command": "cargo test --offline --locked -p screenpipe-db --test eval_wal_read_cleanup -- --test-threads=1"}+ },+ {+ "id": "app-daily-summary-terminal-error",+ "name": "Preserve terminal provider failures in daily summaries",+ "source": {+ "kind": "git_regression",+ "fix_commit": "ee91ade94484a349c98fd7ad9613f4a6018b50bf"+ },+ "base_ref": "e58cc923243682bb8fb9d87fbf5f33d41f122a2f",+ "oracle_ref": "ee91ade94484a349c98fd7ad9613f4a6018b50bf",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/daily-summary-pi.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/daily-summary-pi.ts",+ "apps/screenpipe-app-tauri/lib/events/error-text.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "provider-errors",+ "quota",+ "retry"+ ],+ "prompt": "Daily-summary generation can treat a terminal provider failure as a usable summary or lose the original allowance/rate-limit detail. Repair the actual runDailySummaryWithPi boundary. An assistant error in message_end without an explicit willRetry=true hint, or an assistant error in agent_end, must reject with the original provider detail, including errorMessage and legacy error fields. When detail is absent, return a non-empty failure explanation. A terminal error must beat partial or earlier assistant prose and must not trigger an empty-result reprompt. Preserve successful trimmed summary text, ignore error-like user/tool messages, and allow explicit retry hints to reach a later valid completion. Preserve one bounded empty-completion reprompt: it may recover valid text, but a second empty completion must reject as empty. Ordinary error events must retain provider detail. Unregister the event handler and stop the session when a started request completes or fails. Use synthetic native/event ports; do not call live providers or read personal activity. This task covers the summary orchestration boundary, not native recording, real provider delivery, the activity UI, model behavior or general runtime-start retry policy.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/daily-summary-errors.test.js",+ "destination_path": "evals/coding-agent/graders/daily-summary-errors.test.js"+ }+ ],+ "command": "bun --no-env-file test evals/coding-agent/graders/daily-summary-errors.test.js"+ }}]}diff --git a/evals/coding-agent/graders/daily-summary-errors.test.js b/evals/coding-agent/graders/daily-summary-errors.test.jsnew file mode 100644index 000000000..322f41489--- /dev/null+++ b/evals/coding-agent/graders/daily-summary-errors.test.js@@ -0,0 +1,99 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterEach, expect, mock, test } from "bun:test";+import { existsSync } from "node:fs";+import { posix } from "node:path";++let state;+function appPort(name, factory) {+ mock.module(name, factory);+ const stem = new URL("../../../apps/screenpipe-app-tauri/" + name.slice(2), import.meta.url).pathname;+ for (const suffix of [".ts", ".tsx", ".js"]) {+ if (existsSync(stem + suffix)) { mock.module(stem + suffix, factory); break; }+ }+}+mock.module("@tauri-apps/api/path", () => ({ homeDir: async () => "/synthetic", join: async (...parts) => posix.join(...parts) }));+appPort("@/lib/events/bus", () => ({+ mountAgentEventBus: async () => {},+ registerForeground: (_id, handler) => {+ state.handler = handler;+ return () => { state.unregistered++; state.handler = null; };+ },+}));+appPort("@/lib/utils/tauri", () => ({ commands: {+ piStart: async () => { state.starts++; return { status: "ok", data: { running: true } }; },+ piPrompt: async () => {+ const turn = state.prompts++;+ // Native dispatch acknowledgement completes before the synthetic events.+ setTimeout(() => { for (const event of state.turns[turn] ?? []) state.handler?.({ event }); }, 0);+ return { status: "ok", data: null };+ },+ piStop: async () => { state.stops++; return { status: "ok", data: { running: false } }; },+} }));+globalThis.fetch = async () => { state.network++; throw Error("Live network forbidden"); };+const { runDailySummaryWithPi } = await import("../../../apps/screenpipe-app-tauri/lib/daily-summary-pi.ts");+const assistant = (text, extra = {}) => ({ role: "assistant", content: [{ type: "text", text }], ...extra });+const end = (...messages) => ({ type: "agent_end", messages });+const message = (value, extra = {}) => ({ type: "message_end", message: value, ...extra });+const fail = (text, extra = {}) => assistant("", { stopReason: "error", errorMessage: text, ...extra });+const quota = 'HTTP 429 {"error":{"code":"hosted_ai_allowance_exceeded"}}';+const options = {+ date: new Date("2026-08-18T12:00:00Z"), range: { start: "synthetic-start", end: "synthetic-end" },+ preset: { id: "synthetic", provider: "screenpipe-cloud", url: "", model: "auto", prompt: "" },+ userToken: "synthetic-token", prompt: "Summarize synthetic activity", systemPrompt: "Synthetic test instructions",+};+beforeEach(() => { state = { handler: null, turns: [], starts: 0, prompts: 0, stops: 0, unregistered: 0, network: 0 }; });+afterEach(() => { expect(state.network).toBe(0); });+async function run(turns) {+ state.turns = turns;+ return runDailySummaryWithPi(options).then(value => ({ ok: true, value }), error => ({ ok: false, error: error.message }));+}+function cleaned() {+ expect(state.starts).toBe(1); expect(state.unregistered).toBe(1); expect(state.stops).toBeGreaterThanOrEqual(1);+}+for (const [name, event, expected] of [+ ["message error preserves allowance code", message(fail(quota)), quota],+ ["agent-end error preserves rate-limit code", end(fail("rate_limit_exceeded")), "rate_limit_exceeded"],+ ["message legacy error field is preserved", message(fail(undefined, { error: "legacy-provider-refusal" })), "legacy-provider-refusal"],+ ["agent-end legacy error field is preserved", end(fail(undefined, { error: "legacy-provider-refusal" })), "legacy-provider-refusal"],+ ["missing provider text has an actionable fallback", message(fail("")), "AI request failed"],+ ["terminal error beats partial assistant text", message(fail(quota, { content: [{ type: "text", text: "unfinished partial" }] })), quota],+]) {+ test(name, async () => {+ const result = await run([[event, end(assistant("ordinary completion after failed turn"))]]);+ expect(result.ok).toBe(false);+ if (expected === "AI request failed") expect(result.error?.trim().length).toBeGreaterThan(0);+ else expect(result.error).toContain(expected);+ expect(state.prompts).toBe(1); cleaned();+ });+}+test("agent-end terminal error beats earlier assistant prose", async () => {+ const result = await run([[end(assistant("stale prose"), fail(quota))]]);+ expect(result.ok).toBe(false); expect(result.error).toContain(quota); expect(state.prompts).toBe(1); cleaned();+});+test("explicit retry hint preserves later successful completion", async () => {+ const result = await run([[message(fail(quota), { willRetry: true }), end(assistant(" recovered summary "))]]);+ expect(result).toEqual({ ok: true, value: "recovered summary" }); expect(state.prompts).toBe(1); cleaned();+});+test("non-assistant error-like messages do not suppress a valid summary", async () => {+ const result = await run([[message({ role: "toolResult", stopReason: "error", errorMessage: quota }), end({ role: "user", stopReason: "error", errorMessage: quota }, assistant("valid summary"))]]);+ expect(result).toEqual({ ok: true, value: "valid summary" }); expect(state.prompts).toBe(1); cleaned();+});+test("successful assistant response is preserved and trimmed", async () => {+ expect(await run([[end(assistant(" final summary "))]])).toEqual({ ok: true, value: "final summary" });+ expect(state.prompts).toBe(1); cleaned();+});+test("one empty completion retry may recover valid text", async () => {+ expect(await run([[end()], [end(assistant("recovered after tools"))]])).toEqual({ ok: true, value: "recovered after tools" });+ expect(state.prompts).toBe(2); cleaned();+});+test("repeated empty completion fails after one bounded retry", async () => {+ const result = await run([[end()], [end()]]);+ expect(result.ok).toBe(false); expect(result.error).toMatch(/empty/i);+ expect(state.prompts).toBe(2); cleaned();+});+test("ordinary error events still preserve provider detail", async () => {+ const result = await run([[{ type: "error", errorMessage: quota }]]);+ expect(result.ok).toBe(false); expect(result.error).toContain(quota);+ expect(state.prompts).toBe(1); cleaned();+});