Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat(evals): add second historical regression tranche (#6444)

Louis Beaumont7cea4916ea71
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 9cdc8c631..4c2fe67d5 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,9 +9,14 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 10 git-mined regressions. See
+The current app corpus contains 20 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
-history-mining workflow. The companion website manifest contributes 10 more.
+history-mining workflow. The companion website manifest contributes 20 more.
+
+The regression inventory has an explicit owner and advisory/blocking policy.
+New cases may declare trigger paths for later change-aware selection. Validation
+prints the dataset fingerprint, and scored reports retain both dataset and
+runtime fingerprints for exact-run comparison.
The product tests referenced by the manifest are graders. Passing them directly is not the eval; the evaluated object is an agent trajectory and resulting patch from the historical broken state.
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 8ec027c87..b0b2e08aa 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,8 +1,13 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-08-19.2",
+ "dataset_version": "2026-08-19.3",
"suite_type": "regression",
+ "owner": "screenpipe engineering",
+ "policy": {
+ "stage": "advisory",
+ "promotion_requirement": "matched-environment repeated trials with reliable pass^k"
+ },
"cases": [
{
"id": "app-chat-concurrent-save",
@@ -260,6 +265,241 @@
],
"command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/__tests__/pipe-store-error-state.test.tsx"
}
+ },
+ {
+ "id": "app-learning-handoff-always-finishes",
+ "name": "Always finish the first-run learning handoff",
+ "source": { "kind": "git_regression", "fix_commit": "e3587f5654" },
+ "base_ref": "e3587f5654^",
+ "oracle_ref": "e3587f5654",
+ "gate": "advisory",
+ "trigger_paths": ["apps/screenpipe-app-tauri/components/first-run/**", "apps/screenpipe-app-tauri/lib/first-run/**"],
+ "tags": ["regression", "escaped-failure", "onboarding", "state-machine", "rehydration"],
+ "prompt": "The first-run learning handoff can remain in a fake in-progress state after an empty foreground result, disappear after reload, or reuse stale webview state after setup is completed again. Make every foreground attempt reach useful setup choices, keep background retries out of the interface, preserve unresolved choices across reload, and start a fresh learning window after setup is completed again. Valid completed summaries must keep their existing behavior.",
+ "agent_timeout_seconds": 1200,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 420,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/components/first-run/learning-banner.test.tsx" },
+ { "source_path": "apps/screenpipe-app-tauri/lib/first-run/learning-window.test.ts" },
+ { "source_path": "apps/screenpipe-app-tauri/lib/first-run/use-learning-window.test.ts" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/first-run/learning-banner.test.tsx lib/first-run/learning-window.test.ts lib/first-run/use-learning-window.test.ts"
+ }
+ },
+ {
+ "id": "app-chat-activity-episode-content",
+ "name": "Analyze attached activity episode content instead of its title",
+ "source": { "kind": "git_regression", "fix_commit": "7fa6d1a956" },
+ "base_ref": "7fa6d1a956^",
+ "oracle_ref": "7fa6d1a956",
+ "gate": "advisory",
+ "trigger_paths": ["apps/screenpipe-app-tauri/components/chat/**", "apps/screenpipe-app-tauri/lib/chat/**"],
+ "tags": ["regression", "escaped-failure", "chat", "retrieval", "evidence"],
+ "prompt": "When a user asks about an attached activity-history episode, the agent searches for the generated episode title instead of reading the bounded source content. Mark activity-history attachments as episodes and route questions to the exact attached interval and its content. Reject title-keyword searches and plans that drift outside the interval, while preserving ordinary search attachments and traceable evidence behavior.",
+ "agent_timeout_seconds": 1200,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 420,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/components/chat/standalone/attached-context.test.tsx" },
+ { "source_path": "apps/screenpipe-app-tauri/lib/chat/__tests__/activity-episode-retrieval-eval.test.ts" },
+ { "source_path": "apps/screenpipe-app-tauri/lib/chat/__tests__/system-prompt.test.ts" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/chat/standalone/attached-context.test.tsx lib/chat/__tests__/activity-episode-retrieval-eval.test.ts lib/chat/__tests__/system-prompt.test.ts"
+ }
+ },
+ {
+ "id": "app-recording-stale-permission-recovery",
+ "name": "Recover recording from stale screen-permission state",
+ "source": { "kind": "git_regression", "fix_commit": "8f79280560" },
+ "base_ref": "8f79280560^",
+ "oracle_ref": "8f79280560",
+ "gate": "advisory",
+ "trigger_paths": ["apps/screenpipe-app-tauri/app/shortcut-reminder/**", "apps/screenpipe-app-tauri/lib/hooks/use-health-check.ts"],
+ "tags": ["regression", "escaped-failure", "recording", "permissions", "recovery"],
+ "prompt": "After screen permission is restored, stale permission state can leave the recording overlay showing a broken or restart-only state even though capture can recover. Reconcile permission and health state so the user gets a truthful recovery confirmation and recording resumes without an unnecessary restart. Keep genuine permission denial and real capture failures actionable.",
+ "agent_timeout_seconds": 900,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 300,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/app/shortcut-reminder/page.test.tsx" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts app/shortcut-reminder/page.test.tsx"
+ }
+ },
+ {
+ "id": "app-history-long-generation",
+ "name": "Keep legitimate slow history generation alive",
+ "source": { "kind": "git_regression", "fix_commit": "b402c7be54" },
+ "base_ref": "b402c7be54^",
+ "oracle_ref": "b402c7be54",
+ "gate": "advisory",
+ "trigger_paths": ["apps/screenpipe-app-tauri/components/activity-ledger.tsx", "apps/screenpipe-app-tauri/components/share-logs-button.tsx"],
+ "tags": ["regression", "escaped-failure", "activity", "timeout", "performance"],
+ "prompt": "Generating a long activity-history result can take more than two minutes, but the app times it out as a failure. Keep legitimate slow generation running while retaining a bounded failure path, and load only the five conversations that can actually be attached instead of expanding work without limit. Preserve existing cancellation and successful short-generation behavior.",
+ "agent_timeout_seconds": 1200,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 420,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/components/activity-ledger.test.tsx" },
+ { "source_path": "apps/screenpipe-app-tauri/components/share-logs-button.test.tsx" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/activity-ledger.test.tsx components/share-logs-button.test.tsx"
+ }
+ },
+ {
+ "id": "app-live-view-cadence-write",
+ "name": "Make Live View cadence writes take effect",
+ "source": { "kind": "git_regression", "fix_commit": "f9f2445b94" },
+ "base_ref": "f9f2445b94^",
+ "oracle_ref": "f9f2445b94",
+ "gate": "advisory",
+ "trigger_paths": ["apps/screenpipe-app-tauri/lib/live-views/**", "apps/screenpipe-app-tauri/components/settings/brain-overview.tsx"],
+ "tags": ["regression", "escaped-failure", "live-view", "api-contract", "silent-failure"],
+ "prompt": "Refreshing a paused or manual Live View reports success but leaves its task disabled because the cadence update is wrapped in a config envelope that the API silently ignores. Send the flat configuration shape the endpoint consumes, omit fields that do not need changing, reject an invalid nested front-matter envelope, and surface partial write failures instead of pretending every task refreshed.",
+ "agent_timeout_seconds": 1200,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 420,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/components/settings/__tests__/brain-overview.test.tsx" },
+ { "source_path": "apps/screenpipe-app-tauri/lib/live-views/__tests__/source-cadence.test.ts" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/settings/__tests__/brain-overview.test.tsx lib/live-views/__tests__/source-cadence.test.ts"
+ }
+ },
+ {
+ "id": "app-mcp-traceable-activity-context",
+ "name": "Return bounded, traceable activity context from MCP",
+ "source": { "kind": "git_regression", "fix_commit": "e76dac2120" },
+ "base_ref": "e76dac2120^",
+ "oracle_ref": "e76dac2120",
+ "gate": "advisory",
+ "trigger_paths": ["packages/screenpipe-mcp/src/**"],
+ "tags": ["regression", "escaped-failure", "mcp", "activity", "evidence"],
+ "prompt": "The MCP activity-summary tool returns aggregate time without enough traceable task context, and consumers can confuse missing parsed rows with missing activity. Add optional bounded parsed context while keeping duration authoritative, distinguish disabled context from fetch failure, retain the time result when parsing is unavailable, and expose source paths without making parsed context the default.",
+ "agent_timeout_seconds": 1200,
+ "dependency_links": [
+ { "source_path": "packages/screenpipe-mcp/node_modules", "destination_path": "packages/screenpipe-mcp/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 420,
+ "fixtures": [
+ { "source_path": "packages/screenpipe-mcp/src/activity-summary-format.test.ts" },
+ { "source_path": "packages/screenpipe-mcp/src/activity-summary-tool.test.ts" },
+ { "source_path": "packages/screenpipe-mcp/src/pack-contents.test.ts" },
+ { "source_path": "packages/screenpipe-mcp/src/stdio-startup.test.ts" }
+ ],
+ "command": "cd packages/screenpipe-mcp && node ./node_modules/vitest/vitest.mjs run src/activity-summary-format.test.ts src/activity-summary-tool.test.ts src/pack-contents.test.ts src/stdio-startup.test.ts"
+ }
+ },
+ {
+ "id": "app-chat-inspector-pipe-artifacts",
+ "name": "Keep pipe-run artifacts in the active chat inspector",
+ "source": { "kind": "git_regression", "fix_commit": "d3569717c0" },
+ "base_ref": "d3569717c0^",
+ "oracle_ref": "d3569717c0",
+ "gate": "advisory",
+ "trigger_paths": ["apps/screenpipe-app-tauri/components/chat/**", "apps/screenpipe-app-tauri/lib/hooks/use-chat-inspector.ts"],
+ "tags": ["regression", "escaped-failure", "pipes", "chat", "artifacts"],
+ "prompt": "Artifacts registered by a pipe run do not appear in the chat inspector unless a matching tool call exists, and stale artifacts can leak into a different task. Merge declared run artifacts into the active inspector, deduplicate files already represented by tool output, rewrite absolute artifact links safely, and keep pinned summaries open beside output previews without crossing execution boundaries.",
+ "agent_timeout_seconds": 1200,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 420,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/components/__tests__/markdown-viewer-link.test.ts" },
+ { "source_path": "apps/screenpipe-app-tauri/components/chat/chat-inspector.test.tsx" },
+ { "source_path": "apps/screenpipe-app-tauri/lib/hooks/use-chat-inspector.test.ts" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/__tests__/markdown-viewer-link.test.ts components/chat/chat-inspector.test.tsx lib/hooks/use-chat-inspector.test.ts"
+ }
+ },
+ {
+ "id": "app-sync-legacy-key-recovery",
+ "name": "Recover safely from legacy sync-key mismatches",
+ "source": { "kind": "git_regression", "fix_commit": "0f26a722d2" },
+ "base_ref": "0f26a722d2^",
+ "oracle_ref": "0f26a722d2",
+ "gate": "advisory",
+ "trigger_paths": ["apps/screenpipe-app-tauri/components/settings/**sync**", "apps/screenpipe-app-tauri/lib/hooks/use-cloud-sync.ts"],
+ "tags": ["regression", "escaped-failure", "sync", "encryption", "recovery"],
+ "prompt": "A device with a legacy account-key mismatch cannot resume cloud sync and generic errors can incorrectly trigger destructive recovery UI. Detect only the specific legacy mismatch, keep recovery hidden otherwise, require explicit confirmation before the exact scoped start-fresh request, and leave recovery available when the remote reset fails. Do not weaken ordinary sync error handling.",
+ "agent_timeout_seconds": 900,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 300,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/components/settings/__tests__/sync-key-recovery.test.tsx" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/settings/__tests__/sync-key-recovery.test.tsx"
+ }
+ },
+ {
+ "id": "app-agent-local-calendar-days",
+ "name": "Use local calendar days for agent time ranges",
+ "source": { "kind": "git_regression", "fix_commit": "afed076026" },
+ "base_ref": "afed076026^",
+ "oracle_ref": "afed076026",
+ "gate": "advisory",
+ "trigger_paths": ["packages/screenpipe-mcp/src/**time**", "apps/screenpipe-app-tauri/lib/chat/system-prompt.ts"],
+ "tags": ["regression", "escaped-failure", "agent", "time", "timezone"],
+ "prompt": "Agent time ranges such as today and yesterday are normalized through UTC, so users near a day boundary can query the wrong local date. Normalize calendar literals from the runtime local day, use calendar arithmetic across daylight-saving changes and zones that skip midnight, preserve already supported values and input objects, and advertise the same local-calendar contract in every time field.",
+ "agent_timeout_seconds": 1200,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" },
+ { "source_path": "packages/screenpipe-mcp/node_modules", "destination_path": "packages/screenpipe-mcp/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 420,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/lib/chat/__tests__/system-prompt.test.ts" },
+ { "source_path": "packages/screenpipe-mcp/src/time-normalization.test.ts" },
+ { "source_path": "packages/screenpipe-mcp/src/pack-contents.test.ts" },
+ { "source_path": "packages/screenpipe-mcp/src/stdio-startup.test.ts" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts lib/chat/__tests__/system-prompt.test.ts && cd ../../packages/screenpipe-mcp && node ./node_modules/vitest/vitest.mjs run src/time-normalization.test.ts src/pack-contents.test.ts src/stdio-startup.test.ts"
+ }
+ },
+ {
+ "id": "app-privacy-category-rule-ownership",
+ "name": "Never delete user-owned privacy rules with a category",
+ "source": { "kind": "git_regression", "fix_commit": "4450aaa82d" },
+ "base_ref": "4450aaa82d^",
+ "oracle_ref": "4450aaa82d",
+ "gate": "advisory",
+ "trigger_paths": ["apps/screenpipe-app-tauri/lib/settings/capture-categories.ts"],
+ "tags": ["regression", "escaped-failure", "privacy", "settings", "data-loss"],
+ "prompt": "Disabling a privacy capture category deletes an identical exclusion rule or domain the user created manually. Track category ownership so disabling removes only entries that category actually created, preserves hand-written and other-category entries, and still cleans up all owned filters. Keep a safe compatibility path for legacy state with no provenance.",
+ "agent_timeout_seconds": 900,
+ "dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 300,
+ "fixtures": [
+ { "source_path": "apps/screenpipe-app-tauri/lib/settings/__tests__/capture-categories.test.ts" }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts lib/settings/__tests__/capture-categories.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/run.mjs b/evals/coding-agent/run.mjs
index bfd3b0ac4..91ae720e0 100644
--- a/evals/coding-agent/run.mjs
+++ b/evals/coding-agent/run.mjs
@@ -16,6 +16,7 @@ import {
import { arch, platform, release, tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
import process from "node:process";
const HERE = dirname(new URL(import.meta.url).pathname);
@@ -84,10 +85,18 @@ function shellQuote(value) {
}
function loadManifest() {
- const manifest = JSON.parse(readFileSync(MANIFEST, "utf8"));
+ const body = readFileSync(MANIFEST, "utf8");
+ const manifest = JSON.parse(body);
if (manifest.schema_version !== 1 || !Array.isArray(manifest.cases)) {
throw new Error("unsupported or malformed eval manifest");
}
+ if (manifest.suite_type === "regression") {
+ if (!manifest.owner) throw new Error("regression manifest is missing owner");
+ if (!manifest.policy?.stage || !manifest.policy?.promotion_requirement) {
+ throw new Error("regression manifest is missing policy metadata");
+ }
+ }
+ manifest.dataset_fingerprint = createHash("sha256").update(body).digest("hex");
return manifest;
}
@@ -116,6 +125,17 @@ function validateCase(evalCase) {
}
const baseSha = mustRun("git", ["rev-parse", `${evalCase.base_ref}^{commit}`]).stdout.trim();
if (baseSha === oracleSha) throw new Error(`${evalCase.id} base and oracle are identical`);
+ if (evalCase.gate && !["advisory", "blocking"].includes(evalCase.gate)) {
+ throw new Error(`${evalCase.id} has an invalid gate`);
+ }
+ if (
+ evalCase.trigger_paths &&
+ (!Array.isArray(evalCase.trigger_paths) ||
+ !evalCase.trigger_paths.length ||
+ evalCase.trigger_paths.some((path) => typeof path !== "string" || !path))
+ ) {
+ throw new Error(`${evalCase.id} has invalid trigger_paths`);
+ }
for (const fixture of evalCase.grader.fixtures ?? []) {
if (fixture.local_path) {
const source = join(dirname(MANIFEST), fixture.local_path);
@@ -383,19 +403,24 @@ function summarize(manifest, mode, results, options, runDir) {
observed_all_pass: scoredTrials ? passes === scoredTrials : null,
};
});
+ const runtime = {
+ platform: platform(),
+ release: release(),
+ arch: arch(),
+ node: process.version,
+ repo_head: mustRun("git", ["rev-parse", "HEAD"]).stdout.trim(),
+ agent_command: mode === "agent" ? options.agentCommand : undefined,
+ candidate_patch: mode === "regrade" ? options.candidatePatch : undefined,
+ };
const report = {
suite: manifest.suite,
dataset_version: manifest.dataset_version,
+ dataset_fingerprint: manifest.dataset_fingerprint,
mode,
created_at: new Date().toISOString(),
runtime: {
- platform: platform(),
- release: release(),
- arch: arch(),
- node: process.version,
- repo_head: mustRun("git", ["rev-parse", "HEAD"]).stdout.trim(),
- agent_command: mode === "agent" ? options.agentCommand : undefined,
- candidate_patch: mode === "regrade" ? options.candidatePatch : undefined,
+ ...runtime,
+ fingerprint: createHash("sha256").update(JSON.stringify(runtime)).digest("hex"),
},
cases,
trials: results,
@@ -426,7 +451,10 @@ function main() {
}
for (const evalCase of cases) validateCase(evalCase);
if (options.validate) {
- console.log(`validated ${cases.length} eval cases`);
+ console.log(
+ `validated ${cases.length} eval cases ` +
+ `(dataset sha256:${manifest.dataset_fingerprint.slice(0, 12)})`,
+ );
return;
}