Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover signup-free startup authentication

Louis Beaumontd624fae18aac
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index cd1479d14..4bffe7f9c 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 31 git-mined regressions. See
+The current app corpus contains 32 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest contributes 20 more.
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 051e9aabf..7efe53457 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-01.1",
+ "dataset_version": "2026-09-02.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -870,6 +870,43 @@
],
"command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts components/__tests__/app-log-share-redaction.eval.test.tsx"
}
+ },
+ {
+ "id": "app-startup-auth-not-required",
+ "name": "Start signup-free builds without a false sign-in gate",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "27fb5d84707fc778b6d5740f54943ee25b6128b5",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/6810"
+ },
+ "base_ref": "7ded3c132196a0ce3b347469843b4693d6765ff6",
+ "oracle_ref": "27fb5d84707fc778b6d5740f54943ee25b6128b5",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/app-entitlement-gate.tsx"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/app-entitlement-gate.tsx",
+ "apps/screenpipe-app-tauri/components/providers/app-init-provider.tsx",
+ "apps/screenpipe-app-tauri/src-tauri/src/startup_auth.rs",
+ "apps/screenpipe-app-tauri/src-tauri/src/main.rs"
+ ],
+ "tags": ["regression", "escaped-failure", "auth", "startup", "signup-free", "false-gate", "recording"],
+ "prompt": "A signup-free desktop build has already resolved startup authentication as not required, but the protected app still renders a sign-in gate because no user account exists. Make an explicit no-auth-required startup reach the application without stopping the recorder. Ordinary signed-out builds must still require sign-in; authenticated, enterprise, onboarding, development-skip, entitlement, hydration, recovery, and non-primary-window behavior must remain unchanged.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ { "source_path": "apps/screenpipe-app-tauri/node_modules", "destination_path": "apps/screenpipe-app-tauri/node_modules" }
+ ],
+ "grader": {
+ "timeout_seconds": 420,
+ "fixtures": [
+ {
+ "source_ref": "27fb5d84707fc778b6d5740f54943ee25b6128b5",
+ "source_path": "apps/screenpipe-app-tauri/components/app-entitlement-gate.test.tsx"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts components/app-entitlement-gate.test.tsx"
+ }
}
]
}