Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: cover expired certificate recovery across provider types

louis52706da62980
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index ea7672549..94a66675a 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 58 git-mined regressions. See
+The current app corpus contains 59 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -479,3 +479,20 @@ Missing source remains an import error. Only the provider source is the oracle.
The SDK and error-reporting ports are synthetic; no model or network calls run.
This does not prove upstream API acceptance, gateway authorization/billing,
audio/file/refusal normalization, model quality or trial isolation.
+
+## Expired certificate recovery
+
+`app-expired-certificate-recovery` exercises the public provider-error message
+and presentation interfaces. Eleven outcomes cover the escaped expired-TLS
+signature across hosted, remote and local presets, case normalization, raw-error
+suppression, unknown-text fallback and preserved authentication, throttling and
+safety-refusal guidance. The parent fails five intended outcomes and preserves
+six; the historical reference and selected current code pass all eleven.
+
+Run `bun test evals/coding-agent/calibrate-certificate-error.test.js`. Nine
+controls reject an unused fix, blanket connection classification, raw copy,
+nonretryable recovery and divergent message output, while accepting equivalent
+predicate logic and distinguishing missing source from behavior failure.
+Only the error module is the oracle. These synthetic function outcomes do not
+prove rendered UI, actual retry dispatch, TLS recovery or provider availability.
+No network calls, model trials or native builds run.
diff --git a/evals/coding-agent/calibrate-certificate-error.test.js b/evals/coding-agent/calibrate-certificate-error.test.js
new file mode 100644
index 000000000..b3f6ae396
--- /dev/null
+++ b/evals/coding-agent/calibrate-certificate-error.test.js
@@ -0,0 +1,30 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdirSync,mkdtempSync,readFileSync,rmSync,writeFileSync} from 'node:fs';
+import {dirname,join,resolve} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),root=mkdtempSync(join(tmpdir(),'certificate-calibration-'));
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-expired-certificate-recovery');
+const source='apps/screenpipe-app-tauri/lib/chat/provider-errors.ts';
+const show=ref=>execFileSync('git',['show',`${ref}:${source}`],{cwd:repo,encoding:'utf8'}),fixed=show(item.oracle_ref),broken=show(item.base_ref);
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,body,unused){
+ const cwd=join(root,name);mkdirSync(cwd);execFileSync('tar',['-xf','-','-C',cwd],{input:execFileSync('git',['archive',item.base_ref,source],{cwd:repo,maxBuffer:32*1024*1024})});
+ if(body===null)rmSync(join(cwd,source));else writeFileSync(join(cwd,source),body);
+ if(unused)writeFileSync(join(cwd,source+'.unused.ts'),unused);
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ return spawnSync(process.execPath,['test',item.grader.fixtures[0].destination_path],{cwd,encoding:'utf8',timeout:15000,env:{PATH:process.env.PATH,HOME:cwd,NO_COLOR:'1'}});
+}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toMatch(/Cannot find|ModuleNotFound|SyntaxError/);}
+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}
+test('parent fails five certificate outcomes and preserves six others',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('5 fail');expect(r.stderr).toContain('6 pass');});
+test('reference passes all eleven outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('11 pass');});
+test('unused correct module cannot rescue broken exported interfaces',()=>fails(grade('unused',broken,fixed)));
+test('equivalent case-insensitive exact predicate passes',()=>{const body=replace(fixed,'normalized === "error: certificate has expired"','/^error: certificate has expired$/i.test(errorStr)');expect(grade('equivalent',body).status).toBe(0);});
+test('treating every unknown error as connectivity is rejected',()=>fails(grade('blanket',replace(fixed,'function isConnectionLikeError(errorStr: string): boolean {','function isConnectionLikeError(errorStr: string): boolean { return true;'))));
+test('raw copy without recovery is rejected',()=>fails(grade('raw',replace(fixed,'export function buildCloudConnectionMessage(): string {','export function buildCloudConnectionMessage(): string { return "Error: certificate has expired";'))));
+test('nonretryable connection presentation is rejected',()=>fails(grade('no-retry',replace(fixed,'return message ? { kind: "provider", message, retryable: true } : null;','return message ? { kind: "provider", message, retryable: false } : null;'))));
+test('divergent message interface is rejected',()=>fails(grade('message',replace(fixed,'return buildProviderErrorPresentation(errorStr, preset)?.message ?? null;','return null;'))));
+test('missing module is an import error',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toMatch(/Cannot find|ModuleNotFound/);expect(r.stderr).not.toContain('expect(received)');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 4c3934879..d19b05177 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-20.9",
+ "dataset_version": "2026-09-20.10",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1917,6 +1917,37 @@
],
"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/openai-media-cache.test.js"
}
+ },
+ {
+ "id": "app-expired-certificate-recovery",
+ "name": "Recover from expired TLS certificates with provider-specific guidance",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "bc2d014170e9cd8b81785ddf5fab6441c23d12f4"
+ },
+ "base_ref": "791c2018eecb6ff2d602bf6d9cff3ac5232b6df8",
+ "oracle_ref": "bc2d014170e9cd8b81785ddf5fab6441c23d12f4",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/chat/provider-errors.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "provider-recovery",
+ "preserved-behavior"
+ ],
+ "prompt": "Chat displays an unclassified raw Error: certificate has expired failure. Fix the public provider-error presentation and message interfaces so that error, including letter-case variations, becomes retryable provider-specific connection guidance. Hosted screenpipe-cloud and pi presets need Screenpipe Cloud retry guidance without a quota/upgrade claim; remote providers need connection/retry guidance; native Ollama needs its local service-start instruction and configured model. Do not echo the raw certificate error. Keep the message and presentation interfaces consistent. Preserve null fallback for unrecognized failures and ordinary prose mentioning expired certificates, ordinary cloud TLS recovery, distinct rate-limit guidance, nonretryable model safety refusals and custom-provider authentication guidance about the key and endpoint. No live providers, UI or native build are needed; grade observable classifications and guidance rather than exact sentences.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/certificate-error.test.ts",
+ "destination_path": "evals/coding-agent/graders/certificate-error.test.ts"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/certificate-error.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/certificate-error.test.ts b/evals/coding-agent/graders/certificate-error.test.ts
new file mode 100644
index 000000000..abdda6221
--- /dev/null
+++ b/evals/coding-agent/graders/certificate-error.test.ts
@@ -0,0 +1,27 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {expect,test} from "bun:test";
+import {buildProviderErrorPresentation as present,buildProviderErrorMessage as message} from "../../../apps/screenpipe-app-tauri/lib/chat/provider-errors";
+const raw="Error: certificate has expired";
+for(const [provider,input] of [["screenpipe-cloud",raw],["screenpipe-cloud",raw.toUpperCase()],["pi",raw]]){
+ test(`expired certificate gets hosted recovery for ${provider}: ${input}`,()=>{
+ const result=present(input,{provider}); expect(result).toMatchObject({kind:"provider",retryable:true});
+ const text=result?.message.toLowerCase(); expect(text).toMatch(/screenpipe.*cloud/); expect(text).toMatch(/retry|try again/);
+ expect(text).not.toMatch(/certificate has expired|upgrade|quota|rate.limit/); expect(message(input,{provider})).toBe(result?.message);
+ });
+}
+test("remote certificate failure keeps remote recovery",()=>{
+ const result=present(raw,{provider:"gemini"}); expect(result).toMatchObject({kind:"provider",retryable:true});
+ expect(result?.message.toLowerCase()).toMatch(/provider|gemini/);expect(result?.message.toLowerCase()).toMatch(/internet|connect/);expect(result?.message.toLowerCase()).toMatch(/retry|try again/);expect(result?.message.toLowerCase()).not.toMatch(/screenpipe cloud|ollama|certificate has expired/);
+});
+test("local provider certificate failure retains local instructions",()=>{
+ const result=present(raw,{provider:"native-ollama",model:"synthetic-local-model"});expect(result).toMatchObject({kind:"provider",retryable:true});
+ expect(result?.message.toLowerCase()).toContain("ollama serve");expect(result?.message).toContain("synthetic-local-model");expect(result?.message.toLowerCase()).not.toMatch(/screenpipe cloud|upgrade|certificate has expired/);
+});
+for(const input of ["The attached document says the certificate has expired", "unknown synthetic failure"]){
+ test(`unrecognized text retains null fallback: ${input}`,()=>{expect(present(input,{provider:"screenpipe-cloud"})).toBeNull();expect(message(input,{provider:"screenpipe-cloud"})).toBeNull();});
+}
+test("ordinary cloud connection errors keep recovery",()=>{const result=present("TLS handshake eof",{provider:"screenpipe-cloud"});expect(result).toMatchObject({kind:"provider",retryable:true});expect(result?.message.toLowerCase()).toContain("screenpipe cloud");});
+test("explicit throttling retains its distinct guidance",()=>{const result=present("rate limit exceeded",{provider:"screenpipe-cloud"});expect(result).toMatchObject({kind:"provider",retryable:true});expect(result?.message.toLowerCase()).toMatch(/rate.limit|too many requests/);expect(result?.message.toLowerCase()).toMatch(/wait|retry|try again/);});
+test("model safety refusal remains nonretryable",()=>{expect(present("finish_reason: content_filter",{provider:"screenpipe-cloud"})).toMatchObject({kind:"safety_refusal",retryable:false});});
+test("custom authentication failure retains key and endpoint recovery",()=>{const result=present("401 unauthorized",{provider:"custom"});expect(result?.message.toLowerCase()).toMatch(/api key/);expect(result?.message.toLowerCase()).toMatch(/url|endpoint/);expect(result?.message.toLowerCase()).not.toContain("screenpipe cloud");});