YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: bound Private reasoning at the request adapter
louis91734abede78
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 9f1267d84..e987db96a 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 47 git-mined regressions. See+The current app corpus contains 48 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -244,3 +244,18 @@ blanket denial, skipped installation/removal and missing-source controls.This standalone adapter requires macOS; it does not call Keychain or a provider.It does not verify the native credential-free status cache, browser-cookieconsent, Windows DPAPI, UI interaction or enforced agent isolation.+++`app-private-reasoning-budget` executes the actual Private GLM request adapter+and protocol normalizer with a synthetic verified-client port. Twenty-two+outcomes cover effort translation, bounded reasoning, reserved answer/tool+output, output-limit precedence, body-override refusal and preserved auth, cache+rotation, request destination, cancellation and verification-failure boundaries.+Only the transport file from the historical fix is applied as the oracle.++Run `bun test evals/coding-agent/calibrate-private-reasoning.test.js` for+parent/reference, equivalent local naming, missing output reserve, blanket+budget changes, verification/auth-cache bypass, no-op and missing-source+controls. No dependencies, actual encryption, attestation, model calls, native+app or provider access are exercised; this is request construction, not proof+of sampler enforcement, response quality or agent isolation.diff --git a/evals/coding-agent/calibrate-private-reasoning.test.js b/evals/coding-agent/calibrate-private-reasoning.test.jsnew file mode 100644index 000000000..5b2dbae41--- /dev/null+++ b/evals/coding-agent/calibrate-private-reasoning.test.js@@ -0,0 +1,25 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..'),source='crates/screenpipe-core/assets/extensions/lib/tinfoil-transport.ts',protocol='crates/screenpipe-core/assets/extensions/lib/glm-protocol.ts';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-private-reasoning-budget');+const show=(ref,path)=>execFileSync('git',['-c','core.commitGraph=false','show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});+const broken=show(item.base_ref,source),fixed=show(item.oracle_ref,source),normalizer=show(item.base_ref,protocol);+const fixture='evals/coding-agent/graders/private-reasoning-budget.test.ts',grader=readFileSync(join(repo,fixture));+const root=mkdtempSync(join(tmpdir(),'private-budget-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,text){const cwd=join(root,name);for(const [p,s] of [[source,text],[protocol,normalizer],[fixture,grader]]){if(s===null)continue;mkdirSync(dirname(join(cwd,p)),{recursive:true});writeFileSync(join(cwd,p),s);}return spawnSync(process.execPath,['test',fixture],{cwd,encoding:'utf8',timeout:30_000,env:{PATH:dirname(process.execPath)}});}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');}+function replace(s,old,next){expect(s.split(old)).toHaveLength(2);return s.replace(old,next);}+test('parent fails reasoning policy but preserves request boundaries',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('16 fail');expect(r.stderr).toContain('6 pass');});+test('reference passes all actual-adapter outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('22 pass');});+test('equivalent local names and true expression pass',()=>expect(grade('equivalent',fixed.replaceAll('reasoningBudget','thinkingCap').replace('enable_thinking: true','enable_thinking: !false')).status).toBe(0));+test('discarding all reasoning cannot pass',()=>fails(grade('zero-budget',replace(fixed,'Math.min(reasoningBudget, Math.max(0, outputLimit - 1024))','0'))));+test('omitting reserved answer output cannot pass',()=>fails(grade('no-reserve',replace(fixed,'Math.min(reasoningBudget, Math.max(0, outputLimit - 1024))','reasoningBudget'))));+test('skipping client verification cannot pass',()=>fails(grade('skip-verification',replace(fixed,'await verifiedClient.ready();','/* deliberately broken verification bypass */'))));+test('reusing authenticated cache after token rotation cannot pass',()=>fails(grade('auth-cache',replace(fixed,'if (!client || clientAuth !== auth) {','if (!client) {'))));+test('fake successful response without sending cannot pass',()=>fails(grade('no-op',replace(fixed,'return async (input, init) => {',"return async (input, init) => { return new Response('data: [DONE]\\n\\n');"))));+test('missing source remains setup failure',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex a151a7414..2be4e4627 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-19.4",+ "dataset_version": "2026-09-19.5","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1516,6 +1516,38 @@],"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/grok-installer-consent.test.js"}+ },+ {+ "id": "app-private-reasoning-budget",+ "name": "Bound Private model reasoning while reserving answer and tool output",+ "source": {+ "kind": "git_regression",+ "fix_commit": "6666ed74c051bb536d255fb4520702ae520f9c3d"+ },+ "base_ref": "5e6d3712c4034c67e600a184dcde5b6e19e16ec2",+ "oracle_ref": "6666ed74c051bb536d255fb4520702ae520f9c3d",+ "oracle_paths": [+ "crates/screenpipe-core/assets/extensions/lib/tinfoil-transport.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "private-ai",+ "transport",+ "resource-bounds"+ ],+ "prompt": "Private requests can consume their generation allowance on reasoning or lose the reasoning delimiter. Fix the real GLM request transport to enable the delimiter and translate ordinary reasoning settings into supported low/high/max values: high maps to high, xhigh/max map to max, all other values map to low. Bound reasoning per response at 512/2048/4096 tokens for low/high/max, while reserving 1024 tokens for answer or tool output and never producing a negative budget. Use max_completion_tokens before max_tokens and an 8192 default when neither is supplied; preserve supplied output limits. Body-supplied template or reasoning-budget overrides must not evade this policy. Preserve the existing verified-client boundary: verify before sending, no plaintext fallback, only the configured HTTPS endpoint/model/POST, propagate cancellation, forward the current authorization and content, refuse redirects, reuse a client for unchanged credentials and isolate its cache after credential rotation. Exercise the actual adapter with a synthetic verified-client port. Actual encryption, attestation cryptography and live model output are outside this task.",+ "agent_timeout_seconds": 1200,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/private-reasoning-budget.test.ts",+ "destination_path": "evals/coding-agent/graders/private-reasoning-budget.test.ts"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/private-reasoning-budget.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/private-reasoning-budget.test.ts b/evals/coding-agent/graders/private-reasoning-budget.test.tsnew file mode 100644index 000000000..a5e814e49--- /dev/null+++ b/evals/coding-agent/graders/private-reasoning-budget.test.ts@@ -0,0 +1,40 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, beforeEach, expect, test } from 'bun:test';+import { createGlmEncryptedFetch } from '../../../crates/screenpipe-core/assets/extensions/lib/tinfoil-transport';+const base='https://gateway.invalid/v1',endpoint=`${base}/tinfoil/glm/chat/completions`,model='glm-5.3-flash-reap50-iq3m';+const originalFetch=globalThis.fetch;+let sent:any[],configs:any[],order:string[],readyFailure:boolean,plainCalls:number;+beforeEach(()=>{sent=[];configs=[];order=[];readyFailure=false;plainCalls=0;globalThis.fetch=async()=>{plainCalls++;throw Error('Plaintext fallback forbidden');};});+afterAll(()=>{globalThis.fetch=originalFetch;});+function adapter(){return createGlmEncryptedFetch(base,options=>{+ configs.push(options);return {ready:async()=>{order.push('ready');if(readyFailure)throw Error('Synthetic verification failure');},fetch:async(url,init)=>{+ order.push('fetch');sent.push({url:String(url),headers:new Headers(init?.headers),body:JSON.parse(String(init?.body)),redirect:init?.redirect});+ return new Response('data: [DONE]\n\n',{headers:{'Content-Type':'text/event-stream'}});+ }};+});}+function request(body:any={},token='synthetic-user-token',extra:any={}){return {method:'POST',headers:{'Content-Type':'application/json',Authorization:`Bearer ${token}`},body:JSON.stringify({model,messages:[{role:'user',content:'Synthetic task'}],...body}),...extra};}+async function send(body:any){const response=await adapter()(endpoint,request(body));expect(await response.text()).toContain('[DONE]');expect(sent).toHaveLength(1);expect(order).toEqual(['ready','fetch']);expect(plainCalls).toBe(0);return sent[0].body;}+for(const [effort,expected,budget] of [[undefined,'low',512],['off','low',512],['minimal','low',512],['low','low',512],['medium','low',512],['high','high',2048],['xhigh','max',4096],['max','max',4096],['unknown','low',512]] as const){+ test(`ordinary effort ${String(effort)} has supported reasoning settings`,async()=>{+ const body=await send({reasoning_effort:effort});expect(body.chat_template_kwargs).toMatchObject({enable_thinking:true,reasoning_effort:expected});expect(body.thinking_budget_tokens).toBe(budget);+ });+}+for(const [limits,expected] of [[{max_tokens:8192},4096],[{max_tokens:1536},512],[{max_completion_tokens:128},0],[{max_tokens:1024},0],[{max_tokens:1025},1],[{max_tokens:8192,max_completion_tokens:2048},1024]] as const){+ test(`reserves answer/tool output for ${JSON.stringify(limits)}`,async()=>{+ const body=await send({reasoning_effort:'max',...limits});expect(body.thinking_budget_tokens).toBe(expected);for(const [key,value] of Object.entries(limits))expect(body[key]).toBe(value);+ });+}+test('body-supplied overrides cannot defeat the bounded request policy',async()=>{+ const body=await send({reasoning_effort:'low',thinking_budget_tokens:999999,chat_template_kwargs:{enable_thinking:false,reasoning_effort:'max'}});+ expect(body.chat_template_kwargs).toMatchObject({enable_thinking:true,reasoning_effort:'low'});expect(body.thinking_budget_tokens).toBe(512);+});+test('verified request preserves auth, user content, redirect refusal and same-auth reuse',async()=>{+ const fetch=adapter();await (await fetch(endpoint,request())).text();await (await fetch(endpoint,request())).text();expect(configs).toHaveLength(1);+ expect(configs[0]).toMatchObject({baseURL:endpoint,transport:'ehbp'});expect(sent[0].url).toBe(endpoint);expect(sent[0].headers.get('authorization')).toBe('Bearer synthetic-user-token');expect(sent[0].redirect).toBe('error');expect(sent[0].body.messages).toContainEqual({role:'user',content:'Synthetic task'});+ await (await fetch(endpoint,request({},'rotated-synthetic-token'))).text();expect(configs).toHaveLength(2);expect(configs[0].userCacheSecret).not.toBe(configs[1].userCacheSecret);expect(sent[2].headers.get('authorization')).toBe('Bearer rotated-synthetic-token');expect(plainCalls).toBe(0);+});+for(const kind of ['model','destination','method','aborted','verification'])test(`rejects ${kind} failure without a model or plaintext request`,async()=>{+ const fetch=adapter();let url=endpoint,init=request();if(kind==='model')init=request({model:'other-model'});if(kind==='destination')url='https://other.invalid/v1/tinfoil/glm/chat/completions';if(kind==='method')init={method:'GET'} as any;if(kind==='aborted'){const controller=new AbortController();controller.abort();init=request({},undefined,{signal:controller.signal});}if(kind==='verification')readyFailure=true;+ await expect(fetch(url,init)).rejects.toThrow();expect(sent).toEqual([]);expect(plainCalls).toBe(0);+});