YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover GLM tool history at provider dispatch
louise39ca6ea1f22
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 95e744714..eba796f6c 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 50 git-mined regressions. See+The current app corpus contains 51 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -297,3 +297,16 @@ effects. It does not prove rendered UI rollback, native filesystem durability,cross-process races, or recovery after the final store save fails. No installedpackages or native build is needed. Source setup errors cannot establish theregression. Actual model trials and enforced trial isolation remain separate.++The `ai-gateway-glm-bounded-tool-history` case observes outgoing SDK requests+from the real GLM and OpenAI provider methods, including streaming. It checks+Pi-triggered bounds, retained evidence and tool pairing while preserving small+results, ordinary clients, user/assistant prose, caller inputs and other providers.+The parent fails twelve intended outcomes and preserves fourteen; the historical+fix and current gateway/protocol implementation pass all twenty-six.++Run `bun test evals/coding-agent/calibrate-glm-tool-history.test.js` for bypass,+streaming, boundary, identity-loss and preserved-client controls. A different+valid head/tail allocation is accepted. The SDK transport and telemetry are+synthetic; there is no live inference, token-budget sufficiency, encrypted+transport, gateway authentication, native persistence, model or isolation claim.diff --git a/evals/coding-agent/calibrate-glm-tool-history.test.js b/evals/coding-agent/calibrate-glm-tool-history.test.jsnew file mode 100644index 000000000..f35f6e343--- /dev/null+++ b/evals/coding-agent/calibrate-glm-tool-history.test.js@@ -0,0 +1,32 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), source = 'packages/ai-gateway/src/providers/screenpipe-glm.ts';+const fixture = 'evals/coding-agent/graders/ai-gateway-glm-tool-history.test.js';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'ai-gateway-glm-bounded-tool-history');+const root = mkdtempSync(join(tmpdir(), 'glm-history-calibration-')), archives = new Map();+afterAll(() => rmSync(root, { recursive: true, force: true }));+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) archives.set(ref, execFileSync('git', ['-c', 'core.commitGraph=false', 'archive', ref, 'packages/ai-gateway'], { cwd: repo, maxBuffer: 32 * 1024 * 1024 }));+ execFileSync('tar', ['-x', '-C', cwd], { input: archives.get(ref) });+ mkdirSync(dirname(join(cwd, fixture)), { recursive: true }); writeFileSync(join(cwd, fixture), readFileSync(join(repo, fixture))); mutate(cwd);+ return spawnSync(process.execPath, ['test', fixture], { cwd, encoding: 'utf8', timeout: 20_000, env: { PATH: dirname(process.execPath) } });+}+function change(cwd, from, to) { const path = join(cwd, source), text = readFileSync(path, 'utf8'); expect(text.split(from)).toHaveLength(2); writeFileSync(path, text.replace(from, to)); }+function fails(result) { expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(result.status).toBe(1); expect(result.stderr).toContain('expect(received)'); expect(result.stderr).not.toContain('Cannot find'); expect(result.stderr).not.toContain('Unhandled error'); }+test('broken parent fails twelve wire outcomes and preserves fourteen neighbors', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stderr).toContain('12 fail'); expect(r.stderr).toContain('14 pass'); });+test('reference passes streaming and non-streaming requests', () => { const r = grade('reference'); expect(r.status).toBe(0); expect(r.stderr).toContain('26 pass'); });+test('different valid head-tail allocation is accepted', () => { expect(grade('equivalent', item.oracle_ref, cwd => change(cwd, 'available * 0.82', 'available * 0.65')).status).toBe(0); });+test('unused compaction helper cannot conceal completion bypass', () => fails(grade('completion-bypass', item.oracle_ref, cwd => change(cwd, 'super.createCompletion(normalizeGlmRequest(body))', 'super.createCompletion(body)'))));+test('streaming must also dispatch compacted history', () => fails(grade('stream-bypass', item.oracle_ref, cwd => change(cwd, 'super.createStreamingCompletion(normalizeGlmRequest(body))', 'super.createStreamingCompletion(body)'))));+test('truncating user or assistant text is rejected', () => fails(grade('all-roles', item.oracle_ref, cwd => change(cwd, "message.role !== 'tool' || typeof message.content !== 'string'", "typeof message.content !== 'string'"))));+test('dropping the result tail is rejected', () => fails(grade('lost-tail', item.oracle_ref, cwd => change(cwd, 'content.slice(-tailChars)', "''"))));+test('altering a tool result exactly at the bound is rejected', () => fails(grade('boundary', item.oracle_ref, cwd => change(cwd, 'content.length <= GLM_MAX_TOOL_RESULT_CHARS', 'content.length < GLM_MAX_TOOL_RESULT_CHARS'))));+test('ordinary clients cannot be compacted indiscriminately', () => fails(grade('always-compact', item.oracle_ref, cwd => change(cwd, "body.messages.some(isGlmCatalogMessage)\n\t\t|| (Array.isArray(body.tools) && body.tools.some(isSubagentTool))", 'true'))));+test('losing tool-result identity is rejected', () => fails(grade('lost-identity', item.oracle_ref, cwd => change(cwd, 'return { ...message, content: compactGlmToolResultText(message.content) };', 'return { role: message.role, content: compactGlmToolResultText(message.content) };'))));+test('missing provider remains setup failure', () => { const r = grade('missing', item.oracle_ref, cwd => rmSync(join(cwd, source))); expect(r.status).toBe(1); expect(r.stderr).toContain('Cannot find module'); expect(r.stderr).toContain('0 pass'); });diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 14c02c3b1..4a1d4ceb1 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-19.7",+ "dataset_version": "2026-09-20.1","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1624,6 +1624,39 @@],"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage bun x vitest run --config vitest.config.ts app/onboarding/page.test.tsx -t 'continues immediately when fresh authenticated flags|records a fresh login once|does not count an authenticated settings hydration|records one fresh login without|does not expose the control checkout|falls back explicitly to control|restores the pinned route|assigns a fresh-install treatment user|uses a paid account only|never enrolls an upgraded free install|does not start the standard funnel for managed|does not start a new funnel when standard onboarding resumes|leaves login completion analytics' --maxWorkers=1 --minWorkers=1"}+ },+ {+ "id": "ai-gateway-glm-bounded-tool-history",+ "name": "Bound compacted Pi tool history before GLM dispatch",+ "source": {+ "kind": "git_regression",+ "fix_commit": "7f626da4ffde0e838a022044047a2835660b263d"+ },+ "base_ref": "0f2423d2976e04455ac51fde2879f00bbc491cd6",+ "oracle_ref": "7f626da4ffde0e838a022044047a2835660b263d",+ "oracle_paths": [+ "packages/ai-gateway/src/providers/screenpipe-glm.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "ai-gateway",+ "glm",+ "context",+ "streaming"+ ],+ "prompt": "Large Pi tool results still exceed the GLM context budget after the gateway compacts its skill catalog. Before both streaming and non-streaming GLM provider dispatch, bound oversized string tool results to at most 8,000 characters when the request has a system/developer available-skills catalog (including text-part arrays) or the subagent function tool. Preserve at least the first and last 128 characters, preserve tool-call/result pairing and message order, and mark the omission with guidance to reread a narrower range. Keep results at or below 8,000 characters unchanged. Preserve user and assistant prose, original input objects, normal completion/stream delivery, model routing, ordinary tools/tool-choice, existing subagent removal, and the useful minimum output allowance for compacted Pi requests. Ordinary clients and requests with catalog text only in a user message must retain their full tool results and intentional small output limits. Other OpenAI-compatible providers must retain their original context. Check the real provider methods and outgoing SDK request with synthetic external ports; no live model calls. Any head/tail allocation satisfying these outcomes is acceptable.",+ "agent_timeout_seconds": 1200,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/ai-gateway-glm-tool-history.test.js",+ "destination_path": "evals/coding-agent/graders/ai-gateway-glm-tool-history.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/ai-gateway-glm-tool-history.test.js"+ }}]}diff --git a/evals/coding-agent/graders/ai-gateway-glm-tool-history.test.js b/evals/coding-agent/graders/ai-gateway-glm-tool-history.test.jsnew file mode 100644index 000000000..5b37d5ba3--- /dev/null+++ b/evals/coding-agent/graders/ai-gateway-glm-tool-history.test.js@@ -0,0 +1,84 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterEach, expect, mock, test } from 'bun:test';+let requests, unexpected;+mock.module('openai', () => ({ default: class {+ constructor(options) { this.baseURL = options.baseURL; }+ chat = { completions: { create: async params => {+ requests.push(structuredClone(params));+ if (!params.stream) return { choices: [{ message: { role: 'assistant', content: 'synthetic answer' }, finish_reason: 'stop' }] };+ return { controller: { abort() {} }, async *[Symbol.asyncIterator]() {+ yield { choices: [{ delta: { content: 'synthetic answer' }, finish_reason: null }] };+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };+ } };+ } } };+} }));+mock.module('@sentry/cloudflare', () => ({ captureException: () => { unexpected.push('sentry'); } }));+globalThis.fetch = async () => { unexpected.push('network'); throw Error('Network forbidden'); };+const { ScreenpipeGlmProvider } = await import('../../../packages/ai-gateway/src/providers/screenpipe-glm');+const { OpenAIProvider } = await import('../../../packages/ai-gateway/src/providers/openai');+const catalog = '<available_skills><skill><name>screenpipe-api</name></skill><skill><name>unrelated-synthetic-skill</name></skill></available_skills>';+const readTool = { type: 'function', function: { name: 'read', parameters: { type: 'object', properties: {} } } };+const subagent = { type: 'function', function: { name: 'subagent', parameters: { type: 'object', properties: {} } } };+function body(trigger = 'system', length = 40000) {+ const prefix = 'BEGIN_EVIDENCE_'.repeat(10), suffix = 'END_EVIDENCE'.repeat(10) + 'TAIL_LAST!';+ const content = length < prefix.length + suffix.length ? 'x'.repeat(length) : prefix + 'e'.repeat(length - prefix.length - suffix.length) + suffix;+ expect(content.length).toBe(length);+ return {+ model: 'glm-5.3-flash-reap50-iq3m', max_completion_tokens: 1, tool_choice: 'auto',+ tools: trigger === 'subagent' ? [readTool, subagent] : [readTool],+ messages: [+ { role: trigger === 'developer-array' ? 'developer' : 'system', content: trigger === 'developer-array' ? [{ type: 'text', text: 'Keep policy. ' + catalog }] : 'Keep policy. ' + (trigger === 'system' ? catalog : '') },+ { role: 'user', content: 'Summarize synthetic evidence.' },+ { role: 'assistant', content: '', tool_calls: [{ id: 'call-large', type: 'function', function: { name: 'read', arguments: '{"path":"synthetic.log"}' } }] },+ { role: 'tool', tool_call_id: 'call-large', content },+ { role: 'assistant', content: '', tool_calls: [{ id: 'call-small', type: 'function', function: { name: 'read', arguments: '{"path":"small.log"}' } }] },+ { role: 'tool', tool_call_id: 'call-small', content: 'small result' },+ ],+ };+}+beforeEach(() => { requests = []; unexpected = []; });+afterEach(() => expect(unexpected).toEqual([]));+async function invoke(input, streaming, provider = new ScreenpipeGlmProvider('synthetic-key')) {+ const original = structuredClone(input);+ if (streaming) {+ const stream = await provider.createStreamingCompletion(input);+ const text = await new Response(stream).text(); expect(text).toContain('synthetic answer'); expect(text).toContain('[DONE]');+ } else {+ const response = await provider.createCompletion(input); expect(response.status).toBe(200); expect(JSON.stringify(await response.json())).toContain('synthetic answer');+ }+ expect(input).toEqual(original); expect(requests).toHaveLength(1);+ const request = requests[0]; expect(request.stream).toBe(streaming); expect(request.tool_choice).toBe('auto');+ expect(request.messages.filter(m => m.role === 'assistant').map(m => m.tool_calls)).toEqual(input.messages.filter(m => m.role === 'assistant').map(m => m.tool_calls));+ expect(request.messages.find(m => m.tool_call_id === 'call-small')).toMatchObject({ role: 'tool', content: 'small result' });+ return request;+}+function compacted(request, original) {+ const result = request.messages.find(m => m.tool_call_id === 'call-large');+ expect(result.role).toBe('tool'); expect(result.content.length).toBeLessThanOrEqual(8000);+ expect(result.content.startsWith(original.slice(0, 128))).toBe(true);+ expect(result.content.endsWith(original.slice(-128))).toBe(true);+ expect(result.content).toMatch(/compact|truncat|omitt/i); expect(result.content).toMatch(/narrow|reread|re-read/i);+ expect(request.tools).toEqual([readTool]); expect(request.max_completion_tokens).toBeGreaterThanOrEqual(4096);+ expect(request.model).toBe('glm-5.3-flash-reap50-iq3m');+}+for (const streaming of [false, true]) {+ for (const trigger of ['system', 'developer-array', 'subagent']) for (const length of [8001, 40000]) test(`${streaming ? 'stream' : 'completion'}: ${trigger} compacts ${length} characters at the provider boundary`, async () => {+ const input = body(trigger, length); compacted(await invoke(input, streaming), input.messages[3].content);+ });+ for (const length of [0, 7999, 8000]) test(`${streaming ? 'stream' : 'completion'}: ${length}-character tool result stays intact`, async () => {+ const input = body('system', length), request = await invoke(input, streaming); expect(request.messages.find(m => m.tool_call_id === 'call-large').content).toBe(input.messages[3].content);+ });+ test(`${streaming ? 'stream' : 'completion'}: ordinary client retains large history and intentional small output`, async () => {+ const input = body('none'), request = await invoke(input, streaming); expect(request.messages.find(m => m.tool_call_id === 'call-large').content).toBe(input.messages[3].content); expect(request.max_completion_tokens).toBe(1);+ });+ test(`${streaming ? 'stream' : 'completion'}: user text cannot activate Pi compaction`, async () => {+ const input = body('none'); input.messages[1].content = catalog; const request = await invoke(input, streaming); expect(request.messages.find(m => m.tool_call_id === 'call-large').content).toBe(input.messages[3].content); expect(request.max_completion_tokens).toBe(1);+ });+ test(`${streaming ? 'stream' : 'completion'}: user and assistant prose survive active compaction`, async () => {+ const input = body(); input.messages[1].content = 'user '.repeat(3000); input.messages[2].content = 'assistant '.repeat(2000); const request = await invoke(input, streaming); expect(request.messages[1].content).toBe(input.messages[1].content); expect(request.messages[2].content).toBe(input.messages[2].content);+ });+ test(`${streaming ? 'stream' : 'completion'}: ordinary OpenAI provider retains complete context`, async () => {+ const input = body('subagent'); input.model = 'gpt-4o'; const request = await invoke(input, streaming, new OpenAIProvider('synthetic-key')); expect(request.messages.find(m => m.tool_call_id === 'call-large').content).toBe(input.messages[3].content); expect(request.tools).toEqual(input.tools); expect(request.max_completion_tokens).toBe(1); expect(request.model).toBe('gpt-4o');+ });+}