Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): preserve offsets in chat timeline citations

louis06e0f1e02b4a
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 054a3d0d2..cfbedb5dd 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1137,3 +1137,5 @@ preference, equivalent private names and missing-source setup classification.
Fixtures and dependencies are installed only when grading starts. This does not
establish native environment parsing, actual capture delivery, unknown preference
or failed IPC policy, execution isolation or model performance.
+
+`app-chat-timeline-offset` renders the actual MarkdownBlock and Markdown parser with synthetic native/store ports. Twelve outcomes cover raw and encoded offsets, captured instants, timestamp precedence over start_time, invalid timestamps and preserved frame/artifact/web links. Run `bun test evals/coding-agent/calibrate-chat-timeline-offset.test.js` for grader controls. This is not native event delivery, OS deeplink handling, frame retrieval, sub-millisecond precision, execution isolation or a model trial. Graders and dependency links are installed only when grading begins.
diff --git a/evals/coding-agent/calibrate-chat-timeline-offset.test.js b/evals/coding-agent/calibrate-chat-timeline-offset.test.js
new file mode 100644
index 000000000..a5b069636
--- /dev/null
+++ b/evals/coding-agent/calibrate-chat-timeline-offset.test.js
@@ -0,0 +1,49 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-chat-timeline-offset');
+const component = `${app}/components/chat/markdown-block.tsx`, parser = `${app}/lib/timeline-deeplink.ts`;
+const root = mkdtempSync(join(tmpdir(), 'chat-timeline-offset-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('12 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent fails four offset outcomes and preserves eight nearby outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 8 passed');},60000);
+test('historical reference passes twelve outcomes',()=>passes(grade('reference')),60000);
+test('current caller preserves twelve outcomes',()=>passes(grade('current','HEAD')),60000);
+test('equivalent normalized timestamp output passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,parser,'return candidate;','return new Date(candidate).toISOString();'))),60000);
+test('unused correct component cannot hide broken active caller',()=>fails(grade('unused',item.oracle_ref,cwd=>{
+ writeFileSync(join(cwd,`${app}/unused-correct-markdown.tsx`),readFileSync(join(cwd,component)));
+ writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.base_ref}:${component}`],{cwd:repo}));
+})),60000);
+test('blanket navigation suppression fails',()=>fails(grade('no-navigation',item.oracle_ref,cwd=>replace(cwd,parser,'return candidate;','return null;'))),60000);
+test('discarding raw positive offsets fails',()=>fails(grade('discard-offset',item.oracle_ref,cwd=>replace(cwd,parser,'return decodeURIComponent(value);',"return decodeURIComponent(value).split('+').join(' ');"))),60000);
+test('missing component is setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,component)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index fe3cb64d6..2ef92e299 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -3932,6 +3932,52 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts eval-telemetry-caller.test.tsx"
}
+ },
+ {
+ "id": "app-chat-timeline-offset",
+ "name": "Preserve captured instants when opening chat timeline citations",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "e6df687c7f674fbdda546925be17e1115a8f2fa1"
+ },
+ "base_ref": "afcee637bb263ada9ac34911052f693553c4333a",
+ "oracle_ref": "e6df687c7f674fbdda546925be17e1115a8f2fa1",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/chat/markdown-block.tsx",
+ "apps/screenpipe-app-tauri/lib/timeline-deeplink.ts",
+ "apps/screenpipe-app-tauri/lib/timeline-navigation.ts"
+ ],
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/chat/markdown-block.tsx",
+ "apps/screenpipe-app-tauri/lib/timeline-deeplink.ts",
+ "apps/screenpipe-app-tauri/lib/timeline-navigation.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "chat",
+ "timeline",
+ "navigation"
+ ],
+ "prompt": "Clicking an AI chat timeline citation can silently do nothing when its ISO timestamp has a raw positive timezone offset. Repair the rendered MarkdownBlock click path so timestamp links preserve the captured instant to millisecond precision across raw +00:00 and nonzero positive offsets, percent-encoded offsets, negative offsets and Z timestamps. Support start_time links, with timestamp taking precedence when both are present. Valid links must update pending timeline navigation, show the Main window and emit the existing navigate-to-timestamp event for the same instant; semantically equivalent timestamp formatting is allowed. Invalid timestamps must produce no navigation or viewer effects. Preserve frame navigation, local artifact in-chat preview callbacks and ordinary web-link destinations. Keep public component and native event/command contracts compatible. No particular parser or private helper is required. The task concerns chat clicks with actual React/Markdown rendering and synthetic native/store ports; native event delivery, the separate OS deeplink handler, timeline frame retrieval and sub-millisecond precision are outside this task.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/chat-timeline-offset.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/components/chat/eval-timeline-offset.test.tsx"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/chat/eval-timeline-offset.test.tsx"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/chat-timeline-offset.fixture.tsx.txt b/evals/coding-agent/graders/chat-timeline-offset.fixture.tsx.txt
new file mode 100644
index 000000000..88f2c6a15
--- /dev/null
+++ b/evals/coding-agent/graders/chat-timeline-offset.fixture.tsx.txt
@@ -0,0 +1,28 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import React from 'react';
+import {fireEvent,render,screen,waitFor,cleanup} from '@testing-library/react';
+import {afterEach,beforeEach,expect,it,vi} from 'vitest';
+import {MarkdownBlock} from './markdown-block';
+const ports=vi.hoisted(()=>({emit:vi.fn(async()=>{}),show:vi.fn(async()=>({status:'ok'})),viewer:vi.fn(async()=>({status:'ok'})),pending:vi.fn()}));
+vi.mock('@/lib/utils/tauri',()=>({commands:{showWindow:ports.show,openViewerWindow:ports.viewer}}));
+vi.mock('@tauri-apps/api/event',()=>({emit:ports.emit}));
+vi.mock('@/lib/hooks/use-timeline-store',()=>({useTimelineStore:{getState:()=>({setPendingNavigation:ports.pending})}}));
+beforeEach(()=>{vi.clearAllMocks();vi.stubGlobal('fetch',vi.fn(async()=>{throw Error('unexpected network')}));});
+afterEach(()=>{cleanup();vi.unstubAllGlobals()});
+async function click(href:string){render(<MarkdownBlock text={`[moment](${href})`} isUser={false}/>);fireEvent.click(screen.getByRole('link',{name:'moment'}));await Promise.resolve();}
+async function navigates(href:string,expected:string){await click(href);await waitFor(()=>expect(ports.emit).toHaveBeenCalled());const event=ports.emit.mock.calls.find((c:any[])=>c[0]==='navigate-to-timestamp') as any[]|undefined;expect(event).toBeDefined();expect(Date.parse(event![1])).toBe(Date.parse(expected));expect(ports.show).toHaveBeenCalledWith('Main');const pending=ports.pending.mock.calls.at(-1)?.[0] as any;expect(pending).toBeDefined();expect(Date.parse(pending.timestamp)).toBe(Date.parse(expected));expect(ports.viewer).not.toHaveBeenCalled();}
+it.each([
+ ['raw UTC','2026-01-15T18:12:34.123456+00:00',false],
+ ['raw previous day','2026-01-14T23:58:04.120000+00:00',false],
+ ['raw nonzero offset','2026-01-15T04:12:34.987+05:30',false],
+ ['encoded offset','2026-01-15T04:12:34.987+05:30',true],
+ ['negative offset','2026-01-15T04:12:34.987-07:00',false],
+ ['Z timestamp','2026-01-15T04:12:34.987Z',false],
+])('%s citation reaches its captured instant',async(_label,timestamp,encoded)=>{await navigates(`screenpipe://timeline?timestamp=${encoded?encodeURIComponent(timestamp as string):timestamp}`,timestamp as string)});
+it('start_time preserves a raw positive offset',async()=>{const t='2026-01-16T01:02:03+02:00';await navigates(`screenpipe://timeline?start_time=${t}`,t)});
+it('timestamp takes precedence over start_time',async()=>{const t='2026-01-17T02:03:04Z';await navigates(`screenpipe://timeline?start_time=2026-01-16T00:00:00Z&timestamp=${t}`,t)});
+it('invalid timeline input causes no navigation',async()=>{await click('screenpipe://timeline?timestamp=not-a-date');expect(ports.pending).not.toHaveBeenCalled();expect(ports.emit).not.toHaveBeenCalled();expect(ports.show).not.toHaveBeenCalled();expect(ports.viewer).not.toHaveBeenCalled()});
+it('frame links retain frame navigation',async()=>{await click('screenpipe://frame/42');await waitFor(()=>expect(ports.emit).toHaveBeenCalledWith('navigate-to-frame','42'));expect(ports.pending).toHaveBeenCalledWith({timestamp:'',frameId:'42'});expect(ports.show).toHaveBeenCalledWith('Main')});
+it('local artifacts retain the in-chat preview callback',()=>{const open=vi.fn();render(<MarkdownBlock text='[artifact](file:///tmp/synthetic-preview.md)' isUser={false} onOpenViewerPath={open}/>);fireEvent.click(screen.getByRole('link',{name:'artifact'}));expect(open).toHaveBeenCalledWith('/tmp/synthetic-preview.md');expect(ports.viewer).not.toHaveBeenCalled();expect(ports.emit).not.toHaveBeenCalled()});
+it('ordinary web links retain their destination',()=>{render(<MarkdownBlock text='[web](https://example.invalid/document)' isUser={false}/>);expect(screen.getByRole('link',{name:'web'}).getAttribute('href')).toBe('https://example.invalid/document');expect(ports.pending).not.toHaveBeenCalled();expect(ports.emit).not.toHaveBeenCalled()});