YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): preserve offsets in chat timeline citations
louis06e0f1e02b4a
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 054a3d0d2..cfbedb5dd 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1137,3 +1137,5 @@ preference, equivalent private names and missing-source setup classification.Fixtures and dependencies are installed only when grading starts. This does notestablish native environment parsing, actual capture delivery, unknown preferenceor failed IPC policy, execution isolation or model performance.++`app-chat-timeline-offset` renders the actual MarkdownBlock and Markdown parser with synthetic native/store ports. Twelve outcomes cover raw and encoded offsets, captured instants, timestamp precedence over start_time, invalid timestamps and preserved frame/artifact/web links. Run `bun test evals/coding-agent/calibrate-chat-timeline-offset.test.js` for grader controls. This is not native event delivery, OS deeplink handling, frame retrieval, sub-millisecond precision, execution isolation or a model trial. Graders and dependency links are installed only when grading begins.diff --git a/evals/coding-agent/calibrate-chat-timeline-offset.test.js b/evals/coding-agent/calibrate-chat-timeline-offset.test.jsnew file mode 100644index 000000000..a5b069636--- /dev/null+++ b/evals/coding-agent/calibrate-chat-timeline-offset.test.js@@ -0,0 +1,49 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-chat-timeline-offset');+const component = `${app}/components/chat/markdown-block.tsx`, parser = `${app}/lib/timeline-deeplink.ts`;+const root = mkdtempSync(join(tmpdir(), 'chat-timeline-offset-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);+ for (const path of ['packages/workflows-ui', 'crates/screenpipe-core/assets']) { if (execFileSync('git', ['ls-tree', ref, path], {cwd:repo, encoding:'utf8'}).trim()) paths.push(path); }+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('12 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stderr).toContain('AssertionError'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails four offset outcomes and preserves eight nearby outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 8 passed');},60000);+test('historical reference passes twelve outcomes',()=>passes(grade('reference')),60000);+test('current caller preserves twelve outcomes',()=>passes(grade('current','HEAD')),60000);+test('equivalent normalized timestamp output passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,parser,'return candidate;','return new Date(candidate).toISOString();'))),60000);+test('unused correct component cannot hide broken active caller',()=>fails(grade('unused',item.oracle_ref,cwd=>{+ writeFileSync(join(cwd,`${app}/unused-correct-markdown.tsx`),readFileSync(join(cwd,component)));+ writeFileSync(join(cwd,component),execFileSync('git',['show',`${item.base_ref}:${component}`],{cwd:repo}));+})),60000);+test('blanket navigation suppression fails',()=>fails(grade('no-navigation',item.oracle_ref,cwd=>replace(cwd,parser,'return candidate;','return null;'))),60000);+test('discarding raw positive offsets fails',()=>fails(grade('discard-offset',item.oracle_ref,cwd=>replace(cwd,parser,'return decodeURIComponent(value);',"return decodeURIComponent(value).split('+').join(' ');"))),60000);+test('missing component is setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,component)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex fe3cb64d6..2ef92e299 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -3932,6 +3932,52 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts eval-telemetry-caller.test.tsx"}+ },+ {+ "id": "app-chat-timeline-offset",+ "name": "Preserve captured instants when opening chat timeline citations",+ "source": {+ "kind": "git_regression",+ "fix_commit": "e6df687c7f674fbdda546925be17e1115a8f2fa1"+ },+ "base_ref": "afcee637bb263ada9ac34911052f693553c4333a",+ "oracle_ref": "e6df687c7f674fbdda546925be17e1115a8f2fa1",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/chat/markdown-block.tsx",+ "apps/screenpipe-app-tauri/lib/timeline-deeplink.ts",+ "apps/screenpipe-app-tauri/lib/timeline-navigation.ts"+ ],+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/chat/markdown-block.tsx",+ "apps/screenpipe-app-tauri/lib/timeline-deeplink.ts",+ "apps/screenpipe-app-tauri/lib/timeline-navigation.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "chat",+ "timeline",+ "navigation"+ ],+ "prompt": "Clicking an AI chat timeline citation can silently do nothing when its ISO timestamp has a raw positive timezone offset. Repair the rendered MarkdownBlock click path so timestamp links preserve the captured instant to millisecond precision across raw +00:00 and nonzero positive offsets, percent-encoded offsets, negative offsets and Z timestamps. Support start_time links, with timestamp taking precedence when both are present. Valid links must update pending timeline navigation, show the Main window and emit the existing navigate-to-timestamp event for the same instant; semantically equivalent timestamp formatting is allowed. Invalid timestamps must produce no navigation or viewer effects. Preserve frame navigation, local artifact in-chat preview callbacks and ordinary web-link destinations. Keep public component and native event/command contracts compatible. No particular parser or private helper is required. The task concerns chat clicks with actual React/Markdown rendering and synthetic native/store ports; native event delivery, the separate OS deeplink handler, timeline frame retrieval and sub-millisecond precision are outside this task.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/chat-timeline-offset.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/components/chat/eval-timeline-offset.test.tsx"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/chat/eval-timeline-offset.test.tsx"+ }}]}diff --git a/evals/coding-agent/graders/chat-timeline-offset.fixture.tsx.txt b/evals/coding-agent/graders/chat-timeline-offset.fixture.tsx.txtnew file mode 100644index 000000000..88f2c6a15--- /dev/null+++ b/evals/coding-agent/graders/chat-timeline-offset.fixture.tsx.txt@@ -0,0 +1,28 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import React from 'react';+import {fireEvent,render,screen,waitFor,cleanup} from '@testing-library/react';+import {afterEach,beforeEach,expect,it,vi} from 'vitest';+import {MarkdownBlock} from './markdown-block';+const ports=vi.hoisted(()=>({emit:vi.fn(async()=>{}),show:vi.fn(async()=>({status:'ok'})),viewer:vi.fn(async()=>({status:'ok'})),pending:vi.fn()}));+vi.mock('@/lib/utils/tauri',()=>({commands:{showWindow:ports.show,openViewerWindow:ports.viewer}}));+vi.mock('@tauri-apps/api/event',()=>({emit:ports.emit}));+vi.mock('@/lib/hooks/use-timeline-store',()=>({useTimelineStore:{getState:()=>({setPendingNavigation:ports.pending})}}));+beforeEach(()=>{vi.clearAllMocks();vi.stubGlobal('fetch',vi.fn(async()=>{throw Error('unexpected network')}));});+afterEach(()=>{cleanup();vi.unstubAllGlobals()});+async function click(href:string){render(<MarkdownBlock text={`[moment](${href})`} isUser={false}/>);fireEvent.click(screen.getByRole('link',{name:'moment'}));await Promise.resolve();}+async function navigates(href:string,expected:string){await click(href);await waitFor(()=>expect(ports.emit).toHaveBeenCalled());const event=ports.emit.mock.calls.find((c:any[])=>c[0]==='navigate-to-timestamp') as any[]|undefined;expect(event).toBeDefined();expect(Date.parse(event![1])).toBe(Date.parse(expected));expect(ports.show).toHaveBeenCalledWith('Main');const pending=ports.pending.mock.calls.at(-1)?.[0] as any;expect(pending).toBeDefined();expect(Date.parse(pending.timestamp)).toBe(Date.parse(expected));expect(ports.viewer).not.toHaveBeenCalled();}+it.each([+ ['raw UTC','2026-01-15T18:12:34.123456+00:00',false],+ ['raw previous day','2026-01-14T23:58:04.120000+00:00',false],+ ['raw nonzero offset','2026-01-15T04:12:34.987+05:30',false],+ ['encoded offset','2026-01-15T04:12:34.987+05:30',true],+ ['negative offset','2026-01-15T04:12:34.987-07:00',false],+ ['Z timestamp','2026-01-15T04:12:34.987Z',false],+])('%s citation reaches its captured instant',async(_label,timestamp,encoded)=>{await navigates(`screenpipe://timeline?timestamp=${encoded?encodeURIComponent(timestamp as string):timestamp}`,timestamp as string)});+it('start_time preserves a raw positive offset',async()=>{const t='2026-01-16T01:02:03+02:00';await navigates(`screenpipe://timeline?start_time=${t}`,t)});+it('timestamp takes precedence over start_time',async()=>{const t='2026-01-17T02:03:04Z';await navigates(`screenpipe://timeline?start_time=2026-01-16T00:00:00Z×tamp=${t}`,t)});+it('invalid timeline input causes no navigation',async()=>{await click('screenpipe://timeline?timestamp=not-a-date');expect(ports.pending).not.toHaveBeenCalled();expect(ports.emit).not.toHaveBeenCalled();expect(ports.show).not.toHaveBeenCalled();expect(ports.viewer).not.toHaveBeenCalled()});+it('frame links retain frame navigation',async()=>{await click('screenpipe://frame/42');await waitFor(()=>expect(ports.emit).toHaveBeenCalledWith('navigate-to-frame','42'));expect(ports.pending).toHaveBeenCalledWith({timestamp:'',frameId:'42'});expect(ports.show).toHaveBeenCalledWith('Main')});+it('local artifacts retain the in-chat preview callback',()=>{const open=vi.fn();render(<MarkdownBlock text='[artifact](file:///tmp/synthetic-preview.md)' isUser={false} onOpenViewerPath={open}/>);fireEvent.click(screen.getByRole('link',{name:'artifact'}));expect(open).toHaveBeenCalledWith('/tmp/synthetic-preview.md');expect(ports.viewer).not.toHaveBeenCalled();expect(ports.emit).not.toHaveBeenCalled()});+it('ordinary web links retain their destination',()=>{render(<MarkdownBlock text='[web](https://example.invalid/document)' isUser={false}/>);expect(screen.getByRole('link',{name:'web'}).getAttribute('href')).toBe('https://example.invalid/document');expect(ports.pending).not.toHaveBeenCalled();expect(ports.emit).not.toHaveBeenCalled()});