Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: cover full accessibility trees in telemetry wire

Louis Beaumont3213cc273f7e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index f7c04505c..f2b8abf61 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,18 @@
# Coding-agent regression evals
+`app-telemetry-full-ax-wire` compiles the complete telemetry records module and
+runs its public batch encoder and decoder with synthetic input. Twelve outcomes
+cover retained accessibility trees, empty nodes, nested and unknown properties,
+Unicode and long values, while preserving legacy frames, identity, mixed audio,
+record order and tolerant malformed-line handling. The parent loses four tree
+outcomes and preserves eight; the historical fix and current source pass twelve.
+Run `bun test evals/coding-agent/calibrate-telemetry-ax.test.js` for fifteen
+calibration controls, including equivalent representations, disconnected fixes,
+truncation, lost nodes, lost audio, malformed-line counts and setup errors.
+Hidden fixtures appear only at grading and need no future build-cache links.
+This narrow wire case does not execute native capture, database hydration,
+PII removal, upload consent, persistent gateway storage or actual model trials.
+
`app-recording-status-authority` mounts the actual recording popover with
synthetic device, settings and callback inputs. Fourteen outcomes distinguish
empty or stale status from capture authority, preserve global/device actions,
diff --git a/evals/coding-agent/calibrate-telemetry-ax.test.js b/evals/coding-agent/calibrate-telemetry-ax.test.js
new file mode 100644
index 000000000..a66d5f1f3
--- /dev/null
+++ b/evals/coding-agent/calibrate-telemetry-ax.test.js
@@ -0,0 +1,61 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'app-telemetry-full-ax-wire');
+const path = item.oracle_paths[0];
+const git = ref => execFileSync('git', ['show', `${ref}:${path}`], { cwd: repo, encoding: 'utf8' });
+const parent = git(item.base_ref), fixed = git(item.oracle_ref);
+const hash = bytes => createHash('sha256').update(bytes).digest('hex');
+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }
+test('calibrate full accessibility tree wire outcomes and preserved records', () => {
+ const controls = [
+ ['parent', parent, 'fail'], ['reference', fixed, 'pass'],
+ ['current', readFileSync(join(repo, path), 'utf8'), 'pass'],
+ ['equivalent', change('#[serde(default, skip_serializing_if = "Option::is_none")]\n pub accessibility_tree', '#[serde(default)]\n #[serde(skip_serializing_if = "Option::is_none")]\n pub accessibility_tree'), 'pass'],
+ ['equivalent-field-name', change('pub accessibility_tree: Option<Value>,', '#[serde(rename = "accessibility_tree")]\n pub retained_tree: Option<Value>,'), 'pass'],
+ ['drop-empty-nodes', change('for f in frames {', 'for f in frames {\n let mut f = f.clone();\n if let Some(Value::Array(nodes)) = &mut f.accessibility_tree { nodes.retain(|node| node.get("text").and_then(Value::as_str).is_some_and(|s| !s.is_empty())); }'), 'fail'],
+ ['truncate-long-values', change('for f in frames {', 'for f in frames {\n let mut f = f.clone();\n if let Some(Value::Array(nodes)) = &mut f.accessibility_tree { for node in nodes { if let Some(Value::String(value)) = node.pointer_mut("/properties/value") { value.truncate(100); } } }'), 'fail'],
+ ['unused-correct', parent, 'fail'],
+ ['drop-tree-output', change('#[serde(default, skip_serializing_if = "Option::is_none")]\n pub accessibility_tree', '#[serde(default, skip_serializing)]\n pub accessibility_tree'), 'fail'],
+ ['drop-tree-input', change('#[serde(default, skip_serializing_if = "Option::is_none")]\n pub accessibility_tree', '#[serde(default, skip_deserializing, skip_serializing_if = "Option::is_none")]\n pub accessibility_tree'), 'fail'],
+ ['emit-null-legacy-tree', change('#[serde(default, skip_serializing_if = "Option::is_none")]\n pub accessibility_tree', '#[serde(default)]\n pub accessibility_tree'), 'fail'],
+ ['drop-audio', change('for a in audio {', 'for a in audio.iter().take(0) {'), 'fail'],
+ ['discard-decoded-records', change('Ok(rec) => parsed.records.push(rec),', 'Ok(rec) => { let _ = rec; },'), 'fail'],
+ ['lost-malformed-count', change('Err(_) => parsed.skipped_lines += 1,', 'Err(_) => {},'), 'fail'],
+ ['missing-source', null, 'error'],
+ ];
+ for (const [name, source, expected] of controls) {
+ const root = mkdtempSync(join(tmpdir(), 'telemetry-ax-calibration-'));
+ try {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ if (source !== null) writeFileSync(join(root, path), source);
+ if (name === 'unused-correct') writeFileSync(join(root, 'unused-correct.rs'), fixed);
+ const fixtureHashes = {};
+ for (const fixture of item.grader.fixtures) {
+ const bytes = readFileSync(join(import.meta.dir, fixture.local_path));
+ const destination = join(root, fixture.destination_path);
+ mkdirSync(dirname(destination), { recursive: true }); writeFileSync(destination, bytes);
+ fixtureHashes[fixture.local_path] = hash(bytes);
+ }
+ const result = spawnSync('/bin/bash', ['-c', item.grader.command], { cwd: root, env: { ...process.env, RUSTUP_TOOLCHAIN: '1.94.0' }, encoding: 'utf8', timeout: item.grader.timeout_seconds * 1000, maxBuffer: 8 * 1024 * 1024 });
+ const errorKind = classifyGraderError(result);
+ const observed = result.error || result.signal || errorKind ? 'error' : result.status === 0 ? 'pass' : 'fail';
+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {
+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, status: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: errorKind, source_sha256: source === null ? null : hash(source), fixture_hashes: fixtureHashes, stdout: result.stdout, stderr: result.stderr }, null, 2));
+ }
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);
+ if (expected === 'pass') expect(result.stdout).toContain('12 passed; 0 failed');
+ if (expected === 'fail') { expect(result.status).toBe(101); expect(result.stdout).toContain('test result: FAILED.'); }
+ if (name === 'parent' || name === 'unused-correct') expect(result.stdout).toContain('8 passed; 4 failed');
+ if (name === 'missing-source') expect(errorKind).toBe('rust_compile_error');
+ } finally { rmSync(root, { recursive: true, force: true }); }
+ }
+}, 600_000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index af9b5ac10..6eb1da37d 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6614,6 +6614,50 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config .eval-hidden/pipe-watch-switch.config.mjs"
}
+ },
+ {
+ "id": "app-telemetry-full-ax-wire",
+ "name": "Preserve complete accessibility trees through the telemetry wire format",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "449156e009be5580b0235d52c8543c56e29e37be",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/7659"
+ },
+ "base_ref": "ada8f2fab2f78ea6a9ac8d59f97c932063418a24",
+ "oracle_ref": "449156e009be5580b0235d52c8543c56e29e37be",
+ "oracle_paths": [
+ "crates/screenpipe-telemetry-wire/src/records.rs"
+ ],
+ "trigger_paths": [
+ "crates/screenpipe-telemetry-wire/src/records.rs"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "telemetry",
+ "preserved-behavior"
+ ],
+ "prompt": "Enterprise frame telemetry loses retained accessibility evidence while encoding or decoding a batch. Repair the existing telemetry wire boundary so an optional accessibility_tree survives JSON input, the public batch encoder and the public batch decoder without losing nodes, order, empty text, nested or unknown properties, false/zero/null values, Unicode, multiline text or long values. An empty tree must remain distinct from an absent tree. Older frames without a tree must remain readable; missing and null trees must be omitted from encoded output. Preserve the existing flattened frame fields, timestamps, device identity and labels, visible summary and nullable browser URL. Mixed frame/audio batches must retain both records and speech. Repeated frame IDs must retain input order and distinct payloads; device identities must not merge. Keep tolerant parsing: blank lines are ignored, each malformed or unknown-kind nonblank line is counted and skipped, and valid neighboring records survive. Empty batches remain empty. Use existing public wire interfaces; no particular implementation is required. Grading compiles the complete records module with synthetic inputs. This task does not require native capture, PII redaction, upload authorization, database hydration or persistence, live services, customer data or execution-isolation changes.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 180,
+ "fixtures": [
+ {
+ "local_path": "graders/telemetry-ax.fixture.rs",
+ "destination_path": ".eval-telemetry-ax/outcomes.rs"
+ },
+ {
+ "local_path": "graders/telemetry-ax.lib.rs",
+ "destination_path": ".eval-telemetry-ax/lib.rs"
+ },
+ {
+ "local_path": "graders/telemetry-ax.Cargo.toml.txt",
+ "destination_path": ".eval-telemetry-ax/Cargo.toml"
+ }
+ ],
+ "command": "cargo test --offline --manifest-path .eval-telemetry-ax/Cargo.toml --test outcomes -- --nocapture"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/telemetry-ax.Cargo.toml.txt b/evals/coding-agent/graders/telemetry-ax.Cargo.toml.txt
new file mode 100644
index 000000000..316c32b93
--- /dev/null
+++ b/evals/coding-agent/graders/telemetry-ax.Cargo.toml.txt
@@ -0,0 +1,16 @@
+# screenpipe — AI that knows everything you've seen, said, or heard
+# https://screenpipe.com
+[package]
+name = "telemetry-ax-eval"
+version = "0.0.0"
+edition = "2021"
+[workspace]
+[lib]
+path = "lib.rs"
+[[test]]
+name = "outcomes"
+path = "outcomes.rs"
+[dependencies]
+serde = { version = "=1.0.228", features = ["derive"] }
+serde_json = "=1.0.149"
+tracing = "=0.1.44"
diff --git a/evals/coding-agent/graders/telemetry-ax.fixture.rs b/evals/coding-agent/graders/telemetry-ax.fixture.rs
new file mode 100644
index 000000000..040d6b37c
--- /dev/null
+++ b/evals/coding-agent/graders/telemetry-ax.fixture.rs
@@ -0,0 +1,54 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+use serde_json::{json, Value};
+use telemetry_ax_eval::records::{build_jsonl, parse_jsonl, AudioRow, FrameRow};
+fn frame(id: i64, tree: Option<Value>) -> FrameRow {
+ let mut row = json!({"frame_id":id,"timestamp":"2026-01-02T03:04:05Z","app_name":"Synthetic Editor","window_name":"Example","browser_url":null,"text":"Visible summary"});
+ if let Some(value) = tree { row["accessibility_tree"] = value; }
+ serde_json::from_value(row).unwrap()
+}
+fn wire(rows: &[FrameRow]) -> Vec<u8> { build_jsonl("device-synthetic", "Synthetic device", rows, &[], &[], &[], &[]) }
+fn values(body: &[u8]) -> Vec<Value> {
+ let parsed = parse_jsonl(body); assert_eq!(parsed.skipped_lines, 0);
+ parsed.records.iter().map(|r| serde_json::to_value(r).unwrap()).collect()
+}
+#[test] fn full_tree_keeps_empty_nodes_and_unknown_properties() {
+ let tree = json!([{"role":"Window","text":"","depth":0},{"role":"Button","text":"","depth":1,"bounds":[1,2,3,4],"properties":{"automation_id":"save","future":{"enabled":false,"number":0,"nothing":null}}}]);
+ assert_eq!(values(&wire(&[frame(1,Some(tree.clone()))]))[0]["accessibility_tree"], tree);
+}
+#[test] fn nested_unicode_and_long_values_remain_complete() {
+ let tree = json!([{"role":"TextField","text":"保存 🧪\nsecond line","properties":{"value":"x".repeat(32768),"children":[{"text":""}]}}]);
+ assert_eq!(values(&wire(&[frame(2,Some(tree.clone()))]))[0]["accessibility_tree"], tree);
+}
+#[test] fn empty_tree_remains_distinct_from_absent_tree() {
+ let row=values(&wire(&[frame(3,Some(json!([])))])); assert_eq!(row[0].get("accessibility_tree"),Some(&json!([])));
+}
+#[test] fn decoder_preserves_received_tree() {
+ let tree=json!([{"properties":{"future":[1,false,null,"Ω"]}}]);
+ let raw=json!({"kind":"frame","device_id":"other","device_label":"Other","frame_id":4,"timestamp":"2026-01-02T03:04:05Z","text":"summary","accessibility_tree":tree});
+ assert_eq!(values(raw.to_string().as_bytes())[0]["accessibility_tree"],raw["accessibility_tree"]);
+}
+#[test] fn legacy_frame_without_tree_stays_readable_and_omits_tree() {
+ let row=values(&wire(&[frame(5,None)])); assert!(row[0].get("accessibility_tree").is_none());assert_eq!(row[0]["text"],"Visible summary");
+}
+#[test] fn explicit_null_tree_remains_absent_on_wire() {
+ let row=values(&wire(&[frame(6,Some(Value::Null))]));assert!(row[0].get("accessibility_tree").is_none());
+}
+#[test] fn frame_fields_and_device_identity_remain_unchanged() {
+ let row=values(&wire(&[frame(7,None)]));let row=&row[0];
+ assert_eq!(row["frame_id"],7);assert_eq!(row["timestamp"],"2026-01-02T03:04:05Z");assert_eq!(row["app_name"],"Synthetic Editor");assert_eq!(row["window_name"],"Example");assert_eq!(row["browser_url"],Value::Null);assert_eq!(row["text"],"Visible summary");assert_eq!(row["device_id"],"device-synthetic");assert_eq!(row["device_label"],"Synthetic device");assert_eq!(row["kind"],"frame");assert!(row.get("frame").is_none());
+}
+#[test] fn mixed_audio_survives_frame_encoding() {
+ let audio:AudioRow=serde_json::from_value(json!({"transcription_id":9,"timestamp":"2026-01-02T03:04:06Z","transcription":"Synthetic speech","speaker":"Speaker 1","device":"microphone"})).unwrap();
+ let body=build_jsonl("dev", "label", &[frame(8,None)], &[audio], &[], &[], &[]);let rows=values(&body);assert_eq!(rows.len(),2);assert_eq!(rows[1]["kind"],"audio");assert_eq!(rows[1]["transcription"],"Synthetic speech");assert_eq!(rows[1]["device_id"],"dev");
+}
+#[test] fn malformed_lines_do_not_poison_valid_neighbors() {
+ let mut body=wire(&[frame(9,None)]);body.extend_from_slice(b"\n \t\n{broken\n{\"kind\":\"unknown\"}\n");body.extend(wire(&[frame(10,None)]));let parsed=parse_jsonl(&body);assert_eq!(parsed.skipped_lines,2);assert_eq!(parsed.records.len(),2);assert_eq!(parsed.records[1].device_id(),"device-synthetic");
+}
+#[test] fn duplicate_frame_ids_keep_order_and_distinct_payloads() {
+ let a=frame(11,None);let mut b=frame(11,None);b.text=Some("later version".into());let rows=values(&wire(&[a,b]));assert_eq!(rows.len(),2);assert_eq!(rows[0]["text"],"Visible summary");assert_eq!(rows[1]["text"],"later version");
+}
+#[test] fn distinct_devices_keep_their_identity() {
+ let mut body=build_jsonl("first", "First", &[frame(12,None)], &[], &[], &[], &[]);body.extend(build_jsonl("second", "Second", &[frame(12,None)], &[], &[], &[], &[]));let parsed=parse_jsonl(&body);assert_eq!(parsed.records.len(),2);assert_eq!(parsed.records[0].device_id(),"first");assert_eq!(parsed.records[1].device_label(),"Second");
+}
+#[test] fn empty_batch_has_no_records_or_errors() {assert!(wire(&[]).is_empty());let parsed=parse_jsonl(b"\n \t\r\n");assert!(parsed.records.is_empty());assert_eq!(parsed.skipped_lines,0);}
diff --git a/evals/coding-agent/graders/telemetry-ax.lib.rs b/evals/coding-agent/graders/telemetry-ax.lib.rs
new file mode 100644
index 000000000..72f4490bc
--- /dev/null
+++ b/evals/coding-agent/graders/telemetry-ax.lib.rs
@@ -0,0 +1,4 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+#[path = "../crates/screenpipe-telemetry-wire/src/records.rs"]
+pub mod records;