Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover memory tag filtering around malformed rows (#7499)

Louis Beaumontd3dd27321cf0
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index bb14694c4..ed0e372fe 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.
This does not establish native recording continuity, real authentication,
account-switch race safety, browser integration, isolation or model performance.
-The current app corpus contains 128 git-mined regressions. See
+The current app corpus contains 132 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -1655,3 +1655,16 @@ Saved status and missing-source classification. Hidden fixtures and dependency
links appear only when grading begins. This does not test editor internals,
native stop/deferral release, database durability, global recording pause, full
current dependency parity, execution isolation or model improvement.
+
+## Memory tags around malformed rows
+
+`app-memory-tag-filter-malformed-json` executes actual database list/count methods
+against disposable SQLite databases. Six outcomes cover malformed stored tags,
+exact multi-tag matching, missing tags, full-text search, unfiltered reads, source
+and importance filters, ordering, pagination and preservation of stored values.
+The parent fails four outcomes and preserves two; the historical source-only fix
+passes six. Run `bun test evals/coding-agent/calibrate-memory-tag-filter.test.js`
+for correct, broken, equivalent, disconnected, partial-repair, ignored-filter,
+blanket-filter and missing-source controls. Hidden fixtures appear only at grading.
+No build-cache links are declared. This does not establish sync ingestion, HTTP
+routing, hybrid storage, concurrent writes, execution isolation or model quality.
diff --git a/evals/coding-agent/calibrate-memory-tag-filter.test.js b/evals/coding-agent/calibrate-memory-tag-filter.test.js
new file mode 100644
index 000000000..2f68e5c70
--- /dev/null
+++ b/evals/coding-agent/calibrate-memory-tag-filter.test.js
@@ -0,0 +1,49 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, writeFileSync, rmSync, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir,'cases.json'))).cases.find(c=>c.id==='app-memory-tag-filter-malformed-json');
+const source = item.oracle_paths[0];
+const git = (...args) => execFileSync('git',args,{cwd:repo,encoding:'utf8',maxBuffer:128*1024*1024});
+const parent=git('show',`${item.base_ref}:${source}`), fixed=git('show',`${item.oracle_ref}:${source}`);
+const fixture=readFileSync(join(import.meta.dir,'graders/memory-tag-filter.rs'));
+const hash=x=>createHash('sha256').update(x).digest('hex');
+function change(text,from,to,count=1){expect(text.split(from).length-1).toBe(count);return text.split(from).join(to);}
+test('memory tag grader rejects broken, bypass and preserved-behavior regressions',()=>{
+ const root=mkdtempSync(join(tmpdir(),'memory-tag-calibration-'));
+ try {
+ const archive=execFileSync('git',['archive',item.base_ref],{cwd:repo,maxBuffer:128*1024*1024});
+ execFileSync('tar',['-x','-C',root],{input:archive});
+ writeFileSync(join(root,'crates/screenpipe-db/tests/eval_memory_tag_filter.rs'),fixture);
+ const guarded="json_each(CASE WHEN json_valid({tags_col}) THEN {tags_col} ELSE '[]' END)";
+ const controls=[
+ ['parent',parent,'fail'],['reference',fixed,'pass'],
+ ['equivalent',change(fixed,'json_valid({tags_col}) THEN','json_valid({tags_col}) = 1 THEN',2),'pass'],
+ ['unused',parent,'fail'],
+ ['count-only',fixed.replace(guarded,'json_each({tags_col})'),'fail'],
+ ['list-only',(()=>{const at=fixed.lastIndexOf(guarded);expect(at).toBeGreaterThan(0);return fixed.slice(0,at)+fixed.slice(at).replace(guarded,'json_each({tags_col})');})(),'fail'],
+ ['ignore-tags',change(fixed,'.bind(&tags_all_json)','.bind("[]")',2),'fail'],
+ ['blanket-filter',change(fixed,'.bind(&tags_all_json)',String.raw`.bind("[\"missing\"]")`,2),'fail'],
+ ['missing',null,'error'],
+ ];
+ for(const [name,text,expected] of controls){
+ if(text===null)rmSync(join(root,source));else writeFileSync(join(root,source),text);
+ if(name==='unused')writeFileSync(join(root,'unused-correct.rs'),fixed);else rmSync(join(root,'unused-correct.rs'),{force:true});
+ // Calibration alone reuses its own target. No evaluated agent runs here.
+ const result=spawnSync('/bin/bash',['-c',item.grader.command],{cwd:root,encoding:'utf8',timeout:item.grader.timeout_seconds*1000,maxBuffer:8*1024*1024});
+ const kind=classifyGraderError(result);const observed=result.error||result.signal||kind?'error':result.status===0?'pass':'fail';
+ if(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS){const dir=resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS);mkdirSync(dir,{recursive:true});writeFileSync(join(dir,`${name}.json`),JSON.stringify({expected,observed,command:item.grader.command,status:result.status,signal:result.signal,error:result.error?.message??null,error_kind:kind,source_sha256:text===null?null:hash(text),fixture_sha256:hash(fixture),stdout:result.stdout,stderr:result.stderr},null,2));}
+ expect(result.error).toBeUndefined();expect(result.signal).toBeNull();expect(observed).toBe(expected);
+ if(expected==='pass')expect(result.stdout).toContain('6 passed; 0 failed');
+ if(expected==='fail'){expect(result.status).toBe(101);expect(result.stdout).toContain('test result: FAILED.');}
+ if(name==='parent')expect(result.stdout).toContain('2 passed; 4 failed');
+ if(name==='missing')expect(kind).toBe('rust_compile_error');
+ }
+ }finally{rmSync(root,{recursive:true,force:true});}
+},1500000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 09e0bbd46..f7085b3ef 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -5443,6 +5443,41 @@
"apps/screenpipe-app-tauri/components/meeting-notes/note-view.tsx",
"apps/screenpipe-app-tauri/components/meeting-notes/note-save-queue.ts"
]
+ },
+ {
+ "id": "app-memory-tag-filter-malformed-json",
+ "name": "Keep memory tag filtering usable around malformed stored tags",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "b3ce2f4d751950a85446ac7ca92366687efb4b3d"
+ },
+ "base_ref": "06b856d7395859a38b249eb2bf921524c148f921",
+ "oracle_ref": "b3ce2f4d751950a85446ac7ca92366687efb4b3d",
+ "oracle_paths": [
+ "crates/screenpipe-db/src/db/memories.rs"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "crates/screenpipe-db/src/db/memories.rs"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "database",
+ "search"
+ ],
+ "prompt": "Memory listing and counting fail when an unrelated stored memory has empty, null or malformed JSON tags. Make exact tag-filtered reads tolerate those rows while returning and counting the correct valid memories. Requested tags use exact AND semantics; unknown tags match nothing. Preserve combined full-text search, source and minimum-importance filters, descending importance order and pagination. Empty tag filters must retain all memories, including rows with malformed tags. Preserve ordinary valid-tag behavior and do not delete or rewrite stored memories to make reads succeed. Exercise the actual database methods using disposable synthetic databases. No specific helper, query spelling or implementation is required. Do not access user databases or live services.",
+ "agent_timeout_seconds": 1800,
+ "grader": {
+ "timeout_seconds": 180,
+ "fixtures": [
+ {
+ "local_path": "graders/memory-tag-filter.rs",
+ "destination_path": "crates/screenpipe-db/tests/eval_memory_tag_filter.rs"
+ }
+ ],
+ "command": "cargo test --offline --locked -p screenpipe-db --test eval_memory_tag_filter -- --test-threads=1"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/memory-tag-filter.rs b/evals/coding-agent/graders/memory-tag-filter.rs
new file mode 100644
index 000000000..6393a2623
--- /dev/null
+++ b/evals/coding-agent/graders/memory-tag-filter.rs
@@ -0,0 +1,64 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+use screenpipe_db::DatabaseManager;
+
+async fn db(bad: bool) -> DatabaseManager {
+ let db = DatabaseManager::new("sqlite::memory:", Default::default()).await.unwrap();
+ sqlx::migrate!("./src/migrations").run(&db.pool).await.unwrap();
+ sqlx::query("INSERT INTO memories(id,content,source,tags,importance) VALUES (1,'needle both','local','[\"keep\",\"other\"]',0.9),(2,'needle keep','cloud','[\"keep\"]',0.5),(3,'needle other','local','[\"other\"]',0.7)")
+ .execute(&db.pool).await.unwrap();
+ if bad {
+ sqlx::query("INSERT INTO memories(id,content,source,tags) VALUES (4,'needle empty','local',''),(5,'needle null','local',NULL),(6,'needle malformed','local','not json')")
+ .execute(&db.pool).await.unwrap();
+ }
+ db
+}
+fn tags(values: &[&str]) -> Vec<String> { values.iter().map(|s| s.to_string()).collect() }
+async fn ids(db: &DatabaseManager, q: Option<&str>, source: Option<&str>, tags: &[String], limit: u32, offset: u32) -> Vec<i64> {
+ let rows = db.list_memories(q,source,None,None,None,None,limit,offset,Some("importance"),Some("desc"),tags).await.expect("memory listing must succeed");
+ let mut ids: Vec<i64> = rows.into_iter().map(|row| row.id).collect(); ids.sort(); ids
+}
+async fn unchanged(db: &DatabaseManager) {
+ let stored: Vec<(i64, Option<String>)> = sqlx::query_as("SELECT id,tags FROM memories ORDER BY id").fetch_all(&db.pool).await.unwrap();
+ assert_eq!(stored,vec![(1,Some("[\"keep\",\"other\"]".into())),(2,Some("[\"keep\"]".into())),(3,Some("[\"other\"]".into())),(4,Some("".into())),(5,None),(6,Some("not json".into()))]);
+}
+#[tokio::test]
+async fn malformed_rows_do_not_break_listing() {
+ let db = db(true).await;
+ assert_eq!(ids(&db,None,None,&tags(&["keep"]),50,0).await,vec![1,2]);
+ unchanged(&db).await;
+}
+#[tokio::test]
+async fn malformed_rows_do_not_break_counting() {
+ let db = db(true).await;
+ assert_eq!(db.count_memories(None,None,None,None,None,None,&tags(&["keep"])).await.expect("memory counting must succeed"),2);
+ unchanged(&db).await;
+}
+#[tokio::test]
+async fn all_requested_tags_and_absent_tags_are_respected() {
+ let db = db(true).await;
+ assert_eq!(ids(&db,None,None,&tags(&["keep","other"]),50,0).await,vec![1]);
+ assert_eq!(db.count_memories(None,None,None,None,None,None,&tags(&["keep","other"])).await.unwrap(),1);
+ assert_eq!(ids(&db,None,None,&tags(&["missing"]),50,0).await,Vec::<i64>::new());
+}
+#[tokio::test]
+async fn unfiltered_reads_preserve_malformed_rows() {
+ let db = db(true).await;
+ assert_eq!(ids(&db,None,None,&[],50,0).await,vec![1,2,3,4,5,6]);
+ assert_eq!(db.count_memories(None,None,None,None,None,None,&[]).await.unwrap(),6);
+}
+#[tokio::test]
+async fn text_search_and_tags_work_together() {
+ let db = db(true).await;
+ assert_eq!(ids(&db,Some("needle"),None,&tags(&["keep"]),50,0).await,vec![1,2]);
+ assert_eq!(db.count_memories(Some("needle"),None,None,None,None,None,&tags(&["keep"])).await.unwrap(),2);
+ assert!(ids(&db,Some("unmatched"),None,&tags(&["keep"]),50,0).await.is_empty());
+}
+#[tokio::test]
+async fn valid_rows_keep_filtering_order_and_pagination() {
+ let db = db(false).await;
+ assert_eq!(ids(&db,None,None,&tags(&["keep"]),1,1).await,vec![2]);
+ assert_eq!(ids(&db,None,Some("local"),&tags(&["keep"]),50,0).await,vec![1]);
+ assert_eq!(db.count_memories(None,Some("local"),None,Some(0.8),None,None,&tags(&["keep"])).await.unwrap(),1);
+ assert_eq!(ids(&db,None,None,&tags(&["keep","other"]),50,0).await,vec![1]);
+}