Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: preserve accepted localization during cold recovery

louisef2939db220d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 75888f302..71eef1807 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -621,3 +621,25 @@ The synthetic broken state fails its intended assertion; the reference passes.
The command adapter retains Buffers when `encoding: null` is requested and keeps
normal command output as text. This is harness calibration, not model evaluation
or proof of process isolation; no real recordings or customer files are used.
+
+## Localization recovery preserves accepted output
+
+`app-localization-recovery-preservation` executes the actual translation service
+with synthetic provider responses and disposable catalogs. Cold recovery must
+not overwrite an accepted rich-text or ICU translation with an incompatible
+legacy download. Fourteen outcomes cover arrival order, later valid corrections,
+frontend/native catalogs, multiple locales, disjoint messages, retained rejected
+diagnostics, source identity/format refusal, warm-cache preservation and stale
+loose-output cleanup. The parent fails five persisted-output checks while nine
+preserved behaviors pass; the service-only reference passes all fourteen.
+
+Run `bun test evals/coding-agent/calibrate-localization-recovery.test.js` with
+`@generaltranslation/icu` 0.1.2 available in the desktop dependency directory.
+Ten calibration controls reject unused fixes, first-valid-only merging, dropped
+diagnostics, frontend-only repair, identity bypass and blanket refusal. Equivalent
+object assignment passes; missing service code remains a setup error. Current
+source separately passes the fourteen outcomes and five existing service tests.
+
+Fixtures and dependency links are materialized only for grading. No live provider,
+real user content, native build, release or publication is involved. These are
+corpus checks, not model trials, packaging validation or proof of trial isolation.
diff --git a/evals/coding-agent/calibrate-localization-recovery.test.js b/evals/coding-agent/calibrate-localization-recovery.test.js
new file mode 100644
index 000000000..80018463b
--- /dev/null
+++ b/evals/coding-agent/calibrate-localization-recovery.test.js
@@ -0,0 +1,27 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdirSync,mkdtempSync,readFileSync,rmSync,symlinkSync,writeFileSync} from 'node:fs';
+import {dirname,join,resolve} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),base='apps/screenpipe-app-tauri',folder=base+'/scripts/i18n',target=folder+'/service.mjs';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-localization-recovery-preservation');
+const source=(ref,p)=>execFileSync('git',['show',`${ref}:${p}`],{cwd:repo,encoding:'utf8'});
+const broken=source(item.base_ref,target),fixed=source(item.oracle_ref,target),grader=readFileSync(join(import.meta.dir,'graders/localization-recovery.fixture.js'));
+const root=mkdtempSync(join(tmpdir(),'localization-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const support=['prepare.mjs','validate.mjs','config.mjs'].map(f=>[folder+'/'+f,source(item.oracle_ref,folder+'/'+f)]);support.push([base+'/gt.config.json',source(item.oracle_ref,base+'/gt.config.json')]);
+function grade(name,code,unused){const cwd=join(root,name);for(const [p,b]of support){mkdirSync(dirname(join(cwd,p)),{recursive:true});writeFileSync(join(cwd,p),b);}if(code!==null)writeFileSync(join(cwd,target),code);if(unused)writeFileSync(join(cwd,folder,'unused.mjs'),unused);writeFileSync(join(cwd,folder,'eval-localization-recovery.test.js'),grader);symlinkSync(join(repo,base,'node_modules'),join(cwd,base,'node_modules'),'dir');return spawnSync(process.execPath,['test',folder+'/eval-localization-recovery.test.js'],{cwd,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toMatch(/[1-9][0-9]* fail/);expect(r.stderr).not.toContain('Cannot find');expect(r.stderr).not.toContain('Unhandled error');}
+function change(old,next){expect(fixed.split(old)).toHaveLength(2);return fixed.replace(old,next);}
+const merge='{ ...content, ...downloads[batch.kind][file.locale], ...valid }';
+test('historical parent fails intended persisted outcomes with preserved behavior',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('5 fail');expect(r.stderr).toContain('9 pass');});
+test('historical reference passes all fourteen outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('14 pass');});
+test('equivalent object assignment passes without enforcing merge syntax',()=>expect(grade('equivalent',change(merge,'Object.assign({}, content, downloads[batch.kind][file.locale], valid)')).status).toBe(0));
+test('unused correct service does not fix the real caller',()=>fails(grade('unused',broken,fixed)));
+test('first-valid-only discards later valid corrections',()=>fails(grade('first-valid',change(merge,'{ ...content, ...valid, ...downloads[batch.kind][file.locale] }'))));
+test('discarding invalid diagnostic entries is rejected',()=>fails(grade('diagnostics',change(merge,'{ ...downloads[batch.kind][file.locale], ...valid }'))));
+test('frontend-only repair leaves native placeholder corruption',()=>fails(grade('native',change(merge,"batch.kind === 'native' ? { ...downloads[batch.kind][file.locale], ...content } : "+merge))));
+test('download identity checks cannot be bypassed',()=>fails(grade('identity',change('if (!batch || file.fileFormat !== (batch.kind === "gt" ? "GTJSON" : "JSON"))','if (false)'))));
+test('blanket recovery refusal is rejected',()=>fails(grade('refusal',change(' const dir = path.join(root, ".localization");',' return {exitCode: 1};\n const dir = path.join(root, ".localization");'))));
+test('missing service is setup error rather than behavior evidence',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 5bf334e84..a0a142809 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-21.1",
+ "dataset_version": "2026-09-21.2",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -2112,6 +2112,45 @@
],
"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/mcp-workflow-ids.test.js"
}
+ },
+ {
+ "id": "app-localization-recovery-preservation",
+ "name": "Preserve accepted localization during cold recovery",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "474f641f520b9057be862e3d2ed1cb9f03d1978b",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/7172"
+ },
+ "base_ref": "90ebd317d61df0a81a3ac92ee29a1c25dffaa798",
+ "oracle_ref": "474f641f520b9057be862e3d2ed1cb9f03d1978b",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/scripts/i18n/service.mjs"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "localization",
+ "recovery",
+ "data-preservation"
+ ],
+ "prompt": "Cold localization recovery can report success while writing an incompatible legacy translation over a valid translation already recovered from another provider file. Fix the actual translation service so persisted locale catalogs preserve accepted rich-text structure and ICU placeholders when a later incompatible file is downloaded. Accept a later valid correction, recover regardless of whether the incompatible file arrives first or last, combine disjoint messages, and preserve unmatched rejected entries for diagnostics. Apply these rules independently to frontend and native catalogs and each locale. Fully recovered messages must not trigger translation, branch creation or uploads. Refuse downloaded results with mismatched branch, file, version, locale or format. Preserve the warm-cache path: valid cached frontend/native messages make no provider calls, caller-owned cache stays unchanged, and stale loose output is cleared. Use synthetic provider responses and disposable temporary files; do not contact a translation provider, translate real content, build or publish an app. No particular merge expression or internal helper is required. Native packaging, live provider availability and actual model quality are outside this task.",
+ "agent_timeout_seconds": 1200,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/localization-recovery.fixture.js",
+ "destination_path": "apps/screenpipe-app-tauri/scripts/i18n/eval-localization-recovery.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test apps/screenpipe-app-tauri/scripts/i18n/eval-localization-recovery.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/localization-recovery.fixture.js b/evals/coding-agent/graders/localization-recovery.fixture.js
new file mode 100644
index 000000000..82f107b3d
--- /dev/null
+++ b/evals/coding-agent/graders/localization-recovery.fixture.js
@@ -0,0 +1,46 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterEach, expect, test } from 'bun:test';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { translateMissing } from './service.mjs';
+const roots=[];
+afterEach(async()=>{for(const root of roots.splice(0))await fs.rm(root,{recursive:true,force:true});});
+const source={field:['Correction',{i:1,t:'textarea'}]};
+const good=['修正',{i:1,t:'textarea'}];
+const stale=['古い修正',{i:1,t:'textarea',d:{arl:'Correction'}}];
+async function recover(contents,{kind='gt',locales=['ja'],messages=source,changeFile=x=>x}={}){
+ const root=await fs.mkdtemp(path.join(os.tmpdir(),'localization-outcome-'));roots.push(root);
+ await fs.mkdir(path.join(root,'.localization/source'),{recursive:true});
+ await fs.writeFile(path.join(root,'.localization/source/en.metadata.json'),'{}');
+ const calls=[];
+ const result=await translateMissing({root,source:kind==='gt'?messages:{},native:{messages:kind==='native'?messages:{},metadata:{}},config:{defaultLocale:'en',locales},policy:'eval-policy',cache:null,request:async(endpoint,body)=>{
+ calls.push(endpoint);
+ if(endpoint.endsWith('branches/info'))return {branches:[{id:'synthetic-branch',name:'desktop-eval-policy'}]};
+ if(endpoint.endsWith('files/orphaned'))return {orphanedFiles:contents.map((_,i)=>({fileId:`file-${i}`,versionId:`version-${i}`,fileName:`desktop/${kind}/${i}.json`}))};
+ if(endpoint.endsWith('files/download'))return {files:body.map(ref=>changeFile({...ref,fileFormat:kind==='gt'?'GTJSON':'JSON',data:Buffer.from(JSON.stringify(contents[Number(ref.fileId.slice(5))])).toString('base64')}))};
+ throw new Error(`Unexpected provider mutation: ${endpoint}`);
+ }});
+ const catalogs={};for(const locale of locales)catalogs[locale]=await fs.readFile(path.join(root,'.localization',kind,locale+'.json'),'utf8').then(JSON.parse).catch(()=>null);
+ return {result,catalogs,calls,root};
+}
+function successful(r){expect(r.result.exitCode).toBe(0);expect(r.calls.filter(x=>x==='/v2/translate')).toHaveLength(0);expect(r.calls.some(x=>/create|upload/.test(x))).toBe(false);}
+test('later incompatible legacy rich text cannot overwrite an accepted download',async()=>{const r=await recover([{field:good},{field:stale}]);successful(r);expect(r.catalogs.ja.field).toEqual(good);});
+test('earlier incompatible rich text does not block a later valid correction',async()=>{const r=await recover([{field:stale},{field:good}]);successful(r);expect(r.catalogs.ja.field).toEqual(good);});
+test('latest valid correction replaces earlier accepted wording',async()=>{const newer=['新しい修正',{i:1,t:'textarea'}];const r=await recover([{field:good},{field:newer}]);successful(r);expect(r.catalogs.ja.field).toEqual(newer);});
+for(const kind of ['gt','native'])test(`${kind} missing placeholder cannot overwrite valid wording`,async()=>{const r=await recover([{greet:'こんにちは {name}'},{greet:'こんにちは'}],{kind,messages:{greet:'Hello {name}'}});successful(r);expect(r.catalogs.ja.greet).toBe('こんにちは {name}');});
+test('disjoint files retain both valid messages',async()=>{const r=await recover([{first:'最初'},{second:'次'}],{messages:{first:'First',second:'Second'}});successful(r);expect(r.catalogs.ja).toEqual({first:'最初',second:'次'});});
+test('a legacy file preserves unmatched rejected entries for diagnostics',async()=>{const r=await recover([{field:good},{field:stale,diagnostic:'old entry'}]);successful(r);expect(r.catalogs.ja).toEqual({field:good,diagnostic:'old entry'});});
+test('independent locale downloads retain accepted content',async()=>{const r=await recover([{field:good},{field:stale}],{locales:['ja','fr']});successful(r);expect(r.catalogs.ja.field).toEqual(good);expect(r.catalogs.fr.field).toEqual(good);});
+for(const [label,changeFile] of [
+ ['wrong branch',f=>({...f,branchId:'foreign'})],['wrong version',f=>({...f,versionId:'foreign'})],['wrong file',f=>({...f,fileId:'foreign'})],['wrong locale',f=>({...f,locale:'xx'})],['wrong format',f=>({...f,fileFormat:'JSON'})]
+])test(`${label} is an integrity failure`,async()=>{const r=await recover([{field:good}],{changeFile});expect(r.result.exitCode).toBe(1);expect(r.calls.some(x=>/translate|create|upload/.test(x))).toBe(false);});
+test('valid cached translations make no provider calls and stale loose output is cleared',async()=>{
+ const root=await fs.mkdtemp(path.join(os.tmpdir(),'localization-cache-'));roots.push(root);
+ await fs.mkdir(path.join(root,'.localization/source'),{recursive:true});await fs.mkdir(path.join(root,'.localization/gt'),{recursive:true});
+ await fs.writeFile(path.join(root,'.localization/source/en.metadata.json'),'{}');await fs.writeFile(path.join(root,'.localization/gt/ja.json'),'{"stale":"old"}');
+ const cache={translations:{ja:{title:'設定'}},native:{ja:{toast:'こんにちは {name}'}}};const before=structuredClone(cache);let calls=0;
+ const r=await translateMissing({root,source:{title:'Settings'},native:{messages:{toast:'Hello {name}'},metadata:{}},config:{defaultLocale:'en',locales:['ja']},policy:'eval-policy',cache,request:async()=>{calls++;throw new Error('unexpected request');}});
+ expect(r.exitCode).toBe(0);expect(calls).toBe(0);expect(cache).toEqual(before);expect(JSON.parse(await fs.readFile(path.join(root,'.localization/gt/ja.json'),'utf8'))).toEqual({});
+});