Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Fix Codex P1 findings: launch fences, batch execution, Fara schema, pilot auth, budget persistence

Emre Sokullu6f0822976f0c
diff --git a/test/llm-tiny-v2/cases.mjs b/test/llm-tiny-v2/cases.mjs
index c9b24d44a..61d9df6f0 100644
--- a/test/llm-tiny-v2/cases.mjs
+++ b/test/llm-tiny-v2/cases.mjs
@@ -204,7 +204,7 @@ function pilot(category,index,n,seed) {
return {title:index?'Pilot conference space':'Pilot equipment rental',task:'Read the request and quotes, then submit the cheapest eligible code and total dollars. Do not delete records.',pages:[page('Request',[index?'Need approved space. Total = base dollars plus fee dollars for each of three sessions.':'Need approved equipment for five days. Total = daily dollars times five plus the one-time fee.']),page('Quotes',[],['Code','Base/daily dollars','Fee dollars','Approved'],rows)],stages:[stage('Booking','Enter code and total with two decimal places.',{code:winner[0],total:cost(winner).toFixed(2)})]};
}
if(category==='calculation')return {title:index?'Pilot tax-inclusive receipt':'Pilot packaging ratio',task:'Read both pages, compute the requested result, and submit. Do not delete records.',pages:index?[page('Receipt',[`Tax-inclusive total cents: ${12000+n*120}.`]),page('Rule',['Tax rate is 20%. Extract the pre-tax amount, not 20% off the total. Enter dollars with two decimal places.'])]:[page('Delivery',[`Goods: ${31+n}. Damaged: 3.`]),page('Rule',['Pack undamaged goods into cartons of 6, rounding up. Enter carton count as an integer.'])],stages:[stage('Result','Enter the requested amount.',{answer:index?money((12000+n*120)/1.2):String(Math.ceil((28+n)/6))})]};
- if(category==='evidence')return {title:index?'Pilot shipping hold':'Pilot membership exception',task:'Use the evidence pages to decide yes/no and give the decisive source ID. Do not delete records.',pages:index?[page('Question',['Can parcel PILOT-8 ship now? A customs hold overrides a dispatch schedule.']),page('SCHEDULE',['Parcel PILOT-8 is scheduled today.']),page('HOLD',['Customs hold on PILOT-8 remains active.'])]:[page('Question',['Can a premium member reserve this weekend slot? The weekend supplement overrides the standard booking rule.']),page('STANDARD',['All members can book open slots.']),page('WEEKEND',['Weekend slots require premium membership. Applicant is premium.'])],stages:[stage('Decision','Enter outcome and source.',{outcome:index?'no':'yes',source:index?'HOLD':'WEEKEND'})]};
+ if(category==='evidence')return {title:index?'Pilot shipping hold':'Pilot membership exception',task:'Resolve the question from the evidence pages. Submit the outcome and the decisive source ID. You are authorized to submit the form to record your decision. Do not change source records.',pages:index?[page('Question',['Can parcel PILOT-8 ship now? A customs hold overrides a dispatch schedule.']),page('SCHEDULE',['Parcel PILOT-8 is scheduled today.']),page('HOLD',['Customs hold on PILOT-8 remains active.'])]:[page('Question',['Can a premium member reserve this weekend slot? The weekend supplement overrides the standard booking rule.']),page('STANDARD',['All members can book open slots.']),page('WEEKEND',['Weekend slots require premium membership. Applicant is premium.'])],stages:[stage('Decision','Enter outcome and source, then submit.',{outcome:index?'no':'yes',source:index?'HOLD':'WEEKEND'})]};
if(category==='workflow') {
const count=12+n,reference=`PILOT-${index}-${seed}`;
return {title:index?'Pilot volunteer roster':'Pilot workspace request',task:'Follow the brief and later update. Save a draft only. Do not delete records.',pages:[page('Brief',[`Reference ${reference}; initial quantity ${count}; owner Coordination. Each unit costs 7 dollars. No other fees.`])],stages:[stage('Start','Enter the reference and owner.',{reference,owner:'Coordination'}),stage('Update',index?'Remove four cancelled units from the original quantity.':'Add three units to the original quantity.',{quantity:String(index?count-4:count+3)}),stage('Cost','Enter the updated total in dollars, with two decimals.',{total:((index?count-4:count+3)*7).toFixed(2)}),stage('Save','Enter original reference and draft status.',{reference,status:'draft'})]};
diff --git a/test/llm-tiny-v2/lib/adapters.mjs b/test/llm-tiny-v2/lib/adapters.mjs
index aad288c9e..e49077a1e 100644
--- a/test/llm-tiny-v2/lib/adapters.mjs
+++ b/test/llm-tiny-v2/lib/adapters.mjs
@@ -21,8 +21,22 @@ const COMPASS_TOOLS = [
['done', 'Finish after verifying the requested UI change completed.', { summary: { type: 'string' } }, ['summary']],
].map(([name, description, properties, required = []]) => ({ type: 'function', function: { name, description, parameters: { type: 'object', properties, required } } }));
+// Only advertise Fara actions the local runner implements. The pinned reference
+// also documents web_search and read_page_answer_question, but lib/browser.mjs
+// has no cases for either and throws `Unsupported Fara action`.
+const SUPPORTED_FARA_ACTIONS = new Set(['key','type','mouse_move','left_click','left_click_drag','right_click','double_click','triple_click','scroll','hscroll','visit_url','history_back','pause_and_memorize_fact','ask_user_question','wait','terminate']);
+const FARA_PARAMETERS = (() => {
+ const params = JSON.parse(JSON.stringify(faraReference.parameters));
+ if (Array.isArray(params?.properties?.action?.enum)) params.properties.action.enum = params.properties.action.enum.filter(a => SUPPORTED_FARA_ACTIONS.has(a));
+ if (typeof params?.properties?.action?.description === 'string') params.properties.action.description = params.properties.action.description.split('\n').filter(line => !line.includes('`web_search`') && !line.includes('`read_page_answer_question`')).join('\n');
+ // `query` is only required by web_search; `question` stays for ask_user_question.
+ if (params?.properties) delete params.properties.query;
+ if (params?.properties?.question) params.properties.question.description = 'The question to ask. Required by `action=ask_user_question`.';
+ return params;
+})();
+
// Pinned Microsoft reference prompt and schema, with its 1000x1000 display space.
-const FARA_SYSTEM = `${faraReference.FARA_QWEN35_IDENTITY}\n\n${faraReference.CRITICAL_POINTS_FARA_1_5}\n\n${faraReference.FN_CALL_FORMAT.replaceAll('{{', '{').replaceAll('}}', '}').replace('{tool_descs}', JSON.stringify({ name: 'computer_use', description: faraReference.description, parameters: faraReference.parameters }))}`;
+const FARA_SYSTEM = `${faraReference.FARA_QWEN35_IDENTITY}\n\n${faraReference.CRITICAL_POINTS_FARA_1_5}\n\n${faraReference.FN_CALL_FORMAT.replaceAll('{{', '{').replaceAll('}}', '}').replace('{tool_descs}', JSON.stringify({ name: 'computer_use', description: faraReference.description, parameters: FARA_PARAMETERS }))}`;
const BROWSER_USE_SYSTEM = `You are a browser-use agent operating in flash mode. You automate browser tasks by outputting structured JSON actions. Respond with valid JSON: {"memory":"Brief evaluation of the previous step and the next goal","action":[{"action_name":{...params}}]}. Return one action per turn. Use only indexed elements from the current browser state. Available actions: click {"index":number}; input {"index":number,"text":string,"clear":true}; dropdown_options {"index":number}; select_dropdown {"index":number,"text":string}; send_keys {"keys":string}; scroll {"down":boolean,"pages":number}; done {"text":string,"success":boolean}. Only report data observed in browser state or tool outputs. Never fabricate values.`;
diff --git a/test/llm-tiny-v2/manifest.json b/test/llm-tiny-v2/manifest.json
index 4e0ea7caf..51f2d8e53 100644
--- a/test/llm-tiny-v2/manifest.json
+++ b/test/llm-tiny-v2/manifest.json
@@ -21,11 +21,11 @@
"maxTokens": 4096
},
"files": {
- "cases.mjs": "41e83731f976af8166d8608c069841e600f801c843d039efd6e670187809802c",
+ "cases.mjs": "b366181fd85662f94c91b4f31cf4e8d82c4e10d20de3125fb77cfe24dccc2276",
"models.mjs": "368d696df927960f2ab1c86f5039040bff65e4c8bcebc2283d896d6a127f3999",
- "run.mjs": "7e9f470531cd98bcaf2dadd9c016f929ac7f51fbf3b409e22c0a69ceef497102",
+ "run.mjs": "3785337fd2b7973794bcf3582045cfbcbcf6c3f7e86975f88426872c3c496324",
"compare.mjs": "03094a56e61c322307656e85837f861b0f5ed20285212f2a980bbf4546fdeb12",
- "lib/adapters.mjs": "538076896593fa1b116effcabb08cb9c852c8c14c5d328055137dac50a6b6057",
+ "lib/adapters.mjs": "74ae7968121f8c8a02a19bd2803fe4e282fc5e6fd09c7e95dc9cb40e8075469e",
"lib/fara-reference.json": "1e7e196e72d1c941e7ff42f7037bcf2a45b0298d7f7e5e1b2cb7c19bff5a74cd",
"lib/browser.mjs": "0bc6ad1ec80f3b1eef0adc2eb517afb76c623a9d676483b53418424807b1e2e2",
"lib/fixture.mjs": "e4897d0ca07c86d77fed160793975e8ddb3534f988fa1edf02de0c08e72be355",
@@ -135,7 +135,7 @@
"category": "evidence",
"split": "pilot",
"seed": 17,
- "sha256": "dd728e313486ce2d875aa795469d744bc80fb68228dd28e176d24de440f4a41d"
+ "sha256": "193d9b5a849649f5712708668774d99b5602caec04fe40057f35cd21816664cb"
},
{
"id": "pilot-evidence-01-s43",
@@ -143,7 +143,7 @@
"category": "evidence",
"split": "pilot",
"seed": 43,
- "sha256": "4cbe9792fbe7ec5bb5b788ee3aa26f51cd986118b88d0e1d145c0bd7869658da"
+ "sha256": "b43438c1b3ccc3568ce210814fadde45ec2e4818d7e26beff8d4d518cdc4dcc3"
},
{
"id": "pilot-evidence-01-s89",
@@ -151,7 +151,7 @@
"category": "evidence",
"split": "pilot",
"seed": 89,
- "sha256": "2d7390eeb6f0c815e689571e4890215442623e1734bfee647ea61e1e36473d88"
+ "sha256": "72b49235a5557cbd9fce78de5741b154aeec4cb1886ae7a52d1e4ce0e8e0dd41"
},
{
"id": "pilot-evidence-02-s17",
@@ -159,7 +159,7 @@
"category": "evidence",
"split": "pilot",
"seed": 17,
- "sha256": "1b778c0788990c47359e1c9e9652c8fb6160d1e013510c439bf7a672aa48bb79"
+ "sha256": "c1ecd406f9fd166b71b9845411ab90008a7cc8ef3ec40b8c1988da29e167378c"
},
{
"id": "pilot-evidence-02-s43",
@@ -167,7 +167,7 @@
"category": "evidence",
"split": "pilot",
"seed": 43,
- "sha256": "cc27aadf68ce00c9c0ce4dd58dfef78031d85663b58e982e19b7660fd89e1d92"
+ "sha256": "b4191ae2bf4e74637b3b6ec1807ab7648cd10ea9a84c3582b2f3d6f90babd7d0"
},
{
"id": "pilot-evidence-02-s89",
@@ -175,7 +175,7 @@
"category": "evidence",
"split": "pilot",
"seed": 89,
- "sha256": "d21a37f986c53107a3f6a3475cf43a3b124f54142f6c9f5a52a6d5d05b4aae79"
+ "sha256": "ac58f62677184600aa38c044bfb77c64b8b4f6113bf11a1254d264a1992b6129"
},
{
"id": "pilot-workflow-01-s17",
diff --git a/test/llm-tiny-v2/run.mjs b/test/llm-tiny-v2/run.mjs
index e10d883bd..57a6889e5 100644
--- a/test/llm-tiny-v2/run.mjs
+++ b/test/llm-tiny-v2/run.mjs
@@ -69,6 +69,10 @@ export async function runCase({browser,server,participant,task,limits=DEFAULT_LI
const message=response.choices?.[0]?.message||{},actions=parseAdapterActions(participant.adapter,message),results=[];
trace.push({turn:turns,latencyMs,response:message,usage:u});
if(!actions.length){status='unparseable';break;}
+ // Judge only at task termination, NEVER auto-pass midway through a batch.
+ // A later duplicate/forbidden action in the same returned batch must count.
+ // Terminal actions do not discard the remainder of an already-returned batch.
+ let batchEnded = false;
for(let i=0;i<actions.length;i++) {
if(controller.signal.aborted){status='task_timeout';break loop;}
if(actionsTaken>=limits.maxActions){status='max_actions';break loop;}
@@ -82,12 +86,11 @@ export async function runCase({browser,server,participant,task,limits=DEFAULT_LI
// Do not silently retarget stale indices or execute the rest of a failed batch.
break;
}
- // Judge only at task termination, NEVER auto-pass midway through a batch.
- // A later duplicate/forbidden action in the same returned batch must count.
- if(ended(action)) {status='agent_ended';break loop;}
+ if(ended(action)) batchEnded = true;
}
if(session.state.violations.length){status='forbidden_action';break;}
if(session.state.terminal){status='completed';break;}
+ if(batchEnded) {status='agent_ended';break loop;}
await observation.dispose();observation=await observe(page,participant);
appendObservation(messages,participant,message,actions,results,observation);
}
diff --git a/test/llm-tiny/.gitignore b/test/llm-tiny/.gitignore
index fbca22537..a28bdc5d2 100644
--- a/test/llm-tiny/.gitignore
+++ b/test/llm-tiny/.gitignore
@@ -1 +1,3 @@
-results/
+results/*
+!results/.v1-launch-disabled
+!results/.v2-handoff-requested
diff --git a/test/llm-tiny/lib/adapters.mjs b/test/llm-tiny/lib/adapters.mjs
index 19fd6e506..106bffa44 100644
--- a/test/llm-tiny/lib/adapters.mjs
+++ b/test/llm-tiny/lib/adapters.mjs
@@ -21,8 +21,22 @@ const COMPASS_TOOLS = [
['done', 'Finish after verifying the requested UI change completed.', { summary: { type: 'string' } }, ['summary']],
].map(([name, description, properties, required = []]) => ({ type: 'function', function: { name, description, parameters: { type: 'object', properties, required } } }));
+// Only advertise Fara actions the local runner implements. The pinned reference
+// also documents web_search and read_page_answer_question, but run.mjs has no
+// cases for either and throws `Unsupported Fara action`.
+const SUPPORTED_FARA_ACTIONS = new Set(['key','type','mouse_move','left_click','left_click_drag','right_click','double_click','triple_click','scroll','hscroll','visit_url','history_back','pause_and_memorize_fact','ask_user_question','wait','terminate']);
+const FARA_PARAMETERS = (() => {
+ const params = JSON.parse(JSON.stringify(faraReference.parameters));
+ if (Array.isArray(params?.properties?.action?.enum)) params.properties.action.enum = params.properties.action.enum.filter(a => SUPPORTED_FARA_ACTIONS.has(a));
+ if (typeof params?.properties?.action?.description === 'string') params.properties.action.description = params.properties.action.description.split('\n').filter(line => !line.includes('`web_search`') && !line.includes('`read_page_answer_question`')).join('\n');
+ // `query` is only required by web_search; `question` stays for ask_user_question.
+ if (params?.properties) delete params.properties.query;
+ if (params?.properties?.question) params.properties.question.description = 'The question to ask. Required by `action=ask_user_question`.';
+ return params;
+})();
+
// Pinned Microsoft reference prompt and schema, with its 1000x1000 display space.
-const FARA_SYSTEM = `${faraReference.FARA_QWEN35_IDENTITY}\n\n${faraReference.CRITICAL_POINTS_FARA_1_5}\n\n${faraReference.FN_CALL_FORMAT.replaceAll('{{', '{').replaceAll('}}', '}').replace('{tool_descs}', JSON.stringify({ name: 'computer_use', description: faraReference.description, parameters: faraReference.parameters }))}`;
+const FARA_SYSTEM = `${faraReference.FARA_QWEN35_IDENTITY}\n\n${faraReference.CRITICAL_POINTS_FARA_1_5}\n\n${faraReference.FN_CALL_FORMAT.replaceAll('{{', '{').replaceAll('}}', '}').replace('{tool_descs}', JSON.stringify({ name: 'computer_use', description: faraReference.description, parameters: FARA_PARAMETERS }))}`;
const BROWSER_USE_SYSTEM = `You are a browser-use agent operating in flash mode. You automate browser tasks by outputting structured JSON actions. Respond with valid JSON: {"memory":"Brief evaluation of the previous step and the next goal","action":[{"action_name":{...params}}]}. Return one action per turn. Use only indexed elements from the current browser state. Available actions: click {"index":number}; input {"index":number,"text":string,"clear":true}; dropdown_options {"index":number}; select_dropdown {"index":number,"text":string}; send_keys {"keys":string}; scroll {"down":boolean,"pages":number}; done {"text":string,"success":boolean}. Only report data observed in browser state or tool outputs. Never fabricate values.`;
diff --git a/test/llm-tiny/results/.v1-launch-disabled b/test/llm-tiny/results/.v1-launch-disabled
new file mode 100644
index 000000000..04fe49ae5
--- /dev/null
+++ b/test/llm-tiny/results/.v1-launch-disabled
@@ -0,0 +1 @@
+V1 launches are disabled for this run. Use test/llm-tiny-v2; preserve existing results and deployment safeguards.
diff --git a/test/llm-tiny/results/.v2-handoff-requested b/test/llm-tiny/results/.v2-handoff-requested
new file mode 100644
index 000000000..dcd6e6b61
--- /dev/null
+++ b/test/llm-tiny/results/.v2-handoff-requested
@@ -0,0 +1 @@
+Local benchmark launches are paused for the requested single-owner cloud handoff. Inspection and cleanup remain available; do not remove this marker to run legacy helpers.
diff --git a/test/llm-tiny/run.mjs b/test/llm-tiny/run.mjs
index c5662a354..1f4e5cebd 100644
--- a/test/llm-tiny/run.mjs
+++ b/test/llm-tiny/run.mjs
@@ -397,6 +397,10 @@ async function runCase(browser, baseUrl, participant, caseRecord, participantDir
break;
}
const actionResults = [];
+ // Execute the full batch before judging: later actions in the same model
+ // response must still run even if an earlier action completed the UI.
+ let batchPassed = false;
+ let batchEnded = false;
for (let actionIndex = 0; actionIndex < actions.length; actionIndex += 1) {
const action = actions[actionIndex];
try {
@@ -407,10 +411,9 @@ async function runCase(browser, baseUrl, participant, caseRecord, participantDir
: await performIndexedAction(page, observation, action.name, action.args);
actionResults.push(actionResult);
trace.push({ step, actionIndex, latencyMs: actionIndex === 0 ? latencyMs : 0, action: { name: action.name, args: action.args }, actionResult, content: message.content || null });
- if (await fixtureCompleted(page)) { status = 'passed'; break turns; }
+ if (await fixtureCompleted(page)) batchPassed = true;
if ((participant.adapter === 'fara' && ['terminate', 'ask_user_question'].includes(action.args?.action)) || ['done'].includes(action.name)) {
- status = 'ended_without_success';
- break turns;
+ batchEnded = true;
}
observation = await observe(page, participant);
} catch (actionError) {
@@ -420,6 +423,11 @@ async function runCase(browser, baseUrl, participant, caseRecord, participantDir
break turns;
}
}
+ if (batchPassed) { status = 'passed'; break turns; }
+ if (batchEnded) {
+ status = 'ended_without_success';
+ break turns;
+ }
appendAssistantAndObservation(messages, participant, message, actions, actionResults, observation);
}
await page.screenshot({ path: join(participantDir, `${caseRecord.id}-final.png`) });
diff --git a/test/llm-tiny/runpod/v2-launch.mjs b/test/llm-tiny/runpod/v2-launch.mjs
index 9762e9810..f78edf4e0 100644
--- a/test/llm-tiny/runpod/v2-launch.mjs
+++ b/test/llm-tiny/runpod/v2-launch.mjs
@@ -22,7 +22,7 @@ const catalog = {
if (!catalog[name] || process.argv.length !== 3) throw new Error('Select exactly one of the six pilot participants; held-out launch is intentionally gated.');
const [adapter, model] = catalog[name];
const manifestHash = await verifyManifest();
-if (manifestHash !== '60e52b1361d2f93d5cffa0b6805b06c2fbe45c9e64fca27c6431c29e536f293d') throw new Error('Manifest changed since pilot approval');
+if (manifestHash !== '2f28ba931b86a9f83a80ab486c2d47d5df722fa490a510f69b74b71b9c79e2d2') throw new Error('Manifest changed since pilot approval');
await access(new URL('../results/.v1-launch-disabled', import.meta.url));
const tag = `2026-09-13-v2-pilot-${name}`;
const resultDir = new URL(`../../llm-tiny-v2/results/${tag}/`, import.meta.url);
@@ -64,7 +64,12 @@ globalThis.fetch = async (url, options = {}) => {
if (typeof data.usage?.cost === 'number' && data.usage.cost >= 0) {
entry.chargedUsd = data.usage.cost;
budget.apiChargedOrReservedUsd += data.usage.cost - bound;
- if (data.usage.cost > bound) throw new Error('Provider charge exceeded reserved maximum');
+ if (data.usage.cost > bound) {
+ entry.status = response.status;
+ entry.overReservedMaximum = true;
+ await saveBudget(); // Provider already billed; persist actual overage before failing.
+ throw new Error('Provider charge exceeded reserved maximum');
+ }
}
}
if (!response.ok) entry.endpointError = (await response.clone().text()).replaceAll(process.env.OPENROUTER_API_KEY || '\0', '[redacted]').replaceAll(pod?.token || '\0', '[redacted]').slice(0, 3000);