Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Leave out only structured image blocks from Clef Flash's text budget

Claude5630dedeab8a
diff --git a/src/chrome/src/agent/systemone-judge.js b/src/chrome/src/agent/systemone-judge.js
index af8fb63a7..848cd010d 100644
--- a/src/chrome/src/agent/systemone-judge.js
+++ b/src/chrome/src/agent/systemone-judge.js
@@ -151,9 +151,11 @@ export function estimateTextTokens(text) {
function assertStateFitsModel(model, state) {
const budget = SYSTEM_ONE_TEXT_TOKEN_BUDGETS[model];
if (!budget) return;
- // Screenshots ride inside the state as data URLs; the provider counts images
- // separately, so only the text around them is measured.
- const text = JSON.stringify(state).replace(/data:[\w.+-]+\/[\w.+-]+;base64,[A-Za-z0-9+/=]*/g, 'data:');
+ // Screenshots ride inside the state as structured image_url blocks; the
+ // provider counts images separately, so only those blocks' payloads are left
+ // out. A data URL that sits in ordinary text is text and is measured.
+ const text = JSON.stringify(state, (_key, value) => (value && typeof value === 'object' && value.type === 'image_url'
+ && value.image_url && typeof value.image_url === 'object' ? { type: 'image_url' } : value));
if (estimateTextTokens(text) > budget) {
throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);
}
diff --git a/src/firefox/src/agent/systemone-judge.js b/src/firefox/src/agent/systemone-judge.js
index c670b8b4b..88912b54d 100644
--- a/src/firefox/src/agent/systemone-judge.js
+++ b/src/firefox/src/agent/systemone-judge.js
@@ -151,9 +151,11 @@ export function estimateTextTokens(text) {
function assertStateFitsModel(model, state) {
const budget = SYSTEM_ONE_TEXT_TOKEN_BUDGETS[model];
if (!budget) return;
- // Screenshots ride inside the state as data URLs; the provider counts images
- // separately, so only the text around them is measured.
- const text = JSON.stringify(state).replace(/data:[\w.+-]+\/[\w.+-]+;base64,[A-Za-z0-9+/=]*/g, 'data:');
+ // Screenshots ride inside the state as structured image_url blocks; the
+ // provider counts images separately, so only those blocks' payloads are left
+ // out. A data URL that sits in ordinary text is text and is measured.
+ const text = JSON.stringify(state, (_key, value) => (value && typeof value === 'object' && value.type === 'image_url'
+ && value.image_url && typeof value.image_url === 'object' ? { type: 'image_url' } : value));
if (estimateTextTokens(text) > budget) {
throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);
}
diff --git a/test/systemone.mjs b/test/systemone.mjs
index e0bbfefe2..319c8666b 100644
--- a/test/systemone.mjs
+++ b/test/systemone.mjs
@@ -56,6 +56,11 @@ for (const build of ['chrome', 'firefox']) {
// A screenshot inside the state is an image, not text, so it does not use up the budget.
await judge.evaluate({ config: { ...config, supportsVision: true }, state: [{ task: 'Is it done?' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,' + 'iVBORw0KGgo'.repeat(50000) } }], questions: one });
assert.equal(requests.length, 2, 'an image payload is excluded from the text budget');
+ // The same data URL as ordinary text is text: the model reads it as tokens.
+ await assert.rejects(judge.evaluate({ config: { ...config, supportsVision: true }, questions: one,
+ state: [{ observation: 'data:image/png;base64,' + 'iVBORw0KGgo'.repeat(400) }, { task: 'Is it done?' }] }),
+ error => error.code === 'JEV_STATE_EXCEEDS_MODEL_INPUT');
+ assert.equal(requests.length, 2, 'a data URL in text is not sent');
await judge.evaluate({ config: { ...config, model: 'another/publisher' }, state: dense.cjk, questions: one });
assert.equal(requests.length, 3, 'other models keep the full state');
assert.ok(mod.estimateTextTokens('x'.repeat(100)) >= 50 && mod.estimateTextTokens('æ—¥'.repeat(10)) >= 20);