Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Leave out only structured image blocks from Clef Flash's text budget
Claude5630dedeab8a
diff --git a/src/chrome/src/agent/systemone-judge.js b/src/chrome/src/agent/systemone-judge.jsindex af8fb63a7..848cd010d 100644--- a/src/chrome/src/agent/systemone-judge.js+++ b/src/chrome/src/agent/systemone-judge.js@@ -151,9 +151,11 @@ export function estimateTextTokens(text) {function assertStateFitsModel(model, state) {const budget = SYSTEM_ONE_TEXT_TOKEN_BUDGETS[model];if (!budget) return;- // Screenshots ride inside the state as data URLs; the provider counts images- // separately, so only the text around them is measured.- const text = JSON.stringify(state).replace(/data:[\w.+-]+\/[\w.+-]+;base64,[A-Za-z0-9+/=]*/g, 'data:');+ // Screenshots ride inside the state as structured image_url blocks; the+ // provider counts images separately, so only those blocks' payloads are left+ // out. A data URL that sits in ordinary text is text and is measured.+ const text = JSON.stringify(state, (_key, value) => (value && typeof value === 'object' && value.type === 'image_url'+ && value.image_url && typeof value.image_url === 'object' ? { type: 'image_url' } : value));if (estimateTextTokens(text) > budget) {throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);}diff --git a/src/firefox/src/agent/systemone-judge.js b/src/firefox/src/agent/systemone-judge.jsindex c670b8b4b..88912b54d 100644--- a/src/firefox/src/agent/systemone-judge.js+++ b/src/firefox/src/agent/systemone-judge.js@@ -151,9 +151,11 @@ export function estimateTextTokens(text) {function assertStateFitsModel(model, state) {const budget = SYSTEM_ONE_TEXT_TOKEN_BUDGETS[model];if (!budget) return;- // Screenshots ride inside the state as data URLs; the provider counts images- // separately, so only the text around them is measured.- const text = JSON.stringify(state).replace(/data:[\w.+-]+\/[\w.+-]+;base64,[A-Za-z0-9+/=]*/g, 'data:');+ // Screenshots ride inside the state as structured image_url blocks; the+ // provider counts images separately, so only those blocks' payloads are left+ // out. A data URL that sits in ordinary text is text and is measured.+ const text = JSON.stringify(state, (_key, value) => (value && typeof value === 'object' && value.type === 'image_url'+ && value.image_url && typeof value.image_url === 'object' ? { type: 'image_url' } : value));if (estimateTextTokens(text) > budget) {throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);}diff --git a/test/systemone.mjs b/test/systemone.mjsindex e0bbfefe2..319c8666b 100644--- a/test/systemone.mjs+++ b/test/systemone.mjs@@ -56,6 +56,11 @@ for (const build of ['chrome', 'firefox']) {// A screenshot inside the state is an image, not text, so it does not use up the budget.await judge.evaluate({ config: { ...config, supportsVision: true }, state: [{ task: 'Is it done?' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,' + 'iVBORw0KGgo'.repeat(50000) } }], questions: one });assert.equal(requests.length, 2, 'an image payload is excluded from the text budget');+ // The same data URL as ordinary text is text: the model reads it as tokens.+ await assert.rejects(judge.evaluate({ config: { ...config, supportsVision: true }, questions: one,+ state: [{ observation: 'data:image/png;base64,' + 'iVBORw0KGgo'.repeat(400) }, { task: 'Is it done?' }] }),+ error => error.code === 'JEV_STATE_EXCEEDS_MODEL_INPUT');+ assert.equal(requests.length, 2, 'a data URL in text is not sent');await judge.evaluate({ config: { ...config, model: 'another/publisher' }, state: dense.cjk, questions: one });assert.equal(requests.length, 3, 'other models keep the full state');assert.ok(mod.estimateTextTokens('x'.repeat(100)) >= 50 && mod.estimateTextTokens('æ—¥'.repeat(10)) >= 20);