Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Bound Clef Flash's state by UTF-8 bytes, a provable upper bound
Claude69dce0944774
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.mdindex fa3395854..802f262b9 100644--- a/docs/providers-and-models.md+++ b/docs/providers-and-models.md@@ -90,10 +90,10 @@ provider until changed. Provider keys are stored separately.`cloudflare/clef-flash` reads only the first ~2K tokens of a long text state(OpenRouter's model page says Workers AI truncates it), so-`SYSTEM_ONE_TEXT_TOKEN_BUDGETS` in `systemone-judge.js` caps the state it is sent-at 1,500 *estimated* tokens. The estimate is deliberately high (no tokenizer-ships with the extension): ASCII counts half a token per character, other-characters three (the worst case for CJK and emoji), long unbroken letter/digit runs one each. A longer state is not+`SYSTEM_ONE_TEXT_BYTE_BUDGETS` in `systemone-judge.js` caps the state it is sent+at 1,500 UTF-8 bytes. A byte-level BPE tokenizer never produces more tokens than+bytes, so this holds for any text without shipping a tokenizer; image blocks are+left out because the provider counts them separately. A longer state is notsent; the check takes the same fallback as for an unavailable decision model(failure reason `state_too_long`) instead of judging evidence it only partlyread. In practice Clef Flash answers short decisions, and long completiondiff --git a/src/chrome/src/agent/systemone-judge.js b/src/chrome/src/agent/systemone-judge.jsindex 9f8e3409d..2570b705f 100644--- a/src/chrome/src/agent/systemone-judge.js+++ b/src/chrome/src/agent/systemone-judge.js@@ -132,32 +132,21 @@ function validateState(state, multimodal = false) {// confident and wrong. A state past the budget is not sent at all; the caller// takes the same fallback as for any unavailable decision model. Images are// counted separately by the provider and are not part of this text.-// The budget is in estimated tokens and sits well under the 2K the model reads-// (see estimateTextTokens), because no tokenizer ships with the extension.-export const SYSTEM_ONE_TEXT_TOKEN_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 1500 });--// A deliberately high token estimate, so a state it passes stays under the real-// limit for ordinary text, CJK, and identifier-heavy input alike: ASCII counts-// half a token per character (English and JSON run near a quarter), any other-// character three (CJK and emoji take one to three, so the worst case is-// charged), and long unbroken runs of-// letters and digits (hashes, base64, ids) one per character.-export function estimateTextTokens(text) {- let tokens = 0;- const rest = String(text).replace(/[A-Za-z0-9+/=_-]{24,}/g, run => { tokens += run.length; return ' '; });- for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 3;- return Math.ceil(tokens);-}+// The budget is in UTF-8 bytes: a byte-level BPE tokenizer never produces more+// tokens than bytes, so the byte count is a bound that holds for any text (CJK,+// emoji, punctuation, hashes) without shipping a tokenizer. 1,500 sits well+// under the roughly 2K tokens the model reads.+export const SYSTEM_ONE_TEXT_BYTE_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 1500 });function assertStateFitsModel(model, state) {- const budget = SYSTEM_ONE_TEXT_TOKEN_BUDGETS[model];+ const budget = SYSTEM_ONE_TEXT_BYTE_BUDGETS[model];if (!budget) return;// Screenshots ride inside the state as structured image_url blocks; the// provider counts images separately, so only those blocks' payloads are left// out. A data URL that sits in ordinary text is text and is measured.const text = JSON.stringify(state, (_key, value) => (value && typeof value === 'object' && value.type === 'image_url'&& value.image_url && typeof value.image_url === 'object' ? { type: 'image_url' } : value));- if (estimateTextTokens(text) > budget) {+ if (new TextEncoder().encode(text).length > budget) {throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);}}diff --git a/src/firefox/src/agent/systemone-judge.js b/src/firefox/src/agent/systemone-judge.jsindex aa8238fb1..db94c6ae1 100644--- a/src/firefox/src/agent/systemone-judge.js+++ b/src/firefox/src/agent/systemone-judge.js@@ -132,32 +132,21 @@ function validateState(state, multimodal = false) {// confident and wrong. A state past the budget is not sent at all; the caller// takes the same fallback as for any unavailable decision model. Images are// counted separately by the provider and are not part of this text.-// The budget is in estimated tokens and sits well under the 2K the model reads-// (see estimateTextTokens), because no tokenizer ships with the extension.-export const SYSTEM_ONE_TEXT_TOKEN_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 1500 });--// A deliberately high token estimate, so a state it passes stays under the real-// limit for ordinary text, CJK, and identifier-heavy input alike: ASCII counts-// half a token per character (English and JSON run near a quarter), any other-// character three (CJK and emoji take one to three, so the worst case is-// charged), and long unbroken runs of-// letters and digits (hashes, base64, ids) one per character.-export function estimateTextTokens(text) {- let tokens = 0;- const rest = String(text).replace(/[A-Za-z0-9+/=_-]{24,}/g, run => { tokens += run.length; return ' '; });- for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 3;- return Math.ceil(tokens);-}+// The budget is in UTF-8 bytes: a byte-level BPE tokenizer never produces more+// tokens than bytes, so the byte count is a bound that holds for any text (CJK,+// emoji, punctuation, hashes) without shipping a tokenizer. 1,500 sits well+// under the roughly 2K tokens the model reads.+export const SYSTEM_ONE_TEXT_BYTE_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 1500 });function assertStateFitsModel(model, state) {- const budget = SYSTEM_ONE_TEXT_TOKEN_BUDGETS[model];+ const budget = SYSTEM_ONE_TEXT_BYTE_BUDGETS[model];if (!budget) return;// Screenshots ride inside the state as structured image_url blocks; the// provider counts images separately, so only those blocks' payloads are left// out. A data URL that sits in ordinary text is text and is measured.const text = JSON.stringify(state, (_key, value) => (value && typeof value === 'object' && value.type === 'image_url'&& value.image_url && typeof value.image_url === 'object' ? { type: 'image_url' } : value));- if (estimateTextTokens(text) > budget) {+ if (new TextEncoder().encode(text).length > budget) {throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);}}diff --git a/test/systemone.mjs b/test/systemone.mjsindex 7747b5e4e..2b569e6bc 100644--- a/test/systemone.mjs+++ b/test/systemone.mjs@@ -37,8 +37,8 @@ for (const build of ['chrome', 'firefox']) {return { ok: true, json: async () => ({ model: body.model, answers: { p: { type: 'noul', noul: .9 } }, usage: { input_tokens: 10, output_tokens: 0 } }) }; } });const one = { p: questions.p };const config = { provider: 'openrouter', model: 'cloudflare/clef-flash', apiKey: 'test', url: 'https://example.test/v1/decisions' };- const budget = mod.SYSTEM_ONE_TEXT_TOKEN_BUDGETS['cloudflare/clef-flash'];- assert.ok(budget < 2000, 'the budget stays under the 2K tokens the model reads');+ const budget = mod.SYSTEM_ONE_TEXT_BYTE_BUDGETS['cloudflare/clef-flash'];+ assert.ok(budget < 2000, 'the budget (bytes, an upper bound on byte-level tokens) stays under the 2K tokens the model reads');await judge.evaluate({ config, state: { evidence: 'The post was published and its page shows it. '.repeat(20) }, questions: one });assert.equal(requests.length, 1, 'ordinary text within the budget is sent');// Each of these stays far below the old 6,000-character cap yet can exceed 2K real tokens.@@ -47,6 +47,7 @@ for (const build of ['chrome', 'firefox']) {hashes: { evidence: ('a1b2c3d4e5f60718293a4b5c6d7e8f90 ').repeat(60) },unbroken: { evidence: 'Zm9vYmFy'.repeat(250) },emoji: { evidence: '\u{1F600}'.repeat(700) },+ punctuation: { evidence: '\\"\\n{}[]<>|~^'.repeat(150) },};for (const [name, state] of Object.entries(dense)) {assert.ok(JSON.stringify(state).length < 6000, `${name} is under the old character cap`);@@ -64,7 +65,6 @@ for (const build of ['chrome', 'firefox']) {assert.equal(requests.length, 2, 'a data URL in text is not sent');await judge.evaluate({ config: { ...config, model: 'another/publisher' }, state: dense.cjk, questions: one });assert.equal(requests.length, 3, 'other models keep the full state');- assert.ok(mod.estimateTextTokens('x'.repeat(100)) >= 50 && mod.estimateTextTokens('æ—¥'.repeat(10)) >= 30);});test(`${build}: singleton Choice requests fail locally before a service call`, async () => {let calls = 0;