Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Bound Clef Flash's input in estimated tokens, not characters
Claude46f66d784499
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.mdindex fd9bf4a86..26cb7aa10 100644--- a/docs/providers-and-models.md+++ b/docs/providers-and-models.md@@ -90,10 +90,14 @@ provider until changed. Provider keys are stored separately.`cloudflare/clef-flash` reads only the first ~2K tokens of a long text state(OpenRouter's model page says Workers AI truncates it), so-`SYSTEM_ONE_TEXT_BUDGETS` in `systemone-judge.js` caps the state it is sent at-6,000 characters. A longer state is not sent; the check takes the same fallback-as for an unavailable decision model (failure reason `state_too_long`) instead-of judging evidence it only partly read.+`SYSTEM_ONE_TEXT_TOKEN_BUDGETS` in `systemone-judge.js` caps the state it is sent+at 1,500 *estimated* tokens. The estimate is deliberately high (no tokenizer+ships with the extension): ASCII counts half a token per character, other+characters two, long unbroken letter/digit runs one each. A longer state is not+sent; the check takes the same fallback as for an unavailable decision model+(failure reason `state_too_long`) instead of judging evidence it only partly+read. In practice Clef Flash answers short decisions, and long completion+evidence goes to the main model.Local [Kev](https://github.com/jaredpalmer/kev) defaults to`http://127.0.0.1:8009`, model `kev-latest`, using `/v1/systemone`. A key is optionaldiff --git a/src/chrome/src/agent/systemone-judge.js b/src/chrome/src/agent/systemone-judge.jsindex 34503c37b..a25f1fd5f 100644--- a/src/chrome/src/agent/systemone-judge.js+++ b/src/chrome/src/agent/systemone-judge.js@@ -132,12 +132,26 @@ function validateState(state, multimodal = false) {// confident and wrong. A state past the budget is not sent at all; the caller// takes the same fallback as for any unavailable decision model. Images are// counted separately by the provider and are not part of this text.-export const SYSTEM_ONE_TEXT_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 6000 });+// The budget is in estimated tokens and sits well under the 2K the model reads+// (see estimateTextTokens), because no tokenizer ships with the extension.+export const SYSTEM_ONE_TEXT_TOKEN_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 1500 });++// A deliberately high token estimate, so a state it passes stays under the real+// limit for ordinary text, CJK, and identifier-heavy input alike: ASCII counts+// half a token per character (English and JSON run near a quarter), any other+// character two (CJK and emoji take one to three), and long unbroken runs of+// letters and digits (hashes, base64, ids) one per character.+export function estimateTextTokens(text) {+ let tokens = 0;+ const rest = String(text).replace(/[A-Za-z0-9+/=_-]{24,}/g, run => { tokens += run.length; return ' '; });+ for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 2;+ return Math.ceil(tokens);+}function assertStateFitsModel(model, state) {- const budget = SYSTEM_ONE_TEXT_BUDGETS[model];+ const budget = SYSTEM_ONE_TEXT_TOKEN_BUDGETS[model];if (!budget) return;- if (JSON.stringify(state).length > budget) {+ if (estimateTextTokens(JSON.stringify(state)) > budget) {throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);}}diff --git a/src/firefox/src/agent/systemone-judge.js b/src/firefox/src/agent/systemone-judge.jsindex d71ed1cbe..cbb23013d 100644--- a/src/firefox/src/agent/systemone-judge.js+++ b/src/firefox/src/agent/systemone-judge.js@@ -132,12 +132,26 @@ function validateState(state, multimodal = false) {// confident and wrong. A state past the budget is not sent at all; the caller// takes the same fallback as for any unavailable decision model. Images are// counted separately by the provider and are not part of this text.-export const SYSTEM_ONE_TEXT_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 6000 });+// The budget is in estimated tokens and sits well under the 2K the model reads+// (see estimateTextTokens), because no tokenizer ships with the extension.+export const SYSTEM_ONE_TEXT_TOKEN_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 1500 });++// A deliberately high token estimate, so a state it passes stays under the real+// limit for ordinary text, CJK, and identifier-heavy input alike: ASCII counts+// half a token per character (English and JSON run near a quarter), any other+// character two (CJK and emoji take one to three), and long unbroken runs of+// letters and digits (hashes, base64, ids) one per character.+export function estimateTextTokens(text) {+ let tokens = 0;+ const rest = String(text).replace(/[A-Za-z0-9+/=_-]{24,}/g, run => { tokens += run.length; return ' '; });+ for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 2;+ return Math.ceil(tokens);+}function assertStateFitsModel(model, state) {- const budget = SYSTEM_ONE_TEXT_BUDGETS[model];+ const budget = SYSTEM_ONE_TEXT_TOKEN_BUDGETS[model];if (!budget) return;- if (JSON.stringify(state).length > budget) {+ if (estimateTextTokens(JSON.stringify(state)) > budget) {throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);}}diff --git a/test/systemone.mjs b/test/systemone.mjsindex b5a05f331..75643bf6b 100644--- a/test/systemone.mjs+++ b/test/systemone.mjs@@ -31,21 +31,31 @@ for (const build of ['chrome', 'firefox']) {assert.equal(mod.systemOneFailureReason(Object.assign(new Error(), { code: 'JEV_INVALID_DISTRIBUTION' })), 'invalid_distribution');assert.equal(mod.systemOneFailureReason(Object.assign(new Error(), { status: 429 })), 'rate_limit');});- test(`${build}: a model that clips long text never receives a state past its budget`, async () => {+ test(`${build}: a model that clips long text never receives a state past its token budget`, async () => {const requests = [];- const reply = body => ({ ok: true, json: async () => ({ model: body.model, answers: { p: { type: 'noul', noul: .9 } }, usage: { input_tokens: 10, output_tokens: 0 } }) });- const judge = mod.createSystemOneJudge({ fetchImpl: async (_url, init) => { const body = JSON.parse(init.body); requests.push(body); return reply(body); } });+ const judge = mod.createSystemOneJudge({ fetchImpl: async (_url, init) => { const body = JSON.parse(init.body); requests.push(body);+ return { ok: true, json: async () => ({ model: body.model, answers: { p: { type: 'noul', noul: .9 } }, usage: { input_tokens: 10, output_tokens: 0 } }) }; } });const one = { p: questions.p };const config = { provider: 'openrouter', model: 'cloudflare/clef-flash', apiKey: 'test', url: 'https://example.test/v1/decisions' };- const short = { evidence: 'x'.repeat(1000) };- await judge.evaluate({ config, state: short, questions: one });- assert.equal(requests.length, 1, 'a state within the budget is sent');- const long = { evidence: 'x'.repeat(mod.SYSTEM_ONE_TEXT_BUDGETS['cloudflare/clef-flash'] + 1) };- await assert.rejects(judge.evaluate({ config, state: long, questions: one }),- error => error.code === 'JEV_STATE_EXCEEDS_MODEL_INPUT' && mod.systemOneFailureReason(error) === 'state_too_long');+ const budget = mod.SYSTEM_ONE_TEXT_TOKEN_BUDGETS['cloudflare/clef-flash'];+ assert.ok(budget < 2000, 'the budget stays under the 2K tokens the model reads');+ await judge.evaluate({ config, state: { evidence: 'The post was published and its page shows it. '.repeat(20) }, questions: one });+ assert.equal(requests.length, 1, 'ordinary text within the budget is sent');+ // Each of these stays far below the old 6,000-character cap yet can exceed 2K real tokens.+ const dense = {+ cjk: { evidence: 'ζη¨Ώγε ¬ιγγγΎγγγ'.repeat(100) },+ hashes: { evidence: ('a1b2c3d4e5f60718293a4b5c6d7e8f90 ').repeat(60) },+ unbroken: { evidence: 'Zm9vYmFy'.repeat(250) },+ };+ for (const [name, state] of Object.entries(dense)) {+ assert.ok(JSON.stringify(state).length < 6000, `${name} is under the old character cap`);+ await assert.rejects(judge.evaluate({ config, state, questions: one }),+ error => error.code === 'JEV_STATE_EXCEEDS_MODEL_INPUT' && mod.systemOneFailureReason(error) === 'state_too_long', name);+ }assert.equal(requests.length, 1, 'a state past the budget is not sent');- await judge.evaluate({ config: { ...config, model: 'another/publisher' }, state: long, questions: one });+ await judge.evaluate({ config: { ...config, model: 'another/publisher' }, state: dense.cjk, questions: one });assert.equal(requests.length, 2, 'other models keep the full state');+ assert.ok(mod.estimateTextTokens('x'.repeat(100)) >= 50 && mod.estimateTextTokens('ζ₯'.repeat(10)) >= 20);});test(`${build}: singleton Choice requests fail locally before a service call`, async () => {let calls = 0;