Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Charge the worst-case token count for non-ASCII characters
Claudef3476a527858
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.mdindex 26cb7aa10..fa3395854 100644--- a/docs/providers-and-models.md+++ b/docs/providers-and-models.md@@ -93,7 +93,7 @@ provider until changed. Provider keys are stored separately.`SYSTEM_ONE_TEXT_TOKEN_BUDGETS` in `systemone-judge.js` caps the state it is sentat 1,500 *estimated* tokens. The estimate is deliberately high (no tokenizerships with the extension): ASCII counts half a token per character, other-characters two, long unbroken letter/digit runs one each. A longer state is not+characters three (the worst case for CJK and emoji), long unbroken letter/digit runs one each. A longer state is notsent; the check takes the same fallback as for an unavailable decision model(failure reason `state_too_long`) instead of judging evidence it only partlyread. In practice Clef Flash answers short decisions, and long completiondiff --git a/src/chrome/src/agent/systemone-judge.js b/src/chrome/src/agent/systemone-judge.jsindex 848cd010d..9f8e3409d 100644--- a/src/chrome/src/agent/systemone-judge.js+++ b/src/chrome/src/agent/systemone-judge.js@@ -139,12 +139,13 @@ export const SYSTEM_ONE_TEXT_TOKEN_BUDGETS = Object.freeze({ 'cloudflare/clef-fl// A deliberately high token estimate, so a state it passes stays under the real// limit for ordinary text, CJK, and identifier-heavy input alike: ASCII counts// half a token per character (English and JSON run near a quarter), any other-// character two (CJK and emoji take one to three), and long unbroken runs of+// character three (CJK and emoji take one to three, so the worst case is+// charged), and long unbroken runs of// letters and digits (hashes, base64, ids) one per character.export function estimateTextTokens(text) {let tokens = 0;const rest = String(text).replace(/[A-Za-z0-9+/=_-]{24,}/g, run => { tokens += run.length; return ' '; });- for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 2;+ for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 3;return Math.ceil(tokens);}diff --git a/src/firefox/src/agent/systemone-judge.js b/src/firefox/src/agent/systemone-judge.jsindex 88912b54d..aa8238fb1 100644--- a/src/firefox/src/agent/systemone-judge.js+++ b/src/firefox/src/agent/systemone-judge.js@@ -139,12 +139,13 @@ export const SYSTEM_ONE_TEXT_TOKEN_BUDGETS = Object.freeze({ 'cloudflare/clef-fl// A deliberately high token estimate, so a state it passes stays under the real// limit for ordinary text, CJK, and identifier-heavy input alike: ASCII counts// half a token per character (English and JSON run near a quarter), any other-// character two (CJK and emoji take one to three), and long unbroken runs of+// character three (CJK and emoji take one to three, so the worst case is+// charged), and long unbroken runs of// letters and digits (hashes, base64, ids) one per character.export function estimateTextTokens(text) {let tokens = 0;const rest = String(text).replace(/[A-Za-z0-9+/=_-]{24,}/g, run => { tokens += run.length; return ' '; });- for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 2;+ for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 3;return Math.ceil(tokens);}diff --git a/test/systemone.mjs b/test/systemone.mjsindex 319c8666b..7747b5e4e 100644--- a/test/systemone.mjs+++ b/test/systemone.mjs@@ -46,6 +46,7 @@ for (const build of ['chrome', 'firefox']) {cjk: { evidence: 'ζη¨Ώγε ¬ιγγγΎγγγ'.repeat(100) },hashes: { evidence: ('a1b2c3d4e5f60718293a4b5c6d7e8f90 ').repeat(60) },unbroken: { evidence: 'Zm9vYmFy'.repeat(250) },+ emoji: { evidence: '\u{1F600}'.repeat(700) },};for (const [name, state] of Object.entries(dense)) {assert.ok(JSON.stringify(state).length < 6000, `${name} is under the old character cap`);@@ -63,7 +64,7 @@ for (const build of ['chrome', 'firefox']) {assert.equal(requests.length, 2, 'a data URL in text is not sent');await judge.evaluate({ config: { ...config, model: 'another/publisher' }, state: dense.cjk, questions: one });assert.equal(requests.length, 3, 'other models keep the full state');- assert.ok(mod.estimateTextTokens('x'.repeat(100)) >= 50 && mod.estimateTextTokens('ζ₯'.repeat(10)) >= 20);+ assert.ok(mod.estimateTextTokens('x'.repeat(100)) >= 50 && mod.estimateTextTokens('ζ₯'.repeat(10)) >= 30);});test(`${build}: singleton Choice requests fail locally before a service call`, async () => {let calls = 0;