Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Charge the worst-case token count for non-ASCII characters

Claudef3476a527858
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.md
index 26cb7aa10..fa3395854 100644
--- a/docs/providers-and-models.md
+++ b/docs/providers-and-models.md
@@ -93,7 +93,7 @@ provider until changed. Provider keys are stored separately.
`SYSTEM_ONE_TEXT_TOKEN_BUDGETS` in `systemone-judge.js` caps the state it is sent
at 1,500 *estimated* tokens. The estimate is deliberately high (no tokenizer
ships with the extension): ASCII counts half a token per character, other
-characters two, long unbroken letter/digit runs one each. A longer state is not
+characters three (the worst case for CJK and emoji), long unbroken letter/digit runs one each. A longer state is not
sent; the check takes the same fallback as for an unavailable decision model
(failure reason `state_too_long`) instead of judging evidence it only partly
read. In practice Clef Flash answers short decisions, and long completion
diff --git a/src/chrome/src/agent/systemone-judge.js b/src/chrome/src/agent/systemone-judge.js
index 848cd010d..9f8e3409d 100644
--- a/src/chrome/src/agent/systemone-judge.js
+++ b/src/chrome/src/agent/systemone-judge.js
@@ -139,12 +139,13 @@ export const SYSTEM_ONE_TEXT_TOKEN_BUDGETS = Object.freeze({ 'cloudflare/clef-fl
// A deliberately high token estimate, so a state it passes stays under the real
// limit for ordinary text, CJK, and identifier-heavy input alike: ASCII counts
// half a token per character (English and JSON run near a quarter), any other
-// character two (CJK and emoji take one to three), and long unbroken runs of
+// character three (CJK and emoji take one to three, so the worst case is
+// charged), and long unbroken runs of
// letters and digits (hashes, base64, ids) one per character.
export function estimateTextTokens(text) {
let tokens = 0;
const rest = String(text).replace(/[A-Za-z0-9+/=_-]{24,}/g, run => { tokens += run.length; return ' '; });
- for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 2;
+ for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 3;
return Math.ceil(tokens);
}
diff --git a/src/firefox/src/agent/systemone-judge.js b/src/firefox/src/agent/systemone-judge.js
index 88912b54d..aa8238fb1 100644
--- a/src/firefox/src/agent/systemone-judge.js
+++ b/src/firefox/src/agent/systemone-judge.js
@@ -139,12 +139,13 @@ export const SYSTEM_ONE_TEXT_TOKEN_BUDGETS = Object.freeze({ 'cloudflare/clef-fl
// A deliberately high token estimate, so a state it passes stays under the real
// limit for ordinary text, CJK, and identifier-heavy input alike: ASCII counts
// half a token per character (English and JSON run near a quarter), any other
-// character two (CJK and emoji take one to three), and long unbroken runs of
+// character three (CJK and emoji take one to three, so the worst case is
+// charged), and long unbroken runs of
// letters and digits (hashes, base64, ids) one per character.
export function estimateTextTokens(text) {
let tokens = 0;
const rest = String(text).replace(/[A-Za-z0-9+/=_-]{24,}/g, run => { tokens += run.length; return ' '; });
- for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 2;
+ for (const character of rest) tokens += character.charCodeAt(0) < 128 ? .5 : 3;
return Math.ceil(tokens);
}
diff --git a/test/systemone.mjs b/test/systemone.mjs
index 319c8666b..7747b5e4e 100644
--- a/test/systemone.mjs
+++ b/test/systemone.mjs
@@ -46,6 +46,7 @@ for (const build of ['chrome', 'firefox']) {
cjk: { evidence: 'ζŠ•η¨ΏγŒε…¬ι–‹γ•γ‚ŒγΎγ—γŸγ€‚'.repeat(100) },
hashes: { evidence: ('a1b2c3d4e5f60718293a4b5c6d7e8f90 ').repeat(60) },
unbroken: { evidence: 'Zm9vYmFy'.repeat(250) },
+ emoji: { evidence: '\u{1F600}'.repeat(700) },
};
for (const [name, state] of Object.entries(dense)) {
assert.ok(JSON.stringify(state).length < 6000, `${name} is under the old character cap`);
@@ -63,7 +64,7 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(requests.length, 2, 'a data URL in text is not sent');
await judge.evaluate({ config: { ...config, model: 'another/publisher' }, state: dense.cjk, questions: one });
assert.equal(requests.length, 3, 'other models keep the full state');
- assert.ok(mod.estimateTextTokens('x'.repeat(100)) >= 50 && mod.estimateTextTokens('ζ—₯'.repeat(10)) >= 20);
+ assert.ok(mod.estimateTextTokens('x'.repeat(100)) >= 50 && mod.estimateTextTokens('ζ—₯'.repeat(10)) >= 30);
});
test(`${build}: singleton Choice requests fail locally before a service call`, async () => {
let calls = 0;