Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Publication prefetch: a cost hold counts every field a provider forwards
Barack Sokullu7aee2ae4ca23
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 6906d9298..fae68fff6 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -8727,38 +8727,39 @@ export class Agent extends LoopDetector {// A bound on a request's prompt tokens that holds for any tokenizer: no// token covers less than one byte of UTF-8 text (a byte-level fallback- // spends one token per byte, as CJK or emoji text can), plus framing per- // message and a generous allowance per image. _estimateContextChars stays- // the average estimate that context management uses; admission against- // the cost allowance uses this bound (_holdCostAllowance).+ // spends one token per byte, as CJK or emoji text can). Every field a+ // message carries counts, not only its content: providers forward its tool+ // calls, tool_call_id, name, reasoning and replay state too, and the+ // message's JSON spells each of them at least once. An image counts a+ // generous allowance instead of its data, and each message some framing.+ // _estimateContextChars stays the average estimate that context management+ // uses; admission against the cost allowance uses this bound+ // (_holdCostAllowance)._promptTokenUpperBound(messages, extra = '') {const list = Array.isArray(messages) ? messages : [];try {const encoder = new TextEncoder();const bytes = value => (typeof value === 'string' ? encoder.encode(value).length : 0);- const json = value => bytes(JSON.stringify(value ?? ''));const IMG_DATA_URL = /data:image\/[a-zA-Z0-9+.-]+;base64,[A-Za-z0-9+/=\\]+/g;const imageTokens = 2 * Math.ceil(Agent.IMAGE_CHAR_COST / 4);let tokens = bytes(extra);for (const msg of list) {- tokens += 16;- if (Array.isArray(msg?.content)) {- for (const block of msg.content) {- if (block && (block.type === 'image_url' || block.type === 'image')) tokens += imageTokens;- else if (typeof block?.text === 'string') tokens += bytes(block.text);- else tokens += json(block);- }- } else if (typeof msg?.content === 'string') {- const images = msg.content.match(IMG_DATA_URL) || [];- tokens += images.length * imageTokens + bytes(images.length ? msg.content.replace(IMG_DATA_URL, '') : msg.content);- } else {- tokens += json(msg?.content);- }- if (msg?.tool_calls) tokens += json(msg.tool_calls);- if (msg?.response_items) tokens += json(msg.response_items);- if (typeof msg?.reasoning_content === 'string') tokens += bytes(msg.reasoning_content);- const replay = msg?._reasoning_replay?.providerState?.content;- if (Array.isArray(replay)) tokens += json(replay);+ let images = 0;+ const serialized = JSON.stringify(msg ?? null, (_key, value) => {+ if (value && typeof value === 'object' && (value.type === 'image_url' || value.type === 'image')) {+ images++;+ return null;+ }+ if (typeof value === 'string' && value.includes('data:image/')) {+ const found = value.match(IMG_DATA_URL);+ if (found) {+ images += found.length;+ return value.replace(IMG_DATA_URL, '');+ }+ }+ return value;+ });+ tokens += 16 + bytes(serialized || '') + images * imageTokens;}return Math.max(1, tokens);} catch {diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 26fd1a7f5..72dcb03c7 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -8442,38 +8442,39 @@ export class Agent extends LoopDetector {// A bound on a request's prompt tokens that holds for any tokenizer: no// token covers less than one byte of UTF-8 text (a byte-level fallback- // spends one token per byte, as CJK or emoji text can), plus framing per- // message and a generous allowance per image. _estimateContextChars stays- // the average estimate that context management uses; admission against- // the cost allowance uses this bound (_holdCostAllowance).+ // spends one token per byte, as CJK or emoji text can). Every field a+ // message carries counts, not only its content: providers forward its tool+ // calls, tool_call_id, name, reasoning and replay state too, and the+ // message's JSON spells each of them at least once. An image counts a+ // generous allowance instead of its data, and each message some framing.+ // _estimateContextChars stays the average estimate that context management+ // uses; admission against the cost allowance uses this bound+ // (_holdCostAllowance)._promptTokenUpperBound(messages, extra = '') {const list = Array.isArray(messages) ? messages : [];try {const encoder = new TextEncoder();const bytes = value => (typeof value === 'string' ? encoder.encode(value).length : 0);- const json = value => bytes(JSON.stringify(value ?? ''));const IMG_DATA_URL = /data:image\/[a-zA-Z0-9+.-]+;base64,[A-Za-z0-9+/=\\]+/g;const imageTokens = 2 * Math.ceil(Agent.IMAGE_CHAR_COST / 4);let tokens = bytes(extra);for (const msg of list) {- tokens += 16;- if (Array.isArray(msg?.content)) {- for (const block of msg.content) {- if (block && (block.type === 'image_url' || block.type === 'image')) tokens += imageTokens;- else if (typeof block?.text === 'string') tokens += bytes(block.text);- else tokens += json(block);+ let images = 0;+ const serialized = JSON.stringify(msg ?? null, (_key, value) => {+ if (value && typeof value === 'object' && (value.type === 'image_url' || value.type === 'image')) {+ images++;+ return null;}- } else if (typeof msg?.content === 'string') {- const images = msg.content.match(IMG_DATA_URL) || [];- tokens += images.length * imageTokens + bytes(images.length ? msg.content.replace(IMG_DATA_URL, '') : msg.content);- } else {- tokens += json(msg?.content);- }- if (msg?.tool_calls) tokens += json(msg.tool_calls);- if (msg?.response_items) tokens += json(msg.response_items);- if (typeof msg?.reasoning_content === 'string') tokens += bytes(msg.reasoning_content);- const replay = msg?._reasoning_replay?.providerState?.content;- if (Array.isArray(replay)) tokens += json(replay);+ if (typeof value === 'string' && value.includes('data:image/')) {+ const found = value.match(IMG_DATA_URL);+ if (found) {+ images += found.length;+ return value.replace(IMG_DATA_URL, '');+ }+ }+ return value;+ });+ tokens += 16 + bytes(serialized || '') + images * imageTokens;}return Math.max(1, tokens);} catch {diff --git a/test/publication-prefetch.mjs b/test/publication-prefetch.mjsindex 7d88841db..b3d85c5ff 100644--- a/test/publication-prefetch.mjs+++ b/test/publication-prefetch.mjs@@ -437,6 +437,23 @@ for (const browser of ['chrome', 'firefox']) {`${index}: ${usages[index].prompt_tokens} tokens held for ${bytes(text)} bytes`));assert.equal(usages[0].completion_tokens, 10);assert.equal(agent._costHolds?.size || 0, 0);+ // Every field a provider forwards counts, not only the content: a long+ // model-supplied tool_call_id, a name, the tool calls themselves.+ const id = 'call_' + '市'.repeat(2000);+ const metadata = [+ { role: 'assistant', content: null, tool_calls: [{ id, type: 'function', function: { name: 'click_ax', arguments: '{}' } }] },+ { role: 'tool', tool_call_id: id, name: 'click_ax', content: 'ok' },+ ];+ assert.ok(agent._promptTokenUpperBound(metadata) >= 2 * bytes(id), 'the tool_call_id is held where it is sent');+ // An image counts its allowance, never its data; the text beside it counts.+ const data = 'data:image/png;base64,' + 'A'.repeat(200000);+ const imageBlocks = agent._promptTokenUpperBound([{ role: 'user', content: [+ { type: 'text', text: 'look' }, { type: 'image_url', image_url: { url: data } }] }]);+ const imageInline = agent._promptTokenUpperBound([{ role: 'user', content: `look ${data}` }]);+ for (const held of [imageBlocks, imageInline]) {+ assert.ok(held >= 2 * Math.ceil(Agent.IMAGE_CHAR_COST / 4), `${held}: the image allowance`);+ assert.ok(held < 200000, `${held}: the image data itself is not counted`);+ }});test(`${browser}: a consequential click a card or the permission gate stops still records predispatch_timing`, async () => {