Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Publication prefetch: a cost hold counts every field a provider forwards

Barack Sokullu7aee2ae4ca23
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 6906d9298..fae68fff6 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -8727,38 +8727,39 @@ export class Agent extends LoopDetector {
// A bound on a request's prompt tokens that holds for any tokenizer: no
// token covers less than one byte of UTF-8 text (a byte-level fallback
- // spends one token per byte, as CJK or emoji text can), plus framing per
- // message and a generous allowance per image. _estimateContextChars stays
- // the average estimate that context management uses; admission against
- // the cost allowance uses this bound (_holdCostAllowance).
+ // spends one token per byte, as CJK or emoji text can). Every field a
+ // message carries counts, not only its content: providers forward its tool
+ // calls, tool_call_id, name, reasoning and replay state too, and the
+ // message's JSON spells each of them at least once. An image counts a
+ // generous allowance instead of its data, and each message some framing.
+ // _estimateContextChars stays the average estimate that context management
+ // uses; admission against the cost allowance uses this bound
+ // (_holdCostAllowance).
_promptTokenUpperBound(messages, extra = '') {
const list = Array.isArray(messages) ? messages : [];
try {
const encoder = new TextEncoder();
const bytes = value => (typeof value === 'string' ? encoder.encode(value).length : 0);
- const json = value => bytes(JSON.stringify(value ?? ''));
const IMG_DATA_URL = /data:image\/[a-zA-Z0-9+.-]+;base64,[A-Za-z0-9+/=\\]+/g;
const imageTokens = 2 * Math.ceil(Agent.IMAGE_CHAR_COST / 4);
let tokens = bytes(extra);
for (const msg of list) {
- tokens += 16;
- if (Array.isArray(msg?.content)) {
- for (const block of msg.content) {
- if (block && (block.type === 'image_url' || block.type === 'image')) tokens += imageTokens;
- else if (typeof block?.text === 'string') tokens += bytes(block.text);
- else tokens += json(block);
- }
- } else if (typeof msg?.content === 'string') {
- const images = msg.content.match(IMG_DATA_URL) || [];
- tokens += images.length * imageTokens + bytes(images.length ? msg.content.replace(IMG_DATA_URL, '') : msg.content);
- } else {
- tokens += json(msg?.content);
- }
- if (msg?.tool_calls) tokens += json(msg.tool_calls);
- if (msg?.response_items) tokens += json(msg.response_items);
- if (typeof msg?.reasoning_content === 'string') tokens += bytes(msg.reasoning_content);
- const replay = msg?._reasoning_replay?.providerState?.content;
- if (Array.isArray(replay)) tokens += json(replay);
+ let images = 0;
+ const serialized = JSON.stringify(msg ?? null, (_key, value) => {
+ if (value && typeof value === 'object' && (value.type === 'image_url' || value.type === 'image')) {
+ images++;
+ return null;
+ }
+ if (typeof value === 'string' && value.includes('data:image/')) {
+ const found = value.match(IMG_DATA_URL);
+ if (found) {
+ images += found.length;
+ return value.replace(IMG_DATA_URL, '');
+ }
+ }
+ return value;
+ });
+ tokens += 16 + bytes(serialized || '') + images * imageTokens;
}
return Math.max(1, tokens);
} catch {
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 26fd1a7f5..72dcb03c7 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -8442,38 +8442,39 @@ export class Agent extends LoopDetector {
// A bound on a request's prompt tokens that holds for any tokenizer: no
// token covers less than one byte of UTF-8 text (a byte-level fallback
- // spends one token per byte, as CJK or emoji text can), plus framing per
- // message and a generous allowance per image. _estimateContextChars stays
- // the average estimate that context management uses; admission against
- // the cost allowance uses this bound (_holdCostAllowance).
+ // spends one token per byte, as CJK or emoji text can). Every field a
+ // message carries counts, not only its content: providers forward its tool
+ // calls, tool_call_id, name, reasoning and replay state too, and the
+ // message's JSON spells each of them at least once. An image counts a
+ // generous allowance instead of its data, and each message some framing.
+ // _estimateContextChars stays the average estimate that context management
+ // uses; admission against the cost allowance uses this bound
+ // (_holdCostAllowance).
_promptTokenUpperBound(messages, extra = '') {
const list = Array.isArray(messages) ? messages : [];
try {
const encoder = new TextEncoder();
const bytes = value => (typeof value === 'string' ? encoder.encode(value).length : 0);
- const json = value => bytes(JSON.stringify(value ?? ''));
const IMG_DATA_URL = /data:image\/[a-zA-Z0-9+.-]+;base64,[A-Za-z0-9+/=\\]+/g;
const imageTokens = 2 * Math.ceil(Agent.IMAGE_CHAR_COST / 4);
let tokens = bytes(extra);
for (const msg of list) {
- tokens += 16;
- if (Array.isArray(msg?.content)) {
- for (const block of msg.content) {
- if (block && (block.type === 'image_url' || block.type === 'image')) tokens += imageTokens;
- else if (typeof block?.text === 'string') tokens += bytes(block.text);
- else tokens += json(block);
+ let images = 0;
+ const serialized = JSON.stringify(msg ?? null, (_key, value) => {
+ if (value && typeof value === 'object' && (value.type === 'image_url' || value.type === 'image')) {
+ images++;
+ return null;
}
- } else if (typeof msg?.content === 'string') {
- const images = msg.content.match(IMG_DATA_URL) || [];
- tokens += images.length * imageTokens + bytes(images.length ? msg.content.replace(IMG_DATA_URL, '') : msg.content);
- } else {
- tokens += json(msg?.content);
- }
- if (msg?.tool_calls) tokens += json(msg.tool_calls);
- if (msg?.response_items) tokens += json(msg.response_items);
- if (typeof msg?.reasoning_content === 'string') tokens += bytes(msg.reasoning_content);
- const replay = msg?._reasoning_replay?.providerState?.content;
- if (Array.isArray(replay)) tokens += json(replay);
+ if (typeof value === 'string' && value.includes('data:image/')) {
+ const found = value.match(IMG_DATA_URL);
+ if (found) {
+ images += found.length;
+ return value.replace(IMG_DATA_URL, '');
+ }
+ }
+ return value;
+ });
+ tokens += 16 + bytes(serialized || '') + images * imageTokens;
}
return Math.max(1, tokens);
} catch {
diff --git a/test/publication-prefetch.mjs b/test/publication-prefetch.mjs
index 7d88841db..b3d85c5ff 100644
--- a/test/publication-prefetch.mjs
+++ b/test/publication-prefetch.mjs
@@ -437,6 +437,23 @@ for (const browser of ['chrome', 'firefox']) {
`${index}: ${usages[index].prompt_tokens} tokens held for ${bytes(text)} bytes`));
assert.equal(usages[0].completion_tokens, 10);
assert.equal(agent._costHolds?.size || 0, 0);
+ // Every field a provider forwards counts, not only the content: a long
+ // model-supplied tool_call_id, a name, the tool calls themselves.
+ const id = 'call_' + '市'.repeat(2000);
+ const metadata = [
+ { role: 'assistant', content: null, tool_calls: [{ id, type: 'function', function: { name: 'click_ax', arguments: '{}' } }] },
+ { role: 'tool', tool_call_id: id, name: 'click_ax', content: 'ok' },
+ ];
+ assert.ok(agent._promptTokenUpperBound(metadata) >= 2 * bytes(id), 'the tool_call_id is held where it is sent');
+ // An image counts its allowance, never its data; the text beside it counts.
+ const data = 'data:image/png;base64,' + 'A'.repeat(200000);
+ const imageBlocks = agent._promptTokenUpperBound([{ role: 'user', content: [
+ { type: 'text', text: 'look' }, { type: 'image_url', image_url: { url: data } }] }]);
+ const imageInline = agent._promptTokenUpperBound([{ role: 'user', content: `look ${data}` }]);
+ for (const held of [imageBlocks, imageInline]) {
+ assert.ok(held >= 2 * Math.ceil(Agent.IMAGE_CHAR_COST / 4), `${held}: the image allowance`);
+ assert.ok(held < 200000, `${held}: the image data itself is not counted`);
+ }
});
test(`${browser}: a consequential click a card or the permission gate stops still records predispatch_timing`, async () => {