Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(providers): address codex round 6 - scrub binary in tool_calls args and metadata
Emre Sokulluc1bb16fa785e
diff --git a/src/chrome/src/trace/webbrain-share-outbox.js b/src/chrome/src/trace/webbrain-share-outbox.jsindex 6322424fc..33e457e26 100644--- a/src/chrome/src/trace/webbrain-share-outbox.js+++ b/src/chrome/src/trace/webbrain-share-outbox.js@@ -104,10 +104,69 @@ function requestMessages(messages, responseContent) {return request;}+// Fields that by name always carry binary payloads (any length counts:+// a field literally named imageBase64 is image bytes even when tiny).+const BINARY_VALUE_KEYS = /^(imageBase64|image_base64|screenshot|screenshotBase64|screenshot_data|imageData|image_data|audioBase64|audio_data|fileData|file_data|imageBytes|base64)$/i;++// Scrub tool-call arguments JSON: raw binary rides in fields like+// solve_captcha's imageBase64 for image_to_text, which the content scrubber+// never sees because it lives in tool_calls[].function.arguments.+function scrubToolCallArguments(args) {+ if (typeof args !== 'string' || !args) return args;+ let parsed;+ try { parsed = JSON.parse(args); } catch { return scrubText(args); }+ if (!parsed || typeof parsed !== 'object') return scrubText(args);+ try {+ const scrubbed = JSON.stringify(scrubValue(parsed));+ return typeof scrubbed === 'string' ? scrubbed : scrubText(args);+ } catch { return scrubText(args); }+}++// Recursively scrub non-content message fields (tool_calls, metadata).+// Known-binary keys are redacted at any length; other strings go through the+// data-URI/base64 text scrub so nested blobs cannot leak via copied fields.+function scrubValue(value) {+ if (typeof value === 'string') return scrubText(value);+ if (Array.isArray(value)) return value.map(scrubValue);+ if (value && typeof value === 'object') {+ const out = {};+ for (const [key, entry] of Object.entries(value)) {+ if (typeof entry === 'string' && BINARY_VALUE_KEYS.test(key)) {+ out[key] = '[omitted]';+ continue;+ }+ if (key === 'arguments' && typeof entry === 'string') {+ out[key] = scrubToolCallArguments(entry);+ continue;+ }+ out[key] = scrubValue(entry);+ }+ return out;+ }+ return value;+}+function scrubMessage(message) {if (!message || typeof message !== 'object') return null;- const copy = { ...message };- delete copy.image_url;+ const copy = { role: message.role };+ if (message.tool_call_id !== undefined) copy.tool_call_id = message.tool_call_id;+ if (message.name !== undefined) copy.name = message.name;+ // Assistant tool-call turns carry the trajectory (and, on multi-step runs,+ // raw binary such as solve_captcha imageBase64 in arguments JSON). Scrub+ // the calls instead of copying them verbatim.+ if (Array.isArray(message.tool_calls)) {+ copy.tool_calls = message.tool_calls.map(call =>+ (!call || typeof call !== 'object') ? call : scrubValue(call));+ }+ // Drop known binary attachments outright; scrub any other metadata rather+ // than copying it verbatim into the research record.+ for (const [key, value] of Object.entries(message)) {+ if (key === 'role' || key === 'content' || key === 'tool_calls'+ || key === 'tool_call_id' || key === 'name') continue;+ if (key === 'image_url' || key === '_attachImage' || key === '_attachDocument') continue;+ const scrubbed = scrubValue(value);+ if (scrubbed !== undefined) copy[key] = scrubbed;+ }if (Array.isArray(message.content)) {const items = [];for (const item of message.content) {@@ -128,13 +187,18 @@ function scrubMessage(message) {items.push({ ...item, text: scrubText(item.text) });continue;}- items.push(item);+ // Unrecognized objects still pass through nested strings (data URIs,+ // bare base64) and known-binary keys before being shared.+ items.push(scrubValue(item));}if (!items.length) return { role: message.role, content: '[binary content omitted]' };copy.content = items;} else if (typeof message.content === 'string') {if (!message.content.length) return null;copy.content = scrubText(message.content);+ } else if (message.content == null && Object.hasOwn(message, 'content')) {+ // Assistant tool-call turns carry content: null; preserve the marker.+ copy.content = message.content;}return copy;}diff --git a/src/firefox/src/trace/webbrain-share-outbox.js b/src/firefox/src/trace/webbrain-share-outbox.jsindex 6322424fc..33e457e26 100644--- a/src/firefox/src/trace/webbrain-share-outbox.js+++ b/src/firefox/src/trace/webbrain-share-outbox.js@@ -104,10 +104,69 @@ function requestMessages(messages, responseContent) {return request;}+// Fields that by name always carry binary payloads (any length counts:+// a field literally named imageBase64 is image bytes even when tiny).+const BINARY_VALUE_KEYS = /^(imageBase64|image_base64|screenshot|screenshotBase64|screenshot_data|imageData|image_data|audioBase64|audio_data|fileData|file_data|imageBytes|base64)$/i;++// Scrub tool-call arguments JSON: raw binary rides in fields like+// solve_captcha's imageBase64 for image_to_text, which the content scrubber+// never sees because it lives in tool_calls[].function.arguments.+function scrubToolCallArguments(args) {+ if (typeof args !== 'string' || !args) return args;+ let parsed;+ try { parsed = JSON.parse(args); } catch { return scrubText(args); }+ if (!parsed || typeof parsed !== 'object') return scrubText(args);+ try {+ const scrubbed = JSON.stringify(scrubValue(parsed));+ return typeof scrubbed === 'string' ? scrubbed : scrubText(args);+ } catch { return scrubText(args); }+}++// Recursively scrub non-content message fields (tool_calls, metadata).+// Known-binary keys are redacted at any length; other strings go through the+// data-URI/base64 text scrub so nested blobs cannot leak via copied fields.+function scrubValue(value) {+ if (typeof value === 'string') return scrubText(value);+ if (Array.isArray(value)) return value.map(scrubValue);+ if (value && typeof value === 'object') {+ const out = {};+ for (const [key, entry] of Object.entries(value)) {+ if (typeof entry === 'string' && BINARY_VALUE_KEYS.test(key)) {+ out[key] = '[omitted]';+ continue;+ }+ if (key === 'arguments' && typeof entry === 'string') {+ out[key] = scrubToolCallArguments(entry);+ continue;+ }+ out[key] = scrubValue(entry);+ }+ return out;+ }+ return value;+}+function scrubMessage(message) {if (!message || typeof message !== 'object') return null;- const copy = { ...message };- delete copy.image_url;+ const copy = { role: message.role };+ if (message.tool_call_id !== undefined) copy.tool_call_id = message.tool_call_id;+ if (message.name !== undefined) copy.name = message.name;+ // Assistant tool-call turns carry the trajectory (and, on multi-step runs,+ // raw binary such as solve_captcha imageBase64 in arguments JSON). Scrub+ // the calls instead of copying them verbatim.+ if (Array.isArray(message.tool_calls)) {+ copy.tool_calls = message.tool_calls.map(call =>+ (!call || typeof call !== 'object') ? call : scrubValue(call));+ }+ // Drop known binary attachments outright; scrub any other metadata rather+ // than copying it verbatim into the research record.+ for (const [key, value] of Object.entries(message)) {+ if (key === 'role' || key === 'content' || key === 'tool_calls'+ || key === 'tool_call_id' || key === 'name') continue;+ if (key === 'image_url' || key === '_attachImage' || key === '_attachDocument') continue;+ const scrubbed = scrubValue(value);+ if (scrubbed !== undefined) copy[key] = scrubbed;+ }if (Array.isArray(message.content)) {const items = [];for (const item of message.content) {@@ -128,13 +187,18 @@ function scrubMessage(message) {items.push({ ...item, text: scrubText(item.text) });continue;}- items.push(item);+ // Unrecognized objects still pass through nested strings (data URIs,+ // bare base64) and known-binary keys before being shared.+ items.push(scrubValue(item));}if (!items.length) return { role: message.role, content: '[binary content omitted]' };copy.content = items;} else if (typeof message.content === 'string') {if (!message.content.length) return null;copy.content = scrubText(message.content);+ } else if (message.content == null && Object.hasOwn(message, 'content')) {+ // Assistant tool-call turns carry content: null; preserve the marker.+ copy.content = message.content;}return copy;}diff --git a/test/run.js b/test/run.jsindex 58f7c61a4..d5a5d9909 100644--- a/test/run.js+++ b/test/run.js@@ -12618,6 +12618,22 @@ test('Share-for-research scrub removes embedded data URIs and keeps repeated-ansmodel: 'some-model', mode: 'act', provider: 'anthropic', provider_name: 'Anthropic Claude',});assert.equal(JSON.stringify(answered).includes('iVBORw0K'), false, `${label}: response image bytes escaped the scrub`);+ // Tool-call arguments can embed raw binary (solve_captcha image_to_text):+ // multi-step trajectories keep the calls, never the bytes.+ const captchaBytes = `iVBORw0KGgoAAAANSUhEUg${'A'.repeat(500)}`;+ const captched = outbox.buildShareGenerationItem({+ runId: `run-share-captcha-${label}`,+ finalContent: 'solved it',+ messages: [+ { role: 'user', content: 'solve this captcha' },+ { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name: 'solve_captcha', arguments: JSON.stringify({ type: 'image_to_text', imageBase64: captchaBytes }) } }] },+ { role: 'tool', tool_call_id: 'c1', content: 'abc123' },+ { role: 'assistant', content: 'solved it' },+ ],+ model: 'some-model', mode: 'act', provider: 'anthropic', provider_name: 'Anthropic Claude',+ });+ assert.equal(JSON.stringify(captched).includes('iVBORw0KGgo'), false, `${label}: tool-call image bytes escaped the scrub`);+ assert.ok(captched.request.some(m => Array.isArray(m.tool_calls)), `${label}: scrubbed tool-call turn dropped from request`);}});@@ -12763,6 +12779,8 @@ test('Share-for-research delivery stays opt-in and mirrored across both builds',assert.match(chromeOutbox, /embedded base64 data omitted/, `${browser}: string content must be scrubbed of data URIs`);assert.match(chromeOutbox, /earlier shared messages omitted/, `${browser}: truncation must preserve the tail`);assert.match(chromeOutbox, /scrubText\(responseContent/, `${browser}: shared responses must get the binary scrub`);+ assert.match(chromeOutbox, /_attachImage/, `${browser}: binary metadata attachments must be dropped`);+ assert.match(chromeOutbox, /scrubToolCallArguments/, `${browser}: tool-call arguments must be scrubbed`);assert.match(settings, /shareQueriesForResearch/, `${browser}: share toggle field missing from settings`);assert.match(settings, /!input\.checked[\s\S]*?confirm\(/, `${browser}: consent confirmation must guard turning the share toggle on`);assert.match(provider, /\/improvement\/generations/, `${browser}: share endpoint missing from the Compass provider transport`);