Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(providers): harden research share capture
Emre Sokullu1b149b275658
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex aa10ce4c4..c0ef71ce7 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -18747,6 +18747,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// through this path.const shareTransport = this.providerManager?.getProvider?.('webbrain_cloud');if (shareTransport+ && status === 'done'&& provider?.config?.shareQueriesForResearch === true&& String(provider?.config?.providerName || '').toLowerCase() !== 'webbrain-cloud') {try {diff --git a/src/chrome/src/trace/webbrain-share-outbox.js b/src/chrome/src/trace/webbrain-share-outbox.jsindex 791c0a786..48f8046e1 100644--- a/src/chrome/src/trace/webbrain-share-outbox.js+++ b/src/chrome/src/trace/webbrain-share-outbox.js@@ -12,6 +12,7 @@ const MAX_MESSAGE_CHARS = 10_000;const MAX_REQUEST_BUDGET = 150_000;const MAX_RESPONSE_CHARS = 40_000;let storageQueue = Promise.resolve();+let flushQueue = Promise.resolve();let fallbackRunCounter = 0;function bounded(value, limit) {@@ -26,6 +27,27 @@ function clampText(value, limit = MAX_MESSAGE_CHARS) {return bounded(value, limit);}+function containsBinaryBlock(value) {+ if (!value || typeof value !== 'object') return false;+ if (value.type === 'image_url' || value.type === 'image' || value.type === 'document') return true;+ if (value.source?.type === 'base64') return true;+ if (Array.isArray(value)) return value.some(containsBinaryBlock);+ return Object.values(value).some(containsBinaryBlock);+}++function requestMessages(messages, responseContent) {+ if (!Array.isArray(messages)) return messages;+ const request = [...messages];+ for (let index = request.length - 1; index >= 0; index--) {+ const message = request[index];+ if (message?.role === 'assistant' && message.content === responseContent) {+ request.splice(index, 1);+ break;+ }+ }+ return request;+}+function scrubMessage(message) {if (!message || typeof message !== 'object') return null;const copy = { ...message };@@ -39,14 +61,14 @@ function scrubMessage(message) {}// Drop images entirely so raw screenshot/data-URI bytes never leave the// browser; text blocks are clamped individually.- if (item.type === 'image_url' || item.type === 'image') continue;+ if (containsBinaryBlock(item)) continue;if (item.type === 'text' && typeof item.text === 'string') {items.push({ ...item, text: clampText(item.text) });continue;}items.push(item);}- if (!items.length) return { role: message.role, content: '[image content omitted]' };+ if (!items.length) return { role: message.role, content: '[binary content omitted]' };copy.content = items;} else if (typeof message.content === 'string') {if (!message.content.length) return null;@@ -84,10 +106,10 @@ export function buildShareGenerationItem({provider,provider_name,}) {- const request = scrubMessages(messages);- if (!request?.length) return null;const responseContent = String(finalContent ?? '');if (!responseContent.trim()) return null;+ const request = scrubMessages(requestMessages(messages, responseContent));+ if (!request?.length) return null;const generatedId = globalThis.crypto?.randomUUID?.()|| `${Date.now().toString(36)}_${(++fallbackRunCounter).toString(36)}`;return {@@ -136,7 +158,7 @@ export async function enqueueShareGeneration(item) {return true;}-export async function flushShareOutbox(transportProvider) {+async function flushShareOutboxNow(transportProvider) {if (typeof transportProvider?.sendShareGeneration !== 'function') return 0;await storageQueue.catch(() => {});let snapshot;@@ -165,4 +187,10 @@ export async function flushShareOutbox(transportProvider) {return removeIds.size;}-export const SHARE_OUTBOX_STORAGE_KEY = STORAGE_KEY;\ No newline at end of file+export function flushShareOutbox(transportProvider) {+ const next = flushQueue.catch(() => {}).then(() => flushShareOutboxNow(transportProvider));+ flushQueue = next;+ return next;+}++export const SHARE_OUTBOX_STORAGE_KEY = STORAGE_KEY;diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 16a5bdeb1..d108333bd 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -16541,6 +16541,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d// through this path.const shareTransport = this.providerManager?.getProvider?.('webbrain_cloud');if (shareTransport+ && status === 'done'&& provider?.config?.shareQueriesForResearch === true&& String(provider?.config?.providerName || '').toLowerCase() !== 'webbrain-cloud') {try {diff --git a/src/firefox/src/trace/webbrain-share-outbox.js b/src/firefox/src/trace/webbrain-share-outbox.jsindex 791c0a786..48f8046e1 100644--- a/src/firefox/src/trace/webbrain-share-outbox.js+++ b/src/firefox/src/trace/webbrain-share-outbox.js@@ -12,6 +12,7 @@ const MAX_MESSAGE_CHARS = 10_000;const MAX_REQUEST_BUDGET = 150_000;const MAX_RESPONSE_CHARS = 40_000;let storageQueue = Promise.resolve();+let flushQueue = Promise.resolve();let fallbackRunCounter = 0;function bounded(value, limit) {@@ -26,6 +27,27 @@ function clampText(value, limit = MAX_MESSAGE_CHARS) {return bounded(value, limit);}+function containsBinaryBlock(value) {+ if (!value || typeof value !== 'object') return false;+ if (value.type === 'image_url' || value.type === 'image' || value.type === 'document') return true;+ if (value.source?.type === 'base64') return true;+ if (Array.isArray(value)) return value.some(containsBinaryBlock);+ return Object.values(value).some(containsBinaryBlock);+}++function requestMessages(messages, responseContent) {+ if (!Array.isArray(messages)) return messages;+ const request = [...messages];+ for (let index = request.length - 1; index >= 0; index--) {+ const message = request[index];+ if (message?.role === 'assistant' && message.content === responseContent) {+ request.splice(index, 1);+ break;+ }+ }+ return request;+}+function scrubMessage(message) {if (!message || typeof message !== 'object') return null;const copy = { ...message };@@ -39,14 +61,14 @@ function scrubMessage(message) {}// Drop images entirely so raw screenshot/data-URI bytes never leave the// browser; text blocks are clamped individually.- if (item.type === 'image_url' || item.type === 'image') continue;+ if (containsBinaryBlock(item)) continue;if (item.type === 'text' && typeof item.text === 'string') {items.push({ ...item, text: clampText(item.text) });continue;}items.push(item);}- if (!items.length) return { role: message.role, content: '[image content omitted]' };+ if (!items.length) return { role: message.role, content: '[binary content omitted]' };copy.content = items;} else if (typeof message.content === 'string') {if (!message.content.length) return null;@@ -84,10 +106,10 @@ export function buildShareGenerationItem({provider,provider_name,}) {- const request = scrubMessages(messages);- if (!request?.length) return null;const responseContent = String(finalContent ?? '');if (!responseContent.trim()) return null;+ const request = scrubMessages(requestMessages(messages, responseContent));+ if (!request?.length) return null;const generatedId = globalThis.crypto?.randomUUID?.()|| `${Date.now().toString(36)}_${(++fallbackRunCounter).toString(36)}`;return {@@ -136,7 +158,7 @@ export async function enqueueShareGeneration(item) {return true;}-export async function flushShareOutbox(transportProvider) {+async function flushShareOutboxNow(transportProvider) {if (typeof transportProvider?.sendShareGeneration !== 'function') return 0;await storageQueue.catch(() => {});let snapshot;@@ -165,4 +187,10 @@ export async function flushShareOutbox(transportProvider) {return removeIds.size;}-export const SHARE_OUTBOX_STORAGE_KEY = STORAGE_KEY;\ No newline at end of file+export function flushShareOutbox(transportProvider) {+ const next = flushQueue.catch(() => {}).then(() => flushShareOutboxNow(transportProvider));+ flushQueue = next;+ return next;+}++export const SHARE_OUTBOX_STORAGE_KEY = STORAGE_KEY;diff --git a/test/run.js b/test/run.jsindex ff773a82f..4755488d4 100644--- a/test/run.js+++ b/test/run.js@@ -12523,6 +12523,32 @@ test('Share-for-research item drops empty runs and caps the whole request', () =assert.deepEqual(item.request.at(-1), { role: 'system', content: '[remaining shared message omitted]' });});+test('Share-for-research item excludes terminal answers and binary document blocks', () => {+ for (const [label, outbox] of [['chrome', SHARE_OUTBOX_CH], ['firefox', SHARE_OUTBOX_FX]]) {+ const item = outbox.buildShareGenerationItem({+ runId: `run-share-binary-${label}`,+ finalContent: 'Final answer',+ messages: [+ {+ role: 'user',+ content: [+ { type: 'text', text: 'Read this attachment' },+ { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: 'SECRET_PDF_BYTES' } },+ { type: 'other', payload: { source: { type: 'base64', data: 'UNKNOWN_BINARY_BYTES' } } },+ ],+ },+ { role: 'assistant', content: 'Final answer' },+ ],+ model: 'some-model', mode: 'act', provider: 'anthropic', provider_name: 'Anthropic Claude',+ });+ assert.equal(item.request.length, 1, `${label}: terminal answer stayed in the shared request`);+ assert.deepEqual(item.request[0].content, [{ type: 'text', text: 'Read this attachment' }]);+ const serialized = JSON.stringify(item);+ assert.equal(serialized.includes('SECRET_PDF_BYTES'), false, `${label}: document bytes escaped the scrub`);+ assert.equal(serialized.includes('UNKNOWN_BINARY_BYTES'), false, `${label}: nested base64 bytes escaped the scrub`);+ }+});+test('Share-for-research outbox persists retryable failures and removes acknowledged or rejected entries', async () => {const originalChrome = globalThis.chrome;const storage = {};@@ -12567,6 +12593,23 @@ test('Share-for-research outbox persists retryable failures and removes acknowleassert.equal(await SHARE_OUTBOX_CH.flushShareOutbox(listenedProvider), 1);assert.deepEqual(listenedProvider.sent, { sessionId: 'share_conv_1', payload: { provider: 'anthropic', provider_name: 'x', model: 'm', mode: 'act', request: entry.request, response: entry.response } });assert.equal(storage[SHARE_OUTBOX_CH.SHARE_OUTBOX_STORAGE_KEY].length, 0);+ assert.equal(await SHARE_OUTBOX_CH.enqueueShareGeneration(entry), true);+ let sendCalls = 0;+ let releaseSend;+ const overlappingProvider = {+ async sendShareGeneration() {+ sendCalls++;+ await new Promise(resolve => { releaseSend = resolve; });+ return { ok: true, retryable: false, status: 202 };+ },+ };+ const firstFlush = SHARE_OUTBOX_CH.flushShareOutbox(overlappingProvider);+ const secondFlush = SHARE_OUTBOX_CH.flushShareOutbox(overlappingProvider);+ await new Promise(resolve => setTimeout(resolve, 0));+ assert.equal(sendCalls, 1, 'overlapping flushes must not send one entry twice');+ releaseSend();+ assert.deepEqual(await Promise.all([firstFlush, secondFlush]), [1, 0]);+ assert.equal(storage[SHARE_OUTBOX_CH.SHARE_OUTBOX_STORAGE_KEY].length, 0);} finally {if (originalChrome === undefined) delete globalThis.chrome;else globalThis.chrome = originalChrome;@@ -12622,7 +12665,7 @@ test('Share-for-research delivery stays opt-in and mirrored across both builds',const agent = fs.readFileSync(path.join(ROOT, `src/${browser}/src/agent/agent.js`), 'utf8');const settings = fs.readFileSync(path.join(ROOT, `src/${browser}/src/ui/settings.js`), 'utf8');const provider = fs.readFileSync(path.join(ROOT, `src/${browser}/src/providers/openai.js`), 'utf8');- assert.match(agent, /shareQueriesForResearch === true[\s\S]*enqueueShareGeneration/, `${browser}: capture not gated on the per-provider toggle`);+ assert.match(agent, /status === 'done'[\s\S]*shareQueriesForResearch === true[\s\S]*enqueueShareGeneration/, `${browser}: capture must be successful and gated on the per-provider toggle`);assert.match(agent, /'webbrain-cloud'/, `${browser}: Compass provider must not route through the share path`);assert.match(agent, /void flushShareOutbox\(shareTransport\)/, `${browser}: run-end share flush missing`);assert.match(agent, /_shareSessionId\(/, `${browser}: share session id sanitizer missing`);