Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(providers): harden research share capture

Emre Sokullu1b149b275658
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index aa10ce4c4..c0ef71ce7 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -18747,6 +18747,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// through this path.
const shareTransport = this.providerManager?.getProvider?.('webbrain_cloud');
if (shareTransport
+ && status === 'done'
&& provider?.config?.shareQueriesForResearch === true
&& String(provider?.config?.providerName || '').toLowerCase() !== 'webbrain-cloud') {
try {
diff --git a/src/chrome/src/trace/webbrain-share-outbox.js b/src/chrome/src/trace/webbrain-share-outbox.js
index 791c0a786..48f8046e1 100644
--- a/src/chrome/src/trace/webbrain-share-outbox.js
+++ b/src/chrome/src/trace/webbrain-share-outbox.js
@@ -12,6 +12,7 @@ const MAX_MESSAGE_CHARS = 10_000;
const MAX_REQUEST_BUDGET = 150_000;
const MAX_RESPONSE_CHARS = 40_000;
let storageQueue = Promise.resolve();
+let flushQueue = Promise.resolve();
let fallbackRunCounter = 0;
function bounded(value, limit) {
@@ -26,6 +27,27 @@ function clampText(value, limit = MAX_MESSAGE_CHARS) {
return bounded(value, limit);
}
+function containsBinaryBlock(value) {
+ if (!value || typeof value !== 'object') return false;
+ if (value.type === 'image_url' || value.type === 'image' || value.type === 'document') return true;
+ if (value.source?.type === 'base64') return true;
+ if (Array.isArray(value)) return value.some(containsBinaryBlock);
+ return Object.values(value).some(containsBinaryBlock);
+}
+
+function requestMessages(messages, responseContent) {
+ if (!Array.isArray(messages)) return messages;
+ const request = [...messages];
+ for (let index = request.length - 1; index >= 0; index--) {
+ const message = request[index];
+ if (message?.role === 'assistant' && message.content === responseContent) {
+ request.splice(index, 1);
+ break;
+ }
+ }
+ return request;
+}
+
function scrubMessage(message) {
if (!message || typeof message !== 'object') return null;
const copy = { ...message };
@@ -39,14 +61,14 @@ function scrubMessage(message) {
}
// Drop images entirely so raw screenshot/data-URI bytes never leave the
// browser; text blocks are clamped individually.
- if (item.type === 'image_url' || item.type === 'image') continue;
+ if (containsBinaryBlock(item)) continue;
if (item.type === 'text' && typeof item.text === 'string') {
items.push({ ...item, text: clampText(item.text) });
continue;
}
items.push(item);
}
- if (!items.length) return { role: message.role, content: '[image content omitted]' };
+ if (!items.length) return { role: message.role, content: '[binary content omitted]' };
copy.content = items;
} else if (typeof message.content === 'string') {
if (!message.content.length) return null;
@@ -84,10 +106,10 @@ export function buildShareGenerationItem({
provider,
provider_name,
}) {
- const request = scrubMessages(messages);
- if (!request?.length) return null;
const responseContent = String(finalContent ?? '');
if (!responseContent.trim()) return null;
+ const request = scrubMessages(requestMessages(messages, responseContent));
+ if (!request?.length) return null;
const generatedId = globalThis.crypto?.randomUUID?.()
|| `${Date.now().toString(36)}_${(++fallbackRunCounter).toString(36)}`;
return {
@@ -136,7 +158,7 @@ export async function enqueueShareGeneration(item) {
return true;
}
-export async function flushShareOutbox(transportProvider) {
+async function flushShareOutboxNow(transportProvider) {
if (typeof transportProvider?.sendShareGeneration !== 'function') return 0;
await storageQueue.catch(() => {});
let snapshot;
@@ -165,4 +187,10 @@ export async function flushShareOutbox(transportProvider) {
return removeIds.size;
}
-export const SHARE_OUTBOX_STORAGE_KEY = STORAGE_KEY;
\ No newline at end of file
+export function flushShareOutbox(transportProvider) {
+ const next = flushQueue.catch(() => {}).then(() => flushShareOutboxNow(transportProvider));
+ flushQueue = next;
+ return next;
+}
+
+export const SHARE_OUTBOX_STORAGE_KEY = STORAGE_KEY;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 16a5bdeb1..d108333bd 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -16541,6 +16541,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
// through this path.
const shareTransport = this.providerManager?.getProvider?.('webbrain_cloud');
if (shareTransport
+ && status === 'done'
&& provider?.config?.shareQueriesForResearch === true
&& String(provider?.config?.providerName || '').toLowerCase() !== 'webbrain-cloud') {
try {
diff --git a/src/firefox/src/trace/webbrain-share-outbox.js b/src/firefox/src/trace/webbrain-share-outbox.js
index 791c0a786..48f8046e1 100644
--- a/src/firefox/src/trace/webbrain-share-outbox.js
+++ b/src/firefox/src/trace/webbrain-share-outbox.js
@@ -12,6 +12,7 @@ const MAX_MESSAGE_CHARS = 10_000;
const MAX_REQUEST_BUDGET = 150_000;
const MAX_RESPONSE_CHARS = 40_000;
let storageQueue = Promise.resolve();
+let flushQueue = Promise.resolve();
let fallbackRunCounter = 0;
function bounded(value, limit) {
@@ -26,6 +27,27 @@ function clampText(value, limit = MAX_MESSAGE_CHARS) {
return bounded(value, limit);
}
+function containsBinaryBlock(value) {
+ if (!value || typeof value !== 'object') return false;
+ if (value.type === 'image_url' || value.type === 'image' || value.type === 'document') return true;
+ if (value.source?.type === 'base64') return true;
+ if (Array.isArray(value)) return value.some(containsBinaryBlock);
+ return Object.values(value).some(containsBinaryBlock);
+}
+
+function requestMessages(messages, responseContent) {
+ if (!Array.isArray(messages)) return messages;
+ const request = [...messages];
+ for (let index = request.length - 1; index >= 0; index--) {
+ const message = request[index];
+ if (message?.role === 'assistant' && message.content === responseContent) {
+ request.splice(index, 1);
+ break;
+ }
+ }
+ return request;
+}
+
function scrubMessage(message) {
if (!message || typeof message !== 'object') return null;
const copy = { ...message };
@@ -39,14 +61,14 @@ function scrubMessage(message) {
}
// Drop images entirely so raw screenshot/data-URI bytes never leave the
// browser; text blocks are clamped individually.
- if (item.type === 'image_url' || item.type === 'image') continue;
+ if (containsBinaryBlock(item)) continue;
if (item.type === 'text' && typeof item.text === 'string') {
items.push({ ...item, text: clampText(item.text) });
continue;
}
items.push(item);
}
- if (!items.length) return { role: message.role, content: '[image content omitted]' };
+ if (!items.length) return { role: message.role, content: '[binary content omitted]' };
copy.content = items;
} else if (typeof message.content === 'string') {
if (!message.content.length) return null;
@@ -84,10 +106,10 @@ export function buildShareGenerationItem({
provider,
provider_name,
}) {
- const request = scrubMessages(messages);
- if (!request?.length) return null;
const responseContent = String(finalContent ?? '');
if (!responseContent.trim()) return null;
+ const request = scrubMessages(requestMessages(messages, responseContent));
+ if (!request?.length) return null;
const generatedId = globalThis.crypto?.randomUUID?.()
|| `${Date.now().toString(36)}_${(++fallbackRunCounter).toString(36)}`;
return {
@@ -136,7 +158,7 @@ export async function enqueueShareGeneration(item) {
return true;
}
-export async function flushShareOutbox(transportProvider) {
+async function flushShareOutboxNow(transportProvider) {
if (typeof transportProvider?.sendShareGeneration !== 'function') return 0;
await storageQueue.catch(() => {});
let snapshot;
@@ -165,4 +187,10 @@ export async function flushShareOutbox(transportProvider) {
return removeIds.size;
}
-export const SHARE_OUTBOX_STORAGE_KEY = STORAGE_KEY;
\ No newline at end of file
+export function flushShareOutbox(transportProvider) {
+ const next = flushQueue.catch(() => {}).then(() => flushShareOutboxNow(transportProvider));
+ flushQueue = next;
+ return next;
+}
+
+export const SHARE_OUTBOX_STORAGE_KEY = STORAGE_KEY;
diff --git a/test/run.js b/test/run.js
index ff773a82f..4755488d4 100644
--- a/test/run.js
+++ b/test/run.js
@@ -12523,6 +12523,32 @@ test('Share-for-research item drops empty runs and caps the whole request', () =
assert.deepEqual(item.request.at(-1), { role: 'system', content: '[remaining shared message omitted]' });
});
+test('Share-for-research item excludes terminal answers and binary document blocks', () => {
+ for (const [label, outbox] of [['chrome', SHARE_OUTBOX_CH], ['firefox', SHARE_OUTBOX_FX]]) {
+ const item = outbox.buildShareGenerationItem({
+ runId: `run-share-binary-${label}`,
+ finalContent: 'Final answer',
+ messages: [
+ {
+ role: 'user',
+ content: [
+ { type: 'text', text: 'Read this attachment' },
+ { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: 'SECRET_PDF_BYTES' } },
+ { type: 'other', payload: { source: { type: 'base64', data: 'UNKNOWN_BINARY_BYTES' } } },
+ ],
+ },
+ { role: 'assistant', content: 'Final answer' },
+ ],
+ model: 'some-model', mode: 'act', provider: 'anthropic', provider_name: 'Anthropic Claude',
+ });
+ assert.equal(item.request.length, 1, `${label}: terminal answer stayed in the shared request`);
+ assert.deepEqual(item.request[0].content, [{ type: 'text', text: 'Read this attachment' }]);
+ const serialized = JSON.stringify(item);
+ assert.equal(serialized.includes('SECRET_PDF_BYTES'), false, `${label}: document bytes escaped the scrub`);
+ assert.equal(serialized.includes('UNKNOWN_BINARY_BYTES'), false, `${label}: nested base64 bytes escaped the scrub`);
+ }
+});
+
test('Share-for-research outbox persists retryable failures and removes acknowledged or rejected entries', async () => {
const originalChrome = globalThis.chrome;
const storage = {};
@@ -12567,6 +12593,23 @@ test('Share-for-research outbox persists retryable failures and removes acknowle
assert.equal(await SHARE_OUTBOX_CH.flushShareOutbox(listenedProvider), 1);
assert.deepEqual(listenedProvider.sent, { sessionId: 'share_conv_1', payload: { provider: 'anthropic', provider_name: 'x', model: 'm', mode: 'act', request: entry.request, response: entry.response } });
assert.equal(storage[SHARE_OUTBOX_CH.SHARE_OUTBOX_STORAGE_KEY].length, 0);
+ assert.equal(await SHARE_OUTBOX_CH.enqueueShareGeneration(entry), true);
+ let sendCalls = 0;
+ let releaseSend;
+ const overlappingProvider = {
+ async sendShareGeneration() {
+ sendCalls++;
+ await new Promise(resolve => { releaseSend = resolve; });
+ return { ok: true, retryable: false, status: 202 };
+ },
+ };
+ const firstFlush = SHARE_OUTBOX_CH.flushShareOutbox(overlappingProvider);
+ const secondFlush = SHARE_OUTBOX_CH.flushShareOutbox(overlappingProvider);
+ await new Promise(resolve => setTimeout(resolve, 0));
+ assert.equal(sendCalls, 1, 'overlapping flushes must not send one entry twice');
+ releaseSend();
+ assert.deepEqual(await Promise.all([firstFlush, secondFlush]), [1, 0]);
+ assert.equal(storage[SHARE_OUTBOX_CH.SHARE_OUTBOX_STORAGE_KEY].length, 0);
} finally {
if (originalChrome === undefined) delete globalThis.chrome;
else globalThis.chrome = originalChrome;
@@ -12622,7 +12665,7 @@ test('Share-for-research delivery stays opt-in and mirrored across both builds',
const agent = fs.readFileSync(path.join(ROOT, `src/${browser}/src/agent/agent.js`), 'utf8');
const settings = fs.readFileSync(path.join(ROOT, `src/${browser}/src/ui/settings.js`), 'utf8');
const provider = fs.readFileSync(path.join(ROOT, `src/${browser}/src/providers/openai.js`), 'utf8');
- assert.match(agent, /shareQueriesForResearch === true[\s\S]*enqueueShareGeneration/, `${browser}: capture not gated on the per-provider toggle`);
+ assert.match(agent, /status === 'done'[\s\S]*shareQueriesForResearch === true[\s\S]*enqueueShareGeneration/, `${browser}: capture must be successful and gated on the per-provider toggle`);
assert.match(agent, /'webbrain-cloud'/, `${browser}: Compass provider must not route through the share path`);
assert.match(agent, /void flushShareOutbox\(shareTransport\)/, `${browser}: run-end share flush missing`);
assert.match(agent, /_shareSessionId\(/, `${browser}: share session id sanitizer missing`);