Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Preserve recovery task hashes and recognize localized LinkedIn previews
Emre Sokullu74219e56e5ba
diff --git a/src/chrome/src/agent/conversation-persistence.js b/src/chrome/src/agent/conversation-persistence.jsindex 2477a8b39..d99033e87 100644--- a/src/chrome/src/agent/conversation-persistence.js+++ b/src/chrome/src/agent/conversation-persistence.js@@ -8,6 +8,9 @@ function byteLength(value) {}function capText(value, maxChars, marker, state) {+ // Authority-bearing text must retain the exact parent-task hash. Its data+ // URLs are part of that text; quota handling may fail, but cannot rewrite it.+ if (maxChars === Infinity) return String(value || '');const sanitized = String(value || '').replace(DATA_URL_RE, () => {state.compacted = true;return '[embedded binary data omitted from session recovery]';diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex a1ba69ccf..e9fe24e0b 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -5863,8 +5863,9 @@const verifiedLinkedInImagePreview = clicked => {if (params.adapterName !== 'linkedin') return false;const button = _composedClosestElement(clicked, 'button,[role="button"]');- if (!button || compact(button.getAttribute('aria-label')).toLowerCase()- !== 'click or press enter to display in the image preview') return false;+ if (!button) return false;+ // Reject other controls before preview-specific style and hit probes.+ if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false;const painted = el => {if (!visible(el) || !_hasVisibleBox(el)) return false;for (let node = el; node?.nodeType === 1; node = _composedParent(node)) {@@ -5876,18 +5877,23 @@|| button.getAttribute('aria-disabled') === 'true'|| String(button.getAttribute('type') || '').toLowerCase() !== 'button'|| button.form || button.hasAttribute('form') || button.hasAttribute('download')+ || /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '')|| _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false;// Attachment previews live in history, not in the composer. Proximity// to the composer cannot turn this verified viewer into a send; labels// alone cannot exempt a message commit or an unrelated image button.- if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false;- if ([button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent]+ if ([button.getAttribute('aria-label'), button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent].some(_hasMessageCommitName)) return false;- if (!Array.from(button.querySelectorAll('img[src]')).some(painted)) return false;const rect = button.getBoundingClientRect();- return _isComposedAncestor(button, _shadowAwareElementFromPoint(- rect.left + rect.width / 2, rect.top + rect.height / 2,- ));+ const hit = _shadowAwareElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);+ // LinkedIn localizes the preview label. Verify the attachment itself+ // at the dispatch point instead; small avatar/reaction/menu icons do+ // not prove that a button opens an image viewer.+ return _isComposedAncestor(button, hit) && Array.from(button.querySelectorAll('img[src]')).some(image => {+ const imageRect = image.getBoundingClientRect();+ return painted(image) && imageRect.width >= 64 && imageRect.height >= 64+ && _isComposedAncestor(image, hit);+ });};const classifyLinkedInNavigation = (clicked, blockingModal = null) => {diff --git a/src/firefox/src/agent/conversation-persistence.js b/src/firefox/src/agent/conversation-persistence.jsindex 2477a8b39..d99033e87 100644--- a/src/firefox/src/agent/conversation-persistence.js+++ b/src/firefox/src/agent/conversation-persistence.js@@ -8,6 +8,9 @@ function byteLength(value) {}function capText(value, maxChars, marker, state) {+ // Authority-bearing text must retain the exact parent-task hash. Its data+ // URLs are part of that text; quota handling may fail, but cannot rewrite it.+ if (maxChars === Infinity) return String(value || '');const sanitized = String(value || '').replace(DATA_URL_RE, () => {state.compacted = true;return '[embedded binary data omitted from session recovery]';diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 799746d64..38a0e2216 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -4951,8 +4951,9 @@const verifiedLinkedInImagePreview = clicked => {if (params.adapterName !== 'linkedin') return false;const button = _composedClosestElement(clicked, 'button,[role="button"]');- if (!button || compact(button.getAttribute('aria-label')).toLowerCase()- !== 'click or press enter to display in the image preview') return false;+ if (!button) return false;+ // Reject other controls before preview-specific style and hit probes.+ if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false;const painted = el => {if (!visible(el) || !_hasVisibleBox(el)) return false;for (let node = el; node?.nodeType === 1; node = _composedParent(node)) {@@ -4964,18 +4965,23 @@|| button.getAttribute('aria-disabled') === 'true'|| String(button.getAttribute('type') || '').toLowerCase() !== 'button'|| button.form || button.hasAttribute('form') || button.hasAttribute('download')+ || /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '')|| _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false;// Attachment previews live in history, not in the composer. Proximity// to the composer cannot turn this verified viewer into a send; labels// alone cannot exempt a message commit or an unrelated image button.- if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false;- if ([button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent]+ if ([button.getAttribute('aria-label'), button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent].some(_hasMessageCommitName)) return false;- if (!Array.from(button.querySelectorAll('img[src]')).some(painted)) return false;const rect = button.getBoundingClientRect();- return _isComposedAncestor(button, _shadowAwareElementFromPoint(- rect.left + rect.width / 2, rect.top + rect.height / 2,- ));+ const hit = _shadowAwareElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);+ // LinkedIn localizes the preview label. Verify the attachment itself+ // at the dispatch point instead; small avatar/reaction/menu icons do+ // not prove that a button opens an image viewer.+ return _isComposedAncestor(button, hit) && Array.from(button.querySelectorAll('img[src]')).some(image => {+ const imageRect = image.getBoundingClientRect();+ return painted(image) && imageRect.width >= 64 && imageRect.height >= 64+ && _isComposedAncestor(image, hit);+ });};const classifyLinkedInNavigation = (clicked, blockingModal = null) => {diff --git a/test/chat-steering.mjs b/test/chat-steering.mjsindex cb376a124..c5b8046f0 100644--- a/test/chat-steering.mjs+++ b/test/chat-steering.mjs@@ -360,6 +360,43 @@ for (const build of ['chrome', 'firefox']) {agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId);});+ for (const contentKind of ['string', 'text-block']) {+ test(`${build}: ${contentKind} authority data URLs keep the steering chain after recovery`, async () => {+ const { serializeConversationForSession: serialize } = await import(`../src/${build}/src/agent/conversation-persistence.js`);+ const agent = actSetup(Agent);+ await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options);+ const root = 'Add this invitation to my calendar: data:application/pdf;base64,QUJDRA==\nUse the meeting time in it.';+ const content = contentKind === 'string' ? root : [{ type: 'text', text: root },+ { type: 'image_url', image_url: { url: 'data:image/png;base64,QUJDRA==' } }];+ const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content }];+ agent.conversations.set(tabId, messages);+ agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true });+ steer(agent, 'Use Ipek’s message: data:image/png;base64,QUJDRA==', 'image-correction');+ await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options);+ steer(agent, 'Use the work calendar', 'calendar-correction');+ await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options);+ const live = agent._activeTaskBinding(messages);+ messages.push({ role: 'assistant', content: 'Observed binary: data:image/png;base64,QUJDRA==' });+ messages.push(...Array.from({ length: 70 }, () => ({ role: 'assistant', content: 'Earlier observation '.repeat(1000) })));+ for (const maxBytes of [1_500_000, 450_000]) {+ const snapshot = serialize(messages, { maxBytes, preserveMessageIndices: live.pinnedIndices });+ const restored = actSetup(Agent);+ restored.conversations.set(tabId, snapshot.messages);+ assert.equal(restored._activeTaskBinding(snapshot.messages).text, live.text);+ assert.equal(restored._progressTaskKeyHash(tabId), agent._progressTaskKeyHash(tabId));+ assert.equal(restored._plannerUserAuthoredText(snapshot.messages[1]), root);+ assert.equal(snapshot.messages[live.pinnedIndices[1]].content, messages[live.pinnedIndices[1]].content);+ assert.ok(!snapshot.messages.some(message => message.role === 'assistant' && String(message.content).includes('data:image')),+ 'Non-authority binary text must still be omitted');+ if (contentKind === 'text-block') assert.equal(snapshot.messages[1].content[1].type, 'text', 'Binary attachment blocks remain omitted');+ restored._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true });+ restored._markPlanExecutionToolCall(tabId, 'read_page', { success: true });+ assert.equal(restored._planExecutionGuards.get(tabId).taskDrifted, false);+ }+ agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId);+ });+ }+test(`${build}: steering during planning discards the superseded intent`, async () => {const entered = deferred(), release = deferred(), plans = [];const agent = actSetup(Agent, {}, async (_tab, enriched) => {diff --git a/test/fixtures/message-recipient-navigation.mjs b/test/fixtures/message-recipient-navigation.mjsindex e93bc7c7e..a48a8edd5 100644--- a/test/fixtures/message-recipient-navigation.mjs+++ b/test/fixtures/message-recipient-navigation.mjs@@ -164,9 +164,33 @@ export function registerMessageRecipientNavigationFixtures({assert.equal(await page.evaluate(() => window.sentMessages), 0);});+ register(`${kind}: localized LinkedIn attachment viewers open across selector, text, AX, and coordinate targeting`, async page => {+ const { guard } = await setupMessageHistory(page);+ for (const label of ['Görsel önizlemesinde görüntülemek için tıklayın veya Enter tuşuna basın',+ 'Cliquez ou appuyez sur Entrée pour afficher un aperçu de l’image',+ 'Clique ou pressione Enter para exibir na visualização da imagem']) {+ await page.locator('#preview').evaluate((el, label) => el.setAttribute('aria-label', label), label);+ const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxChars: 20000 });+ const ref = tree.pageContent.split('\n').find(line => line.includes(`button "${label}"`))?.match(/\[([^\]]+)\]/)?.[1];+ assert.ok(ref, tree.pageContent);+ const point = await page.locator('#preview').evaluate(el => {+ const r = el.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 };+ });+ for (const [tool, args] of [['click', { selector: '#preview' }], ['click', { text: label, textMatch: 'exact' }],+ ['click_ax', { ref_id: ref }], ['click', point], ['click', { selector: '#preview-image' }]]) {+ assert.equal(await guard(tool, args), null, `${label}/${tool}`);+ assert.equal((await call(page, tool, args)).success, true);+ assert.equal(await page.locator('#viewer').isVisible(), true);+ await page.locator('#close-viewer').click();+ }+ }+ assert.equal(await page.evaluate(() => window.sentMessages), 0);+ });+register(`${kind}: LinkedIn preview lookalikes and hidden or occluded images stay protected`, async page => {const { guard } = await setupMessageHistory(page);- for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text']) {+ for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text',+ 'localized-send', 'submit', 'external-form', 'outside-history', 'small-icon', 'menu', 'image-hidden']) {await page.evaluate(change => {const button=document.querySelector('#preview');window.originalPreview=button.cloneNode(true);@@ -178,6 +202,13 @@ export function registerMessageRecipientNavigationFixtures({if(change==='disabled')button.disabled=true;if(change==='commit-label')button.title='Send';if(change==='commit-text'){button.title='View image';button.append('Send');}+ if(change==='localized-send')button.setAttribute('aria-label','Gönder');+ if(change==='submit')button.type='submit';+ if(change==='external-form')button.setAttribute('form','composer');+ if(change==='outside-history')document.body.append(button);+ if(change==='small-icon')button.querySelector('img').style='width:20px;height:20px';+ if(change==='menu')button.setAttribute('aria-haspopup','menu');+ if(change==='image-hidden')button.querySelector('img').style.visibility='hidden';if(change==='occluded') {const cover=document.createElement('div');cover.id='cover';cover.style='position:fixed;left:350px;top:80px;width:400px;height:340px;z-index:20;background:white';document.body.append(cover);