Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Preserve recovery task hashes and recognize localized LinkedIn previews

Emre Sokullu74219e56e5ba
diff --git a/src/chrome/src/agent/conversation-persistence.js b/src/chrome/src/agent/conversation-persistence.js
index 2477a8b39..d99033e87 100644
--- a/src/chrome/src/agent/conversation-persistence.js
+++ b/src/chrome/src/agent/conversation-persistence.js
@@ -8,6 +8,9 @@ function byteLength(value) {
}
function capText(value, maxChars, marker, state) {
+ // Authority-bearing text must retain the exact parent-task hash. Its data
+ // URLs are part of that text; quota handling may fail, but cannot rewrite it.
+ if (maxChars === Infinity) return String(value || '');
const sanitized = String(value || '').replace(DATA_URL_RE, () => {
state.compacted = true;
return '[embedded binary data omitted from session recovery]';
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index a1ba69ccf..e9fe24e0b 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -5863,8 +5863,9 @@
const verifiedLinkedInImagePreview = clicked => {
if (params.adapterName !== 'linkedin') return false;
const button = _composedClosestElement(clicked, 'button,[role="button"]');
- if (!button || compact(button.getAttribute('aria-label')).toLowerCase()
- !== 'click or press enter to display in the image preview') return false;
+ if (!button) return false;
+ // Reject other controls before preview-specific style and hit probes.
+ if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false;
const painted = el => {
if (!visible(el) || !_hasVisibleBox(el)) return false;
for (let node = el; node?.nodeType === 1; node = _composedParent(node)) {
@@ -5876,18 +5877,23 @@
|| button.getAttribute('aria-disabled') === 'true'
|| String(button.getAttribute('type') || '').toLowerCase() !== 'button'
|| button.form || button.hasAttribute('form') || button.hasAttribute('download')
+ || /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '')
|| _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false;
// Attachment previews live in history, not in the composer. Proximity
// to the composer cannot turn this verified viewer into a send; labels
// alone cannot exempt a message commit or an unrelated image button.
- if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false;
- if ([button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent]
+ if ([button.getAttribute('aria-label'), button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent]
.some(_hasMessageCommitName)) return false;
- if (!Array.from(button.querySelectorAll('img[src]')).some(painted)) return false;
const rect = button.getBoundingClientRect();
- return _isComposedAncestor(button, _shadowAwareElementFromPoint(
- rect.left + rect.width / 2, rect.top + rect.height / 2,
- ));
+ const hit = _shadowAwareElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);
+ // LinkedIn localizes the preview label. Verify the attachment itself
+ // at the dispatch point instead; small avatar/reaction/menu icons do
+ // not prove that a button opens an image viewer.
+ return _isComposedAncestor(button, hit) && Array.from(button.querySelectorAll('img[src]')).some(image => {
+ const imageRect = image.getBoundingClientRect();
+ return painted(image) && imageRect.width >= 64 && imageRect.height >= 64
+ && _isComposedAncestor(image, hit);
+ });
};
const classifyLinkedInNavigation = (clicked, blockingModal = null) => {
diff --git a/src/firefox/src/agent/conversation-persistence.js b/src/firefox/src/agent/conversation-persistence.js
index 2477a8b39..d99033e87 100644
--- a/src/firefox/src/agent/conversation-persistence.js
+++ b/src/firefox/src/agent/conversation-persistence.js
@@ -8,6 +8,9 @@ function byteLength(value) {
}
function capText(value, maxChars, marker, state) {
+ // Authority-bearing text must retain the exact parent-task hash. Its data
+ // URLs are part of that text; quota handling may fail, but cannot rewrite it.
+ if (maxChars === Infinity) return String(value || '');
const sanitized = String(value || '').replace(DATA_URL_RE, () => {
state.compacted = true;
return '[embedded binary data omitted from session recovery]';
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index 799746d64..38a0e2216 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -4951,8 +4951,9 @@
const verifiedLinkedInImagePreview = clicked => {
if (params.adapterName !== 'linkedin') return false;
const button = _composedClosestElement(clicked, 'button,[role="button"]');
- if (!button || compact(button.getAttribute('aria-label')).toLowerCase()
- !== 'click or press enter to display in the image preview') return false;
+ if (!button) return false;
+ // Reject other controls before preview-specific style and hit probes.
+ if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false;
const painted = el => {
if (!visible(el) || !_hasVisibleBox(el)) return false;
for (let node = el; node?.nodeType === 1; node = _composedParent(node)) {
@@ -4964,18 +4965,23 @@
|| button.getAttribute('aria-disabled') === 'true'
|| String(button.getAttribute('type') || '').toLowerCase() !== 'button'
|| button.form || button.hasAttribute('form') || button.hasAttribute('download')
+ || /^(?:true|menu|listbox|tree|grid)$/.test(button.getAttribute('aria-haspopup') || '')
|| _composedClosestElement(button, 'form,.msg-form,[contenteditable]:not([contenteditable="false"])')) return false;
// Attachment previews live in history, not in the composer. Proximity
// to the composer cannot turn this verified viewer into a send; labels
// alone cannot exempt a message commit or an unrelated image button.
- if (!_composedClosestElement(button, '[role="log"],.msg-s-message-list,.msg-s-event-listitem')) return false;
- if ([button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent]
+ if ([button.getAttribute('aria-label'), button.getAttribute('title'), button.getAttribute('data-tooltip'), button.innerText || button.textContent]
.some(_hasMessageCommitName)) return false;
- if (!Array.from(button.querySelectorAll('img[src]')).some(painted)) return false;
const rect = button.getBoundingClientRect();
- return _isComposedAncestor(button, _shadowAwareElementFromPoint(
- rect.left + rect.width / 2, rect.top + rect.height / 2,
- ));
+ const hit = _shadowAwareElementFromPoint(rect.left + rect.width / 2, rect.top + rect.height / 2);
+ // LinkedIn localizes the preview label. Verify the attachment itself
+ // at the dispatch point instead; small avatar/reaction/menu icons do
+ // not prove that a button opens an image viewer.
+ return _isComposedAncestor(button, hit) && Array.from(button.querySelectorAll('img[src]')).some(image => {
+ const imageRect = image.getBoundingClientRect();
+ return painted(image) && imageRect.width >= 64 && imageRect.height >= 64
+ && _isComposedAncestor(image, hit);
+ });
};
const classifyLinkedInNavigation = (clicked, blockingModal = null) => {
diff --git a/test/chat-steering.mjs b/test/chat-steering.mjs
index cb376a124..c5b8046f0 100644
--- a/test/chat-steering.mjs
+++ b/test/chat-steering.mjs
@@ -360,6 +360,43 @@ for (const build of ['chrome', 'firefox']) {
agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId);
});
+ for (const contentKind of ['string', 'text-block']) {
+ test(`${build}: ${contentKind} authority data URLs keep the steering chain after recovery`, async () => {
+ const { serializeConversationForSession: serialize } = await import(`../src/${build}/src/agent/conversation-persistence.js`);
+ const agent = actSetup(Agent);
+ await agent._claimRunEntry(tabId, 'interactive', options); agent._beginSteeringRun(tabId, () => {}, options);
+ const root = 'Add this invitation to my calendar: data:application/pdf;base64,QUJDRA==\nUse the meeting time in it.';
+ const content = contentKind === 'string' ? root : [{ type: 'text', text: root },
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,QUJDRA==' } }];
+ const messages = [{ role: 'system', content: 'sys' }, { role: 'user', content }];
+ agent.conversations.set(tabId, messages);
+ agent._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true });
+ steer(agent, 'Use Ipek’s message: data:image/png;base64,QUJDRA==', 'image-correction');
+ await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options);
+ steer(agent, 'Use the work calendar', 'calendar-correction');
+ await agent._revalidatePendingSteering(tabId, messages, () => {}, 'act', null, null, options);
+ const live = agent._activeTaskBinding(messages);
+ messages.push({ role: 'assistant', content: 'Observed binary: data:image/png;base64,QUJDRA==' });
+ messages.push(...Array.from({ length: 70 }, () => ({ role: 'assistant', content: 'Earlier observation '.repeat(1000) })));
+ for (const maxBytes of [1_500_000, 450_000]) {
+ const snapshot = serialize(messages, { maxBytes, preserveMessageIndices: live.pinnedIndices });
+ const restored = actSetup(Agent);
+ restored.conversations.set(tabId, snapshot.messages);
+ assert.equal(restored._activeTaskBinding(snapshot.messages).text, live.text);
+ assert.equal(restored._progressTaskKeyHash(tabId), agent._progressTaskKeyHash(tabId));
+ assert.equal(restored._plannerUserAuthoredText(snapshot.messages[1]), root);
+ assert.equal(snapshot.messages[live.pinnedIndices[1]].content, messages[live.pinnedIndices[1]].content);
+ assert.ok(!snapshot.messages.some(message => message.role === 'assistant' && String(message.content).includes('data:image')),
+ 'Non-authority binary text must still be omitted');
+ if (contentKind === 'text-block') assert.equal(snapshot.messages[1].content[1].type, 'text', 'Binary attachment blocks remain omitted');
+ restored._startPlanExecutionGuard(tabId, 'act', { requiresStateChange: true });
+ restored._markPlanExecutionToolCall(tabId, 'read_page', { success: true });
+ assert.equal(restored._planExecutionGuards.get(tabId).taskDrifted, false);
+ }
+ agent._finishSteeringRun(tabId); agent._releaseRunEntry(tabId);
+ });
+ }
+
test(`${build}: steering during planning discards the superseded intent`, async () => {
const entered = deferred(), release = deferred(), plans = [];
const agent = actSetup(Agent, {}, async (_tab, enriched) => {
diff --git a/test/fixtures/message-recipient-navigation.mjs b/test/fixtures/message-recipient-navigation.mjs
index e93bc7c7e..a48a8edd5 100644
--- a/test/fixtures/message-recipient-navigation.mjs
+++ b/test/fixtures/message-recipient-navigation.mjs
@@ -164,9 +164,33 @@ export function registerMessageRecipientNavigationFixtures({
assert.equal(await page.evaluate(() => window.sentMessages), 0);
});
+ register(`${kind}: localized LinkedIn attachment viewers open across selector, text, AX, and coordinate targeting`, async page => {
+ const { guard } = await setupMessageHistory(page);
+ for (const label of ['Görsel önizlemesinde görüntülemek için tıklayın veya Enter tuşuna basın',
+ 'Cliquez ou appuyez sur Entrée pour afficher un aperçu de l’image',
+ 'Clique ou pressione Enter para exibir na visualização da imagem']) {
+ await page.locator('#preview').evaluate((el, label) => el.setAttribute('aria-label', label), label);
+ const tree = await call(page, 'get_accessibility_tree', { filter: 'visible', maxChars: 20000 });
+ const ref = tree.pageContent.split('\n').find(line => line.includes(`button "${label}"`))?.match(/\[([^\]]+)\]/)?.[1];
+ assert.ok(ref, tree.pageContent);
+ const point = await page.locator('#preview').evaluate(el => {
+ const r = el.getBoundingClientRect(); return { x: r.x + r.width / 2, y: r.y + r.height / 2 };
+ });
+ for (const [tool, args] of [['click', { selector: '#preview' }], ['click', { text: label, textMatch: 'exact' }],
+ ['click_ax', { ref_id: ref }], ['click', point], ['click', { selector: '#preview-image' }]]) {
+ assert.equal(await guard(tool, args), null, `${label}/${tool}`);
+ assert.equal((await call(page, tool, args)).success, true);
+ assert.equal(await page.locator('#viewer').isVisible(), true);
+ await page.locator('#close-viewer').click();
+ }
+ }
+ assert.equal(await page.evaluate(() => window.sentMessages), 0);
+ });
+
register(`${kind}: LinkedIn preview lookalikes and hidden or occluded images stay protected`, async page => {
const { guard } = await setupMessageHistory(page);
- for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text']) {
+ for (const change of ['no-image', 'composer', 'hidden', 'transparent', 'ancestor-transparent', 'occluded', 'disabled', 'commit-label', 'commit-text',
+ 'localized-send', 'submit', 'external-form', 'outside-history', 'small-icon', 'menu', 'image-hidden']) {
await page.evaluate(change => {
const button=document.querySelector('#preview');
window.originalPreview=button.cloneNode(true);
@@ -178,6 +202,13 @@ export function registerMessageRecipientNavigationFixtures({
if(change==='disabled')button.disabled=true;
if(change==='commit-label')button.title='Send';
if(change==='commit-text'){button.title='View image';button.append('Send');}
+ if(change==='localized-send')button.setAttribute('aria-label','Gönder');
+ if(change==='submit')button.type='submit';
+ if(change==='external-form')button.setAttribute('form','composer');
+ if(change==='outside-history')document.body.append(button);
+ if(change==='small-icon')button.querySelector('img').style='width:20px;height:20px';
+ if(change==='menu')button.setAttribute('aria-haspopup','menu');
+ if(change==='image-hidden')button.querySelector('img').style.visibility='hidden';
if(change==='occluded') {
const cover=document.createElement('div');cover.id='cover';
cover.style='position:fixed;left:350px;top:80px;width:400px;height:340px;z-index:20;background:white';document.body.append(cover);