Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(linkedin): permit opening post composer
Emre Sokulluc8c15d9fdb83
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 726fe5070..6604b6de8 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -22878,16 +22878,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d: false;}+ const missingLinkedInComposer = policy.adapterName === 'linkedin'+ && probe?.success === true && probe?.composerAvailable === false;return {success: false,blocked: true,noDispatch: true,dispatched: false,messageRecipientGuard: true,+ ...(missingLinkedInComposer ? { retryable: false } : {}),reasonCode: probe?.success !== true || probe?.conclusive !== true || probe?.messageSend !== true? 'message_send_classification_inconclusive': (target ? 'active_recipient_unverified' : 'authorized_recipient_missing'),- error: probe?.success !== true || probe?.conclusive !== true || probe?.messageSend !== true+ error: missingLinkedInComposer+ ? 'Message action blocked: no message composer is visible and this control is not a verified non-send action. Changing click targeting methods will not resolve this classification failure. Re-read the page to find a supported navigation or composer-opening control; if none is available, report the blocker instead of repeating the click.'+ : probe?.success !== true || probe?.conclusive !== true || probe?.messageSend !== true? 'Message action blocked: WebBrain could not conclusively resolve the target control and active composer. Re-read the page and retry with an exact visible control or fresh ref_id.': target? 'Message send blocked: the active conversation does not exactly match the recipient authorized by the user. Select the intended conversation, re-read its visible header, then retry the send action.'diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 8387e4be1..613dd56a9 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -1371,11 +1371,36 @@'input[type="submit"]', 'summary', 'label', '[onclick]', '[data-action]',..._siteInteractiveSelectors(),].join(', ');- return Array.from(scope.querySelectorAll(selectors))+ // LinkedIn can render the feed itself inside an open shadow root. Keep+ // preflight and dispatch on the same candidate set, including duplicate+ // labels across roots, without searching outside the selected modal.+ const candidates = [];+ const visit = (root) => {+ candidates.push(...root.querySelectorAll(selectors));+ if (root.shadowRoot) visit(root.shadowRoot);+ for (const host of root.querySelectorAll('*')) {+ if (host.shadowRoot) visit(host.shadowRoot);+ }+ };+ visit(scope);+ return candidates.map(e => ({ e, txt: _siteInteractionText(e).toLowerCase() })).filter(candidate => candidate.txt);}+ function _shadowAwareElementFromPoint(x, y) {+ let topmost = document.elementFromPoint(x, y);+ const seen = new Set();+ while (topmost && topmost.shadowRoot && !seen.has(topmost)) {+ seen.add(topmost);+ let inner = null;+ try { inner = topmost.shadowRoot.elementFromPoint(x, y); } catch {}+ if (!inner || inner === topmost) break;+ topmost = inner;+ }+ return topmost;+ }+let _lastClickIdent = null;/**@@ -1601,7 +1626,7 @@el = interactive[params.index];if (!el) return { ..._staleIndexError(params.index, interactive), dispatched: false };} else if (params.x != null && params.y != null) {- el = document.elementFromPoint(params.x, params.y);+ el = _shadowAwareElementFromPoint(params.x, params.y);}// A selector miss is a discovery failure, not a changed bound target. Do@@ -1741,8 +1766,8 @@if (r.width >= 1 && r.height >= 1 && r.top >= 0 && r.left >= 0 && r.bottom <= window.innerHeight && r.right <= window.innerWidth) {const cx = Math.round(r.left + r.width / 2);const cy = Math.round(r.top + r.height / 2);- const topmost = document.elementFromPoint(cx, cy);- if (topmost && topmost !== el && !el.contains(topmost) && !topmost.contains(el)) {+ const topmost = _shadowAwareElementFromPoint(cx, cy);+ if (topmost && !_isComposedAncestor(el, topmost) && !_isComposedAncestor(topmost, el)) {// Another element is painted on top. Give the model actionable// info (what's blocking, where, what to do) instead of silently// clicking the wrong thing.@@ -5080,7 +5105,7 @@} else if (Number.isInteger(args.index) && args.index >= 0) {target = queryInteractiveForToolIndex()[args.index] || null;} else if (Number.isFinite(args.x) && Number.isFinite(args.y)) {- target = document.elementFromPoint(args.x, args.y);+ target = _shadowAwareElementFromPoint(args.x, args.y);}targetResolved = !!target;}@@ -5280,6 +5305,32 @@}};+ const verifiedLinkedInPostEntry = (clicked) => {+ if (params.adapterName !== 'linkedin' || !/^\/feed\/?$/.test(location.pathname)) return false;+ const button = _composedClosestElement(clicked, 'button,[role="button"]');+ if (!button || !visible(button) || button.disabled+ || button.getAttribute?.('aria-disabled') === 'true'+ || String(button.getAttribute?.('type') || 'button').toLowerCase() !== 'button'+ || button.form || button.hasAttribute?.('form')) return false;+ // The feed entry only opens a public-post composer. Do not infer this+ // from the requested text, a nearby editor, or a control inside a post,+ // conversation, or dialog: those can send or publish existing drafts.+ if (!_composedClosestElement(button, 'main,[role="main"]')+ || _composedClosestElement(button,+ 'form,article,dialog,[role="dialog"],[role="alertdialog"],[role="log"],'+ + '[data-message-id],[data-thread-id],[data-conversation-id],'+ + '[contenteditable]:not([contenteditable="false"])')) return false;+ const labels = [button.innerText || button.textContent, button.getAttribute?.('aria-label')]+ .map(value => compact(value).toLowerCase()).filter(Boolean);+ if (!labels.length || !labels.every(label => label === 'start a post')) return false;+ // A shadow-root dialog may not appear in the document's modal query.+ // Require the actual painted control, descending through open roots.+ const rect = button.getBoundingClientRect();+ const x = rect.left + rect.width / 2, y = rect.top + rect.height / 2;+ const hit = _shadowAwareElementFromPoint(x, y);+ return !!hit && _isComposedAncestor(button, hit);+ };+let composer = null;let messageSend = null;if (observationOnly) {@@ -5320,6 +5371,9 @@if (!visible(control) || (modal && !_isComposedAncestor(modal, target))) {return { success: true, messageSend: null, conclusive: false, identityCandidates: [] };}+ if (verifiedLinkedInPostEntry(target)) {+ return { success: true, messageSend: false, conclusive: true, composerSetup: true, identityCandidates: [] };+ }const linkedInNavigation = classifyLinkedInNavigation(target, modal);if (linkedInNavigation === 'navigation') {return { success: true, messageSend: false, conclusive: true, navigation: true, identityCandidates: [] };diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex 3f7d7e097..777386a59 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -20643,16 +20643,21 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d: false;}+ const missingLinkedInComposer = policy.adapterName === 'linkedin'+ && probe?.success === true && probe?.composerAvailable === false;return {success: false,blocked: true,noDispatch: true,dispatched: false,messageRecipientGuard: true,+ ...(missingLinkedInComposer ? { retryable: false } : {}),reasonCode: probe?.success !== true || probe?.conclusive !== true || probe?.messageSend !== true? 'message_send_classification_inconclusive': (target ? 'active_recipient_unverified' : 'authorized_recipient_missing'),- error: probe?.success !== true || probe?.conclusive !== true || probe?.messageSend !== true+ error: missingLinkedInComposer+ ? 'Message action blocked: no message composer is visible and this control is not a verified non-send action. Changing click targeting methods will not resolve this classification failure. Re-read the page to find a supported navigation or composer-opening control; if none is available, report the blocker instead of repeating the click.'+ : probe?.success !== true || probe?.conclusive !== true || probe?.messageSend !== true? 'Message action blocked: WebBrain could not conclusively resolve the target control and active composer. Re-read the page and retry with an exact visible control or fresh ref_id.': target? 'Message send blocked: the active conversation does not exactly match the recipient authorized by the user. Select the intended conversation, re-read its visible header, then retry the send action.'diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex 6708b81cc..31d91b8fe 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -1346,7 +1346,19 @@const siteText = _isSiteInteractive(el) ? _siteInteractionText(el) : '';return (siteText || el.innerText || (_valIsLabel(el) ? el.value : '') || el.placeholder || el.ariaLabel || '').trim().toLowerCase();};- const all = Array.from(scope.querySelectorAll(sels)).filter(_keepCandidate);+ // LinkedIn can render the feed itself inside an open shadow root. Keep+ // preflight and dispatch on the same candidate set, including duplicate+ // labels across roots, without searching outside the selected modal.+ const candidates = [];+ const visit = (root) => {+ candidates.push(...root.querySelectorAll(sels));+ if (root.shadowRoot) visit(root.shadowRoot);+ for (const host of root.querySelectorAll('*')) {+ if (host.shadowRoot) visit(host.shadowRoot);+ }+ };+ visit(scope);+ const all = candidates.filter(_keepCandidate);return all.map(e => ({ e, txt: _normTxt(e) })).filter(x => !!x.txt);}@@ -1914,7 +1926,7 @@el = interactive[params.index];if (!el) return { ..._staleIndexError(params.index, interactive), dispatched: false };} else if (params.x != null && params.y != null) {- el = document.elementFromPoint(params.x, params.y);+ el = _shadowAwareElementFromPoint(params.x, params.y);}// A selector miss is a discovery failure, not a changed bound target. Do@@ -4266,7 +4278,7 @@} else if (Number.isInteger(args.index) && args.index >= 0) {target = queryInteractiveForToolIndex()[args.index] || null;} else if (Number.isFinite(args.x) && Number.isFinite(args.y)) {- target = document.elementFromPoint(args.x, args.y);+ target = _shadowAwareElementFromPoint(args.x, args.y);}targetResolved = !!target;}@@ -4466,6 +4478,32 @@}};+ const verifiedLinkedInPostEntry = (clicked) => {+ if (params.adapterName !== 'linkedin' || !/^\/feed\/?$/.test(location.pathname)) return false;+ const button = _composedClosestElement(clicked, 'button,[role="button"]');+ if (!button || !visible(button) || button.disabled+ || button.getAttribute?.('aria-disabled') === 'true'+ || String(button.getAttribute?.('type') || 'button').toLowerCase() !== 'button'+ || button.form || button.hasAttribute?.('form')) return false;+ // The feed entry only opens a public-post composer. Do not infer this+ // from the requested text, a nearby editor, or a control inside a post,+ // conversation, or dialog: those can send or publish existing drafts.+ if (!_composedClosestElement(button, 'main,[role="main"]')+ || _composedClosestElement(button,+ 'form,article,dialog,[role="dialog"],[role="alertdialog"],[role="log"],'+ + '[data-message-id],[data-thread-id],[data-conversation-id],'+ + '[contenteditable]:not([contenteditable="false"])')) return false;+ const labels = [button.innerText || button.textContent, button.getAttribute?.('aria-label')]+ .map(value => compact(value).toLowerCase()).filter(Boolean);+ if (!labels.length || !labels.every(label => label === 'start a post')) return false;+ // A shadow-root dialog may not appear in the document's modal query.+ // Require the actual painted control, descending through open roots.+ const rect = button.getBoundingClientRect();+ const x = rect.left + rect.width / 2, y = rect.top + rect.height / 2;+ const hit = _shadowAwareElementFromPoint(x, y);+ return !!hit && _isComposedAncestor(button, hit);+ };+let composer = null;let messageSend = null;if (observationOnly) {@@ -4506,6 +4544,9 @@if (!visible(control) || (modal && !_isComposedAncestor(modal, target))) {return { success: true, messageSend: null, conclusive: false, identityCandidates: [] };}+ if (verifiedLinkedInPostEntry(target)) {+ return { success: true, messageSend: false, conclusive: true, composerSetup: true, identityCandidates: [] };+ }const linkedInNavigation = classifyLinkedInNavigation(target, modal);if (linkedInNavigation === 'navigation') {return { success: true, messageSend: false, conclusive: true, navigation: true, identityCandidates: [] };diff --git a/test/fixtures/message-recipient-navigation.mjs b/test/fixtures/message-recipient-navigation.mjsindex f24d79afe..314578e07 100644--- a/test/fixtures/message-recipient-navigation.mjs+++ b/test/fixtures/message-recipient-navigation.mjs@@ -52,6 +52,130 @@ export function registerMessageRecipientNavigationFixtures({return { agent, guard, probe };};+ const addPostEntry = async (page, shadow = false) => page.evaluate((shadow) => {+ const host = document.createElement('section');+ host.id = 'post-entry';+ document.querySelector('main').prepend(host);+ const root = shadow ? host.attachShadow({ mode: 'open' }) : host;+ root.innerHTML = '<button id="start-post" type="button" style="padding:12px"><span>Start a post</span></button>';+ const button = root.querySelector('button');+ button.addEventListener('click', () => {+ const dialog = document.createElement('div');+ dialog.id = 'post-composer';+ dialog.setAttribute('role', 'dialog');+ dialog.setAttribute('aria-modal', 'true');+ dialog.style.cssText = 'position:fixed;inset:100px;background:white';+ dialog.innerHTML = '<div contenteditable="true">Draft</div><button type="button">Post</button>';+ document.body.append(dialog);+ });+ const rect = button.getBoundingClientRect();+ return { ref_id: window.__wb_ax_ref(button.querySelector('span')), x: rect.x + 10, y: rect.y + 10 };+ }, shadow);++ register(`${kind}: LinkedIn Start a post opens without a message recipient across targeting methods`, async (page) => {+ const { guard, probe } = await setup(page);+ for (const shadow of [false, true]) {+ const target = await addPostEntry(page, shadow);+ for (const chatOpen of [false, true]) {+ await page.evaluate(open => { document.querySelector('#chat').hidden = !open; }, chatOpen);+ // Both agents expose the full collector, including open shadow roots.+ const elements = await call(page, 'get_interactive_elements_cdp', {});+ const index = elements.find(element => element.text === 'Start a post')?.index;+ assert.ok(Number.isInteger(index), JSON.stringify(elements));+ const strategies = [+ ['click', { text: 'Start a post' }],+ ['click_ax', { ref_id: target.ref_id }],+ ['click', { index }],+ ['click', { x: target.x, y: target.y, coordinate_space: 'css' }],+ ...(!shadow ? [['click', { selector: '#start-post' }]] : []),+ ];+ for (const [tool, args] of strategies) {+ const result = await probe(tool, args);+ assert.equal(result.conclusive, true, JSON.stringify({ shadow, chatOpen, tool, args, result }));+ assert.equal(result.messageSend, false);+ assert.equal(await guard(tool, args), null);+ const clicked = await call(page, tool, args);+ assert.equal(clicked.success, true, JSON.stringify({ shadow, chatOpen, tool, args, clicked }));+ assert.equal(await page.locator('#post-composer').count(), 1,+ JSON.stringify({ shadow, chatOpen, tool, args, clicked }));+ // Opening a composer must not authorize its eventual publication.+ assert.equal((await guard('click', { text: 'Post' }))?.noDispatch, true);+ await page.locator('#post-composer').evaluate(el => el.remove());+ }+ }+ await page.locator('#post-entry').evaluate(el => el.remove());+ }+ });++ register(`${kind}: LinkedIn post entry classification rejects send and publish lookalikes`, async (page) => {+ const { guard } = await setup(page);+ await addPostEntry(page);+ for (const [attribute, value] of [+ ['type', 'submit'], ['aria-label', 'Send'], ['form', 'chat'], ['disabled', ''], ['aria-disabled', 'true'],+ ]) {+ await page.locator('#start-post').evaluate((el, [key, value]) => el.setAttribute(key, value), [attribute, value]);+ assert.equal((await guard('click', { selector: '#start-post' }))?.noDispatch, true, attribute);+ await page.locator('#start-post').evaluate((el, key) => el.removeAttribute(key), attribute);+ }+ for (const label of ['Post', 'Send', 'Start a post and send', 'Photo']) {+ await page.locator('#start-post').evaluate((el, text) => { el.textContent = text; }, label);+ assert.equal((await guard('click', { selector: '#start-post' }))?.noDispatch, true, label);+ }+ await page.locator('#start-post').evaluate(el => { el.textContent = 'Start a post'; });+ await page.evaluate(() => history.replaceState(null, '', '/messaging/'));+ assert.equal((await guard('click', { text: 'Start a post' }))?.noDispatch, true, 'messaging route');+ await page.evaluate(() => history.replaceState(null, '', '/feed/'));+ for (const markup of ['<form></form>', '<article></article>', '<div role="dialog"></div>', '<div role="log"></div>']) {+ await page.evaluate(markup => {+ const wrapper = document.createElement('div');+ wrapper.id = 'entry-wrapper';+ wrapper.innerHTML = markup;+ document.querySelector('main').append(wrapper);+ wrapper.firstChild.append(document.querySelector('#post-entry'));+ }, markup);+ assert.equal((await guard('click', { text: 'Start a post' }))?.noDispatch, true, markup);+ await page.evaluate(() => {+ document.querySelector('main').append(document.querySelector('#post-entry'));+ document.querySelector('#entry-wrapper').remove();+ });+ }+ await page.evaluate(() => document.body.append(document.querySelector('#post-entry')));+ assert.equal((await guard('click', { text: 'Start a post' }))?.noDispatch, true, 'outside feed main');+ assert.deepEqual(await page.evaluate(() => window.fixtureClicks), []);+ });++ register(`${kind}: LinkedIn post entry respects blocking shadow dialogs and ambiguous targets`, async (page) => {+ const { guard } = await setup(page);+ const { ref_id } = await addPostEntry(page, true);+ await page.evaluate(() => {+ const host = document.createElement('div');+ host.id = 'shadow-modal';+ document.body.append(host);+ host.attachShadow({ mode: 'open' }).innerHTML = '<div role="dialog" aria-modal="true" style="position:fixed;inset:0;background:white"><button>Start a post</button></div>';+ });+ assert.equal((await guard('click_ax', { ref_id }))?.noDispatch, true, 'background control');+ assert.equal((await guard('click', { text: 'Start a post' }))?.noDispatch, true, 'dialog control');+ await page.locator('#shadow-modal').evaluate(el => el.remove());+ await page.evaluate(() => document.querySelector('main').insertAdjacentHTML('beforeend', '<button>Start a post</button>'));+ assert.equal((await guard('click', { text: 'Start a post' }))?.noDispatch, true, 'ambiguous controls');+ await page.locator('#post-entry').evaluate(el => el.remove());+ assert.equal((await guard('click_ax', { ref_id }))?.noDispatch, true, 'stale reference');+ });++ register(`${kind}: LinkedIn missing composer does not recommend alternate click retries`, async (page) => {+ const { guard } = await setup(page);+ const result = await guard('click', { selector: '#close-contact-info' });+ // Use a visible unresolved action so this is a classification failure,+ // not a stale/missing target that a fresh page read could repair.+ await page.evaluate(() => document.querySelector('main').insertAdjacentHTML('beforeend', '<button id="unknown">Unknown action</button>'));+ const missingComposer = await guard('click', { selector: '#unknown' });+ assert.equal(missingComposer?.noDispatch, true);+ assert.equal(missingComposer?.reasonCode, 'message_send_classification_inconclusive');+ assert.equal(missingComposer?.retryable, false);+ assert.match(missingComposer?.error, /no message composer/i);+ assert.notEqual(result?.retryable, false, 'unresolved target should retain fresh-target recovery');+ });+register(`${kind}: LinkedIn Home and Jobs navigate with closed and open message composers (#2999)`, async (page) => {const { guard, probe } = await setup(page);for (const open of [false, true]) {