Open-source AI browser agent for Chrome and Firefox (monorepo) π§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Let each Gmail Send and attachment bind to its own message
Emre Sokullu432caabe6138
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.jsindex 093f5da32..40b8baa35 100644--- a/src/chrome/src/content/content.js+++ b/src/chrome/src/content/content.js@@ -7789,8 +7789,176 @@nonMessagingTarget: true, reasonCode: 'non_messaging_target', identityCandidates: [],};}+ // ββ Gmail: the message a composer, a Send or an attachment belongs to ββ+ const gmailRecipientMode = params.adapterName === 'gmail';+ // The To, Cc or Bcc row an element names. A compose window, its+ // chips and its typed addresses below use it.+ const roleFromAttribute = (attribute, value) => {+ const text = compact(value, 120).toLowerCase();+ if (!text) return '';+ // A row's or field's label names its role in the user's language,+ // read by a model; the app's own names below do not.+ if (attribute === 'aria-label') {+ const role = _pageReading('message_field', compact(value, 120));+ return ['to', 'cc', 'bcc'].includes(role) ? role : '';+ }+ if (attribute === 'name') {+ return /^(?:to|cc|bcc)$/.test(text) ? text : '';+ }+ const tokens = text.split(/[^a-z]+/).filter(Boolean);+ if (tokens.includes('bcc')) return 'bcc';+ if (tokens.includes('cc')) return 'cc';+ if (tokens.length === 1 && tokens[0] === 'to') return 'to';+ return '';+ };+ const recipientRole = (el, root = null) => {+ let current = el;+ for (let depth = 0; current && depth < 7; depth++, current = current.parentElement) {+ for (const attribute of ['data-recipient-type', 'data-type', 'name', 'aria-label']) {+ const role = roleFromAttribute(attribute, current.getAttribute?.(attribute));+ if (role) return role;+ }+ if (root && current === root) break;+ try {+ const descendantRoles = new Set();+ for (const marker of current.querySelectorAll?.(+ 'input[name="to"],input[name="cc"],input[name="bcc"],[aria-label]'+ ) || []) {+ const role = roleFromAttribute('name', marker.getAttribute?.('name'))+ || roleFromAttribute('aria-label', marker.getAttribute?.('aria-label'));+ if (role) descendantRoles.add(role);+ }+ // A nearest row containing one role marker can classify its+ // chips. A shared ancestor containing multiple rows cannot.+ if (descendantRoles.size === 1) return [...descendantRoles][0];+ } catch {}+ }+ return '';+ };+ const collectGmailRecipients = (root, editor = composer) => {+ const collected = new Map();+ if (!root?.querySelectorAll) return collected;+ for (const el of root.querySelectorAll('[email],[data-hovercard-id],[data-email]')) {+ if (!visible(el) || editable(el)) continue;+ if (el === editor || editor?.contains?.(el)) continue;+ const email = [+ el.getAttribute?.('email'),+ el.getAttribute?.('data-email'),+ el.getAttribute?.('data-hovercard-id'),+ ].map(value => compact(value, 240)).find(value => /@/.test(value)) || '';+ if (!email) continue;+ const aliases = new Map();+ const addAlias = (val) => {+ const alias = compact(val, 240);+ const normalized = normalizedIdentity(alias);+ if (alias && normalized && !aliases.has(normalized)) aliases.set(normalized, alias);+ const angleMatch = alias.match(/^([^<@]+)<[^>]+>$/);+ if (angleMatch) {+ const nameOnly = compact(angleMatch[1], 240);+ const normName = normalizedIdentity(nameOnly);+ if (nameOnly && normName && !aliases.has(normName)) aliases.set(normName, nameOnly);+ }+ };+ for (const value of [+ email,+ el.getAttribute?.('name'),+ el.getAttribute?.('aria-label'),+ el.getAttribute?.('title'),+ el.innerText,+ el.textContent,+ ]) {+ addAlias(value);+ }+ const key = normalizedIdentity(email);+ if (!key) continue;+ const role = recipientRole(el, root);+ if (!role) continue;+ const recipientKey = `${role}:${key}`;+ const prior = collected.get(recipientKey) || { identity: email, role, aliases: new Map() };+ for (const [normalized, alias] of aliases) prior.aliases.set(normalized, alias);+ collected.set(recipientKey, prior);+ }+ return collected;+ };+ // A message's header field (To, Cc, Bcc or subject) belongs to its+ // compose window and is never a message's body: Gmail's own field+ // names, or a label a model reads as one of those fields.+ const gmailHeaderField = el => {+ if (el?.matches?.('[name="to"],[name="cc"],[name="bcc"],[name="subjectbox"]')) return true;+ const label = compact(el?.getAttribute?.('aria-label'), 120);+ return !!label && ['to', 'cc', 'bcc', 'subject'].includes(_pageReading('message_field', label));+ };+ // Another message's editor than `editor`: not nested with it, not a+ // search or navigation field and not a message's header field.+ const otherMessageEditor = (other, editor) => other !== editor && !editor?.contains?.(other)+ && !other.contains?.(editor) && !verifiedNavigationEditable(other) && !gmailHeaderField(other);+ // What marks a container as holding a message's recipients: Gmail's+ // own row names, a chip in a To/Cc/Bcc row, or a field read as one of+ // those rows, typed into or not.+ const gmailRecipientMarkers = (root, editor) => !!root && (+ !!root.querySelector?.('[name="to"],[name="cc"],[name="bcc"]')+ || collectGmailRecipients(root, editor).size > 0+ || Array.from(root.querySelectorAll?.('input, textarea, [contenteditable="true"], [contenteditable=""]') || [])+ .slice(0, 40).some(input => visible(input) && input !== editor && !editor?.contains?.(input)+ && !!(roleFromAttribute('name', input.getAttribute?.('name'))+ || roleFromAttribute('aria-label', input.getAttribute?.('aria-label')))));+ // In Gmail a send has recipients only from its composer's own mail+ // compose window: a popup or full-screen compose is the editor's+ // dialog (else its form), marked by recipient rows, a subject or+ // Gmail's own compose body; an inline thread reply, which has neither,+ // is the nearest ancestor with recipient rows that stays inside the+ // page's main region and holds no visible window or other message+ // editor. A dialog or form without them (a chat) is none.+ const gmailComposeWindowOf = editor => {+ if (!gmailRecipientMode || !editor) return null;+ const explicit = editor.closest?.('[role="dialog"]') || editor.closest?.('form') || null;+ if (explicit) {+ return gmailRecipientMarkers(explicit, editor) || !!explicit.querySelector?.('input[name="subjectbox"]')+ || editor.getAttribute?.('g_editable') === 'true' ? explicit : null;+ }+ let current = editor.parentElement;+ for (let depth = 0; current && depth < 16; depth += 1, current = current.parentElement) {+ if (current === document.body || current === document.documentElement+ || current.matches?.('main,[role="main"]')+ || Array.from(current.querySelectorAll?.('[role="dialog"]') || []).some(visible)+ || composerCandidates.some(other => current.contains?.(other) && otherMessageEditor(other, editor))) break;+ if (gmailRecipientMarkers(current, editor)) return current;+ }+ return null;+ };+ // Google Chat inside Gmail is a chat: on a Chat route, a composer in+ // the page's main region, in no compose window and not Gmail's mail+ // body, is identified by its own conversation header, as any chat is.+ const gmailChatRoute = gmailRecipientMode && (() => {+ try {+ return location.hostname === 'mail.google.com'+ && (/^\/chat(?:\/|$)/.test(String(location.pathname || ''))+ || /^#chat(?:[/?]|$)/.test(String(location.hash || '')));+ } catch { return false; }+ })();+ const gmailChatMainOf = (editor, ownWindow = gmailComposeWindowOf(editor)) => gmailChatRoute && editor && !ownWindow+ && editor.getAttribute?.('g_editable') !== 'true' && !editor.closest?.('[role="dialog"]')+ ? editor.closest?.('main,[role="main"]') || null : null;+ // The composer a control (a Send, an attachment input) belongs to: the+ // one message editor in the control's nearest ancestor holding any,+ // when that editor is a Gmail compose or Chat composer. Its message,+ // not the page's lowest editor, is what the control sends. None, or+ // more than one there, leaves the pairing to the page's layout.+ const gmailControlComposer = control => {+ if (!gmailRecipientMode || !control) return null;+ for (let node = control; node && node !== document.documentElement; node = node.parentElement) {+ const inside = composerCandidates.filter(editor => node.contains?.(editor)+ && !verifiedNavigationEditable(editor) && !gmailHeaderField(editor));+ if (!inside.length) continue;+ const owners = inside.filter(editor => !inside.some(other => other !== editor && other.contains?.(editor)));+ return owners.length === 1 && (gmailComposeWindowOf(owners[0]) || gmailChatMainOf(owners[0]))+ ? owners[0] : null;+ }+ return null;+ };if (tool === 'upload_file') {- composer = layoutComposer;+ // In Gmail the attachment input's own message decides its composer.+ composer = gmailControlComposer(target) || layoutComposer;const uploadOwner = _messageRecipientUploadOwner(target, composer);const linkedInPrivateComposer = params.adapterName !== 'linkedin'|| (/^\/messaging(?:\/|$)/.test(location.pathname)@@ -7904,7 +8072,9 @@identityCandidates: [],};}- composer = layoutComposer;+ // In Gmail the Send's own message decides its composer; elsewhere,+ // and where no one message owns it, the page's layout pairs them.+ composer = gmailControlComposer(control) || layoutComposer;if (!composer) {const actionLabel = compact(control.getAttribute?.('aria-label')@@ -8048,7 +8218,6 @@const observedRecipientCandidates = [];let conversationRecipientEvidence = null;let nativeChatMessageBaseline = null;- const gmailRecipientMode = params.adapterName === 'gmail';const inConversationHeaderBand = (el) => {if (headerBandBottom <= 0) return false;const rect = el.getBoundingClientRect();@@ -8072,159 +8241,23 @@};let gmailComposeRoot = null;- // ββ Gmail: the message a composer and its Send belong to βββββββββββββ- // The To, Cc or Bcc row an element names. The compose window, its- // chips and its typed addresses below use it.- const roleFromAttribute = (attribute, value) => {- const text = compact(value, 120).toLowerCase();- if (!text) return '';- // A row's or field's label names its role in the user's language,- // read by a model; the app's own names below do not.- if (attribute === 'aria-label') {- const role = _pageReading('message_field', compact(value, 120));- return ['to', 'cc', 'bcc'].includes(role) ? role : '';- }- if (attribute === 'name') {- return /^(?:to|cc|bcc)$/.test(text) ? text : '';- }- const tokens = text.split(/[^a-z]+/).filter(Boolean);- if (tokens.includes('bcc')) return 'bcc';- if (tokens.includes('cc')) return 'cc';- if (tokens.length === 1 && tokens[0] === 'to') return 'to';- return '';- };- const recipientRole = (el, root = null) => {- let current = el;- for (let depth = 0; current && depth < 7; depth++, current = current.parentElement) {- for (const attribute of ['data-recipient-type', 'data-type', 'name', 'aria-label']) {- const role = roleFromAttribute(attribute, current.getAttribute?.(attribute));- if (role) return role;- }- if (root && current === root) break;- try {- const descendantRoles = new Set();- for (const marker of current.querySelectorAll?.(- 'input[name="to"],input[name="cc"],input[name="bcc"],[aria-label]'- ) || []) {- const role = roleFromAttribute('name', marker.getAttribute?.('name'))- || roleFromAttribute('aria-label', marker.getAttribute?.('aria-label'));- if (role) descendantRoles.add(role);- }- // A nearest row containing one role marker can classify its- // chips. A shared ancestor containing multiple rows cannot.- if (descendantRoles.size === 1) return [...descendantRoles][0];- } catch {}- }- return '';- };- const collectGmailRecipients = (root) => {- const collected = new Map();- if (!root?.querySelectorAll) return collected;- for (const el of root.querySelectorAll('[email],[data-hovercard-id],[data-email]')) {- if (!visible(el) || editable(el)) continue;- if (el === composer || composer.contains?.(el)) continue;- const email = [- el.getAttribute?.('email'),- el.getAttribute?.('data-email'),- el.getAttribute?.('data-hovercard-id'),- ].map(value => compact(value, 240)).find(value => /@/.test(value)) || '';- if (!email) continue;- const aliases = new Map();- const addAlias = (val) => {- const alias = compact(val, 240);- const normalized = normalizedIdentity(alias);- if (alias && normalized && !aliases.has(normalized)) aliases.set(normalized, alias);- const angleMatch = alias.match(/^([^<@]+)<[^>]+>$/);- if (angleMatch) {- const nameOnly = compact(angleMatch[1], 240);- const normName = normalizedIdentity(nameOnly);- if (nameOnly && normName && !aliases.has(normName)) aliases.set(normName, nameOnly);- }- };- for (const value of [- email,- el.getAttribute?.('name'),- el.getAttribute?.('aria-label'),- el.getAttribute?.('title'),- el.innerText,- el.textContent,- ]) {- addAlias(value);- }- const key = normalizedIdentity(email);- if (!key) continue;- const role = recipientRole(el, root);- if (!role) continue;- const recipientKey = `${role}:${key}`;- const prior = collected.get(recipientKey) || { identity: email, role, aliases: new Map() };- for (const [normalized, alias] of aliases) prior.aliases.set(normalized, alias);- collected.set(recipientKey, prior);- }- return collected;- };- // Another message's editor: not this composer, not nested with it,- // not a search or navigation field and not a To/Cc/Bcc row's field.- const otherMessageEditor = editor => editor !== composer && !composer.contains?.(editor)- && !editor.contains?.(composer) && !verifiedNavigationEditable(editor)- && !editor.matches?.('[name="to"],[name="cc"],[name="bcc"]');- // In Gmail a send has recipients only from its composer's own mail- // compose window: a popup or full-screen compose is the composer's- // dialog (else its form), and an inline thread reply, which has- // neither, the nearest ancestor with To/Cc/Bcc chips that stays inside- // the page's main region and holds no other window or message editor.- // Gmail's own field names, its compose body, a chip in a row or a- // field read as a row mark that window; one without them (a chat) is- // none.- const gmailComposeWindow = (() => {- if (!gmailRecipientMode) return null;- const recipientRows = root => !!root && (- !!root.querySelector?.('[name="to"],[name="cc"],[name="bcc"],input[name="subjectbox"]')- || composer.getAttribute?.('g_editable') === 'true'- || collectGmailRecipients(root).size > 0- || Array.from(root.querySelectorAll?.('input, textarea, [contenteditable="true"], [contenteditable=""]') || [])- .slice(0, 40).some(input => visible(input) && input !== composer && !composer.contains?.(input)- && !!(roleFromAttribute('name', input.getAttribute?.('name'))- || roleFromAttribute('aria-label', input.getAttribute?.('aria-label')))));- const explicit = composer.closest?.('[role="dialog"]') || composer.closest?.('form') || null;- if (explicit) return recipientRows(explicit) ? explicit : null;- let current = composer.parentElement;- for (let depth = 0; current && depth < 16; depth += 1, current = current.parentElement) {- if (current === document.body || current === document.documentElement- || current.matches?.('main,[role="main"]') || current.querySelector?.('[role="dialog"]')- || composerCandidates.some(editor => current.contains?.(editor) && otherMessageEditor(editor))) break;- if (collectGmailRecipients(current).size > 0) return current;- }- return null;- })();- // The control that sends must be this composer's own: the two share an+ // The composer's own Gmail compose window, and whether what sends (a+ // Send, an attachment input) is that composer's own: the two share an// owner holding no message editor beyond this composer's own window.- // A Send the page's layout pairs with another message's composer (a- // Chat's Send beside a compose window's body, or the reverse) belongs- // to that other message.+ // A control the page's layout pairs with another message's composer+ // belongs to that other message.+ const gmailComposeWindow = gmailComposeWindowOf(composer);let gmailSendOwner = null;- if (gmailRecipientMode && (tool === 'click' || tool === 'click_ax') && dispatchTarget- && dispatchTarget !== composer && !composer.contains?.(dispatchTarget)) {+ if (gmailRecipientMode && dispatchTarget && dispatchTarget !== composer && !composer.contains?.(dispatchTarget)) {for (let node = composer.parentElement; node; node = node.parentElement) {if (node.contains?.(dispatchTarget)) { gmailSendOwner = node; break; }}}const gmailEditorScope = composer.closest?.('[role="dialog"]') || gmailComposeWindow || composer;const gmailForeignSend = !!gmailSendOwner && composerCandidates.some(editor => gmailSendOwner.contains(editor)- && !gmailEditorScope.contains?.(editor) && otherMessageEditor(editor));+ && !gmailEditorScope.contains?.(editor) && otherMessageEditor(editor, composer));const gmailMailSend = !!gmailComposeWindow && !gmailForeignSend;- // Google Chat inside Gmail is a chat: on a Chat route, a composer in- // the page's main region and in no compose window is identified by its- // own conversation header, as any chat is (below).- const gmailChatRoute = gmailRecipientMode && (() => {- try {- return location.hostname === 'mail.google.com'- && (/^\/chat(?:\/|$)/.test(String(location.pathname || ''))- || /^#chat(?:[/?]|$)/.test(String(location.hash || '')));- } catch { return false; }- })();- const gmailChatMain = gmailChatRoute && !gmailComposeWindow && !gmailForeignSend- && composer.getAttribute?.('g_editable') !== 'true' && !composer.closest?.('[role="dialog"]')- ? composer.closest?.('main,[role="main"]') || null : null;+ const gmailChatMain = gmailForeignSend ? null : gmailChatMainOf(composer, gmailComposeWindow);const gmailChatComposer = !!gmailChatMain;// Gmail's recipient sets come from that window alone, on any route.const recipientSets = gmailRecipientMode ? gmailMailSend : params.supportsRecipientSets === true;diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.jsindex fbb39bfa8..657251a65 100644--- a/src/firefox/src/content/content.js+++ b/src/firefox/src/content/content.js@@ -6970,8 +6970,176 @@nonMessagingTarget: true, reasonCode: 'non_messaging_target', identityCandidates: [],};}+ // ββ Gmail: the message a composer, a Send or an attachment belongs to ββ+ const gmailRecipientMode = params.adapterName === 'gmail';+ // The To, Cc or Bcc row an element names. A compose window, its+ // chips and its typed addresses below use it.+ const roleFromAttribute = (attribute, value) => {+ const text = compact(value, 120).toLowerCase();+ if (!text) return '';+ // A row's or field's label names its role in the user's language,+ // read by a model; the app's own names below do not.+ if (attribute === 'aria-label') {+ const role = _pageReading('message_field', compact(value, 120));+ return ['to', 'cc', 'bcc'].includes(role) ? role : '';+ }+ if (attribute === 'name') {+ return /^(?:to|cc|bcc)$/.test(text) ? text : '';+ }+ const tokens = text.split(/[^a-z]+/).filter(Boolean);+ if (tokens.includes('bcc')) return 'bcc';+ if (tokens.includes('cc')) return 'cc';+ if (tokens.length === 1 && tokens[0] === 'to') return 'to';+ return '';+ };+ const recipientRole = (el, root = null) => {+ let current = el;+ for (let depth = 0; current && depth < 7; depth++, current = current.parentElement) {+ for (const attribute of ['data-recipient-type', 'data-type', 'name', 'aria-label']) {+ const role = roleFromAttribute(attribute, current.getAttribute?.(attribute));+ if (role) return role;+ }+ if (root && current === root) break;+ try {+ const descendantRoles = new Set();+ for (const marker of current.querySelectorAll?.(+ 'input[name="to"],input[name="cc"],input[name="bcc"],[aria-label]'+ ) || []) {+ const role = roleFromAttribute('name', marker.getAttribute?.('name'))+ || roleFromAttribute('aria-label', marker.getAttribute?.('aria-label'));+ if (role) descendantRoles.add(role);+ }+ // A nearest row containing one role marker can classify its+ // chips. A shared ancestor containing multiple rows cannot.+ if (descendantRoles.size === 1) return [...descendantRoles][0];+ } catch {}+ }+ return '';+ };+ const collectGmailRecipients = (root, editor = composer) => {+ const collected = new Map();+ if (!root?.querySelectorAll) return collected;+ for (const el of root.querySelectorAll('[email],[data-hovercard-id],[data-email]')) {+ if (!visible(el) || editable(el)) continue;+ if (el === editor || editor?.contains?.(el)) continue;+ const email = [+ el.getAttribute?.('email'),+ el.getAttribute?.('data-email'),+ el.getAttribute?.('data-hovercard-id'),+ ].map(value => compact(value, 240)).find(value => /@/.test(value)) || '';+ if (!email) continue;+ const aliases = new Map();+ const addAlias = (val) => {+ const alias = compact(val, 240);+ const normalized = normalizedIdentity(alias);+ if (alias && normalized && !aliases.has(normalized)) aliases.set(normalized, alias);+ const angleMatch = alias.match(/^([^<@]+)<[^>]+>$/);+ if (angleMatch) {+ const nameOnly = compact(angleMatch[1], 240);+ const normName = normalizedIdentity(nameOnly);+ if (nameOnly && normName && !aliases.has(normName)) aliases.set(normName, nameOnly);+ }+ };+ for (const value of [+ email,+ el.getAttribute?.('name'),+ el.getAttribute?.('aria-label'),+ el.getAttribute?.('title'),+ el.innerText,+ el.textContent,+ ]) {+ addAlias(value);+ }+ const key = normalizedIdentity(email);+ if (!key) continue;+ const role = recipientRole(el, root);+ if (!role) continue;+ const recipientKey = `${role}:${key}`;+ const prior = collected.get(recipientKey) || { identity: email, role, aliases: new Map() };+ for (const [normalized, alias] of aliases) prior.aliases.set(normalized, alias);+ collected.set(recipientKey, prior);+ }+ return collected;+ };+ // A message's header field (To, Cc, Bcc or subject) belongs to its+ // compose window and is never a message's body: Gmail's own field+ // names, or a label a model reads as one of those fields.+ const gmailHeaderField = el => {+ if (el?.matches?.('[name="to"],[name="cc"],[name="bcc"],[name="subjectbox"]')) return true;+ const label = compact(el?.getAttribute?.('aria-label'), 120);+ return !!label && ['to', 'cc', 'bcc', 'subject'].includes(_pageReading('message_field', label));+ };+ // Another message's editor than `editor`: not nested with it, not a+ // search or navigation field and not a message's header field.+ const otherMessageEditor = (other, editor) => other !== editor && !editor?.contains?.(other)+ && !other.contains?.(editor) && !verifiedNavigationEditable(other) && !gmailHeaderField(other);+ // What marks a container as holding a message's recipients: Gmail's+ // own row names, a chip in a To/Cc/Bcc row, or a field read as one of+ // those rows, typed into or not.+ const gmailRecipientMarkers = (root, editor) => !!root && (+ !!root.querySelector?.('[name="to"],[name="cc"],[name="bcc"]')+ || collectGmailRecipients(root, editor).size > 0+ || Array.from(root.querySelectorAll?.('input, textarea, [contenteditable="true"], [contenteditable=""]') || [])+ .slice(0, 40).some(input => visible(input) && input !== editor && !editor?.contains?.(input)+ && !!(roleFromAttribute('name', input.getAttribute?.('name'))+ || roleFromAttribute('aria-label', input.getAttribute?.('aria-label')))));+ // In Gmail a send has recipients only from its composer's own mail+ // compose window: a popup or full-screen compose is the editor's+ // dialog (else its form), marked by recipient rows, a subject or+ // Gmail's own compose body; an inline thread reply, which has neither,+ // is the nearest ancestor with recipient rows that stays inside the+ // page's main region and holds no visible window or other message+ // editor. A dialog or form without them (a chat) is none.+ const gmailComposeWindowOf = editor => {+ if (!gmailRecipientMode || !editor) return null;+ const explicit = editor.closest?.('[role="dialog"]') || editor.closest?.('form') || null;+ if (explicit) {+ return gmailRecipientMarkers(explicit, editor) || !!explicit.querySelector?.('input[name="subjectbox"]')+ || editor.getAttribute?.('g_editable') === 'true' ? explicit : null;+ }+ let current = editor.parentElement;+ for (let depth = 0; current && depth < 16; depth += 1, current = current.parentElement) {+ if (current === document.body || current === document.documentElement+ || current.matches?.('main,[role="main"]')+ || Array.from(current.querySelectorAll?.('[role="dialog"]') || []).some(visible)+ || composerCandidates.some(other => current.contains?.(other) && otherMessageEditor(other, editor))) break;+ if (gmailRecipientMarkers(current, editor)) return current;+ }+ return null;+ };+ // Google Chat inside Gmail is a chat: on a Chat route, a composer in+ // the page's main region, in no compose window and not Gmail's mail+ // body, is identified by its own conversation header, as any chat is.+ const gmailChatRoute = gmailRecipientMode && (() => {+ try {+ return location.hostname === 'mail.google.com'+ && (/^\/chat(?:\/|$)/.test(String(location.pathname || ''))+ || /^#chat(?:[/?]|$)/.test(String(location.hash || '')));+ } catch { return false; }+ })();+ const gmailChatMainOf = (editor, ownWindow = gmailComposeWindowOf(editor)) => gmailChatRoute && editor && !ownWindow+ && editor.getAttribute?.('g_editable') !== 'true' && !editor.closest?.('[role="dialog"]')+ ? editor.closest?.('main,[role="main"]') || null : null;+ // The composer a control (a Send, an attachment input) belongs to: the+ // one message editor in the control's nearest ancestor holding any,+ // when that editor is a Gmail compose or Chat composer. Its message,+ // not the page's lowest editor, is what the control sends. None, or+ // more than one there, leaves the pairing to the page's layout.+ const gmailControlComposer = control => {+ if (!gmailRecipientMode || !control) return null;+ for (let node = control; node && node !== document.documentElement; node = node.parentElement) {+ const inside = composerCandidates.filter(editor => node.contains?.(editor)+ && !verifiedNavigationEditable(editor) && !gmailHeaderField(editor));+ if (!inside.length) continue;+ const owners = inside.filter(editor => !inside.some(other => other !== editor && other.contains?.(editor)));+ return owners.length === 1 && (gmailComposeWindowOf(owners[0]) || gmailChatMainOf(owners[0]))+ ? owners[0] : null;+ }+ return null;+ };if (tool === 'upload_file') {- composer = layoutComposer;+ // In Gmail the attachment input's own message decides its composer.+ composer = gmailControlComposer(target) || layoutComposer;const uploadOwner = _messageRecipientUploadOwner(target, composer);const linkedInPrivateComposer = params.adapterName !== 'linkedin'|| (/^\/messaging(?:\/|$)/.test(location.pathname)@@ -7071,7 +7239,9 @@identityCandidates: [],};}- composer = layoutComposer;+ // In Gmail the Send's own message decides its composer; elsewhere,+ // and where no one message owns it, the page's layout pairs them.+ composer = gmailControlComposer(control) || layoutComposer;if (!composer) {const actionLabel = compact(control.getAttribute?.('aria-label')@@ -7202,7 +7372,6 @@const observedRecipientCandidates = [];let conversationRecipientEvidence = null;let nativeChatMessageBaseline = null;- const gmailRecipientMode = params.adapterName === 'gmail';const inConversationHeaderBand = (el) => {if (headerBandBottom <= 0) return false;const rect = el.getBoundingClientRect();@@ -7226,159 +7395,23 @@};let gmailComposeRoot = null;- // ββ Gmail: the message a composer and its Send belong to βββββββββββββ- // The To, Cc or Bcc row an element names. The compose window, its- // chips and its typed addresses below use it.- const roleFromAttribute = (attribute, value) => {- const text = compact(value, 120).toLowerCase();- if (!text) return '';- // A row's or field's label names its role in the user's language,- // read by a model; the app's own names below do not.- if (attribute === 'aria-label') {- const role = _pageReading('message_field', compact(value, 120));- return ['to', 'cc', 'bcc'].includes(role) ? role : '';- }- if (attribute === 'name') {- return /^(?:to|cc|bcc)$/.test(text) ? text : '';- }- const tokens = text.split(/[^a-z]+/).filter(Boolean);- if (tokens.includes('bcc')) return 'bcc';- if (tokens.includes('cc')) return 'cc';- if (tokens.length === 1 && tokens[0] === 'to') return 'to';- return '';- };- const recipientRole = (el, root = null) => {- let current = el;- for (let depth = 0; current && depth < 7; depth++, current = current.parentElement) {- for (const attribute of ['data-recipient-type', 'data-type', 'name', 'aria-label']) {- const role = roleFromAttribute(attribute, current.getAttribute?.(attribute));- if (role) return role;- }- if (root && current === root) break;- try {- const descendantRoles = new Set();- for (const marker of current.querySelectorAll?.(- 'input[name="to"],input[name="cc"],input[name="bcc"],[aria-label]'- ) || []) {- const role = roleFromAttribute('name', marker.getAttribute?.('name'))- || roleFromAttribute('aria-label', marker.getAttribute?.('aria-label'));- if (role) descendantRoles.add(role);- }- // A nearest row containing one role marker can classify its- // chips. A shared ancestor containing multiple rows cannot.- if (descendantRoles.size === 1) return [...descendantRoles][0];- } catch {}- }- return '';- };- const collectGmailRecipients = (root) => {- const collected = new Map();- if (!root?.querySelectorAll) return collected;- for (const el of root.querySelectorAll('[email],[data-hovercard-id],[data-email]')) {- if (!visible(el) || editable(el)) continue;- if (el === composer || composer.contains?.(el)) continue;- const email = [- el.getAttribute?.('email'),- el.getAttribute?.('data-email'),- el.getAttribute?.('data-hovercard-id'),- ].map(value => compact(value, 240)).find(value => /@/.test(value)) || '';- if (!email) continue;- const aliases = new Map();- const addAlias = (val) => {- const alias = compact(val, 240);- const normalized = normalizedIdentity(alias);- if (alias && normalized && !aliases.has(normalized)) aliases.set(normalized, alias);- const angleMatch = alias.match(/^([^<@]+)<[^>]+>$/);- if (angleMatch) {- const nameOnly = compact(angleMatch[1], 240);- const normName = normalizedIdentity(nameOnly);- if (nameOnly && normName && !aliases.has(normName)) aliases.set(normName, nameOnly);- }- };- for (const value of [- email,- el.getAttribute?.('name'),- el.getAttribute?.('aria-label'),- el.getAttribute?.('title'),- el.innerText,- el.textContent,- ]) {- addAlias(value);- }- const key = normalizedIdentity(email);- if (!key) continue;- const role = recipientRole(el, root);- if (!role) continue;- const recipientKey = `${role}:${key}`;- const prior = collected.get(recipientKey) || { identity: email, role, aliases: new Map() };- for (const [normalized, alias] of aliases) prior.aliases.set(normalized, alias);- collected.set(recipientKey, prior);- }- return collected;- };- // Another message's editor: not this composer, not nested with it,- // not a search or navigation field and not a To/Cc/Bcc row's field.- const otherMessageEditor = editor => editor !== composer && !composer.contains?.(editor)- && !editor.contains?.(composer) && !verifiedNavigationEditable(editor)- && !editor.matches?.('[name="to"],[name="cc"],[name="bcc"]');- // In Gmail a send has recipients only from its composer's own mail- // compose window: a popup or full-screen compose is the composer's- // dialog (else its form), and an inline thread reply, which has- // neither, the nearest ancestor with To/Cc/Bcc chips that stays inside- // the page's main region and holds no other window or message editor.- // Gmail's own field names, its compose body, a chip in a row or a- // field read as a row mark that window; one without them (a chat) is- // none.- const gmailComposeWindow = (() => {- if (!gmailRecipientMode) return null;- const recipientRows = root => !!root && (- !!root.querySelector?.('[name="to"],[name="cc"],[name="bcc"],input[name="subjectbox"]')- || composer.getAttribute?.('g_editable') === 'true'- || collectGmailRecipients(root).size > 0- || Array.from(root.querySelectorAll?.('input, textarea, [contenteditable="true"], [contenteditable=""]') || [])- .slice(0, 40).some(input => visible(input) && input !== composer && !composer.contains?.(input)- && !!(roleFromAttribute('name', input.getAttribute?.('name'))- || roleFromAttribute('aria-label', input.getAttribute?.('aria-label')))));- const explicit = composer.closest?.('[role="dialog"]') || composer.closest?.('form') || null;- if (explicit) return recipientRows(explicit) ? explicit : null;- let current = composer.parentElement;- for (let depth = 0; current && depth < 16; depth += 1, current = current.parentElement) {- if (current === document.body || current === document.documentElement- || current.matches?.('main,[role="main"]') || current.querySelector?.('[role="dialog"]')- || composerCandidates.some(editor => current.contains?.(editor) && otherMessageEditor(editor))) break;- if (collectGmailRecipients(current).size > 0) return current;- }- return null;- })();- // The control that sends must be this composer's own: the two share an+ // The composer's own Gmail compose window, and whether what sends (a+ // Send, an attachment input) is that composer's own: the two share an// owner holding no message editor beyond this composer's own window.- // A Send the page's layout pairs with another message's composer (a- // Chat's Send beside a compose window's body, or the reverse) belongs- // to that other message.+ // A control the page's layout pairs with another message's composer+ // belongs to that other message.+ const gmailComposeWindow = gmailComposeWindowOf(composer);let gmailSendOwner = null;- if (gmailRecipientMode && (tool === 'click' || tool === 'click_ax') && dispatchTarget- && dispatchTarget !== composer && !composer.contains?.(dispatchTarget)) {+ if (gmailRecipientMode && dispatchTarget && dispatchTarget !== composer && !composer.contains?.(dispatchTarget)) {for (let node = composer.parentElement; node; node = node.parentElement) {if (node.contains?.(dispatchTarget)) { gmailSendOwner = node; break; }}}const gmailEditorScope = composer.closest?.('[role="dialog"]') || gmailComposeWindow || composer;const gmailForeignSend = !!gmailSendOwner && composerCandidates.some(editor => gmailSendOwner.contains(editor)- && !gmailEditorScope.contains?.(editor) && otherMessageEditor(editor));+ && !gmailEditorScope.contains?.(editor) && otherMessageEditor(editor, composer));const gmailMailSend = !!gmailComposeWindow && !gmailForeignSend;- // Google Chat inside Gmail is a chat: on a Chat route, a composer in- // the page's main region and in no compose window is identified by its- // own conversation header, as any chat is (below).- const gmailChatRoute = gmailRecipientMode && (() => {- try {- return location.hostname === 'mail.google.com'- && (/^\/chat(?:\/|$)/.test(String(location.pathname || ''))- || /^#chat(?:[/?]|$)/.test(String(location.hash || '')));- } catch { return false; }- })();- const gmailChatMain = gmailChatRoute && !gmailComposeWindow && !gmailForeignSend- && composer.getAttribute?.('g_editable') !== 'true' && !composer.closest?.('[role="dialog"]')- ? composer.closest?.('main,[role="main"]') || null : null;+ const gmailChatMain = gmailForeignSend ? null : gmailChatMainOf(composer, gmailComposeWindow);const gmailChatComposer = !!gmailChatMain;// Gmail's recipient sets come from that window alone, on any route.const recipientSets = gmailRecipientMode ? gmailMailSend : params.supportsRecipientSets === true;diff --git a/test/improver-content-recipient.mjs b/test/improver-content-recipient.mjsindex bd9d74032..fb03e2915 100644--- a/test/improver-content-recipient.mjs+++ b/test/improver-content-recipient.mjs@@ -65,14 +65,16 @@ const GMAIL_COMPOSE_OVER_CHAT = `<!doctype html><style>body{margin:0;font:16px sconst TO_ADA = '<div name="to"><span email="[email protected]">Ada</span></div>';const CC_CEM = '<div name="cc"><span email="[email protected]">Cem</span></div>';const BCC_DEE = '<div name="bcc"><span email="[email protected]">Dee</span></div>';-const gmailChat = ({ rows = '', raise = 0 } = {}) => `<!doctype html><style>body{margin:0;font:16px sans-serif}+const TO_FIELD = value => `<input id="to-input" role="combobox" aria-label="To recipients" value="${value}">`;+const gmailChat = ({ rows = '', raise = 0, chatFile = false, appFile = false } = {}) => `<!doctype html><style>body{margin:0;font:16px sans-serif}#chat{position:absolute;left:0;top:0;width:640px;height:780px}#chat>header{height:70px}#messages{height:580px;overflow:auto}footer{display:flex;gap:15px;align-items:center}#body{width:500px;min-height:32px}button{min-height:28px}[role=dialog]{position:absolute;left:660px;bottom:${raise}px;width:560px}</style><div id="app"><main id="chat" role="main"><header><h2>Bob</h2></header><section id="messages"><div class="msg">Bob: please also write to <a href="mailto:[email protected]">[email protected]</a></div></section><footer><div id="body" contenteditable="true" role="textbox" aria-label="Type a message">Hi Bob</div>- <button id="send" type="button" aria-label="Send">Send</button></footer></main>+ ${chatFile ? '<input id="chat-file" type="file">' : ''}<button id="send" type="button" aria-label="Send">Send</button></footer></main>+ ${appFile ? '<input id="app-file" type="file">' : ''}${rows ? `<div role="dialog" aria-label="New Message"><form onsubmit="return false">${rows}<div contenteditable="true" role="textbox" aria-label="Message Body">Draft</div><button id="mail-send" type="button" aria-label="Send">Send</button></form></div>` : ''}</div>`;@@ -338,26 +340,58 @@ for (const [build, Agent] of [['chrome', ChromeAgent], ['firefox', FirefoxAgent]});}- test(`${build}: beside an open compose window a Gmail Chat send binds to its own header, and the window's Send is not the chat's`, async () => {- const url = 'https://mail.google.com/mail/u/0/#chat/dm/abc?compose=new';- const f = await fixture(build, Agent, { url, recipients: ['Bob'], html: gmailChat({ rows: TO_ADA, raise: 150 }) });+ // Each Send sends its own message, whichever editor is the page's lowest:+ // the compose window's Send goes to its own To row, the Chat's Send to the+ // chat's person and never to the window's recipients.+ for (const raise of [0, 150]) {+ test(`${build}: beside an open compose window (${raise ? 'raised' : 'docked'}) each Send binds to its own message`, async () => {+ const url = 'https://mail.google.com/mail/u/0/#chat/dm/abc?compose=new';+ const mailSend = { selector: '#mail-send' };+ const f = await fixture(build, Agent, { url, recipients: ['[email protected]'], html: gmailChat({ rows: TO_ADA, raise }) });+ try {+ await f.page.evaluate(() => {+ window.mailSent = 0;+ document.querySelector('#mail-send').addEventListener('click', () => window.mailSent++);+ });+ const chat = await f.bind();+ assert.equal(chat.blocked?.success, false, JSON.stringify(chat.blocked));+ assert.deepEqual(chat.blocked.recipientConfirmation?.recipients, [{ identity: 'Bob', role: 'to' }], JSON.stringify(chat.blocked));+ assert.equal(chat.context.messageRecipientDispatchBinding, undefined);+ const mailContext = {};+ const mailBlocked = await f.agent._messageRecipientGuardBlock(71, 'click', mailSend, url, mailContext);+ assert.equal(mailBlocked, null, JSON.stringify(mailBlocked));+ const result = await call(f.page, 'click', { ...mailSend, ...mailContext });+ assert.equal(result.success, true, JSON.stringify(result));+ assert.equal(await f.page.evaluate(() => [window.sent, window.mailSent]).then(String), '0,1');+ } finally { await f.browser.close(); }+ const bob = await fixture(build, Agent, { url, recipients: ['Bob'], html: gmailChat({ rows: TO_ADA, raise }) });+ try {+ const { blocked, context } = await bob.bind();+ assert.equal(blocked, null, JSON.stringify(blocked));+ const result = await call(bob.page, 'click', { ...sendArgs, ...context });+ assert.equal(result.success, true, JSON.stringify(result));+ assert.equal(await bob.page.evaluate(() => window.sent), 1);+ } finally { await bob.browser.close(); }+ });+ }++ // An attachment goes with the message its file input belongs to: the+ // Chat's input to the chat's person, an input no one message owns to no+ // one, and neither to a compose window's recipients beside it.+ test(`${build}: a file attached in a Gmail Chat never goes to a compose window's recipients`, async () => {+ const recipients = [{ identity: '[email protected]', role: 'to' }, { identity: '[email protected]', role: 'cc' }];+ const f = await fixture(build, Agent, { url: 'https://mail.google.com/mail/u/0/#chat/dm/abc', recipients,+ html: gmailChat({ rows: TO_ADA + CC_CEM, chatFile: true, appFile: true }) });try {- await f.page.evaluate(() => {- window.mailSent = 0;- document.querySelector('#mail-send').addEventListener('click', () => window.mailSent++);- });- // The layout pairs the window's Send with the chat's composer: it- // identifies no one, so neither the chat's person nor the window's.- const mailContext = {};- const mailBlocked = await f.agent._messageRecipientGuardBlock(71, 'click', { selector: '#mail-send' }, url, mailContext);- assert.equal(mailBlocked?.success, false, JSON.stringify(mailBlocked));- assert.equal(mailContext.messageRecipientDispatchBinding, undefined);- assert.equal(await f.page.evaluate(() => window.mailSent), 0);- const { blocked, context } = await f.bind();- assert.equal(blocked, null, JSON.stringify(blocked));- const result = await call(f.page, 'click', { ...sendArgs, ...context });- assert.equal(result.success, true, JSON.stringify(result));- assert.equal(await f.page.evaluate(() => [window.sent, window.mailSent]).then(String), '1,0');+ const upload = selector => f.probe({ adapterName: 'gmail', tool: 'upload_file', args: { selector },+ uploadSourceKey: 'download:1', expectedRecipients: recipients });+ const chatUpload = await upload('#chat-file');+ assert.equal(chatUpload.recipientSets, undefined, JSON.stringify(chatUpload));+ assert.deepEqual(chatUpload.strongRecipientCandidates, [{ identity: 'Bob', role: 'to' }], JSON.stringify(chatUpload));+ const appUpload = await upload('#app-file');+ assert.equal(appUpload.recipientSets, undefined, JSON.stringify(appUpload));+ assert.deepEqual(appUpload.strongRecipientCandidates || [], [], JSON.stringify(appUpload));+ assert.equal(appUpload.messageRecipientDispatchBinding, undefined);} finally { await f.browser.close(); }});@@ -399,6 +433,51 @@ for (const [build, Agent] of [['chrome', ChromeAgent], ['firefox', FirefoxAgent]});}+ // Gmail's To field is a combobox named by its label: focused, typed into+ // and not yet a chip, it is still the reply's own row, and a hidden+ // dialog inside the reply does not end it.+ for (const [kind, rows, focus, extra] of [+ ['a chip and the focused To field', TO_ADA + TO_FIELD(''), '#to-input', ''],+ ['a typed To address, the body focused', TO_FIELD('[email protected]'), '#body', ''],+ ['a typed To address, the To field focused', TO_FIELD('[email protected]'), '#to-input', ''],+ ['a hidden dialog inside it', TO_ADA, '#body', '<div role="dialog" aria-label="Insert link" style="display:none"></div>'],+ ]) {+ test(`${build}: a Gmail inline reply with ${kind} binds to its own To`, async () => {+ const html = `<!doctype html><style>body{margin:0;font:16px sans-serif}main{position:absolute;left:0;top:0;width:900px;height:780px}+ .reply{position:absolute;left:0;top:560px}#body{display:block;width:560px;min-height:40px}</style>+ <main role="main"><h2>Invoice</h2><div class="message">Ada: here is the invoice</div>+ <div class="reply">${extra}<div class="rows">${rows}</div>+ <div id="body" g_editable="true" contenteditable="true" role="textbox" aria-label="Message Body">Thanks Ada</div>+ <div class="toolbar"><button id="send" type="button" aria-label="Send">Send</button></div></div></main>`;+ const f = await fixture(build, Agent, { url: 'https://mail.google.com/mail/u/0/#inbox/FMfc123', recipients: ['[email protected]'], html });+ try {+ await f.page.locator(focus).focus();+ const { blocked, context } = await f.bind();+ assert.equal(blocked, null, JSON.stringify(blocked));+ const result = await call(f.page, 'click', { ...sendArgs, ...context });+ assert.equal(result.success, true, JSON.stringify(result));+ assert.equal(await f.page.evaluate(() => window.sent), 1);+ } finally { await f.browser.close(); }+ });+ }++ test(`${build}: a Gmail popup compose with a typed To address binds with the To field focused`, async () => {+ const html = `<!doctype html><style>body{margin:0;font:16px sans-serif}[role=dialog]{position:absolute;right:20px;bottom:0;width:560px}+ #body{display:block;width:500px;min-height:60px}</style><main role="main"><h2>Inbox</h2></main>+ <div role="dialog" aria-label="New Message">${TO_FIELD('[email protected]')}<input name="subjectbox" value="Hi">+ <div id="body" g_editable="true" contenteditable="true" role="textbox" aria-label="Message Body">Hello</div>+ <button id="send" type="button" aria-label="Send">Send</button></div>`;+ const f = await fixture(build, Agent, { url: 'https://mail.google.com/mail/u/0/#inbox?compose=new', recipients: ['[email protected]'], html });+ try {+ await f.page.locator('#to-input').focus();+ const { blocked, context } = await f.bind();+ assert.equal(blocked, null, JSON.stringify(blocked));+ const result = await call(f.page, 'click', { ...sendArgs, ...context });+ assert.equal(result.success, true, JSON.stringify(result));+ assert.equal(await f.page.evaluate(() => window.sent), 1);+ } finally { await f.browser.close(); }+ });+test(`${build}: an ordinary Gmail compose still binds and sends to its own To, Cc and Bcc`, async () => {const recipients = [{ identity: '[email protected]', role: 'to' }, { identity: '[email protected]', role: 'cc' },{ identity: '[email protected]', role: 'bcc' }];