Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Let each Gmail Send and attachment bind to its own message

Emre Sokullu432caabe6138
diff --git a/src/chrome/src/content/content.js b/src/chrome/src/content/content.js
index 093f5da32..40b8baa35 100644
--- a/src/chrome/src/content/content.js
+++ b/src/chrome/src/content/content.js
@@ -7789,8 +7789,176 @@
nonMessagingTarget: true, reasonCode: 'non_messaging_target', identityCandidates: [],
};
}
+ // ── Gmail: the message a composer, a Send or an attachment belongs to ──
+ const gmailRecipientMode = params.adapterName === 'gmail';
+ // The To, Cc or Bcc row an element names. A compose window, its
+ // chips and its typed addresses below use it.
+ const roleFromAttribute = (attribute, value) => {
+ const text = compact(value, 120).toLowerCase();
+ if (!text) return '';
+ // A row's or field's label names its role in the user's language,
+ // read by a model; the app's own names below do not.
+ if (attribute === 'aria-label') {
+ const role = _pageReading('message_field', compact(value, 120));
+ return ['to', 'cc', 'bcc'].includes(role) ? role : '';
+ }
+ if (attribute === 'name') {
+ return /^(?:to|cc|bcc)$/.test(text) ? text : '';
+ }
+ const tokens = text.split(/[^a-z]+/).filter(Boolean);
+ if (tokens.includes('bcc')) return 'bcc';
+ if (tokens.includes('cc')) return 'cc';
+ if (tokens.length === 1 && tokens[0] === 'to') return 'to';
+ return '';
+ };
+ const recipientRole = (el, root = null) => {
+ let current = el;
+ for (let depth = 0; current && depth < 7; depth++, current = current.parentElement) {
+ for (const attribute of ['data-recipient-type', 'data-type', 'name', 'aria-label']) {
+ const role = roleFromAttribute(attribute, current.getAttribute?.(attribute));
+ if (role) return role;
+ }
+ if (root && current === root) break;
+ try {
+ const descendantRoles = new Set();
+ for (const marker of current.querySelectorAll?.(
+ 'input[name="to"],input[name="cc"],input[name="bcc"],[aria-label]'
+ ) || []) {
+ const role = roleFromAttribute('name', marker.getAttribute?.('name'))
+ || roleFromAttribute('aria-label', marker.getAttribute?.('aria-label'));
+ if (role) descendantRoles.add(role);
+ }
+ // A nearest row containing one role marker can classify its
+ // chips. A shared ancestor containing multiple rows cannot.
+ if (descendantRoles.size === 1) return [...descendantRoles][0];
+ } catch {}
+ }
+ return '';
+ };
+ const collectGmailRecipients = (root, editor = composer) => {
+ const collected = new Map();
+ if (!root?.querySelectorAll) return collected;
+ for (const el of root.querySelectorAll('[email],[data-hovercard-id],[data-email]')) {
+ if (!visible(el) || editable(el)) continue;
+ if (el === editor || editor?.contains?.(el)) continue;
+ const email = [
+ el.getAttribute?.('email'),
+ el.getAttribute?.('data-email'),
+ el.getAttribute?.('data-hovercard-id'),
+ ].map(value => compact(value, 240)).find(value => /@/.test(value)) || '';
+ if (!email) continue;
+ const aliases = new Map();
+ const addAlias = (val) => {
+ const alias = compact(val, 240);
+ const normalized = normalizedIdentity(alias);
+ if (alias && normalized && !aliases.has(normalized)) aliases.set(normalized, alias);
+ const angleMatch = alias.match(/^([^<@]+)<[^>]+>$/);
+ if (angleMatch) {
+ const nameOnly = compact(angleMatch[1], 240);
+ const normName = normalizedIdentity(nameOnly);
+ if (nameOnly && normName && !aliases.has(normName)) aliases.set(normName, nameOnly);
+ }
+ };
+ for (const value of [
+ email,
+ el.getAttribute?.('name'),
+ el.getAttribute?.('aria-label'),
+ el.getAttribute?.('title'),
+ el.innerText,
+ el.textContent,
+ ]) {
+ addAlias(value);
+ }
+ const key = normalizedIdentity(email);
+ if (!key) continue;
+ const role = recipientRole(el, root);
+ if (!role) continue;
+ const recipientKey = `${role}:${key}`;
+ const prior = collected.get(recipientKey) || { identity: email, role, aliases: new Map() };
+ for (const [normalized, alias] of aliases) prior.aliases.set(normalized, alias);
+ collected.set(recipientKey, prior);
+ }
+ return collected;
+ };
+ // A message's header field (To, Cc, Bcc or subject) belongs to its
+ // compose window and is never a message's body: Gmail's own field
+ // names, or a label a model reads as one of those fields.
+ const gmailHeaderField = el => {
+ if (el?.matches?.('[name="to"],[name="cc"],[name="bcc"],[name="subjectbox"]')) return true;
+ const label = compact(el?.getAttribute?.('aria-label'), 120);
+ return !!label && ['to', 'cc', 'bcc', 'subject'].includes(_pageReading('message_field', label));
+ };
+ // Another message's editor than `editor`: not nested with it, not a
+ // search or navigation field and not a message's header field.
+ const otherMessageEditor = (other, editor) => other !== editor && !editor?.contains?.(other)
+ && !other.contains?.(editor) && !verifiedNavigationEditable(other) && !gmailHeaderField(other);
+ // What marks a container as holding a message's recipients: Gmail's
+ // own row names, a chip in a To/Cc/Bcc row, or a field read as one of
+ // those rows, typed into or not.
+ const gmailRecipientMarkers = (root, editor) => !!root && (
+ !!root.querySelector?.('[name="to"],[name="cc"],[name="bcc"]')
+ || collectGmailRecipients(root, editor).size > 0
+ || Array.from(root.querySelectorAll?.('input, textarea, [contenteditable="true"], [contenteditable=""]') || [])
+ .slice(0, 40).some(input => visible(input) && input !== editor && !editor?.contains?.(input)
+ && !!(roleFromAttribute('name', input.getAttribute?.('name'))
+ || roleFromAttribute('aria-label', input.getAttribute?.('aria-label')))));
+ // In Gmail a send has recipients only from its composer's own mail
+ // compose window: a popup or full-screen compose is the editor's
+ // dialog (else its form), marked by recipient rows, a subject or
+ // Gmail's own compose body; an inline thread reply, which has neither,
+ // is the nearest ancestor with recipient rows that stays inside the
+ // page's main region and holds no visible window or other message
+ // editor. A dialog or form without them (a chat) is none.
+ const gmailComposeWindowOf = editor => {
+ if (!gmailRecipientMode || !editor) return null;
+ const explicit = editor.closest?.('[role="dialog"]') || editor.closest?.('form') || null;
+ if (explicit) {
+ return gmailRecipientMarkers(explicit, editor) || !!explicit.querySelector?.('input[name="subjectbox"]')
+ || editor.getAttribute?.('g_editable') === 'true' ? explicit : null;
+ }
+ let current = editor.parentElement;
+ for (let depth = 0; current && depth < 16; depth += 1, current = current.parentElement) {
+ if (current === document.body || current === document.documentElement
+ || current.matches?.('main,[role="main"]')
+ || Array.from(current.querySelectorAll?.('[role="dialog"]') || []).some(visible)
+ || composerCandidates.some(other => current.contains?.(other) && otherMessageEditor(other, editor))) break;
+ if (gmailRecipientMarkers(current, editor)) return current;
+ }
+ return null;
+ };
+ // Google Chat inside Gmail is a chat: on a Chat route, a composer in
+ // the page's main region, in no compose window and not Gmail's mail
+ // body, is identified by its own conversation header, as any chat is.
+ const gmailChatRoute = gmailRecipientMode && (() => {
+ try {
+ return location.hostname === 'mail.google.com'
+ && (/^\/chat(?:\/|$)/.test(String(location.pathname || ''))
+ || /^#chat(?:[/?]|$)/.test(String(location.hash || '')));
+ } catch { return false; }
+ })();
+ const gmailChatMainOf = (editor, ownWindow = gmailComposeWindowOf(editor)) => gmailChatRoute && editor && !ownWindow
+ && editor.getAttribute?.('g_editable') !== 'true' && !editor.closest?.('[role="dialog"]')
+ ? editor.closest?.('main,[role="main"]') || null : null;
+ // The composer a control (a Send, an attachment input) belongs to: the
+ // one message editor in the control's nearest ancestor holding any,
+ // when that editor is a Gmail compose or Chat composer. Its message,
+ // not the page's lowest editor, is what the control sends. None, or
+ // more than one there, leaves the pairing to the page's layout.
+ const gmailControlComposer = control => {
+ if (!gmailRecipientMode || !control) return null;
+ for (let node = control; node && node !== document.documentElement; node = node.parentElement) {
+ const inside = composerCandidates.filter(editor => node.contains?.(editor)
+ && !verifiedNavigationEditable(editor) && !gmailHeaderField(editor));
+ if (!inside.length) continue;
+ const owners = inside.filter(editor => !inside.some(other => other !== editor && other.contains?.(editor)));
+ return owners.length === 1 && (gmailComposeWindowOf(owners[0]) || gmailChatMainOf(owners[0]))
+ ? owners[0] : null;
+ }
+ return null;
+ };
if (tool === 'upload_file') {
- composer = layoutComposer;
+ // In Gmail the attachment input's own message decides its composer.
+ composer = gmailControlComposer(target) || layoutComposer;
const uploadOwner = _messageRecipientUploadOwner(target, composer);
const linkedInPrivateComposer = params.adapterName !== 'linkedin'
|| (/^\/messaging(?:\/|$)/.test(location.pathname)
@@ -7904,7 +8072,9 @@
identityCandidates: [],
};
}
- composer = layoutComposer;
+ // In Gmail the Send's own message decides its composer; elsewhere,
+ // and where no one message owns it, the page's layout pairs them.
+ composer = gmailControlComposer(control) || layoutComposer;
if (!composer) {
const actionLabel = compact(
control.getAttribute?.('aria-label')
@@ -8048,7 +8218,6 @@
const observedRecipientCandidates = [];
let conversationRecipientEvidence = null;
let nativeChatMessageBaseline = null;
- const gmailRecipientMode = params.adapterName === 'gmail';
const inConversationHeaderBand = (el) => {
if (headerBandBottom <= 0) return false;
const rect = el.getBoundingClientRect();
@@ -8072,159 +8241,23 @@
};
let gmailComposeRoot = null;
- // ── Gmail: the message a composer and its Send belong to ─────────────
- // The To, Cc or Bcc row an element names. The compose window, its
- // chips and its typed addresses below use it.
- const roleFromAttribute = (attribute, value) => {
- const text = compact(value, 120).toLowerCase();
- if (!text) return '';
- // A row's or field's label names its role in the user's language,
- // read by a model; the app's own names below do not.
- if (attribute === 'aria-label') {
- const role = _pageReading('message_field', compact(value, 120));
- return ['to', 'cc', 'bcc'].includes(role) ? role : '';
- }
- if (attribute === 'name') {
- return /^(?:to|cc|bcc)$/.test(text) ? text : '';
- }
- const tokens = text.split(/[^a-z]+/).filter(Boolean);
- if (tokens.includes('bcc')) return 'bcc';
- if (tokens.includes('cc')) return 'cc';
- if (tokens.length === 1 && tokens[0] === 'to') return 'to';
- return '';
- };
- const recipientRole = (el, root = null) => {
- let current = el;
- for (let depth = 0; current && depth < 7; depth++, current = current.parentElement) {
- for (const attribute of ['data-recipient-type', 'data-type', 'name', 'aria-label']) {
- const role = roleFromAttribute(attribute, current.getAttribute?.(attribute));
- if (role) return role;
- }
- if (root && current === root) break;
- try {
- const descendantRoles = new Set();
- for (const marker of current.querySelectorAll?.(
- 'input[name="to"],input[name="cc"],input[name="bcc"],[aria-label]'
- ) || []) {
- const role = roleFromAttribute('name', marker.getAttribute?.('name'))
- || roleFromAttribute('aria-label', marker.getAttribute?.('aria-label'));
- if (role) descendantRoles.add(role);
- }
- // A nearest row containing one role marker can classify its
- // chips. A shared ancestor containing multiple rows cannot.
- if (descendantRoles.size === 1) return [...descendantRoles][0];
- } catch {}
- }
- return '';
- };
- const collectGmailRecipients = (root) => {
- const collected = new Map();
- if (!root?.querySelectorAll) return collected;
- for (const el of root.querySelectorAll('[email],[data-hovercard-id],[data-email]')) {
- if (!visible(el) || editable(el)) continue;
- if (el === composer || composer.contains?.(el)) continue;
- const email = [
- el.getAttribute?.('email'),
- el.getAttribute?.('data-email'),
- el.getAttribute?.('data-hovercard-id'),
- ].map(value => compact(value, 240)).find(value => /@/.test(value)) || '';
- if (!email) continue;
- const aliases = new Map();
- const addAlias = (val) => {
- const alias = compact(val, 240);
- const normalized = normalizedIdentity(alias);
- if (alias && normalized && !aliases.has(normalized)) aliases.set(normalized, alias);
- const angleMatch = alias.match(/^([^<@]+)<[^>]+>$/);
- if (angleMatch) {
- const nameOnly = compact(angleMatch[1], 240);
- const normName = normalizedIdentity(nameOnly);
- if (nameOnly && normName && !aliases.has(normName)) aliases.set(normName, nameOnly);
- }
- };
- for (const value of [
- email,
- el.getAttribute?.('name'),
- el.getAttribute?.('aria-label'),
- el.getAttribute?.('title'),
- el.innerText,
- el.textContent,
- ]) {
- addAlias(value);
- }
- const key = normalizedIdentity(email);
- if (!key) continue;
- const role = recipientRole(el, root);
- if (!role) continue;
- const recipientKey = `${role}:${key}`;
- const prior = collected.get(recipientKey) || { identity: email, role, aliases: new Map() };
- for (const [normalized, alias] of aliases) prior.aliases.set(normalized, alias);
- collected.set(recipientKey, prior);
- }
- return collected;
- };
- // Another message's editor: not this composer, not nested with it,
- // not a search or navigation field and not a To/Cc/Bcc row's field.
- const otherMessageEditor = editor => editor !== composer && !composer.contains?.(editor)
- && !editor.contains?.(composer) && !verifiedNavigationEditable(editor)
- && !editor.matches?.('[name="to"],[name="cc"],[name="bcc"]');
- // In Gmail a send has recipients only from its composer's own mail
- // compose window: a popup or full-screen compose is the composer's
- // dialog (else its form), and an inline thread reply, which has
- // neither, the nearest ancestor with To/Cc/Bcc chips that stays inside
- // the page's main region and holds no other window or message editor.
- // Gmail's own field names, its compose body, a chip in a row or a
- // field read as a row mark that window; one without them (a chat) is
- // none.
- const gmailComposeWindow = (() => {
- if (!gmailRecipientMode) return null;
- const recipientRows = root => !!root && (
- !!root.querySelector?.('[name="to"],[name="cc"],[name="bcc"],input[name="subjectbox"]')
- || composer.getAttribute?.('g_editable') === 'true'
- || collectGmailRecipients(root).size > 0
- || Array.from(root.querySelectorAll?.('input, textarea, [contenteditable="true"], [contenteditable=""]') || [])
- .slice(0, 40).some(input => visible(input) && input !== composer && !composer.contains?.(input)
- && !!(roleFromAttribute('name', input.getAttribute?.('name'))
- || roleFromAttribute('aria-label', input.getAttribute?.('aria-label')))));
- const explicit = composer.closest?.('[role="dialog"]') || composer.closest?.('form') || null;
- if (explicit) return recipientRows(explicit) ? explicit : null;
- let current = composer.parentElement;
- for (let depth = 0; current && depth < 16; depth += 1, current = current.parentElement) {
- if (current === document.body || current === document.documentElement
- || current.matches?.('main,[role="main"]') || current.querySelector?.('[role="dialog"]')
- || composerCandidates.some(editor => current.contains?.(editor) && otherMessageEditor(editor))) break;
- if (collectGmailRecipients(current).size > 0) return current;
- }
- return null;
- })();
- // The control that sends must be this composer's own: the two share an
+ // The composer's own Gmail compose window, and whether what sends (a
+ // Send, an attachment input) is that composer's own: the two share an
// owner holding no message editor beyond this composer's own window.
- // A Send the page's layout pairs with another message's composer (a
- // Chat's Send beside a compose window's body, or the reverse) belongs
- // to that other message.
+ // A control the page's layout pairs with another message's composer
+ // belongs to that other message.
+ const gmailComposeWindow = gmailComposeWindowOf(composer);
let gmailSendOwner = null;
- if (gmailRecipientMode && (tool === 'click' || tool === 'click_ax') && dispatchTarget
- && dispatchTarget !== composer && !composer.contains?.(dispatchTarget)) {
+ if (gmailRecipientMode && dispatchTarget && dispatchTarget !== composer && !composer.contains?.(dispatchTarget)) {
for (let node = composer.parentElement; node; node = node.parentElement) {
if (node.contains?.(dispatchTarget)) { gmailSendOwner = node; break; }
}
}
const gmailEditorScope = composer.closest?.('[role="dialog"]') || gmailComposeWindow || composer;
const gmailForeignSend = !!gmailSendOwner && composerCandidates.some(editor => gmailSendOwner.contains(editor)
- && !gmailEditorScope.contains?.(editor) && otherMessageEditor(editor));
+ && !gmailEditorScope.contains?.(editor) && otherMessageEditor(editor, composer));
const gmailMailSend = !!gmailComposeWindow && !gmailForeignSend;
- // Google Chat inside Gmail is a chat: on a Chat route, a composer in
- // the page's main region and in no compose window is identified by its
- // own conversation header, as any chat is (below).
- const gmailChatRoute = gmailRecipientMode && (() => {
- try {
- return location.hostname === 'mail.google.com'
- && (/^\/chat(?:\/|$)/.test(String(location.pathname || ''))
- || /^#chat(?:[/?]|$)/.test(String(location.hash || '')));
- } catch { return false; }
- })();
- const gmailChatMain = gmailChatRoute && !gmailComposeWindow && !gmailForeignSend
- && composer.getAttribute?.('g_editable') !== 'true' && !composer.closest?.('[role="dialog"]')
- ? composer.closest?.('main,[role="main"]') || null : null;
+ const gmailChatMain = gmailForeignSend ? null : gmailChatMainOf(composer, gmailComposeWindow);
const gmailChatComposer = !!gmailChatMain;
// Gmail's recipient sets come from that window alone, on any route.
const recipientSets = gmailRecipientMode ? gmailMailSend : params.supportsRecipientSets === true;
diff --git a/src/firefox/src/content/content.js b/src/firefox/src/content/content.js
index fbb39bfa8..657251a65 100644
--- a/src/firefox/src/content/content.js
+++ b/src/firefox/src/content/content.js
@@ -6970,8 +6970,176 @@
nonMessagingTarget: true, reasonCode: 'non_messaging_target', identityCandidates: [],
};
}
+ // ── Gmail: the message a composer, a Send or an attachment belongs to ──
+ const gmailRecipientMode = params.adapterName === 'gmail';
+ // The To, Cc or Bcc row an element names. A compose window, its
+ // chips and its typed addresses below use it.
+ const roleFromAttribute = (attribute, value) => {
+ const text = compact(value, 120).toLowerCase();
+ if (!text) return '';
+ // A row's or field's label names its role in the user's language,
+ // read by a model; the app's own names below do not.
+ if (attribute === 'aria-label') {
+ const role = _pageReading('message_field', compact(value, 120));
+ return ['to', 'cc', 'bcc'].includes(role) ? role : '';
+ }
+ if (attribute === 'name') {
+ return /^(?:to|cc|bcc)$/.test(text) ? text : '';
+ }
+ const tokens = text.split(/[^a-z]+/).filter(Boolean);
+ if (tokens.includes('bcc')) return 'bcc';
+ if (tokens.includes('cc')) return 'cc';
+ if (tokens.length === 1 && tokens[0] === 'to') return 'to';
+ return '';
+ };
+ const recipientRole = (el, root = null) => {
+ let current = el;
+ for (let depth = 0; current && depth < 7; depth++, current = current.parentElement) {
+ for (const attribute of ['data-recipient-type', 'data-type', 'name', 'aria-label']) {
+ const role = roleFromAttribute(attribute, current.getAttribute?.(attribute));
+ if (role) return role;
+ }
+ if (root && current === root) break;
+ try {
+ const descendantRoles = new Set();
+ for (const marker of current.querySelectorAll?.(
+ 'input[name="to"],input[name="cc"],input[name="bcc"],[aria-label]'
+ ) || []) {
+ const role = roleFromAttribute('name', marker.getAttribute?.('name'))
+ || roleFromAttribute('aria-label', marker.getAttribute?.('aria-label'));
+ if (role) descendantRoles.add(role);
+ }
+ // A nearest row containing one role marker can classify its
+ // chips. A shared ancestor containing multiple rows cannot.
+ if (descendantRoles.size === 1) return [...descendantRoles][0];
+ } catch {}
+ }
+ return '';
+ };
+ const collectGmailRecipients = (root, editor = composer) => {
+ const collected = new Map();
+ if (!root?.querySelectorAll) return collected;
+ for (const el of root.querySelectorAll('[email],[data-hovercard-id],[data-email]')) {
+ if (!visible(el) || editable(el)) continue;
+ if (el === editor || editor?.contains?.(el)) continue;
+ const email = [
+ el.getAttribute?.('email'),
+ el.getAttribute?.('data-email'),
+ el.getAttribute?.('data-hovercard-id'),
+ ].map(value => compact(value, 240)).find(value => /@/.test(value)) || '';
+ if (!email) continue;
+ const aliases = new Map();
+ const addAlias = (val) => {
+ const alias = compact(val, 240);
+ const normalized = normalizedIdentity(alias);
+ if (alias && normalized && !aliases.has(normalized)) aliases.set(normalized, alias);
+ const angleMatch = alias.match(/^([^<@]+)<[^>]+>$/);
+ if (angleMatch) {
+ const nameOnly = compact(angleMatch[1], 240);
+ const normName = normalizedIdentity(nameOnly);
+ if (nameOnly && normName && !aliases.has(normName)) aliases.set(normName, nameOnly);
+ }
+ };
+ for (const value of [
+ email,
+ el.getAttribute?.('name'),
+ el.getAttribute?.('aria-label'),
+ el.getAttribute?.('title'),
+ el.innerText,
+ el.textContent,
+ ]) {
+ addAlias(value);
+ }
+ const key = normalizedIdentity(email);
+ if (!key) continue;
+ const role = recipientRole(el, root);
+ if (!role) continue;
+ const recipientKey = `${role}:${key}`;
+ const prior = collected.get(recipientKey) || { identity: email, role, aliases: new Map() };
+ for (const [normalized, alias] of aliases) prior.aliases.set(normalized, alias);
+ collected.set(recipientKey, prior);
+ }
+ return collected;
+ };
+ // A message's header field (To, Cc, Bcc or subject) belongs to its
+ // compose window and is never a message's body: Gmail's own field
+ // names, or a label a model reads as one of those fields.
+ const gmailHeaderField = el => {
+ if (el?.matches?.('[name="to"],[name="cc"],[name="bcc"],[name="subjectbox"]')) return true;
+ const label = compact(el?.getAttribute?.('aria-label'), 120);
+ return !!label && ['to', 'cc', 'bcc', 'subject'].includes(_pageReading('message_field', label));
+ };
+ // Another message's editor than `editor`: not nested with it, not a
+ // search or navigation field and not a message's header field.
+ const otherMessageEditor = (other, editor) => other !== editor && !editor?.contains?.(other)
+ && !other.contains?.(editor) && !verifiedNavigationEditable(other) && !gmailHeaderField(other);
+ // What marks a container as holding a message's recipients: Gmail's
+ // own row names, a chip in a To/Cc/Bcc row, or a field read as one of
+ // those rows, typed into or not.
+ const gmailRecipientMarkers = (root, editor) => !!root && (
+ !!root.querySelector?.('[name="to"],[name="cc"],[name="bcc"]')
+ || collectGmailRecipients(root, editor).size > 0
+ || Array.from(root.querySelectorAll?.('input, textarea, [contenteditable="true"], [contenteditable=""]') || [])
+ .slice(0, 40).some(input => visible(input) && input !== editor && !editor?.contains?.(input)
+ && !!(roleFromAttribute('name', input.getAttribute?.('name'))
+ || roleFromAttribute('aria-label', input.getAttribute?.('aria-label')))));
+ // In Gmail a send has recipients only from its composer's own mail
+ // compose window: a popup or full-screen compose is the editor's
+ // dialog (else its form), marked by recipient rows, a subject or
+ // Gmail's own compose body; an inline thread reply, which has neither,
+ // is the nearest ancestor with recipient rows that stays inside the
+ // page's main region and holds no visible window or other message
+ // editor. A dialog or form without them (a chat) is none.
+ const gmailComposeWindowOf = editor => {
+ if (!gmailRecipientMode || !editor) return null;
+ const explicit = editor.closest?.('[role="dialog"]') || editor.closest?.('form') || null;
+ if (explicit) {
+ return gmailRecipientMarkers(explicit, editor) || !!explicit.querySelector?.('input[name="subjectbox"]')
+ || editor.getAttribute?.('g_editable') === 'true' ? explicit : null;
+ }
+ let current = editor.parentElement;
+ for (let depth = 0; current && depth < 16; depth += 1, current = current.parentElement) {
+ if (current === document.body || current === document.documentElement
+ || current.matches?.('main,[role="main"]')
+ || Array.from(current.querySelectorAll?.('[role="dialog"]') || []).some(visible)
+ || composerCandidates.some(other => current.contains?.(other) && otherMessageEditor(other, editor))) break;
+ if (gmailRecipientMarkers(current, editor)) return current;
+ }
+ return null;
+ };
+ // Google Chat inside Gmail is a chat: on a Chat route, a composer in
+ // the page's main region, in no compose window and not Gmail's mail
+ // body, is identified by its own conversation header, as any chat is.
+ const gmailChatRoute = gmailRecipientMode && (() => {
+ try {
+ return location.hostname === 'mail.google.com'
+ && (/^\/chat(?:\/|$)/.test(String(location.pathname || ''))
+ || /^#chat(?:[/?]|$)/.test(String(location.hash || '')));
+ } catch { return false; }
+ })();
+ const gmailChatMainOf = (editor, ownWindow = gmailComposeWindowOf(editor)) => gmailChatRoute && editor && !ownWindow
+ && editor.getAttribute?.('g_editable') !== 'true' && !editor.closest?.('[role="dialog"]')
+ ? editor.closest?.('main,[role="main"]') || null : null;
+ // The composer a control (a Send, an attachment input) belongs to: the
+ // one message editor in the control's nearest ancestor holding any,
+ // when that editor is a Gmail compose or Chat composer. Its message,
+ // not the page's lowest editor, is what the control sends. None, or
+ // more than one there, leaves the pairing to the page's layout.
+ const gmailControlComposer = control => {
+ if (!gmailRecipientMode || !control) return null;
+ for (let node = control; node && node !== document.documentElement; node = node.parentElement) {
+ const inside = composerCandidates.filter(editor => node.contains?.(editor)
+ && !verifiedNavigationEditable(editor) && !gmailHeaderField(editor));
+ if (!inside.length) continue;
+ const owners = inside.filter(editor => !inside.some(other => other !== editor && other.contains?.(editor)));
+ return owners.length === 1 && (gmailComposeWindowOf(owners[0]) || gmailChatMainOf(owners[0]))
+ ? owners[0] : null;
+ }
+ return null;
+ };
if (tool === 'upload_file') {
- composer = layoutComposer;
+ // In Gmail the attachment input's own message decides its composer.
+ composer = gmailControlComposer(target) || layoutComposer;
const uploadOwner = _messageRecipientUploadOwner(target, composer);
const linkedInPrivateComposer = params.adapterName !== 'linkedin'
|| (/^\/messaging(?:\/|$)/.test(location.pathname)
@@ -7071,7 +7239,9 @@
identityCandidates: [],
};
}
- composer = layoutComposer;
+ // In Gmail the Send's own message decides its composer; elsewhere,
+ // and where no one message owns it, the page's layout pairs them.
+ composer = gmailControlComposer(control) || layoutComposer;
if (!composer) {
const actionLabel = compact(
control.getAttribute?.('aria-label')
@@ -7202,7 +7372,6 @@
const observedRecipientCandidates = [];
let conversationRecipientEvidence = null;
let nativeChatMessageBaseline = null;
- const gmailRecipientMode = params.adapterName === 'gmail';
const inConversationHeaderBand = (el) => {
if (headerBandBottom <= 0) return false;
const rect = el.getBoundingClientRect();
@@ -7226,159 +7395,23 @@
};
let gmailComposeRoot = null;
- // ── Gmail: the message a composer and its Send belong to ─────────────
- // The To, Cc or Bcc row an element names. The compose window, its
- // chips and its typed addresses below use it.
- const roleFromAttribute = (attribute, value) => {
- const text = compact(value, 120).toLowerCase();
- if (!text) return '';
- // A row's or field's label names its role in the user's language,
- // read by a model; the app's own names below do not.
- if (attribute === 'aria-label') {
- const role = _pageReading('message_field', compact(value, 120));
- return ['to', 'cc', 'bcc'].includes(role) ? role : '';
- }
- if (attribute === 'name') {
- return /^(?:to|cc|bcc)$/.test(text) ? text : '';
- }
- const tokens = text.split(/[^a-z]+/).filter(Boolean);
- if (tokens.includes('bcc')) return 'bcc';
- if (tokens.includes('cc')) return 'cc';
- if (tokens.length === 1 && tokens[0] === 'to') return 'to';
- return '';
- };
- const recipientRole = (el, root = null) => {
- let current = el;
- for (let depth = 0; current && depth < 7; depth++, current = current.parentElement) {
- for (const attribute of ['data-recipient-type', 'data-type', 'name', 'aria-label']) {
- const role = roleFromAttribute(attribute, current.getAttribute?.(attribute));
- if (role) return role;
- }
- if (root && current === root) break;
- try {
- const descendantRoles = new Set();
- for (const marker of current.querySelectorAll?.(
- 'input[name="to"],input[name="cc"],input[name="bcc"],[aria-label]'
- ) || []) {
- const role = roleFromAttribute('name', marker.getAttribute?.('name'))
- || roleFromAttribute('aria-label', marker.getAttribute?.('aria-label'));
- if (role) descendantRoles.add(role);
- }
- // A nearest row containing one role marker can classify its
- // chips. A shared ancestor containing multiple rows cannot.
- if (descendantRoles.size === 1) return [...descendantRoles][0];
- } catch {}
- }
- return '';
- };
- const collectGmailRecipients = (root) => {
- const collected = new Map();
- if (!root?.querySelectorAll) return collected;
- for (const el of root.querySelectorAll('[email],[data-hovercard-id],[data-email]')) {
- if (!visible(el) || editable(el)) continue;
- if (el === composer || composer.contains?.(el)) continue;
- const email = [
- el.getAttribute?.('email'),
- el.getAttribute?.('data-email'),
- el.getAttribute?.('data-hovercard-id'),
- ].map(value => compact(value, 240)).find(value => /@/.test(value)) || '';
- if (!email) continue;
- const aliases = new Map();
- const addAlias = (val) => {
- const alias = compact(val, 240);
- const normalized = normalizedIdentity(alias);
- if (alias && normalized && !aliases.has(normalized)) aliases.set(normalized, alias);
- const angleMatch = alias.match(/^([^<@]+)<[^>]+>$/);
- if (angleMatch) {
- const nameOnly = compact(angleMatch[1], 240);
- const normName = normalizedIdentity(nameOnly);
- if (nameOnly && normName && !aliases.has(normName)) aliases.set(normName, nameOnly);
- }
- };
- for (const value of [
- email,
- el.getAttribute?.('name'),
- el.getAttribute?.('aria-label'),
- el.getAttribute?.('title'),
- el.innerText,
- el.textContent,
- ]) {
- addAlias(value);
- }
- const key = normalizedIdentity(email);
- if (!key) continue;
- const role = recipientRole(el, root);
- if (!role) continue;
- const recipientKey = `${role}:${key}`;
- const prior = collected.get(recipientKey) || { identity: email, role, aliases: new Map() };
- for (const [normalized, alias] of aliases) prior.aliases.set(normalized, alias);
- collected.set(recipientKey, prior);
- }
- return collected;
- };
- // Another message's editor: not this composer, not nested with it,
- // not a search or navigation field and not a To/Cc/Bcc row's field.
- const otherMessageEditor = editor => editor !== composer && !composer.contains?.(editor)
- && !editor.contains?.(composer) && !verifiedNavigationEditable(editor)
- && !editor.matches?.('[name="to"],[name="cc"],[name="bcc"]');
- // In Gmail a send has recipients only from its composer's own mail
- // compose window: a popup or full-screen compose is the composer's
- // dialog (else its form), and an inline thread reply, which has
- // neither, the nearest ancestor with To/Cc/Bcc chips that stays inside
- // the page's main region and holds no other window or message editor.
- // Gmail's own field names, its compose body, a chip in a row or a
- // field read as a row mark that window; one without them (a chat) is
- // none.
- const gmailComposeWindow = (() => {
- if (!gmailRecipientMode) return null;
- const recipientRows = root => !!root && (
- !!root.querySelector?.('[name="to"],[name="cc"],[name="bcc"],input[name="subjectbox"]')
- || composer.getAttribute?.('g_editable') === 'true'
- || collectGmailRecipients(root).size > 0
- || Array.from(root.querySelectorAll?.('input, textarea, [contenteditable="true"], [contenteditable=""]') || [])
- .slice(0, 40).some(input => visible(input) && input !== composer && !composer.contains?.(input)
- && !!(roleFromAttribute('name', input.getAttribute?.('name'))
- || roleFromAttribute('aria-label', input.getAttribute?.('aria-label')))));
- const explicit = composer.closest?.('[role="dialog"]') || composer.closest?.('form') || null;
- if (explicit) return recipientRows(explicit) ? explicit : null;
- let current = composer.parentElement;
- for (let depth = 0; current && depth < 16; depth += 1, current = current.parentElement) {
- if (current === document.body || current === document.documentElement
- || current.matches?.('main,[role="main"]') || current.querySelector?.('[role="dialog"]')
- || composerCandidates.some(editor => current.contains?.(editor) && otherMessageEditor(editor))) break;
- if (collectGmailRecipients(current).size > 0) return current;
- }
- return null;
- })();
- // The control that sends must be this composer's own: the two share an
+ // The composer's own Gmail compose window, and whether what sends (a
+ // Send, an attachment input) is that composer's own: the two share an
// owner holding no message editor beyond this composer's own window.
- // A Send the page's layout pairs with another message's composer (a
- // Chat's Send beside a compose window's body, or the reverse) belongs
- // to that other message.
+ // A control the page's layout pairs with another message's composer
+ // belongs to that other message.
+ const gmailComposeWindow = gmailComposeWindowOf(composer);
let gmailSendOwner = null;
- if (gmailRecipientMode && (tool === 'click' || tool === 'click_ax') && dispatchTarget
- && dispatchTarget !== composer && !composer.contains?.(dispatchTarget)) {
+ if (gmailRecipientMode && dispatchTarget && dispatchTarget !== composer && !composer.contains?.(dispatchTarget)) {
for (let node = composer.parentElement; node; node = node.parentElement) {
if (node.contains?.(dispatchTarget)) { gmailSendOwner = node; break; }
}
}
const gmailEditorScope = composer.closest?.('[role="dialog"]') || gmailComposeWindow || composer;
const gmailForeignSend = !!gmailSendOwner && composerCandidates.some(editor => gmailSendOwner.contains(editor)
- && !gmailEditorScope.contains?.(editor) && otherMessageEditor(editor));
+ && !gmailEditorScope.contains?.(editor) && otherMessageEditor(editor, composer));
const gmailMailSend = !!gmailComposeWindow && !gmailForeignSend;
- // Google Chat inside Gmail is a chat: on a Chat route, a composer in
- // the page's main region and in no compose window is identified by its
- // own conversation header, as any chat is (below).
- const gmailChatRoute = gmailRecipientMode && (() => {
- try {
- return location.hostname === 'mail.google.com'
- && (/^\/chat(?:\/|$)/.test(String(location.pathname || ''))
- || /^#chat(?:[/?]|$)/.test(String(location.hash || '')));
- } catch { return false; }
- })();
- const gmailChatMain = gmailChatRoute && !gmailComposeWindow && !gmailForeignSend
- && composer.getAttribute?.('g_editable') !== 'true' && !composer.closest?.('[role="dialog"]')
- ? composer.closest?.('main,[role="main"]') || null : null;
+ const gmailChatMain = gmailForeignSend ? null : gmailChatMainOf(composer, gmailComposeWindow);
const gmailChatComposer = !!gmailChatMain;
// Gmail's recipient sets come from that window alone, on any route.
const recipientSets = gmailRecipientMode ? gmailMailSend : params.supportsRecipientSets === true;
diff --git a/test/improver-content-recipient.mjs b/test/improver-content-recipient.mjs
index bd9d74032..fb03e2915 100644
--- a/test/improver-content-recipient.mjs
+++ b/test/improver-content-recipient.mjs
@@ -65,14 +65,16 @@ const GMAIL_COMPOSE_OVER_CHAT = `<!doctype html><style>body{margin:0;font:16px s
const TO_ADA = '<div name="to"><span email="[email protected]">Ada</span></div>';
const CC_CEM = '<div name="cc"><span email="[email protected]">Cem</span></div>';
const BCC_DEE = '<div name="bcc"><span email="[email protected]">Dee</span></div>';
-const gmailChat = ({ rows = '', raise = 0 } = {}) => `<!doctype html><style>body{margin:0;font:16px sans-serif}
+const TO_FIELD = value => `<input id="to-input" role="combobox" aria-label="To recipients" value="${value}">`;
+const gmailChat = ({ rows = '', raise = 0, chatFile = false, appFile = false } = {}) => `<!doctype html><style>body{margin:0;font:16px sans-serif}
#chat{position:absolute;left:0;top:0;width:640px;height:780px}#chat>header{height:70px}
#messages{height:580px;overflow:auto}footer{display:flex;gap:15px;align-items:center}
#body{width:500px;min-height:32px}button{min-height:28px}[role=dialog]{position:absolute;left:660px;bottom:${raise}px;width:560px}</style>
<div id="app"><main id="chat" role="main"><header><h2>Bob</h2></header>
<section id="messages"><div class="msg">Bob: please also write to <a href="mailto:[email protected]">[email protected]</a></div></section>
<footer><div id="body" contenteditable="true" role="textbox" aria-label="Type a message">Hi Bob</div>
- <button id="send" type="button" aria-label="Send">Send</button></footer></main>
+ ${chatFile ? '<input id="chat-file" type="file">' : ''}<button id="send" type="button" aria-label="Send">Send</button></footer></main>
+ ${appFile ? '<input id="app-file" type="file">' : ''}
${rows ? `<div role="dialog" aria-label="New Message"><form onsubmit="return false">${rows}
<div contenteditable="true" role="textbox" aria-label="Message Body">Draft</div>
<button id="mail-send" type="button" aria-label="Send">Send</button></form></div>` : ''}</div>`;
@@ -338,26 +340,58 @@ for (const [build, Agent] of [['chrome', ChromeAgent], ['firefox', FirefoxAgent]
});
}
- test(`${build}: beside an open compose window a Gmail Chat send binds to its own header, and the window's Send is not the chat's`, async () => {
- const url = 'https://mail.google.com/mail/u/0/#chat/dm/abc?compose=new';
- const f = await fixture(build, Agent, { url, recipients: ['Bob'], html: gmailChat({ rows: TO_ADA, raise: 150 }) });
+ // Each Send sends its own message, whichever editor is the page's lowest:
+ // the compose window's Send goes to its own To row, the Chat's Send to the
+ // chat's person and never to the window's recipients.
+ for (const raise of [0, 150]) {
+ test(`${build}: beside an open compose window (${raise ? 'raised' : 'docked'}) each Send binds to its own message`, async () => {
+ const url = 'https://mail.google.com/mail/u/0/#chat/dm/abc?compose=new';
+ const mailSend = { selector: '#mail-send' };
+ const f = await fixture(build, Agent, { url, recipients: ['[email protected]'], html: gmailChat({ rows: TO_ADA, raise }) });
+ try {
+ await f.page.evaluate(() => {
+ window.mailSent = 0;
+ document.querySelector('#mail-send').addEventListener('click', () => window.mailSent++);
+ });
+ const chat = await f.bind();
+ assert.equal(chat.blocked?.success, false, JSON.stringify(chat.blocked));
+ assert.deepEqual(chat.blocked.recipientConfirmation?.recipients, [{ identity: 'Bob', role: 'to' }], JSON.stringify(chat.blocked));
+ assert.equal(chat.context.messageRecipientDispatchBinding, undefined);
+ const mailContext = {};
+ const mailBlocked = await f.agent._messageRecipientGuardBlock(71, 'click', mailSend, url, mailContext);
+ assert.equal(mailBlocked, null, JSON.stringify(mailBlocked));
+ const result = await call(f.page, 'click', { ...mailSend, ...mailContext });
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(await f.page.evaluate(() => [window.sent, window.mailSent]).then(String), '0,1');
+ } finally { await f.browser.close(); }
+ const bob = await fixture(build, Agent, { url, recipients: ['Bob'], html: gmailChat({ rows: TO_ADA, raise }) });
+ try {
+ const { blocked, context } = await bob.bind();
+ assert.equal(blocked, null, JSON.stringify(blocked));
+ const result = await call(bob.page, 'click', { ...sendArgs, ...context });
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(await bob.page.evaluate(() => window.sent), 1);
+ } finally { await bob.browser.close(); }
+ });
+ }
+
+ // An attachment goes with the message its file input belongs to: the
+ // Chat's input to the chat's person, an input no one message owns to no
+ // one, and neither to a compose window's recipients beside it.
+ test(`${build}: a file attached in a Gmail Chat never goes to a compose window's recipients`, async () => {
+ const recipients = [{ identity: '[email protected]', role: 'to' }, { identity: '[email protected]', role: 'cc' }];
+ const f = await fixture(build, Agent, { url: 'https://mail.google.com/mail/u/0/#chat/dm/abc', recipients,
+ html: gmailChat({ rows: TO_ADA + CC_CEM, chatFile: true, appFile: true }) });
try {
- await f.page.evaluate(() => {
- window.mailSent = 0;
- document.querySelector('#mail-send').addEventListener('click', () => window.mailSent++);
- });
- // The layout pairs the window's Send with the chat's composer: it
- // identifies no one, so neither the chat's person nor the window's.
- const mailContext = {};
- const mailBlocked = await f.agent._messageRecipientGuardBlock(71, 'click', { selector: '#mail-send' }, url, mailContext);
- assert.equal(mailBlocked?.success, false, JSON.stringify(mailBlocked));
- assert.equal(mailContext.messageRecipientDispatchBinding, undefined);
- assert.equal(await f.page.evaluate(() => window.mailSent), 0);
- const { blocked, context } = await f.bind();
- assert.equal(blocked, null, JSON.stringify(blocked));
- const result = await call(f.page, 'click', { ...sendArgs, ...context });
- assert.equal(result.success, true, JSON.stringify(result));
- assert.equal(await f.page.evaluate(() => [window.sent, window.mailSent]).then(String), '1,0');
+ const upload = selector => f.probe({ adapterName: 'gmail', tool: 'upload_file', args: { selector },
+ uploadSourceKey: 'download:1', expectedRecipients: recipients });
+ const chatUpload = await upload('#chat-file');
+ assert.equal(chatUpload.recipientSets, undefined, JSON.stringify(chatUpload));
+ assert.deepEqual(chatUpload.strongRecipientCandidates, [{ identity: 'Bob', role: 'to' }], JSON.stringify(chatUpload));
+ const appUpload = await upload('#app-file');
+ assert.equal(appUpload.recipientSets, undefined, JSON.stringify(appUpload));
+ assert.deepEqual(appUpload.strongRecipientCandidates || [], [], JSON.stringify(appUpload));
+ assert.equal(appUpload.messageRecipientDispatchBinding, undefined);
} finally { await f.browser.close(); }
});
@@ -399,6 +433,51 @@ for (const [build, Agent] of [['chrome', ChromeAgent], ['firefox', FirefoxAgent]
});
}
+ // Gmail's To field is a combobox named by its label: focused, typed into
+ // and not yet a chip, it is still the reply's own row, and a hidden
+ // dialog inside the reply does not end it.
+ for (const [kind, rows, focus, extra] of [
+ ['a chip and the focused To field', TO_ADA + TO_FIELD(''), '#to-input', ''],
+ ['a typed To address, the body focused', TO_FIELD('[email protected]'), '#body', ''],
+ ['a typed To address, the To field focused', TO_FIELD('[email protected]'), '#to-input', ''],
+ ['a hidden dialog inside it', TO_ADA, '#body', '<div role="dialog" aria-label="Insert link" style="display:none"></div>'],
+ ]) {
+ test(`${build}: a Gmail inline reply with ${kind} binds to its own To`, async () => {
+ const html = `<!doctype html><style>body{margin:0;font:16px sans-serif}main{position:absolute;left:0;top:0;width:900px;height:780px}
+ .reply{position:absolute;left:0;top:560px}#body{display:block;width:560px;min-height:40px}</style>
+ <main role="main"><h2>Invoice</h2><div class="message">Ada: here is the invoice</div>
+ <div class="reply">${extra}<div class="rows">${rows}</div>
+ <div id="body" g_editable="true" contenteditable="true" role="textbox" aria-label="Message Body">Thanks Ada</div>
+ <div class="toolbar"><button id="send" type="button" aria-label="Send">Send</button></div></div></main>`;
+ const f = await fixture(build, Agent, { url: 'https://mail.google.com/mail/u/0/#inbox/FMfc123', recipients: ['[email protected]'], html });
+ try {
+ await f.page.locator(focus).focus();
+ const { blocked, context } = await f.bind();
+ assert.equal(blocked, null, JSON.stringify(blocked));
+ const result = await call(f.page, 'click', { ...sendArgs, ...context });
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(await f.page.evaluate(() => window.sent), 1);
+ } finally { await f.browser.close(); }
+ });
+ }
+
+ test(`${build}: a Gmail popup compose with a typed To address binds with the To field focused`, async () => {
+ const html = `<!doctype html><style>body{margin:0;font:16px sans-serif}[role=dialog]{position:absolute;right:20px;bottom:0;width:560px}
+ #body{display:block;width:500px;min-height:60px}</style><main role="main"><h2>Inbox</h2></main>
+ <div role="dialog" aria-label="New Message">${TO_FIELD('[email protected]')}<input name="subjectbox" value="Hi">
+ <div id="body" g_editable="true" contenteditable="true" role="textbox" aria-label="Message Body">Hello</div>
+ <button id="send" type="button" aria-label="Send">Send</button></div>`;
+ const f = await fixture(build, Agent, { url: 'https://mail.google.com/mail/u/0/#inbox?compose=new', recipients: ['[email protected]'], html });
+ try {
+ await f.page.locator('#to-input').focus();
+ const { blocked, context } = await f.bind();
+ assert.equal(blocked, null, JSON.stringify(blocked));
+ const result = await call(f.page, 'click', { ...sendArgs, ...context });
+ assert.equal(result.success, true, JSON.stringify(result));
+ assert.equal(await f.page.evaluate(() => window.sent), 1);
+ } finally { await f.browser.close(); }
+ });
+
test(`${build}: an ordinary Gmail compose still binds and sends to its own To, Cc and Bcc`, async () => {
const recipients = [{ identity: '[email protected]', role: 'to' }, { identity: '[email protected]', role: 'cc' },
{ identity: '[email protected]', role: 'bcc' }];