Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Address Discord observation review feedback
Emre Sokulluf7bab31b77b0
diff --git a/src/chrome/src/content/chat-observation.js b/src/chrome/src/content/chat-observation.jsindex 8f7da8d28..cd1127bbc 100644--- a/src/chrome/src/content/chat-observation.js+++ b/src/chrome/src/content/chat-observation.js@@ -138,20 +138,22 @@const discordChannel = (composer) => {if (!composer || !/^(?:www\.)?discord\.com$/.test(window.location?.hostname || '')) return null;const route = /^\/channels\/(\d+)\/(\d+)\/?$/.exec(window.location?.pathname || '');- const name = /^Message #(.+)$/.exec(attribute(composer, 'aria-label'))?.[1];- if (!route || !name || !composer.matches?.('[data-slate-editor="true"][contenteditable="true"][role="textbox"]')) return null;+ if (!route || !composer.matches?.('[data-slate-editor="true"][contenteditable="true"][role="textbox"]')) return null;const root = composer.closest('main,[role="main"]');- if (!root || attribute(root, 'aria-label') !== `${name} (channel)` || !visible(root)) return null;- const transcript = query(root, '[role="list"],ol,ul').find(node =>- attribute(node, 'aria-label') === `Messages in ${name}` && visible(node) && !contains(node, composer));+ if (!root || !visible(root)) return null;+ const transcript = query(root, '[data-list-id="chat-messages"]').find(node =>+ visible(node) && !contains(node, composer));const channelLink = query(document, 'nav a[href],[role="navigation"] a[href]').find(node => {try {const url = new URL(attribute(node, 'href'), window.location.href);return visible(node) && attribute(node, 'aria-current') === 'page' && url.origin === window.location.origin- && url.pathname.replace(/\/$/, '') === `/channels/${route[1]}/${route[2]}`- && attribute(node, 'aria-label') === `${name} (text channel)`;+ && url.pathname.replace(/\/$/, '') === `/channels/${route[1]}/${route[2]}`;} catch { return false; }});+ const name = /^Message #(.+)$/.exec(attribute(composer, 'aria-label'))?.[1]+ || /^Messages in (.+)$/.exec(attribute(transcript, 'aria-label'))?.[1]+ || /^(.+) \(channel\)$/.exec(attribute(root, 'aria-label'))?.[1]+ || `channel-${route[2]}`;return transcript && channelLink? { root, transcript, name, channelId: route[2], id: `discord:${route[1]}:${route[2]}` }: null;@@ -280,10 +282,52 @@return normalizeText(read(content));};+ const discordAttachmentDescriptions = (row, messageId) => {+ const accessories = query(row, `[id="message-accessories-${messageId}"]`)[0];+ if (!accessories) return [];+ const reactions = query(accessories, `[id="message-reactions-${messageId}"]`)[0];+ const selectors = [+ '[role="img"][aria-label]',+ 'img[alt]',+ '[data-filename]',+ '[data-name]',+ '[class*="fileName"]',+ '[class*="filename"]',+ '[class*="embedTitle"]',+ '[class*="embedDescription"]',+ 'a[href*="/attachments/"]',+ 'a[href*="/ephemeral-attachments/"]',+ ];+ const seen = new Set();+ const descriptions = [];+ for (const node of queryMany(accessories, selectors)) {+ if (!visible(node) || (reactions && contains(reactions, node))) continue;+ const labelledMedia = node.closest?.('[role="img"][aria-label]');+ if (labelledMedia && labelledMedia !== node) continue;+ const value = compact(+ attribute(node, 'aria-label')+ || attribute(node, 'alt')+ || attribute(node, 'title')+ || attribute(node, 'data-filename')+ || attribute(node, 'data-name')+ || node.innerText+ || node.textContent,+ 240,+ );+ const key = canonicalText(value);+ if (!value || seen.has(key)) continue;+ seen.add(key);+ descriptions.push(value);+ }+ return descriptions;+ };+const collectDiscordMessages = (channel) => {- const panel = query(document, 'section[aria-label="User status and settings"]')[0];- const ownAvatarId = discordAvatarUserId(query(panel, 'img[src*="/avatars/"]')[0]);- const ownName = compact(query(panel, '[class*="panelTitle"]')[0]?.textContent, 240);+ const account = query(document, '[class*="accountPopoutButtonWrapper"]')[0]?.parentElement;+ const ownAvatarId = discordAvatarUserId(query(account, 'img[src*="/avatars/"]')[0]);+ const ownNames = new Set(query(account, '[class*="panelTitle"],[class*="hovered"]')+ .map(node => canonicalText(node.textContent))+ .filter(Boolean));const rows = query(channel.transcript, '[role="article"][data-list-item-id]');const messages = [];let previousAuthor = '';@@ -301,14 +345,16 @@continue;}const content = query(row, `[id="message-content-${messageId}"]`)[0];- const body = discordMessageText(content);+ const text = discordMessageText(content);+ const attachments = discordAttachmentDescriptions(row, messageId);+ const body = [text, ...attachments.map(value => `Attachment: ${value}`)].filter(Boolean).join('\n');const author = compact(attribute(username, 'data-text') || username?.textContent, 240)|| (!username ? previousAuthor : '');const avatar = username?.closest('h3')?.parentElement?.querySelector('img[src]');const avatarId = discordAvatarUserId(avatar);let direction = 'unknown';if (ownAvatarId && avatarId) direction = ownAvatarId === avatarId ? 'outgoing' : 'incoming';- else if (author && ownName && author !== ownName) direction = 'incoming';+ else if (author && ownNames.size) direction = ownNames.has(canonicalText(author)) ? 'outgoing' : 'incoming';else if (!username) direction = previousDirection;previousAuthor = author;previousDirection = direction;diff --git a/src/firefox/src/content/chat-observation.js b/src/firefox/src/content/chat-observation.jsindex 8f7da8d28..cd1127bbc 100644--- a/src/firefox/src/content/chat-observation.js+++ b/src/firefox/src/content/chat-observation.js@@ -138,20 +138,22 @@const discordChannel = (composer) => {if (!composer || !/^(?:www\.)?discord\.com$/.test(window.location?.hostname || '')) return null;const route = /^\/channels\/(\d+)\/(\d+)\/?$/.exec(window.location?.pathname || '');- const name = /^Message #(.+)$/.exec(attribute(composer, 'aria-label'))?.[1];- if (!route || !name || !composer.matches?.('[data-slate-editor="true"][contenteditable="true"][role="textbox"]')) return null;+ if (!route || !composer.matches?.('[data-slate-editor="true"][contenteditable="true"][role="textbox"]')) return null;const root = composer.closest('main,[role="main"]');- if (!root || attribute(root, 'aria-label') !== `${name} (channel)` || !visible(root)) return null;- const transcript = query(root, '[role="list"],ol,ul').find(node =>- attribute(node, 'aria-label') === `Messages in ${name}` && visible(node) && !contains(node, composer));+ if (!root || !visible(root)) return null;+ const transcript = query(root, '[data-list-id="chat-messages"]').find(node =>+ visible(node) && !contains(node, composer));const channelLink = query(document, 'nav a[href],[role="navigation"] a[href]').find(node => {try {const url = new URL(attribute(node, 'href'), window.location.href);return visible(node) && attribute(node, 'aria-current') === 'page' && url.origin === window.location.origin- && url.pathname.replace(/\/$/, '') === `/channels/${route[1]}/${route[2]}`- && attribute(node, 'aria-label') === `${name} (text channel)`;+ && url.pathname.replace(/\/$/, '') === `/channels/${route[1]}/${route[2]}`;} catch { return false; }});+ const name = /^Message #(.+)$/.exec(attribute(composer, 'aria-label'))?.[1]+ || /^Messages in (.+)$/.exec(attribute(transcript, 'aria-label'))?.[1]+ || /^(.+) \(channel\)$/.exec(attribute(root, 'aria-label'))?.[1]+ || `channel-${route[2]}`;return transcript && channelLink? { root, transcript, name, channelId: route[2], id: `discord:${route[1]}:${route[2]}` }: null;@@ -280,10 +282,52 @@return normalizeText(read(content));};+ const discordAttachmentDescriptions = (row, messageId) => {+ const accessories = query(row, `[id="message-accessories-${messageId}"]`)[0];+ if (!accessories) return [];+ const reactions = query(accessories, `[id="message-reactions-${messageId}"]`)[0];+ const selectors = [+ '[role="img"][aria-label]',+ 'img[alt]',+ '[data-filename]',+ '[data-name]',+ '[class*="fileName"]',+ '[class*="filename"]',+ '[class*="embedTitle"]',+ '[class*="embedDescription"]',+ 'a[href*="/attachments/"]',+ 'a[href*="/ephemeral-attachments/"]',+ ];+ const seen = new Set();+ const descriptions = [];+ for (const node of queryMany(accessories, selectors)) {+ if (!visible(node) || (reactions && contains(reactions, node))) continue;+ const labelledMedia = node.closest?.('[role="img"][aria-label]');+ if (labelledMedia && labelledMedia !== node) continue;+ const value = compact(+ attribute(node, 'aria-label')+ || attribute(node, 'alt')+ || attribute(node, 'title')+ || attribute(node, 'data-filename')+ || attribute(node, 'data-name')+ || node.innerText+ || node.textContent,+ 240,+ );+ const key = canonicalText(value);+ if (!value || seen.has(key)) continue;+ seen.add(key);+ descriptions.push(value);+ }+ return descriptions;+ };+const collectDiscordMessages = (channel) => {- const panel = query(document, 'section[aria-label="User status and settings"]')[0];- const ownAvatarId = discordAvatarUserId(query(panel, 'img[src*="/avatars/"]')[0]);- const ownName = compact(query(panel, '[class*="panelTitle"]')[0]?.textContent, 240);+ const account = query(document, '[class*="accountPopoutButtonWrapper"]')[0]?.parentElement;+ const ownAvatarId = discordAvatarUserId(query(account, 'img[src*="/avatars/"]')[0]);+ const ownNames = new Set(query(account, '[class*="panelTitle"],[class*="hovered"]')+ .map(node => canonicalText(node.textContent))+ .filter(Boolean));const rows = query(channel.transcript, '[role="article"][data-list-item-id]');const messages = [];let previousAuthor = '';@@ -301,14 +345,16 @@continue;}const content = query(row, `[id="message-content-${messageId}"]`)[0];- const body = discordMessageText(content);+ const text = discordMessageText(content);+ const attachments = discordAttachmentDescriptions(row, messageId);+ const body = [text, ...attachments.map(value => `Attachment: ${value}`)].filter(Boolean).join('\n');const author = compact(attribute(username, 'data-text') || username?.textContent, 240)|| (!username ? previousAuthor : '');const avatar = username?.closest('h3')?.parentElement?.querySelector('img[src]');const avatarId = discordAvatarUserId(avatar);let direction = 'unknown';if (ownAvatarId && avatarId) direction = ownAvatarId === avatarId ? 'outgoing' : 'incoming';- else if (author && ownName && author !== ownName) direction = 'incoming';+ else if (author && ownNames.size) direction = ownNames.has(canonicalText(author)) ? 'outgoing' : 'incoming';else if (!username) direction = previousDirection;previousAuthor = author;previousDirection = direction;diff --git a/test/discord-guard.mjs b/test/discord-guard.mjsindex d2cb549fc..7d985a149 100644--- a/test/discord-guard.mjs+++ b/test/discord-guard.mjs@@ -6,7 +6,7 @@ import { Agent } from '../src/chrome/src/agent/agent.js';import { Agent as FirefoxAgent } from '../src/firefox/src/agent/agent.js';import { getMessageRecipientGuardPolicy } from '../src/chrome/src/agent/adapters.js';import { getMessageRecipientGuardPolicy as firefoxPolicy } from '../src/firefox/src/agent/adapters.js';-import { advanceChatSession, createChatSession } from '../src/chrome/src/agent/chat-workflow.js';+import { advanceChatSession, createChatSession, markChatSendPending } from '../src/chrome/src/agent/chat-workflow.js';const url = 'https://discord.com/channels/123/456';const fixture = `<!doctype html><style>@@ -24,7 +24,7 @@ ol {min-height:30px} main {position:fixed;left:320px;top:0;width:550px;height:70<a id="channel" href="/channels/123/456" aria-label="general (text channel)" aria-current="page">general</a><button id="edit" type="button" aria-label="Edit Channel">Edit Channel</button><button id="nav-send" type="button">Send message</button></nav>-<section aria-label="User status and settings"><div class="panelTitleContainer_fixture">WebBrain</div><img src="https://cdn.discordapp.com/avatars/11/self.webp?size=56"></section>+<section aria-label="User status and settings"><div class="accountPopoutButtonWrapper_fixture"><img id="profile-avatar" src="https://cdn.discordapp.com/avatars/11/self.webp?size=56"></div><div class="nameTag_fixture"><div class="panelTitleContainer_fixture">WebBrain</div><div class="panelSubtext_fixture"><span class="hovered_fixture">webbrain_one</span></div></div></section><main aria-label="general (channel)"><h2>general chat</h2><ol role="list" aria-label="Messages in general" data-list-id="chat-messages"><li><div id="message" role="article" data-list-item-id="chat-messages___chat-messages-456-1001"><div class="contents"><img src="https://cdn.discordapp.com/avatars/22/other.webp?size=160"><h3><span id="message-username-1001"><span data-text="Ficsit">Ficsit</span></span><time id="message-timestamp-1001" datetime="2026-09-29T01:00:00.000Z"></time></h3><div id="message-content-1001">Hello from the fixture</div></div><div role="group" aria-label="Message Actions"><button id="wave">Wave to say hi!</button><button id="lookalike" aria-expanded="false">Test, server actions</button></div></div></li></ol><div id="composer" role="textbox" aria-label="Message #general" contenteditable="true" data-slate-editor="true">Draft</div><button id="send" type="button">Send</button></main>`;@@ -192,7 +192,22 @@ for (const [kind, engine, AgentClass, policy] of [id:'discord:456:1001', direction:'incoming', text:'Hello from the fixture',author:'Ficsit', timestamp:'2026-09-29T01:00:00.000Z',}]);- let advanced=advanceChatSession(createChatSession({threadKey:result.threadKey}),result);+ await page.locator('#composer').evaluate(el=>el.setAttribute('aria-label','Nachricht #general'));+ await page.locator('main').evaluate(el=>el.setAttribute('aria-label','general (Kanal)'));+ await page.locator('[data-list-id=chat-messages]').evaluate(el=>el.setAttribute('aria-label','Nachrichten in general'));+ await page.locator('#channel').evaluate(el=>el.setAttribute('aria-label','general (Textkanal)'));+ await page.locator('section').evaluate(el=>el.setAttribute('aria-label','Benutzerstatus und Einstellungen'));+ const localized=await observe();+ assert.equal(localized.success,true,JSON.stringify(localized));+ assert.equal(localized.conversationId,result.conversationId);+ assert.equal(localized.threadKey,result.threadKey);+ assert.deepEqual(localized.messages,result.messages);++ await page.locator('#profile-avatar').evaluate(el=>el.setAttribute('src','/assets/embed/avatars/3.png'));+ await page.locator('#message .contents img').evaluate(el=>el.setAttribute('src','/assets/embed/avatars/3.png'));+ const defaultAvatars=await observe();+ assert.equal(defaultAvatars.messages[0].direction,'incoming','a shared Discord default avatar does not make a different author outgoing');+ let advanced=advanceChatSession(createChatSession({threadKey:result.threadKey}),defaultAvatars);await page.locator('#wave').evaluate(el=>el.textContent='Changed hover action');const afterHover=await observe();assert.deepEqual(afterHover.messages,result.messages,'message actions do not alter observed content');@@ -201,16 +216,20 @@ for (const [kind, engine, AgentClass, policy] of [await page.locator('[data-list-id=chat-messages]').evaluate(el=>el.insertAdjacentHTML('beforeend',`<li><div role="article" data-list-item-id="chat-messages___chat-messages-456-1002"><div class="contents">- <img src="https://cdn.discordapp.com/avatars/11/self.webp?size=160"><h3><span id="message-username-1002"><span data-text="WebBrain">WebBrain</span></span><time id="message-timestamp-1002" datetime="2026-09-29T01:01:00.000Z"></time></h3>+ <img src="/assets/embed/avatars/3.png"><h3><span id="message-username-1002"><span data-text="WebBrain">WebBrain</span></span><time id="message-timestamp-1002" datetime="2026-09-29T01:01:00.000Z"></time></h3><div id="message-content-1002">My answer</div></div></div></li><li><div role="article" data-list-item-id="chat-messages___chat-messages-999-1003"><div id="message-content-1003">Wrong channel</div></div></li><li><div role="article" class="isSystemMessage_fixture" data-list-item-id="chat-messages___chat-messages-456-1004"><div id="message-content-1004">A member joined</div></div></li>`));const afterSend=await observe();assert.deepEqual(afterSend.messages.map(item=>item.id),['discord:456:1001','discord:456:1002']);assert.equal(afterSend.messages[1].direction,'outgoing');- advanced=advanceChatSession(advanced.session,afterSend);+ const pending=markChatSendPending(advanced.session,{+ ok:true,messageKey:'test-pending',threadKey:result.threadKey,text:'My answer',+ });+ advanced=advanceChatSession(pending,afterSend);assert.equal(advanced.newMessages[0].direction,'outgoing');assert.equal(advanced.session.state,'we_responded');+ assert.equal(advanced.session.pendingOutbound,null,'self-authored default-avatar message clears pending send');await page.locator('[data-list-id=chat-messages]').evaluate(el=>el.insertAdjacentHTML('beforeend',`<li><div role="article" data-list-item-id="chat-messages___chat-messages-456-1005">@@ -223,6 +242,18 @@ for (const [kind, engine, AgentClass, policy] of [advanced=advanceChatSession(advanced.session,afterReply);assert.deepEqual(advanced.newMessages.map(item=>item.id),['discord:456:1005']);assert.equal(advanced.nextAction,'reply');+ await page.locator('[data-list-id=chat-messages]').evaluate(el=>el.insertAdjacentHTML('beforeend',`+ <li><div role="article" data-list-item-id="chat-messages___chat-messages-456-1006"><div class="contents">+ <img src="/assets/embed/avatars/3.png"><h3><span id="message-username-1006"><span data-text="Ficsit">Ficsit</span></span><time id="message-timestamp-1006" datetime="2026-09-29T01:03:00.000Z"></time></h3>+ <div id="message-content-1006"></div></div><div id="message-accessories-1006"><div class="attachment_fixture"><img width="64" height="64" alt="support-error.png" src="/assets/support-error.png"></div>+ <div role="img" aria-label="Sticker, Wave" style="display:block;width:24px;height:24px"></div><div class="embedTitle_fixture">Build error report</div><div class="embedDescription_fixture">Setup fails on startup</div></div></div></li>`));+ const afterAttachment=await observe();+ assert.equal(afterAttachment.messages[3].direction,'incoming');+ assert.equal(afterAttachment.messages[3].text,+ 'Attachment: support-error.png\nAttachment: Sticker, Wave\nAttachment: Build error report\nAttachment: Setup fails on startup');+ advanced=advanceChatSession(advanced.session,afterAttachment);+ assert.deepEqual(advanced.newMessages.map(item=>item.id),['discord:456:1006']);+ assert.equal(advanced.nextAction,'reply');await page.locator('[data-list-id=chat-messages]').evaluate(el=>el.replaceChildren());assert.equal((await observe()).success,true);await page.locator('#channel').evaluate(el=>el.setAttribute('href','/channels/123/999'));