Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Side panel: no icon when a favicon fails; accept PNG .ico files
Emre Sokulluc5ad237c81f0
diff --git a/docs/privacy-and-data-flow.md b/docs/privacy-and-data-flow.mdindex 93d4193b0..2c617365a 100644--- a/docs/privacy-and-data-flow.md+++ b/docs/privacy-and-data-flow.md@@ -625,7 +625,7 @@ The only outbound HTTP requests are:Icons are cached in memory while the panel is open, with bounded requestsand response sizes. Local hosts, IP addresses, code, Markdown image syntaxand download links do not trigger these requests. Strict Secret Mode disables- remote icon lookups; unavailable icons use a local globe.+ remote icon lookups; a link whose icon is unavailable shows no icon.11. **Feedback trace attachments** uploaded by GitHub's native issue editor,only after the user chooses **Upload trace and open GitHub** in the extension.Preparing, viewing, and downloading the trace are local operations. Thediff --git a/docs/prompt-injection-defense.md b/docs/prompt-injection-defense.mdindex 98c811a4b..682482d64 100644--- a/docs/prompt-injection-defense.md+++ b/docs/prompt-injection-defense.md@@ -51,7 +51,7 @@ them in sync — the test suite asserts the pure modules are byte-identical.icons come only from a fixed favicon service using a validated publichostname, without URL paths, query strings, credentials or a referrer.Responses are bounded raster images rendered through local blob URLs;- failed lookups keep a local globe icon.+ a link whose icon cannot load shows no icon.- Code: `ui/markdown-link.js`, `ui/markdown-render.js`, `ui/chat-link-icons.js`.---diff --git a/src/chrome/src/ui/chat-link-icons.js b/src/chrome/src/ui/chat-link-icons.jsindex e22e76cdc..8c0f30821 100644--- a/src/chrome/src/ui/chat-link-icons.js+++ b/src/chrome/src/ui/chat-link-icons.js@@ -21,12 +21,16 @@ function publicHostname(href, baseURI) {} catch { return ''; }}-function rasterSignature(bytes, mime) {- if (mime === 'image/png') return [137, 80, 78, 71, 13, 10, 26, 10].every((value, index) => bytes[index] === value);- if (mime === 'image/jpeg') return bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255;+// The type the bytes really have, or '' when they do not match the declared+// one. Many sites serve a PNG as favicon.ico, so an icon type accepts PNG too.+function rasterType(bytes, mime) {+ const png = [137, 80, 78, 71, 13, 10, 26, 10].every((value, index) => bytes[index] === value);+ if (mime === 'image/png') return png ? mime : '';+ if (mime === 'image/jpeg') return bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255 ? mime : '';if (mime === 'image/webp') return String.fromCharCode(...bytes.slice(0, 4)) === 'RIFF'- && String.fromCharCode(...bytes.slice(8, 12)) === 'WEBP';- return bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 1 && bytes[3] === 0;+ && String.fromCharCode(...bytes.slice(8, 12)) === 'WEBP' ? mime : '';+ if (png) return 'image/png';+ return bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 1 && bytes[3] === 0 ? mime : '';}async function readIcon(response) {@@ -58,14 +62,21 @@ async function readIcon(response) {const bytes = new Uint8Array(total);let offset = 0;for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; }- if (!rasterSignature(bytes, mime)) throw new Error('Invalid favicon');- return new Blob([bytes], { type: mime });+ const type = rasterType(bytes, mime);+ if (!type) throw new Error('Invalid favicon');+ return new Blob([bytes], { type });+}++// A link whose favicon cannot load shows no icon and gives back its space.+function hideIcon(link) {+ link.classList.remove('chat-link-icon-ready');+ link.classList.add('chat-link-no-icon');}/*** Decorate chat links without allowing model text to choose an image URL.* Only a public hostname reaches the fixed favicon service; paths, queries,- * fragments, cookies and referrers are never sent. Failures retain the CSS icon.+ * fragments, cookies and referrers are never sent. A failed icon shows nothing.* The returned function releases observers, requests and local image URLs.*/export function installChatLinkIcons(root, {@@ -104,7 +115,7 @@ export function installChatLinkIcons(root, {if (stillCurrent(image, entry)) image.parentElement.classList.add('chat-link-icon-ready');};image.onerror = () => {- if (stillCurrent(image, entry)) image.parentElement.classList.remove('chat-link-icon-ready');+ if (stillCurrent(image, entry)) hideIcon(image.parentElement);image.removeAttribute('src');};image.src = entry.url;@@ -135,6 +146,7 @@ export function installChatLinkIcons(root, {for (const image of entry.images) applyImage(image, entry);} catch {entry.status = 'failed';+ for (const image of entry.images) if (stillCurrent(image, entry)) hideIcon(image.parentElement);} finally {clearTimeout(timer);controllers.delete(controller);@@ -160,13 +172,14 @@ export function installChatLinkIcons(root, {if (!entry) {// A hard per-panel bound prevents long chats or streamed replacements// from causing unbounded lookups or an eviction/re-fetch loop.- if (entries.size >= cacheLimit || typeof fetchImpl !== 'function') return;+ if (entries.size >= cacheLimit || typeof fetchImpl !== 'function') { hideIcon(link); return; }entry = { hostname: state.hostname, status: 'queued', url: '', images: new Set() };entries.set(state.hostname, entry);queue.push(entry);}if (entry.status === 'ready') applyImage(state.image, entry);- else if (entry.status !== 'failed') {+ else if (entry.status === 'failed') hideIcon(link);+ else {for (const image of entry.images) if (!stillCurrent(image, entry)) entry.images.delete(image);entry.images.add(state.image);}@@ -222,7 +235,7 @@ export function installChatLinkIcons(root, {previous?.image?.remove();// Restored chat HTML may contain an object URL from a closed panel.for (const image of link.querySelectorAll('.chat-link-icon')) image.remove();- link.classList.remove('chat-link-icon-ready');+ link.classList.remove('chat-link-icon-ready', 'chat-link-no-icon');link.classList.toggle('chat-link', Boolean(hostname));const state = { href, hostname, image: null };decorated.set(link, state);diff --git a/src/firefox/src/ui/chat-link-icons.js b/src/firefox/src/ui/chat-link-icons.jsindex e22e76cdc..8c0f30821 100644--- a/src/firefox/src/ui/chat-link-icons.js+++ b/src/firefox/src/ui/chat-link-icons.js@@ -21,12 +21,16 @@ function publicHostname(href, baseURI) {} catch { return ''; }}-function rasterSignature(bytes, mime) {- if (mime === 'image/png') return [137, 80, 78, 71, 13, 10, 26, 10].every((value, index) => bytes[index] === value);- if (mime === 'image/jpeg') return bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255;+// The type the bytes really have, or '' when they do not match the declared+// one. Many sites serve a PNG as favicon.ico, so an icon type accepts PNG too.+function rasterType(bytes, mime) {+ const png = [137, 80, 78, 71, 13, 10, 26, 10].every((value, index) => bytes[index] === value);+ if (mime === 'image/png') return png ? mime : '';+ if (mime === 'image/jpeg') return bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255 ? mime : '';if (mime === 'image/webp') return String.fromCharCode(...bytes.slice(0, 4)) === 'RIFF'- && String.fromCharCode(...bytes.slice(8, 12)) === 'WEBP';- return bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 1 && bytes[3] === 0;+ && String.fromCharCode(...bytes.slice(8, 12)) === 'WEBP' ? mime : '';+ if (png) return 'image/png';+ return bytes[0] === 0 && bytes[1] === 0 && bytes[2] === 1 && bytes[3] === 0 ? mime : '';}async function readIcon(response) {@@ -58,14 +62,21 @@ async function readIcon(response) {const bytes = new Uint8Array(total);let offset = 0;for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; }- if (!rasterSignature(bytes, mime)) throw new Error('Invalid favicon');- return new Blob([bytes], { type: mime });+ const type = rasterType(bytes, mime);+ if (!type) throw new Error('Invalid favicon');+ return new Blob([bytes], { type });+}++// A link whose favicon cannot load shows no icon and gives back its space.+function hideIcon(link) {+ link.classList.remove('chat-link-icon-ready');+ link.classList.add('chat-link-no-icon');}/*** Decorate chat links without allowing model text to choose an image URL.* Only a public hostname reaches the fixed favicon service; paths, queries,- * fragments, cookies and referrers are never sent. Failures retain the CSS icon.+ * fragments, cookies and referrers are never sent. A failed icon shows nothing.* The returned function releases observers, requests and local image URLs.*/export function installChatLinkIcons(root, {@@ -104,7 +115,7 @@ export function installChatLinkIcons(root, {if (stillCurrent(image, entry)) image.parentElement.classList.add('chat-link-icon-ready');};image.onerror = () => {- if (stillCurrent(image, entry)) image.parentElement.classList.remove('chat-link-icon-ready');+ if (stillCurrent(image, entry)) hideIcon(image.parentElement);image.removeAttribute('src');};image.src = entry.url;@@ -135,6 +146,7 @@ export function installChatLinkIcons(root, {for (const image of entry.images) applyImage(image, entry);} catch {entry.status = 'failed';+ for (const image of entry.images) if (stillCurrent(image, entry)) hideIcon(image.parentElement);} finally {clearTimeout(timer);controllers.delete(controller);@@ -160,13 +172,14 @@ export function installChatLinkIcons(root, {if (!entry) {// A hard per-panel bound prevents long chats or streamed replacements// from causing unbounded lookups or an eviction/re-fetch loop.- if (entries.size >= cacheLimit || typeof fetchImpl !== 'function') return;+ if (entries.size >= cacheLimit || typeof fetchImpl !== 'function') { hideIcon(link); return; }entry = { hostname: state.hostname, status: 'queued', url: '', images: new Set() };entries.set(state.hostname, entry);queue.push(entry);}if (entry.status === 'ready') applyImage(state.image, entry);- else if (entry.status !== 'failed') {+ else if (entry.status === 'failed') hideIcon(link);+ else {for (const image of entry.images) if (!stillCurrent(image, entry)) entry.images.delete(image);entry.images.add(state.image);}@@ -222,7 +235,7 @@ export function installChatLinkIcons(root, {previous?.image?.remove();// Restored chat HTML may contain an object URL from a closed panel.for (const image of link.querySelectorAll('.chat-link-icon')) image.remove();- link.classList.remove('chat-link-icon-ready');+ link.classList.remove('chat-link-icon-ready', 'chat-link-no-icon');link.classList.toggle('chat-link', Boolean(hostname));const state = { href, hostname, image: null };decorated.set(link, state);diff --git a/test/chat-links.mjs b/test/chat-links.mjsindex fc58b05d7..37343799d 100644--- a/test/chat-links.mjs+++ b/test/chat-links.mjs@@ -626,11 +626,11 @@ for (const build of ['chrome', 'firefox']) {const { installChatLinkIcons } = await import('/chat-link-icons.js');const dependencies = { ...helpers, ...links, escapeHtml, t: key => key, scheduleMathRender() {} };const format = new Function(...Object.keys(dependencies), `return (${formatter})`)(...Object.values(dependencies));- // x.com gets a favicon image; the other hosts keep the CSS globe.+ // x.com gets a favicon image; the other hosts have none to show.document.querySelector('#chat-test-content').innerHTML = format(['Verified post URL: https://x.com/EmreSokullu/status/2114988545588736396','Try it: https://webbrain.one',- 'Fallback icon URL: https://twitter.com/EmreSokullu/status/2114988545588736396',+ 'Unavailable icon URL: https://twitter.com/EmreSokullu/status/2114988545588736396',].join('\n\n'), { enhance: false });const canvas = document.createElement('canvas');canvas.width = 14; canvas.height = 14;@@ -638,12 +638,14 @@ for (const build of ['chrome', 'firefox']) {const png = Uint8Array.from(atob(canvas.toDataURL('image/png').split(',')[1]), char => char.charCodeAt(0));installChatLinkIcons(document.querySelector('#messages'), {intersectionObserverImpl: null,+ // The live icon service sends x.com's PNG labelled as an .ico.fetchImpl: async url => String(url).endsWith('/x.com.ico')- ? new Response(png, { headers: { 'content-type': 'image/png' } })+ ? new Response(png, { headers: { 'content-type': 'image/x-icon' } }): new Response('', { status: 404 }),});}, { formatter: formatterSource(build) });- await page.waitForFunction(() => document.querySelector('#chat-test-content a.chat-link-icon-ready > img')?.complete);+ await page.waitForFunction(() => document.querySelector('#chat-test-content a.chat-link-icon-ready > img')?.complete+ && document.querySelectorAll('#chat-test-content a.chat-link-no-icon').length === 2);const layout = await page.evaluate(() => {const box = ({ left, right, top, bottom }) => ({ left, right, top, bottom });const content = box(document.querySelector('#chat-test-content').getBoundingClientRect());@@ -661,7 +663,8 @@ for (const build of ['chrome', 'firefox']) {firstCharacter: box(firstCharacter.getBoundingClientRect()),label: box([...label.getClientRects()].at(-1)),icon: icon && box(icon.getBoundingClientRect()),- fallbackIcon: icon ? null : getComputedStyle(link, '::before').position,+ noIcon: link.classList.contains('chat-link-no-icon'),+ generatedIcon: getComputedStyle(link, '::before').content,};});});@@ -670,20 +673,22 @@ for (const build of ['chrome', 'firefox']) {for (const link of layout) {const [first] = link.fragments;assert.ok(sameLine(first, link.firstCharacter), `${link.href}: the link's first line holds its first character`);- assert.ok(link.firstCharacter.left - first.left >= 14, `${link.href}: icon space precedes the first character`);+ assert.equal(link.generatedIcon, 'none', `${link.href}: no generic placeholder icon`);for (const fragment of link.fragments) {assert.ok(fragment.left >= link.content.left - 1 && fragment.right <= link.content.right + 1, `${link.href}: no overflow`);}if (link.icon) {+ assert.ok(link.firstCharacter.left - first.left >= 14, `${link.href}: icon space precedes the first character`);assert.ok(sameLine(link.icon, link.firstCharacter), `${link.href}: favicon shares the first line`);assert.ok(Math.abs(link.icon.left - first.left) <= 1 && link.icon.right <= link.firstCharacter.left, `${link.href}: favicon leads the link`);} else {- assert.equal(link.fallbackIcon, 'absolute', `${link.href}: the CSS icon is drawn in the reserved space`);+ assert.ok(link.noIcon, `${link.href}: a link without a favicon is marked iconless`);+ assert.ok(Math.abs(link.firstCharacter.left - first.left) <= 1, `${link.href}: no empty icon space`);}}- const [post, tryIt, fallback] = layout;+ const [post, tryIt, noIcon] = layout;assert.ok(post.icon, 'the x.com link uses its favicon image');- assert.ok(post.fragments.length > 1 && fallback.fragments.length > 1, 'long post URLs wrap inside the panel');+ assert.ok(post.fragments.length > 1 && noIcon.fragments.length > 1, 'long post URLs wrap inside the panel');assert.equal(tryIt.fragments.length, 1);assert.ok(sameLine(tryIt.label, tryIt.fragments[0]), '"Try it:" and its short link share one line');@@ -713,4 +718,79 @@ for (const build of ['chrome', 'firefox']) {await browser.close();}});++ if (process.env.WEBBRAIN_CHAT_LINKS_DOM === '1') test(`${build}: favicon responses show the site icon or no icon at all`, async () => {+ const { chromium, firefox } = await import('playwright');+ const browser = await (build === 'chrome' ? chromium : firefox).launch({ headless: true });+ try {+ const page = await browser.newPage();+ await page.route('**/*', async route => {+ const file = new URL(route.request().url()).pathname.slice(1);+ if (/^[\w-]+\.js$/.test(file)) await route.fulfill({ contentType: 'text/javascript', body: readUi(build, file) });+ else await route.fulfill({ contentType: 'text/html', body: '<html data-theme="light"><body><div id="messages"><div class="message assistant"><div class="message-content"><div id="chat-test-content" class="message-text"></div></div></div></div><div class="message-content"><div id="limited" class="message-text"></div></div></body></html>' });+ });+ await page.goto('http://chat-links.test/');+ await page.addStyleTag({ content: fs.readFileSync(new URL(`../src/${build}/styles/sidepanel.css`, import.meta.url), 'utf8') });+ const icons = await page.evaluate(async () => {+ const { installChatLinkIcons } = await import('/chat-link-icons.js');+ const canvas = document.createElement('canvas');+ canvas.width = 14; canvas.height = 14;+ canvas.getContext('2d').fillRect(0, 0, 14, 14);+ const png = Uint8Array.from(atob(canvas.toDataURL('image/png').split(',')[1]), char => char.charCodeAt(0));+ // A real .ico container holding that PNG.+ const ico = new Uint8Array(22 + png.length);+ ico.set([0, 0, 1, 0, 1, 0, 14, 14, 0, 0, 1, 0, 32, 0]);+ new DataView(ico.buffer).setUint32(14, png.length, true);+ new DataView(ico.buffer).setUint32(18, 22, true);+ ico.set(png, 22);+ const responses = {+ 'x.com': () => new Response(png, { headers: { 'content-type': 'image/x-icon' } }),+ 'github.com': () => new Response(ico, { headers: { 'content-type': 'image/x-icon' } }),+ 'stackoverflow.com': () => new Response('<html>not an icon</html>', { headers: { 'content-type': 'image/x-icon' } }),+ 'wikipedia.org': () => new Response(png, { headers: { 'content-type': 'image/jpeg' } }),+ 'mozilla.org': () => new Response('', { status: 404 }),+ 'apple.com': () => { throw new Error('Strict Secret Mode'); },+ };+ const fetchImpl = async url => responses[/\/ip3\/(.+)\.ico$/.exec(String(url))[1]]();+ const links = hosts => hosts.map(host => `<a href="https://${host}/page">${host}</a>`).join(' ');+ document.querySelector('#chat-test-content').innerHTML = links(Object.keys(responses));+ installChatLinkIcons(document.querySelector('#messages'), { intersectionObserverImpl: null, fetchImpl });+ // Past the per-panel domain limit a link shows no icon instead of waiting.+ document.querySelector('#limited').innerHTML = links(['example.com', 'example.net']);+ installChatLinkIcons(document.querySelector('#limited'), { intersectionObserverImpl: null, fetchImpl: async () => new Response(png, { headers: { 'content-type': 'image/png' } }), maxDomains: 1 });+ const settled = () => [...document.querySelectorAll('a')].every(link => /chat-link-(?:icon-ready|no-icon)/.test(link.className));+ for (let wait = 0; wait < 100 && !settled(); wait++) await new Promise(resolve => setTimeout(resolve, 50));+ return Promise.all([...document.querySelectorAll('a')].map(async link => {+ const image = link.querySelector('img.chat-link-icon');+ return {+ host: link.textContent,+ ready: link.classList.contains('chat-link-icon-ready'),+ noIcon: link.classList.contains('chat-link-no-icon'),+ imageShown: !!image && getComputedStyle(image).display !== 'none',+ imageType: image?.src ? (await (await fetch(image.src)).blob()).type : '',+ space: getComputedStyle(link).paddingInlineStart,+ generatedIcon: getComputedStyle(link, '::before').content,+ };+ }));+ });+ const byHost = Object.fromEntries(icons.map(icon => [icon.host, icon]));+ for (const host of ['x.com', 'github.com', 'example.com']) {+ assert.equal(byHost[host].ready, true, `${host}: favicon shown`);+ assert.equal(byHost[host].imageShown, true, host);+ assert.notEqual(byHost[host].space, '0px', `${host}: room for the favicon`);+ }+ assert.equal(byHost['x.com'].imageType, 'image/png', 'a PNG served as an .ico is stored as the PNG it is');+ assert.equal(byHost['github.com'].imageType, 'image/x-icon');+ for (const host of ['stackoverflow.com', 'wikipedia.org', 'mozilla.org', 'apple.com', 'example.net']) {+ assert.deepEqual(+ { ready: byHost[host].ready, noIcon: byHost[host].noIcon, imageShown: byHost[host].imageShown, space: byHost[host].space },+ { ready: false, noIcon: true, imageShown: false, space: '0px' },+ `${host}: no icon and no empty space`,+ );+ }+ for (const icon of icons) assert.equal(icon.generatedIcon, 'none', `${icon.host}: no generic placeholder icon`);+ } finally {+ await browser.close();+ }+ });}