Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
fix(agent): match specific attachment names without substring collisions and require each media type for mixed requests
Emre Sokullu9777a9df861a
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.jsindex 68e450ffa..e9f0cb24a 100644--- a/src/chrome/src/agent/agent.js+++ b/src/chrome/src/agent/agent.js@@ -2391,12 +2391,104 @@ export class Agent extends LoopDetector {}_cleanSpecificAttachmentTarget(value) {- const text = String(value || '').trim();- const match = text.match(/(?:(?:an?|the|\d+|one|two|three|four|five|six|seven|eight|nine|ten)\s+)?(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?)\s+(?:of|named|called|with\s+name)\s+(.+)$/i);- if (match) return match[1].trim().toLowerCase();+ let text = String(value || '').trim();+ text = text.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();+ const prefixMatch = text.match(/(?:(?:an?|the|\d+|one|two|three|four|five|six|seven|eight|nine|ten|une?|ein(?:e|en)?|un[ao]?|el|la|le|l'|gli|il)\s+)?(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)\s*(?:of|named|called|with\s+name|de|von|d'|d’|di|con\s+nombre|이름의|名為|名为|名前の|:|:)\s*(.+)$/i)+ || text.match(/^(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)\s*[::]\s*(.+)$/i);+ if (prefixMatch) {+ let target = prefixMatch[1].trim();+ target = target.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();+ if (target) return target.toLowerCase();+ }+ const suffixMatch = text.match(/^(.+?)\s*(?:の|의)?\s*(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)$/i);+ if (suffixMatch) {+ let target = suffixMatch[1].trim();+ target = target.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();+ if (target) return target.toLowerCase();+ }return text.toLowerCase();}+ _targetAttachmentNames(specificTarget) {+ const targets = new Set();+ if (!specificTarget) return [];+ const clean = String(specificTarget).trim().replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim().toLowerCase();+ if (clean) {+ targets.add(clean);+ const segs = clean.split(/[?#]/)[0].split(/[/\\]/).filter(Boolean);+ if (segs.length > 0) {+ targets.add(segs[segs.length - 1]);+ }+ }+ return Array.from(targets);+ }++ _extractAttachmentCandidateNames(att) {+ const rawCandidates = [];+ if (typeof att === 'string') {+ rawCandidates.push(att);+ } else if (att && typeof att === 'object') {+ if (att.name) rawCandidates.push(String(att.name));+ if (att.src) rawCandidates.push(String(att.src));+ if (att.url) rawCandidates.push(String(att.url));+ if (att.alt) rawCandidates.push(String(att.alt));+ if (att.title) rawCandidates.push(String(att.title));+ if (att.ariaLabel) rawCandidates.push(String(att.ariaLabel));+ if (att.text) rawCandidates.push(String(att.text));+ }++ const candidateNames = new Set();++ for (const raw of rawCandidates) {+ if (!raw) continue;+ const trimmed = String(raw).trim();+ if (!trimmed) continue;++ const cleanDirect = trimmed.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim().toLowerCase();+ if (cleanDirect) {+ candidateNames.add(cleanDirect);+ }++ const cleanedPhrase = this._cleanSpecificAttachmentTarget(cleanDirect);+ if (cleanedPhrase) {+ candidateNames.add(cleanedPhrase);+ }++ try {+ const urlObj = new URL(trimmed, 'https://example.invalid');+ const pathname = urlObj.pathname;+ const segments = pathname.split('/').filter(Boolean);+ if (segments.length > 0) {+ let last = segments[segments.length - 1];+ try { last = decodeURIComponent(last); } catch {}+ last = last.trim().toLowerCase();+ if (last) candidateNames.add(last);+ }+ } catch {+ const withoutQuery = trimmed.split(/[?#]/)[0];+ const segments = withoutQuery.split(/[/\\]/).filter(Boolean);+ if (segments.length > 0) {+ const last = segments[segments.length - 1].trim().toLowerCase();+ if (last) candidateNames.add(last);+ }+ }++ const tokens = trimmed.split(/[\s,;:()[\]{}<>"'“”«»`]+/);+ for (let token of tokens) {+ token = token.replace(/^[('"`“«]+|[)'"`”».!?,;:]+$/g, '').trim().toLowerCase();+ if (token) {+ candidateNames.add(token);+ const tokenSegments = token.split(/[?#]/)[0].split(/[/\\]/).filter(Boolean);+ if (tokenSegments.length > 0) {+ candidateNames.add(tokenSegments[tokenSegments.length - 1]);+ }+ }+ }+ }++ return Array.from(candidateNames);+ }+_parseWorkflowAttachmentRequirement(value) {const text = String(value || '').trim().toLowerCase();if (!text) return { isGeneric: true, expectedCount: 1, expectedImageCount: 0, expectedVideoCount: 0, wantsImage: false, wantsVideo: false, wantsGif: false, normalized: '' };@@ -2450,9 +2542,9 @@ export class Agent extends LoopDetector {if (expectedImageCount > 0 && expectedVideoCount > 0) {expectedCount = expectedImageCount + expectedVideoCount;} else if (expectedImageCount > 0) {- expectedCount = expectedImageCount;+ expectedCount = expectedImageCount + (wantsVideo ? 1 : 0);} else if (expectedVideoCount > 0) {- expectedCount = expectedVideoCount;+ expectedCount = expectedVideoCount + (wantsImage ? 1 : 0);} else {const genericNounCountMatch = text.match(new RegExp(`(?:^|[\\s,;and+와과및])(${countPrefix})\\s+(?:attachments?|files?|media|uploads?|pieces?|items?|assets?|enclosures?|documents?|fichiers?|archivos?|dateien?|allegati?|anexos?|вложения?|вложение|첨부(?:파일)?|파일|미디어)`, 'i'))|| text.match(new RegExp(`(?:^|[\\s,;and+와과및])(${countPrefix})\\s*(?:枚|つ|本|张|條|条|个|個|장|개|건|편)`, 'i'))@@ -2465,7 +2557,7 @@ export class Agent extends LoopDetector {}}- if (expectedImageCount === 0 && expectedVideoCount === 0 && wantsImage && wantsVideo && expectedCount < 2) {+ if (wantsImage && wantsVideo && expectedCount < 2) {expectedCount = 2;}@@ -2498,10 +2590,12 @@ export class Agent extends LoopDetector {};let matchingAttachments = rawAttachments;- if (parsed.expectedImageCount > 0 && parsed.expectedVideoCount > 0) {+ if (parsed.wantsImage && parsed.wantsVideo) {+ const minImages = parsed.expectedImageCount > 0 ? parsed.expectedImageCount : 1;+ const minVideos = parsed.expectedVideoCount > 0 ? parsed.expectedVideoCount : 1;const imageCount = rawAttachments.filter(isImageAttachment).length;const videoCount = rawAttachments.filter(isVideoAttachment).length;- if (imageCount < parsed.expectedImageCount || videoCount < parsed.expectedVideoCount) {+ if (imageCount < minImages || videoCount < minVideos) {return false;}} else if (parsed.wantsVideo && !parsed.wantsImage) {@@ -2519,12 +2613,21 @@ export class Agent extends LoopDetector {}const specificTarget = this._cleanSpecificAttachmentTarget(parsed.normalized);+ const targetNames = this._targetAttachmentNames(specificTarget);+const matchesSpecific = matchingAttachments.some(att => {- const src = String(att?.src || att?.url || '').toLowerCase();- const alt = String(att?.alt || att?.name || '').toLowerCase();- const text = String(att?.text || '').toLowerCase();- return src.includes(parsed.normalized) || alt.includes(parsed.normalized) || text.includes(parsed.normalized)- || (specificTarget && (src.includes(specificTarget) || alt.includes(specificTarget) || text.includes(specificTarget)));+ const candidates = this._extractAttachmentCandidateNames(att);+ return targetNames.some(target => {+ const targetHasExt = /\.[a-z0-9]+$/i.test(target);+ return candidates.some(candidate => {+ if (candidate === target) return true;+ if (!targetHasExt) {+ const candidateBase = candidate.replace(/\.[a-z0-9]+$/i, '');+ if (candidateBase === target) return true;+ }+ return false;+ });+ });});return matchesSpecific;diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.jsindex ea9c52b99..827074b2a 100644--- a/src/firefox/src/agent/agent.js+++ b/src/firefox/src/agent/agent.js@@ -2105,12 +2105,104 @@ export class Agent extends LoopDetector {}_cleanSpecificAttachmentTarget(value) {- const text = String(value || '').trim();- const match = text.match(/(?:(?:an?|the|\d+|one|two|three|four|five|six|seven|eight|nine|ten)\s+)?(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?)\s+(?:of|named|called|with\s+name)\s+(.+)$/i);- if (match) return match[1].trim().toLowerCase();+ let text = String(value || '').trim();+ text = text.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();+ const prefixMatch = text.match(/(?:(?:an?|the|\d+|one|two|three|four|five|six|seven|eight|nine|ten|une?|ein(?:e|en)?|un[ao]?|el|la|le|l'|gli|il)\s+)?(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)\s*(?:of|named|called|with\s+name|de|von|d'|d’|di|con\s+nombre|이름의|名為|名为|名前の|:|:)\s*(.+)$/i)+ || text.match(/^(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)\s*[::]\s*(.+)$/i);+ if (prefixMatch) {+ let target = prefixMatch[1].trim();+ target = target.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();+ if (target) return target.toLowerCase();+ }+ const suffixMatch = text.match(/^(.+?)\s*(?:の|의)?\s*(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)$/i);+ if (suffixMatch) {+ let target = suffixMatch[1].trim();+ target = target.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();+ if (target) return target.toLowerCase();+ }return text.toLowerCase();}+ _targetAttachmentNames(specificTarget) {+ const targets = new Set();+ if (!specificTarget) return [];+ const clean = String(specificTarget).trim().replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim().toLowerCase();+ if (clean) {+ targets.add(clean);+ const segs = clean.split(/[?#]/)[0].split(/[/\\]/).filter(Boolean);+ if (segs.length > 0) {+ targets.add(segs[segs.length - 1]);+ }+ }+ return Array.from(targets);+ }++ _extractAttachmentCandidateNames(att) {+ const rawCandidates = [];+ if (typeof att === 'string') {+ rawCandidates.push(att);+ } else if (att && typeof att === 'object') {+ if (att.name) rawCandidates.push(String(att.name));+ if (att.src) rawCandidates.push(String(att.src));+ if (att.url) rawCandidates.push(String(att.url));+ if (att.alt) rawCandidates.push(String(att.alt));+ if (att.title) rawCandidates.push(String(att.title));+ if (att.ariaLabel) rawCandidates.push(String(att.ariaLabel));+ if (att.text) rawCandidates.push(String(att.text));+ }++ const candidateNames = new Set();++ for (const raw of rawCandidates) {+ if (!raw) continue;+ const trimmed = String(raw).trim();+ if (!trimmed) continue;++ const cleanDirect = trimmed.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim().toLowerCase();+ if (cleanDirect) {+ candidateNames.add(cleanDirect);+ }++ const cleanedPhrase = this._cleanSpecificAttachmentTarget(cleanDirect);+ if (cleanedPhrase) {+ candidateNames.add(cleanedPhrase);+ }++ try {+ const urlObj = new URL(trimmed, 'https://example.invalid');+ const pathname = urlObj.pathname;+ const segments = pathname.split('/').filter(Boolean);+ if (segments.length > 0) {+ let last = segments[segments.length - 1];+ try { last = decodeURIComponent(last); } catch {}+ last = last.trim().toLowerCase();+ if (last) candidateNames.add(last);+ }+ } catch {+ const withoutQuery = trimmed.split(/[?#]/)[0];+ const segments = withoutQuery.split(/[/\\]/).filter(Boolean);+ if (segments.length > 0) {+ const last = segments[segments.length - 1].trim().toLowerCase();+ if (last) candidateNames.add(last);+ }+ }++ const tokens = trimmed.split(/[\s,;:()[\]{}<>"'“”«»`]+/);+ for (let token of tokens) {+ token = token.replace(/^[('"`“«]+|[)'"`”».!?,;:]+$/g, '').trim().toLowerCase();+ if (token) {+ candidateNames.add(token);+ const tokenSegments = token.split(/[?#]/)[0].split(/[/\\]/).filter(Boolean);+ if (tokenSegments.length > 0) {+ candidateNames.add(tokenSegments[tokenSegments.length - 1]);+ }+ }+ }+ }++ return Array.from(candidateNames);+ }+_parseWorkflowAttachmentRequirement(value) {const text = String(value || '').trim().toLowerCase();if (!text) return { isGeneric: true, expectedCount: 1, expectedImageCount: 0, expectedVideoCount: 0, wantsImage: false, wantsVideo: false, wantsGif: false, normalized: '' };@@ -2164,9 +2256,9 @@ export class Agent extends LoopDetector {if (expectedImageCount > 0 && expectedVideoCount > 0) {expectedCount = expectedImageCount + expectedVideoCount;} else if (expectedImageCount > 0) {- expectedCount = expectedImageCount;+ expectedCount = expectedImageCount + (wantsVideo ? 1 : 0);} else if (expectedVideoCount > 0) {- expectedCount = expectedVideoCount;+ expectedCount = expectedVideoCount + (wantsImage ? 1 : 0);} else {const genericNounCountMatch = text.match(new RegExp(`(?:^|[\\s,;and+와과및])(${countPrefix})\\s+(?:attachments?|files?|media|uploads?|pieces?|items?|assets?|enclosures?|documents?|fichiers?|archivos?|dateien?|allegati?|anexos?|вложения?|вложение|첨부(?:파일)?|파일|미디어)`, 'i'))|| text.match(new RegExp(`(?:^|[\\s,;and+와과및])(${countPrefix})\\s*(?:枚|つ|本|张|條|条|个|個|장|개|건|편)`, 'i'))@@ -2179,7 +2271,7 @@ export class Agent extends LoopDetector {}}- if (expectedImageCount === 0 && expectedVideoCount === 0 && wantsImage && wantsVideo && expectedCount < 2) {+ if (wantsImage && wantsVideo && expectedCount < 2) {expectedCount = 2;}@@ -2212,10 +2304,12 @@ export class Agent extends LoopDetector {};let matchingAttachments = rawAttachments;- if (parsed.expectedImageCount > 0 && parsed.expectedVideoCount > 0) {+ if (parsed.wantsImage && parsed.wantsVideo) {+ const minImages = parsed.expectedImageCount > 0 ? parsed.expectedImageCount : 1;+ const minVideos = parsed.expectedVideoCount > 0 ? parsed.expectedVideoCount : 1;const imageCount = rawAttachments.filter(isImageAttachment).length;const videoCount = rawAttachments.filter(isVideoAttachment).length;- if (imageCount < parsed.expectedImageCount || videoCount < parsed.expectedVideoCount) {+ if (imageCount < minImages || videoCount < minVideos) {return false;}} else if (parsed.wantsVideo && !parsed.wantsImage) {@@ -2233,12 +2327,21 @@ export class Agent extends LoopDetector {}const specificTarget = this._cleanSpecificAttachmentTarget(parsed.normalized);+ const targetNames = this._targetAttachmentNames(specificTarget);+const matchesSpecific = matchingAttachments.some(att => {- const src = String(att?.src || att?.url || '').toLowerCase();- const alt = String(att?.alt || att?.name || '').toLowerCase();- const text = String(att?.text || '').toLowerCase();- return src.includes(parsed.normalized) || alt.includes(parsed.normalized) || text.includes(parsed.normalized)- || (specificTarget && (src.includes(specificTarget) || alt.includes(specificTarget) || text.includes(specificTarget)));+ const candidates = this._extractAttachmentCandidateNames(att);+ return targetNames.some(target => {+ const targetHasExt = /\.[a-z0-9]+$/i.test(target);+ return candidates.some(candidate => {+ if (candidate === target) return true;+ if (!targetHasExt) {+ const candidateBase = candidate.replace(/\.[a-z0-9]+$/i, '');+ if (candidateBase === target) return true;+ }+ return false;+ });+ });});return matchesSpecific;diff --git a/test/run.js b/test/run.jsindex 2d41ba7b4..7b20416b9 100644--- a/test/run.js+++ b/test/run.js@@ -87592,6 +87592,89 @@ test('attachment requirement parser recognizes Korean generic media nouns and co}});+test('attachment verification requires each media type in unquantified mixed-media requests', () => {+ for (const AgentClass of [AgentCh, AgentFx]) {+ const agent = new AgentClass({});+ const req = agent._parseWorkflowAttachmentRequirement('image and video');+ assert.equal(req.isGeneric, true, AgentClass.name + ': image and video should be generic');+ assert.equal(req.wantsImage, true, AgentClass.name + ': should want image');+ assert.equal(req.wantsVideo, true, AgentClass.name + ': should want video');+ assert.equal(req.expectedCount, 2, AgentClass.name + ': should expect at least 2 attachments');++ const twoImages = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'image and video' },+ { attachments: [{ type: 'image', src: 'pic1.png' }, { type: 'image', src: 'pic2.png' }] }+ );+ assert.equal(twoImages, false, AgentClass.name + ': two images without video should be rejected for image and video');++ const twoVideos = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'image and video' },+ { attachments: [{ type: 'video', src: 'clip1.mp4' }, { type: 'video', src: 'clip2.mp4' }] }+ );+ assert.equal(twoVideos, false, AgentClass.name + ': two videos without image should be rejected for image and video');++ const imageAndVideo = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'image and video' },+ { attachments: [{ type: 'image', src: 'pic.png' }, { type: 'video', src: 'clip.mp4' }] }+ );+ assert.equal(imageAndVideo, true, AgentClass.name + ': image plus video should satisfy image and video');++ const twoImagesAndVideo = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'image and video' },+ { attachments: [{ type: 'image', src: 'pic1.png' }, { type: 'image', src: 'pic2.png' }, { type: 'video', src: 'clip.mp4' }] }+ );+ assert.equal(twoImagesAndVideo, true, AgentClass.name + ': 2 images plus video should satisfy image and video');+ }+});++test('attachment verification matches specific attachment names without substring collisions', () => {+ for (const AgentClass of [AgentCh, AgentFx]) {+ const agent = new AgentClass({});++ const oldChartUrl = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'chart.png' },+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/old-chart.png', alt: 'old-chart.png' }] }+ );+ assert.equal(oldChartUrl, false, AgentClass.name + ': old-chart.png should not satisfy chart.png');++ const notChartUrl = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'chart.png' },+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/not-chart.png', alt: 'not-chart.png' }] }+ );+ assert.equal(notChartUrl, false, AgentClass.name + ': not-chart.png should not satisfy chart.png');++ const oldChartAlt = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'chart.png' },+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/random-id.jpg', alt: 'Uploaded old-chart.png' }] }+ );+ assert.equal(oldChartAlt, false, AgentClass.name + ': alt with old-chart.png should not satisfy chart.png');++ const exactChartUrl = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'chart.png' },+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/chart.png' }] }+ );+ assert.equal(exactChartUrl, true, AgentClass.name + ': chart.png in URL should satisfy chart.png');++ const exactChartAlt = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'chart.png' },+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/random-id.jpg', alt: 'Uploaded chart.png' }] }+ );+ assert.equal(exactChartAlt, true, AgentClass.name + ': alt with chart.png should satisfy chart.png');++ const cleanedPrefixAccepted = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'an image of chart.png' },+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/chart.png' }] }+ );+ assert.equal(cleanedPrefixAccepted, true, AgentClass.name + ': an image of chart.png should accept chart.png');++ const cleanedPrefixRejected = agent._workflowSocialPublishedAttachmentObserved(+ { value: 'an image of chart.png' },+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/old-chart.png' }] }+ );+ assert.equal(cleanedPrefixRejected, false, AgentClass.name + ': an image of chart.png should reject old-chart.png');+ }+});+test('post body extraction preserves nested quotation marks and does not overwrite full body with prefix', async () => {for (const [index, AgentClass] of [AgentCh, AgentFx].entries()) {const agent = new AgentClass({});