Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(agent): match specific attachment names without substring collisions and require each media type for mixed requests

Emre Sokullu9777a9df861a
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 68e450ffa..e9f0cb24a 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -2391,12 +2391,104 @@ export class Agent extends LoopDetector {
}
_cleanSpecificAttachmentTarget(value) {
- const text = String(value || '').trim();
- const match = text.match(/(?:(?:an?|the|\d+|one|two|three|four|five|six|seven|eight|nine|ten)\s+)?(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?)\s+(?:of|named|called|with\s+name)\s+(.+)$/i);
- if (match) return match[1].trim().toLowerCase();
+ let text = String(value || '').trim();
+ text = text.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();
+ const prefixMatch = text.match(/(?:(?:an?|the|\d+|one|two|three|four|five|six|seven|eight|nine|ten|une?|ein(?:e|en)?|un[ao]?|el|la|le|l'|gli|il)\s+)?(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)\s*(?:of|named|called|with\s+name|de|von|d'|d’|di|con\s+nombre|이름의|名為|名为|名前の|:|:)\s*(.+)$/i)
+ || text.match(/^(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)\s*[::]\s*(.+)$/i);
+ if (prefixMatch) {
+ let target = prefixMatch[1].trim();
+ target = target.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();
+ if (target) return target.toLowerCase();
+ }
+ const suffixMatch = text.match(/^(.+?)\s*(?:の|의)?\s*(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)$/i);
+ if (suffixMatch) {
+ let target = suffixMatch[1].trim();
+ target = target.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();
+ if (target) return target.toLowerCase();
+ }
return text.toLowerCase();
}
+ _targetAttachmentNames(specificTarget) {
+ const targets = new Set();
+ if (!specificTarget) return [];
+ const clean = String(specificTarget).trim().replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim().toLowerCase();
+ if (clean) {
+ targets.add(clean);
+ const segs = clean.split(/[?#]/)[0].split(/[/\\]/).filter(Boolean);
+ if (segs.length > 0) {
+ targets.add(segs[segs.length - 1]);
+ }
+ }
+ return Array.from(targets);
+ }
+
+ _extractAttachmentCandidateNames(att) {
+ const rawCandidates = [];
+ if (typeof att === 'string') {
+ rawCandidates.push(att);
+ } else if (att && typeof att === 'object') {
+ if (att.name) rawCandidates.push(String(att.name));
+ if (att.src) rawCandidates.push(String(att.src));
+ if (att.url) rawCandidates.push(String(att.url));
+ if (att.alt) rawCandidates.push(String(att.alt));
+ if (att.title) rawCandidates.push(String(att.title));
+ if (att.ariaLabel) rawCandidates.push(String(att.ariaLabel));
+ if (att.text) rawCandidates.push(String(att.text));
+ }
+
+ const candidateNames = new Set();
+
+ for (const raw of rawCandidates) {
+ if (!raw) continue;
+ const trimmed = String(raw).trim();
+ if (!trimmed) continue;
+
+ const cleanDirect = trimmed.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim().toLowerCase();
+ if (cleanDirect) {
+ candidateNames.add(cleanDirect);
+ }
+
+ const cleanedPhrase = this._cleanSpecificAttachmentTarget(cleanDirect);
+ if (cleanedPhrase) {
+ candidateNames.add(cleanedPhrase);
+ }
+
+ try {
+ const urlObj = new URL(trimmed, 'https://example.invalid');
+ const pathname = urlObj.pathname;
+ const segments = pathname.split('/').filter(Boolean);
+ if (segments.length > 0) {
+ let last = segments[segments.length - 1];
+ try { last = decodeURIComponent(last); } catch {}
+ last = last.trim().toLowerCase();
+ if (last) candidateNames.add(last);
+ }
+ } catch {
+ const withoutQuery = trimmed.split(/[?#]/)[0];
+ const segments = withoutQuery.split(/[/\\]/).filter(Boolean);
+ if (segments.length > 0) {
+ const last = segments[segments.length - 1].trim().toLowerCase();
+ if (last) candidateNames.add(last);
+ }
+ }
+
+ const tokens = trimmed.split(/[\s,;:()[\]{}<>"'“”«»`]+/);
+ for (let token of tokens) {
+ token = token.replace(/^[('"`“«]+|[)'"`”».!?,;:]+$/g, '').trim().toLowerCase();
+ if (token) {
+ candidateNames.add(token);
+ const tokenSegments = token.split(/[?#]/)[0].split(/[/\\]/).filter(Boolean);
+ if (tokenSegments.length > 0) {
+ candidateNames.add(tokenSegments[tokenSegments.length - 1]);
+ }
+ }
+ }
+ }
+
+ return Array.from(candidateNames);
+ }
+
_parseWorkflowAttachmentRequirement(value) {
const text = String(value || '').trim().toLowerCase();
if (!text) return { isGeneric: true, expectedCount: 1, expectedImageCount: 0, expectedVideoCount: 0, wantsImage: false, wantsVideo: false, wantsGif: false, normalized: '' };
@@ -2450,9 +2542,9 @@ export class Agent extends LoopDetector {
if (expectedImageCount > 0 && expectedVideoCount > 0) {
expectedCount = expectedImageCount + expectedVideoCount;
} else if (expectedImageCount > 0) {
- expectedCount = expectedImageCount;
+ expectedCount = expectedImageCount + (wantsVideo ? 1 : 0);
} else if (expectedVideoCount > 0) {
- expectedCount = expectedVideoCount;
+ expectedCount = expectedVideoCount + (wantsImage ? 1 : 0);
} else {
const genericNounCountMatch = text.match(new RegExp(`(?:^|[\\s,;and+와과및])(${countPrefix})\\s+(?:attachments?|files?|media|uploads?|pieces?|items?|assets?|enclosures?|documents?|fichiers?|archivos?|dateien?|allegati?|anexos?|вложения?|вложение|첨부(?:파일)?|파일|미디어)`, 'i'))
|| text.match(new RegExp(`(?:^|[\\s,;and+와과및])(${countPrefix})\\s*(?:枚|つ|本|张|條|条|个|個|장|개|건|편)`, 'i'))
@@ -2465,7 +2557,7 @@ export class Agent extends LoopDetector {
}
}
- if (expectedImageCount === 0 && expectedVideoCount === 0 && wantsImage && wantsVideo && expectedCount < 2) {
+ if (wantsImage && wantsVideo && expectedCount < 2) {
expectedCount = 2;
}
@@ -2498,10 +2590,12 @@ export class Agent extends LoopDetector {
};
let matchingAttachments = rawAttachments;
- if (parsed.expectedImageCount > 0 && parsed.expectedVideoCount > 0) {
+ if (parsed.wantsImage && parsed.wantsVideo) {
+ const minImages = parsed.expectedImageCount > 0 ? parsed.expectedImageCount : 1;
+ const minVideos = parsed.expectedVideoCount > 0 ? parsed.expectedVideoCount : 1;
const imageCount = rawAttachments.filter(isImageAttachment).length;
const videoCount = rawAttachments.filter(isVideoAttachment).length;
- if (imageCount < parsed.expectedImageCount || videoCount < parsed.expectedVideoCount) {
+ if (imageCount < minImages || videoCount < minVideos) {
return false;
}
} else if (parsed.wantsVideo && !parsed.wantsImage) {
@@ -2519,12 +2613,21 @@ export class Agent extends LoopDetector {
}
const specificTarget = this._cleanSpecificAttachmentTarget(parsed.normalized);
+ const targetNames = this._targetAttachmentNames(specificTarget);
+
const matchesSpecific = matchingAttachments.some(att => {
- const src = String(att?.src || att?.url || '').toLowerCase();
- const alt = String(att?.alt || att?.name || '').toLowerCase();
- const text = String(att?.text || '').toLowerCase();
- return src.includes(parsed.normalized) || alt.includes(parsed.normalized) || text.includes(parsed.normalized)
- || (specificTarget && (src.includes(specificTarget) || alt.includes(specificTarget) || text.includes(specificTarget)));
+ const candidates = this._extractAttachmentCandidateNames(att);
+ return targetNames.some(target => {
+ const targetHasExt = /\.[a-z0-9]+$/i.test(target);
+ return candidates.some(candidate => {
+ if (candidate === target) return true;
+ if (!targetHasExt) {
+ const candidateBase = candidate.replace(/\.[a-z0-9]+$/i, '');
+ if (candidateBase === target) return true;
+ }
+ return false;
+ });
+ });
});
return matchesSpecific;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index ea9c52b99..827074b2a 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -2105,12 +2105,104 @@ export class Agent extends LoopDetector {
}
_cleanSpecificAttachmentTarget(value) {
- const text = String(value || '').trim();
- const match = text.match(/(?:(?:an?|the|\d+|one|two|three|four|five|six|seven|eight|nine|ten)\s+)?(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?)\s+(?:of|named|called|with\s+name)\s+(.+)$/i);
- if (match) return match[1].trim().toLowerCase();
+ let text = String(value || '').trim();
+ text = text.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();
+ const prefixMatch = text.match(/(?:(?:an?|the|\d+|one|two|three|four|five|six|seven|eight|nine|ten|une?|ein(?:e|en)?|un[ao]?|el|la|le|l'|gli|il)\s+)?(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)\s*(?:of|named|called|with\s+name|de|von|d'|d’|di|con\s+nombre|이름의|名為|名为|名前の|:|:)\s*(.+)$/i)
+ || text.match(/^(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)\s*[::]\s*(.+)$/i);
+ if (prefixMatch) {
+ let target = prefixMatch[1].trim();
+ target = target.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();
+ if (target) return target.toLowerCase();
+ }
+ const suffixMatch = text.match(/^(.+?)\s*(?:の|의)?\s*(?:images?|photos?|pictures?|videos?|gifs?|animated\s+gifs?|attachments?|files?|bilder?|fotos?|vidéos?|archivos?|dateien?|allegat[oi]?|anexos?|вложения?|사진|이미지|동영상|画像|写真|视频|影片)$/i);
+ if (suffixMatch) {
+ let target = suffixMatch[1].trim();
+ target = target.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim();
+ if (target) return target.toLowerCase();
+ }
return text.toLowerCase();
}
+ _targetAttachmentNames(specificTarget) {
+ const targets = new Set();
+ if (!specificTarget) return [];
+ const clean = String(specificTarget).trim().replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim().toLowerCase();
+ if (clean) {
+ targets.add(clean);
+ const segs = clean.split(/[?#]/)[0].split(/[/\\]/).filter(Boolean);
+ if (segs.length > 0) {
+ targets.add(segs[segs.length - 1]);
+ }
+ }
+ return Array.from(targets);
+ }
+
+ _extractAttachmentCandidateNames(att) {
+ const rawCandidates = [];
+ if (typeof att === 'string') {
+ rawCandidates.push(att);
+ } else if (att && typeof att === 'object') {
+ if (att.name) rawCandidates.push(String(att.name));
+ if (att.src) rawCandidates.push(String(att.src));
+ if (att.url) rawCandidates.push(String(att.url));
+ if (att.alt) rawCandidates.push(String(att.alt));
+ if (att.title) rawCandidates.push(String(att.title));
+ if (att.ariaLabel) rawCandidates.push(String(att.ariaLabel));
+ if (att.text) rawCandidates.push(String(att.text));
+ }
+
+ const candidateNames = new Set();
+
+ for (const raw of rawCandidates) {
+ if (!raw) continue;
+ const trimmed = String(raw).trim();
+ if (!trimmed) continue;
+
+ const cleanDirect = trimmed.replace(/^['"“”«»`]+|['"“”«»`]+$/g, '').trim().toLowerCase();
+ if (cleanDirect) {
+ candidateNames.add(cleanDirect);
+ }
+
+ const cleanedPhrase = this._cleanSpecificAttachmentTarget(cleanDirect);
+ if (cleanedPhrase) {
+ candidateNames.add(cleanedPhrase);
+ }
+
+ try {
+ const urlObj = new URL(trimmed, 'https://example.invalid');
+ const pathname = urlObj.pathname;
+ const segments = pathname.split('/').filter(Boolean);
+ if (segments.length > 0) {
+ let last = segments[segments.length - 1];
+ try { last = decodeURIComponent(last); } catch {}
+ last = last.trim().toLowerCase();
+ if (last) candidateNames.add(last);
+ }
+ } catch {
+ const withoutQuery = trimmed.split(/[?#]/)[0];
+ const segments = withoutQuery.split(/[/\\]/).filter(Boolean);
+ if (segments.length > 0) {
+ const last = segments[segments.length - 1].trim().toLowerCase();
+ if (last) candidateNames.add(last);
+ }
+ }
+
+ const tokens = trimmed.split(/[\s,;:()[\]{}<>"'“”«»`]+/);
+ for (let token of tokens) {
+ token = token.replace(/^[('"`“«]+|[)'"`”».!?,;:]+$/g, '').trim().toLowerCase();
+ if (token) {
+ candidateNames.add(token);
+ const tokenSegments = token.split(/[?#]/)[0].split(/[/\\]/).filter(Boolean);
+ if (tokenSegments.length > 0) {
+ candidateNames.add(tokenSegments[tokenSegments.length - 1]);
+ }
+ }
+ }
+ }
+
+ return Array.from(candidateNames);
+ }
+
_parseWorkflowAttachmentRequirement(value) {
const text = String(value || '').trim().toLowerCase();
if (!text) return { isGeneric: true, expectedCount: 1, expectedImageCount: 0, expectedVideoCount: 0, wantsImage: false, wantsVideo: false, wantsGif: false, normalized: '' };
@@ -2164,9 +2256,9 @@ export class Agent extends LoopDetector {
if (expectedImageCount > 0 && expectedVideoCount > 0) {
expectedCount = expectedImageCount + expectedVideoCount;
} else if (expectedImageCount > 0) {
- expectedCount = expectedImageCount;
+ expectedCount = expectedImageCount + (wantsVideo ? 1 : 0);
} else if (expectedVideoCount > 0) {
- expectedCount = expectedVideoCount;
+ expectedCount = expectedVideoCount + (wantsImage ? 1 : 0);
} else {
const genericNounCountMatch = text.match(new RegExp(`(?:^|[\\s,;and+와과및])(${countPrefix})\\s+(?:attachments?|files?|media|uploads?|pieces?|items?|assets?|enclosures?|documents?|fichiers?|archivos?|dateien?|allegati?|anexos?|вложения?|вложение|첨부(?:파일)?|파일|미디어)`, 'i'))
|| text.match(new RegExp(`(?:^|[\\s,;and+와과및])(${countPrefix})\\s*(?:枚|つ|本|张|條|条|个|個|장|개|건|편)`, 'i'))
@@ -2179,7 +2271,7 @@ export class Agent extends LoopDetector {
}
}
- if (expectedImageCount === 0 && expectedVideoCount === 0 && wantsImage && wantsVideo && expectedCount < 2) {
+ if (wantsImage && wantsVideo && expectedCount < 2) {
expectedCount = 2;
}
@@ -2212,10 +2304,12 @@ export class Agent extends LoopDetector {
};
let matchingAttachments = rawAttachments;
- if (parsed.expectedImageCount > 0 && parsed.expectedVideoCount > 0) {
+ if (parsed.wantsImage && parsed.wantsVideo) {
+ const minImages = parsed.expectedImageCount > 0 ? parsed.expectedImageCount : 1;
+ const minVideos = parsed.expectedVideoCount > 0 ? parsed.expectedVideoCount : 1;
const imageCount = rawAttachments.filter(isImageAttachment).length;
const videoCount = rawAttachments.filter(isVideoAttachment).length;
- if (imageCount < parsed.expectedImageCount || videoCount < parsed.expectedVideoCount) {
+ if (imageCount < minImages || videoCount < minVideos) {
return false;
}
} else if (parsed.wantsVideo && !parsed.wantsImage) {
@@ -2233,12 +2327,21 @@ export class Agent extends LoopDetector {
}
const specificTarget = this._cleanSpecificAttachmentTarget(parsed.normalized);
+ const targetNames = this._targetAttachmentNames(specificTarget);
+
const matchesSpecific = matchingAttachments.some(att => {
- const src = String(att?.src || att?.url || '').toLowerCase();
- const alt = String(att?.alt || att?.name || '').toLowerCase();
- const text = String(att?.text || '').toLowerCase();
- return src.includes(parsed.normalized) || alt.includes(parsed.normalized) || text.includes(parsed.normalized)
- || (specificTarget && (src.includes(specificTarget) || alt.includes(specificTarget) || text.includes(specificTarget)));
+ const candidates = this._extractAttachmentCandidateNames(att);
+ return targetNames.some(target => {
+ const targetHasExt = /\.[a-z0-9]+$/i.test(target);
+ return candidates.some(candidate => {
+ if (candidate === target) return true;
+ if (!targetHasExt) {
+ const candidateBase = candidate.replace(/\.[a-z0-9]+$/i, '');
+ if (candidateBase === target) return true;
+ }
+ return false;
+ });
+ });
});
return matchesSpecific;
diff --git a/test/run.js b/test/run.js
index 2d41ba7b4..7b20416b9 100644
--- a/test/run.js
+++ b/test/run.js
@@ -87592,6 +87592,89 @@ test('attachment requirement parser recognizes Korean generic media nouns and co
}
});
+test('attachment verification requires each media type in unquantified mixed-media requests', () => {
+ for (const AgentClass of [AgentCh, AgentFx]) {
+ const agent = new AgentClass({});
+ const req = agent._parseWorkflowAttachmentRequirement('image and video');
+ assert.equal(req.isGeneric, true, AgentClass.name + ': image and video should be generic');
+ assert.equal(req.wantsImage, true, AgentClass.name + ': should want image');
+ assert.equal(req.wantsVideo, true, AgentClass.name + ': should want video');
+ assert.equal(req.expectedCount, 2, AgentClass.name + ': should expect at least 2 attachments');
+
+ const twoImages = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'image and video' },
+ { attachments: [{ type: 'image', src: 'pic1.png' }, { type: 'image', src: 'pic2.png' }] }
+ );
+ assert.equal(twoImages, false, AgentClass.name + ': two images without video should be rejected for image and video');
+
+ const twoVideos = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'image and video' },
+ { attachments: [{ type: 'video', src: 'clip1.mp4' }, { type: 'video', src: 'clip2.mp4' }] }
+ );
+ assert.equal(twoVideos, false, AgentClass.name + ': two videos without image should be rejected for image and video');
+
+ const imageAndVideo = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'image and video' },
+ { attachments: [{ type: 'image', src: 'pic.png' }, { type: 'video', src: 'clip.mp4' }] }
+ );
+ assert.equal(imageAndVideo, true, AgentClass.name + ': image plus video should satisfy image and video');
+
+ const twoImagesAndVideo = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'image and video' },
+ { attachments: [{ type: 'image', src: 'pic1.png' }, { type: 'image', src: 'pic2.png' }, { type: 'video', src: 'clip.mp4' }] }
+ );
+ assert.equal(twoImagesAndVideo, true, AgentClass.name + ': 2 images plus video should satisfy image and video');
+ }
+});
+
+test('attachment verification matches specific attachment names without substring collisions', () => {
+ for (const AgentClass of [AgentCh, AgentFx]) {
+ const agent = new AgentClass({});
+
+ const oldChartUrl = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'chart.png' },
+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/old-chart.png', alt: 'old-chart.png' }] }
+ );
+ assert.equal(oldChartUrl, false, AgentClass.name + ': old-chart.png should not satisfy chart.png');
+
+ const notChartUrl = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'chart.png' },
+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/not-chart.png', alt: 'not-chart.png' }] }
+ );
+ assert.equal(notChartUrl, false, AgentClass.name + ': not-chart.png should not satisfy chart.png');
+
+ const oldChartAlt = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'chart.png' },
+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/random-id.jpg', alt: 'Uploaded old-chart.png' }] }
+ );
+ assert.equal(oldChartAlt, false, AgentClass.name + ': alt with old-chart.png should not satisfy chart.png');
+
+ const exactChartUrl = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'chart.png' },
+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/chart.png' }] }
+ );
+ assert.equal(exactChartUrl, true, AgentClass.name + ': chart.png in URL should satisfy chart.png');
+
+ const exactChartAlt = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'chart.png' },
+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/random-id.jpg', alt: 'Uploaded chart.png' }] }
+ );
+ assert.equal(exactChartAlt, true, AgentClass.name + ': alt with chart.png should satisfy chart.png');
+
+ const cleanedPrefixAccepted = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'an image of chart.png' },
+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/chart.png' }] }
+ );
+ assert.equal(cleanedPrefixAccepted, true, AgentClass.name + ': an image of chart.png should accept chart.png');
+
+ const cleanedPrefixRejected = agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'an image of chart.png' },
+ { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/old-chart.png' }] }
+ );
+ assert.equal(cleanedPrefixRejected, false, AgentClass.name + ': an image of chart.png should reject old-chart.png');
+ }
+});
+
test('post body extraction preserves nested quotation marks and does not overwrite full body with prefix', async () => {
for (const [index, AgentClass] of [AgentCh, AgentFx].entries()) {
const agent = new AgentClass({});