Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(agent): verify GIF subtype and read singular articles as exact counts

Emre Sokullu8197fe967539
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 17e750605..16c63511c 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -296,7 +296,7 @@ const GENERIC_ATTACHMENT_WORDS = new Set([
'o', 'os', 'as', 'do', 'da', 'dos', 'das', 'un', 'une', 'deux', 'trois', 'quatre', 'cinq',
'uno', 'una', 'unos', 'unas', 'dos', 'tres', 'cuatro', 'cinco', 'due', 'tre', 'quattro', 'cinque',
'ein', 'eine', 'einen', 'einer', 'zwei', 'drei', 'vier', 'fünf', 'um', 'uma', 'dois', 'duas', 'três',
- 'imagen', 'imagenes', 'imágenes', 'foto', 'fotos', 'vidéo', 'vidéos', 'bild', 'bilder',
+ 'imagen', 'imagenes', 'imágenes', 'imagem', 'imagens', 'foto', 'fotos', 'vidéo', 'vidéos', 'bild', 'bilder',
'fichier', 'fichiers', 'archivo', 'archivos', 'datei', 'dateien', 'allegato', 'allegati', 'anexo', 'anexos',
'et', 'ou', 'y', 'e', 'und', 'oder', 'ed', 've', 'veya', 'ile',
'один', 'одна', 'одно', 'два', 'две', 'три', 'четыре', 'пять',
@@ -403,8 +403,8 @@ const TOPIC_NOUN_AFTER_PLATFORM = new RegExp(
);
const IMAGE_NEGATION_REGEX = new RegExp(
- `(?<![${SOCIAL_WORD_EDGE}])(?:no|not|without|without\\s+any|0|zero|none|sin|sans|sem|senza|ohne|kein|keine|keinen|aucun|aucune|ningun|ningún|ninguna|nenhum|nenhuma|nessun|nessuno|nessuna|nie|без|нет)\\s+(?:any\\s+)?(?:images?|photos?|pictures?|pics?|fotos?|bilder?|imagen(?:es)?|imágenes|pièces?\\s+jointes?|изображени[яй]|фото(?:графий)?|resim|fotoğraf)(?![${SOCIAL_WORD_EDGE}])`
- + `|(?:images?|photos?|pictures?|pics?|fotos?|bilder?|imagen(?:es)?|imágenes)\\s*:\\s*(?:none|no|0|zero|false)`
+ `(?<![${SOCIAL_WORD_EDGE}])(?:no|not|without|without\\s+any|0|zero|none|sin|sans|sem|senza|ohne|kein|keine|keinen|aucun|aucune|ningun|ningún|ninguna|nenhum|nenhuma|nessun|nessuno|nessuna|nie|без|нет)\\s+(?:any\\s+)?(?:images?|photos?|pictures?|pics?|fotos?|bilder?|imagen(?:es)?|imágenes|imagem|imagens|pièces?\\s+jointes?|изображени[яй]|фото(?:графий)?|resim|fotoğraf)(?![${SOCIAL_WORD_EDGE}])`
+ + `|(?:images?|photos?|pictures?|pics?|fotos?|bilder?|imagen(?:es)?|imágenes|imagem|imagens)\\s*:\\s*(?:none|no|0|zero|false)`
+ `|(?:无|没有|不带|零个|0个|0)\\s*(?:图片|照片|圖片)`
+ `|(?:图片|照片|圖片)\\s*(?:无|没有|为0|为零|0个|零个|0)`
+ `|(?:なし|無し|ゼロ|0)\\s*(?:画像|写真)`
@@ -2782,7 +2782,7 @@ export class Agent extends LoopDetector {
const hasRawGif = /\b(?:gif|gifs)\b|\.gif(?:[?#]|$)|(?:animated[-_ ]gif|動圖|动图|움짤)/i.test(text);
const hasRawVideo = hasRawGif || /\b(?:video|videos|mp4|mov|webm|mkv|clip|clips|recording|recordings|vidéo|vidéos)\b|(?:動画|视频|影片|видео|동영상|비디오|영상)/i.test(text);
- const hasRawImage = /\b(?:image|images|photo|photos|picture|pictures|pic|pics|png|jpg|jpeg|webp|foto|fotos|bild|bilder|imagen(?:es)?|imágenes)\b|(?:画像|写真|图片|照片|圖片|изображение|фото|사진|이미지|포토)/i.test(text);
+ const hasRawImage = /\b(?:image|images|photo|photos|picture|pictures|pic|pics|png|jpg|jpeg|webp|foto|fotos|bild|bilder|imagen(?:es)?|imágenes|imagem|imagens)\b|(?:画像|写真|图片|照片|圖片|изображение|фото|사진|이미지|포토)/i.test(text);
const wantsGif = hasRawGif && !isGifNegated;
const wantsVideo = (hasRawVideo && !isVideoNegated) || wantsGif;
@@ -2819,6 +2819,9 @@ export class Agent extends LoopDetector {
'zero', 'cero', 'zéro', 'null', 'ноль', 'нуль',
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',
'single', 'both',
+ // "a video" asks for one video, the same as "one video" does.
+ 'a', 'an', 'un', 'une', 'uno', 'una', 'ein', 'eine', 'einen', 'einer',
+ 'um', 'uma', 'один', 'одна', 'одно',
'deux', 'trois', 'quatre', 'cinq',
'dos', 'tres', 'cuatro', 'cinco',
'due', 'tre', 'quattro', 'cinque',
@@ -2854,7 +2857,7 @@ export class Agent extends LoopDetector {
const countSeparator = '(?:^|[\\s,;+&/|、,。;:]|(?:\\b(?:and|und|et|e|y)\\b\\s*)|[와과및])';
const imageCountMatch = isGeneric
- ? (countText.match(new RegExp(`${countSeparator}(${countPrefix})(?:\\s*(?:枚|つ|本|张|條|条|个|個|장|개|건|편))?\\s*(?:images?|photos?|pictures?|pics?|foto|fotos|bild|bilder|imagen(?:es)?|imágenes|画像|写真|图片|照片|圖片|изображение|фото|사진|이미지|포토)(?:\\s*(?:장|개|건))?`, 'i'))
+ ? (countText.match(new RegExp(`${countSeparator}(${countPrefix})(?:\\s*(?:枚|つ|本|张|條|条|个|個|장|개|건|편))?\\s*(?:images?|photos?|pictures?|pics?|foto|fotos|bild|bilder|imagen(?:es)?|imágenes|imagem|imagens|画像|写真|图片|照片|圖片|изображение|фото|사진|이미지|포토)(?:\\s*(?:장|개|건))?`, 'i'))
|| countText.match(/(?:사진|이미지|포토)\s*([0-9하나둘셋넷다섯]+|[한두세네일이삼사오](?=\\s*(?:장|개|건|편|개의|장의)))\s*(?:장|개|건)?/i)
|| countText.match(/(?:画像|写真|图片|照片|圖片)\s*([0-9一二两三四五六七八九十]+)\s*(?:枚|つ|张|條|条|个|個)?/i))
: null;
@@ -2951,6 +2954,7 @@ export class Agent extends LoopDetector {
hasExplicitCardinality,
isImageNegated,
isVideoNegated,
+ isGifNegated,
isAlternative,
isMinimumCount,
wantsImage,
@@ -2989,8 +2993,25 @@ export class Agent extends LoopDetector {
return !isVideoAttachment(att);
};
+ // A GIF is a video, but not every video is a GIF: an mp4 must not satisfy
+ // "a GIF", and a GIF must not slip past "one video and no GIFs". X serves
+ // animated GIFs as mp4 from its tweet_video path, so the subtype is read
+ // from the type, the path, and the name rather than the container.
+ const isGifAttachment = (att) => {
+ const type = attachmentType(att);
+ if (type === 'animated_gif' || type === 'gif') return true;
+ const src = String(att?.src || att?.url || '');
+ const alt = String(att?.alt || att?.description || '');
+ return /\.gif(?:\.mp4)?(?:[?#]|$)/i.test(src)
+ || /\/tweet_video(?:_thumb)?\//i.test(src)
+ || /\.gif(?:[?#\s]|$)/i.test(alt);
+ };
+
const imageCount = rawAttachments.filter(isImageAttachment).length;
const videoCount = rawAttachments.filter(isVideoAttachment).length;
+ const gifCount = rawAttachments.filter(isGifAttachment).length;
+ if (parsed.wantsGif && gifCount < 1) return false;
+ if (parsed.isGifNegated && gifCount > 0) return false;
let matchingAttachments = rawAttachments;
if (parsed.wantsImage && parsed.wantsVideo) {
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 2824ed2db..01184a144 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -229,7 +229,7 @@ const GENERIC_ATTACHMENT_WORDS = new Set([
'o', 'os', 'as', 'do', 'da', 'dos', 'das', 'un', 'une', 'deux', 'trois', 'quatre', 'cinq',
'uno', 'una', 'unos', 'unas', 'dos', 'tres', 'cuatro', 'cinco', 'due', 'tre', 'quattro', 'cinque',
'ein', 'eine', 'einen', 'einer', 'zwei', 'drei', 'vier', 'fünf', 'um', 'uma', 'dois', 'duas', 'três',
- 'imagen', 'imagenes', 'imágenes', 'foto', 'fotos', 'vidéo', 'vidéos', 'bild', 'bilder',
+ 'imagen', 'imagenes', 'imágenes', 'imagem', 'imagens', 'foto', 'fotos', 'vidéo', 'vidéos', 'bild', 'bilder',
'fichier', 'fichiers', 'archivo', 'archivos', 'datei', 'dateien', 'allegato', 'allegati', 'anexo', 'anexos',
'et', 'ou', 'y', 'e', 'und', 'oder', 'ed', 've', 'veya', 'ile',
'один', 'одна', 'одно', 'два', 'две', 'три', 'четыре', 'пять',
@@ -336,8 +336,8 @@ const TOPIC_NOUN_AFTER_PLATFORM = new RegExp(
);
const IMAGE_NEGATION_REGEX = new RegExp(
- `(?<![${SOCIAL_WORD_EDGE}])(?:no|not|without|without\\s+any|0|zero|none|sin|sans|sem|senza|ohne|kein|keine|keinen|aucun|aucune|ningun|ningún|ninguna|nenhum|nenhuma|nessun|nessuno|nessuna|nie|без|нет)\\s+(?:any\\s+)?(?:images?|photos?|pictures?|pics?|fotos?|bilder?|imagen(?:es)?|imágenes|pièces?\\s+jointes?|изображени[яй]|фото(?:графий)?|resim|fotoğraf)(?![${SOCIAL_WORD_EDGE}])`
- + `|(?:images?|photos?|pictures?|pics?|fotos?|bilder?|imagen(?:es)?|imágenes)\\s*:\\s*(?:none|no|0|zero|false)`
+ `(?<![${SOCIAL_WORD_EDGE}])(?:no|not|without|without\\s+any|0|zero|none|sin|sans|sem|senza|ohne|kein|keine|keinen|aucun|aucune|ningun|ningún|ninguna|nenhum|nenhuma|nessun|nessuno|nessuna|nie|без|нет)\\s+(?:any\\s+)?(?:images?|photos?|pictures?|pics?|fotos?|bilder?|imagen(?:es)?|imágenes|imagem|imagens|pièces?\\s+jointes?|изображени[яй]|фото(?:графий)?|resim|fotoğraf)(?![${SOCIAL_WORD_EDGE}])`
+ + `|(?:images?|photos?|pictures?|pics?|fotos?|bilder?|imagen(?:es)?|imágenes|imagem|imagens)\\s*:\\s*(?:none|no|0|zero|false)`
+ `|(?:无|没有|不带|零个|0个|0)\\s*(?:图片|照片|圖片)`
+ `|(?:图片|照片|圖片)\\s*(?:无|没有|为0|为零|0个|零个|0)`
+ `|(?:なし|無し|ゼロ|0)\\s*(?:画像|写真)`
@@ -2496,7 +2496,7 @@ export class Agent extends LoopDetector {
const hasRawGif = /\b(?:gif|gifs)\b|\.gif(?:[?#]|$)|(?:animated[-_ ]gif|動圖|动图|움짤)/i.test(text);
const hasRawVideo = hasRawGif || /\b(?:video|videos|mp4|mov|webm|mkv|clip|clips|recording|recordings|vidéo|vidéos)\b|(?:動画|视频|影片|видео|동영상|비디오|영상)/i.test(text);
- const hasRawImage = /\b(?:image|images|photo|photos|picture|pictures|pic|pics|png|jpg|jpeg|webp|foto|fotos|bild|bilder|imagen(?:es)?|imágenes)\b|(?:画像|写真|图片|照片|圖片|изображение|фото|사진|이미지|포토)/i.test(text);
+ const hasRawImage = /\b(?:image|images|photo|photos|picture|pictures|pic|pics|png|jpg|jpeg|webp|foto|fotos|bild|bilder|imagen(?:es)?|imágenes|imagem|imagens)\b|(?:画像|写真|图片|照片|圖片|изображение|фото|사진|이미지|포토)/i.test(text);
const wantsGif = hasRawGif && !isGifNegated;
const wantsVideo = (hasRawVideo && !isVideoNegated) || wantsGif;
@@ -2533,6 +2533,9 @@ export class Agent extends LoopDetector {
'zero', 'cero', 'zéro', 'null', 'ноль', 'нуль',
'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten',
'single', 'both',
+ // "a video" asks for one video, the same as "one video" does.
+ 'a', 'an', 'un', 'une', 'uno', 'una', 'ein', 'eine', 'einen', 'einer',
+ 'um', 'uma', 'один', 'одна', 'одно',
'deux', 'trois', 'quatre', 'cinq',
'dos', 'tres', 'cuatro', 'cinco',
'due', 'tre', 'quattro', 'cinque',
@@ -2568,7 +2571,7 @@ export class Agent extends LoopDetector {
const countSeparator = '(?:^|[\\s,;+&/|、,。;:]|(?:\\b(?:and|und|et|e|y)\\b\\s*)|[와과및])';
const imageCountMatch = isGeneric
- ? (countText.match(new RegExp(`${countSeparator}(${countPrefix})(?:\\s*(?:枚|つ|本|张|條|条|个|個|장|개|건|편))?\\s*(?:images?|photos?|pictures?|pics?|foto|fotos|bild|bilder|imagen(?:es)?|imágenes|画像|写真|图片|照片|圖片|изображение|фото|사진|이미지|포토)(?:\\s*(?:장|개|건))?`, 'i'))
+ ? (countText.match(new RegExp(`${countSeparator}(${countPrefix})(?:\\s*(?:枚|つ|本|张|條|条|个|個|장|개|건|편))?\\s*(?:images?|photos?|pictures?|pics?|foto|fotos|bild|bilder|imagen(?:es)?|imágenes|imagem|imagens|画像|写真|图片|照片|圖片|изображение|фото|사진|이미지|포토)(?:\\s*(?:장|개|건))?`, 'i'))
|| countText.match(/(?:사진|이미지|포토)\s*([0-9하나둘셋넷다섯]+|[한두세네일이삼사오](?=\\s*(?:장|개|건|편|개의|장의)))\s*(?:장|개|건)?/i)
|| countText.match(/(?:画像|写真|图片|照片|圖片)\s*([0-9一二两三四五六七八九十]+)\s*(?:枚|つ|张|條|条|个|個)?/i))
: null;
@@ -2665,6 +2668,7 @@ export class Agent extends LoopDetector {
hasExplicitCardinality,
isImageNegated,
isVideoNegated,
+ isGifNegated,
isAlternative,
isMinimumCount,
wantsImage,
@@ -2703,8 +2707,25 @@ export class Agent extends LoopDetector {
return !isVideoAttachment(att);
};
+ // A GIF is a video, but not every video is a GIF: an mp4 must not satisfy
+ // "a GIF", and a GIF must not slip past "one video and no GIFs". X serves
+ // animated GIFs as mp4 from its tweet_video path, so the subtype is read
+ // from the type, the path, and the name rather than the container.
+ const isGifAttachment = (att) => {
+ const type = attachmentType(att);
+ if (type === 'animated_gif' || type === 'gif') return true;
+ const src = String(att?.src || att?.url || '');
+ const alt = String(att?.alt || att?.description || '');
+ return /\.gif(?:\.mp4)?(?:[?#]|$)/i.test(src)
+ || /\/tweet_video(?:_thumb)?\//i.test(src)
+ || /\.gif(?:[?#\s]|$)/i.test(alt);
+ };
+
const imageCount = rawAttachments.filter(isImageAttachment).length;
const videoCount = rawAttachments.filter(isVideoAttachment).length;
+ const gifCount = rawAttachments.filter(isGifAttachment).length;
+ if (parsed.wantsGif && gifCount < 1) return false;
+ if (parsed.isGifNegated && gifCount > 0) return false;
let matchingAttachments = rawAttachments;
if (parsed.wantsImage && parsed.wantsVideo) {
diff --git a/test/run.js b/test/run.js
index bfb3db9e3..96333ef8a 100644
--- a/test/run.js
+++ b/test/run.js
@@ -88110,7 +88110,7 @@ test('X and Bluesky same-route publication accepts only one new permalink with t
pageStateWithVideo,
fixture.feedUrl,
submit,
- ), true, AgentClass.name + ': video attachment satisfying GIF requirement was rejected');
+ ), false, AgentClass.name + ': an ordinary mp4 satisfied a requirement for a GIF');
const pageStateWithSpecificGif = {
...exactPageState,
@@ -89181,6 +89181,77 @@ test('attachment verification matches specific attachment names without substrin
assert.deepEqual(mixedListReq.specificTargets, ['research and development.png', 'cover.jpg'],
AgentClass.name + ': a trailing conjunction between two filenames still splits');
+ // A GIF is a video, but an ordinary video is not a GIF.
+ const gifMedia = {
+ mp4: { type: 'video', src: 'https://video.twimg.com/media/clip.mp4' },
+ typed: { type: 'animated_gif', src: 'https://video.twimg.com/tweet_video/loop.mp4' },
+ named: { type: 'video', src: 'https://video.twimg.com/tweet_video/animation.gif.mp4', alt: 'animation.gif' },
+ plain: { type: 'video', src: 'https://cdn.example.com/loop.gif' },
+ };
+ for (const [gifKey, gifLabel] of [['typed', 'a typed animated GIF'], ['named', 'an mp4-served GIF'], ['plain', 'a plain .gif']]) {
+ assert.equal(
+ agent._workflowSocialPublishedAttachmentObserved({ value: 'a GIF' }, { attachments: [gifMedia[gifKey]] }),
+ true,
+ AgentClass.name + `: ${gifLabel} should satisfy a GIF requirement`,
+ );
+ assert.equal(
+ agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one video and no GIFs' },
+ { attachments: [gifMedia[gifKey]] },
+ ),
+ false,
+ AgentClass.name + `: ${gifLabel} should not pass a negated GIF requirement`,
+ );
+ }
+ assert.equal(
+ agent._workflowSocialPublishedAttachmentObserved({ value: 'a GIF' }, { attachments: [gifMedia.mp4] }),
+ false,
+ AgentClass.name + ': an ordinary mp4 should not satisfy a GIF requirement',
+ );
+ assert.equal(
+ agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one video and no GIFs' },
+ { attachments: [gifMedia.mp4] },
+ ),
+ true,
+ AgentClass.name + ': an ordinary mp4 should pass a negated GIF requirement',
+ );
+
+ // A singular article counts as exactly one, the same as the word "one".
+ const oneImage = { type: 'image', src: 'https://pbs.twimg.com/media/first.png' };
+ const otherImage = { type: 'image', src: 'https://pbs.twimg.com/media/second.png' };
+ const oneVideo = { type: 'video', src: 'https://video.twimg.com/media/clip.mp4' };
+ for (const articleReq of ['a video', 'an image', 'un video', 'une image', 'ein bild', 'uma imagem']) {
+ assert.equal(agent._parseWorkflowAttachmentRequirement({ value: articleReq }).hasExplicitCardinality, true,
+ AgentClass.name + `: "${articleReq}" should read as an explicit count of one`);
+ }
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved({ value: 'a video' }, { attachments: [oneVideo] }), true,
+ AgentClass.name + ': one video should satisfy "a video"');
+ assert.equal(
+ agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'a video' },
+ { attachments: [oneVideo, { type: 'video', src: 'https://video.twimg.com/media/other.mp4' }] },
+ ),
+ false,
+ AgentClass.name + ': two videos should not satisfy "a video"',
+ );
+ assert.equal(
+ agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'an image and a video' },
+ { attachments: [oneImage, oneVideo] },
+ ),
+ true,
+ AgentClass.name + ': one image and one video should satisfy "an image and a video"',
+ );
+ assert.equal(
+ agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'an image and a video' },
+ { attachments: [oneImage, otherImage, oneVideo] },
+ ),
+ false,
+ AgentClass.name + ': an extra image should not satisfy "an image and a video"',
+ );
+
// Minimum-count qualifiers bound the count from below instead of naming a file.
for (const [minReq, minCount] of [
['at least two images', 2],