Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(social-publish): evaluate typed media alternatives

Barack Sokullufa7a53ec146f
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index dfca5369e..9ea3ef732 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -308,7 +308,7 @@ const GENERIC_ATTACHMENT_WORDS = new Set([
'item', 'items', 'piece', 'pieces', 'upload', 'uploads', 'asset', 'assets',
'enclosure', 'enclosures', 'document', 'documents', 'documento', 'documentos',
'a', 'an', 'the', 'of', 'in', 'with', 'some', 'any',
- 'and', 'or', 'plus', 'also', 'as', 'well', 'between', 'from', 'to',
+ 'and', 'or', 'either', 'plus', 'also', 'as', 'well', 'between', 'from', 'to',
'no', 'not', 'without', 'zero', 'none',
'only', 'just', 'solely', 'exactly', 'exact', 'precisely',
'sin', 'sans', 'sem', 'senza', 'ohne', 'kein', 'keine', 'keinen', 'aucun', 'aucune', 'ningun', 'ninguna', 'ningún', 'nenhum', 'nenhuma', 'nessun', 'nessuno', 'nessuna', 'nie', 'без', 'нет',
@@ -3640,7 +3640,9 @@ export class Agent extends LoopDetector {
: rawAttachments.length;
if (boundedCount < parsed.minimumCount || boundedCount > parsed.maximumCount) return false;
}
- if (parsed.wantsGif) {
+ const gifIsAlternativeToAnotherType = parsed.isAlternative
+ && (parsed.wantsImage || parsed.wantsOrdinaryVideo);
+ if (parsed.wantsGif && !gifIsAlternativeToAnotherType) {
const requiredGifCount = parsed.alternativeCounts?.length
? Math.min(...parsed.alternativeCounts)
: (parsed.isGifMaximum ? 0 : (parsed.expectedGifCount > 0 ? parsed.expectedGifCount : 1));
@@ -3649,7 +3651,7 @@ export class Agent extends LoopDetector {
&& !parsed.isGifMinimum && !parsed.isGifMaximum && gifCount !== parsed.expectedGifCount) return false;
if (parsed.hasExplicitGifCount && parsed.isGifMaximum && gifCount > parsed.expectedGifCount) return false;
}
- if (parsed.wantsGif && parsed.wantsOrdinaryVideo) {
+ if (parsed.wantsGif && parsed.wantsOrdinaryVideo && !parsed.isAlternative) {
const requiredOrdinaryVideoCount = parsed.isVideoMaximum ? 0 : (parsed.expectedVideoCount > 0 ? parsed.expectedVideoCount : 1);
if (ordinaryVideoCount < requiredOrdinaryVideoCount) return false;
if (parsed.hasExplicitVideoCount && !parsed.isVideoMinimum && !parsed.isVideoMaximum
@@ -3780,7 +3782,26 @@ export class Agent extends LoopDetector {
// cardinalities. Their typed checks above are authoritative when both
// subtypes were requested; comparing their combined count with one
// subtype's exact/maximum mode would turn a maximum into an exact total.
- if (!(parsed.wantsGif && parsed.wantsOrdinaryVideo)) {
+ if (parsed.isAlternative && parsed.wantsGif && parsed.wantsOrdinaryVideo) {
+ const minGifs = parsed.isGifMaximum ? 0 : (parsed.expectedGifCount > 0 ? parsed.expectedGifCount : 1);
+ const minOrdinaryVideos = parsed.isVideoMaximum ? 0 : (parsed.expectedVideoCount > 0 ? parsed.expectedVideoCount : 1);
+ const exactGifAlt = parsed.hasExplicitGifCount && !parsed.isGifMinimum && !parsed.isGifMaximum;
+ const exactVideoAlt = parsed.hasExplicitVideoCount && !parsed.isVideoMinimum && !parsed.isVideoMaximum;
+ const matchesGifAlt = gifCount >= minGifs
+ && (!exactGifAlt || gifCount === parsed.expectedGifCount)
+ && (!parsed.isGifMaximum || gifCount <= parsed.expectedGifCount)
+ && ordinaryVideoCount === 0
+ && rawAttachments.length === gifCount;
+ const matchesOrdinaryVideoAlt = ordinaryVideoCount >= minOrdinaryVideos
+ && (!exactVideoAlt || ordinaryVideoCount === parsed.expectedVideoCount)
+ && (!parsed.isVideoMaximum || ordinaryVideoCount <= parsed.expectedVideoCount)
+ && gifCount === 0
+ && rawAttachments.length === ordinaryVideoCount;
+ if (!matchesGifAlt && !matchesOrdinaryVideoAlt) return false;
+ matchingAttachments = matchesGifAlt
+ ? rawAttachments.filter(isGifAttachment)
+ : rawAttachments.filter(att => isVideoAttachment(att) && !isGifAttachment(att));
+ } else if (!(parsed.wantsGif && parsed.wantsOrdinaryVideo)) {
const hasExactVideoCount = (parsed.hasExplicitVideoCount && !parsed.isVideoNegated && !parsed.isVideoMinimum && !parsed.isVideoMaximum)
|| (parsed.hasExplicitGifCount && !parsed.isGifNegated && !parsed.isGifMinimum && !parsed.isGifMaximum)
|| (parsed.hasExplicitGenericCount && !parsed.isMaximumCount);
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 0deef186a..ce7ad03aa 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -242,7 +242,7 @@ const GENERIC_ATTACHMENT_WORDS = new Set([
'item', 'items', 'piece', 'pieces', 'upload', 'uploads', 'asset', 'assets',
'enclosure', 'enclosures', 'document', 'documents', 'documento', 'documentos',
'a', 'an', 'the', 'of', 'in', 'with', 'some', 'any',
- 'and', 'or', 'plus', 'also', 'as', 'well', 'between', 'from', 'to',
+ 'and', 'or', 'either', 'plus', 'also', 'as', 'well', 'between', 'from', 'to',
'no', 'not', 'without', 'zero', 'none',
'only', 'just', 'solely', 'exactly', 'exact', 'precisely',
'sin', 'sans', 'sem', 'senza', 'ohne', 'kein', 'keine', 'keinen', 'aucun', 'aucune', 'ningun', 'ninguna', 'ningún', 'nenhum', 'nenhuma', 'nessun', 'nessuno', 'nessuna', 'nie', 'без', 'нет',
@@ -3355,7 +3355,9 @@ export class Agent extends LoopDetector {
: rawAttachments.length;
if (boundedCount < parsed.minimumCount || boundedCount > parsed.maximumCount) return false;
}
- if (parsed.wantsGif) {
+ const gifIsAlternativeToAnotherType = parsed.isAlternative
+ && (parsed.wantsImage || parsed.wantsOrdinaryVideo);
+ if (parsed.wantsGif && !gifIsAlternativeToAnotherType) {
const requiredGifCount = parsed.alternativeCounts?.length
? Math.min(...parsed.alternativeCounts)
: (parsed.isGifMaximum ? 0 : (parsed.expectedGifCount > 0 ? parsed.expectedGifCount : 1));
@@ -3364,7 +3366,7 @@ export class Agent extends LoopDetector {
&& !parsed.isGifMinimum && !parsed.isGifMaximum && gifCount !== parsed.expectedGifCount) return false;
if (parsed.hasExplicitGifCount && parsed.isGifMaximum && gifCount > parsed.expectedGifCount) return false;
}
- if (parsed.wantsGif && parsed.wantsOrdinaryVideo) {
+ if (parsed.wantsGif && parsed.wantsOrdinaryVideo && !parsed.isAlternative) {
const requiredOrdinaryVideoCount = parsed.isVideoMaximum ? 0 : (parsed.expectedVideoCount > 0 ? parsed.expectedVideoCount : 1);
if (ordinaryVideoCount < requiredOrdinaryVideoCount) return false;
if (parsed.hasExplicitVideoCount && !parsed.isVideoMinimum && !parsed.isVideoMaximum
@@ -3495,7 +3497,26 @@ export class Agent extends LoopDetector {
// cardinalities. Their typed checks above are authoritative when both
// subtypes were requested; comparing their combined count with one
// subtype's exact/maximum mode would turn a maximum into an exact total.
- if (!(parsed.wantsGif && parsed.wantsOrdinaryVideo)) {
+ if (parsed.isAlternative && parsed.wantsGif && parsed.wantsOrdinaryVideo) {
+ const minGifs = parsed.isGifMaximum ? 0 : (parsed.expectedGifCount > 0 ? parsed.expectedGifCount : 1);
+ const minOrdinaryVideos = parsed.isVideoMaximum ? 0 : (parsed.expectedVideoCount > 0 ? parsed.expectedVideoCount : 1);
+ const exactGifAlt = parsed.hasExplicitGifCount && !parsed.isGifMinimum && !parsed.isGifMaximum;
+ const exactVideoAlt = parsed.hasExplicitVideoCount && !parsed.isVideoMinimum && !parsed.isVideoMaximum;
+ const matchesGifAlt = gifCount >= minGifs
+ && (!exactGifAlt || gifCount === parsed.expectedGifCount)
+ && (!parsed.isGifMaximum || gifCount <= parsed.expectedGifCount)
+ && ordinaryVideoCount === 0
+ && rawAttachments.length === gifCount;
+ const matchesOrdinaryVideoAlt = ordinaryVideoCount >= minOrdinaryVideos
+ && (!exactVideoAlt || ordinaryVideoCount === parsed.expectedVideoCount)
+ && (!parsed.isVideoMaximum || ordinaryVideoCount <= parsed.expectedVideoCount)
+ && gifCount === 0
+ && rawAttachments.length === ordinaryVideoCount;
+ if (!matchesGifAlt && !matchesOrdinaryVideoAlt) return false;
+ matchingAttachments = matchesGifAlt
+ ? rawAttachments.filter(isGifAttachment)
+ : rawAttachments.filter(att => isVideoAttachment(att) && !isGifAttachment(att));
+ } else if (!(parsed.wantsGif && parsed.wantsOrdinaryVideo)) {
const hasExactVideoCount = (parsed.hasExplicitVideoCount && !parsed.isVideoNegated && !parsed.isVideoMinimum && !parsed.isVideoMaximum)
|| (parsed.hasExplicitGifCount && !parsed.isGifNegated && !parsed.isGifMinimum && !parsed.isGifMaximum)
|| (parsed.hasExplicitGenericCount && !parsed.isMaximumCount);
diff --git a/test/run.js b/test/run.js
index 6f4ebb255..4843650ad 100644
--- a/test/run.js
+++ b/test/run.js
@@ -94002,6 +94002,45 @@ test('attachment verification matches specific attachment names without substrin
AgentClass.name + ': simultaneous image and video should reject "one image or one video"'
);
+ const alternativeImage = index => ({ type: 'image', src: `https://example.test/alternative-${index}.png` });
+ const alternativeVideo = index => ({ type: 'video', src: `https://example.test/alternative-${index}.mp4` });
+ const alternativeGif = index => ({ type: 'animated_gif', src: `https://example.test/alternative-${index}.gif` });
+ for (const { requirement, valid, invalid } of [
+ {
+ requirement: 'either one image or two videos',
+ valid: [[alternativeImage(1)], [alternativeVideo(1), alternativeVideo(2)]],
+ invalid: [[alternativeVideo(1)]],
+ },
+ {
+ requirement: 'one image or two GIFs',
+ valid: [[alternativeImage(1)], [alternativeGif(1), alternativeGif(2)]],
+ invalid: [[alternativeGif(1)], [alternativeVideo(1), alternativeVideo(2)], [alternativeImage(1), alternativeGif(1)]],
+ },
+ {
+ requirement: 'one GIF or two videos',
+ valid: [[alternativeGif(1)], [alternativeVideo(1), alternativeVideo(2)]],
+ invalid: [[alternativeVideo(1)], [alternativeGif(1), alternativeVideo(1)], [alternativeGif(1), alternativeGif(2)]],
+ },
+ ]) {
+ const parsedTypedAlternative = agent._parseWorkflowAttachmentRequirement({ value: requirement });
+ assert.equal(parsedTypedAlternative.isGeneric, true,
+ AgentClass.name + `: typed alternative was parsed as filenames for "${requirement}"`);
+ assert.equal(parsedTypedAlternative.isAlternative, true,
+ AgentClass.name + `: typed alternative grammar was lost for "${requirement}"`);
+ for (const attachments of valid) {
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: requirement },
+ { attachments },
+ ), true, AgentClass.name + `: a permitted branch did not satisfy "${requirement}"`);
+ }
+ for (const attachments of invalid) {
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: requirement },
+ { attachments },
+ ), false, AgentClass.name + `: a non-permitted branch satisfied "${requirement}"`);
+ }
+ }
+
for (const [sameTypeRequirement, attachment] of [
['one or two images', index => ({ type: 'image', src: `https://example.test/${index}.png` })],
['one image or two images', index => ({ type: 'image', src: `https://example.test/${index}.png` })],