Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(social-publish): retain positive and forbidden media

Barack Sokullud2d08e307d97
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 4b652b59c..c4e511b55 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -3103,7 +3103,8 @@ export class Agent extends LoopDetector {
const isGifNegated = GIF_NEGATION_REGEX.test(negationText)
|| Boolean(coordinatedNegatedMedia && RAW_GIF_NOUN_REGEX.test(coordinatedNegatedMedia));
- if (isImageNegated && isVideoNegated) {
+ const hasPositiveGif = RAW_GIF_NOUN_REGEX.test(text) && !isGifNegated;
+ if (isImageNegated && isVideoNegated && !hasPositiveGif) {
return {
isGeneric: true,
isNegative: true,
@@ -3134,6 +3135,12 @@ export class Agent extends LoopDetector {
const videoFormatQualifier = `${VIDEO_ATTACHMENT_FORMAT}(?:\\s*(?:or|and|/|,)\\s*${VIDEO_ATTACHMENT_FORMAT})*`;
const formatPostfixLead = '(?:in|as)\\s+(?:the\\s+)?';
const formatPostfixSuffix = '(?:\\s+formats?)?';
+ const mediaFormatMatchIsNegated = (start) => {
+ const precedingSegment = negationText.slice(0, start)
+ .split(/(?:[,;]|\\s+(?:and|but|plus|also|as\\s+well\\s+as)\\s+)/iu)
+ .pop() || '';
+ return /(?<![\p{L}\p{N}_])(?:no|not(?!\s+only)|without(?:\s+any)?|zero|0)(?![\p{L}\p{N}_])/iu.test(precedingSegment);
+ };
const collectMediaFormatMatches = (formatQualifier, nounPattern) => {
const matches = [];
for (const match of text.matchAll(new RegExp(
@@ -3141,7 +3148,12 @@ export class Agent extends LoopDetector {
'giu',
))) {
const start = match.index || 0;
- matches.push({ formats: match[1], start, end: start + match[1].length });
+ matches.push({
+ formats: match[1],
+ start,
+ end: start + match[1].length,
+ isNegated: mediaFormatMatchIsNegated(start),
+ });
}
for (const match of text.matchAll(new RegExp(
`(?<![${SOCIAL_WORD_EDGE}])(?:${nounPattern})(?![${SOCIAL_WORD_EDGE}])\\s+${formatPostfixLead}(${formatQualifier})(?![${SOCIAL_WORD_EDGE}])${formatPostfixSuffix}`,
@@ -3149,7 +3161,12 @@ export class Agent extends LoopDetector {
))) {
const formatOffset = match[0].lastIndexOf(match[1]);
const start = (match.index || 0) + Math.max(0, formatOffset);
- matches.push({ formats: match[1], start, end: start + match[1].length });
+ matches.push({
+ formats: match[1],
+ start,
+ end: start + match[1].length,
+ isNegated: mediaFormatMatchIsNegated(start),
+ });
}
return matches;
};
@@ -3157,16 +3174,26 @@ export class Agent extends LoopDetector {
imageFormatQualifier,
'images?|photos?|pictures?|pics?',
);
- const requestedImageFormats = imageFormatMatches.flatMap(match => [...match.formats.matchAll(new RegExp(IMAGE_ATTACHMENT_FORMAT, 'giu'))]
+ const requestedImageFormats = imageFormatMatches.filter(match => !match.isNegated)
+ .flatMap(match => [...match.formats.matchAll(new RegExp(IMAGE_ATTACHMENT_FORMAT, 'giu'))]
.map(formatMatch => normalizeAttachmentFormat(formatMatch[0])))
.filter((format, index, formats) => formats.indexOf(format) === index);
+ const forbiddenImageFormats = imageFormatMatches.filter(match => match.isNegated)
+ .flatMap(match => [...match.formats.matchAll(new RegExp(IMAGE_ATTACHMENT_FORMAT, 'giu'))]
+ .map(formatMatch => normalizeAttachmentFormat(formatMatch[0])))
+ .filter((format, index, formats) => formats.indexOf(format) === index);
const videoFormatMatches = collectMediaFormatMatches(
videoFormatQualifier,
'videos?|clips?|recordings?',
);
- const requestedVideoFormats = videoFormatMatches.flatMap(match => [...match.formats.matchAll(new RegExp(VIDEO_ATTACHMENT_FORMAT, 'giu'))]
+ const requestedVideoFormats = videoFormatMatches.filter(match => !match.isNegated)
+ .flatMap(match => [...match.formats.matchAll(new RegExp(VIDEO_ATTACHMENT_FORMAT, 'giu'))]
.map(formatMatch => normalizeAttachmentFormat(formatMatch[0])))
.filter((format, index, formats) => formats.indexOf(format) === index);
+ const forbiddenVideoFormats = videoFormatMatches.filter(match => match.isNegated)
+ .flatMap(match => [...match.formats.matchAll(new RegExp(VIDEO_ATTACHMENT_FORMAT, 'giu'))]
+ .map(formatMatch => normalizeAttachmentFormat(formatMatch[0])))
+ .filter((format, index, formats) => formats.indexOf(format) === index);
const mediaFormatQualifierSpans = [...imageFormatMatches, ...videoFormatMatches]
.map(match => [match.start, match.end]);
@@ -3471,9 +3498,15 @@ export class Agent extends LoopDetector {
`(?<![${SOCIAL_WORD_EDGE}])(${countPrefix})(?![${SOCIAL_WORD_EDGE}])\\s+(?:${nounPattern})(?![${SOCIAL_WORD_EDGE}])\\s+${formatPostfixLead}(${formatPattern})(?![${SOCIAL_WORD_EDGE}])${formatPostfixSuffix}`,
'giu',
)),
- ].map(match => parseMediaCountMatch(match, normalizeAttachmentFormat(match[2])))
- .filter(requirement => requirement.exactCount > 0
- || requirement.minimumCount > 0 || requirement.maximumCount > 0);
+ ].map((match) => {
+ const requirement = parseMediaCountMatch(match, normalizeAttachmentFormat(match[2]));
+ const formatOffset = match[0].lastIndexOf(match[2]);
+ return {
+ ...requirement,
+ isNegated: mediaFormatMatchIsNegated((match.index || 0) + Math.max(0, formatOffset)),
+ };
+ }).filter(requirement => !requirement.isNegated && (requirement.exactCount > 0
+ || requirement.minimumCount > 0 || requirement.maximumCount > 0));
const collectUnrestrictedCountRequirements = nounPattern => [...text.matchAll(new RegExp(
`(?<![${SOCIAL_WORD_EDGE}])(${countPrefix})(?![${SOCIAL_WORD_EDGE}])\\s+(?:${nounPattern})(?![${SOCIAL_WORD_EDGE}])`,
'giu',
@@ -3931,6 +3964,8 @@ export class Agent extends LoopDetector {
wantsGif,
requestedImageFormats,
requestedVideoFormats,
+ forbiddenImageFormats,
+ forbiddenVideoFormats,
imageFormatCounts,
videoFormatCounts,
imageFormatConstraints,
@@ -4173,6 +4208,8 @@ export class Agent extends LoopDetector {
const videoCount = rawAttachments.filter(isVideoAttachment).length;
const gifCount = rawAttachments.filter(isGifAttachment).length;
const ordinaryVideoCount = ordinaryVideoAttachments.length;
+ if (imageAttachments.some(att => parsed.forbiddenImageFormats?.includes(attachmentFormat(att)))) return false;
+ if (ordinaryVideoAttachments.some(att => parsed.forbiddenVideoFormats?.includes(attachmentFormat(att)))) return false;
if (!parsed.imageUnrestrictedConstraint
&& !attachmentsMatchFormats(parsed.requestedImageFormats, imageAttachments)) return false;
if (!parsed.videoUnrestrictedConstraint
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 68dcd78da..e6cb2de5e 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -2818,7 +2818,8 @@ export class Agent extends LoopDetector {
const isGifNegated = GIF_NEGATION_REGEX.test(negationText)
|| Boolean(coordinatedNegatedMedia && RAW_GIF_NOUN_REGEX.test(coordinatedNegatedMedia));
- if (isImageNegated && isVideoNegated) {
+ const hasPositiveGif = RAW_GIF_NOUN_REGEX.test(text) && !isGifNegated;
+ if (isImageNegated && isVideoNegated && !hasPositiveGif) {
return {
isGeneric: true,
isNegative: true,
@@ -2849,6 +2850,12 @@ export class Agent extends LoopDetector {
const videoFormatQualifier = `${VIDEO_ATTACHMENT_FORMAT}(?:\\s*(?:or|and|/|,)\\s*${VIDEO_ATTACHMENT_FORMAT})*`;
const formatPostfixLead = '(?:in|as)\\s+(?:the\\s+)?';
const formatPostfixSuffix = '(?:\\s+formats?)?';
+ const mediaFormatMatchIsNegated = (start) => {
+ const precedingSegment = negationText.slice(0, start)
+ .split(/(?:[,;]|\\s+(?:and|but|plus|also|as\\s+well\\s+as)\\s+)/iu)
+ .pop() || '';
+ return /(?<![\p{L}\p{N}_])(?:no|not(?!\s+only)|without(?:\s+any)?|zero|0)(?![\p{L}\p{N}_])/iu.test(precedingSegment);
+ };
const collectMediaFormatMatches = (formatQualifier, nounPattern) => {
const matches = [];
for (const match of text.matchAll(new RegExp(
@@ -2856,7 +2863,12 @@ export class Agent extends LoopDetector {
'giu',
))) {
const start = match.index || 0;
- matches.push({ formats: match[1], start, end: start + match[1].length });
+ matches.push({
+ formats: match[1],
+ start,
+ end: start + match[1].length,
+ isNegated: mediaFormatMatchIsNegated(start),
+ });
}
for (const match of text.matchAll(new RegExp(
`(?<![${SOCIAL_WORD_EDGE}])(?:${nounPattern})(?![${SOCIAL_WORD_EDGE}])\\s+${formatPostfixLead}(${formatQualifier})(?![${SOCIAL_WORD_EDGE}])${formatPostfixSuffix}`,
@@ -2864,7 +2876,12 @@ export class Agent extends LoopDetector {
))) {
const formatOffset = match[0].lastIndexOf(match[1]);
const start = (match.index || 0) + Math.max(0, formatOffset);
- matches.push({ formats: match[1], start, end: start + match[1].length });
+ matches.push({
+ formats: match[1],
+ start,
+ end: start + match[1].length,
+ isNegated: mediaFormatMatchIsNegated(start),
+ });
}
return matches;
};
@@ -2872,16 +2889,26 @@ export class Agent extends LoopDetector {
imageFormatQualifier,
'images?|photos?|pictures?|pics?',
);
- const requestedImageFormats = imageFormatMatches.flatMap(match => [...match.formats.matchAll(new RegExp(IMAGE_ATTACHMENT_FORMAT, 'giu'))]
+ const requestedImageFormats = imageFormatMatches.filter(match => !match.isNegated)
+ .flatMap(match => [...match.formats.matchAll(new RegExp(IMAGE_ATTACHMENT_FORMAT, 'giu'))]
.map(formatMatch => normalizeAttachmentFormat(formatMatch[0])))
.filter((format, index, formats) => formats.indexOf(format) === index);
+ const forbiddenImageFormats = imageFormatMatches.filter(match => match.isNegated)
+ .flatMap(match => [...match.formats.matchAll(new RegExp(IMAGE_ATTACHMENT_FORMAT, 'giu'))]
+ .map(formatMatch => normalizeAttachmentFormat(formatMatch[0])))
+ .filter((format, index, formats) => formats.indexOf(format) === index);
const videoFormatMatches = collectMediaFormatMatches(
videoFormatQualifier,
'videos?|clips?|recordings?',
);
- const requestedVideoFormats = videoFormatMatches.flatMap(match => [...match.formats.matchAll(new RegExp(VIDEO_ATTACHMENT_FORMAT, 'giu'))]
+ const requestedVideoFormats = videoFormatMatches.filter(match => !match.isNegated)
+ .flatMap(match => [...match.formats.matchAll(new RegExp(VIDEO_ATTACHMENT_FORMAT, 'giu'))]
.map(formatMatch => normalizeAttachmentFormat(formatMatch[0])))
.filter((format, index, formats) => formats.indexOf(format) === index);
+ const forbiddenVideoFormats = videoFormatMatches.filter(match => match.isNegated)
+ .flatMap(match => [...match.formats.matchAll(new RegExp(VIDEO_ATTACHMENT_FORMAT, 'giu'))]
+ .map(formatMatch => normalizeAttachmentFormat(formatMatch[0])))
+ .filter((format, index, formats) => formats.indexOf(format) === index);
const mediaFormatQualifierSpans = [...imageFormatMatches, ...videoFormatMatches]
.map(match => [match.start, match.end]);
@@ -3186,9 +3213,15 @@ export class Agent extends LoopDetector {
`(?<![${SOCIAL_WORD_EDGE}])(${countPrefix})(?![${SOCIAL_WORD_EDGE}])\\s+(?:${nounPattern})(?![${SOCIAL_WORD_EDGE}])\\s+${formatPostfixLead}(${formatPattern})(?![${SOCIAL_WORD_EDGE}])${formatPostfixSuffix}`,
'giu',
)),
- ].map(match => parseMediaCountMatch(match, normalizeAttachmentFormat(match[2])))
- .filter(requirement => requirement.exactCount > 0
- || requirement.minimumCount > 0 || requirement.maximumCount > 0);
+ ].map((match) => {
+ const requirement = parseMediaCountMatch(match, normalizeAttachmentFormat(match[2]));
+ const formatOffset = match[0].lastIndexOf(match[2]);
+ return {
+ ...requirement,
+ isNegated: mediaFormatMatchIsNegated((match.index || 0) + Math.max(0, formatOffset)),
+ };
+ }).filter(requirement => !requirement.isNegated && (requirement.exactCount > 0
+ || requirement.minimumCount > 0 || requirement.maximumCount > 0));
const collectUnrestrictedCountRequirements = nounPattern => [...text.matchAll(new RegExp(
`(?<![${SOCIAL_WORD_EDGE}])(${countPrefix})(?![${SOCIAL_WORD_EDGE}])\\s+(?:${nounPattern})(?![${SOCIAL_WORD_EDGE}])`,
'giu',
@@ -3646,6 +3679,8 @@ export class Agent extends LoopDetector {
wantsGif,
requestedImageFormats,
requestedVideoFormats,
+ forbiddenImageFormats,
+ forbiddenVideoFormats,
imageFormatCounts,
videoFormatCounts,
imageFormatConstraints,
@@ -3888,6 +3923,8 @@ export class Agent extends LoopDetector {
const videoCount = rawAttachments.filter(isVideoAttachment).length;
const gifCount = rawAttachments.filter(isGifAttachment).length;
const ordinaryVideoCount = ordinaryVideoAttachments.length;
+ if (imageAttachments.some(att => parsed.forbiddenImageFormats?.includes(attachmentFormat(att)))) return false;
+ if (ordinaryVideoAttachments.some(att => parsed.forbiddenVideoFormats?.includes(attachmentFormat(att)))) return false;
if (!parsed.imageUnrestrictedConstraint
&& !attachmentsMatchFormats(parsed.requestedImageFormats, imageAttachments)) return false;
if (!parsed.videoUnrestrictedConstraint
diff --git a/test/run.js b/test/run.js
index 076c67087..2f29dcbdd 100644
--- a/test/run.js
+++ b/test/run.js
@@ -93069,6 +93069,26 @@ test('upper-bound attachment qualifiers verify as maximum counts', () => {
assert.equal(agent._workflowSocialPublishedAttachmentObserved(
{ value: 'one video in MP4 format' }, { attachments: [{ type: 'video', src: 'https://cdn.example/a.webm' }] }), false,
AgentClass.name + ': a WebM video satisfied a postfix MP4 contract');
+ const pngWithoutJpeg = agent._parseWorkflowAttachmentRequirement('one PNG image and no JPEG images');
+ assert.deepEqual(pngWithoutJpeg.requestedImageFormats, ['png'],
+ AgentClass.name + ': a negated JPEG qualifier was folded into the allowed image formats');
+ assert.deepEqual(pngWithoutJpeg.forbiddenImageFormats, ['jpeg'],
+ AgentClass.name + ': a negated JPEG qualifier was not retained');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one PNG image and no JPEG images' }, { attachments: [image(1)] }), true,
+ AgentClass.name + ': the required PNG was rejected by a separate JPEG prohibition');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one PNG image and no JPEG images' }, { attachments: [{ type: 'image', src: 'https://cdn.example/a.jpg' }] }), false,
+ AgentClass.name + ': an explicitly prohibited JPEG satisfied the image contract');
+ const mp4WithoutWebm = agent._parseWorkflowAttachmentRequirement('one video in MP4 format and no WebM videos');
+ assert.deepEqual(mp4WithoutWebm.requestedVideoFormats, ['mp4']);
+ assert.deepEqual(mp4WithoutWebm.forbiddenVideoFormats, ['webm']);
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one video in MP4 format and no WebM videos' }, { attachments: [video(1)] }), true,
+ AgentClass.name + ': the required MP4 was rejected by a separate WebM prohibition');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one video in MP4 format and no WebM videos' }, { attachments: [{ type: 'video', src: 'https://cdn.example/a.webm' }] }), false,
+ AgentClass.name + ': an explicitly prohibited WebM satisfied the video contract');
const pngOrJpegImage = agent._parseWorkflowAttachmentRequirement('one PNG or JPEG image');
assert.equal(pngOrJpegImage.isGeneric, true,
AgentClass.name + ': a coordinated image-format choice was parsed as filenames');
@@ -93338,6 +93358,22 @@ test('upper-bound attachment qualifiers verify as maximum counts', () => {
AgentClass.name + `: forbidden media satisfied "${coordinatedNegative}"`);
}
}
+ const gifWithOtherTypesProhibited = agent._parseWorkflowAttachmentRequirement(
+ 'one GIF but no images or videos',
+ );
+ assert.notEqual(gifWithOtherTypesProhibited.wantsNone, true,
+ AgentClass.name + ': broad media negation discarded an affirmative GIF');
+ assert.equal(gifWithOtherTypesProhibited.wantsGif, true,
+ AgentClass.name + ': an affirmative GIF was treated as negated');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one GIF but no images or videos' }, { attachments: [gif(1)] }), true,
+ AgentClass.name + ': a GIF did not satisfy the positive GIF-only contract');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one GIF but no images or videos' }, { attachments: [video(1)] }), false,
+ AgentClass.name + ': an ordinary video satisfied the positive GIF-only contract');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one GIF but no images or videos' }, { attachments: [gif(1), image(1)] }), false,
+ AgentClass.name + ': a prohibited image survived beside the required GIF');
assert.equal(agent._workflowSocialPublishedAttachmentObserved(
{ value: 'one image and no videos or GIFs' }, { attachments: [image(1)] }), true,
AgentClass.name + ': embedded coordinated media negation rejected the required image');