Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(social-publish): tighten destination and media evidence

Barack Sokullua253d5061619
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 0c03e1741..ab9f93a8b 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -274,7 +274,7 @@ const SOCIAL_CLAUSE_BREAK = new RegExp(
// variable, not a destination. The platform pattern is case-insensitive, so
// the test is on the following noun (duration, variable, field, or similar),
// not on the letter's case.
-const PLACEHOLDER_X_AFTER_PLATFORM = /^\s*(?:days?|hours?|hrs?|minutes?|mins?|seconds?|secs?|weeks?|months?|years?|variables?|values?|fields?|inputs?|columns?|rows?|cells?|axis|coordinates?|params?(?:eters?)?|placeholders?|amounts?|counts?|numbers?|digits?|items?|steps?)\b|^\s*[:=+\-*/]|^\s*\d/iu;
+const PLACEHOLDER_X_AFTER_PLATFORM = /^\s*(?:(?:business|calendar|working|work|more|additional)\s+)*(?:days?|hours?|hrs?|minutes?|mins?|seconds?|secs?|weeks?|months?|years?|variables?|values?|fields?|inputs?|columns?|rows?|cells?|axis|coordinates?|params?(?:eters?)?|placeholders?|amounts?|counts?|numbers?|digits?|items?|steps?)\b|^\s*[:=+\-*/]|^\s*\d/iu;
const isPlaceholderBareXMatch = (matchedText, afterText) => {
const matched = String(matchedText || '');
if (/twitter/i.test(matched) || /x\.com/i.test(matched)) return false;
@@ -1695,14 +1695,20 @@ export class Agent extends LoopDetector {
if (socialUploadEvidence && result && typeof result === 'object') {
result.socialUploadEvidence = socialUploadEvidence;
}
- // A removal in the social composer shows as the old filename disappearing
- // from later observations. Prune it there so the pre-dispatch binding
- // snapshots only active uploads instead of every file ever attached.
+ // Only a complete root AX read proves that an omitted composer filename was
+ // removed. Partial, paginated, and viewport-filtered observations preserve
+ // all active provenance.
if (result && typeof result === 'object') {
const socialObservationText = [result.pageContent, result.text, result.content, result.pageText]
.filter(value => typeof value === 'string' && value.trim())
.join('\n');
- if (socialObservationText) this._pruneStaleSocialPublishUploadEvidence(tabId, socialObservationText);
+ const completeComposerSnapshot = name === 'get_accessibility_tree'
+ && isExhaustiveAccessibilityInventoryRead(args, result).rootReadComplete;
+ if (socialObservationText) {
+ this._pruneStaleSocialPublishUploadEvidence(tabId, socialObservationText, {
+ completeComposerSnapshot,
+ });
+ }
}
const workflowControlEvidence = this._rememberWorkflowControlActionEvidence(
tabId,
@@ -3313,17 +3319,12 @@ export class Agent extends LoopDetector {
// never create one. Require one upload name per DOM-observed media node and
// a type-compatible bijection before adding the names to the record.
if (!attachments.length) return record;
- let effectiveNames = names;
- if (names.length !== attachments.length) {
- // An upload, removal, and replacement can leave both filenames in the
- // binding for one DOM-observed attachment when the removal never
- // surfaced as observation text. Fall back to the most recent uploads so
- // the replacement can still verify instead of refusing to join either
- // name and pushing the run toward a duplicate post. Fewer names than
- // nodes still fails closed: provenance may never invent media.
- if (names.length <= attachments.length) return record;
- effectiveNames = names.slice(-attachments.length);
- }
+ // Upload order alone cannot prove which files remain in the composer after
+ // a removal. Join names only when active provenance and observed media have
+ // the same cardinality; otherwise leave the CDN record unnamed and fail
+ // closed for filename-specific requirements.
+ if (names.length !== attachments.length) return record;
+ const effectiveNames = names;
const nameKind = (name) => (
/\.gif(?:[?#]|$)/i.test(name) ? 'video'
: /\.(?:mp4|mov|webm|mkv)(?:[?#]|$)/i.test(name) ? 'video'
@@ -3538,22 +3539,26 @@ export class Agent extends LoopDetector {
matchingAttachments = rawAttachments.filter(isImageAttachment);
} else if (parsed.wantsVideo && !parsed.wantsImage) {
if (imageCount > 0) return false;
- const hasExactVideoCount = (parsed.hasExplicitVideoCount && !parsed.isVideoNegated && !parsed.isVideoMinimum && !parsed.isVideoMaximum)
- || (parsed.hasExplicitGifCount && !parsed.isGifNegated && !parsed.isGifMinimum && !parsed.isGifMaximum)
- || (parsed.hasExplicitGenericCount && !parsed.isMaximumCount);
- if (hasExactVideoCount) {
- if (videoCount !== parsed.expectedCount || rawAttachments.length !== parsed.expectedCount) {
- return false;
- }
- } else if (parsed.isVideoMaximum || parsed.isGifMaximum || parsed.isMaximumCount) {
- if (videoCount > parsed.expectedCount || rawAttachments.length > parsed.expectedCount) {
- return false;
- }
- if (videoCount < 1) {
- return false;
- }
- } else {
- if (videoCount < parsed.expectedCount) {
+ // Ordinary videos and GIFs share the DOM video class but have separate
+ // cardinalities. Their typed checks above are authoritative when both
+ // subtypes were requested; comparing their combined count with one
+ // subtype's exact/maximum mode would turn a maximum into an exact total.
+ if (!(parsed.wantsGif && parsed.wantsOrdinaryVideo)) {
+ const hasExactVideoCount = (parsed.hasExplicitVideoCount && !parsed.isVideoNegated && !parsed.isVideoMinimum && !parsed.isVideoMaximum)
+ || (parsed.hasExplicitGifCount && !parsed.isGifNegated && !parsed.isGifMinimum && !parsed.isGifMaximum)
+ || (parsed.hasExplicitGenericCount && !parsed.isMaximumCount);
+ if (hasExactVideoCount) {
+ if (videoCount !== parsed.expectedCount || rawAttachments.length !== parsed.expectedCount) {
+ return false;
+ }
+ } else if (parsed.isVideoMaximum || parsed.isGifMaximum || parsed.isMaximumCount) {
+ if (videoCount > parsed.expectedCount || rawAttachments.length > parsed.expectedCount) {
+ return false;
+ }
+ if (videoCount < 1) {
+ return false;
+ }
+ } else if (videoCount < parsed.expectedCount) {
return false;
}
}
@@ -16331,13 +16336,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return evidence;
}
- // An upload, removal, and replacement in the social composer leaves every
- // uploaded filename in evidence while only the replacement is still active.
- // The pre-dispatch binding would then carry two names for one observed
- // attachment and refuse to join either, so a correctly published replacement
- // can never verify. Prune filenames the latest observation no longer shows
- // so the dispatch snapshot keeps only active uploads.
- _pruneStaleSocialPublishUploadEvidence(tabId, observationText) {
+ // Reconcile upload provenance only from a complete composer snapshot. A
+ // partial, paginated, or viewport-filtered observation can omit an active
+ // filename, so absence there is not evidence that the media was removed.
+ _pruneStaleSocialPublishUploadEvidence(tabId, observationText, {
+ completeComposerSnapshot = false,
+ } = {}) {
const guard = this._planExecutionGuards.get(tabId);
const siteWorkflow = guard?.siteWorkflow;
if (!guard?.enabled
@@ -16346,7 +16350,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const prior = Array.isArray(guard.workflowSocialUploadEvidence)
? guard.workflowSocialUploadEvidence
: [];
- if (prior.length < 2) return 0;
+ if (prior.length < 2 || !completeComposerSnapshot) return 0;
let normalized = String(observationText || '');
if (!normalized.trim()) return 0;
try { normalized = normalized.normalize('NFKC'); } catch {}
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index e676aab39..cca23486e 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -208,7 +208,7 @@ const SOCIAL_CLAUSE_BREAK = new RegExp(
// variable, not a destination. The platform pattern is case-insensitive, so
// the test is on the following noun (duration, variable, field, or similar),
// not on the letter's case.
-const PLACEHOLDER_X_AFTER_PLATFORM = /^\s*(?:days?|hours?|hrs?|minutes?|mins?|seconds?|secs?|weeks?|months?|years?|variables?|values?|fields?|inputs?|columns?|rows?|cells?|axis|coordinates?|params?(?:eters?)?|placeholders?|amounts?|counts?|numbers?|digits?|items?|steps?)\b|^\s*[:=+\-*/]|^\s*\d/iu;
+const PLACEHOLDER_X_AFTER_PLATFORM = /^\s*(?:(?:business|calendar|working|work|more|additional)\s+)*(?:days?|hours?|hrs?|minutes?|mins?|seconds?|secs?|weeks?|months?|years?|variables?|values?|fields?|inputs?|columns?|rows?|cells?|axis|coordinates?|params?(?:eters?)?|placeholders?|amounts?|counts?|numbers?|digits?|items?|steps?)\b|^\s*[:=+\-*/]|^\s*\d/iu;
const isPlaceholderBareXMatch = (matchedText, afterText) => {
const matched = String(matchedText || '');
if (/twitter/i.test(matched) || /x\.com/i.test(matched)) return false;
@@ -1411,14 +1411,20 @@ export class Agent extends LoopDetector {
if (socialUploadEvidence && result && typeof result === 'object') {
result.socialUploadEvidence = socialUploadEvidence;
}
- // A removal in the social composer shows as the old filename disappearing
- // from later observations. Prune it there so the pre-dispatch binding
- // snapshots only active uploads instead of every file ever attached.
+ // Only a complete root AX read proves that an omitted composer filename was
+ // removed. Partial, paginated, and viewport-filtered observations preserve
+ // all active provenance.
if (result && typeof result === 'object') {
const socialObservationText = [result.pageContent, result.text, result.content, result.pageText]
.filter(value => typeof value === 'string' && value.trim())
.join('\n');
- if (socialObservationText) this._pruneStaleSocialPublishUploadEvidence(tabId, socialObservationText);
+ const completeComposerSnapshot = name === 'get_accessibility_tree'
+ && isExhaustiveAccessibilityInventoryRead(args, result).rootReadComplete;
+ if (socialObservationText) {
+ this._pruneStaleSocialPublishUploadEvidence(tabId, socialObservationText, {
+ completeComposerSnapshot,
+ });
+ }
}
const workflowControlEvidence = this._rememberWorkflowControlActionEvidence(
tabId,
@@ -3028,17 +3034,12 @@ export class Agent extends LoopDetector {
// never create one. Require one upload name per DOM-observed media node and
// a type-compatible bijection before adding the names to the record.
if (!attachments.length) return record;
- let effectiveNames = names;
- if (names.length !== attachments.length) {
- // An upload, removal, and replacement can leave both filenames in the
- // binding for one DOM-observed attachment when the removal never
- // surfaced as observation text. Fall back to the most recent uploads so
- // the replacement can still verify instead of refusing to join either
- // name and pushing the run toward a duplicate post. Fewer names than
- // nodes still fails closed: provenance may never invent media.
- if (names.length <= attachments.length) return record;
- effectiveNames = names.slice(-attachments.length);
- }
+ // Upload order alone cannot prove which files remain in the composer after
+ // a removal. Join names only when active provenance and observed media have
+ // the same cardinality; otherwise leave the CDN record unnamed and fail
+ // closed for filename-specific requirements.
+ if (names.length !== attachments.length) return record;
+ const effectiveNames = names;
const nameKind = (name) => (
/\.gif(?:[?#]|$)/i.test(name) ? 'video'
: /\.(?:mp4|mov|webm|mkv)(?:[?#]|$)/i.test(name) ? 'video'
@@ -3253,22 +3254,26 @@ export class Agent extends LoopDetector {
matchingAttachments = rawAttachments.filter(isImageAttachment);
} else if (parsed.wantsVideo && !parsed.wantsImage) {
if (imageCount > 0) return false;
- const hasExactVideoCount = (parsed.hasExplicitVideoCount && !parsed.isVideoNegated && !parsed.isVideoMinimum && !parsed.isVideoMaximum)
- || (parsed.hasExplicitGifCount && !parsed.isGifNegated && !parsed.isGifMinimum && !parsed.isGifMaximum)
- || (parsed.hasExplicitGenericCount && !parsed.isMaximumCount);
- if (hasExactVideoCount) {
- if (videoCount !== parsed.expectedCount || rawAttachments.length !== parsed.expectedCount) {
- return false;
- }
- } else if (parsed.isVideoMaximum || parsed.isGifMaximum || parsed.isMaximumCount) {
- if (videoCount > parsed.expectedCount || rawAttachments.length > parsed.expectedCount) {
- return false;
- }
- if (videoCount < 1) {
- return false;
- }
- } else {
- if (videoCount < parsed.expectedCount) {
+ // Ordinary videos and GIFs share the DOM video class but have separate
+ // cardinalities. Their typed checks above are authoritative when both
+ // subtypes were requested; comparing their combined count with one
+ // subtype's exact/maximum mode would turn a maximum into an exact total.
+ if (!(parsed.wantsGif && parsed.wantsOrdinaryVideo)) {
+ const hasExactVideoCount = (parsed.hasExplicitVideoCount && !parsed.isVideoNegated && !parsed.isVideoMinimum && !parsed.isVideoMaximum)
+ || (parsed.hasExplicitGifCount && !parsed.isGifNegated && !parsed.isGifMinimum && !parsed.isGifMaximum)
+ || (parsed.hasExplicitGenericCount && !parsed.isMaximumCount);
+ if (hasExactVideoCount) {
+ if (videoCount !== parsed.expectedCount || rawAttachments.length !== parsed.expectedCount) {
+ return false;
+ }
+ } else if (parsed.isVideoMaximum || parsed.isGifMaximum || parsed.isMaximumCount) {
+ if (videoCount > parsed.expectedCount || rawAttachments.length > parsed.expectedCount) {
+ return false;
+ }
+ if (videoCount < 1) {
+ return false;
+ }
+ } else if (videoCount < parsed.expectedCount) {
return false;
}
}
@@ -14130,13 +14135,12 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
return evidence;
}
- // An upload, removal, and replacement in the social composer leaves every
- // uploaded filename in evidence while only the replacement is still active.
- // The pre-dispatch binding would then carry two names for one observed
- // attachment and refuse to join either, so a correctly published replacement
- // can never verify. Prune filenames the latest observation no longer shows
- // so the dispatch snapshot keeps only active uploads.
- _pruneStaleSocialPublishUploadEvidence(tabId, observationText) {
+ // Reconcile upload provenance only from a complete composer snapshot. A
+ // partial, paginated, or viewport-filtered observation can omit an active
+ // filename, so absence there is not evidence that the media was removed.
+ _pruneStaleSocialPublishUploadEvidence(tabId, observationText, {
+ completeComposerSnapshot = false,
+ } = {}) {
const guard = this._planExecutionGuards.get(tabId);
const siteWorkflow = guard?.siteWorkflow;
if (!guard?.enabled
@@ -14145,7 +14149,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
const prior = Array.isArray(guard.workflowSocialUploadEvidence)
? guard.workflowSocialUploadEvidence
: [];
- if (prior.length < 2) return 0;
+ if (prior.length < 2 || !completeComposerSnapshot) return 0;
let normalized = String(observationText || '');
if (!normalized.trim()) return 0;
try { normalized = normalized.normalize('NFKC'); } catch {}
diff --git a/test/run.js b/test/run.js
index 2f1e2cb31..39717404f 100644
--- a/test/run.js
+++ b/test/run.js
@@ -92468,6 +92468,13 @@ test('placeholder x values are not social destinations', () => {
['bluesky'],
AgentClass.name + ': placeholder x variable bound an unintended X destination',
);
+ for (const qualifier of ['business', 'calendar', 'more']) {
+ assert.deepEqual(
+ [...agent._trustedSocialPublishTargetAdapters({ taskText: `Publish this in x ${qualifier} days on Bluesky` })].sort(),
+ ['bluesky'],
+ AgentClass.name + `: placeholder x ${qualifier} days bound an unintended X destination`,
+ );
+ }
assert.deepEqual(
[...agent._trustedSocialPublishTargetAdapters({ taskText: 'Post on X' })].sort(),
['twitter'],
@@ -92481,7 +92488,7 @@ test('placeholder x values are not social destinations', () => {
}
});
-test('replaced composer media drops stale upload provenance', () => {
+test('social upload provenance reconciles only from complete composer snapshots', () => {
for (const [index, AgentClass] of [AgentCh, AgentFx].entries()) {
const agent = new AgentClass({});
agent.useSiteAdapters = true;
@@ -92501,34 +92508,68 @@ test('replaced composer media drops stale upload provenance', () => {
siteWorkflow,
siteWorkflowUrl: 'https://x.com/compose/post',
});
- guard.workflowSocialUploadEvidence = [
- { name: 'old.png', attachmentState: 'input_attached', actionSequence: 1 },
- { name: 'replacement.png', attachmentState: 'input_attached', actionSequence: 3 },
- ];
- // The composer now shows only the replacement: the removed filename is
- // pruned so the dispatch snapshot keeps only active uploads.
+ const upload = (name, actionSequence) => ({
+ name,
+ attachmentState: 'input_attached',
+ actionSequence,
+ });
+ guard.workflowSocialUploadEvidence = [upload('a.png', 1), upload('b.png', 2)];
+
+ // A partial/paginated observation can omit an active filename. Preserve
+ // both rather than treating absence from one page as a removal signal.
assert.equal(
- agent._pruneStaleSocialPublishUploadEvidence(tabId, 'composer shows replacement.png with a Remove button'),
+ agent._pruneStaleSocialPublishUploadEvidence(tabId, 'composer page shows a.png'),
+ 0,
+ AgentClass.name + ': a partial composer observation pruned active provenance',
+ );
+ assert.deepEqual(
+ guard.workflowSocialUploadEvidence.map(item => item.name),
+ ['a.png', 'b.png'],
+ AgentClass.name + ': an omitted active upload was lost after a partial observation',
+ );
+
+ // Upload recency cannot reveal which of two earlier files was removed.
+ // Refuse to assign the last two history entries to the two published nodes.
+ guard.workflowSocialUploadEvidence.push(upload('c.png', 3));
+ const cdnRecord = {
+ attachments: [
+ { type: 'image', src: 'https://pbs.twimg.com/media/XYZ123?format=png' },
+ { type: 'image', src: 'https://pbs.twimg.com/media/XYZ456?format=png' },
+ ],
+ };
+ const ambiguous = agent._workflowSocialRecordWithUploadedAttachmentNames(cdnRecord, {
+ uploadedAttachmentNames: ['a.png', 'b.png', 'c.png'],
+ });
+ assert.equal(ambiguous, cdnRecord,
+ AgentClass.name + ': upload recency was used to guess active composer media');
+ assert.equal(ambiguous.attachments.some(item => item.name), false,
+ AgentClass.name + ': ambiguous upload history supplied a false filename');
+
+ // A complete, untruncated root snapshot does establish that a.png and
+ // c.png remain active, so it can safely prune b.png before dispatch.
+ assert.equal(
+ agent._pruneStaleSocialPublishUploadEvidence(
+ tabId,
+ 'complete composer shows a.png and c.png with Remove buttons',
+ { completeComposerSnapshot: true },
+ ),
1,
- AgentClass.name + ': a removed composer file was retained in upload provenance',
+ AgentClass.name + ': a complete composer snapshot did not prune removed media',
);
assert.deepEqual(
guard.workflowSocialUploadEvidence.map(item => item.name),
- ['replacement.png'],
- AgentClass.name + ': pruning kept the removed file instead of the replacement',
- );
- // When the removal never surfaced as observation text, the join still
- // prefers the most recent upload so the replacement can verify instead of
- // refusing both names and pushing the run toward a duplicate post.
- const staleBinding = { uploadedAttachmentNames: ['old.png', 'replacement.png'] };
- const cdnRecord = { attachments: [{ type: 'image', src: 'https://pbs.twimg.com/media/XYZ123?format=png' }] };
- const joined = agent._workflowSocialRecordWithUploadedAttachmentNames(cdnRecord, staleBinding);
- assert.equal(joined.attachments?.[0]?.name, 'replacement.png',
- AgentClass.name + ': stale provenance did not fall back to the replacement upload');
- assert.equal(
- agent._workflowSocialPublishedAttachmentObserved({ value: 'replacement.png' }, joined),
+ ['a.png', 'c.png'],
+ AgentClass.name + ': complete snapshot retained the wrong active provenance',
+ );
+ const joined = agent._workflowSocialRecordWithUploadedAttachmentNames(cdnRecord, {
+ uploadedAttachmentNames: guard.workflowSocialUploadEvidence.map(item => item.name),
+ });
+ assert.deepEqual(joined.attachments.map(item => item.name), ['a.png', 'c.png'],
+ AgentClass.name + ': reconciled active provenance did not join published media');
+ assert.equal(
+ agent._workflowSocialPublishedAttachmentObserved({ value: 'a.png and c.png' }, joined),
true,
- AgentClass.name + ': the replacement file could not verify after a removal',
+ AgentClass.name + ': active replacement media could not verify after reconciliation',
);
}
});
@@ -92549,6 +92590,7 @@ test('upper-bound attachment qualifiers verify as maximum counts', () => {
assert.equal(capped.isImageMaximum, true, AgentClass.name + ': the capped maximum was not retained');
const image = index => ({ type: 'image', src: `https://pbs.twimg.com/media/image${index}.png` });
const video = index => ({ type: 'video', src: `https://video.twimg.com/media/video${index}.mp4` });
+ const gif = index => ({ type: 'animated_gif', src: `https://video.twimg.com/tweet_video/gif${index}.mp4` });
assert.equal(agent._workflowSocialPublishedAttachmentObserved(
{ value: 'up to two images' }, { attachments: [image(1)] }), true,
AgentClass.name + ': one image should satisfy "up to two images"');
@@ -92570,6 +92612,23 @@ test('upper-bound attachment qualifiers verify as maximum counts', () => {
assert.equal(agent._workflowSocialPublishedAttachmentObserved(
{ value: 'one video and at most two images' }, { attachments: [video(1), image(1), video(2)] }), false,
AgentClass.name + ': the image maximum incorrectly loosened the exact video count');
+ const cappedGifs = agent._parseWorkflowAttachmentRequirement('one video and at most two GIFs');
+ assert.equal(cappedGifs.expectedVideoCount, 1, AgentClass.name + ': the exact ordinary-video count was lost');
+ assert.equal(cappedGifs.expectedGifCount, 2, AgentClass.name + ': the GIF maximum count was lost');
+ assert.equal(cappedGifs.isVideoMaximum, false, AgentClass.name + ': the GIF maximum leaked onto ordinary videos');
+ assert.equal(cappedGifs.isGifMaximum, true, AgentClass.name + ': the GIF upper bound was not retained');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one video and at most two GIFs' }, { attachments: [video(1), gif(1)] }), true,
+ AgentClass.name + ': one video plus one GIF should satisfy the GIF maximum');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one video and at most two GIFs' }, { attachments: [video(1), gif(1), gif(2)] }), true,
+ AgentClass.name + ': one video plus two GIFs should satisfy the GIF maximum');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one video and at most two GIFs' }, { attachments: [video(1), gif(1), gif(2), gif(3)] }), false,
+ AgentClass.name + ': three GIFs exceeded the requested maximum');
+ assert.equal(agent._workflowSocialPublishedAttachmentObserved(
+ { value: 'one video and at most two GIFs' }, { attachments: [video(1), video(2), gif(1)] }), false,
+ AgentClass.name + ': the GIF maximum loosened the exact ordinary-video count');
}
});