Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

fix(agent): support CJK word segmentation and expose recipient clarify purposes

Emre Sokullu5a74e78b208e
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index 2c7660d85..c82fb3510 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -17868,20 +17868,51 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
};
}
- _isRecipientClarification(context, pendingType = 'message_recipient') {
+ _isRecipientClarification(context, pendingType = 'message_recipient', observedCandidates = []) {
if (!context || typeof context !== 'object') return false;
const purpose = String(context.purpose || '').trim();
if (pendingType === 'recipient_change') {
if (purpose === 'recipient_change') return true;
- const text = [context.question, context.reason].filter(Boolean).join(' ').toLowerCase();
- return /(change|switch|different|instead|update|replace).*(recipient|conversation|contact|send to|person|address)/i.test(text)
- || /(recipient|conversation|contact).*(change|switch|different)/i.test(text)
- || /(send|deliver).*(to .* instead|to someone else)/i.test(text)
- || /(send|reply).*(instead of)/i.test(text);
+ if (purpose === 'research_escalation') return false;
+ const text = [
+ context.question,
+ context.reason,
+ ...(Array.isArray(context.options) ? context.options : []),
+ ].filter(Boolean).join(' ').toLowerCase();
+ const changeWord = /(change|switch|different|instead|update|replace|another|new|更改|更换|切换|替换|换成)/i.test(text);
+ const targetWord = /(recipient|conversation|contact|send to|person|address|target|someone else|收件人|联系人|发送)/i.test(text);
+ return changeWord && targetWord;
}
if (purpose === 'message_recipient' || purpose === 'recipient_change') return true;
- const text = [context.question, context.reason].filter(Boolean).join(' ').toLowerCase();
- return /(recipient|who (is|to|should|are)|send (the |this )?(message|email|draft) to|message (to|for)|deliver to|email to|draft to|authorize.*conversation)/i.test(text);
+ if (purpose === 'research_escalation') return false;
+
+ // Check if the clarify question, reason, or options explicitly mention any observed candidate identity
+ if (Array.isArray(observedCandidates) && observedCandidates.length > 0) {
+ const allCandidateText = [
+ context.question,
+ context.reason,
+ ...(Array.isArray(context.options) ? context.options : []),
+ ].filter(Boolean).join(' ');
+ for (const candidate of observedCandidates) {
+ const id = typeof candidate === 'string' ? candidate : (candidate?.identity ?? candidate?.recipient);
+ if (id && answerNamesIdentity(allCandidateText, id)) {
+ return true;
+ }
+ }
+ }
+
+ const text = [
+ context.question,
+ context.reason,
+ ...(Array.isArray(context.options) ? context.options : []),
+ ].filter(Boolean).join(' ').toLowerCase();
+
+ if (/recipient|收件人/i.test(text)) return true;
+
+ const hasRecipientTerm = /(contact|person|address|who|whom|which contact|which person|which recipient|which user|which address|to whom|for whom|someone|联系人|哪位|谁)/i.test(text);
+ const hasMessagingTerm = /(send|receive|deliver|message|email|mail|draft|reply|chat|conversation|发送|邮件|信息|消息|寄|回复)/i.test(text);
+
+ return hasRecipientTerm && hasMessagingTerm;
}
/**
@@ -17922,10 +17953,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!pending || !pendingType || observed.length === 0) return false;
if (!answer || source === 'timeout' || source === 'auto') return false;
if (pendingType === 'recipient_change') {
- if (!clarifyContext || !this._isRecipientClarification(clarifyContext, 'recipient_change')) {
+ if (!clarifyContext || !this._isRecipientClarification(clarifyContext, 'recipient_change', observed)) {
return false;
}
- } else if (clarifyContext && !this._isRecipientClarification(clarifyContext, 'message_recipient')) {
+ } else if (clarifyContext && !this._isRecipientClarification(clarifyContext, 'message_recipient', observed)) {
return false;
}
const normalizedAnswer = normalizeRecipientIdentity(answer);
diff --git a/src/chrome/src/agent/message-recipient-guard.js b/src/chrome/src/agent/message-recipient-guard.js
index d72ae26a6..1bd051726 100644
--- a/src/chrome/src/agent/message-recipient-guard.js
+++ b/src/chrome/src/agent/message-recipient-guard.js
@@ -58,9 +58,40 @@ export function recipientMatchesObservedIdentity(recipient, observedIdentity) {
const IDENTITY_WORD_CHAR = /[\p{L}\p{N}]/u;
+let cachedWordSegmenter = null;
+
+function getWordSegmenter() {
+ if (cachedWordSegmenter) return cachedWordSegmenter;
+ if (typeof Intl !== 'undefined' && typeof Intl.Segmenter === 'function') {
+ try {
+ cachedWordSegmenter = new Intl.Segmenter(undefined, { granularity: 'word' });
+ } catch {
+ cachedWordSegmenter = null;
+ }
+ }
+ return cachedWordSegmenter;
+}
+
+export function getWordBoundaries(text) {
+ const boundaries = new Set([0, text.length]);
+ const segmenter = getWordSegmenter();
+ if (segmenter) {
+ try {
+ for (const seg of segmenter.segment(text)) {
+ boundaries.add(seg.index);
+ boundaries.add(seg.index + seg.segment.length);
+ }
+ } catch {
+ // fallback
+ }
+ }
+ return boundaries;
+}
+
/**
* Return all start/end spans where normalizedIdentity occurs in normalizedAnswer
- * at word boundaries, or as an exact full-string match for short identities (< 3 chars).
+ * at word boundaries (including script-aware segmentation for unsegmented scripts),
+ * or as an exact full-string match for short identities (< 3 chars).
*/
export function findCandidateAnswerSpans(normalizedAnswer, normalizedIdentity) {
const answer = normalizeRecipientIdentity(normalizedAnswer);
@@ -69,12 +100,17 @@ export function findCandidateAnswerSpans(normalizedAnswer, normalizedIdentity) {
if (identity.length < 3) {
return answer === identity ? [{ start: 0, end: answer.length }] : [];
}
+ const boundaries = getWordBoundaries(answer);
const spans = [];
for (let at = answer.indexOf(identity); at >= 0; at = answer.indexOf(identity, at + 1)) {
+ const end = at + identity.length;
const before = at > 0 ? answer[at - 1] : '';
- const after = answer[at + identity.length] || '';
- if (!IDENTITY_WORD_CHAR.test(before) && !IDENTITY_WORD_CHAR.test(after)) {
- spans.push({ start: at, end: at + identity.length });
+ const after = answer[end] || '';
+ const charBoundary = (!before || !IDENTITY_WORD_CHAR.test(before))
+ && (!after || !IDENTITY_WORD_CHAR.test(after));
+ const segmentBoundary = boundaries.has(at) && boundaries.has(end);
+ if (charBoundary || segmentBoundary) {
+ spans.push({ start: at, end });
}
}
return spans;
diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.js
index e86fe8e57..c9dbbbcd6 100644
--- a/src/chrome/src/agent/tools.js
+++ b/src/chrome/src/agent/tools.js
@@ -810,8 +810,8 @@ export const AGENT_TOOLS = [
},
purpose: {
type: 'string',
- enum: ['research_escalation'],
- description: 'Set to research_escalation only when asking to delegate a read-only research subtask.',
+ enum: ['research_escalation', 'message_recipient', 'recipient_change'],
+ description: 'Optional purpose of the clarification. Use message_recipient to ask the user to confirm or specify the message/email recipient, recipient_change to switch an already-authorized recipient, or research_escalation to delegate a read-only research subtask.',
},
research_request: {
type: 'string',
@@ -1595,7 +1595,11 @@ function askResearchConsentTool(tool) {
...properties.reason,
description: 'Optional one-sentence explanation of why this unusually complex read-only research subtask would benefit from ChatGPT.',
},
- purpose: properties.purpose,
+ purpose: {
+ ...properties.purpose,
+ enum: ['research_escalation'],
+ description: 'Must be set to research_escalation.',
+ },
research_request: properties.research_request,
approve_option: properties.approve_option,
},
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index e5d694e52..df9971a55 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -15621,20 +15621,51 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
};
}
- _isRecipientClarification(context, pendingType = 'message_recipient') {
+ _isRecipientClarification(context, pendingType = 'message_recipient', observedCandidates = []) {
if (!context || typeof context !== 'object') return false;
const purpose = String(context.purpose || '').trim();
if (pendingType === 'recipient_change') {
if (purpose === 'recipient_change') return true;
- const text = [context.question, context.reason].filter(Boolean).join(' ').toLowerCase();
- return /(change|switch|different|instead|update|replace).*(recipient|conversation|contact|send to|person|address)/i.test(text)
- || /(recipient|conversation|contact).*(change|switch|different)/i.test(text)
- || /(send|deliver).*(to .* instead|to someone else)/i.test(text)
- || /(send|reply).*(instead of)/i.test(text);
+ if (purpose === 'research_escalation') return false;
+ const text = [
+ context.question,
+ context.reason,
+ ...(Array.isArray(context.options) ? context.options : []),
+ ].filter(Boolean).join(' ').toLowerCase();
+ const changeWord = /(change|switch|different|instead|update|replace|another|new|更改|更换|切换|替换|换成)/i.test(text);
+ const targetWord = /(recipient|conversation|contact|send to|person|address|target|someone else|收件人|联系人|发送)/i.test(text);
+ return changeWord && targetWord;
}
if (purpose === 'message_recipient' || purpose === 'recipient_change') return true;
- const text = [context.question, context.reason].filter(Boolean).join(' ').toLowerCase();
- return /(recipient|who (is|to|should|are)|send (the |this )?(message|email|draft) to|message (to|for)|deliver to|email to|draft to|authorize.*conversation)/i.test(text);
+ if (purpose === 'research_escalation') return false;
+
+ // Check if the clarify question, reason, or options explicitly mention any observed candidate identity
+ if (Array.isArray(observedCandidates) && observedCandidates.length > 0) {
+ const allCandidateText = [
+ context.question,
+ context.reason,
+ ...(Array.isArray(context.options) ? context.options : []),
+ ].filter(Boolean).join(' ');
+ for (const candidate of observedCandidates) {
+ const id = typeof candidate === 'string' ? candidate : (candidate?.identity ?? candidate?.recipient);
+ if (id && answerNamesIdentity(allCandidateText, id)) {
+ return true;
+ }
+ }
+ }
+
+ const text = [
+ context.question,
+ context.reason,
+ ...(Array.isArray(context.options) ? context.options : []),
+ ].filter(Boolean).join(' ').toLowerCase();
+
+ if (/recipient|收件人/i.test(text)) return true;
+
+ const hasRecipientTerm = /(contact|person|address|who|whom|which contact|which person|which recipient|which user|which address|to whom|for whom|someone|联系人|哪位|谁)/i.test(text);
+ const hasMessagingTerm = /(send|receive|deliver|message|email|mail|draft|reply|chat|conversation|发送|邮件|信息|消息|寄|回复)/i.test(text);
+
+ return hasRecipientTerm && hasMessagingTerm;
}
/**
@@ -15675,10 +15706,10 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
if (!pending || !pendingType || observed.length === 0) return false;
if (!answer || source === 'timeout' || source === 'auto') return false;
if (pendingType === 'recipient_change') {
- if (!clarifyContext || !this._isRecipientClarification(clarifyContext, 'recipient_change')) {
+ if (!clarifyContext || !this._isRecipientClarification(clarifyContext, 'recipient_change', observed)) {
return false;
}
- } else if (clarifyContext && !this._isRecipientClarification(clarifyContext, 'message_recipient')) {
+ } else if (clarifyContext && !this._isRecipientClarification(clarifyContext, 'message_recipient', observed)) {
return false;
}
const normalizedAnswer = normalizeRecipientIdentity(answer);
diff --git a/src/firefox/src/agent/message-recipient-guard.js b/src/firefox/src/agent/message-recipient-guard.js
index d72ae26a6..1bd051726 100644
--- a/src/firefox/src/agent/message-recipient-guard.js
+++ b/src/firefox/src/agent/message-recipient-guard.js
@@ -58,9 +58,40 @@ export function recipientMatchesObservedIdentity(recipient, observedIdentity) {
const IDENTITY_WORD_CHAR = /[\p{L}\p{N}]/u;
+let cachedWordSegmenter = null;
+
+function getWordSegmenter() {
+ if (cachedWordSegmenter) return cachedWordSegmenter;
+ if (typeof Intl !== 'undefined' && typeof Intl.Segmenter === 'function') {
+ try {
+ cachedWordSegmenter = new Intl.Segmenter(undefined, { granularity: 'word' });
+ } catch {
+ cachedWordSegmenter = null;
+ }
+ }
+ return cachedWordSegmenter;
+}
+
+export function getWordBoundaries(text) {
+ const boundaries = new Set([0, text.length]);
+ const segmenter = getWordSegmenter();
+ if (segmenter) {
+ try {
+ for (const seg of segmenter.segment(text)) {
+ boundaries.add(seg.index);
+ boundaries.add(seg.index + seg.segment.length);
+ }
+ } catch {
+ // fallback
+ }
+ }
+ return boundaries;
+}
+
/**
* Return all start/end spans where normalizedIdentity occurs in normalizedAnswer
- * at word boundaries, or as an exact full-string match for short identities (< 3 chars).
+ * at word boundaries (including script-aware segmentation for unsegmented scripts),
+ * or as an exact full-string match for short identities (< 3 chars).
*/
export function findCandidateAnswerSpans(normalizedAnswer, normalizedIdentity) {
const answer = normalizeRecipientIdentity(normalizedAnswer);
@@ -69,12 +100,17 @@ export function findCandidateAnswerSpans(normalizedAnswer, normalizedIdentity) {
if (identity.length < 3) {
return answer === identity ? [{ start: 0, end: answer.length }] : [];
}
+ const boundaries = getWordBoundaries(answer);
const spans = [];
for (let at = answer.indexOf(identity); at >= 0; at = answer.indexOf(identity, at + 1)) {
+ const end = at + identity.length;
const before = at > 0 ? answer[at - 1] : '';
- const after = answer[at + identity.length] || '';
- if (!IDENTITY_WORD_CHAR.test(before) && !IDENTITY_WORD_CHAR.test(after)) {
- spans.push({ start: at, end: at + identity.length });
+ const after = answer[end] || '';
+ const charBoundary = (!before || !IDENTITY_WORD_CHAR.test(before))
+ && (!after || !IDENTITY_WORD_CHAR.test(after));
+ const segmentBoundary = boundaries.has(at) && boundaries.has(end);
+ if (charBoundary || segmentBoundary) {
+ spans.push({ start: at, end });
}
}
return spans;
diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.js
index 4aa4b4ea9..26289ce3c 100644
--- a/src/firefox/src/agent/tools.js
+++ b/src/firefox/src/agent/tools.js
@@ -672,8 +672,8 @@ export const AGENT_TOOLS = [
},
purpose: {
type: 'string',
- enum: ['research_escalation'],
- description: 'Set to research_escalation only when asking to delegate a read-only research subtask.',
+ enum: ['research_escalation', 'message_recipient', 'recipient_change'],
+ description: 'Optional purpose of the clarification. Use message_recipient to ask the user to confirm or specify the message/email recipient, recipient_change to switch an already-authorized recipient, or research_escalation to delegate a read-only research subtask.',
},
research_request: {
type: 'string',
@@ -1442,7 +1442,11 @@ function askResearchConsentTool(tool) {
...properties.reason,
description: 'Optional one-sentence explanation of why this unusually complex read-only research subtask would benefit from ChatGPT.',
},
- purpose: properties.purpose,
+ purpose: {
+ ...properties.purpose,
+ enum: ['research_escalation'],
+ description: 'Must be set to research_escalation.',
+ },
research_request: properties.research_request,
approve_option: properties.approve_option,
},
diff --git a/test/run.js b/test/run.js
index 132f2e557..ead5a0401 100644
--- a/test/run.js
+++ b/test/run.js
@@ -2715,6 +2715,11 @@ test('research escalation is opt-in, tier-complete when enabled, and absent othe
for (const field of researchOnlyClarifyFields) {
assert.ok(actClarify.function.parameters.properties[field], `${label}: enabled Act clarification lost ${field}`);
}
+ assert.deepEqual(
+ actClarify.function.parameters.properties.purpose.enum,
+ ['research_escalation', 'message_recipient', 'recipient_change'],
+ `${label}: clarify purpose did not expose recipient purposes`,
+ );
assert.deepEqual(actClarify.function.parameters.required, ['question']);
assert.doesNotMatch(actClarify.function.description, /not a generic clarification tool/i);
@@ -4646,6 +4651,8 @@ test('direct-message recipient guard uses structured intent and exact active ide
assert.equal(helper.answerNamesIdentity('Lisa', 'Li'), false);
assert.equal(helper.answerNamesIdentity('王', '王'), true);
assert.equal(helper.answerNamesIdentity('王小明', '王'), false);
+ assert.equal(helper.answerNamesIdentity('请发送给王小明', '王小明'), true, 'natural Chinese sentence must match CJK recipient');
+ assert.equal(helper.answerNamesIdentity('请发送给王小明', '王'), false, 'sub-fragment of CJK name must not match short identity');
assert.equal(helper.answerNamesAllObservedRecipients(
'Send to Ann Smith',
[{ identity: 'Ann' }, { identity: 'Ann Smith' }],
@@ -5248,6 +5255,26 @@ test('direct-message recipient guard uses structured intent and exact active ide
`${label}: valid recipient clarification context was rejected`,
);
+ // Natural recipient question without explicit purpose (e.g. "Which contact should receive this message?").
+ agent._planExecutionGuards.set(tabId, clarifyGuardState([{ identity: '[email protected]', role: 'to' }]));
+ assert.equal(
+ agent._bindClarifiedMessageRecipient(tabId, '[email protected]', 'user', {
+ question: 'Which contact should receive this message?',
+ }),
+ true,
+ `${label}: natural recipient question ("Which contact should receive this message?") was rejected`,
+ );
+
+ // Natural-language Chinese answer for CJK recipient.
+ agent._planExecutionGuards.set(tabId, clarifyGuardState([{ identity: '王小明', role: 'to' }]));
+ assert.equal(
+ agent._bindClarifiedMessageRecipient(tabId, '请发送给王小明', 'user', {
+ question: '请确认要发送给哪位联系人?',
+ }),
+ true,
+ `${label}: natural Chinese clarification question/answer was rejected`,
+ );
+
// Recipient-change clarification check: when a target was already set, an explicit
// recipient_change context is required.
const recipientChangeGuardState = (candidates) => ({