Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Classify complete long follow-ups without losing task context

Emre Sokullu3de7b143d3ba
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index b5f56590b..306282f7f 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -11,7 +11,7 @@ import { completionResultSnapshot, completionResultPixelRect } from './completio
import { captureSemanticNavigationDocument, semanticCommitNavigationRequest, recordSemanticCommitNavigation, semanticCommitIsCurrent, canRecordSemanticCommitDispatch } from './semantic-commit-navigation.js';
import { installPageFeedback, beforePageAgentDispatch, capturePageAgentInputCompletion, hasPageAgentDispatchOwner, pageFeedbackPendingResult } from './page-feedback.js';
import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';
-import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText } from './systemone-evidence.js';
+import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText, systemOneStateBytes } from './systemone-evidence.js';
import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
import { SEMANTIC_BATCH_LIMITS, SEMANTIC_DECISION_VERSION, acceptSemanticAnswer, acceptSemanticBatchAnswers, boundedSemanticData, controlEffectCommits, semanticBatchChunks, semanticBatchDecisionMessages, semanticDecisionMessages, semanticQuestion, semanticSystemOneBatchQuestions, semanticSystemOneQuestion, WORKFLOW_STATES } from './semantic-decisions.js';
import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationContract, resolvePublicationText, publicationProgress, exactPublicationText, publicationMediaMatches, publicationContractMessages, publicationAuditMessages, parsePublicationJson, publicationAuditValidation } from './social-publish-contract.js';
@@ -7863,7 +7863,9 @@ export class Agent extends LoopDetector {
*/
async _semanticDecision(tabId, kind, { request = '', page = {} } = {}, { provider = this._activeProvider(tabId), signal = null, mainModel = true } = {}) {
const spec = semanticQuestion(kind);
- const trusted = String(request ?? '').slice(0, 6000);
+ // Relationship decisions must read the entire latest user message. A
+ // prefix-only verdict can mistake a correction or a new task at its end.
+ const trusted = kind === 'user_turn_kind' ? String(request ?? '') : String(request ?? '').slice(0, 6000);
const data = boundedSemanticData(page);
const cacheKey = this._sha256TextSync(JSON.stringify([SEMANTIC_DECISION_VERSION, kind, trusted, data]));
this._semanticDecisionCache ??= new Map();
@@ -7878,10 +7880,14 @@ export class Agent extends LoopDetector {
const context = this.systemOneContext(tabId);
try {
const settings = await this._jevSettings(tabId, provider);
- if (settings && !this._checkAbort(tabId)) {
+ const decisionText = redactSystemOneText(JSON.stringify({ request: trusted, page: data }));
+ // Keep the existing bounded fast route, but use the full-text main
+ // model directly when a relationship question cannot fit in it.
+ const decisionFits = kind !== 'user_turn_kind' || systemOneStateBytes(decisionText) <= 12000;
+ if (settings && decisionFits && !this._checkAbort(tabId)) {
const verdict = await this.evaluateSystemOne(tabId, createSystemOneJudge({ timeoutMs: 2500, maxRetries: 0 }), {
config: settings.decisionConfig, apiKey: settings.typesafeApiKey, signal: runSignal,
- state: { context: wrapSystemOneData(boundedSystemOneText(redactSystemOneText(JSON.stringify({ request: trusted, page: data })), 12000)) },
+ state: { context: wrapSystemOneData(boundedSystemOneText(decisionText, 12000)) },
questions: { [kind]: semanticSystemOneQuestion(kind) },
}, context);
const answer = verdict.answers?.[kind];
@@ -7889,6 +7895,8 @@ export class Agent extends LoopDetector {
this.recordSystemOneVerdict(tabId, { decision: choice ? 'semantic' : 'fallback', reason: choice ? kind : `${kind}_low_confidence`,
model: verdict.model }, context);
if (choice) return remember({ choice, evidence: '', source: 'decision_model' });
+ } else if (settings && !decisionFits) {
+ this.recordSystemOneVerdict(tabId, { decision: 'fallback', reason: `${kind}_context_budget` }, context);
}
} catch (error) {
if (this._isCostAllowanceError(error) || this._checkAbort(tabId)) throw error;
@@ -33479,7 +33487,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
async _userTurnKind(tabId, messages, text, mode, runOptions = {}, sourceBound = false) {
if (runOptions?.trustedContinuation === true) return 'continue';
const value = typeof text === 'string' ? text.trim() : '';
- if (sourceBound || !this._isActionMode(mode) || !value || value.length > 6000
+ if (sourceBound || !this._isActionMode(mode) || !value
|| this._isScheduledResumeTurn(value) || this._activeTaskBinding(messages || []).index < 0) return 'request';
const reference = this._userTurnReference(messages);
const verdict = await this._semanticDecision(tabId, 'user_turn_kind', { request: value, page: reference });
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 2c2dca314..699b59edf 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -9,7 +9,7 @@ import { completionResultSnapshot, completionResultPixelRect } from './completio
import { captureSemanticNavigationDocument, semanticCommitNavigationRequest, recordSemanticCommitNavigation, semanticCommitIsCurrent, canRecordSemanticCommitDispatch } from './semantic-commit-navigation.js';
import { installPageFeedback, beforePageAgentDispatch, hasPageAgentDispatchOwner, pageFeedbackPendingResult } from './page-feedback.js';
import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';
-import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText } from './systemone-evidence.js';
+import { redactSystemOneText, wrapSystemOneData, boundedSystemOneText, systemOneStateBytes } from './systemone-evidence.js';
import { createSystemOneJudge, isSystemOneResponseContractError, systemOneFailureReason, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
import { SEMANTIC_BATCH_LIMITS, SEMANTIC_DECISION_VERSION, acceptSemanticAnswer, acceptSemanticBatchAnswers, boundedSemanticData, controlEffectCommits, semanticBatchChunks, semanticBatchDecisionMessages, semanticDecisionMessages, semanticQuestion, semanticSystemOneBatchQuestions, semanticSystemOneQuestion, WORKFLOW_STATES } from './semantic-decisions.js';
import { firefoxBidi } from '../bidi/client.js';
@@ -7688,7 +7688,9 @@ export class Agent extends LoopDetector {
*/
async _semanticDecision(tabId, kind, { request = '', page = {} } = {}, { provider = this._activeProvider(tabId), signal = null, mainModel = true } = {}) {
const spec = semanticQuestion(kind);
- const trusted = String(request ?? '').slice(0, 6000);
+ // Relationship decisions must read the entire latest user message. A
+ // prefix-only verdict can mistake a correction or a new task at its end.
+ const trusted = kind === 'user_turn_kind' ? String(request ?? '') : String(request ?? '').slice(0, 6000);
const data = boundedSemanticData(page);
const cacheKey = this._sha256TextSync(JSON.stringify([SEMANTIC_DECISION_VERSION, kind, trusted, data]));
this._semanticDecisionCache ??= new Map();
@@ -7703,10 +7705,14 @@ export class Agent extends LoopDetector {
const context = this.systemOneContext(tabId);
try {
const settings = await this._jevSettings(tabId, provider);
- if (settings && !this._checkAbort(tabId)) {
+ const decisionText = redactSystemOneText(JSON.stringify({ request: trusted, page: data }));
+ // Keep the existing bounded fast route, but use the full-text main
+ // model directly when a relationship question cannot fit in it.
+ const decisionFits = kind !== 'user_turn_kind' || systemOneStateBytes(decisionText) <= 12000;
+ if (settings && decisionFits && !this._checkAbort(tabId)) {
const verdict = await this.evaluateSystemOne(tabId, createSystemOneJudge({ timeoutMs: 2500, maxRetries: 0 }), {
config: settings.decisionConfig, apiKey: settings.typesafeApiKey, signal: runSignal,
- state: { context: wrapSystemOneData(boundedSystemOneText(redactSystemOneText(JSON.stringify({ request: trusted, page: data })), 12000)) },
+ state: { context: wrapSystemOneData(boundedSystemOneText(decisionText, 12000)) },
questions: { [kind]: semanticSystemOneQuestion(kind) },
}, context);
const answer = verdict.answers?.[kind];
@@ -7714,6 +7720,8 @@ export class Agent extends LoopDetector {
this.recordSystemOneVerdict(tabId, { decision: choice ? 'semantic' : 'fallback', reason: choice ? kind : `${kind}_low_confidence`,
model: verdict.model }, context);
if (choice) return remember({ choice, evidence: '', source: 'decision_model' });
+ } else if (settings && !decisionFits) {
+ this.recordSystemOneVerdict(tabId, { decision: 'fallback', reason: `${kind}_context_budget` }, context);
}
} catch (error) {
if (this._isCostAllowanceError(error) || this._checkAbort(tabId)) throw error;
@@ -30758,7 +30766,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d
async _userTurnKind(tabId, messages, text, mode, runOptions = {}, sourceBound = false) {
if (runOptions?.trustedContinuation === true) return 'continue';
const value = typeof text === 'string' ? text.trim() : '';
- if (sourceBound || !this._isActionMode(mode) || !value || value.length > 6000
+ if (sourceBound || !this._isActionMode(mode) || !value
|| this._isScheduledResumeTurn(value) || this._activeTaskBinding(messages || []).index < 0) return 'request';
const reference = this._userTurnReference(messages);
const verdict = await this._semanticDecision(tabId, 'user_turn_kind', { request: value, page: reference });
diff --git a/test/follow-up-task-context.mjs b/test/follow-up-task-context.mjs
index 62aa7cdaa..e6866e1c5 100644
--- a/test/follow-up-task-context.mjs
+++ b/test/follow-up-task-context.mjs
@@ -304,6 +304,64 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(agent._progressTaskAnchorText(tab), root);
});
+ for (const [length, character, fast] of [[6001, 'x', true], [14000, 'x', false], [4500, '界', false], [6001, '"', false]]) {
+ test(`${build}: ${length}-character ${character} corrections retain the original task after full-text classification`, async () => {
+ const { agent, messages, provider } = fixture();
+ agent._recordTaskClarification(tab, 'Which bot?', 'Chat with @emrebot', 'option');
+ const earlier = 'Use a short greeting, then keep talking.';
+ messages.push(user(earlier, 'update'));
+ const tail = ' END-OF-LONG-CORRECTION: keep this same task.';
+ const correction = character.repeat(length - tail.length) + tail;
+ const calls = installModelStub(agent, { provider, decide: (_kind, { request }) => {
+ assert.equal(request, correction, 'Fallback must read the entire message, including the final instruction.');
+ return { choice: 'update', evidence: tail.trim() };
+ } });
+ let decisions = 0;
+ agent._jevSettings = async () => ({ decisionConfig: { enabled: true, local: true } });
+ agent.evaluateSystemOne = async (_tab, _judge, input) => {
+ decisions++;
+ assert.ok(JSON.stringify(input.state).includes(tail), 'Fast decision must also see the final instruction.');
+ return { model: 'owned-decider', answers: { user_turn_kind: { type: 'choice', choice: 'update', confidence: .97,
+ probabilities: { continue: .01, update: .97, acknowledge: .01, request: .01 } } } };
+ };
+ const turn = user(correction);
+ await agent._markUserTurnKind(tab, messages, turn, correction, 'act');
+ assert.equal(turn.webbrainTurnKind, 'update');
+ assert.equal(decisions, fast ? 1 : 0, 'Oversized decision payloads go directly to the full-text LLM fallback.');
+ assert.equal(calls.length, fast ? 0 : 1);
+ messages.push(turn);
+ assert.equal(agent._activeTaskBinding(messages).taskIndex, 1);
+ const task = agent._plannerAuthenticTaskText(messages);
+ for (const text of [root, earlier, 'Chat with @emrebot']) assert.ok(task.includes(text));
+ assert.equal(JSON.parse(task.slice(task.indexOf('{'))).latest_request, correction);
+ const checked = JSON.parse(buildPlannerAlignmentMessages(task, chatPlan(), { taskBinding: 'long-related-task' }).at(-1).content);
+ assert.equal(checked.current_request, task);
+ });
+ }
+
+ test(`${build}: long separate requests and cache entries keep their distinguishing suffix`, async () => {
+ const { agent, messages, provider } = fixture();
+ const prefix = 'x'.repeat(14000);
+ const suffixes = [' Keep working on this same dialog.', ' Instead start a separate task: summarize the article.'];
+ let decisions = 0;
+ agent._jevSettings = async () => ({ decisionConfig: { enabled: true, local: true } });
+ agent.evaluateSystemOne = async () => { decisions++; throw new Error('A truncated decision is not permitted'); };
+ const calls = installModelStub(agent, { provider, decide: (_kind, { request }) => {
+ assert.ok(suffixes.some(suffix => request.endsWith(suffix)));
+ return { choice: request.endsWith(suffixes[0]) ? 'update' : 'request', evidence: request.slice(prefix.length).trim() };
+ } });
+ for (const [index, suffix] of suffixes.entries()) {
+ const text = prefix + suffix, turn = user(text);
+ await agent._markUserTurnKind(tab, messages, turn, text, 'act');
+ assert.equal(turn.webbrainTurnKind, index === 0 ? 'update' : 'request');
+ const view = [...messages, turn];
+ assert.equal(agent._activeTaskBinding(view).taskIndex, index === 0 ? 1 : messages.length);
+ if (index === 1) assert.equal(agent._plannerAuthenticTaskText(view), text);
+ }
+ assert.equal(calls.length, 2, 'Messages sharing a long prefix must not reuse the other suffix verdict.');
+ assert.equal(decisions, 0);
+ });
+
test(`${build}: legacy turns and classification failure preserve the entire latest request`, async () => {
const { agent, messages } = fixture();
installModelStub(agent, { decide: () => null });
diff --git a/test/user-turn-kind.mjs b/test/user-turn-kind.mjs
index 827011f53..7edcb922b 100644
--- a/test/user-turn-kind.mjs
+++ b/test/user-turn-kind.mjs
@@ -85,7 +85,6 @@ for (const build of ['chrome', 'firefox']) {
assert.equal((await f.mark('Please continue from where you left off.', 'act', { trustedContinuation: true })).webbrainTurnKind, 'continue');
assert.equal((await f.mark('devam et', 'ask')).webbrainTurnKind, 'request', 'Ask turns are requests');
assert.equal((await f.mark('devam et', 'act', {}, { sourceBound: true })).webbrainTurnKind, 'request', 'Shortcuts are requests');
- assert.equal((await f.mark('x'.repeat(6001))).webbrainTurnKind, 'request', 'Long turns are requests');
assert.equal((await f.mark('[Scheduled resume] check the queue')).webbrainTurnKind, 'request', 'Scheduled resumes keep their own rule');
f.agent.conversations.set(f.tab, [{ role: 'system', content: 'system' }]);
assert.equal((await f.mark('devam et')).webbrainTurnKind, 'request', 'A first turn has nothing to continue');