Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Keep Jev skips transient and reconcile superseded verification runs

release-verification4fc52e75e095
diff --git a/docs/architecture.md b/docs/architecture.md
index 651217170..502728dcd 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -1019,4 +1019,10 @@ Scheduler evaluation returns keep/downgrade/skip plus metadata, revalidates the
execution after awaiting the response and never requeues a downgraded action.
The agent supplies the original run cost state and trace ID to the sidecar.
No evidence is written to additional diagnostic logs. Watch baselines alone
-persist a bounded observation to support change comparisons.
+persist a bounded observation to support change comparisons after an actual
+judgment. Skips leave the stored verdict and Jev baseline unchanged and do not
+create verdict trace notes; billable responses still record usage. If the tab's
+run changes during verification, the still-owned scheduler execution enters
+reconciliation instead of remaining running. Post-response cost enforcement
+rejects a judgment that crosses the allowance; an already sent request can
+still incur charges.
diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js
index ea25ad9f5..3f2f217e6 100644
--- a/src/chrome/src/agent/agent.js
+++ b/src/chrome/src/agent/agent.js
@@ -6801,8 +6801,9 @@ export class Agent extends LoopDetector {
if (message) throw this._costAllowanceError(message);
},
onUsage: async metadata => {
- await this._recordCostUsage(SYSTEM_ONE_COST_PROVIDER, metadata.usage, costState);
+ const stopped = await this._recordCostUsage(SYSTEM_ONE_COST_PROVIDER, metadata.usage, costState);
this.recordSystemOneVerdict(tabId, { decision: 'usage', ...metadata }, context);
+ if (stopped) throw this._costAllowanceError(stopped);
},
});
}
diff --git a/src/chrome/src/agent/scheduler.js b/src/chrome/src/agent/scheduler.js
index 11c28eacc..3fd9eaa7c 100644
--- a/src/chrome/src/agent/scheduler.js
+++ b/src/chrome/src/agent/scheduler.js
@@ -1484,7 +1484,7 @@ export class ScheduledJobManager {
async _applySystemOne(job, result, outcome, runMeta) {
const verdict = await this._evaluateSystemOne(job, result, outcome, runMeta);
if (runMeta.signal?.aborted || runMeta.judgeContext?.isCurrent?.() === false) return { stop: true };
- if (verdict.reason === 'disabled') return { stop: false, outcome };
+ if (verdict.decision === 'skip') return { stop: false, outcome };
const current = await this._updateJobIf(job.id, prev => (
['running', 'needs_user_input'].includes(prev.status)
&& (!runMeta.executionId || prev.executionId === runMeta.executionId)
@@ -1656,6 +1656,26 @@ export class ScheduledJobManager {
}
async _complete(job, result, outcome = null, runMeta = {}) {
+ try {
+ return await this._completeCurrentExecution(job, result, outcome, runMeta);
+ } finally {
+ // Discard a stale judgment, but settle the scheduler execution we still
+ // own. Never overwrite cancellation, user input, or a newer execution.
+ if (runMeta.executionId && !runMeta.signal?.aborted && runMeta.judgeContext?.isCurrent?.() === false) {
+ const waiting = await this._updateJobIf(job.id, prev => (
+ prev.status === 'running' && prev.executionId === runMeta.executionId && !runMeta.signal?.aborted
+ ), () => ({
+ status: 'needs_user_input', reconciliationRequired: true,
+ clarificationRequired: true, clarificationAuthorizationRequired: true,
+ lastOutcome: 'partial', nextRunAt: null, pendingClarify: null,
+ lastError: 'The tab started another run while completion was being verified. Reconcile the earlier result before running this task again.',
+ }));
+ if (waiting) this._emit(waiting, 'clarification_required');
+ }
+ }
+ }
+
+ async _completeCurrentExecution(job, result, outcome = null, runMeta = {}) {
if (job.source === 'watch') {
await this._completeWatch(job, result, outcome, runMeta);
return;
diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js
index 845f348b9..0cdfb6627 100644
--- a/src/firefox/src/agent/agent.js
+++ b/src/firefox/src/agent/agent.js
@@ -6585,8 +6585,9 @@ export class Agent extends LoopDetector {
if (message) throw this._costAllowanceError(message);
},
onUsage: async metadata => {
- await this._recordCostUsage(SYSTEM_ONE_COST_PROVIDER, metadata.usage, costState);
+ const stopped = await this._recordCostUsage(SYSTEM_ONE_COST_PROVIDER, metadata.usage, costState);
this.recordSystemOneVerdict(tabId, { decision: 'usage', ...metadata }, context);
+ if (stopped) throw this._costAllowanceError(stopped);
},
});
}
diff --git a/src/firefox/src/agent/scheduler.js b/src/firefox/src/agent/scheduler.js
index c26cbd74e..f4dc06552 100644
--- a/src/firefox/src/agent/scheduler.js
+++ b/src/firefox/src/agent/scheduler.js
@@ -1460,7 +1460,7 @@ export class ScheduledJobManager {
async _applySystemOne(job, result, outcome, runMeta) {
const verdict = await this._evaluateSystemOne(job, result, outcome, runMeta);
if (runMeta.signal?.aborted || runMeta.judgeContext?.isCurrent?.() === false) return { stop: true };
- if (verdict.reason === 'disabled') return { stop: false, outcome };
+ if (verdict.decision === 'skip') return { stop: false, outcome };
const current = await this._updateJobIf(job.id, prev => (
['running', 'needs_user_input'].includes(prev.status)
&& (!runMeta.executionId || prev.executionId === runMeta.executionId)
@@ -1632,6 +1632,26 @@ export class ScheduledJobManager {
}
async _complete(job, result, outcome = null, runMeta = {}) {
+ try {
+ return await this._completeCurrentExecution(job, result, outcome, runMeta);
+ } finally {
+ // Discard a stale judgment, but settle the scheduler execution we still
+ // own. Never overwrite cancellation, user input, or a newer execution.
+ if (runMeta.executionId && !runMeta.signal?.aborted && runMeta.judgeContext?.isCurrent?.() === false) {
+ const waiting = await this._updateJobIf(job.id, prev => (
+ prev.status === 'running' && prev.executionId === runMeta.executionId && !runMeta.signal?.aborted
+ ), () => ({
+ status: 'needs_user_input', reconciliationRequired: true,
+ clarificationRequired: true, clarificationAuthorizationRequired: true,
+ lastOutcome: 'partial', nextRunAt: null, pendingClarify: null,
+ lastError: 'The tab started another run while completion was being verified. Reconcile the earlier result before running this task again.',
+ }));
+ if (waiting) this._emit(waiting, 'clarification_required');
+ }
+ }
+ }
+
+ async _completeCurrentExecution(job, result, outcome = null, runMeta = {}) {
if (job.source === 'watch') {
await this._completeWatch(job, result, outcome, runMeta);
return;
diff --git a/test/systemone.mjs b/test/systemone.mjs
index 35610eb1c..9e907de05 100644
--- a/test/systemone.mjs
+++ b/test/systemone.mjs
@@ -2,6 +2,8 @@ import assert from 'node:assert/strict';
import { test } from 'node:test';
import { makeSchedulerHarness } from './lib/scheduler-harness.mjs';
import './systemone-trace.mjs';
+const area = { get: async () => ({}), set: async () => {} };
+globalThis.chrome = globalThis.browser = { storage: { local: area, session: area }, runtime: { getURL: x => x, sendMessage: async () => ({}) } };
const deferred = () => { let resolve; const promise = new Promise(r => { resolve = r; }); return { promise, resolve }; };
for (const build of ['chrome', 'firefox']) {
const mod = await import(`../src/${build}/src/agent/systemone-judge.js`);
@@ -56,7 +58,7 @@ for (const build of ['chrome', 'firefox']) {
assert.match(state.latest_observation[0].data, /untrusted_page_content/);
assert.ok(JSON.stringify(state).length <= 16000);
});
- async function setup({ mutate = false, noEvidence = false, judge, recurring = false } = {}) {
+ async function setup({ mutate = false, noEvidence = false, judge, recurring = false, outcome = 'success' } = {}) {
let calls = 0;
const h = makeSchedulerHarness(scheduler, {
systemOneEnabled: true, systemOneWatchEnabled: true, systemOneCompletionEnabled: true, systemOneApiKey: 'test',
@@ -68,7 +70,7 @@ for (const build of ['chrome', 'firefox']) {
await options.afterConsequentialTool({ name: 'click_ax', result: { success: true } });
}
if (!noEvidence) emit('tool_result', { name: 'get_accessibility_tree', result: { pageContent: 'Status: not confirmed' } });
- emit('tool_result', { name: 'done', result: { done: true, outcome: 'success', summary: 'Succeeded' } });
+ emit('tool_result', { name: 'done', result: { done: true, outcome, summary: 'Succeeded' } });
return 'Succeeded';
},
});
@@ -80,7 +82,8 @@ for (const build of ['chrome', 'firefox']) {
}
test(`${build}: no evidence skips Jev; read-only downgraded watch keeps polling`, async () => {
const missing = await setup({ noEvidence: true }); await missing.run();
- assert.equal(missing.calls(), 0); assert.equal(missing.jobs()[0].systemOneVerdict.reason, 'no_evidence');
+ assert.equal(missing.calls(), 0); assert.equal(missing.jobs()[0].systemOneVerdict, undefined);
+ assert.equal(missing.jobs()[0].watch.systemOneBaseline, undefined);
const h = await setup(); await h.run();
assert.equal(h.jobs()[0].status, 'pending'); assert.equal(h.jobs()[0].lastOutcome, 'partial');
assert.match(h.jobs()[0].watch.systemOneBaseline[0].data, /Status: not confirmed/);
@@ -97,7 +100,10 @@ for (const build of ['chrome', 'firefox']) {
for (const cancel of [true, false]) {
const entered = deferred(); const response = deferred();
const h = await setup({ judge: { evaluate: async () => { entered.resolve(); return response.promise; } } });
+ let current = true;
+ h.manager.agent.systemOneContext = () => ({ isCurrent: () => current });
const running = h.run(); await entered.promise;
+ current = false;
if (cancel) await h.manager.cancelJob(h.id);
else await h.manager._updateJob(h.id, () => ({ executionId: 'new-execution', lastResult: 'new result' }));
response.resolve({ answers: { condition_met: { noul: .1 }, task_completeness: { score: 0 } } }); await running;
@@ -105,4 +111,54 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(cancel ? job.status : job.lastResult, cancel ? 'cancelled' : 'new result');
}
});
+
+ test(`${build}: every skipped judgment preserves prior verdict and observation without a verdict trace`, async () => {
+ for (const reason of ['disabled', 'no_evidence', 'not_success', 'service_unavailable', 'cost_limit', 'unavailable', 'cancelled']) {
+ const h = await setup();
+ const prior = { decision: 'keep', reason: 'evidence_judgment' };
+ const baseline = [{ tool: 'read_page', data: 'Previously judged observation' }];
+ await h.manager._updateJob(h.id, job => ({ systemOneVerdict: prior, watch: { ...job.watch, systemOneBaseline: baseline } }));
+ h.manager._evaluateSystemOne = async () => ({ decision: 'skip', reason });
+ let notes = 0; h.manager.agent.recordSystemOneVerdict = () => { notes++; };
+ await h.run();
+ assert.deepEqual(h.jobs()[0].systemOneVerdict, prior, reason);
+ assert.deepEqual(h.jobs()[0].watch.systemOneBaseline, baseline, reason);
+ assert.equal(h.jobs()[0].status, 'pending', reason);
+ assert.equal(notes, 0, reason);
+ }
+ });
+
+ test(`${build}: generation loss settles the owned watch or recurring execution without replay`, async () => {
+ for (const recurring of [false, true]) for (const phase of ['judge', 'terminal']) {
+ const entered = deferred(); const response = deferred(); let current = true;
+ const h = await setup({ mutate: true, recurring, judge: { evaluate: async () => { entered.resolve(); return response.promise; } } });
+ h.manager.agent.systemOneContext = () => ({ isCurrent: () => current });
+ h.manager.agent.recordSystemOneVerdict = () => { if (phase === 'terminal') current = false; };
+ const running = h.run(); await entered.promise;
+ if (phase === 'judge') current = false;
+ response.resolve({ answers: { condition_met: { noul: .99 }, task_complete: { noul: .99 }, task_completeness: { score: 2 } } });
+ await running;
+ const job = h.jobs()[0];
+ assert.equal(job.status, 'needs_user_input');
+ assert.equal(job.reconciliationRequired, true);
+ assert.equal(job.completedConsequentialAction.name, 'click_ax');
+ assert.equal(job.nextRunAt, null);
+ if (phase === 'judge') assert.equal(job.systemOneVerdict, undefined);
+ await h.run(); assert.equal(h.jobs()[0].status, 'needs_user_input');
+ }
+ });
+
+ test(`${build}: response that crosses the allowance is charged but cannot supply a judgment`, async () => {
+ const { Agent } = await import(`../src/${build}/src/agent/agent.js`);
+ const agent = new Agent({ getActive: () => ({ name: 'test', config: {} }) });
+ agent._checkCostAllowance = async () => null;
+ let charged = 0; const notes = [];
+ agent._recordCostUsage = async () => { charged++; return 'Cost allowance exceeded'; };
+ agent.recordSystemOneVerdict = (_tab, data) => notes.push(data);
+ const client = mod.createSystemOneJudge({ fetchImpl: async () => ({ ok: true, json: async () => ({ model: mod.SYSTEM_ONE_MODEL, usage: { input_tokens: 1000, output_tokens: 1 }, answers: answers() }) }) });
+ await assert.rejects(agent.evaluateSystemOne(77, client, { apiKey: 'test', state: {}, questions }), error => error.code === 'WB_COST_ALLOWANCE');
+ assert.equal(charged, 1);
+ assert.equal(notes.length, 1);
+ assert.equal(notes[0].decision, 'usage');
+ });
}