Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Bound Clef Flash's input: never send a state it would clip

Claude98718dfcad6e
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.md
index 7e2e16335..fd9bf4a86 100644
--- a/docs/providers-and-models.md
+++ b/docs/providers-and-models.md
@@ -88,6 +88,13 @@ all publishers. The default is `cloudflare/clef-flash`; existing
TypeSafe keys, enabled features and scheduler thresholds stay on their original
provider until changed. Provider keys are stored separately.
+`cloudflare/clef-flash` reads only the first ~2K tokens of a long text state
+(OpenRouter's model page says Workers AI truncates it), so
+`SYSTEM_ONE_TEXT_BUDGETS` in `systemone-judge.js` caps the state it is sent at
+6,000 characters. A longer state is not sent; the check takes the same fallback
+as for an unavailable decision model (failure reason `state_too_long`) instead
+of judging evidence it only partly read.
+
Local [Kev](https://github.com/jaredpalmer/kev) defaults to
`http://127.0.0.1:8009`, model `kev-latest`, using `/v1/systemone`. A key is optional
and loopback requests work offline. Custom remote endpoints require HTTPS.
diff --git a/src/chrome/src/agent/systemone-judge.js b/src/chrome/src/agent/systemone-judge.js
index a8f22cb55..34503c37b 100644
--- a/src/chrome/src/agent/systemone-judge.js
+++ b/src/chrome/src/agent/systemone-judge.js
@@ -70,6 +70,7 @@ export function systemOneFailureReason(error) {
D1_MODEL_LOADING: 'model_loading',
D1_MODEL_NOT_LOADED: 'model_not_loaded',
D1_REQUEST_CANCELLED: 'cancelled',
+ JEV_STATE_EXCEEDS_MODEL_INPUT: 'state_too_long',
};
if (mapped[error?.code]) return mapped[error.code];
const status = Number(error?.status || 0);
@@ -125,6 +126,22 @@ function validateState(state, multimodal = false) {
if (new TextEncoder().encode(serialized).length > (multimodal ? 8 * 1024 * 1024 : 16_000)) throw new Error('TypeSafe System One state is too large.');
}
+// Models that read only the start of a long text state. OpenRouter's Clef Flash
+// page says Workers AI clips it to roughly the first 2K tokens, and completion
+// evidence puts its results last, so a verdict from a clipped read could be
+// confident and wrong. A state past the budget is not sent at all; the caller
+// takes the same fallback as for any unavailable decision model. Images are
+// counted separately by the provider and are not part of this text.
+export const SYSTEM_ONE_TEXT_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 6000 });
+
+function assertStateFitsModel(model, state) {
+ const budget = SYSTEM_ONE_TEXT_BUDGETS[model];
+ if (!budget) return;
+ if (JSON.stringify(state).length > budget) {
+ throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);
+ }
+}
+
function validateQuestions(questions) {
if (!isRecord(questions) || !Object.keys(questions).length) {
throw new Error('TypeSafe System One requires at least one question.');
@@ -288,6 +305,7 @@ export function createSystemOneJudge({
const model = config?.model || SYSTEM_ONE_MODEL;
validateState(state, config?.supportsVision === true);
validateQuestions(questions);
+ assertStateFitsModel(model, state);
const started = now();
const cap = config?.provider === 'webgpu_chat' ? SYSTEM_ONE_WEBGPU_CHAT_TIMEOUT_MS : SYSTEM_ONE_TIMEOUT_MS;
const request = requestSignal(signal, Math.max(1, Math.min(cap, Number(timeoutMs) || cap)));
diff --git a/src/firefox/src/agent/systemone-judge.js b/src/firefox/src/agent/systemone-judge.js
index 07515d878..d71ed1cbe 100644
--- a/src/firefox/src/agent/systemone-judge.js
+++ b/src/firefox/src/agent/systemone-judge.js
@@ -70,6 +70,7 @@ export function systemOneFailureReason(error) {
D1_MODEL_LOADING: 'model_loading',
D1_MODEL_NOT_LOADED: 'model_not_loaded',
D1_REQUEST_CANCELLED: 'cancelled',
+ JEV_STATE_EXCEEDS_MODEL_INPUT: 'state_too_long',
};
if (mapped[error?.code]) return mapped[error.code];
const status = Number(error?.status || 0);
@@ -125,6 +126,22 @@ function validateState(state, multimodal = false) {
if (new TextEncoder().encode(serialized).length > (multimodal ? 8 * 1024 * 1024 : 16_000)) throw new Error('TypeSafe System One state is too large.');
}
+// Models that read only the start of a long text state. OpenRouter's Clef Flash
+// page says Workers AI clips it to roughly the first 2K tokens, and completion
+// evidence puts its results last, so a verdict from a clipped read could be
+// confident and wrong. A state past the budget is not sent at all; the caller
+// takes the same fallback as for any unavailable decision model. Images are
+// counted separately by the provider and are not part of this text.
+export const SYSTEM_ONE_TEXT_BUDGETS = Object.freeze({ 'cloudflare/clef-flash': 6000 });
+
+function assertStateFitsModel(model, state) {
+ const budget = SYSTEM_ONE_TEXT_BUDGETS[model];
+ if (!budget) return;
+ if (JSON.stringify(state).length > budget) {
+ throw systemOneError('JEV_STATE_EXCEEDS_MODEL_INPUT', `The decision state is longer than ${model} reads; the fallback answers instead.`);
+ }
+}
+
function validateQuestions(questions) {
if (!isRecord(questions) || !Object.keys(questions).length) {
throw new Error('TypeSafe System One requires at least one question.');
@@ -287,6 +304,7 @@ export function createSystemOneJudge({
const model = config?.model || SYSTEM_ONE_MODEL;
validateState(state, config?.supportsVision === true);
validateQuestions(questions);
+ assertStateFitsModel(model, state);
const started = now();
const cap = config?.provider === 'webgpu_chat' ? SYSTEM_ONE_WEBGPU_CHAT_TIMEOUT_MS : SYSTEM_ONE_TIMEOUT_MS;
const request = requestSignal(signal, Math.max(1, Math.min(cap, Number(timeoutMs) || cap)));
diff --git a/test/systemone.mjs b/test/systemone.mjs
index 404aa4ff0..b5a05f331 100644
--- a/test/systemone.mjs
+++ b/test/systemone.mjs
@@ -31,6 +31,22 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(mod.systemOneFailureReason(Object.assign(new Error(), { code: 'JEV_INVALID_DISTRIBUTION' })), 'invalid_distribution');
assert.equal(mod.systemOneFailureReason(Object.assign(new Error(), { status: 429 })), 'rate_limit');
});
+ test(`${build}: a model that clips long text never receives a state past its budget`, async () => {
+ const requests = [];
+ const reply = body => ({ ok: true, json: async () => ({ model: body.model, answers: { p: { type: 'noul', noul: .9 } }, usage: { input_tokens: 10, output_tokens: 0 } }) });
+ const judge = mod.createSystemOneJudge({ fetchImpl: async (_url, init) => { const body = JSON.parse(init.body); requests.push(body); return reply(body); } });
+ const one = { p: questions.p };
+ const config = { provider: 'openrouter', model: 'cloudflare/clef-flash', apiKey: 'test', url: 'https://example.test/v1/decisions' };
+ const short = { evidence: 'x'.repeat(1000) };
+ await judge.evaluate({ config, state: short, questions: one });
+ assert.equal(requests.length, 1, 'a state within the budget is sent');
+ const long = { evidence: 'x'.repeat(mod.SYSTEM_ONE_TEXT_BUDGETS['cloudflare/clef-flash'] + 1) };
+ await assert.rejects(judge.evaluate({ config, state: long, questions: one }),
+ error => error.code === 'JEV_STATE_EXCEEDS_MODEL_INPUT' && mod.systemOneFailureReason(error) === 'state_too_long');
+ assert.equal(requests.length, 1, 'a state past the budget is not sent');
+ await judge.evaluate({ config: { ...config, model: 'another/publisher' }, state: long, questions: one });
+ assert.equal(requests.length, 2, 'other models keep the full state');
+ });
test(`${build}: singleton Choice requests fail locally before a service call`, async () => {
let calls = 0;
const judge = mod.createSystemOneJudge({ fetchImpl: async () => { calls++; throw new Error('must not call'); } });