AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(field-report): enforce the raw request byte limit (#2845)
Qiu50ea0bb5859e
diff --git a/.github/scripts/functions-field-report-test.mjs b/.github/scripts/functions-field-report-test.mjsnew file mode 100644index 000000000..2e7174c9e--- /dev/null+++ b/.github/scripts/functions-field-report-test.mjs@@ -0,0 +1,175 @@+// Plain-Node tests for the field-report HTTP fallback. No network, deployment,+// or XERJ process: only R2 is replaced with an in-memory stand-in.+// node .github/scripts/functions-field-report-test.mjs+import assert from 'node:assert/strict';+import { readFileSync } from 'node:fs';+import { test } from 'node:test';++const source = readFileSync(new URL('../../functions/api/field-report.js', import.meta.url), 'utf8');+const { onRequest } = await import(`data:text/javascript;base64,${Buffer.from(source).toString('base64')}`);+const LIMIT = 16 * 1024;+const encoder = new TextEncoder();+const valid = { agent: 'test-agent', used_for: 'local validation', verdict: 'works' };++function fakeR2() {+ const store = new Map();+ const reads = [];+ const writes = [];+ return {+ store, reads, writes,+ async get(key) {+ reads.push(key);+ return store.has(key) ? { text: async () => store.get(key) } : null;+ },+ async put(key, value) {+ writes.push(key);+ store.set(key, value);+ },+ };+}++function request(body, headers = {}) {+ return new Request('https://xerj.example.invalid/api/field-report', {+ method: 'POST', headers: { 'content-type': 'application/json', ...headers }, body,+ });+}++function paddedBody(bytes, { multibyte = false, bom = false } = {}) {+ const raw = JSON.stringify({ ...valid, notes: (multibyte ? '界' : 'x').repeat(1000) });+ const prefix = bom ? new Uint8Array([0xef, 0xbb, 0xbf]) : new Uint8Array();+ const json = encoder.encode(raw);+ assert.ok(bytes >= prefix.byteLength + json.byteLength);+ const body = new Uint8Array(bytes).fill(0x20);+ body.set(prefix);+ body.set(json, prefix.byteLength);+ return body;+}++function assertNoStorage(bucket) {+ assert.deepEqual(bucket.reads, []);+ assert.deepEqual(bucket.writes, []);+ assert.equal(bucket.store.size, 0);+}++async function assertTooLarge(body, headers = {}) {+ const FIELD_REPORTS = fakeR2();+ const response = await onRequest({ request: request(body, headers), env: { FIELD_REPORTS } });+ assert.equal(response.status, 413);+ assert.deepEqual(await response.json(), { ok: false, error: 'body too large' });+ assert.equal(response.headers.get('access-control-allow-origin'), '*');+ assertNoStorage(FIELD_REPORTS);+}++test('multibyte JSON over 16 KiB is rejected without Content-Length or R2 access', async () => {+ const body = JSON.stringify({ agent: 'test', used_for: '界'.repeat(1000), verdict: '界'.repeat(1000), notes: '界'.repeat(4000) });+ assert.ok(body.length < LIMIT);+ assert.equal(encoder.encode(body).byteLength, 18054);+ assert.equal(request(body).headers.get('content-length'), null);+ await assertTooLarge(body);+});++test('exactly 16384 UTF-8 bytes are accepted and keep all three storage writes', async () => {+ const FIELD_REPORTS = fakeR2();+ const body = paddedBody(LIMIT, { multibyte: true });+ const response = await onRequest({ request: request(body), env: { FIELD_REPORTS } });+ assert.equal(body.byteLength, 16384);+ assert.equal(response.status, 200);+ const result = await response.json();+ assert.equal(result.ok, true);+ assert.equal(result.stored, true);+ assert.match(result.note, /github\.com\/xerj-org\/xerj/);+ assert.equal(response.headers.get('content-type'), 'application/json');+ assert.equal(response.headers.get('cache-control'), 'no-store');+ assert.equal(response.headers.get('access-control-allow-origin'), '*');+ assert.equal(FIELD_REPORTS.reads.length, 2);+ assert.equal(FIELD_REPORTS.writes.length, 3);+ assert.equal(FIELD_REPORTS.store.size, 3);+ assert.ok(FIELD_REPORTS.writes.some((key) => key.startsWith('rl/')));+ assert.ok(FIELD_REPORTS.writes.some((key) => key.startsWith('meta/count-')));+ const reportKey = FIELD_REPORTS.writes.find((key) => key.startsWith('reports/'));+ const report = JSON.parse(FIELD_REPORTS.store.get(reportKey));+ assert.equal(report.agent, valid.agent);+ assert.equal(report.notes, '界'.repeat(1000));+});++test('16385 UTF-8 bytes are rejected even with an understated Content-Length', async () => {+ const body = paddedBody(LIMIT + 1, { multibyte: true });+ assert.equal(body.byteLength, 16385);+ await assertTooLarge(body, { 'content-length': '1' });+});++test('oversized malformed JSON is rejected by byte size before JSON parsing', async () => {+ const body = '界'.repeat(6000);+ assert.ok(body.length < LIMIT);+ assert.equal(encoder.encode(body).byteLength, 18000);+ await assertTooLarge(body);+});++test('ASCII bodies keep the same exact byte boundary', async () => {+ const FIELD_REPORTS = fakeR2();+ const response = await onRequest({ request: request(paddedBody(LIMIT)), env: { FIELD_REPORTS } });+ assert.equal(response.status, 200);+ assert.equal(FIELD_REPORTS.writes.length, 3);+ await assertTooLarge(paddedBody(LIMIT + 1));+});++test('the UTF-8 BOM counts toward the limit before text decoding removes it', async () => {+ const body = paddedBody(LIMIT + 1, { bom: true });+ assert.equal(body.byteLength, 16385);+ assert.equal((await request(body).text()).length, 16382);+ await assertTooLarge(body);+ const response = await onRequest({ request: request(paddedBody(LIMIT, { bom: true })), env: {} });+ assert.equal(response.status, 200);+});++test('a declared oversize body is rejected before storage regardless of its actual size', async () => {+ await assertTooLarge(JSON.stringify(valid), { 'content-length': String(LIMIT + 1) });+});++test('valid reports without a bucket still return the unstored canonical response', async () => {+ const response = await onRequest({ request: request(JSON.stringify(valid)), env: {} });+ assert.equal(response.status, 200);+ const result = await response.json();+ assert.equal(result.ok, true);+ assert.equal(result.stored, false);+ assert.match(result.note, /github\.com\/xerj-org\/xerj/);+});++test('method and OPTIONS responses preserve CORS and never touch storage', async () => {+ for (const [method, status] of [['GET', 405], ['PUT', 405], ['OPTIONS', 204]]) {+ const FIELD_REPORTS = fakeR2();+ const response = await onRequest({ request: new Request('https://xerj.example.invalid/api/field-report', { method }), env: { FIELD_REPORTS } });+ assert.equal(response.status, status);+ assert.equal(response.headers.get('access-control-allow-origin'), '*');+ assert.equal(response.headers.get('access-control-allow-methods'), 'POST, OPTIONS');+ assertNoStorage(FIELD_REPORTS);+ }+});++test('wrong Content-Type is rejected without storage', async () => {+ const FIELD_REPORTS = fakeR2();+ const response = await onRequest({ request: request(JSON.stringify(valid), { 'content-type': 'text/plain' }), env: { FIELD_REPORTS } });+ assert.equal(response.status, 415);+ assertNoStorage(FIELD_REPORTS);+});++test('malformed JSON, non-object bodies and invalid fields still return 400 before storage', async () => {+ for (const body of ['{', 'null', '[]', '{}', JSON.stringify({ ...valid, agent: '' }), JSON.stringify({ ...valid, notes: 1 })]) {+ const FIELD_REPORTS = fakeR2();+ const response = await onRequest({ request: request(body), env: { FIELD_REPORTS } });+ assert.equal(response.status, 400);+ assertNoStorage(FIELD_REPORTS);+ }+});++test('a body read failure still returns 400 without storage', async () => {+ const FIELD_REPORTS = fakeR2();+ const body = new ReadableStream({ start(controller) { controller.error(new Error('read failed')); } });+ const response = await onRequest({+ request: new Request('https://xerj.example.invalid/api/field-report', { method: 'POST', headers: { 'content-type': 'application/json' }, body, duplex: 'half' }),+ env: { FIELD_REPORTS },+ });+ assert.equal(response.status, 400);+ assert.deepEqual(await response.json(), { ok: false, error: 'could not read body' });+ assertNoStorage(FIELD_REPORTS);+});diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex dea9c0fee..fbe88bd35 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -848,6 +848,9 @@ jobs:- name: Lead request validation (functions/lead.js)run: node .github/scripts/functions-lead-test.mjs+ - name: Field-report request byte limits (functions/api/field-report.js)+ run: node .github/scripts/functions-field-report-test.mjs+- name: Install Rustuses: dtolnay/rust-toolchain@032958afbdc797a9164d3bc0b56325c1308924a5 # 1.97.1diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5e90c5aed..f6824f485 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -111,6 +111,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- The field-report HTTP fallback now enforces its 16 KiB body limit on raw+ bytes before decoding or storage access. Multibyte JSON without a reliable+ `Content-Length` no longer bypasses the limit by having fewer characters.+- `xerj autoindex` no longer stalls near the end of a run at high`--workers`. When all 8 bulk slots were busy, a worker whose chunk crossedthe flush size parked; if another worker then shipped the shared buffer,diff --git a/functions/README.md b/functions/README.mdindex ec64ad42b..5425309dc 100644--- a/functions/README.md+++ b/functions/README.md@@ -31,12 +31,16 @@ Content-Type: application/json```- `200 { ok:true, stored:true, note:"...GitHub PR is canonical..." }` on success.-- `405` non-POST · `415` wrong Content-Type · `413` body > ~16 KB ·+- `405` non-POST · `415` wrong Content-Type · `413` body > 16 KiB (16,384 bytes) ·`400` bad JSON / missing required field · `429` rate limited ·`503` daily cap reached (use the GitHub PR path).- CORS is open for POST (`Access-Control-Allow-Origin: *`); `OPTIONS` preflightreturns `204`.+The body limit counts raw request bytes, including a UTF-8 BOM, before decoding,+JSON parsing or storage access. `Content-Length` enables early rejection; the+actual body is still checked when that header is missing or understates its size.+**Rate limit** — at most **1 accepted report per IP per minute** (`rl:<ip>`,60s TTL). A second request inside the minute → `429`.diff --git a/functions/api/field-report.js b/functions/api/field-report.jsindex dc7f07fce..579366d98 100644--- a/functions/api/field-report.js+++ b/functions/api/field-report.js@@ -80,12 +80,12 @@ async function handlePost({ request, env }) {const declared = parseInt(request.headers.get('content-length') || '0', 10);if (Number.isFinite(declared) && declared > MAX_BODY) return json({ ok:false, error:'body too large' }, 413);- let raw;- try { raw = await request.text(); } catch (_) { return json({ ok:false, error:'could not read body' }, 400); }- if (raw.length > MAX_BODY) return json({ ok:false, error:'body too large' }, 413);+ let bytes;+ try { bytes = await request.arrayBuffer(); } catch (_) { return json({ ok:false, error:'could not read body' }, 400); }+ if (bytes.byteLength > MAX_BODY) return json({ ok:false, error:'body too large' }, 413);let body;- try { body = JSON.parse(raw); } catch (_) { return json({ ok:false, error:'invalid JSON body' }, 400); }+ try { body = JSON.parse(new TextDecoder().decode(bytes)); } catch (_) { return json({ ok:false, error:'invalid JSON body' }, 400); }if (body === null || typeof body !== 'object' || Array.isArray(body)) return json({ ok:false, error:'body must be a JSON object' }, 400);const report = {};