~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
/**
* Runtime contracts for the full benchmark page's inline FastAPI client.
*
* This covers configuration serialization, start recovery, and reconnection
* using the production functions extracted from the checked-in template.
*/
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
const TEMPLATE_PATH = resolve(
__dirname,
'../../../src/local_deep_research/web/templates/pages/benchmark.html',
);
function deferred() {
let resolvePromise;
const promise = new Promise(resolveDeferred => {
resolvePromise = resolveDeferred;
});
return { promise, resolve: resolvePromise };
}
function extractFunction(source, name) {
const signature = new RegExp(`(?:async\\s+)?function\\s+${name}\\s*\\(`);
const match = signature.exec(source);
if (!match) throw new Error(`Function ${name} not found in template`);
const openBrace = source.indexOf('{', match.index + match[0].length);
let depth = 0;
for (let index = openBrace; index < source.length; index += 1) {
if (source[index] === '{') depth += 1;
if (source[index] === '}') {
depth -= 1;
if (depth === 0) return source.slice(match.index, index + 1);
}
}
throw new Error(`Function ${name} has an unterminated body`);
}
function compileBenchmarkClient(dependencies) {
const template = readFileSync(TEMPLATE_PATH, 'utf8');
const functions = [
'getConfigurationData',
'startBenchmark',
'checkForRunningBenchmark',
].map(name => extractFunction(template, name)).join('\n');
const dependencyNames = Object.keys(dependencies);
// The extracted source is repository-owned production code from the
// template above, not user-controlled input.
const factory = new Function( // eslint-disable-line no-new-func
'csrfToken',
...dependencyNames,
`
let currentBenchmarkId = null;
let benchmarkIntentGeneration = 0;
let benchmarkStartInFlight = false;
${functions}
return {
getConfigurationData,
startBenchmark,
checkForRunningBenchmark,
getCurrentBenchmarkId: () => currentBenchmarkId,
};
`,
);
return factory(
'csrf-benchmark',
...Object.values(dependencies),
);
}
function renderBenchmarkForm() {
document.body.innerHTML = `
<form id="benchmark-form"></form>
<section id="benchmark-progress" style="display: none"></section>
<input id="run_name" value="migration sweep">
<input id="sampling_seed" value="17">
<input id="simpleqa_enabled" type="checkbox" checked>
<input id="simpleqa_count" value="4">
<input id="xbench_deepsearch_enabled" type="checkbox">
<input id="xbench_deepsearch_count" value="99">
<input id="browsecomp_enabled" type="checkbox" checked>
<input id="browsecomp_count" value="2">
`;
}
function createDependencies() {
return {
showAlert: vi.fn(),
startProgressTracking: vi.fn(),
resetForm: vi.fn(),
};
}
beforeEach(() => {
renderBenchmarkForm();
});
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
document.body.replaceChildren();
});
it('serializes only enabled datasets and applies the shared sampling seed', () => {
const harness = compileBenchmarkClient(createDependencies());
expect(harness.getConfigurationData()).toEqual({
run_name: 'migration sweep',
datasets_config: {
simpleqa: { count: 4, seed: 17 },
browsecomp: { count: 2, seed: 17 },
},
});
});
it('omits the seed for a fresh random sample', () => {
document.getElementById('sampling_seed').value = '';
expect(compileBenchmarkClient(createDependencies()).getConfigurationData())
.toEqual({
run_name: 'migration sweep',
datasets_config: {
simpleqa: { count: 4 },
browsecomp: { count: 2 },
},
});
});
it('starts a run with the serialized config and tracks the returned run ID', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 3299,
}),
});
vi.stubGlobal('fetch', fetchMock);
const dependencies = createDependencies();
const harness = compileBenchmarkClient(dependencies);
harness.startBenchmark();
await vi.waitFor(() => {
expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
});
expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/start', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-CSRFToken': 'csrf-benchmark',
},
body: JSON.stringify({
run_name: 'migration sweep',
datasets_config: {
simpleqa: { count: 4, seed: 17 },
browsecomp: { count: 2, seed: 17 },
},
}),
});
expect(harness.getCurrentBenchmarkId()).toBe(3299);
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Benchmark started successfully!',
'success',
);
expect(document.getElementById('benchmark-form').style.display).toBe('none');
expect(document.getElementById('benchmark-progress').style.display)
.toBe('block');
});
it('resets the browser state after a rejected start response', async () => {
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
ok: true,
json: vi.fn().mockResolvedValue({
success: false,
error: 'benchmark worker unavailable',
}),
}));
const dependencies = createDependencies();
const harness = compileBenchmarkClient(dependencies);
harness.startBenchmark();
await vi.waitFor(() => {
expect(dependencies.resetForm).toHaveBeenCalledOnce();
});
expect(dependencies.startProgressTracking).not.toHaveBeenCalled();
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Error starting benchmark: benchmark worker unavailable',
'error',
);
});
it('reconnects the page to the run reported by the migrated running endpoint', async () => {
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 'existing-12',
}),
});
vi.stubGlobal('fetch', fetchMock);
const dependencies = createDependencies();
const harness = compileBenchmarkClient(dependencies);
harness.checkForRunningBenchmark();
await vi.waitFor(() => {
expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
});
expect(fetchMock).toHaveBeenCalledOnce();
expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/running');
expect(harness.getCurrentBenchmarkId()).toBe('existing-12');
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Reconnected to running benchmark #existing-12',
'info',
);
expect(document.getElementById('benchmark-form').style.display).toBe('none');
expect(document.getElementById('benchmark-progress').style.display)
.toBe('block');
});
it('does not let a delayed running probe replace a user-started benchmark', async () => {
const runningProbe = deferred();
const fetchMock = vi.fn()
.mockImplementationOnce(() => runningProbe.promise)
.mockResolvedValueOnce({
ok: true,
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 'user-run',
}),
});
vi.stubGlobal('fetch', fetchMock);
const dependencies = createDependencies();
const harness = compileBenchmarkClient(dependencies);
harness.checkForRunningBenchmark();
harness.startBenchmark();
await vi.waitFor(() => {
expect(harness.getCurrentBenchmarkId()).toBe('user-run');
});
expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
runningProbe.resolve({
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 'stale-running-probe',
}),
});
await Promise.resolve();
await Promise.resolve();
expect(harness.getCurrentBenchmarkId()).toBe('user-run');
expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
expect(dependencies.showAlert).not.toHaveBeenCalledWith(
'Reconnected to running benchmark #stale-running-probe',
'info',
);
});
it.each([
{
label: 'FastAPI error envelope',
response: {
ok: false,
status: 409,
json: vi.fn().mockResolvedValue({
error: 'benchmark worker unavailable',
}),
},
expectedError: 'Error starting benchmark: benchmark worker unavailable',
},
{
label: 'FastAPI detail envelope',
response: {
ok: false,
status: 422,
json: vi.fn().mockResolvedValue({
detail: 'dataset configuration is invalid',
}),
},
expectedError: 'Error starting benchmark: dataset configuration is invalid',
},
{
label: 'FastAPI message envelope',
response: {
ok: false,
status: 503,
json: vi.fn().mockResolvedValue({
message: 'benchmark queue is paused',
}),
},
expectedError: 'Error starting benchmark: benchmark queue is paused',
},
{
label: 'non-ok success-shaped response',
response: {
ok: false,
status: 503,
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 'must-not-start',
}),
},
expectedError: 'Error starting benchmark: HTTP 503',
},
{
label: 'success response without a run ID',
response: {
ok: true,
status: 200,
json: vi.fn().mockResolvedValue({ success: true }),
},
expectedError: 'Error starting benchmark: response did not include a benchmark run ID',
},
])('rejects a $label', async ({ response, expectedError }) => {
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response));
const dependencies = createDependencies();
const harness = compileBenchmarkClient(dependencies);
await harness.startBenchmark();
expect(harness.getCurrentBenchmarkId()).toBeNull();
expect(dependencies.startProgressTracking).not.toHaveBeenCalled();
expect(dependencies.resetForm).toHaveBeenCalledOnce();
expect(dependencies.showAlert).toHaveBeenCalledWith(expectedError, 'error');
});
it('coalesces duplicate full benchmark submissions while the POST is pending', async () => {
const pendingStart = deferred();
const fetchMock = vi.fn(() => pendingStart.promise);
vi.stubGlobal('fetch', fetchMock);
const dependencies = createDependencies();
const harness = compileBenchmarkClient(dependencies);
const firstStart = harness.startBenchmark();
const duplicateStart = harness.startBenchmark();
expect(fetchMock).toHaveBeenCalledOnce();
await duplicateStart;
pendingStart.resolve({
ok: true,
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 'single-run',
}),
});
await firstStart;
expect(fetchMock).toHaveBeenCalledOnce();
expect(harness.getCurrentBenchmarkId()).toBe('single-run');
expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
});