~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
/**
* Runtime contracts for the simple benchmark page's inline start workflow.
*
* These tests execute the checked-in template function so the FastAPI route,
* CSRF header, selected-dataset payload, and recovery behavior stay aligned.
*/
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
const TEMPLATE_PATH = resolve(
__dirname,
'../../../src/local_deep_research/web/templates/pages/benchmark_simple.html',
);
function extractFunction(source, name) {
const signature = new RegExp(`(?:async\\s+)?function\\s+${name}\\s*\\(`);
const match = signature.exec(source);
if (!match) throw new Error(`Function ${name} not found in template`);
const openBrace = source.indexOf('{', match.index + match[0].length);
let depth = 0;
for (let index = openBrace; index < source.length; index += 1) {
if (source[index] === '{') depth += 1;
if (source[index] === '}') {
depth -= 1;
if (depth === 0) return source.slice(match.index, index + 1);
}
}
throw new Error(`Function ${name} has an unterminated body`);
}
function compileStartBenchmark(dependencies) {
const template = readFileSync(TEMPLATE_PATH, 'utf8');
const startBenchmarkSource = extractFunction(template, 'startBenchmark');
const dependencyNames = Object.keys(dependencies);
// The extracted source is repository-owned production code from the
// template above, not user-controlled input.
const factory = new Function( // eslint-disable-line no-new-func
...dependencyNames,
`
let currentBenchmarkId = null;
${startBenchmarkSource}
return {
startBenchmark,
getCurrentBenchmarkId: () => currentBenchmarkId,
};
`,
);
return factory(...Object.values(dependencies));
}
function renderBenchmarkSelection() {
document.head.innerHTML = '<meta name="csrf-token" content="csrf-3299">';
document.body.innerHTML = `
<div class="ldr-dataset-option ldr-selected" data-dataset="simpleqa">
<input value="3">
</div>
<div class="ldr-dataset-option ldr-selected" data-dataset="browsecomp">
<input value="0">
</div>
<button id="start-benchmark"></button>
`;
}
function createDependencies() {
return {
showAlert: vi.fn(),
showProgress: vi.fn(),
startProgressTracking: vi.fn(),
};
}
function deferred() {
let resolvePromise;
const promise = new Promise(resolvePromiseArgument => {
resolvePromise = resolvePromiseArgument;
});
return { promise, resolve: resolvePromise };
}
function compileProgressRuntime(dependencies) {
const template = readFileSync(TEMPLATE_PATH, 'utf8');
const functions = [
'startProgressTracking',
'formatBenchmarkAccuracy',
'updateProgress',
]
.map(name => extractFunction(template, name))
.join('\n');
const dependencyNames = Object.keys(dependencies);
const factory = new Function( // eslint-disable-line no-new-func
...dependencyNames,
`
let currentBenchmarkId = 'run-a';
let progressInterval = null;
let progressRunGeneration = 0;
let progressStatusRequestId = 0;
let latestAppliedProgressStatusRequestId = 0;
let terminalProgressRunGeneration = null;
${functions}
return {
startProgressTracking,
updateProgress,
setCurrentBenchmarkId: value => { currentBenchmarkId = value; },
getCurrentBenchmarkId: () => currentBenchmarkId,
getProgressInterval: () => progressInterval,
};
`,
);
return factory(...Object.values(dependencies));
}
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
vi.useRealTimers();
document.head.replaceChildren();
document.body.replaceChildren();
});
it('starts the selected datasets through the FastAPI simple-benchmark route', async () => {
renderBenchmarkSelection();
const fetchMock = vi.fn().mockResolvedValue({
ok: true,
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 'run-3299',
}),
});
vi.stubGlobal('fetch', fetchMock);
const dependencies = createDependencies();
const harness = compileStartBenchmark(dependencies);
await harness.startBenchmark();
expect(fetchMock).toHaveBeenCalledOnce();
expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/start-simple', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-CSRFToken': 'csrf-3299',
},
body: JSON.stringify({
datasets_config: { simpleqa: { count: 3 } },
}),
});
expect(harness.getCurrentBenchmarkId()).toBe('run-3299');
expect(dependencies.showProgress).toHaveBeenCalledOnce();
expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Benchmark started!',
'success',
);
});
it('does not call the backend when every selected dataset has a zero count', async () => {
renderBenchmarkSelection();
document.querySelector('[data-dataset="simpleqa"] input').value = '0';
const fetchMock = vi.fn();
vi.stubGlobal('fetch', fetchMock);
const dependencies = createDependencies();
await compileStartBenchmark(dependencies).startBenchmark();
expect(fetchMock).not.toHaveBeenCalled();
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Please select at least one dataset',
'error',
);
expect(document.getElementById('start-benchmark').disabled).toBe(false);
});
it('reenables the start action when the backend rejects the configuration', async () => {
renderBenchmarkSelection();
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
ok: true,
json: vi.fn().mockResolvedValue({
success: false,
error: 'dataset unavailable',
}),
}));
const dependencies = createDependencies();
await compileStartBenchmark(dependencies).startBenchmark();
expect(dependencies.showProgress).not.toHaveBeenCalled();
expect(dependencies.startProgressTracking).not.toHaveBeenCalled();
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Error: dataset unavailable',
'error',
);
expect(document.getElementById('start-benchmark').disabled).toBe(false);
});
it.each([
{
label: 'FastAPI error envelope',
response: {
ok: false,
status: 409,
json: vi.fn().mockResolvedValue({
error: 'benchmark worker unavailable',
}),
},
expectedError: 'Failed to start benchmark: benchmark worker unavailable',
},
{
label: 'FastAPI detail envelope',
response: {
ok: false,
status: 422,
json: vi.fn().mockResolvedValue({
detail: 'dataset configuration is invalid',
}),
},
expectedError: 'Failed to start benchmark: dataset configuration is invalid',
},
{
label: 'FastAPI message envelope',
response: {
ok: false,
status: 503,
json: vi.fn().mockResolvedValue({
message: 'benchmark queue is paused',
}),
},
expectedError: 'Failed to start benchmark: benchmark queue is paused',
},
{
label: 'non-ok success-shaped response',
response: {
ok: false,
status: 503,
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 'must-not-start',
}),
},
expectedError: 'Failed to start benchmark: HTTP 503',
},
{
label: 'success response without a run ID',
response: {
ok: true,
status: 200,
json: vi.fn().mockResolvedValue({ success: true }),
},
expectedError: 'Error: response did not include a benchmark run ID',
},
])('rejects a $label from the simple start endpoint', async ({
response,
expectedError,
}) => {
renderBenchmarkSelection();
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response));
const dependencies = createDependencies();
const harness = compileStartBenchmark(dependencies);
await harness.startBenchmark();
expect(harness.getCurrentBenchmarkId()).toBeNull();
expect(dependencies.showProgress).not.toHaveBeenCalled();
expect(dependencies.startProgressTracking).not.toHaveBeenCalled();
expect(dependencies.showAlert).toHaveBeenCalledWith(expectedError, 'error');
expect(document.getElementById('start-benchmark').disabled).toBe(false);
});
it('coalesces duplicate simple starts while the first POST is pending', async () => {
renderBenchmarkSelection();
const pendingStart = deferred();
const fetchMock = vi.fn(() => pendingStart.promise);
vi.stubGlobal('fetch', fetchMock);
const dependencies = createDependencies();
const harness = compileStartBenchmark(dependencies);
const firstStart = harness.startBenchmark();
await harness.startBenchmark();
expect(fetchMock).toHaveBeenCalledOnce();
pendingStart.resolve({
ok: true,
json: vi.fn().mockResolvedValue({
success: true,
benchmark_run_id: 'simple-single-run',
}),
});
await firstStart;
expect(fetchMock).toHaveBeenCalledOnce();
expect(harness.getCurrentBenchmarkId()).toBe('simple-single-run');
expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
});
it('ignores a terminal response from an older run after a newer run starts', async () => {
vi.useFakeTimers();
document.body.innerHTML = `
<div id="progress-fill" style="width: 0%"></div>
<span id="stat-accuracy">--%</span>
<span id="stat-completed">0</span>
<span id="stat-rate">--</span>
<span id="stat-remaining">--</span>
`;
const staleStatus = deferred();
const fetchMock = vi.fn(url => {
if (url === '/benchmark/api/status/run-a') return staleStatus.promise;
return new Promise(() => {});
});
vi.stubGlobal('fetch', fetchMock);
const dependencies = {
showAlert: vi.fn(),
hideProgress: vi.fn(),
};
const harness = compileProgressRuntime(dependencies);
const staleRequest = harness.startProgressTracking();
expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/status/run-a');
harness.setCurrentBenchmarkId('run-b');
harness.startProgressTracking();
const runBInterval = harness.getProgressInterval();
expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/status/run-b');
staleStatus.resolve({
json: vi.fn().mockResolvedValue({
success: true,
status: {
status: 'completed',
completed_examples: 5,
total_examples: 5,
overall_accuracy: 90,
processing_rate: 2,
},
}),
});
await staleRequest;
expect(document.getElementById('progress-fill').style.width).toBe('0%');
expect(dependencies.showAlert).not.toHaveBeenCalled();
expect(dependencies.hideProgress).not.toHaveBeenCalled();
expect(harness.getProgressInterval()).toBe(runBInterval);
clearInterval(runBInterval);
vi.useRealTimers();
});
it('ignores an older same-run poll after a newer status renders', async () => {
document.body.innerHTML = `
<div id="progress-fill" style="width: 0%"></div>
<span id="stat-accuracy">--%</span>
<span id="stat-completed">0</span>
<span id="stat-rate">--</span>
<span id="stat-remaining">--</span>
`;
const olderStatus = deferred();
const newerStatus = deferred();
vi.stubGlobal('fetch', vi.fn()
.mockImplementationOnce(() => olderStatus.promise)
.mockImplementationOnce(() => newerStatus.promise));
const dependencies = {
showAlert: vi.fn(),
hideProgress: vi.fn(),
};
const harness = compileProgressRuntime(dependencies);
const olderRequest = harness.startProgressTracking();
const newerRequest = harness.updateProgress();
const currentStatus = {
status: 'in_progress',
completed_examples: 6,
total_examples: 10,
overall_accuracy: 80,
processing_rate: 2,
};
newerStatus.resolve({
json: vi.fn().mockResolvedValue({ success: true, status: currentStatus }),
});
await newerRequest;
expect(document.getElementById('progress-fill').style.width).toBe('60%');
olderStatus.resolve({
json: vi.fn().mockResolvedValue({
success: true,
status: {
status: 'in_progress',
completed_examples: 2,
total_examples: 10,
overall_accuracy: 10,
processing_rate: 2,
},
}),
});
await olderRequest;
expect(document.getElementById('progress-fill').style.width).toBe('60%');
expect(document.getElementById('stat-accuracy').textContent).toBe('80.0%');
expect(dependencies.showAlert).not.toHaveBeenCalled();
clearInterval(harness.getProgressInterval());
});
it('accepts an older terminal snapshot after a newer nonterminal poll', async () => {
vi.useFakeTimers();
document.body.innerHTML = `
<div id="progress-fill" style="width: 0%"></div>
<span id="stat-accuracy">--%</span>
<span id="stat-completed">0</span>
<span id="stat-rate">--</span>
<span id="stat-remaining">--</span>
`;
const terminalResponse = deferred();
const nonterminalResponse = deferred();
vi.stubGlobal('fetch', vi.fn()
.mockImplementationOnce(() => terminalResponse.promise)
.mockImplementationOnce(() => nonterminalResponse.promise));
const dependencies = {
showAlert: vi.fn(),
hideProgress: vi.fn(),
};
const harness = compileProgressRuntime(dependencies);
const terminalLoad = harness.updateProgress();
const nonterminalLoad = harness.updateProgress();
nonterminalResponse.resolve({
json: vi.fn().mockResolvedValue({
success: true,
status: {
status: 'in_progress',
completed_examples: 8,
total_examples: 10,
overall_accuracy: 80,
processing_rate: 2,
},
}),
});
await nonterminalLoad;
terminalResponse.resolve({
json: vi.fn().mockResolvedValue({
success: true,
status: {
status: 'completed',
completed_examples: 10,
total_examples: 10,
overall_accuracy: 90,
processing_rate: 2,
},
}),
});
await terminalLoad;
expect(document.getElementById('progress-fill').style.width).toBe('100%');
expect(document.getElementById('stat-accuracy').textContent).toBe('90.0%');
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Benchmark completed! Final accuracy: 90.0%',
'success',
);
vi.clearAllTimers();
vi.useRealTimers();
});
it('keeps the first applied terminal snapshot authoritative for a run', async () => {
vi.useFakeTimers();
document.body.innerHTML = `
<div id="progress-fill" style="width: 0%"></div>
<span id="stat-accuracy">--%</span>
<span id="stat-completed">0</span>
<span id="stat-rate">--</span>
<span id="stat-remaining">--</span>
`;
const olderTerminal = deferred();
const newerTerminal = deferred();
vi.stubGlobal('fetch', vi.fn()
.mockImplementationOnce(() => olderTerminal.promise)
.mockImplementationOnce(() => newerTerminal.promise));
const dependencies = {
showAlert: vi.fn(),
hideProgress: vi.fn(),
};
const harness = compileProgressRuntime(dependencies);
const olderLoad = harness.updateProgress();
const newerLoad = harness.updateProgress();
newerTerminal.resolve({
json: vi.fn().mockResolvedValue({
success: true,
status: {
status: 'completed',
completed_examples: 10,
total_examples: 10,
overall_accuracy: 92,
processing_rate: 2,
},
}),
});
await newerLoad;
olderTerminal.resolve({
json: vi.fn().mockResolvedValue({
success: true,
status: {
status: 'failed',
completed_examples: 3,
total_examples: 10,
overall_accuracy: 20,
processing_rate: 1,
},
}),
});
await olderLoad;
expect(document.getElementById('progress-fill').style.width).toBe('100%');
expect(document.getElementById('stat-accuracy').textContent).toBe('92.0%');
expect(dependencies.showAlert).toHaveBeenCalledOnce();
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Benchmark completed! Final accuracy: 92.0%',
'success',
);
await vi.advanceTimersByTimeAsync(3000);
expect(dependencies.hideProgress).toHaveBeenCalledOnce();
});
it('completes and schedules cleanup when final accuracy is unavailable', async () => {
vi.useFakeTimers();
document.body.innerHTML = `
<div id="progress-fill" style="width: 0%"></div>
<span id="stat-accuracy">--%</span>
<span id="stat-completed">0</span>
<span id="stat-rate">--</span>
<span id="stat-remaining">--</span>
`;
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
json: vi.fn().mockResolvedValue({
success: true,
status: {
status: 'completed',
completed_examples: 0,
total_examples: 0,
overall_accuracy: null,
processing_rate: 0,
},
}),
}));
const dependencies = {
showAlert: vi.fn(),
hideProgress: vi.fn(),
};
const harness = compileProgressRuntime(dependencies);
await harness.updateProgress();
expect(document.getElementById('stat-accuracy').textContent).toBe('N/A');
expect(dependencies.showAlert).toHaveBeenCalledWith(
'Benchmark completed! Final accuracy: N/A',
'success',
);
await vi.advanceTimersByTimeAsync(3000);
expect(dependencies.hideProgress).toHaveBeenCalledOnce();
expect(harness.getCurrentBenchmarkId()).toBeNull();
});
it('does not let an old terminal cleanup hide a replacement run', async () => {
vi.useFakeTimers();
document.body.innerHTML = `
<div id="progress-fill" style="width: 0%"></div>
<span id="stat-accuracy">--%</span>
<span id="stat-completed">0</span>
<span id="stat-rate">--</span>
<span id="stat-remaining">--</span>
`;
const fetchMock = vi.fn()
.mockResolvedValueOnce({
json: vi.fn().mockResolvedValue({
success: true,
status: {
status: 'completed',
completed_examples: 4,
total_examples: 4,
overall_accuracy: 75,
processing_rate: 2,
},
}),
})
.mockImplementation(() => new Promise(() => {}));
vi.stubGlobal('fetch', fetchMock);
const dependencies = {
showAlert: vi.fn(),
hideProgress: vi.fn(),
};
const harness = compileProgressRuntime(dependencies);
await harness.updateProgress();
harness.setCurrentBenchmarkId('run-b');
harness.startProgressTracking();
const replacementInterval = harness.getProgressInterval();
await vi.advanceTimersByTimeAsync(3000);
expect(dependencies.hideProgress).not.toHaveBeenCalled();
expect(harness.getCurrentBenchmarkId()).toBe('run-b');
expect(harness.getProgressInterval()).toBe(replacementInterval);
clearInterval(replacementInterval);
});