~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
/**
* Browser contract for the benchmark page's Socket.IO adapter.
*
* The generic socket-service tests cover room/event names. This test executes
* the checked-in inline page functions so the benchmark's run ID, callback
* rendering, and teardown cannot drift from that service unnoticed.
*/
import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';
import {
compileTemplateHarness,
extractJavaScriptBlock,
} from '../helpers/template-harness.js';
const TEMPLATE_PATH = resolve(
__dirname,
'../../../src/local_deep_research/web/templates/pages/benchmark.html',
);
function extractDOMContentLoadedCallback(source) {
const marker = "document.addEventListener('DOMContentLoaded', ";
const listenerIndex = source.indexOf(marker);
if (listenerIndex === -1) {
throw new Error('DOMContentLoaded listener not found in template');
}
const functionIndex = source.indexOf('function', listenerIndex);
return extractJavaScriptBlock(source, functionIndex);
}
function compileDOMContentLoadedHandler(dependencies) {
const template = readFileSync(TEMPLATE_PATH, 'utf8');
const callback = extractDOMContentLoadedCallback(template);
const dependencyNames = Object.keys(dependencies);
// The extracted source is repository-owned production code from the
// template above, not user-controlled input.
const factory = new Function( // eslint-disable-line no-new-func
...dependencyNames,
`return (${callback});`,
);
return factory(...Object.values(dependencies));
}
function createBenchmarkHarness(runId, dependencies) {
return compileTemplateHarness({
templatePath: TEMPLATE_PATH,
functionNames: [
'handleDetailedProgress',
'startProgressTracking',
'resetForm',
],
dependencies: {
runId,
initializeCharts: dependencies.initializeCharts,
loadHistoricalChartData: dependencies.loadHistoricalChartData,
updateBenchmarkProgress: dependencies.updateBenchmarkProgress,
},
preamble: `
let currentBenchmarkId = runId;
let progressInterval = null;
let progressRunGeneration = 0;
let progressStatusRequestId = 0;
let latestAppliedProgressStatusRequestId = 0;
let terminalBenchmarkRunGeneration = 0;
let hydratedTerminalBenchmarkRunGeneration = 0;
let recentResultsRequestId = 0;
let historicalChartRequestId = 0;
let searchQualityRequestId = 0;
let chartData = {};
let recentSearchCounts = [];
let searchQualityAlert = false;
let accuracyChart = null;
let timingChart = null;
let searchResultsChart = null;
`,
returnExpression: `({
handleDetailedProgress,
startProgressTracking,
resetForm,
getCurrentBenchmarkId: () => currentBenchmarkId,
})`,
});
}
beforeEach(() => {
vi.useFakeTimers();
document.body.innerHTML = `
<form id="benchmark-form" style="display: none"></form>
<section id="benchmark-progress"></section>
<section id="performance-charts-section" style="display: none"></section>
<div id="current-task"></div>
`;
});
afterEach(() => {
vi.clearAllTimers();
vi.useRealTimers();
vi.restoreAllMocks();
delete window.socket;
delete window.modelsLoading;
document.body.replaceChildren();
});
it('subscribes, renders progress, and unsubscribes with the original numeric run ID', async () => {
const subscribeToResearch = vi.fn();
const unsubscribeFromResearch = vi.fn();
window.socket = {
init: vi.fn(),
getSocketInstance: vi.fn(() => null),
subscribeToResearch,
unsubscribeFromResearch,
};
const harness = createBenchmarkHarness(42, {
initializeCharts: vi.fn(),
loadHistoricalChartData: vi.fn(),
updateBenchmarkProgress: vi.fn(),
});
harness.startProgressTracking();
await vi.advanceTimersByTimeAsync(500);
expect(window.socket.init).toHaveBeenCalledOnce();
expect(subscribeToResearch).toHaveBeenCalledTimes(1);
expect(subscribeToResearch).toHaveBeenCalledWith(42, expect.any(Function));
const progressCallback = subscribeToResearch.mock.calls[0][1];
progressCallback({
status: 'in_progress',
message: 'grading',
log_entry: {
message: 'Example 7: grading',
metadata: { example_id: 7 },
},
});
expect(document.getElementById('current-task').textContent)
.toBe('Example 7: grading');
harness.resetForm();
expect(unsubscribeFromResearch).toHaveBeenCalledTimes(1);
expect(unsubscribeFromResearch).toHaveBeenCalledWith(42);
expect(harness.getCurrentBenchmarkId()).toBeNull();
});
it('uses the migrated socket init API at page bootstrap and keeps initializing after a socket failure', () => {
const initError = new Error('transport unavailable');
window.socket = { init: vi.fn(() => { throw initError; }) };
const dependencies = {
initializeBenchmarkForm: vi.fn(),
initializeEvaluationSettings: vi.fn(),
loadCurrentSettings: vi.fn(),
updateConfigSummary: vi.fn(),
checkForRunningBenchmark: vi.fn(),
};
const warning = vi.spyOn(console, 'warn').mockImplementation(() => {});
compileDOMContentLoadedHandler(dependencies)();
expect(window.socket.init).toHaveBeenCalledOnce();
expect(warning).toHaveBeenCalledWith(
'Socket initialization failed, continuing without real-time updates',
);
for (const dependency of Object.values(dependencies)) {
expect(dependency).toHaveBeenCalledOnce();
}
});
it('reuses an existing socket and keeps terminal progress authoritative', async () => {
const subscribeToResearch = vi.fn();
window.socket = {
init: vi.fn(),
getSocketInstance: vi.fn(() => ({ connected: true })),
subscribeToResearch,
unsubscribeFromResearch: vi.fn(),
};
const harness = createBenchmarkHarness('run-9', {
initializeCharts: vi.fn(),
loadHistoricalChartData: vi.fn(),
updateBenchmarkProgress: vi.fn(),
});
harness.startProgressTracking();
await vi.advanceTimersByTimeAsync(500);
expect(window.socket.init).not.toHaveBeenCalled();
const progressCallback = subscribeToResearch.mock.calls[0][1];
progressCallback({ message: 'Scoring answer' });
expect(document.getElementById('current-task').textContent)
.toBe('Scoring answer');
progressCallback({ status: 'completed' });
expect(document.getElementById('current-task').textContent)
.toBe('completed');
progressCallback({
status: 'ignored status',
message: 'ignored message',
log_entry: { message: 'Preferred structured log' },
});
expect(document.getElementById('current-task').textContent)
.toBe('completed');
});