~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
/**
* Browser contracts for the domain-classification workflow embedded in
* link_analytics.html. The harness compiles the checked-in template functions
* so endpoint, response-shape, CSRF, sequencing, and lifecycle assertions stay
* tied to the code shipped to the browser.
*/
import { readFileSync } from 'node:fs';
import { resolve as resolvePath } from 'node:path';
import { compileTemplateHarness } from '../helpers/template-harness.js';
const TEMPLATE_PATH = resolvePath(
__dirname,
'../../../src/local_deep_research/web/templates/pages/link_analytics.html',
);
const PROGRESS_URL = '/metrics/api/domain-classifications/progress';
const CLASSIFICATIONS_URL = '/metrics/api/domain-classifications';
const CLASSIFY_URL = '/metrics/api/domain-classifications/classify';
function jsonResponse(payload, { ok = true, status = 200, text = '' } = {}) {
return {
ok,
status,
json: vi.fn().mockResolvedValue(payload),
text: vi.fn().mockResolvedValue(text),
};
}
function deferred() {
let resolvePromise;
let rejectPromise;
const promise = new Promise((resolve, reject) => {
resolvePromise = resolve;
rejectPromise = reject;
});
return { promise, resolve: resolvePromise, reject: rejectPromise };
}
function mountClassificationUi() {
document.head.innerHTML = '<meta name="csrf-token" content="csrf-link">';
document.body.innerHTML = `
<button id="start-classification">Start</button>
<button id="stop-classification">Stop</button>
<input id="force-reclassify" type="checkbox">
<section id="classification-info" style="display: block"></section>
<section id="classification-progress" style="display: none">
<div id="progress-fill" style="width: 0%">0%</div>
<span id="current-domain"></span>
<div id="classification-log"></div>
</section>
<section id="classification-complete" style="display: none">
<span id="final-classified">0</span>
</section>
<span id="total-domains-count">0</span>
<span id="classified-count">0</span>
<span id="classification-status"></span>
`;
}
function extractStartRegistration() {
const template = readFileSync(TEMPLATE_PATH, 'utf8');
const registration = template.match(
/^\s*startBtn\.addEventListener\(\s*['"]click['"]\s*,\s*startDomainClassification\s*\);\s*$/m,
);
if (!registration) {
throw new Error('start-classification click registration not found');
}
return registration[0];
}
function extractStopRegistration() {
const template = readFileSync(TEMPLATE_PATH, 'utf8');
const registration = template.match(
/(stopBtn\.addEventListener\('click', \(\) => \{[\s\S]*?\n\}\);)/,
);
if (!registration) {
throw new Error('stop-classification click registration not found');
}
return registration[1];
}
function compileClassificationRuntime(delay = (callback) => callback()) {
return compileTemplateHarness({
templatePath: TEMPLATE_PATH,
functionNames: [
'loadClassificationProgress',
'startDomainClassification',
'classifyAllDomains',
'addLogEntry',
'showClassificationComplete',
],
dependencies: { setTimeout: delay },
preamble: `
let classificationInProgress = false;
let domainsToClassify = [];
let classificationRunId = 0;
let classificationAbortController = null;
const startBtn = document.getElementById('start-classification');
const stopBtn = document.getElementById('stop-classification');
${extractStartRegistration()}
${extractStopRegistration()}
`,
returnExpression: `({
loadClassificationProgress,
startDomainClassification,
getClassificationInProgress: () => classificationInProgress,
getDomainsToClassify: () => domainsToClassify,
})`,
});
}
beforeEach(() => {
mountClassificationUi();
});
afterEach(() => {
vi.restoreAllMocks();
vi.unstubAllGlobals();
document.head.replaceChildren();
document.body.replaceChildren();
});
it('loads the progress envelope and renders its counts', async () => {
const fetchMock = vi.fn().mockResolvedValue(jsonResponse({
status: 'success',
progress: { total_domains: 12, classified: 7 },
}));
vi.stubGlobal('fetch', fetchMock);
const runtime = compileClassificationRuntime();
await runtime.loadClassificationProgress();
expect(fetchMock).toHaveBeenCalledOnce();
expect(fetchMock).toHaveBeenCalledWith(PROGRESS_URL);
expect(document.getElementById('total-domains-count').textContent).toBe('12');
expect(document.getElementById('classified-count').textContent).toBe('7');
expect(document.getElementById('classification-status').textContent)
.toBe('7/12 domains classified');
});
it('classifies unhandled domains sequentially with CSRF and completes the UI', async () => {
const delay = vi.fn((callback) => callback());
const postSnapshots = [];
let progressCalls = 0;
const fetchMock = vi.fn((url, options) => {
if (url === PROGRESS_URL) {
progressCalls += 1;
if (progressCalls === 1) {
return Promise.resolve(jsonResponse({
status: 'success',
progress: { total_domains: 3, classified: 1 },
}));
}
return Promise.resolve(jsonResponse({
status: 'success',
progress: {
all_domains: ['known.test', 'alpha.test', 'beta.test'],
},
}));
}
if (url === CLASSIFICATIONS_URL) {
return Promise.resolve(jsonResponse({
status: 'success',
classifications: [{ domain: 'known.test' }],
}));
}
if (url === CLASSIFY_URL) {
const body = JSON.parse(options.body);
postSnapshots.push({
domain: body.domain,
progress: document.getElementById('progress-fill').textContent,
current: document.getElementById('current-domain').textContent,
});
return Promise.resolve(jsonResponse({
status: 'success',
classification: {
category: 'Research',
subcategory: body.domain === 'alpha.test' ? 'Journal' : 'Index',
},
}));
}
throw new Error(`Unexpected request: ${url}`);
});
vi.stubGlobal('fetch', fetchMock);
const runtime = compileClassificationRuntime(delay);
await runtime.startDomainClassification();
expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
PROGRESS_URL,
PROGRESS_URL,
CLASSIFICATIONS_URL,
CLASSIFY_URL,
CLASSIFY_URL,
]);
const postOptions = fetchMock.mock.calls
.filter(([url]) => url === CLASSIFY_URL)
.map(([, options]) => options);
expect(postOptions).toHaveLength(2);
postOptions.forEach((options, index) => {
expect(options).toEqual({
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-CSRFToken': 'csrf-link',
},
credentials: 'same-origin',
signal: expect.anything(),
body: JSON.stringify({
domain: index === 0 ? 'alpha.test' : 'beta.test',
force_update: false,
}),
});
});
expect(postSnapshots).toEqual([
{
domain: 'alpha.test',
progress: '50%',
current: 'Classifying: alpha.test',
},
{
domain: 'beta.test',
progress: '100%',
current: 'Classifying: beta.test',
},
]);
expect(delay).toHaveBeenCalledTimes(2);
expect(delay.mock.calls.every(([, milliseconds]) => milliseconds === 1000))
.toBe(true);
expect(runtime.getDomainsToClassify()).toEqual(['alpha.test', 'beta.test']);
expect(runtime.getClassificationInProgress()).toBe(false);
expect(document.getElementById('start-classification').disabled).toBe(false);
expect(document.getElementById('classification-progress').style.display)
.toBe('none');
expect(document.getElementById('classification-complete').style.display)
.toBe('block');
expect(document.getElementById('final-classified').textContent).toBe('2');
expect(document.getElementById('classification-log').textContent)
.toContain('✓ alpha.test: Research / Journal');
expect(document.getElementById('classification-log').textContent)
.toContain('✓ beta.test: Research / Index');
});
it('continues after an HTTP failure and reports only successful classifications', async () => {
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
const delay = vi.fn((callback) => callback());
let progressCalls = 0;
const postedDomains = [];
const fetchMock = vi.fn((url, options) => {
if (url === PROGRESS_URL) {
progressCalls += 1;
return Promise.resolve(jsonResponse({
status: 'success',
progress: progressCalls === 1
? { total_domains: 2, classified: 0 }
: { all_domains: ['unavailable.test', 'healthy.test'] },
}));
}
if (url === CLASSIFICATIONS_URL) {
return Promise.resolve(jsonResponse({
status: 'success',
classifications: [],
}));
}
if (url === CLASSIFY_URL) {
const { domain } = JSON.parse(options.body);
postedDomains.push(domain);
if (domain === 'unavailable.test') {
return Promise.resolve(jsonResponse({}, {
ok: false,
status: 503,
text: 'classifier unavailable',
}));
}
return Promise.resolve(jsonResponse({
status: 'success',
classification: { category: 'News', subcategory: 'Publisher' },
}));
}
throw new Error(`Unexpected request: ${url}`);
});
vi.stubGlobal('fetch', fetchMock);
const runtime = compileClassificationRuntime(delay);
await runtime.startDomainClassification();
expect(postedDomains).toEqual(['unavailable.test', 'healthy.test']);
expect(errorSpy).toHaveBeenCalledWith(
'HTTP error! status: 503, body: classifier unavailable',
);
expect(delay).toHaveBeenCalledOnce();
expect(document.getElementById('classification-log').textContent)
.toContain('✗ unavailable.test: HTTP 503 error');
expect(document.getElementById('classification-log').textContent)
.toContain('✓ healthy.test: News / Publisher');
expect(document.getElementById('final-classified').textContent).toBe('1');
expect(runtime.getClassificationInProgress()).toBe(false);
});
it('uses the checked-in click registration and restores retry after rejection', async () => {
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
const firstProgress = deferred();
const fetchMock = vi.fn().mockImplementation(() => firstProgress.promise);
vi.stubGlobal('fetch', fetchMock);
const runtime = compileClassificationRuntime();
const startButton = document.getElementById('start-classification');
startButton.click();
startButton.dispatchEvent(new MouseEvent('click'));
expect(fetchMock).toHaveBeenCalledOnce();
expect(runtime.getClassificationInProgress()).toBe(true);
expect(startButton.disabled).toBe(true);
firstProgress.reject(new Error('temporarily offline'));
await vi.waitFor(() => {
expect(runtime.getClassificationInProgress()).toBe(false);
});
expect(errorSpy).toHaveBeenCalledWith(
'Error starting classification:',
expect.objectContaining({ message: 'temporarily offline' }),
);
expect(runtime.getClassificationInProgress()).toBe(false);
expect(startButton.disabled).toBe(false);
expect(document.getElementById('classification-info').style.display)
.toBe('block');
expect(document.getElementById('classification-progress').style.display)
.toBe('none');
expect(document.getElementById('classification-log').textContent)
.toContain('Error: temporarily offline');
fetchMock.mockReset();
fetchMock
.mockResolvedValueOnce(jsonResponse({
status: 'success',
progress: { total_domains: 1, classified: 1 },
}))
.mockResolvedValueOnce(jsonResponse({
status: 'success',
progress: { all_domains: ['known.test'] },
}))
.mockResolvedValueOnce(jsonResponse({
status: 'success',
classifications: [{ domain: 'known.test' }],
}));
startButton.click();
await vi.waitFor(() => {
expect(document.getElementById('classification-complete').style.display)
.toBe('block');
});
expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
PROGRESS_URL,
PROGRESS_URL,
CLASSIFICATIONS_URL,
]);
expect(document.getElementById('classification-log').textContent)
.toContain('All domains are already classified!');
expect(document.getElementById('classification-complete').style.display)
.toBe('block');
expect(Number(document.getElementById('final-classified').textContent)).toBe(0);
expect(runtime.getClassificationInProgress()).toBe(false);
expect(startButton.disabled).toBe(false);
});
it('uses the real stop control to retire a deferred classification run', async () => {
const domainsResponse = deferred();
const fetchMock = vi.fn()
.mockResolvedValueOnce(jsonResponse({
status: 'success',
progress: { total_domains: 2, classified: 0 },
}))
.mockImplementationOnce(() => domainsResponse.promise);
vi.stubGlobal('fetch', fetchMock);
const runtime = compileClassificationRuntime();
const startButton = document.getElementById('start-classification');
const stopButton = document.getElementById('stop-classification');
const run = runtime.startDomainClassification();
await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2));
const deferredSignal = fetchMock.mock.calls[1][1].signal;
stopButton.click();
expect(deferredSignal.aborted).toBe(true);
expect(runtime.getClassificationInProgress()).toBe(false);
expect(startButton.disabled).toBe(false);
expect(document.getElementById('classification-log').textContent)
.toContain('Classification stopped by user');
domainsResponse.resolve(jsonResponse({
status: 'success',
progress: { all_domains: ['stale.test'] },
}));
await run;
expect(fetchMock).toHaveBeenCalledTimes(2);
expect(document.getElementById('classification-log').textContent)
.toContain('Classification stopped by user');
expect(document.getElementById('classification-log').textContent)
.not.toContain('Starting classification');
expect(document.getElementById('classification-complete').style.display)
.toBe('none');
});
it.each([
{
label: 'HTTP failure',
classificationsResponse: jsonResponse({}, {
ok: false,
status: 503,
text: 'unavailable',
}),
expectedMessage: 'Unable to load existing classifications (HTTP 503)',
},
{
label: 'error envelope',
classificationsResponse: jsonResponse({
status: 'error',
message: 'classification index unavailable',
}),
expectedMessage: 'classification index unavailable',
},
])('does not classify every domain after a lookup $label', async ({
classificationsResponse,
expectedMessage,
}) => {
vi.spyOn(console, 'error').mockImplementation(() => {});
const fetchMock = vi.fn()
.mockResolvedValueOnce(jsonResponse({
status: 'success',
progress: { total_domains: 1, classified: 0 },
}))
.mockResolvedValueOnce(jsonResponse({
status: 'success',
progress: { all_domains: ['must-not-post.test'] },
}))
.mockResolvedValueOnce(classificationsResponse);
vi.stubGlobal('fetch', fetchMock);
const runtime = compileClassificationRuntime();
await runtime.startDomainClassification();
expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
PROGRESS_URL,
PROGRESS_URL,
CLASSIFICATIONS_URL,
]);
expect(fetchMock).not.toHaveBeenCalledWith(
CLASSIFY_URL,
expect.anything(),
);
expect(document.getElementById('classification-log').textContent)
.toContain(`Fatal error: ${expectedMessage}`);
expect(document.getElementById('classification-log').textContent)
.toContain(`Error: ${expectedMessage}`);
expect(runtime.getClassificationInProgress()).toBe(false);
expect(document.getElementById('start-classification').disabled).toBe(false);
expect(document.getElementById('classification-info').style.display)
.toBe('block');
});
it.each([
{
failure: 'rejects',
secondProgress: () => Promise.reject(new Error('refresh offline')),
message: 'refresh offline',
},
{
failure: 'returns an error envelope',
secondProgress: () => Promise.resolve(jsonResponse({
status: 'error',
message: 'progress unavailable',
})),
message: 'progress unavailable',
},
])('restores retry controls when the classification progress refresh $failure', async ({
secondProgress,
message,
}) => {
vi.spyOn(console, 'error').mockImplementation(() => {});
const fetchMock = vi.fn()
.mockResolvedValueOnce(jsonResponse({
status: 'success',
progress: { total_domains: 2, classified: 0 },
}))
.mockImplementationOnce(secondProgress);
vi.stubGlobal('fetch', fetchMock);
const runtime = compileClassificationRuntime();
const startButton = document.getElementById('start-classification');
await runtime.startDomainClassification();
expect(fetchMock.mock.calls.map(([url]) => url)).toEqual([
PROGRESS_URL,
PROGRESS_URL,
]);
expect(runtime.getClassificationInProgress()).toBe(false);
expect(startButton.disabled).toBe(false);
expect(document.getElementById('classification-info').style.display)
.toBe('block');
expect(document.getElementById('classification-progress').style.display)
.toBe('none');
expect(document.getElementById('classification-log').textContent)
.toContain(`Fatal error: ${message}`);
expect(document.getElementById('classification-log').textContent)
.toContain(`Error: ${message}`);
});