Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

6.9 KB
Raw Blame
/**
 * Browser contract for the benchmark page's Socket.IO adapter.
 *
 * The generic socket-service tests cover room/event names. This test executes
 * the checked-in inline page functions so the benchmark's run ID, callback
 * rendering, and teardown cannot drift from that service unnoticed.
 */

import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';

import {
    compileTemplateHarness,
    extractJavaScriptBlock,
} from '../helpers/template-harness.js';

const TEMPLATE_PATH = resolve(
    __dirname,
    '../../../src/local_deep_research/web/templates/pages/benchmark.html',
);

function extractDOMContentLoadedCallback(source) {
    const marker = "document.addEventListener('DOMContentLoaded', ";
    const listenerIndex = source.indexOf(marker);
    if (listenerIndex === -1) {
        throw new Error('DOMContentLoaded listener not found in template');
    }

    const functionIndex = source.indexOf('function', listenerIndex);
    return extractJavaScriptBlock(source, functionIndex);
}

function compileDOMContentLoadedHandler(dependencies) {
    const template = readFileSync(TEMPLATE_PATH, 'utf8');
    const callback = extractDOMContentLoadedCallback(template);
    const dependencyNames = Object.keys(dependencies);
    // The extracted source is repository-owned production code from the
    // template above, not user-controlled input.
    const factory = new Function( // eslint-disable-line no-new-func
        ...dependencyNames,
        `return (${callback});`,
    );
    return factory(...Object.values(dependencies));
}

function createBenchmarkHarness(runId, dependencies) {
    return compileTemplateHarness({
        templatePath: TEMPLATE_PATH,
        functionNames: [
            'handleDetailedProgress',
            'startProgressTracking',
            'resetForm',
        ],
        dependencies: {
            runId,
            initializeCharts: dependencies.initializeCharts,
            loadHistoricalChartData: dependencies.loadHistoricalChartData,
            updateBenchmarkProgress: dependencies.updateBenchmarkProgress,
        },
        preamble: `
            let currentBenchmarkId = runId;
            let progressInterval = null;
            let progressRunGeneration = 0;
            let progressStatusRequestId = 0;
            let latestAppliedProgressStatusRequestId = 0;
            let terminalBenchmarkRunGeneration = 0;
            let hydratedTerminalBenchmarkRunGeneration = 0;
            let recentResultsRequestId = 0;
            let historicalChartRequestId = 0;
            let searchQualityRequestId = 0;
            let chartData = {};
            let recentSearchCounts = [];
            let searchQualityAlert = false;
            let accuracyChart = null;
            let timingChart = null;
            let searchResultsChart = null;
        `,
        returnExpression: `({
            handleDetailedProgress,
            startProgressTracking,
            resetForm,
            getCurrentBenchmarkId: () => currentBenchmarkId,
        })`,
    });
}

beforeEach(() => {
    vi.useFakeTimers();
    document.body.innerHTML = `
        <form id="benchmark-form" style="display: none"></form>
        <section id="benchmark-progress"></section>
        <section id="performance-charts-section" style="display: none"></section>
        <div id="current-task"></div>
    `;
});

afterEach(() => {
    vi.clearAllTimers();
    vi.useRealTimers();
    vi.restoreAllMocks();
    delete window.socket;
    delete window.modelsLoading;
    document.body.replaceChildren();
});

it('subscribes, renders progress, and unsubscribes with the original numeric run ID', async () => {
    const subscribeToResearch = vi.fn();
    const unsubscribeFromResearch = vi.fn();
    window.socket = {
        init: vi.fn(),
        getSocketInstance: vi.fn(() => null),
        subscribeToResearch,
        unsubscribeFromResearch,
    };

    const harness = createBenchmarkHarness(42, {
        initializeCharts: vi.fn(),
        loadHistoricalChartData: vi.fn(),
        updateBenchmarkProgress: vi.fn(),
    });

    harness.startProgressTracking();
    await vi.advanceTimersByTimeAsync(500);

    expect(window.socket.init).toHaveBeenCalledOnce();
    expect(subscribeToResearch).toHaveBeenCalledTimes(1);
    expect(subscribeToResearch).toHaveBeenCalledWith(42, expect.any(Function));

    const progressCallback = subscribeToResearch.mock.calls[0][1];
    progressCallback({
        status: 'in_progress',
        message: 'grading',
        log_entry: {
            message: 'Example 7: grading',
            metadata: { example_id: 7 },
        },
    });
    expect(document.getElementById('current-task').textContent)
        .toBe('Example 7: grading');

    harness.resetForm();

    expect(unsubscribeFromResearch).toHaveBeenCalledTimes(1);
    expect(unsubscribeFromResearch).toHaveBeenCalledWith(42);
    expect(harness.getCurrentBenchmarkId()).toBeNull();
});

it('uses the migrated socket init API at page bootstrap and keeps initializing after a socket failure', () => {
    const initError = new Error('transport unavailable');
    window.socket = { init: vi.fn(() => { throw initError; }) };
    const dependencies = {
        initializeBenchmarkForm: vi.fn(),
        initializeEvaluationSettings: vi.fn(),
        loadCurrentSettings: vi.fn(),
        updateConfigSummary: vi.fn(),
        checkForRunningBenchmark: vi.fn(),
    };
    const warning = vi.spyOn(console, 'warn').mockImplementation(() => {});

    compileDOMContentLoadedHandler(dependencies)();

    expect(window.socket.init).toHaveBeenCalledOnce();
    expect(warning).toHaveBeenCalledWith(
        'Socket initialization failed, continuing without real-time updates',
    );
    for (const dependency of Object.values(dependencies)) {
        expect(dependency).toHaveBeenCalledOnce();
    }
});

it('reuses an existing socket and keeps terminal progress authoritative', async () => {
    const subscribeToResearch = vi.fn();
    window.socket = {
        init: vi.fn(),
        getSocketInstance: vi.fn(() => ({ connected: true })),
        subscribeToResearch,
        unsubscribeFromResearch: vi.fn(),
    };
    const harness = createBenchmarkHarness('run-9', {
        initializeCharts: vi.fn(),
        loadHistoricalChartData: vi.fn(),
        updateBenchmarkProgress: vi.fn(),
    });

    harness.startProgressTracking();
    await vi.advanceTimersByTimeAsync(500);

    expect(window.socket.init).not.toHaveBeenCalled();
    const progressCallback = subscribeToResearch.mock.calls[0][1];
    progressCallback({ message: 'Scoring answer' });
    expect(document.getElementById('current-task').textContent)
        .toBe('Scoring answer');

    progressCallback({ status: 'completed' });
    expect(document.getElementById('current-task').textContent)
        .toBe('completed');

    progressCallback({
        status: 'ignored status',
        message: 'ignored message',
        log_entry: { message: 'Preferred structured log' },
    });
    expect(document.getElementById('current-task').textContent)
        .toBe('completed');
});