Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

11.5 KB
Raw Blame
/**
 * Runtime contracts for the full benchmark page's inline FastAPI client.
 *
 * This covers configuration serialization, start recovery, and reconnection
 * using the production functions extracted from the checked-in template.
 */

import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';

const TEMPLATE_PATH = resolve(
    __dirname,
    '../../../src/local_deep_research/web/templates/pages/benchmark.html',
);

function deferred() {
    let resolvePromise;
    const promise = new Promise(resolveDeferred => {
        resolvePromise = resolveDeferred;
    });
    return { promise, resolve: resolvePromise };
}

function extractFunction(source, name) {
    const signature = new RegExp(`(?:async\\s+)?function\\s+${name}\\s*\\(`);
    const match = signature.exec(source);
    if (!match) throw new Error(`Function ${name} not found in template`);

    const openBrace = source.indexOf('{', match.index + match[0].length);
    let depth = 0;
    for (let index = openBrace; index < source.length; index += 1) {
        if (source[index] === '{') depth += 1;
        if (source[index] === '}') {
            depth -= 1;
            if (depth === 0) return source.slice(match.index, index + 1);
        }
    }

    throw new Error(`Function ${name} has an unterminated body`);
}

function compileBenchmarkClient(dependencies) {
    const template = readFileSync(TEMPLATE_PATH, 'utf8');
    const functions = [
        'getConfigurationData',
        'startBenchmark',
        'checkForRunningBenchmark',
    ].map(name => extractFunction(template, name)).join('\n');
    const dependencyNames = Object.keys(dependencies);
    // The extracted source is repository-owned production code from the
    // template above, not user-controlled input.
    const factory = new Function( // eslint-disable-line no-new-func
        'csrfToken',
        ...dependencyNames,
        `
            let currentBenchmarkId = null;
            let benchmarkIntentGeneration = 0;
            let benchmarkStartInFlight = false;
            ${functions}
            return {
                getConfigurationData,
                startBenchmark,
                checkForRunningBenchmark,
                getCurrentBenchmarkId: () => currentBenchmarkId,
            };
        `,
    );
    return factory(
        'csrf-benchmark',
        ...Object.values(dependencies),
    );
}

function renderBenchmarkForm() {
    document.body.innerHTML = `
        <form id="benchmark-form"></form>
        <section id="benchmark-progress" style="display: none"></section>
        <input id="run_name" value="migration sweep">
        <input id="sampling_seed" value="17">
        <input id="simpleqa_enabled" type="checkbox" checked>
        <input id="simpleqa_count" value="4">
        <input id="xbench_deepsearch_enabled" type="checkbox">
        <input id="xbench_deepsearch_count" value="99">
        <input id="browsecomp_enabled" type="checkbox" checked>
        <input id="browsecomp_count" value="2">
    `;
}

function createDependencies() {
    return {
        showAlert: vi.fn(),
        startProgressTracking: vi.fn(),
        resetForm: vi.fn(),
    };
}

beforeEach(() => {
    renderBenchmarkForm();
});

afterEach(() => {
    vi.restoreAllMocks();
    vi.unstubAllGlobals();
    document.body.replaceChildren();
});

it('serializes only enabled datasets and applies the shared sampling seed', () => {
    const harness = compileBenchmarkClient(createDependencies());

    expect(harness.getConfigurationData()).toEqual({
        run_name: 'migration sweep',
        datasets_config: {
            simpleqa: { count: 4, seed: 17 },
            browsecomp: { count: 2, seed: 17 },
        },
    });
});

it('omits the seed for a fresh random sample', () => {
    document.getElementById('sampling_seed').value = '';

    expect(compileBenchmarkClient(createDependencies()).getConfigurationData())
        .toEqual({
            run_name: 'migration sweep',
            datasets_config: {
                simpleqa: { count: 4 },
                browsecomp: { count: 2 },
            },
        });
});

it('starts a run with the serialized config and tracks the returned run ID', async () => {
    const fetchMock = vi.fn().mockResolvedValue({
        ok: true,
        json: vi.fn().mockResolvedValue({
            success: true,
            benchmark_run_id: 3299,
        }),
    });
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = createDependencies();
    const harness = compileBenchmarkClient(dependencies);

    harness.startBenchmark();

    await vi.waitFor(() => {
        expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
    });
    expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/start', {
        method: 'POST',
        headers: {
            'Content-Type': 'application/json',
            'X-CSRFToken': 'csrf-benchmark',
        },
        body: JSON.stringify({
            run_name: 'migration sweep',
            datasets_config: {
                simpleqa: { count: 4, seed: 17 },
                browsecomp: { count: 2, seed: 17 },
            },
        }),
    });
    expect(harness.getCurrentBenchmarkId()).toBe(3299);
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Benchmark started successfully!',
        'success',
    );
    expect(document.getElementById('benchmark-form').style.display).toBe('none');
    expect(document.getElementById('benchmark-progress').style.display)
        .toBe('block');
});

it('resets the browser state after a rejected start response', async () => {
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
        ok: true,
        json: vi.fn().mockResolvedValue({
            success: false,
            error: 'benchmark worker unavailable',
        }),
    }));
    const dependencies = createDependencies();
    const harness = compileBenchmarkClient(dependencies);

    harness.startBenchmark();

    await vi.waitFor(() => {
        expect(dependencies.resetForm).toHaveBeenCalledOnce();
    });
    expect(dependencies.startProgressTracking).not.toHaveBeenCalled();
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Error starting benchmark: benchmark worker unavailable',
        'error',
    );
});

it('reconnects the page to the run reported by the migrated running endpoint', async () => {
    const fetchMock = vi.fn().mockResolvedValue({
        ok: true,
        json: vi.fn().mockResolvedValue({
            success: true,
            benchmark_run_id: 'existing-12',
        }),
    });
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = createDependencies();
    const harness = compileBenchmarkClient(dependencies);

    harness.checkForRunningBenchmark();

    await vi.waitFor(() => {
        expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
    });
    expect(fetchMock).toHaveBeenCalledOnce();
    expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/running');
    expect(harness.getCurrentBenchmarkId()).toBe('existing-12');
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Reconnected to running benchmark #existing-12',
        'info',
    );
    expect(document.getElementById('benchmark-form').style.display).toBe('none');
    expect(document.getElementById('benchmark-progress').style.display)
        .toBe('block');
});

it('does not let a delayed running probe replace a user-started benchmark', async () => {
    const runningProbe = deferred();
    const fetchMock = vi.fn()
        .mockImplementationOnce(() => runningProbe.promise)
        .mockResolvedValueOnce({
            ok: true,
            json: vi.fn().mockResolvedValue({
                success: true,
                benchmark_run_id: 'user-run',
            }),
        });
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = createDependencies();
    const harness = compileBenchmarkClient(dependencies);

    harness.checkForRunningBenchmark();
    harness.startBenchmark();
    await vi.waitFor(() => {
        expect(harness.getCurrentBenchmarkId()).toBe('user-run');
    });
    expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();

    runningProbe.resolve({
        json: vi.fn().mockResolvedValue({
            success: true,
            benchmark_run_id: 'stale-running-probe',
        }),
    });
    await Promise.resolve();
    await Promise.resolve();

    expect(harness.getCurrentBenchmarkId()).toBe('user-run');
    expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
    expect(dependencies.showAlert).not.toHaveBeenCalledWith(
        'Reconnected to running benchmark #stale-running-probe',
        'info',
    );
});

it.each([
    {
        label: 'FastAPI error envelope',
        response: {
            ok: false,
            status: 409,
            json: vi.fn().mockResolvedValue({
                error: 'benchmark worker unavailable',
            }),
        },
        expectedError: 'Error starting benchmark: benchmark worker unavailable',
    },
    {
        label: 'FastAPI detail envelope',
        response: {
            ok: false,
            status: 422,
            json: vi.fn().mockResolvedValue({
                detail: 'dataset configuration is invalid',
            }),
        },
        expectedError: 'Error starting benchmark: dataset configuration is invalid',
    },
    {
        label: 'FastAPI message envelope',
        response: {
            ok: false,
            status: 503,
            json: vi.fn().mockResolvedValue({
                message: 'benchmark queue is paused',
            }),
        },
        expectedError: 'Error starting benchmark: benchmark queue is paused',
    },
    {
        label: 'non-ok success-shaped response',
        response: {
            ok: false,
            status: 503,
            json: vi.fn().mockResolvedValue({
                success: true,
                benchmark_run_id: 'must-not-start',
            }),
        },
        expectedError: 'Error starting benchmark: HTTP 503',
    },
    {
        label: 'success response without a run ID',
        response: {
            ok: true,
            status: 200,
            json: vi.fn().mockResolvedValue({ success: true }),
        },
        expectedError: 'Error starting benchmark: response did not include a benchmark run ID',
    },
])('rejects a $label', async ({ response, expectedError }) => {
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response));
    const dependencies = createDependencies();
    const harness = compileBenchmarkClient(dependencies);

    await harness.startBenchmark();

    expect(harness.getCurrentBenchmarkId()).toBeNull();
    expect(dependencies.startProgressTracking).not.toHaveBeenCalled();
    expect(dependencies.resetForm).toHaveBeenCalledOnce();
    expect(dependencies.showAlert).toHaveBeenCalledWith(expectedError, 'error');
});

it('coalesces duplicate full benchmark submissions while the POST is pending', async () => {
    const pendingStart = deferred();
    const fetchMock = vi.fn(() => pendingStart.promise);
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = createDependencies();
    const harness = compileBenchmarkClient(dependencies);

    const firstStart = harness.startBenchmark();
    const duplicateStart = harness.startBenchmark();

    expect(fetchMock).toHaveBeenCalledOnce();
    await duplicateStart;
    pendingStart.resolve({
        ok: true,
        json: vi.fn().mockResolvedValue({
            success: true,
            benchmark_run_id: 'single-run',
        }),
    });
    await firstStart;

    expect(fetchMock).toHaveBeenCalledOnce();
    expect(harness.getCurrentBenchmarkId()).toBe('single-run');
    expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
});