Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

19.9 KB
Raw Blame
/**
 * Runtime contracts for the simple benchmark page's inline start workflow.
 *
 * These tests execute the checked-in template function so the FastAPI route,
 * CSRF header, selected-dataset payload, and recovery behavior stay aligned.
 */

import { readFileSync } from 'node:fs';
import { resolve } from 'node:path';

const TEMPLATE_PATH = resolve(
    __dirname,
    '../../../src/local_deep_research/web/templates/pages/benchmark_simple.html',
);

function extractFunction(source, name) {
    const signature = new RegExp(`(?:async\\s+)?function\\s+${name}\\s*\\(`);
    const match = signature.exec(source);
    if (!match) throw new Error(`Function ${name} not found in template`);

    const openBrace = source.indexOf('{', match.index + match[0].length);
    let depth = 0;
    for (let index = openBrace; index < source.length; index += 1) {
        if (source[index] === '{') depth += 1;
        if (source[index] === '}') {
            depth -= 1;
            if (depth === 0) return source.slice(match.index, index + 1);
        }
    }

    throw new Error(`Function ${name} has an unterminated body`);
}

function compileStartBenchmark(dependencies) {
    const template = readFileSync(TEMPLATE_PATH, 'utf8');
    const startBenchmarkSource = extractFunction(template, 'startBenchmark');
    const dependencyNames = Object.keys(dependencies);
    // The extracted source is repository-owned production code from the
    // template above, not user-controlled input.
    const factory = new Function( // eslint-disable-line no-new-func
        ...dependencyNames,
        `
            let currentBenchmarkId = null;
            ${startBenchmarkSource}
            return {
                startBenchmark,
                getCurrentBenchmarkId: () => currentBenchmarkId,
            };
        `,
    );
    return factory(...Object.values(dependencies));
}

function renderBenchmarkSelection() {
    document.head.innerHTML = '<meta name="csrf-token" content="csrf-3299">';
    document.body.innerHTML = `
        <div class="ldr-dataset-option ldr-selected" data-dataset="simpleqa">
            <input value="3">
        </div>
        <div class="ldr-dataset-option ldr-selected" data-dataset="browsecomp">
            <input value="0">
        </div>
        <button id="start-benchmark"></button>
    `;
}

function createDependencies() {
    return {
        showAlert: vi.fn(),
        showProgress: vi.fn(),
        startProgressTracking: vi.fn(),
    };
}

function deferred() {
    let resolvePromise;
    const promise = new Promise(resolvePromiseArgument => {
        resolvePromise = resolvePromiseArgument;
    });
    return { promise, resolve: resolvePromise };
}

function compileProgressRuntime(dependencies) {
    const template = readFileSync(TEMPLATE_PATH, 'utf8');
    const functions = [
        'startProgressTracking',
        'formatBenchmarkAccuracy',
        'updateProgress',
    ]
        .map(name => extractFunction(template, name))
        .join('\n');
    const dependencyNames = Object.keys(dependencies);
    const factory = new Function( // eslint-disable-line no-new-func
        ...dependencyNames,
        `
            let currentBenchmarkId = 'run-a';
            let progressInterval = null;
            let progressRunGeneration = 0;
            let progressStatusRequestId = 0;
            let latestAppliedProgressStatusRequestId = 0;
            let terminalProgressRunGeneration = null;
            ${functions}
            return {
                startProgressTracking,
                updateProgress,
                setCurrentBenchmarkId: value => { currentBenchmarkId = value; },
                getCurrentBenchmarkId: () => currentBenchmarkId,
                getProgressInterval: () => progressInterval,
            };
        `,
    );
    return factory(...Object.values(dependencies));
}

afterEach(() => {
    vi.restoreAllMocks();
    vi.unstubAllGlobals();
    vi.useRealTimers();
    document.head.replaceChildren();
    document.body.replaceChildren();
});

it('starts the selected datasets through the FastAPI simple-benchmark route', async () => {
    renderBenchmarkSelection();
    const fetchMock = vi.fn().mockResolvedValue({
        ok: true,
        json: vi.fn().mockResolvedValue({
            success: true,
            benchmark_run_id: 'run-3299',
        }),
    });
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = createDependencies();
    const harness = compileStartBenchmark(dependencies);

    await harness.startBenchmark();

    expect(fetchMock).toHaveBeenCalledOnce();
    expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/start-simple', {
        method: 'POST',
        headers: {
            'Content-Type': 'application/json',
            'X-CSRFToken': 'csrf-3299',
        },
        body: JSON.stringify({
            datasets_config: { simpleqa: { count: 3 } },
        }),
    });
    expect(harness.getCurrentBenchmarkId()).toBe('run-3299');
    expect(dependencies.showProgress).toHaveBeenCalledOnce();
    expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Benchmark started!',
        'success',
    );
});

it('does not call the backend when every selected dataset has a zero count', async () => {
    renderBenchmarkSelection();
    document.querySelector('[data-dataset="simpleqa"] input').value = '0';
    const fetchMock = vi.fn();
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = createDependencies();

    await compileStartBenchmark(dependencies).startBenchmark();

    expect(fetchMock).not.toHaveBeenCalled();
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Please select at least one dataset',
        'error',
    );
    expect(document.getElementById('start-benchmark').disabled).toBe(false);
});

it('reenables the start action when the backend rejects the configuration', async () => {
    renderBenchmarkSelection();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
        ok: true,
        json: vi.fn().mockResolvedValue({
            success: false,
            error: 'dataset unavailable',
        }),
    }));
    const dependencies = createDependencies();

    await compileStartBenchmark(dependencies).startBenchmark();

    expect(dependencies.showProgress).not.toHaveBeenCalled();
    expect(dependencies.startProgressTracking).not.toHaveBeenCalled();
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Error: dataset unavailable',
        'error',
    );
    expect(document.getElementById('start-benchmark').disabled).toBe(false);
});

it.each([
    {
        label: 'FastAPI error envelope',
        response: {
            ok: false,
            status: 409,
            json: vi.fn().mockResolvedValue({
                error: 'benchmark worker unavailable',
            }),
        },
        expectedError: 'Failed to start benchmark: benchmark worker unavailable',
    },
    {
        label: 'FastAPI detail envelope',
        response: {
            ok: false,
            status: 422,
            json: vi.fn().mockResolvedValue({
                detail: 'dataset configuration is invalid',
            }),
        },
        expectedError: 'Failed to start benchmark: dataset configuration is invalid',
    },
    {
        label: 'FastAPI message envelope',
        response: {
            ok: false,
            status: 503,
            json: vi.fn().mockResolvedValue({
                message: 'benchmark queue is paused',
            }),
        },
        expectedError: 'Failed to start benchmark: benchmark queue is paused',
    },
    {
        label: 'non-ok success-shaped response',
        response: {
            ok: false,
            status: 503,
            json: vi.fn().mockResolvedValue({
                success: true,
                benchmark_run_id: 'must-not-start',
            }),
        },
        expectedError: 'Failed to start benchmark: HTTP 503',
    },
    {
        label: 'success response without a run ID',
        response: {
            ok: true,
            status: 200,
            json: vi.fn().mockResolvedValue({ success: true }),
        },
        expectedError: 'Error: response did not include a benchmark run ID',
    },
])('rejects a $label from the simple start endpoint', async ({
    response,
    expectedError,
}) => {
    renderBenchmarkSelection();
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response));
    const dependencies = createDependencies();
    const harness = compileStartBenchmark(dependencies);

    await harness.startBenchmark();

    expect(harness.getCurrentBenchmarkId()).toBeNull();
    expect(dependencies.showProgress).not.toHaveBeenCalled();
    expect(dependencies.startProgressTracking).not.toHaveBeenCalled();
    expect(dependencies.showAlert).toHaveBeenCalledWith(expectedError, 'error');
    expect(document.getElementById('start-benchmark').disabled).toBe(false);
});

it('coalesces duplicate simple starts while the first POST is pending', async () => {
    renderBenchmarkSelection();
    const pendingStart = deferred();
    const fetchMock = vi.fn(() => pendingStart.promise);
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = createDependencies();
    const harness = compileStartBenchmark(dependencies);

    const firstStart = harness.startBenchmark();
    await harness.startBenchmark();

    expect(fetchMock).toHaveBeenCalledOnce();
    pendingStart.resolve({
        ok: true,
        json: vi.fn().mockResolvedValue({
            success: true,
            benchmark_run_id: 'simple-single-run',
        }),
    });
    await firstStart;

    expect(fetchMock).toHaveBeenCalledOnce();
    expect(harness.getCurrentBenchmarkId()).toBe('simple-single-run');
    expect(dependencies.startProgressTracking).toHaveBeenCalledOnce();
});

it('ignores a terminal response from an older run after a newer run starts', async () => {
    vi.useFakeTimers();
    document.body.innerHTML = `
        <div id="progress-fill" style="width: 0%"></div>
        <span id="stat-accuracy">--%</span>
        <span id="stat-completed">0</span>
        <span id="stat-rate">--</span>
        <span id="stat-remaining">--</span>
    `;
    const staleStatus = deferred();
    const fetchMock = vi.fn(url => {
        if (url === '/benchmark/api/status/run-a') return staleStatus.promise;
        return new Promise(() => {});
    });
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = {
        showAlert: vi.fn(),
        hideProgress: vi.fn(),
    };
    const harness = compileProgressRuntime(dependencies);

    const staleRequest = harness.startProgressTracking();
    expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/status/run-a');
    harness.setCurrentBenchmarkId('run-b');
    harness.startProgressTracking();
    const runBInterval = harness.getProgressInterval();
    expect(fetchMock).toHaveBeenCalledWith('/benchmark/api/status/run-b');

    staleStatus.resolve({
        json: vi.fn().mockResolvedValue({
            success: true,
            status: {
                status: 'completed',
                completed_examples: 5,
                total_examples: 5,
                overall_accuracy: 90,
                processing_rate: 2,
            },
        }),
    });
    await staleRequest;

    expect(document.getElementById('progress-fill').style.width).toBe('0%');
    expect(dependencies.showAlert).not.toHaveBeenCalled();
    expect(dependencies.hideProgress).not.toHaveBeenCalled();
    expect(harness.getProgressInterval()).toBe(runBInterval);
    clearInterval(runBInterval);
    vi.useRealTimers();
});

it('ignores an older same-run poll after a newer status renders', async () => {
    document.body.innerHTML = `
        <div id="progress-fill" style="width: 0%"></div>
        <span id="stat-accuracy">--%</span>
        <span id="stat-completed">0</span>
        <span id="stat-rate">--</span>
        <span id="stat-remaining">--</span>
    `;
    const olderStatus = deferred();
    const newerStatus = deferred();
    vi.stubGlobal('fetch', vi.fn()
        .mockImplementationOnce(() => olderStatus.promise)
        .mockImplementationOnce(() => newerStatus.promise));
    const dependencies = {
        showAlert: vi.fn(),
        hideProgress: vi.fn(),
    };
    const harness = compileProgressRuntime(dependencies);

    const olderRequest = harness.startProgressTracking();
    const newerRequest = harness.updateProgress();
    const currentStatus = {
        status: 'in_progress',
        completed_examples: 6,
        total_examples: 10,
        overall_accuracy: 80,
        processing_rate: 2,
    };
    newerStatus.resolve({
        json: vi.fn().mockResolvedValue({ success: true, status: currentStatus }),
    });
    await newerRequest;
    expect(document.getElementById('progress-fill').style.width).toBe('60%');

    olderStatus.resolve({
        json: vi.fn().mockResolvedValue({
            success: true,
            status: {
                status: 'in_progress',
                completed_examples: 2,
                total_examples: 10,
                overall_accuracy: 10,
                processing_rate: 2,
            },
        }),
    });
    await olderRequest;

    expect(document.getElementById('progress-fill').style.width).toBe('60%');
    expect(document.getElementById('stat-accuracy').textContent).toBe('80.0%');
    expect(dependencies.showAlert).not.toHaveBeenCalled();
    clearInterval(harness.getProgressInterval());
});

it('accepts an older terminal snapshot after a newer nonterminal poll', async () => {
    vi.useFakeTimers();
    document.body.innerHTML = `
        <div id="progress-fill" style="width: 0%"></div>
        <span id="stat-accuracy">--%</span>
        <span id="stat-completed">0</span>
        <span id="stat-rate">--</span>
        <span id="stat-remaining">--</span>
    `;
    const terminalResponse = deferred();
    const nonterminalResponse = deferred();
    vi.stubGlobal('fetch', vi.fn()
        .mockImplementationOnce(() => terminalResponse.promise)
        .mockImplementationOnce(() => nonterminalResponse.promise));
    const dependencies = {
        showAlert: vi.fn(),
        hideProgress: vi.fn(),
    };
    const harness = compileProgressRuntime(dependencies);

    const terminalLoad = harness.updateProgress();
    const nonterminalLoad = harness.updateProgress();
    nonterminalResponse.resolve({
        json: vi.fn().mockResolvedValue({
            success: true,
            status: {
                status: 'in_progress',
                completed_examples: 8,
                total_examples: 10,
                overall_accuracy: 80,
                processing_rate: 2,
            },
        }),
    });
    await nonterminalLoad;

    terminalResponse.resolve({
        json: vi.fn().mockResolvedValue({
            success: true,
            status: {
                status: 'completed',
                completed_examples: 10,
                total_examples: 10,
                overall_accuracy: 90,
                processing_rate: 2,
            },
        }),
    });
    await terminalLoad;

    expect(document.getElementById('progress-fill').style.width).toBe('100%');
    expect(document.getElementById('stat-accuracy').textContent).toBe('90.0%');
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Benchmark completed! Final accuracy: 90.0%',
        'success',
    );
    vi.clearAllTimers();
    vi.useRealTimers();
});

it('keeps the first applied terminal snapshot authoritative for a run', async () => {
    vi.useFakeTimers();
    document.body.innerHTML = `
        <div id="progress-fill" style="width: 0%"></div>
        <span id="stat-accuracy">--%</span>
        <span id="stat-completed">0</span>
        <span id="stat-rate">--</span>
        <span id="stat-remaining">--</span>
    `;
    const olderTerminal = deferred();
    const newerTerminal = deferred();
    vi.stubGlobal('fetch', vi.fn()
        .mockImplementationOnce(() => olderTerminal.promise)
        .mockImplementationOnce(() => newerTerminal.promise));
    const dependencies = {
        showAlert: vi.fn(),
        hideProgress: vi.fn(),
    };
    const harness = compileProgressRuntime(dependencies);

    const olderLoad = harness.updateProgress();
    const newerLoad = harness.updateProgress();
    newerTerminal.resolve({
        json: vi.fn().mockResolvedValue({
            success: true,
            status: {
                status: 'completed',
                completed_examples: 10,
                total_examples: 10,
                overall_accuracy: 92,
                processing_rate: 2,
            },
        }),
    });
    await newerLoad;

    olderTerminal.resolve({
        json: vi.fn().mockResolvedValue({
            success: true,
            status: {
                status: 'failed',
                completed_examples: 3,
                total_examples: 10,
                overall_accuracy: 20,
                processing_rate: 1,
            },
        }),
    });
    await olderLoad;

    expect(document.getElementById('progress-fill').style.width).toBe('100%');
    expect(document.getElementById('stat-accuracy').textContent).toBe('92.0%');
    expect(dependencies.showAlert).toHaveBeenCalledOnce();
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Benchmark completed! Final accuracy: 92.0%',
        'success',
    );

    await vi.advanceTimersByTimeAsync(3000);
    expect(dependencies.hideProgress).toHaveBeenCalledOnce();
});

it('completes and schedules cleanup when final accuracy is unavailable', async () => {
    vi.useFakeTimers();
    document.body.innerHTML = `
        <div id="progress-fill" style="width: 0%"></div>
        <span id="stat-accuracy">--%</span>
        <span id="stat-completed">0</span>
        <span id="stat-rate">--</span>
        <span id="stat-remaining">--</span>
    `;
    vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
        json: vi.fn().mockResolvedValue({
            success: true,
            status: {
                status: 'completed',
                completed_examples: 0,
                total_examples: 0,
                overall_accuracy: null,
                processing_rate: 0,
            },
        }),
    }));
    const dependencies = {
        showAlert: vi.fn(),
        hideProgress: vi.fn(),
    };
    const harness = compileProgressRuntime(dependencies);

    await harness.updateProgress();

    expect(document.getElementById('stat-accuracy').textContent).toBe('N/A');
    expect(dependencies.showAlert).toHaveBeenCalledWith(
        'Benchmark completed! Final accuracy: N/A',
        'success',
    );

    await vi.advanceTimersByTimeAsync(3000);
    expect(dependencies.hideProgress).toHaveBeenCalledOnce();
    expect(harness.getCurrentBenchmarkId()).toBeNull();
});

it('does not let an old terminal cleanup hide a replacement run', async () => {
    vi.useFakeTimers();
    document.body.innerHTML = `
        <div id="progress-fill" style="width: 0%"></div>
        <span id="stat-accuracy">--%</span>
        <span id="stat-completed">0</span>
        <span id="stat-rate">--</span>
        <span id="stat-remaining">--</span>
    `;
    const fetchMock = vi.fn()
        .mockResolvedValueOnce({
            json: vi.fn().mockResolvedValue({
                success: true,
                status: {
                    status: 'completed',
                    completed_examples: 4,
                    total_examples: 4,
                    overall_accuracy: 75,
                    processing_rate: 2,
                },
            }),
        })
        .mockImplementation(() => new Promise(() => {}));
    vi.stubGlobal('fetch', fetchMock);
    const dependencies = {
        showAlert: vi.fn(),
        hideProgress: vi.fn(),
    };
    const harness = compileProgressRuntime(dependencies);

    await harness.updateProgress();
    harness.setCurrentBenchmarkId('run-b');
    harness.startProgressTracking();
    const replacementInterval = harness.getProgressInterval();

    await vi.advanceTimersByTimeAsync(3000);

    expect(dependencies.hideProgress).not.toHaveBeenCalled();
    expect(harness.getCurrentBenchmarkId()).toBe('run-b');
    expect(harness.getProgressInterval()).toBe(replacementInterval);
    clearInterval(replacementInterval);
});