Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

5.2 KB
Raw Blame
import { EventEmitter } from 'node:events';
import { readFileSync } from 'node:fs';
import path from 'node:path';
import { clearTimeout, setTimeout } from 'node:timers';
import { fileURLToPath } from 'node:url';
import { runInNewContext } from 'node:vm';
import { describe, expect, it, vi } from 'vitest';

const runnerPath = path.resolve(
    path.dirname(fileURLToPath(import.meta.url)), '../ui_tests/run_all_tests.js'
);
const runnerSource = readFileSync(runnerPath, 'utf8');
const smokeFile = 'test_research_completion_reconnect_ci.js';

// Execute the real runner loop, replacing only its server, child processes,
// console and process boundary. A separate VM keeps argv/env/exit isolated
// from Vitest; appending the function name exposes it without copying logic.
async function runSuite({ args = [], env = {}, failingFile } = {}) {
    const output = [];
    const stopped = new Error('Runner called process.exit');
    const exit = vi.fn(() => { throw stopped; });
    const spawn = vi.fn((_command, [file]) => {
        const child = new EventEmitter();
        Promise.resolve().then(() => child.emit('close', file === failingFile ? 1 : 0, null));
        return child;
    });
    const get = vi.fn((_url, _options, onResponse) => {
        onResponse({ statusCode: 200, resume() {} });
        return new EventEmitter();
    });
    const modules = { child_process: { spawn }, http: { get }, path };
    const runAllTests = runInNewContext(`${runnerSource}\nrunAllTests;`, {
        require(name) {
            if (!Object.hasOwn(modules, name)) throw new Error(`Unexpected require: ${name}`);
            return modules[name];
        },
        module: { exports: {} },
        __dirname: path.dirname(runnerPath),
        process: { argv: ['node', runnerPath, ...args], env, exit },
        console: {
            log: (...parts) => output.push(parts.join(' ')),
            error: (...parts) => output.push(parts.join(' ')),
        },
        setTimeout,
        clearTimeout,
    }, { filename: runnerPath });

    await expect(runAllTests()).rejects.toBe(stopped);
    expect(exit).toHaveBeenCalledTimes(1);
    return {
        files: spawn.mock.calls.map(([, [file]]) => file),
        spawn,
        healthChecks: get.mock.calls.length,
        exitCode: exit.mock.calls[0][0],
        output: output.join('\n'),
    };
}

describe('UI test runner shard selection', () => {
    it.each([
        ['without a fixture directory', {}],
        ['with a leftover fixture directory', { LDR_RESEARCH_SMOKE_DIR: '/tmp/research-smoke' }],
    ])('default local run skips the dedicated smoke %s', async (_name, env) => {
        const result = await runSuite({ env, failingFile: smokeFile });

        expect(result.files).not.toContain(smokeFile);
        expect(result.files).toContain('test_auth_flow.js');
        expect(result.files).toContain('test_research_cancellation_ci.js');
        // Existing local-only tests must remain enabled outside CI.
        expect(result.files).toContain('test_research_submit.js');
        expect(result.healthChecks).toBe(result.files.length);
        expect(result.exitCode).toBe(0);
        expect(result.output).toContain('requires --shard=research-completion');
    });

    it.each([
        ['local', {}],
        ['CI', { CI: 'true' }],
    ])('explicit %s smoke shard still runs with its fixture environment', async (_name, env) => {
        const fixtureEnv = { ...env, LDR_RESEARCH_SMOKE_DIR: '/tmp/research-smoke' };
        const result = await runSuite({ args: ['--shard=research-completion'], env: fixtureEnv });

        expect(result.files).toEqual([smokeFile]);
        expect(result.healthChecks).toBe(1);
        expect(result.exitCode).toBe(0);
        expect(result.spawn).toHaveBeenCalledWith('node', [smokeFile], expect.objectContaining({
            cwd: path.dirname(runnerPath),
            env: expect.objectContaining(fixtureEnv),
        }));
    });

    it.each([
        ['local', {}],
        ['CI', { CI: 'true' }],
    ])('explicit %s smoke shard propagates failure even without fixture configuration', async (_name, env) => {
        const result = await runSuite({
            args: ['--shard=research-completion'], env, failingFile: smokeFile,
        });

        expect(result.files).toEqual([smokeFile]);
        expect(result.exitCode).toBe(1);
    });

    it('keeps other CI shards and their local-only exclusions intact', async () => {
        const result = await runSuite({ args: ['--shard=research-workflow'], env: { CI: 'true' } });

        expect(result.files).toContain('test_research_cancellation_ci.js');
        expect(result.files).not.toContain(smokeFile);
        expect(result.files).not.toContain('test_auth_flow.js');
        expect(result.files).not.toContain('test_concurrent_limit.js');
        expect(result.exitCode).toBe(0);
    });

    it.each([
        ['CI without a shard', [], { CI: 'true' }, 'no --shard flag provided'],
        ['an unknown shard', ['--shard=unknown'], {}, 'unknown shard'],
    ])('rejects %s before starting any tests', async (_name, args, env, message) => {
        const result = await runSuite({ args, env });

        expect(result.files).toEqual([]);
        expect(result.healthChecks).toBe(0);
        expect(result.exitCode).toBe(1);
        expect(result.output).toContain(message);
    });
});