Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

19.9 KB
Raw Blame
/**
 * Regression tests for the chunk-overlap=0 truthy-check bug.
 *
 * The page used to write `if (settings.chunk_overlap) { ... }` to hydrate
 * the chunk-overlap input from `/library/api/rag/settings`. Because `0`
 * is falsy in JavaScript, a saved overlap of 0 was dropped and the HTML
 * default (value="200") leaked through, making the form lie to the user.
 * The same `value || default` collapse existed in the change-tracking
 * `originalValues` snapshot and in `refreshSavedDefaults`, so an unchanged
 * blur could spuriously fire a save and the "Saved Defaults" panel could
 * flash 200 after a save of 0.
 *
 * These tests exercise the literal production hydration code against a
 * DOM constructed from `embedding_settings.html` and stub the API surface
 * via `safeFetchWithAuth`, so any regression of the truthy-check pattern
 * for numeric settings will fail loudly.
 *
 * Note: the categorical string fields (provider, model, splitter_type,
 * distance_metric, index_type) keep their plain `if (settings.X)` truthy
 * checks because their valid values are always non-empty strings —
 * `0`/`""`/`false` are not legitimate saved values for them, so the
 * narrower fix is correct. `normalize_vectors` already used
 * `!== undefined`. Only the numeric fields are vulnerable.
 */

// --- Per-test mutable API response ------------------------------------------
const apiState = {
    ragSettings: null,
    ollamaUrl: '',
    ollamaNumCtx: 8192,
    openaiBatchSize: 5,
    putCalls: [],
};

function jsonResponse(body) {
    return {
        ok: true,
        status: 200,
        json: () => Promise.resolve(body),
    };
}

globalThis.safeFetchWithAuth = vi.fn(async (url, opts = {}) => {
    if (opts.method === 'PUT' && url.startsWith('/settings/api/')) {
        const body = opts.body ? JSON.parse(opts.body) : {};
        apiState.putCalls.push({ url, body });
        const key = url.replace(/^\/settings\/api\//, '');
        if (key === 'embeddings.ollama.url') apiState.ollamaUrl = body.value;
        if (key === 'embeddings.ollama.num_ctx') apiState.ollamaNumCtx = body.value;
        if (key === 'embeddings.openai.chunk_size') apiState.openaiBatchSize = body.value;
        return jsonResponse({ success: true });
    }
    if (url === '/library/api/rag/models') {
        return jsonResponse({
            success: true,
            provider_options: [
                { value: 'openai', label: 'OpenAI', available: true },
                { value: 'sentence_transformers', label: 'ST', available: true },
                { value: 'ollama', label: 'Ollama', available: true },
            ],
            providers: {
                openai: [
                    { value: 'text-embedding-nomic-embed-text-v1.5@q8_0', label: 'nomic', is_embedding: true },
                ],
                sentence_transformers: [
                    { value: 'all-MiniLM-L6-v2', label: 'mini', is_embedding: true },
                ],
                ollama: [
                    { value: 'nomic-embed-text', label: 'nomic-embed', is_embedding: true },
                ],
            },
        });
    }
    if (url === '/library/api/rag/settings') {
        return jsonResponse({ success: true, settings: apiState.ragSettings });
    }
    if (url === '/settings/api/embeddings.ollama.url') {
        return jsonResponse({ value: apiState.ollamaUrl });
    }
    if (url === '/settings/api/embeddings.ollama.num_ctx') {
        return jsonResponse({ value: apiState.ollamaNumCtx });
    }
    if (url === '/settings/api/embeddings.openai.chunk_size') {
        return jsonResponse({ value: apiState.openaiBatchSize });
    }
    return jsonResponse({ success: true });
});

// --- DOM scaffolding matching the production template ----------------------
function buildDom() {
    document.body.innerHTML = `
        <div class="ldr-library-container">
            <div id="saved-default-settings"></div>
            <select id="embedding-provider"></select>
            <select id="embedding-model"></select>
            <input type="number" id="chunk-size" value="1000">
            <input type="number" id="chunk-overlap" value="200">
            <select id="splitter-type"></select>
            <select id="distance-metric"></select>
            <select id="index-type"></select>
            <input type="checkbox" id="normalize-vectors">
            <textarea id="text-separators"></textarea>
            <div id="ollama-url-group"><input id="ollama-url"></div>
            <div id="ollama-num-ctx-group"><input type="number" id="ollama-num-ctx" value="8192"></div>
            <div id="openai-embedding-batch-size-group">
                <input type="number" id="openai-embedding-batch-size" value="5">
                <div id="openai-embedding-batch-size-error"></div>
            </div>
            <button id="test-config-btn"></button>
            <div id="provider-info"></div>
            <div id="model-description"></div>
            <div id="text-separators-error"></div>
        </div>
    `;
    for (const id of ['embedding-provider', 'splitter-type', 'distance-metric', 'index-type']) {
        document.getElementById(id).innerHTML = `
            <option value="openai">OpenAI</option>
            <option value="sentence_transformers">Sentence Transformers</option>
            <option value="ollama">Ollama</option>
            <option value="recursive">Recursive</option>
            <option value="token">Token</option>
            <option value="sentence">Sentence</option>
            <option value="semantic">Semantic</option>
            <option value="cosine">Cosine</option>
            <option value="l2">Euclidean</option>
            <option value="dot_product">Dot Product</option>
            <option value="flat">Flat</option>
            <option value="hnsw">HNSW</option>
            <option value="ivf">IVF</option>
        `;
    }
    document.getElementById('embedding-model').innerHTML = `
        <option value="all-MiniLM-L6-v2">all-MiniLM-L6-v2</option>
        <option value="text-embedding-nomic-embed-text-v1.5@q8_0">nomic</option>
        <option value="nomic-embed-text">nomic-embed</option>
    `;
}

function defaultSettings(overrides = {}) {
    return {
        embedding_provider: 'openai',
        embedding_model: 'text-embedding-nomic-embed-text-v1.5@q8_0',
        chunk_size: 1000,
        chunk_overlap: 0, // the bug case
        splitter_type: 'token',
        distance_metric: 'cosine',
        index_type: 'flat',
        normalize_vectors: true,
        text_separators: ['\n\n', '\n', '. ', ' ', ''],
        ...overrides,
    };
}

// --- Module setup ----------------------------------------------------------
globalThis.window.escapeHtml = (s) => String(s ?? '');

// Re-import the module on every reload so module-level state
// (`autoSaveListenersAttached`, `originalValues`, `providerOptions`,
// `availableModels`) is reset to a fresh page load. Without this the
// guard at the top of `attachAutoSaveListeners()` short-circuits on
// subsequent reloads and no event listeners are wired up.
async function loadModuleFresh() {
    vi.resetModules();
    // eslint-disable-next-line no-unsanitized/method
    return import('@js/embedding_settings.js?ts=' + Date.now() + Math.random());
}

async function reloadPage() {
    buildDom();
    await loadModuleFresh();
    document.dispatchEvent(new Event('DOMContentLoaded'));
    // Let the awaited chain drain: loadAvailableModels -> loadCurrentSettings
    // -> loadOllamaUrl -> loadOllamaNumCtx -> toggleProviderFields ->
    // renderSavedDefaults -> attachAutoSaveListeners.
    for (let i = 0; i < 25; i++) {
        await new Promise((r) => setTimeout(r, 0));
    }
}

beforeEach(async () => {
    apiState.ragSettings = defaultSettings();
    apiState.putCalls.length = 0;
    await reloadPage();
});

afterEach(() => {
    vi.restoreAllMocks();
    buildDom();
});

// ============================================================================
// Bug-class: settings.chunk_overlap === 0 must populate the input.
// ============================================================================
describe('loadCurrentSettings — numeric settings hydrate inputs', () => {
    it('chunk_overlap=0: input reflects 0, not the HTML default of 200', () => {
        apiState.ragSettings = defaultSettings({ chunk_overlap: 0 });
        buildDom();
        document.dispatchEvent(new Event('DOMContentLoaded'));
        return new Promise((r) => setTimeout(r, 0))
            .then(() => new Promise((r) => setTimeout(r, 0)))
            .then(() => {
                const overlap = document.getElementById('chunk-overlap');
                expect(overlap.value).toBe('0');
                expect(Number(overlap.value)).toBe(0);
            });
    });

    it('chunk_overlap=200: input reflects 200 (no regression)', () => {
        apiState.ragSettings = defaultSettings({ chunk_overlap: 200 });
        buildDom();
        document.dispatchEvent(new Event('DOMContentLoaded'));
        return new Promise((r) => setTimeout(r, 0))
            .then(() => new Promise((r) => setTimeout(r, 0)))
            .then(() => {
                expect(document.getElementById('chunk-overlap').value).toBe('200');
            });
    });

    it('chunk_overlap missing from API: input keeps the HTML default of 200', () => {
        const settings = defaultSettings();
        delete settings.chunk_overlap;
        apiState.ragSettings = settings;
        buildDom();
        document.dispatchEvent(new Event('DOMContentLoaded'));
        return new Promise((r) => setTimeout(r, 0))
            .then(() => new Promise((r) => setTimeout(r, 0)))
            .then(() => {
                expect(document.getElementById('chunk-overlap').value).toBe('200');
            });
    });

    it('chunk_overlap=null from API: input keeps the HTML default of 200', () => {
        apiState.ragSettings = defaultSettings({ chunk_overlap: null });
        buildDom();
        document.dispatchEvent(new Event('DOMContentLoaded'));
        return new Promise((r) => setTimeout(r, 0))
            .then(() => new Promise((r) => setTimeout(r, 0)))
            .then(() => {
                expect(document.getElementById('chunk-overlap').value).toBe('200');
            });
    });

    it('chunk_size=0: input reflects 0 (defensive parity with chunk_overlap)', () => {
        apiState.ragSettings = defaultSettings({ chunk_size: 0 });
        buildDom();
        document.dispatchEvent(new Event('DOMContentLoaded'));
        return new Promise((r) => setTimeout(r, 0))
            .then(() => new Promise((r) => setTimeout(r, 0)))
            .then(() => {
                expect(document.getElementById('chunk-size').value).toBe('0');
            });
    });
});

// ============================================================================
// Categorical string fields keep their truthy checks (they're never falsy).
// ============================================================================
describe('loadCurrentSettings — categorical fields round-trip correctly', () => {
    it('provider/model/splitter/distance/index round-trip through truthy check', () => {
        apiState.ragSettings = defaultSettings({
            embedding_provider: 'sentence_transformers',
            embedding_model: 'all-MiniLM-L6-v2',
            splitter_type: 'semantic',
            distance_metric: 'l2',
            index_type: 'hnsw',
        });
        buildDom();
        document.dispatchEvent(new Event('DOMContentLoaded'));
        return new Promise((r) => setTimeout(r, 0))
            .then(() => new Promise((r) => setTimeout(r, 0)))
            .then(() => {
                expect(document.getElementById('embedding-provider').value).toBe('sentence_transformers');
                expect(document.getElementById('embedding-model').value).toBe('all-MiniLM-L6-v2');
                expect(document.getElementById('splitter-type').value).toBe('semantic');
                expect(document.getElementById('distance-metric').value).toBe('l2');
                expect(document.getElementById('index-type').value).toBe('hnsw');
            });
    });

    it('normalize_vectors=false: checkbox correctly unchecked', () => {
        apiState.ragSettings = defaultSettings({ normalize_vectors: false });
        buildDom();
        document.dispatchEvent(new Event('DOMContentLoaded'));
        return new Promise((r) => setTimeout(r, 0))
            .then(() => new Promise((r) => setTimeout(r, 0)))
            .then(() => {
                expect(document.getElementById('normalize-vectors').checked).toBe(false);
            });
    });
});

// ============================================================================
// Change-tracking: originalValues snapshot must preserve 0 so an unchanged
// blur is correctly detected as a no-op.
// ============================================================================
describe('change-tracking — originalValues snapshot preserves 0', () => {
    it('blurring the chunk-overlap input without changing it does NOT PUT', async () => {
        // Page loaded in beforeEach with chunk_overlap=0; input shows "0".
        const overlap = document.getElementById('chunk-overlap');
        expect(overlap.value).toBe('0');

        const baselinePuts = apiState.putCalls.filter((c) =>
            c.url === '/settings/api/local_search_chunk_overlap'
        ).length;

        overlap.focus();
        overlap.dispatchEvent(new Event('blur'));
        await new Promise((r) => setTimeout(r, 300));

        const afterPuts = apiState.putCalls.filter((c) =>
            c.url === '/settings/api/local_search_chunk_overlap'
        ).length;
        expect(afterPuts).toBe(baselinePuts);
    });

    it('blurring the chunk-size input without changing it does NOT PUT', async () => {
        const size = document.getElementById('chunk-size');
        expect(size.value).toBe('1000');

        const baselinePuts = apiState.putCalls.filter((c) =>
            c.url === '/settings/api/local_search_chunk_size'
        ).length;

        size.focus();
        size.dispatchEvent(new Event('blur'));
        await new Promise((r) => setTimeout(r, 300));

        const afterPuts = apiState.putCalls.filter((c) =>
            c.url === '/settings/api/local_search_chunk_size'
        ).length;
        expect(afterPuts).toBe(baselinePuts);
    });

    it('listener is actually attached: a real change DOES PUT (sanity check)', async () => {
        // Catches a broken test setup where the listener was never wired,
        // which would make the no-op tests above false-pass.
        const overlap = document.getElementById('chunk-overlap');
        expect(overlap.value).toBe('0');

        apiState.putCalls.length = 0;
        overlap.focus();
        overlap.value = '75';
        overlap.dispatchEvent(new Event('blur'));
        await new Promise((r) => setTimeout(r, 300));

        const changePuts = apiState.putCalls.filter((c) =>
            c.url === '/settings/api/local_search_chunk_overlap'
        );
        expect(changePuts).toHaveLength(1);
        expect(changePuts[0].body.value).toBe(75);
    });
});

// ============================================================================
// Source-code guard: the truthy-collapse pattern must not regress.
// ============================================================================
describe('source-code guard against truthy-check regression', () => {
    const { readFileSync } = require('node:fs');
    const { resolve } = require('node:path');
    const SRC = readFileSync(
        resolve(
            __dirname,
            '../../src/local_deep_research/web/static/js/embedding_settings.js',
        ),
        'utf8',
    );

    it('loadCurrentSettings hydrates chunk_size / chunk_overlap via nullish check', () => {
        const start = SRC.indexOf('async function loadCurrentSettings');
        const end = SRC.indexOf('async function renderSavedDefaults');
        const body = start >= 0 && end > start ? SRC.slice(start, end) : SRC;
        // The numeric fields that have a legitimate 0 value MUST use the
        // nullish form — a re-introduction of `if (settings.chunk_overlap)`
        // is the original bug.
        expect(body).not.toMatch(/if\s*\(\s*settings\.chunk_overlap\s*\)/);
        expect(body).not.toMatch(/if\s*\(\s*settings\.chunk_size\s*\)/);
    });

    it('categorical fields keep their truthy check (they\'re never falsy)', () => {
        // Documenting the intentional asymmetry: the categorical fields
        // use `if (settings.X)` because valid values are non-empty
        // strings. A future refactor that switches them to nullish
        // checks is fine (and would make the test in the previous
        // block cover them) but flipping them BACK to truthy while also
        // changing the schema to allow empty values would re-introduce
        // the bug class. Pin the intent here.
        const start = SRC.indexOf('async function loadCurrentSettings');
        const end = SRC.indexOf('async function renderSavedDefaults');
        const body = start >= 0 && end > start ? SRC.slice(start, end) : SRC;
        expect(body).toMatch(/if\s*\(\s*settings\.embedding_provider\s*\)/);
        expect(body).toMatch(/if\s*\(\s*settings\.embedding_model\s*\)/);
        expect(body).toMatch(/if\s*\(\s*settings\.splitter_type\s*\)/);
        expect(body).toMatch(/if\s*\(\s*settings\.distance_metric\s*\)/);
        expect(body).toMatch(/if\s*\(\s*settings\.index_type\s*\)/);
        expect(body).toMatch(/if\s*\(\s*settings\.text_separators\s*\)/);
    });

    it('originalValues / refreshSavedDefaults do not use `||` as a parseInt fallback (the bug class)', () => {
        // The bug is `parseInt(x) || Y` — parseInt("0") is 0, and `0 || Y`
        // evaluates to Y, dropping the legitimate 0. The fix uses
        // `Number.isNaN(v) ? Y : v` instead.
        //
        // The regex matches `parseInt...||` on the same line, requiring the
        // `||` to come AFTER parseInt (not before, which is the common shape
        // of unrelated `data.value === null || (...)` checks).
        const offendingLines = SRC.split('\n').filter(
            (line) => /parseInt[^\n]*\|\|/.test(line),
        );
        expect(offendingLines).toEqual([]);
    });

    it('renderSavedDefaults does not hardcode categorical fallback strings (HTML is the source)', () => {
        // The categorical fallbacks ('recursive', 'cosine', 'flat') must come
        // from the <select>'s first option, not from a hardcoded literal.
        // If someone changes the HTML default to a new value, the JS fallback
        // should follow automatically — otherwise the saved-defaults panel
        // could lie about the actual default.
        //
        // Extract renderSavedDefaults function body so we don't catch the
        // same string literals used elsewhere (e.g. in tests or comments).
        const start = SRC.indexOf('function renderSavedDefaults(');
        if (start < 0) throw new Error('renderSavedDefaults not found');
        let depth = 0;
        let end = start;
        for (let i = start; i < SRC.length; i++) {
            if (SRC[i] === '{') depth++;
            else if (SRC[i] === '}') {
                depth--;
                if (depth === 0) {
                    end = i + 1;
                    break;
                }
            }
        }
        const body = SRC.slice(start, end);

        // `|| 'recursive'`, `|| "cosine"`, `|| 'flat'` are the bug pattern.
        expect(body).not.toMatch(/\|\|\s*['"]recursive['"]/);
        expect(body).not.toMatch(/\|\|\s*['"]cosine['"]/);
        expect(body).not.toMatch(/\|\|\s*['"]flat['"]/);
    });

    it('ollama.num_ctx and openai batch size use HTML defaultValue, not hardcoded literals', () => {
        // The numeric fallbacks (8192, 5) must come from the HTML defaultValue,
        // not from a hardcoded literal. The previous bug had similar
        // hardcoded values for chunk_size / chunk_overlap — keep that pattern
        // consistent across all settings.
        const start = SRC.indexOf('originalValues = {');
        const end = SRC.indexOf('};', start) + 2;
        const body = SRC.slice(start, end);
        expect(body).not.toMatch(/\)\s*:\s*8192/);
        expect(body).not.toMatch(/\)\s*:\s*5/);
    });
});