Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

42.5 KB
Raw Blame
/**
 * UI Test Suite Runner
 *
 * Runs UI tests in sequence and provides a summary report.
 * This script executes each test individually and tracks pass/fail status.
 *
 * Prerequisites: Web server running on http://127.0.0.1:5000
 * The research-completion shard requires research_completion_server.py and
 * LDR_RESEARCH_SMOKE_DIR, and only runs when explicitly selected.
 *
 * Usage:
 *   node tests/ui_tests/run_all_tests.js                      # default local suite
 *   node tests/ui_tests/run_all_tests.js --shard=auth-login   # run one shard
 *
 * Valid shards: auth-login, auth-register, auth-pages, research-workflow,
 *   research-completion,
 *   research-form, research-metrics, settings-core, settings-pages,
 *   library, history-news, mobile, api-crud, error-benchmark, accessibility,
 *   chat-core, chat-lifecycle, link-analytics
 */

// Keep in sync with `strategy.matrix.shard` in .github/workflows/docker-tests.yml.
// A mismatch would cause silent test misrouting.
//
// Shard design (18 shards, ~4 tests each):
//   Each shard runs in its own Docker container with a dedicated server.
//   Keeping shards small prevents cascade failures when one test stresses
//   the server (e.g., encrypted DB creation in auth-register).
const VALID_SHARDS = [
    'auth-login',          // login/auth flow tests
    'auth-register',       // registration (isolated — heavy SQLCipher DB creation)
    'auth-pages',          // page browsing, navigation, comprehensive auth
    'research-workflow',   // core research lifecycle
    'research-completion', // deterministic real-worker browser qualification
    'research-form',       // research form interactions + results
    'research-metrics',    // metrics charts, dashboard, progress
    'settings-core',       // settings page, errors, save, interactions
    'settings-pages',      // settings tabs, star reviews
    'library',             // collections, documents
    'history-news',        // history page, news subscriptions
    'mobile',              // mobile navigation, interactions, UI functionality
    'api-crud',            // API endpoints, CRUD operations, rate limiting
    'error-benchmark',     // error handling/recovery, benchmark, context overflow
    'accessibility',       // keyboard navigation & ARIA
    'chat-core',           // chat-mode v2 input + a11y + chips + nav
    'chat-lifecycle',      // chat-mode v2 session lifecycle + export + persistence
    'link-analytics',      // /metrics/links full-page render + XSS runtime check
];

const { spawn } = require('child_process');
const http = require('http');
const path = require('path');

/** Format a Date as HH:MM:SS for log timestamps */
function ts(date = new Date()) {
    return date.toISOString().slice(11, 19);
}

/**
 * Wait for the server to be responsive before starting the next test.
 * Prevents cascade failures when a previous test stressed the server
 * (e.g., registration creating encrypted databases).
 */
async function waitForServer(maxWaitMs = 60000) {
    const startTime = Date.now();
    let delay = 1000;
    let wasDown = false;
    while (Date.now() - startTime < maxWaitMs) {
        try {
            const ok = await new Promise((resolve) => {
                const req = http.get('http://127.0.0.1:5000/api/v1/health', { timeout: 5000 }, (res) => {
                    resolve(res.statusCode >= 200 && res.statusCode < 400);
                    res.resume();
                });
                req.on('error', () => resolve(false));
                req.on('timeout', () => { req.destroy(); resolve(false); });
            });
            if (ok) {
                if (wasDown) console.log('Server recovered after being unresponsive');
                return true;
            }
            wasDown = true;
        } catch {
            wasDown = true;
        }
        // Capture into const so the closure isn't flagged as no-loop-func.
        const sleepFor = delay;
        await new Promise(r => setTimeout(r, sleepFor));
        delay = Math.min(delay * 2, 8000);
    }
    console.log(`Server did not respond within ${maxWaitMs/1000}s — subsequent tests may fail`);
    return false;
}

const tests = [
    // =====================================================================
    // Shard: auth-login (2 tests)
    // =====================================================================
    {
        name: 'Authentication Flow Test',
        file: 'test_auth_flow.js',
        shard: 'auth-login',
        description: 'Tests registration, login, and logout functionality'
    },
    {
        name: 'Login Validation Test',
        file: 'test_login_validation.js',
        shard: 'auth-login',
        description: 'Tests login form validation'
    },

    // =====================================================================
    // Shard: auth-register (2 tests)
    // Register Full Flow is isolated because it creates an encrypted
    // SQLCipher database (CPU-intensive key derivation + 58 tables +
    // 500+ settings) which can block the server for 2+ minutes.
    // =====================================================================
    {
        name: 'Register Validation Test',
        file: 'test_register_validation.js',
        shard: 'auth-register',
        description: 'Tests registration form validation without auth'
    },
    {
        name: 'Register Full Flow Test',
        file: 'test_register_full_flow.js',
        shard: 'auth-register',
        description: 'Tests complete registration flow (CPU-heavy SQLCipher DB creation)'
    },

    // =====================================================================
    // Shard: auth-pages (4 tests)
    // =====================================================================
    {
        name: 'Frontend Bundle Integrity',
        file: 'test_frontend_bundle_integrity_ci.js',
        shard: 'auth-pages',
        description:
            'Asserts the Vite bundle is served AND executes. dist/ is a ' +
            'gitignored build artefact; with it missing every page still ' +
            'returns 200 and the rest of the suite still passes, so a ' +
            'broken build would otherwise ship green.'
    },
    {
        name: 'All Pages Browser Test',
        file: 'test_pages_browser.js',
        shard: 'auth-pages',
        description: 'Tests all main pages for basic functionality'
    },
    {
        name: 'Full Navigation Test',
        file: 'test_full_navigation.js',
        shard: 'auth-pages',
        description: 'Tests full app navigation flow'
    },
    {
        name: 'Auth Comprehensive CI Tests',
        file: 'test_auth_comprehensive_ci.js',
        shard: 'auth-pages',
        description: 'Tests password strength, form validation, remember me, sessions'
    },
    {
        name: 'Navigation & Theme CI Tests',
        file: 'test_navigation_and_theme_ci.js',
        shard: 'auth-pages',
        description:
            'Flask->FastAPI migration: walks every sidebar link discovered live from ' +
            'the DOM (not a hardcoded list) asserting HTTP 2xx/HTML/no-JS-errors, ' +
            'checks active-nav .active state matches the current page, exercises the ' +
            'header theme dropdown (data-theme attribute + localStorage + reload ' +
            'persistence, restored after), and compares sidebar-click vs. fresh-tab ' +
            'direct-load parity for two pages.'
    },

    // =====================================================================
    // Shard: research-workflow (5 tests)
    // =====================================================================
    {
        name: 'Research Submission Flow Test',
        file: 'test_research_submit_flow_ci.js',
        shard: 'research-workflow',
        description:
            'Flask->FastAPI migration: drives the real research-submit form ' +
            '(real typing + real click, no synthetic fetch) through the ' +
            'CSRF-guarded async /api/start_research route (run_db_sync offload, ' +
            'per-user encrypted DB write), the client-side redirect to ' +
            '/progress/<id>, the Socket.IO realtime channel, /history rendering, ' +
            'and cross-user DB isolation. No LLM required — the research thread ' +
            'failing after being created is an accepted outcome; only the ' +
            'submission path itself is asserted.'
    },
    {
        name: 'Research Workflow Test',
        file: 'test_research_workflow.js',
        shard: 'research-workflow',
        description: 'Tests the complete research lifecycle from submission to results'
    },
    {
        name: 'Research Workflow CI Tests',
        file: 'test_research_workflow_ci.js',
        shard: 'research-workflow',
        description: 'Tests research form, progress page, results, exports'
    },
    {
        name: 'Follow-up Research CI Tests',
        file: 'test_followup_research_ci.js',
        shard: 'research-workflow',
        description: 'Tests follow-up research flow'
    },
    {
        name: 'Research Cancellation CI Tests',
        file: 'test_research_cancellation_ci.js',
        shard: 'research-workflow',
        description: 'Tests cancel/terminate lifecycle (button → SUSPENDED, idempotency, queued branch, not-found). Seeds rows, no LLM needed.'
    },

    // =====================================================================
    // Shard: research-completion (1 test)
    // =====================================================================
    {
        name: 'Research completion and reconnect release smoke',
        file: 'test_research_completion_reconnect_ci.js',
        shard: 'research-completion',
        // A normal local server cannot provide the dedicated fixture's gates.
        requiresExplicitShard: true,
        description: 'Real form and worker, Socket.IO completion after reconnect, and HTTP polling fallback'
    },

    // =====================================================================
    // Shard: research-form (4 tests)
    // =====================================================================
    {
        name: 'Research Form CI Tests',
        file: 'test_research_form_ci.js',
        shard: 'research-form',
        description: 'Tests advanced options, mode toggle, dropdowns, validation'
    },
    {
        name: 'Research Results Test',
        file: 'test_research_results.js',
        shard: 'research-form',
        description: 'Tests error handling for non-existent research and history page structure'
    },
    {
        name: 'Results & Exports CI Tests',
        file: 'test_results_exports_ci.js',
        shard: 'research-form',
        description: 'Tests star ratings, export buttons, download functionality'
    },
    {
        name: 'Search-Engine Dropdown Scope + Strategy CI Tests',
        file: 'test_search_engine_dropdown_scope_ci.js',
        shard: 'research-form',
        description:
            'Covers #5221/#5204 (scope- and strategy-aware search engine dropdown) whose ' +
            'FastAPI backend (_classify_options_for_egress, the agent_enabled field on ' +
            'GET /settings/api/available-search-engines) was hand-ported after the merge ' +
            'dropped it. Asserts the API contract (egress_scope stamps egress:{allowed,reason}; ' +
            'no params / blank params stay unfiltered), the rendered dropdown disables exactly ' +
            'the denied options with a visible reason, a live scope change reconciles a now-' +
            'invalid primary selection without a reload, and an agent_enabled:false collection ' +
            'is disabled only under the LangGraph strategy and re-enables live when switched away.'
    },

    // =====================================================================
    // Shard: research-metrics (3 tests)
    // =====================================================================
    {
        name: 'Metrics Charts Test',
        file: 'test_metrics_charts.js',
        shard: 'research-metrics',
        description: 'Tests Chart.js rendering for token and search charts'
    },
    {
        name: 'Metrics Dashboard CI Tests',
        file: 'test_metrics_dashboard_ci.js',
        shard: 'research-metrics',
        description: 'Tests metrics dashboard, cost analytics, star reviews, links'
    },
    {
        name: 'Realtime Progress CI Tests',
        file: 'test_realtime_progress_ci.js',
        shard: 'research-metrics',
        description: 'Tests progress page and real-time elements'
    },
    {
        name: 'Streaming/Realtime Browser Tests',
        file: 'test_streaming_realtime_ci.js',
        shard: 'research-metrics',
        description:
            'Flask->FastAPI migration: proves a real browser receives data over ' +
            'the rewritten transports (python-socketio ASGI at /ws, Starlette SSE) ' +
            'rather than just asserting DOM/route structure. Socket.IO handshake ' +
            'auth (accept authenticated, reject unauthenticated), SSE streaming ' +
            'contract on POST /library/api/rag/index-all (headers + multi-chunk ' +
            'delivery + parsed events), and console-error-free load of the pages ' +
            'that depend on these subsystems.'
    },

    // =====================================================================
    // Shard: link-analytics (1 test — release-pipeline only)
    // =====================================================================
    {
        name: 'Link Analytics Full Page Test',
        file: 'test_link_analytics_full.js',
        shard: 'link-analytics',
        description: 'Verifies /metrics/links renders, Recent Researches header shows numeric total, no console errors, no script element leaks'
    },

    // =====================================================================
    // Shard: settings-core (6 tests)
    // =====================================================================
    {
        name: 'Settings Page Test',
        file: 'test_settings_page.js',
        shard: 'settings-core',
        description: 'Tests settings page loading and API integration'
    },
    {
        name: 'Settings Error Detection Test',
        file: 'test_settings_errors.js',
        shard: 'settings-core',
        description: 'Tests error handling when changing settings'
    },
    {
        name: 'Settings Save Test',
        file: 'test_settings_save.js',
        shard: 'settings-core',
        description: 'Tests settings save workflow and validation'
    },
    {
        name: 'Settings Interactions CI Tests',
        file: 'test_settings_interactions_ci.js',
        shard: 'settings-core',
        description: 'Tests tabs, search, toggles, save, raw config'
    },
    {
        name: 'Settings Save Error CI Tests',
        file: 'test_settings_save_error_ci.js',
        shard: 'settings-core',
        description: 'Injects 5xx on save endpoint via request interception, asserts error toast renders in #notification-banner-assertive.'
    },
    {
        name: 'Settings Form POST Fallback CI Tests',
        file: 'test_settings_form_post_fallback_ci.js',
        shard: 'settings-core',
        description:
            'Flask->FastAPI migration: #settings-form\'s real (non-AJAX) POST /settings/save_settings ' +
            '"no-JS fallback" round trip, untouched by every other settings suite (all AJAX-path). ' +
            'The handler\'s own comment flags this as a migration casualty that had to be hand-restored: ' +
            'web/dependencies/flash.py replaces Flask\'s flash()/get_flashed_messages() with a session-' +
            'backed equivalent. Drives a genuine form.submit() (bypasses the JS submit interceptor by ' +
            'spec) and asserts the 302 redirect + success flash, a disallowed new setting key producing ' +
            'a visible warning flash instead of a silent drop (and confirms the key was NOT created), ' +
            'and that the flash is one-shot (session .pop(), not .get()) across a subsequent reload.'
    },

    // =====================================================================
    // Shard: settings-pages (4 tests)
    // =====================================================================
    {
        name: 'Settings Pages CI Tests',
        file: 'test_settings_pages_ci.js',
        shard: 'settings-pages',
        description: 'Tests settings tabs, navigation, provider/engine settings'
    },
    {
        name: 'Star Reviews Test',
        file: 'test_star_reviews.js',
        shard: 'settings-pages',
        description: 'Tests star reviews analytics page and visualizations'
    },
    {
        name: 'Journal Quality CI Tests',
        file: 'test_journal_quality_ci.js',
        shard: 'settings-pages',
        description: 'Tests journal quality dashboard: tabs, threshold slider settings round-trip, data-status APIs'
    },
    {
        name: 'Settings Persistence CI Tests',
        file: 'test_settings_persistence_ci.js',
        shard: 'settings-pages',
        description: 'Real-UI settings round trip: change survives reload, matches settings API, per-user isolation via incognito context, search filter narrows + restores'
    },

    // =====================================================================
    // Shard: library (10 tests)
    // =====================================================================
    {
        name: 'Library Collections CI Tests',
        file: 'test_library_collections_ci.js',
        shard: 'library',
        description: 'Tests library page, collections, document details'
    },
    {
        name: 'Collections CRUD Lifecycle CI Tests',
        file: 'test_collections_crud_ci.js',
        shard: 'library',
        description: 'Real-UI collection lifecycle: create via the form, appears in the list, open via click-through, edit (is_public toggle), delete guarded by confirm() then confirmed, gone from list + API'
    },
    {
        name: 'Library Documents CI Tests',
        file: 'test_library_documents_ci.js',
        shard: 'library',
        description: 'Tests filters, views, PDF/text viewers, bulk actions'
    },
    {
        name: 'Library Collections Page Test',
        file: 'library/test_collections_page.js',
        shard: 'library',
        description: 'Tests library collections page'
    },
    {
        name: 'Download Manager CI Tests',
        file: 'test_download_manager_ci.js',
        shard: 'library',
        description: 'Tests download manager: stats, filters, selection cycle, collections API'
    },
    {
        name: 'Zotero Integration CI Tests',
        file: 'test_zotero_integration_ci.js',
        shard: 'library',
        description: 'Tests Zotero page structure, config/status APIs, API-key non-leak contract, unconfigured negative paths'
    },
    {
        name: 'RAG Index + Semantic Search E2E',
        file: 'test_rag_index_search_ci.js',
        shard: 'library',
        description: 'End-to-end: create collection, upload a doc, index it, and semantic-search it back through the UI'
    },
    {
        name: 'Notes CRUD Lifecycle CI Tests',
        file: 'test_notes_crud_ci.js',
        shard: 'library',
        description: 'Real-UI notes lifecycle: create via the modal, appears in the list, open via click-through, edit title/content, add + remove a tag (each persisted), delete guarded by confirm() then confirmed, gone from list + API'
    },
    {
        name: 'Legacy .pkl RAG Migration E2E',
        file: 'test_legacy_pkl_migration_ci.js',
        shard: 'library',
        description: 'End-to-end: seed legacy plaintext .pkl docstores (2 collections), let the server run phase-1 at startup + phase-2 at login, and assert the .pkl is gone, a text-free sidecar was written, and search still rehydrates the RIGHT seeded content by id',
        // Requires a Python seed BEFORE the server boots (phase-1 runs only at
        // startup). Locally: tests/ui_tests/run_legacy_pkl_migration.sh. In CI:
        // the library-shard seed step in .github/workflows/docker-tests.yml.
        // The manifest path defaults to <LDR_DATA_DIR>/migtest_manifest.json;
        // if the manifest is absent the test fails fast asking you to seed.
        needsSeed: 'legacy_pkl_migration',
    },
    {
        name: 'Pagination Query-Param Robustness CI Tests',
        file: 'test_pagination_query_params_ci.js',
        shard: 'library',
        description:
            'Flask->FastAPI migration: /library/, /library/download-manager, and ' +
            '/library/api/documents each hand-parse page/limit/offset straight off ' +
            'request.query_params (int() wrapped in try/except + clamp, mirroring Flask\'s ' +
            'forgiving request.args.get(..., type=int)). Drives a battery of adversarial ' +
            'values (non-numeric, negative, zero, absurdly large) no rendered pagination ' +
            'link would ever produce and asserts none of them 500 or leak the raw JSON ' +
            'catch-all error body into the page.'
    },

    // =====================================================================
    // Shard: history-news (5 tests)
    // =====================================================================
    {
        name: 'History Page CI Tests',
        file: 'test_history_page_ci.js',
        shard: 'history-news',
        description: 'Tests history table, actions, search/filter'
    },
    {
        name: 'History Page Test',
        file: 'test_history_page.js',
        shard: 'history-news',
        description: 'Tests history page functionality'
    },
    {
        name: 'News Subscriptions CI Tests',
        file: 'test_news_subscriptions_ci.js',
        shard: 'history-news',
        description: 'Tests news feeds, subscription CRUD, form validation'
    },
    {
        name: 'History Delete CI Tests',
        file: 'test_history_delete_ci.js',
        shard: 'history-news',
        description: 'Clicks delete button, handles confirm dialog, asserts item removed from DOM and DELETE API fires.'
    },
    {
        name: 'News Subscription Form Dropdown Scope CI Tests',
        file: 'test_news_subscription_engine_dropdown_ci.js',
        shard: 'history-news',
        description:
            'Covers #5221/#5204 scope-aware search engine dropdown on the OTHER consumer of ' +
            'GET /settings/api/available-search-engines: the news-subscription create/edit ' +
            'forms, which read the saved policy.egress_scope server-side (web/routers/' +
            'news_pages.py) rather than exposing their own scope selector. Asserts default/' +
            'adaptive scope stays unfiltered (zero impact), a saved private_only scope disables ' +
            'non-local engines with a reason on BOTH /news/subscriptions/new and ' +
            '/news/subscriptions/<id>/edit, and the two forms agree on the exact disabled set.'
    },

    // =====================================================================
    // Shard: mobile (4 tests)
    // =====================================================================
    {
        name: 'Mobile Interactions CI Tests',
        file: 'test_mobile_interactions_ci.js',
        shard: 'mobile',
        description: 'Tests mobile modals, navigation, forms'
    },
    {
        name: 'Mobile Navigation CI Test',
        file: 'mobile/test_mobile_navigation_ci.js',
        shard: 'mobile',
        description: 'Tests mobile navigation patterns'
    },
    {
        name: 'UI Functionality CI Tests',
        file: 'mobile/test_ui_functionality_ci.js',
        shard: 'mobile',
        description: 'Tests forms, dropdowns, modals, navigation, buttons'
    },
    {
        name: 'Loading & Feedback CI Tests',
        file: 'test_loading_feedback_ci.js',
        shard: 'mobile',
        description: 'Tests spinners, toasts, progress bars, hover states'
    },

    // =====================================================================
    // Shard: api-crud (3 tests)
    // =====================================================================
    {
        name: 'API Endpoints CI Tests',
        file: 'test_api_endpoints_ci.js',
        shard: 'api-crud',
        description: 'Tests all major API endpoints'
    },
    {
        name: 'CRUD Operations CI Tests',
        file: 'test_crud_operations_ci.js',
        shard: 'api-crud',
        description: 'Tests collections, subscriptions, documents CRUD'
    },
    {
        name: 'Rate Limiting Functionality Test',
        file: 'test_rate_limiting_settings.js',
        shard: 'api-crud',
        description: 'Tests authentication rate limits and 429 response bodies and headers',
        // The test file itself no-ops (early `return`, exit 0) unless
        // ENABLE_RATE_LIMITING=true, because CI always launches the server
        // with LDR_DISABLE_RATE_LIMITING=true (see docker-tests.yml — every
        // job sets it, none sets ENABLE_RATE_LIMITING). That no-op used to
        // reach this runner as an ordinary exit-0 pass, indistinguishable in
        // the summary from a run that actually exercised rate limiting.
        // skipCI here makes that show up as a skip, matching the file's own
        // console message instead of contradicting it.
        skipCI: true,
    },
    {
        name: 'Download & CSRF Flow CI Tests',
        file: 'test_download_and_csrf_flows_ci.js',
        shard: 'api-crud',
        description: 'Tests CSRF middleware rejects/accepts real mutations (JSON + multipart) and a binary download (PDF) delivers correct Content-Length/Content-Disposition/bytes; also session cookie flags, external-link login preservation, external chat links with ?q= sending and pre-filling nothing, and logout invalidation. No LLM needed.'
    },

    // =====================================================================
    // Shard: error-benchmark (6 tests)
    // =====================================================================
    {
        name: 'UI Runner Timeout Regression',
        file: 'test_run_all_tests_timeout.test.js',
        shard: 'error-benchmark',
        description: 'A timed-out child fails even when it exits zero on SIGTERM, and is killed if it ignores SIGTERM'
    },
    {
        name: 'Error Recovery Test',
        file: 'test_error_recovery.js',
        shard: 'error-benchmark',
        description: 'Tests how the UI handles various error conditions gracefully'
    },
    {
        name: 'Error Handling CI Tests',
        file: 'test_error_handling_ci.js',
        shard: 'error-benchmark',
        description: 'Tests 404, 401, 429, validation errors'
    },
    {
        name: 'Error / Browser Session-Loss UX CI Tests',
        file: 'test_error_and_session_ux_ci.js',
        shard: 'error-benchmark',
        description: 'CSRF-rejected mutation shows a visible error toast; unrouted URL renders as an HTML page, not raw JSON; ' +
            'after mid-session cookie invalidation, a WRITE (401 from the CSRF middleware, no token refresh) and a GET (401) ' +
            'both redirect to a rendered login form at /auth/login with next=/notes/; no uncaught JS errors.'
    },
    {
        name: 'Benchmark CI Tests',
        file: 'test_benchmark_ci.js',
        shard: 'error-benchmark',
        description: 'Tests benchmark dashboard and results pages'
    },
    {
        name: 'Context Overflow CI Tests',
        file: 'test_context_overflow_ci.js',
        shard: 'error-benchmark',
        description: 'Tests context overflow analytics page'
    },

    // =====================================================================
    // Shard: accessibility (1 test)
    // =====================================================================
    {
        name: 'Keyboard & Accessibility CI Tests',
        file: 'test_keyboard_accessibility_ci.js',
        shard: 'accessibility',
        description: 'Tests keyboard navigation, shortcuts, ARIA, focus management'
    },

    // =====================================================================
    // Shard: chat-core (7 tests)
    // chat-mode v2 — input, a11y, security, navigation. These tests do
    // not require an LLM backend; they exercise the chat page's
    // client-side behavior + the chat HTTP routes.
    // =====================================================================
    {
        name: 'Chat ARIA Live Region Test',
        file: 'chat/test_chat_aria_live.js',
        shard: 'chat-core',
        description: 'Tests role=log + aria-live on .ldr-chat-messages'
    },
    {
        name: 'Chat Keyboard & Input Test',
        file: 'chat/test_chat_keyboard_and_input.js',
        shard: 'chat-core',
        description: 'Tests Enter-to-send, Shift+Enter newline, textarea state',
        // Re-enabled: the "CDP input not delivered" failure (#4430) was
        // Chrome's password-leak-detection dialog after login, now disabled
        // via the seeded profile in chrome_profile.js.
    },
    {
        name: 'Chat CSRF Required Test',
        file: 'chat/test_chat_csrf_required.js',
        shard: 'chat-core',
        description: 'Tests that state-mutating chat endpoints reject missing CSRF tokens'
    },
    {
        name: 'Chat Suggestion Chips Test',
        file: 'chat/test_chat_suggestion_chips.js',
        shard: 'chat-core',
        description: 'Tests suggestion-chip click dispatches a chat message'
    },
    {
        name: 'Chat New Chat Button Test',
        file: 'chat/test_chat_new_chat_button.js',
        shard: 'chat-core',
        description: 'Tests "New Chat" button starts a fresh session'
    },
    {
        name: 'Chat URL ?q= Param Test',
        file: 'chat/test_chat_url_q_param.js',
        shard: 'chat-core',
        description: 'Tests /chat?q=... only pre-fills an unfocused new or empty chat and main-page chat mode sends its query once'
    },
    {
        name: 'Chat Page Navigation Test',
        file: 'chat/test_chat_page_navigation.js',
        shard: 'chat-core',
        description: 'Tests sidebar navigation to /chat works'
    },

    // =====================================================================
    // Shard: chat-lifecycle (6 tests)
    // chat-mode v2 — session lifecycle (edit/archive/export), error
    // surfacing, reload persistence. LLM-dependent tests live in the
    // skipCI section below.
    // =====================================================================
    {
        name: 'Chat Archived Session Rejects Send Test',
        file: 'chat/test_chat_archived_session_rejects.js',
        shard: 'chat-lifecycle',
        description: 'Tests an archived session rejects POST /api/chat/sessions/<id>/messages with 409'
    },
    {
        name: 'Chat Edit Title Test',
        file: 'chat/test_chat_edit_title.js',
        shard: 'chat-lifecycle',
        description: 'Tests in-place rename of a chat session via PATCH'
    },
    {
        name: 'Chat Export Markdown Test',
        file: 'chat/test_chat_export_markdown.js',
        shard: 'chat-lifecycle',
        description: 'Tests export-to-markdown endpoint + UI flow'
    },
    {
        name: 'Chat Reload Persistence Test',
        file: 'chat/test_chat_reload_persistence.js',
        shard: 'chat-lifecycle',
        description: 'Tests messages persist across a page reload'
    },
    {
        name: 'Chat Session Management Test',
        file: 'chat/test_chat_session_management.js',
        shard: 'chat-lifecycle',
        description: 'Tests list/archive/reactivate/delete via the chat UI'
    },
    {
        name: 'Chat Error States Test',
        file: 'chat/test_chat_error_states.js',
        shard: 'chat-lifecycle',
        description: 'Tests 404/429/network-down error rendering'
    },

    // =====================================================================
    // Skipped tests (skipCI: true) — still need shard assignments for
    // local runs. Shard names can be anything valid since they never run
    // in CI; assigned to the closest active shard.
    // =====================================================================
    {
        name: 'Research Submit Test',
        file: 'test_research_submit.js',
        shard: 'research-form',
        description: 'Tests research submission',
        skipCI: true,  // Requires LLM backend
    },
    {
        name: 'Export Functionality Test',
        file: 'test_export_functionality.js',
        shard: 'research-form',
        description: 'Tests export features',
        skipCI: true,  // Auth hangs with "Navigating frame was detached" in Docker
    },
    {
        name: 'Concurrent Limit Test',
        file: 'test_concurrent_limit.js',
        shard: 'research-workflow',
        description: 'Tests concurrent research limits',
        skipCI: true,  // Requires LLM backend — always fails without model server
    },
    {
        name: 'News Feed CI Tests',
        file: 'test_news_feed_ci.js',
        shard: 'history-news',
        description: 'Tests feed, filters, templates, subscription management',
        skipCI: true,  // Intermittent 60s navigation timeouts; core coverage in test_news_subscriptions_ci.js
    },
    {
        name: 'Settings Validation Test',
        file: 'test_settings_validation.js',
        shard: 'settings-core',
        description: 'Tests settings input validation',
        skipCI: true,  // Same frame-detachment issue as Export test in Docker
    },
    {
        name: 'Research Form Validation Test',
        file: 'test_research_form_validation.js',
        shard: 'research-form',
        description: 'Tests research form field validation',
        skipCI: true,  // Redundant with test_research_form_ci.js; auth frame-detachment in Docker
    },
    {
        name: 'Form Validation ARIA Tests',
        file: 'test_form_validation_aria_ci.js',
        shard: 'accessibility',
        description: 'Tests inline form validation with ARIA support',
        skipCI: true,  // Auth frame-detachment in Docker causes intermittent 120s timeout
    },
    {
        name: 'Research Simple Test',
        file: 'test_research_simple.js',
        shard: 'research-workflow',
        description: 'Tests basic research flow',
        skipCI: true,  // Requires LLM backend to complete research submission
    },
    {
        name: 'Research Form Test',
        file: 'test_research_form.js',
        shard: 'research-form',
        description: 'Tests research form interactions',
        skipCI: true,  // Diagnostic test — requires LLM for form submission
    },
    {
        name: 'Research API Test',
        file: 'test_research_api.js',
        shard: 'research-workflow',
        description: 'Tests research API endpoints via UI',
        skipCI: true,  // Diagnostic test — requires functioning LLM API
    },
    {
        name: 'Queue Simple Test',
        file: 'test_queue_simple.js',
        shard: 'research-workflow',
        description: 'Tests research queue functionality',
        skipCI: true,  // Requires LLM backend — always fails without model server
    },
    {
        name: 'Chat Message Flow E2E Test',
        file: 'chat/test_chat_message_flow.js',
        shard: 'chat-lifecycle',
        description: 'End-to-end: send message, watch research streaming, assert assistant response',
        skipCI: true,  // Requires LDR_TEST_LLM_URL + LDR_TEST_LLM_MODEL backend
    },
    {
        name: 'Chat report_content Refactor Test',
        file: 'chat/test_chat_report_content_refactor.js',
        shard: 'chat-lifecycle',
        description: 'Verifies report_content shape change: chat shows answer-only, /results assembles full',
        skipCI: true,  // Requires LDR_TEST_LLM_URL + LDR_TEST_LLM_MODEL backend
    },
];

async function runTest(test, timeoutMs = process.env.CI ? 300000 : 60000) {
    return new Promise((resolve) => {
        const startTime = Date.now();
        console.log(`\n[${ts()}] Running: ${test.name}`);

        const testProcess = spawn('node', [test.file], {
            cwd: path.join(__dirname),
            stdio: 'inherit',
            env: { ...process.env, NODE_OPTIONS: '--max-old-space-size=4096' }
        });

        // Add timeout for individual tests
        // 300 seconds in CI to handle slow registration/auth tests.
        // In Docker, new user registration creates encrypted SQLCipher databases
        // with key derivation + 58 tables + 500+ settings, which can take 60-120s.
        // Subsequent tests may also be slow while the server recovers.
        // 60 seconds locally for faster feedback.
        let timedOut = false;
        let closed = false;
        let forceKillTimeout;
        const timeout = setTimeout(() => {
            timedOut = true;
            const elapsed = Math.round((Date.now() - startTime) / 1000);
            console.log(`\n⏱️ Test timeout: ${test.name} exceeded ${timeoutMs/1000} seconds (${elapsed}s elapsed)`);
            console.log(`🔪 Sending SIGTERM to PID ${testProcess.pid}...`);
            testProcess.kill('SIGTERM');
            forceKillTimeout = setTimeout(() => {
                if (!closed) {
                    console.log(`🔫 Process still alive, sending SIGKILL to PID ${testProcess.pid}...`);
                    testProcess.kill('SIGKILL');
                }
            }, 5000);
        }, timeoutMs);

        testProcess.on('close', (code, signal) => {
            closed = true;
            clearTimeout(timeout);
            clearTimeout(forceKillTimeout);
            const elapsed = Math.round((Date.now() - startTime) / 1000);
            // Puppeteer's SIGTERM handler can close Chrome and let Node exit 0.
            // A killed test never completed its assertions, regardless of code.
            const success = code === 0 && !timedOut;
            console.log(`[${ts()}] ${success ? '✅' : '❌'} ${test.name}: ${success ? 'PASSED' : 'FAILED'} (${elapsed}s)`);
            // Grep-friendly line for post-run duration analysis (used to rebalance shards).
            console.log(`TIMING: ${test.name}: ${elapsed}`);
            if (code !== 0 && code !== null) {
                console.log(`   Exit code: ${code}`);
            }
            resolve({
                name: test.name,
                success,
                code,
                signal,
                error: timedOut ? `Timed out after ${timeoutMs/1000} seconds` : undefined,
                duration: elapsed
            });
        });

        testProcess.on('error', (error) => {
            clearTimeout(timeout);
            const elapsed = Math.round((Date.now() - startTime) / 1000);
            console.log(`[${ts()}] ❌ ${test.name}: ERROR - ${error.message} (${elapsed}s)`);
            resolve({
                name: test.name,
                success: false,
                error: error.message,
                duration: elapsed
            });
        });
    });
}

function parseShardArg() {
    const shardArg = process.argv.find(arg => arg.startsWith('--shard='));
    return shardArg ? shardArg.split('=')[1] : null;
}

function validateBeforeRun(requestedShard, isCI) {
    // Every test must declare a shard so filtering can't silently skip it.
    const untagged = tests.filter(t => !t.shard);
    if (untagged.length > 0) {
        console.error('FATAL: the following tests are missing a `shard:` property:');
        untagged.forEach(t => console.error(`  - ${t.name} (${t.file})`));
        console.error(`Valid shards: ${VALID_SHARDS.join(', ')}`);
        process.exit(1);
    }

    if (requestedShard && !VALID_SHARDS.includes(requestedShard)) {
        console.error(`FATAL: unknown shard "${requestedShard}".`);
        console.error(`Valid shards: ${VALID_SHARDS.join(', ')}`);
        process.exit(1);
    }

    // Matrix misconfiguration guard: if CI forgot to pass --shard, every
    // matrix cell would otherwise run the full suite. Fail loud.
    if (isCI && !requestedShard) {
        console.error('FATAL: CI=true but no --shard flag provided.');
        console.error('Matrix is misconfigured — each cell must pass --shard=<name>.');
        console.error(`Valid shards: ${VALID_SHARDS.join(', ')}`);
        process.exit(1);
    }
}

async function runAllTests() {
    const suiteStart = new Date();
    const isCI = !!process.env.CI;
    const requestedShard = parseShardArg();

    validateBeforeRun(requestedShard, isCI);

    const shardLabel = requestedShard ? ` [shard: ${requestedShard}]` : '';
    console.log(`[${ts(suiteStart)}] Starting UI Test Suite${shardLabel}\n`);

    const results = [];

    for (const test of tests) {
        // skipCI takes priority: a test marked skipCI stays skipped in CI even
        // if its shard matches. Keeps existing skip semantics intact.
        if (test.skipCI && isCI) {
            // Only log the skip when this shard would have run the test.
            if (!requestedShard || test.shard === requestedShard) {
                console.log(`\n[${ts()}] ⏭️  Skipping: ${test.name} (not supported in CI Docker)`);
                results.push({ name: test.name, success: true, duration: 0, skipped: true });
            }
            continue;
        }
        // Shard filter: silently drop tests that don't belong to the requested shard.
        if (requestedShard && test.shard !== requestedShard) {
            continue;
        }
        if (test.requiresExplicitShard && !requestedShard) {
            console.log(`\n[${ts()}] ⏭️  Skipping: ${test.name} (requires --shard=${test.shard} and its fixture server)`);
            results.push({ name: test.name, success: true, duration: 0, skipped: true });
            continue;
        }
        // Ensure server is responsive before starting each test.
        // Prevents cascade failures when a previous test stressed the server.
        await waitForServer();
        const result = await runTest(test);
        results.push(result);
    }

    // Print summary
    const suiteEnd = new Date();
    const wallTime = Math.round((suiteEnd - suiteStart) / 1000);
    console.log(`\n[${ts(suiteEnd)}] TEST SUMMARY`);

    const passed = results.filter(r => r.success).length;
    const failed = results.filter(r => !r.success).length;
    const totalDuration = results.reduce((sum, r) => sum + (r.duration || 0), 0);

    // Sort by duration descending so slowest tests are easy to spot
    const sorted = [...results].sort((a, b) => (b.duration || 0) - (a.duration || 0));
    sorted.forEach(result => {
        const status = result.success ? '✅ PASS' : '❌ FAIL';
        const duration = result.duration ? ` (${result.duration}s)` : '';
        console.log(`${status} ${result.name}${duration}`);
        if (result.error) {
            console.log(`       Error: ${result.error}`);
        }
    });

    const shardSuffix = requestedShard ? ` [shard: ${requestedShard}]` : '';
    const rate = results.length === 0 ? 0 : Math.round((passed / results.length) * 100);
    console.log(`Total: ${results.length} | Passed: ${passed} | Failed: ${failed} | Duration: ${totalDuration}s | Wall: ${wallTime}s | Rate: ${rate}%${shardSuffix}`);

    if (failed === 0) {
        console.log('All tests passed!');
    } else {
        console.log(`${failed} test(s) failed.`);
    }

    process.exit(failed > 0 ? 1 : 0);
}

if (require.main === module) {
    runAllTests().catch(error => {
        console.error('💥 Test runner error:', error);
        process.exit(1);
    });
}

module.exports = { runTest };