Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

8.5 KB
Raw Blame
/**
 * Playwright Configuration for Local Deep Research
 *
 * This configuration enables proper mobile UI testing with:
 * - Visual regression testing (screenshot comparison)
 * - Multiple mobile device profiles
 * - Cross-browser testing (Chromium, Firefox, WebKit/Safari)
 * - Parallel execution for faster CI runs
 * - Single authentication to avoid rate limiting
 *
 * Migration from Puppeteer: This provides the "real" testing
 * that Puppeteer emulation cannot deliver.
 */

import { defineConfig, devices } from '@playwright/test';
import path from 'path';

const authFile = path.join(__dirname, '.auth/user.json');

/**
 * Notes feature specs run ONLY in the dedicated `notes-paths` project
 * (Desktop Chrome). Without this ignore on every device project,
 * testDir './tests' made all of them pick the notes specs up too — 34
 * spec files × 14 projects per unfiltered `npm test`, including the
 * @slow live-LLM specs whose `--grep-invert @slow` filter exists only
 * in `npm run test:notes`.
 */
const NOTES_SPECS = /\/tests\/notes\//;
// Real-browser sanitizer regression (security/mxss-reparse) targets desktop
// browsers via a fixture HTML loaded from disk; mobile/visual-regression
// projects have no value running it.
const MXSS_SPECS = /\/tests\/security-mxss-reparse/;

/**
 * See https://playwright.dev/docs/test-configuration
 */
export default defineConfig({
  testDir: './tests',

  /* Run tests in files in parallel */
  fullyParallel: true,

  /* Global teardown wipes test_admin notes belonging to the current run.
     Idempotent and harmless for non-notes runs. */
  globalTeardown: './tests/helpers/global-teardown.js',

  /* Fail the build on CI if you accidentally left test.only in the source code */
  forbidOnly: !!process.env.CI,

  /* Retry failed tests - helps with flaky timing-sensitive tests */
  retries: process.env.CI ? 2 : 1,

  /* Opt out of parallel tests on CI for more consistent results */
  workers: process.env.CI ? 1 : undefined,

  /* Reporter to use */
  reporter: [
    ['html', { open: 'never' }],
    ['json', { outputFile: 'test-results/results.json' }],
    ['junit', { outputFile: 'test-results/results.xml' }],
  ],

  /* Shared settings for all projects */
  use: {
    /* Base URL to use in actions like `await page.goto('/')` */
    baseURL: process.env.TEST_BASE_URL || 'http://127.0.0.1:5000',

    /* Collect trace when retrying the failed test */
    trace: 'on-first-retry',

    /* Take screenshot on failure */
    screenshot: 'only-on-failure',

    /* Record video on failure */
    video: 'on-first-retry',

    /*
     * Cap navigation-class waits (goto, reload, waitForLoadState,
     * waitForURL) at 15s. Playwright's default is 30s — equal to our
     * test timeout — which means a single slow nav can silently consume
     * the entire budget, especially when wrapped in `.catch(() => {})`.
     * See PR #4215 / issue #4060 for the concrete failure shape.
     */
    navigationTimeout: 15000,
  },

  /* Configure projects for major browsers and mobile devices */
  projects: [
    // ==========================================
    // AUTHENTICATION SETUP (runs first)
    // ==========================================
    {
      name: 'setup',
      testMatch: /auth\.setup\.js/,
      use: {
        ...devices['Desktop Chrome'],
      },
    },

    // ==========================================
    // MOBILE DEVICES - Primary Testing Targets
    // ==========================================

    // Small iPhones (most constrained viewport)
    {
      name: 'iPhone SE',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['iPhone SE'],
        // Additional settings for thorough testing
        hasTouch: true,
        isMobile: true,
        storageState: authFile,
      },
    },

    // Standard iPhone (most popular)
    {
      name: 'iPhone 14',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['iPhone 14'],
        hasTouch: true,
        isMobile: true,
        storageState: authFile,
      },
    },

    // Large iPhone (Pro Max)
    {
      name: 'iPhone 14 Pro Max',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['iPhone 14 Pro Max'],
        hasTouch: true,
        isMobile: true,
        storageState: authFile,
      },
    },

    // Android - Small (budget phones)
    {
      name: 'Pixel 5',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['Pixel 5'],
        hasTouch: true,
        isMobile: true,
        storageState: authFile,
      },
    },

    // Android - Large (Samsung Galaxy)
    {
      name: 'Galaxy S23',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        viewport: { width: 360, height: 780 },
        deviceScaleFactor: 3,
        hasTouch: true,
        isMobile: true,
        userAgent: 'Mozilla/5.0 (Linux; Android 13; SM-S911B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36',
        storageState: authFile,
      },
    },

    // ==========================================
    // TABLETS
    // ==========================================

    {
      name: 'iPad Mini',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['iPad Mini'],
        hasTouch: true,
        isMobile: true,
        storageState: authFile,
      },
    },

    {
      name: 'iPad Pro 11',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['iPad Pro 11'],
        hasTouch: true,
        isMobile: true,
        storageState: authFile,
      },
    },

    // ==========================================
    // LANDSCAPE ORIENTATIONS (Critical for layout bugs)
    // ==========================================

    {
      name: 'iPhone SE Landscape',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['iPhone SE landscape'],
        hasTouch: true,
        isMobile: true,
        storageState: authFile,
      },
    },

    {
      name: 'iPhone 14 Landscape',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['iPhone 14 landscape'],
        hasTouch: true,
        isMobile: true,
        storageState: authFile,
      },
    },

    // ==========================================
    // DESKTOP BROWSERS
    // ==========================================

    {
      name: 'Desktop Chrome',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['Desktop Chrome'],
        storageState: authFile,
      },
    },

    {
      name: 'Desktop Firefox',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['Desktop Firefox'],
        storageState: authFile,
      },
    },

    {
      name: 'Desktop Safari',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['Desktop Safari'],
        storageState: authFile,
      },
    },

    // ==========================================
    // WEBKIT/SAFARI MOBILE (Critical for iOS bugs)
    // ==========================================

    {
      name: 'Mobile Safari',
      dependencies: ['setup'],
      testIgnore: [NOTES_SPECS, MXSS_SPECS],
      use: {
        ...devices['iPhone 14'],
        browserName: 'webkit', // This uses WebKit, closer to real Safari
        storageState: authFile,
      },
    },

    // ==========================================
    // NOTES FEATURE PATH COVERAGE
    // ==========================================
    // Dedicated project for notes/ specs. Runs only on Desktop Chrome; the
    // 12 device projects above are for layout / visual regression and don't
    // need to re-run feature paths. Pinned to the tests/notes/ directory by
    // the testMatch regex (anchored to the path, not bare 'notes/').
    {
      name: 'notes-paths',
      dependencies: ['setup'],
      testMatch: /\/tests\/notes\/[^/]+\.spec\.js$/,
      testIgnore: MXSS_SPECS,
      use: {
        ...devices['Desktop Chrome'],
        storageState: authFile,
      },
    },
  ],

  /* Run your local dev server before starting the tests */
  webServer: process.env.CI ? undefined : {
    command: 'cd ../../../ && pdm run python -m local_deep_research.web.app',
    url: 'http://127.0.0.1:5000',
    reuseExistingServer: !process.env.CI,
    timeout: 120000,
  },
});