Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

8.3 KB
Raw Blame
import type { Page } from '@playwright/test';
import { expect as baseExpect, test } from './fixtures';

// The dev server shares a machine with other suites; default timeouts are too tight.
const expect = baseExpect.configure({ timeout: 30_000 });
test.describe.configure({ timeout: 240_000 });

const SHOTS = 'test-results/data/shots';
// The seed's SaaS project is the one whose users talk to an assistant.
const SAAS = '/acme/acme-saas';
const NOISE = [
  /api\.openpanel\.dev/,
  /favicon/,
  /sentry/i,
  /\[vite\]/,
  /Failed to load resource/,
  /A tree hydrated but some attributes/,
  /requires a `DialogTitle`/,
];

function watch(page: Page) {
  const problems: string[] = [];
  page.on('console', (message) => {
    if (message.type() !== 'error') {
      return;
    }
    const text = message.text();
    if (!NOISE.some((pattern) => pattern.test(text))) {
      problems.push(`console: ${text.slice(0, 400)}`);
    }
  });
  page.on('pageerror', (error) => {
    problems.push(`pageerror: ${error.message.slice(0, 400)}`);
  });
  page.on('response', (response) => {
    const url = response.url();
    if (response.status() >= 400 && !NOISE.some((p) => p.test(url))) {
      problems.push(`http ${response.status()}: ${url.slice(0, 200)}`);
    }
  });
  return problems;
}

async function hydrated(page: Page) {
  await page.waitForLoadState('networkidle');
  await expect(page.getByRole('combobox').first()).toContainText('Acme', {
    timeout: 45_000,
  });
}

async function open(page: Page, path: string) {
  await page.context().addCookies([
    {
      name: 'feedback-prompt-seen',
      value: new Date().toISOString(),
      url: process.env.DASHBOARD_URL ?? 'http://localhost:3000',
    },
  ]);
  for (const attempt of [1, 2, 3]) {
    await page.goto(path);
    try {
      await hydrated(page);
      return;
    } catch (error) {
      if (attempt === 3) {
        throw error;
      }
    }
  }
}

const tableRows = (page: Page) => page.locator('tbody tr');

test.describe('ai observability', () => {
  test('overview shows cost, tokens and breakdowns for the seeded assistant', async ({
    page,
  }) => {
    const problems = watch(page);
    await open(page, `${SAAS}/ai?range=30d`);

    await expect(
      page.getByRole('heading', { level: 1, name: /^AI/ })
    ).toBeVisible();
    await expect(page.getByText('AI observability is in beta')).toBeVisible();
    await expect(
      page.getByRole('button', { name: 'Give feedback' }).last()
    ).toBeVisible();
    const cards = page.locator('.card').first();
    await expect(cards).toContainText('Cost');
    await expect(cards).toContainText('Calls');
    await expect(cards).toContainText('Latency p95');
    // The seeded SaaS project has four models; the cost widget lists them.
    await expect(page.getByText('Cost by model')).toBeVisible();
    await expect(page.getByText('gpt-5-mini').first()).toBeVisible();
    await expect(page.getByText('claude-sonnet-5-5').first()).toBeVisible();
    await expect(page.getByText('Top users')).toBeVisible();
    await page.screenshot({ path: `${SHOTS}/ai-overview.png`, fullPage: true });

    // "Open as report" lands in the editor with the widget's series and breakdown.
    await page.getByRole('link', { name: 'Open as report' }).nth(1).click();
    await expect(page).toHaveURL(/\/reports\?.*draft=/);
    await expect(page.getByText('gpt-5-mini').first()).toBeVisible();
    await page.screenshot({
      path: `${SHOTS}/ai-open-as-report.png`,
      fullPage: true,
    });

    expect(problems).toEqual([]);
  });

  test('traces list opens a trace with its timeline and conversation', async ({
    page,
  }) => {
    const problems = watch(page);
    await open(page, `${SAAS}/ai/traces?range=30d`);

    await expect(tableRows(page).first()).toBeVisible();
    // A model without a catalog price shows — instead of a cost.
    // A seeded chat trace (it has a conversation); real SDK calls in the same project come first.
    const seededTrace = tableRows(page)
      .filter({ hasText: 'assistant-chat' })
      .first();
    await expect(seededTrace).toContainText('$');
    await page.screenshot({ path: `${SHOTS}/ai-traces.png`, fullPage: true });

    // The profile cell is a link of its own; the start time opens the trace.
    await seededTrace.locator('td').first().click();
    await expect(page).toHaveURL(/\/ai\/traces\//);
    await expect(page.getByText('Timeline')).toBeVisible();
    await expect(
      page.getByRole('button', { name: 'Conversation', exact: true })
    ).toBeVisible();
    // A trace may start with an embedding; pick the model call and expect its
    // conversation to render a user message.
    await page
      .getByRole('button', { name: /gpt-|claude-/ })
      .first()
      .click();
    await expect(
      page.locator('span', { hasText: /^user$/ }).first()
    ).toBeVisible();
    await page.screenshot({ path: `${SHOTS}/ai-trace.png`, fullPage: true });

    // The conversation link filters the traces list by the trace's conversation.
    await page.getByRole('link', { name: 'Conversation' }).click();
    await expect(page).toHaveURL(/\/ai\/traces\?.*conversation=/);
    await expect(tableRows(page).first()).toBeVisible();
    await expect(page.getByText('Something went wrong')).toHaveCount(0);

    expect(problems).toEqual([]);
  });

  test('generations open in the event drawer and filters narrow the list', async ({
    page,
  }) => {
    const problems = watch(page);
    await open(page, `${SAAS}/ai/generations?range=30d`);

    const seededGeneration = tableRows(page)
      .filter({ hasText: 'insight-summary' })
      .first();
    await expect(seededGeneration).toBeVisible();
    await seededGeneration.click();
    const drawer = page.getByRole('complementary', { name: 'Event details' });
    await expect(drawer).toBeVisible();
    await expect(drawer.getByText('Input', { exact: true })).toBeVisible();
    await expect(drawer.getByText('Output', { exact: true })).toBeVisible();
    await page.screenshot({
      path: `${SHOTS}/ai-generation-drawer.png`,
      fullPage: true,
    });

    // A picked model round-trips through the URL and narrows the rows.
    await page.getByRole('button', { name: 'Model', exact: true }).click();
    const modelItem = page.getByRole('menuitemcheckbox', {
      // Anchored: real SDK calls add dated snapshots like gpt-5-mini-2025-08-07.
      name: /^gpt-5-mini\b(?!-)/,
    });
    await modelItem.click();
    await expect(page).toHaveURL(/models=/);
    await expect(tableRows(page).first()).toContainText('gpt-5-mini');
    await expect(page.getByText('Something went wrong')).toHaveCount(0);
    await modelItem.click();
    await expect(page).not.toHaveURL(/models=/);
    await page.keyboard.press('Escape');

    await page.getByRole('button', { name: 'Errors only' }).click();
    await expect(page).toHaveURL(/errors=true/);
    await expect(tableRows(page).first()).toContainText(/\b[45]\d\d\b/);
    await page.screenshot({
      path: `${SHOTS}/ai-generations-errors.png`,
      fullPage: true,
    });

    expect(problems).toEqual([]);
  });

  test('the AI settings tab toggles content storage', async ({ page }) => {
    const problems = watch(page);
    await open(page, `${SAAS}/settings/ai`);

    const toggle = page.getByRole('switch', {
      name: 'Store prompts and completions',
    });
    await expect(toggle).toBeChecked();
    await toggle.click();
    await expect(page.locator('[data-sonner-toast]')).toContainText(
      'AI settings updated'
    );
    await expect(toggle).not.toBeChecked();
    await toggle.click();
    await expect(toggle).toBeChecked();
    await page.screenshot({ path: `${SHOTS}/ai-settings.png`, fullPage: true });

    expect(problems).toEqual([]);
  });

  test('a profile that used the assistant shows its AI usage', async ({
    page,
  }) => {
    const problems = watch(page);
    await open(page, `${SAAS}/ai/traces?range=30d`);
    const profileLink = tableRows(page)
      .first()
      .locator('a[href*="/profiles/"]')
      .first();
    await expect(profileLink).toBeVisible();
    await profileLink.click();
    await expect(page).toHaveURL(/\/profiles\//);
    await expect(page.getByText('AI usage')).toBeVisible();
    await expect(page.getByText('Calls', { exact: true })).toBeVisible();
    await page.screenshot({
      path: `${SHOTS}/ai-profile-usage.png`,
      fullPage: true,
    });

    expect(problems).toEqual([]);
  });
});