OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
import type { Page } from '@playwright/test';
import { expect as baseExpect, test } from './fixtures';
// The dev server shares a machine with other suites; default timeouts are too tight.
const expect = baseExpect.configure({ timeout: 30_000 });
test.describe.configure({ timeout: 240_000 });
const SHOTS = 'test-results/data/shots';
// The seed's SaaS project is the one whose users talk to an assistant.
const SAAS = '/acme/acme-saas';
const NOISE = [
/api\.openpanel\.dev/,
/favicon/,
/sentry/i,
/\[vite\]/,
/Failed to load resource/,
/A tree hydrated but some attributes/,
/requires a `DialogTitle`/,
];
function watch(page: Page) {
const problems: string[] = [];
page.on('console', (message) => {
if (message.type() !== 'error') {
return;
}
const text = message.text();
if (!NOISE.some((pattern) => pattern.test(text))) {
problems.push(`console: ${text.slice(0, 400)}`);
}
});
page.on('pageerror', (error) => {
problems.push(`pageerror: ${error.message.slice(0, 400)}`);
});
page.on('response', (response) => {
const url = response.url();
if (response.status() >= 400 && !NOISE.some((p) => p.test(url))) {
problems.push(`http ${response.status()}: ${url.slice(0, 200)}`);
}
});
return problems;
}
async function hydrated(page: Page) {
await page.waitForLoadState('networkidle');
await expect(page.getByRole('combobox').first()).toContainText('Acme', {
timeout: 45_000,
});
}
async function open(page: Page, path: string) {
await page.context().addCookies([
{
name: 'feedback-prompt-seen',
value: new Date().toISOString(),
url: process.env.DASHBOARD_URL ?? 'http://localhost:3000',
},
]);
for (const attempt of [1, 2, 3]) {
await page.goto(path);
try {
await hydrated(page);
return;
} catch (error) {
if (attempt === 3) {
throw error;
}
}
}
}
const tableRows = (page: Page) => page.locator('tbody tr');
test.describe('ai observability', () => {
test('overview shows cost, tokens and breakdowns for the seeded assistant', async ({
page,
}) => {
const problems = watch(page);
await open(page, `${SAAS}/ai?range=30d`);
await expect(
page.getByRole('heading', { level: 1, name: /^AI/ })
).toBeVisible();
await expect(page.getByText('AI observability is in beta')).toBeVisible();
await expect(
page.getByRole('button', { name: 'Give feedback' }).last()
).toBeVisible();
const cards = page.locator('.card').first();
await expect(cards).toContainText('Cost');
await expect(cards).toContainText('Calls');
await expect(cards).toContainText('Latency p95');
// The seeded SaaS project has four models; the cost widget lists them.
await expect(page.getByText('Cost by model')).toBeVisible();
await expect(page.getByText('gpt-5-mini').first()).toBeVisible();
await expect(page.getByText('claude-sonnet-5-5').first()).toBeVisible();
await expect(page.getByText('Top users')).toBeVisible();
await page.screenshot({ path: `${SHOTS}/ai-overview.png`, fullPage: true });
// "Open as report" lands in the editor with the widget's series and breakdown.
await page.getByRole('link', { name: 'Open as report' }).nth(1).click();
await expect(page).toHaveURL(/\/reports\?.*draft=/);
await expect(page.getByText('gpt-5-mini').first()).toBeVisible();
await page.screenshot({
path: `${SHOTS}/ai-open-as-report.png`,
fullPage: true,
});
expect(problems).toEqual([]);
});
test('traces list opens a trace with its timeline and conversation', async ({
page,
}) => {
const problems = watch(page);
await open(page, `${SAAS}/ai/traces?range=30d`);
await expect(tableRows(page).first()).toBeVisible();
// A model without a catalog price shows — instead of a cost.
// A seeded chat trace (it has a conversation); real SDK calls in the same project come first.
const seededTrace = tableRows(page)
.filter({ hasText: 'assistant-chat' })
.first();
await expect(seededTrace).toContainText('$');
await page.screenshot({ path: `${SHOTS}/ai-traces.png`, fullPage: true });
// The profile cell is a link of its own; the start time opens the trace.
await seededTrace.locator('td').first().click();
await expect(page).toHaveURL(/\/ai\/traces\//);
await expect(page.getByText('Timeline')).toBeVisible();
await expect(
page.getByRole('button', { name: 'Conversation', exact: true })
).toBeVisible();
// A trace may start with an embedding; pick the model call and expect its
// conversation to render a user message.
await page
.getByRole('button', { name: /gpt-|claude-/ })
.first()
.click();
await expect(
page.locator('span', { hasText: /^user$/ }).first()
).toBeVisible();
await page.screenshot({ path: `${SHOTS}/ai-trace.png`, fullPage: true });
// The conversation link filters the traces list by the trace's conversation.
await page.getByRole('link', { name: 'Conversation' }).click();
await expect(page).toHaveURL(/\/ai\/traces\?.*conversation=/);
await expect(tableRows(page).first()).toBeVisible();
await expect(page.getByText('Something went wrong')).toHaveCount(0);
expect(problems).toEqual([]);
});
test('generations open in the event drawer and filters narrow the list', async ({
page,
}) => {
const problems = watch(page);
await open(page, `${SAAS}/ai/generations?range=30d`);
const seededGeneration = tableRows(page)
.filter({ hasText: 'insight-summary' })
.first();
await expect(seededGeneration).toBeVisible();
await seededGeneration.click();
const drawer = page.getByRole('complementary', { name: 'Event details' });
await expect(drawer).toBeVisible();
await expect(drawer.getByText('Input', { exact: true })).toBeVisible();
await expect(drawer.getByText('Output', { exact: true })).toBeVisible();
await page.screenshot({
path: `${SHOTS}/ai-generation-drawer.png`,
fullPage: true,
});
// A picked model round-trips through the URL and narrows the rows.
await page.getByRole('button', { name: 'Model', exact: true }).click();
const modelItem = page.getByRole('menuitemcheckbox', {
// Anchored: real SDK calls add dated snapshots like gpt-5-mini-2025-08-07.
name: /^gpt-5-mini\b(?!-)/,
});
await modelItem.click();
await expect(page).toHaveURL(/models=/);
await expect(tableRows(page).first()).toContainText('gpt-5-mini');
await expect(page.getByText('Something went wrong')).toHaveCount(0);
await modelItem.click();
await expect(page).not.toHaveURL(/models=/);
await page.keyboard.press('Escape');
await page.getByRole('button', { name: 'Errors only' }).click();
await expect(page).toHaveURL(/errors=true/);
await expect(tableRows(page).first()).toContainText(/\b[45]\d\d\b/);
await page.screenshot({
path: `${SHOTS}/ai-generations-errors.png`,
fullPage: true,
});
expect(problems).toEqual([]);
});
test('the AI settings tab toggles content storage', async ({ page }) => {
const problems = watch(page);
await open(page, `${SAAS}/settings/ai`);
const toggle = page.getByRole('switch', {
name: 'Store prompts and completions',
});
await expect(toggle).toBeChecked();
await toggle.click();
await expect(page.locator('[data-sonner-toast]')).toContainText(
'AI settings updated'
);
await expect(toggle).not.toBeChecked();
await toggle.click();
await expect(toggle).toBeChecked();
await page.screenshot({ path: `${SHOTS}/ai-settings.png`, fullPage: true });
expect(problems).toEqual([]);
});
test('a profile that used the assistant shows its AI usage', async ({
page,
}) => {
const problems = watch(page);
await open(page, `${SAAS}/ai/traces?range=30d`);
const profileLink = tableRows(page)
.first()
.locator('a[href*="/profiles/"]')
.first();
await expect(profileLink).toBeVisible();
await profileLink.click();
await expect(page).toHaveURL(/\/profiles\//);
await expect(page.getByText('AI usage')).toBeVisible();
await expect(page.getByText('Calls', { exact: true })).toBeVisible();
await page.screenshot({
path: `${SHOTS}/ai-profile-usage.png`,
fullPage: true,
});
expect(problems).toEqual([]);
});
});