Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

19.7 KB
Raw Blame
import { mkdirSync, writeFileSync } from 'node:fs';
import { dirname, resolve } from 'node:path';
import type { APIRequestContext, Page, Response } from '@playwright/test';

export const ORGANIZATION_ID = 'acme';
export const API_URL =
  process.env.API_URL ?? 'https://api.main.local.openpanel.cc';
export const OUTPUT_DIR = resolve(
  import.meta.dirname,
  '..',
  'test-results',
  'sweep'
);
export const SLOW_LOAD_MS = 5000;
const NETWORK_IDLE_TIMEOUT_MS = 12_000;
const BLANK_PAGE_MAX_CHARS = 20;
const SNIPPET_RADIUS = 40;

type IdKind =
  | 'profile'
  | 'session'
  | 'group'
  | 'cohort'
  | 'dashboard'
  | 'report';

export type ProjectIds = Record<IdKind, string | null>;

export interface RouteSpec {
  /** Route file pattern, used as the row label. */
  route: string;
  /** Path below `/$organizationId/$projectId`, or an absolute path when `scope` is not project. */
  path: (ids: ProjectIds) => string | null;
  scope: 'project' | 'organization' | 'global';
}

const fixed = (path: string) => () => path;
const withId = (kind: IdKind, suffix = '') => {
  const plural = `${kind}s`;
  return (ids: ProjectIds) =>
    ids[kind] ? `/${plural}/${encodeURIComponent(ids[kind])}${suffix}` : null;
};

export const PROJECT_ROUTES: RouteSpec[] = [
  { route: '$projectId/ (overview)', path: fixed(''), scope: 'project' },
  { route: 'pages', path: fixed('/pages'), scope: 'project' },
  { route: 'realtime', path: fixed('/realtime'), scope: 'project' },
  { route: 'insights', path: fixed('/insights'), scope: 'project' },
  { route: 'seo', path: fixed('/seo'), scope: 'project' },
  { route: 'references', path: fixed('/references'), scope: 'project' },
  { route: 'dashboards', path: fixed('/dashboards'), scope: 'project' },
  {
    route: 'dashboards/$dashboardId',
    path: withId('dashboard'),
    scope: 'project',
  },
  { route: 'reports (new)', path: fixed('/reports'), scope: 'project' },
  { route: 'reports/$reportId', path: withId('report'), scope: 'project' },
  { route: 'events', path: fixed('/events'), scope: 'project' },
  { route: 'events/events', path: fixed('/events/events'), scope: 'project' },
  {
    route: 'events/conversions',
    path: fixed('/events/conversions'),
    scope: 'project',
  },
  { route: 'events/stats', path: fixed('/events/stats'), scope: 'project' },
  { route: 'sessions', path: fixed('/sessions'), scope: 'project' },
  { route: 'sessions/$sessionId', path: withId('session'), scope: 'project' },
  { route: 'profiles', path: fixed('/profiles'), scope: 'project' },
  {
    route: 'profiles/identified',
    path: fixed('/profiles/identified'),
    scope: 'project',
  },
  {
    route: 'profiles/anonymous',
    path: fixed('/profiles/anonymous'),
    scope: 'project',
  },
  {
    route: 'profiles/power-users',
    path: fixed('/profiles/power-users'),
    scope: 'project',
  },
  { route: 'profiles/$profileId', path: withId('profile'), scope: 'project' },
  {
    route: 'profiles/$profileId/events',
    path: withId('profile', '/events'),
    scope: 'project',
  },
  {
    route: 'profiles/$profileId/sessions',
    path: withId('profile', '/sessions'),
    scope: 'project',
  },
  { route: 'groups', path: fixed('/groups'), scope: 'project' },
  { route: 'groups/$groupId', path: withId('group'), scope: 'project' },
  {
    route: 'groups/$groupId/events',
    path: withId('group', '/events'),
    scope: 'project',
  },
  {
    route: 'groups/$groupId/members',
    path: withId('group', '/members'),
    scope: 'project',
  },
  { route: 'cohorts', path: fixed('/cohorts'), scope: 'project' },
  { route: 'cohorts/$cohortId', path: withId('cohort'), scope: 'project' },
  {
    route: 'cohorts/$cohortId/events',
    path: withId('cohort', '/events'),
    scope: 'project',
  },
  {
    route: 'cohorts/$cohortId/members',
    path: withId('cohort', '/members'),
    scope: 'project',
  },
  { route: 'integrations', path: fixed('/integrations'), scope: 'project' },
  {
    route: 'integrations/available',
    path: fixed('/integrations/available'),
    scope: 'project',
  },
  {
    route: 'integrations/installed',
    path: fixed('/integrations/installed'),
    scope: 'project',
  },
  { route: 'notifications', path: fixed('/notifications'), scope: 'project' },
  {
    route: 'notifications/notifications',
    path: fixed('/notifications/notifications'),
    scope: 'project',
  },
  {
    route: 'notifications/rules',
    path: fixed('/notifications/rules'),
    scope: 'project',
  },
  { route: 'settings', path: fixed('/settings'), scope: 'project' },
  {
    route: 'settings/details',
    path: fixed('/settings/details'),
    scope: 'project',
  },
  {
    route: 'settings/events',
    path: fixed('/settings/events'),
    scope: 'project',
  },
  {
    route: 'settings/clients',
    path: fixed('/settings/clients'),
    scope: 'project',
  },
  {
    route: 'settings/tracking',
    path: fixed('/settings/tracking'),
    scope: 'project',
  },
  {
    route: 'settings/widgets',
    path: fixed('/settings/widgets'),
    scope: 'project',
  },
  {
    route: 'settings/imports',
    path: fixed('/settings/imports'),
    scope: 'project',
  },
  { route: 'settings/mcp', path: fixed('/settings/mcp'), scope: 'project' },
  { route: 'settings/gsc', path: fixed('/settings/gsc'), scope: 'project' },
];

export const ORGANIZATION_ROUTES: RouteSpec[] = [
  { route: '/ (index)', path: fixed('/'), scope: 'global' },
  { route: '$organizationId/', path: fixed(''), scope: 'organization' },
  {
    route: '$organizationId/settings',
    path: fixed('/settings'),
    scope: 'organization',
  },
  {
    route: '$organizationId/billing',
    path: fixed('/billing'),
    scope: 'organization',
  },
  {
    route: '$organizationId/members',
    path: fixed('/members'),
    scope: 'organization',
  },
  {
    route: '$organizationId/members/members',
    path: fixed('/members/members'),
    scope: 'organization',
  },
  {
    route: '$organizationId/members/invitations',
    path: fixed('/members/invitations'),
    scope: 'organization',
  },
  {
    route: '$organizationId/account',
    path: fixed('/account'),
    scope: 'organization',
  },
  {
    route: '$organizationId/account/email-preferences',
    path: fixed('/account/email-preferences'),
    scope: 'organization',
  },
  {
    route: '$organizationId/account/two-factor',
    path: fixed('/account/two-factor'),
    scope: 'organization',
  },
  { route: '/account', path: fixed('/account'), scope: 'global' },
  { route: '/login (signed in)', path: fixed('/login'), scope: 'global' },
  {
    route: '/reset-password (no token)',
    path: fixed('/reset-password'),
    scope: 'global',
  },
  { route: '/verify', path: fixed('/verify'), scope: 'global' },
  { route: '/onboarding', path: fixed('/onboarding'), scope: 'global' },
  {
    route: '/onboarding/project',
    path: fixed('/onboarding/project'),
    scope: 'global',
  },
  {
    route: '/onboarding/$projectId/connect',
    path: fixed('/onboarding/acme-shop/connect'),
    scope: 'global',
  },
  {
    route: '/onboarding/$projectId/verify',
    path: fixed('/onboarding/acme-shop/verify'),
    scope: 'global',
  },
  {
    route: '/unsubscribe (no token)',
    path: fixed('/unsubscribe'),
    scope: 'global',
  },
];

export function resolveUrl(
  spec: RouteSpec,
  projectId: string,
  ids: ProjectIds
): string | null {
  const path = spec.path(ids);
  if (path === null) {
    return null;
  }
  if (spec.scope === 'project') {
    return `/${ORGANIZATION_ID}/${projectId}${path}`;
  }
  if (spec.scope === 'organization') {
    return `/${ORGANIZATION_ID}${path}`;
  }
  return path;
}

export async function trpcQuery<T>(
  request: APIRequestContext,
  procedure: string,
  input: unknown
): Promise<T | null> {
  const response = await request.get(`${API_URL}/trpc/${procedure}`, {
    params: { input: JSON.stringify({ json: input }) },
    ignoreHTTPSErrors: true,
  });
  if (!response.ok()) {
    return null;
  }
  const body = await response.json();
  return (body?.result?.data?.json ?? null) as T | null;
}

export async function resolveProjectIds(
  request: APIRequestContext,
  projectId: string
): Promise<ProjectIds> {
  const [profiles, sessions, groups, cohorts, dashboards] = await Promise.all([
    trpcQuery<{ data: { id: string }[] }>(request, 'profile.list', {
      projectId,
      take: 1,
    }),
    trpcQuery<{ items: { id: string }[] }>(request, 'session.list', {
      projectId,
      take: 1,
    }),
    trpcQuery<{ data: { id: string }[] }>(request, 'group.list', {
      projectId,
      take: 1,
    }),
    trpcQuery<{ id: string }[]>(request, 'cohort.list', { projectId }),
    trpcQuery<{ id: string }[]>(request, 'dashboard.list', { projectId }),
  ]);
  const dashboard = dashboards?.[0]?.id ?? null;
  const reports = dashboard
    ? await trpcQuery<{ id: string }[]>(request, 'report.list', {
        projectId,
        dashboardId: dashboard,
      })
    : null;
  return {
    profile: profiles?.data?.[0]?.id ?? null,
    session: sessions?.items?.[0]?.id ?? null,
    group: groups?.data?.[0]?.id ?? null,
    cohort: cohorts?.[0]?.id ?? null,
    dashboard,
    report: reports?.[0]?.id ?? null,
  };
}

export interface BadResponse {
  status: number;
  procedure: string;
  message: string;
}

export interface Recorder {
  consoleErrors: string[];
  consoleWarnings: string[];
  pageErrors: string[];
  badResponses: BadResponse[];
  failedRequests: string[];
  /** Every 401 and every document navigation of the whole run; never reset, to explain surprise redirects. */
  authTrail: string[];
  reset: () => void;
  snapshot: () => Omit<Recorder, 'reset' | 'snapshot' | 'authTrail'>;
}

const NOISE = [
  /\[vite\]/i,
  /favicon/i,
  /sentry/i,
  /Download the React DevTools/i,
  /hot[- ]update/i,
  /api\.openpanel\.dev\/track/,
];

const isNoise = (text: string) => NOISE.some((pattern) => pattern.test(text));

async function describeBadResponse(response: Response): Promise<BadResponse> {
  const url = new URL(response.url());
  const procedure = url.pathname.includes('/trpc/')
    ? url.pathname.split('/trpc/')[1]
    : `${response.request().method()} ${url.host}${url.pathname}`;
  let message = '';
  try {
    const text = await response.text();
    const parsed = JSON.parse(text);
    const first = Array.isArray(parsed) ? parsed.find((p) => p.error) : parsed;
    message =
      first?.error?.json?.message ??
      first?.error?.message ??
      first?.message ??
      text.slice(0, 200);
  } catch {
    message = '(no readable body)';
  }
  return {
    status: response.status(),
    procedure,
    message: String(message).slice(0, 300),
  };
}

export function attachRecorder(page: Page): Recorder {
  const recorder: Recorder = {
    consoleErrors: [],
    consoleWarnings: [],
    pageErrors: [],
    badResponses: [],
    failedRequests: [],
    authTrail: [],
    reset() {
      recorder.consoleErrors = [];
      recorder.consoleWarnings = [];
      recorder.pageErrors = [];
      recorder.badResponses = [];
      recorder.failedRequests = [];
    },
    snapshot() {
      return {
        consoleErrors: [...recorder.consoleErrors],
        consoleWarnings: [...recorder.consoleWarnings],
        pageErrors: [...recorder.pageErrors],
        badResponses: [...recorder.badResponses],
        failedRequests: [...recorder.failedRequests],
      };
    },
  };
  page.on('console', (message) => {
    const text = message.text();
    if (isNoise(`${text} ${message.location().url}`)) {
      return;
    }
    if (message.type() === 'error') {
      recorder.consoleErrors.push(text.slice(0, 6000));
    }
    if (message.type() === 'warning') {
      recorder.consoleWarnings.push(text.slice(0, 600));
    }
  });
  page.on('pageerror', (error) => {
    recorder.pageErrors.push(error.message.slice(0, 8000));
  });
  page.on('response', (response) => {
    const isDocument = response.request().resourceType() === 'document';
    if (response.status() === 401 || isDocument) {
      recorder.authTrail.push(
        `${new Date().toISOString()} ${response.status()} ${response.url().slice(0, 160)} (page at ${page.url()})`
      );
    }
    if (response.status() < 400 || isNoise(response.url())) {
      return;
    }
    describeBadResponse(response).then((bad) =>
      recorder.badResponses.push(bad)
    );
  });
  page.on('requestfailed', (request) => {
    const failure = request.failure()?.errorText ?? '';
    const isAborted = failure.includes('ERR_ABORTED');
    if (isAborted || isNoise(request.url())) {
      return;
    }
    recorder.failedRequests.push(`${request.url()} ${failure}`.slice(0, 300));
  });
  return recorder;
}

const SUSPICIOUS_TEXT: [string, RegExp][] = [
  ['undefined', /\bundefined\b/],
  ['NaN', /\bNaN\b/],
  ['null', /\bnull\b/],
  ['Invalid Date', /Invalid Date/i],
  ['No match', /No match/],
  ['[object Object]', /\[object Object\]/],
  ['Something went wrong', /Something went wrong/i],
  ['Not found', /\bnot found\b/i],
  ['error boundary', /unexpected error|an error occurred|try again/i],
];

export interface PageState {
  suspicious: { label: string; snippet: string }[];
  isBlank: boolean;
  textLength: number;
  horizontalOverflowPx: number;
  title: string;
}

export async function inspectPage(page: Page): Promise<PageState> {
  const { text, overflow, title } = await page.evaluate(() => ({
    text: document.body?.innerText ?? '',
    overflow:
      document.documentElement.scrollWidth -
      document.documentElement.clientWidth,
    title: document.title,
  }));
  const suspicious: PageState['suspicious'] = [];
  for (const [label, pattern] of SUSPICIOUS_TEXT) {
    const match = pattern.exec(text);
    if (!match) {
      continue;
    }
    const start = Math.max(0, match.index - SNIPPET_RADIUS);
    const end = match.index + match[0].length + SNIPPET_RADIUS;
    suspicious.push({
      label,
      snippet: text.slice(start, end).replace(/\s+/g, ' '),
    });
  }
  return {
    suspicious,
    isBlank: text.trim().length < BLANK_PAGE_MAX_CHARS,
    textLength: text.trim().length,
    horizontalOverflowPx: overflow,
    title,
  };
}

const HYDRATION_TIMEOUT_MS = 45_000;
const CLIENT_NAVIGATION_START_MS = 700;
const SKELETON_TIMEOUT_MS = 15_000;
const SKELETON_SELECTOR = '.animate-pulse';

/** The dev server ships ~1200 unbundled modules, so React attaches many seconds after `load`. */
export async function waitForHydration(page: Page): Promise<boolean> {
  try {
    await page.waitForFunction(
      () =>
        Array.from(document.querySelectorAll('body div, body a, body button'))
          .slice(0, 40)
          .some((element) =>
            Object.keys(element).some((key) => key.startsWith('__react'))
          ),
      undefined,
      { timeout: HYDRATION_TIMEOUT_MS }
    );
    return true;
  } catch {
    return false;
  }
}

/** Waits for the page to settle; returns false when the network never went idle. */
export async function settle(page: Page): Promise<boolean> {
  await waitForHydration(page);
  let wentIdle = true;
  try {
    await page.waitForLoadState('networkidle', {
      timeout: NETWORK_IDLE_TIMEOUT_MS,
    });
  } catch {
    wentIdle = false;
  }
  await page
    .waitForFunction(
      (selector) => document.querySelectorAll(selector).length === 0,
      SKELETON_SELECTOR,
      { timeout: SKELETON_TIMEOUT_MS }
    )
    .catch(() => undefined);
  return wentIdle;
}

export const countSkeletons = (page: Page) =>
  page.locator(SKELETON_SELECTOR).count();

/** Client-side navigation: TanStack Router patches `history.pushState`, so this runs loaders without a document load. */
export async function clientNavigate(page: Page, url: string) {
  await page.evaluate((target) => {
    window.history.pushState({}, '', target);
  }, url);
}

export interface Visit {
  route: string;
  project: string;
  url: string | null;
  finalUrl?: string;
  status?: number | null;
  loadMs?: number;
  wentIdle?: boolean;
  state?: PageState;
  consoleErrors?: string[];
  consoleWarnings?: string[];
  pageErrors?: string[];
  badResponses?: BadResponse[];
  failedRequests?: string[];
  screenshot?: string;
  navigationError?: string;
  hydrateMs?: number;
  hydrated?: boolean;
  attempts?: number;
  skeletonsLeft?: number;
}

const MAX_LOAD_ATTEMPTS = 3;

/**
 * The dev server answers 502 for some of its ~1200 modules when the machine is
 * saturated; the client entry then never runs. That is not an app defect, so a
 * visit that never hydrated is loaded again before it is recorded.
 */
export async function visit(
  page: Page,
  recorder: Recorder,
  label: { route: string; project: string },
  url: string,
  screenshotPath?: string
): Promise<Visit> {
  const result: Visit = { ...label, url };
  for (let attempt = 1; attempt <= MAX_LOAD_ATTEMPTS; attempt++) {
    recorder.reset();
    const startedAt = Date.now();
    delete result.navigationError;
    result.attempts = attempt;
    try {
      const response = await page.goto(url, { waitUntil: 'load' });
      result.status = response?.status() ?? null;
      result.hydrated = await waitForHydration(page);
      result.hydrateMs = Date.now() - startedAt;
      if (!result.hydrated && attempt < MAX_LOAD_ATTEMPTS) {
        continue;
      }
      result.wentIdle = await settle(page);
      result.skeletonsLeft = await countSkeletons(page);
      result.loadMs = Date.now() - startedAt;
      result.finalUrl =
        new URL(page.url()).pathname + new URL(page.url()).search;
      result.state = await inspectPage(page);
      if (screenshotPath) {
        mkdirSync(dirname(screenshotPath), { recursive: true });
        await page.screenshot({ path: screenshotPath, fullPage: true });
        result.screenshot = screenshotPath;
      }
      break;
    } catch (error) {
      result.navigationError = String(error).slice(0, 500);
    }
  }
  Object.assign(result, recorder.snapshot());
  logProgress(result);
  return result;
}

/** Same record as `visit`, reached by client-side navigation from an already hydrated page. */
export async function visitClientSide(
  page: Page,
  recorder: Recorder,
  label: { route: string; project: string },
  url: string,
  screenshotPath?: string
): Promise<Visit> {
  recorder.reset();
  const startedAt = Date.now();
  const result: Visit = { ...label, url };
  try {
    await clientNavigate(page, url);
    // Loaders and queries start a tick after the history change; without this
    // pause `networkidle` resolves against the previous page.
    await page.waitForTimeout(CLIENT_NAVIGATION_START_MS);
    result.wentIdle = await settle(page);
    result.skeletonsLeft = await countSkeletons(page);
    result.loadMs = Date.now() - startedAt;
    result.finalUrl = new URL(page.url()).pathname + new URL(page.url()).search;
    result.state = await inspectPage(page);
    if (screenshotPath) {
      mkdirSync(dirname(screenshotPath), { recursive: true });
      await page.screenshot({ path: screenshotPath, fullPage: true });
      result.screenshot = screenshotPath;
    }
  } catch (error) {
    result.navigationError = String(error).slice(0, 500);
  }
  Object.assign(result, recorder.snapshot());
  logProgress(result);
  return result;
}

/** One line per visit on stdout, so a long sweep shows where it is. */
export function logProgress(result: Visit) {
  process.stdout.write(
    `${result.project} ${result.route} -> ${result.finalUrl ?? result.navigationError} ${result.loadMs ?? '?'}ms skeletons=${result.skeletonsLeft ?? '?'}\n`
  );
}

export function isClean(result: Visit): boolean {
  return (
    !result.navigationError &&
    (result.status ?? 0) < 400 &&
    !result.pageErrors?.length &&
    !result.consoleErrors?.length &&
    !result.badResponses?.length &&
    !result.state?.isBlank &&
    !result.state?.suspicious.length
  );
}

export function writeJson(name: string, data: unknown) {
  const file = resolve(OUTPUT_DIR, name);
  mkdirSync(dirname(file), { recursive: true });
  writeFileSync(file, JSON.stringify(data, null, 2));
}

export const slug = (route: string) =>
  route
    .replace(/[^a-zA-Z0-9]+/g, '-')
    .replace(/^-|-$/g, '')
    .toLowerCase() || 'root';