Skip to content

AI-native design editor. Open-source Figma alternative.

7.8 KB
Raw Blame
import { readFile } from 'node:fs/promises'

import type { Page } from '@playwright/test'
import * as v from 'valibot'

import { CanvasHelper } from '#tests/helpers/canvas'
import { expect, test } from '#tests/helpers/chat/fixture'
import { openAppMenu } from '#tests/helpers/menu'

/** The JSON that Copy diagnostics puts on the clipboard, as far as these tests read it. */
const reportSchema = v.pipe(
  v.string(),
  v.parseJson(),
  v.object({
    environment: v.object({ app: v.string(), shell: v.string() }),
    events: v.array(
      v.object({
        name: v.string(),
        sessionId: v.optional(v.string()),
        runId: v.optional(v.string()),
        attributes: v.record(v.string(), v.unknown())
      })
    )
  })
)

test('Settings exports correlated chat telemetry without conversation content', async ({
  configuredChat: chat,
  page,
  context
}) => {
  await context.grantPermissions(['clipboard-read', 'clipboard-write'])
  await chat.submit('Private conversation content must not enter diagnostics')
  await expect(chat.assistantMessage()).toBeVisible()
  await expect(chat.input).toBeEnabled()
  await chat.submit('A second request in the same conversation')
  await expect(page.getByTestId('chat-message-assistant')).toHaveCount(2)
  await page.getByTestId('app-settings-trigger').click()
  await page.getByTestId('settings-section-diagnostics').click()
  await page.getByRole('button', { name: 'Copy diagnostics', exact: true }).click()
  await expect(page.getByRole('button', { name: 'Copied', exact: true })).toBeVisible()
  const text = await page.evaluate(() => navigator.clipboard.readText())
  const { environment, events } = v.parse(reportSchema, text)
  expect(environment).toMatchObject({ app: expect.any(String), shell: 'browser' })
  const completed = events.filter((event) => event.name === 'chat.completed')
  expect(completed).toHaveLength(2)
  expect(completed[0].sessionId).toEqual(expect.any(String))
  expect(completed[1].sessionId).toBe(completed[0].sessionId)
  expect(completed[0].runId).toEqual(expect.any(String))
  expect(completed[1].runId).toEqual(expect.any(String))
  expect(completed[1].runId).not.toBe(completed[0].runId)
  expect(text).not.toContain('Private conversation content')
  expect(text).not.toContain('A second request')
})

test('Settings saves the diagnostics report as a file', async ({ configuredChat: chat, page }) => {
  // Without a save picker the browser downloads the file.
  await page.evaluate(() => Reflect.deleteProperty(window, 'showSaveFilePicker'))
  await chat.submit('A request to record')
  await expect(chat.assistantMessage()).toBeVisible()
  await openDiagnostics(page)

  const [download] = await Promise.all([
    page.waitForEvent('download'),
    page.getByRole('button', { name: 'Save diagnostics…', exact: true }).click()
  ])

  expect(download.suggestedFilename()).toMatch(/\.json$/)
  const { environment, events } = v.parse(
    reportSchema,
    await readFile(await download.path(), 'utf8')
  )
  expect(environment).toMatchObject({ shell: 'browser' })
  expect(events.map((event) => event.name)).toContain('chat.completed')
})

async function openDiagnostics(page: Page) {
  await page.keyboard.press(process.platform === 'darwin' ? 'Meta+,' : 'Control+,')
  await page.getByTestId('settings-section-diagnostics').click()
}

async function reloadAndOpenDiagnostics(page: Page) {
  await page.reload()
  await new CanvasHelper(page).waitForInit()
  await openDiagnostics(page)
}

test('diagnostics retention offers presets and accepts a bounded custom value', async ({
  page
}) => {
  await page.goto('/?test')
  await new CanvasHelper(page).waitForInit()
  await openDiagnostics(page)
  const row = page
    .locator('[data-slot="settings-row"]')
    .filter({ hasText: 'Diagnostics retention' })
  await expect(row).toContainText('Keep up to this many recent events locally.')
  const retention = row.getByRole('combobox', { name: 'Diagnostics retention' })
  await expect(retention).toHaveText('500')

  // A custom value is validated against the supported range before it is kept.
  await retention.click()
  await page.getByRole('option', { name: 'Custom…' }).click()
  const custom = row.getByRole('spinbutton', { name: /Diagnostics retention/ })
  await custom.fill('10')
  await custom.press('Enter')
  await expect(custom).toHaveAttribute('aria-invalid', 'true')
  await expect(custom).toHaveAccessibleDescription(/Enter a whole number from 50 to 20000/)

  await custom.fill('750')
  await custom.press('Enter')
  await expect(custom).toHaveAttribute('aria-invalid', 'false')

  await reloadAndOpenDiagnostics(page)
  await expect(retention).toHaveText('Custom…')
  await expect(row.getByRole('spinbutton', { name: /Diagnostics retention/ })).toHaveValue('750')

  // Presets remain one click and persist without revealing the field.
  await retention.click()
  await page.getByRole('option', { name: '1000', exact: true }).click()
  await expect(retention).toHaveText('1000')
  await reloadAndOpenDiagnostics(page)
  await expect(retention).toHaveText('1000')
  await expect(row.getByRole('spinbutton')).toHaveCount(0)
})

test('uncaught errors are listed with their stack and can be filtered and copied', async ({
  configuredChat: chat,
  page,
  context
}) => {
  await context.grantPermissions(['clipboard-read', 'clipboard-write'])
  await chat.submit('A request that records info events')
  await expect(chat.assistantMessage()).toBeVisible()
  await page.evaluate(() => {
    setTimeout(() => {
      throw new TypeError('Attempting to define property on object that is not extensible.')
    })
  })
  await openDiagnostics(page)

  const events = page.locator('[data-slot="diagnostics-events"]')
  const failure = events.getByRole('button', { name: /Error: TypeError/ })
  await expect(failure).toContainText(
    'Attempting to define property on object that is not extensible.'
  )
  await failure.click()
  await expect(events.locator('pre')).toContainText('TypeError')
  // Events recorded before Settings opened are counted.
  await expect(page.getByText(/^[1-9]\d* events · /)).toBeVisible()

  // Info events such as the completed chat drop out when only problems are shown.
  await expect(events.getByRole('button', { name: /AI chat completed/ })).toBeVisible()
  await events.getByRole('button', { name: 'Errors and warnings' }).click()
  await expect(events.getByRole('button', { name: /AI chat completed/ })).toHaveCount(0)
  await expect(failure).toBeVisible()

  await page.getByRole('button', { name: 'Copy diagnostics', exact: true }).click()
  // The clipboard outlives the test's page, so read it only once this copy has finished.
  await expect(page.getByRole('button', { name: 'Copied', exact: true })).toBeVisible()
  const report = v.parse(reportSchema, await page.evaluate(() => navigator.clipboard.readText()))
  const runtime = report.events.find((event) => event.name === 'runtime.error')
  expect(runtime?.attributes).toMatchObject({ source: 'window', errorName: 'TypeError' })
  expect(String(runtime?.attributes.stack)).toContain('TypeError')
})

test('File › Diagnostics opens its own page, and usage history is switched on the Usage page', async ({
  page
}) => {
  await page.goto('/?test')
  await new CanvasHelper(page).waitForInit()
  const menu = await openAppMenu(page, 'File')
  await menu.getByRole('menuitem', { name: 'Diagnostics…' }).click()
  await expect(page.getByTestId('settings-section-diagnostics')).toHaveAttribute(
    'aria-selected',
    'true'
  )
  await expect(page.getByTestId('settings-diagnostics-panel')).toBeVisible()
  await expect(page.getByRole('switch', { name: 'Usage history' })).toHaveCount(0)

  await page.getByTestId('settings-section-usage').click()
  const history = page
    .getByTestId('settings-usage-panel')
    .getByRole('switch', { name: 'Usage history' })
  await expect(history).toHaveAttribute('data-state', 'checked')
})