Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
16.8 KB
Raw Blame
import { describe, expect, it } from 'vitest'
import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import {
  ADMIN_PASSWORD_KEYS,
  ADMIN_USER_KEYS,
  CONFIRM_MUTATIONS_FLAG,
  REQUIRED_JOURNEY_INVENTORY,
  collectGateProblems,
  collectReportJourneys,
  evaluateGateReport,
  invalidatePreviousAttemptArtifacts,
  missingRequiredJourneyInventories,
  tallyReportJourneys,
  writeSanitizedRunSummary,
} from '../../e2e/run-gfs-drag-and-drop.mjs'

/**
 * The EvenDrive drag-and-drop gate is the frontend-owned entry point that
 * makes a missing setup LOUD. These regressions pin the three halves of that
 * contract: the gate refuses to run without the required setup, it never
 * reads a zero-journey or skipped-journey run as a green result, and it
 * never reads an INCOMPLETE report as one.
 */

const completeSetup = {
  [CONFIRM_MUTATIONS_FLAG]: '1',
  [ADMIN_USER_KEYS[0]]: 'admin',
  [ADMIN_PASSWORD_KEYS[0]]: 'secret-value',
  CONTROL_UI_URL: 'http://127.0.0.1:3100',
  CONTROL_API_URL: 'http://127.0.0.1:8090',
}

type Outcome = 'expected' | 'skipped' | 'unexpected' | 'flaky'

const LANE = 'control-ui'
const COMMAND = 'npm run e2e:gfs-drag-and-drop'
const FILE = 'gfs-drag-and-drop.spec.ts'

/**
 * A report shaped like the one Playwright writes: the suite tree with one
 * spec per journey, and the counters that summarise it.
 */
function journeyReport(
  journeys: Array<{ title: string; outcome?: Outcome | null; results?: number }>,
  counterOverrides: Record<string, unknown> = {}
) {
  const stats = { expected: 0, skipped: 0, unexpected: 0, flaky: 0, ...counterOverrides }
  for (const journey of journeys) {
    const outcome = journey.outcome ?? 'expected'
    if (outcome in stats) stats[outcome] += 1
  }
  return {
    suites: [
      {
        title: FILE,
        file: FILE,
        specs: journeys.map(journey => ({
          title: journey.title,
          file: FILE,
          tests: [
            {
              ...(journey.outcome === null ? {} : { status: journey.outcome ?? 'expected' }),
              expectedStatus: 'passed',
              results:
                journey.results === undefined
                  ? [{ status: journey.outcome === 'skipped' ? 'skipped' : 'passed' }]
                  : Array.from({ length: journey.results }, () => ({ status: 'passed' })),
            },
          ],
        })),
        suites: [],
      },
    ],
    stats,
  }
}

describe('EvenDrive drag-and-drop E2E gate', () => {
  describe('required setup', () => {
    it('accepts a complete setup', () => {
      expect(collectGateProblems(completeSetup)).toEqual([])
    })

    it('refuses without the live-fixture mutation opt-in', () => {
      const problems = collectGateProblems({
        ...completeSetup,
        [CONFIRM_MUTATIONS_FLAG]: undefined,
      })
      expect(problems.join('\n')).toContain(CONFIRM_MUTATIONS_FLAG)
    })

    it('refuses without an admin username', () => {
      const env = { ...completeSetup }
      for (const key of ADMIN_USER_KEYS) delete env[key]
      expect(collectGateProblems(env).join('\n')).toContain('admin username')
    })

    it('refuses without an admin password', () => {
      const env = { ...completeSetup }
      for (const key of ADMIN_PASSWORD_KEYS) delete env[key]
      expect(collectGateProblems(env).join('\n')).toContain('admin password')
    })

    it('refuses a non-loopback target unless the remote opt-in is set', () => {
      const problems = collectGateProblems({
        ...completeSetup,
        CONTROL_API_URL: 'https://api.example.test',
      })
      expect(problems.join('\n')).toContain('QA_RECORDER_ALLOW_REMOTE')
    })

    it('accepts a non-loopback target with the remote opt-in', () => {
      expect(
        collectGateProblems({
          ...completeSetup,
          CONTROL_API_URL: 'https://api.example.test',
          QA_RECORDER_ALLOW_REMOTE: '1',
        })
      ).toEqual([])
    })

    it('reports every missing piece at once instead of the first one', () => {
      const problems = collectGateProblems({})
      expect(problems.length).toBeGreaterThanOrEqual(3)
    })
  })

  describe('run verdict', () => {
    const passingReport = journeyReport([{ title: 'journey one' }, { title: 'journey two' }])

    it.each([
      { status: 1 },
      { status: null, signal: 'SIGTERM' },
      { status: null, error: new Error('spawn failed') },
    ])('rejects a green report when Playwright did not exit zero (%j)', run => {
      expect(evaluateGateReport(passingReport, run).problems.join('\n')).toContain(
        'did not exit successfully'
      )
    })

    it('rejects a run-level error even when individual journeys passed', () => {
      const report = { ...passingReport, errors: [{ message: 'teardown failed' }] }
      expect(evaluateGateReport(report).problems.join('\n')).toContain('run-level error')
    })

    it('rejects flaky journeys', () => {
      const report = journeyReport([{ title: 'journey one', outcome: 'flaky' }])
      expect(evaluateGateReport(report).problems.join('\n')).toContain('flaky')
    })

    it('rejects malformed result counts', () => {
      const report = journeyReport([{ title: 'journey one' }], { skipped: 'invalid' })
      expect(evaluateGateReport(report).problems.join('\n')).toContain('invalid result counts')
    })

    it('treats a run that executed zero journeys as a failure', () => {
      const verdict = evaluateGateReport(
        journeyReport([{ title: 'journey one', outcome: 'skipped' }])
      )
      expect(verdict.problems.join('\n')).toContain('executed zero journeys')
    })

    it('treats a skipped journey as a failure', () => {
      const verdict = evaluateGateReport(
        journeyReport([{ title: 'journey one' }, { title: 'journey two', outcome: 'skipped' }])
      )
      expect(verdict.problems.join('\n')).toContain('skipped')
    })

    it('treats a failed journey as a failure', () => {
      const verdict = evaluateGateReport(
        journeyReport([{ title: 'journey one' }, { title: 'journey two', outcome: 'unexpected' }])
      )
      expect(verdict.problems.join('\n')).toContain('failed')
    })

    it('accepts a clean run', () => {
      const verdict = evaluateGateReport(passingReport)
      expect(verdict.problems).toEqual([])
      expect(verdict.stats.expected).toBe(2)
    })

    it('treats a missing report as a failure', () => {
      expect(evaluateGateReport(undefined).problems.join('\n')).toContain('executed zero journeys')
    })
  })

  describe('incomplete reports', () => {
    it('rejects a report whose counters are incomplete', () => {
      // The reproduced defect: the missing counters used to be read as
      // successful zero counts.
      const verdict = evaluateGateReport({ stats: { expected: 1 } }, { status: 0 })
      expect(verdict.problems.length).toBeGreaterThan(0)
      expect(verdict.problems.join('\n')).toContain('invalid result counts')
    })

    it('rejects a report that lists no journeys', () => {
      const verdict = evaluateGateReport({
        stats: { expected: 1, skipped: 0, unexpected: 0, flaky: 0 },
      })
      expect(verdict.problems.join('\n')).toContain('lists no journeys')
    })

    it('rejects a report whose counters do not reconcile with its inventory', () => {
      const report = journeyReport([{ title: 'journey one' }], { expected: 5 })
      expect(evaluateGateReport(report).problems.join('\n')).toContain('does not reconcile')
    })

    it('rejects a journey the report carries no outcome for', () => {
      const report = journeyReport([{ title: 'journey one', outcome: null }])
      const problems = evaluateGateReport(report).problems.join('\n')
      expect(problems).toContain('carry no outcome')
    })

    it('rejects a journey whose recorded outcome contradicts the counters', () => {
      // Playwright counts a journey with no result as skipped; a report that
      // claims otherwise does not reconcile.
      const report = journeyReport([{ title: 'journey one', results: 0 }])
      expect(evaluateGateReport(report).problems.join('\n')).toContain('does not reconcile')
    })

    it('rejects a report that is missing a selected journey', () => {
      const selected = collectReportJourneys(
        journeyReport([{ title: 'journey one' }, { title: 'journey two' }])
      )
      const report = journeyReport([{ title: 'journey one' }])
      expect(evaluateGateReport(report, { status: 0 }, selected).problems.join('\n')).toContain(
        'missing 1 selected journey'
      )
    })

    it('rejects a report that carries a journey that was not selected', () => {
      const selected = collectReportJourneys(journeyReport([{ title: 'journey one' }]))
      const report = journeyReport([{ title: 'journey one' }, { title: 'journey two' }])
      expect(evaluateGateReport(report, { status: 0 }, selected).problems.join('\n')).toContain(
        'not selected'
      )
    })

    it('accepts a report that matches the selected journeys exactly', () => {
      const report = journeyReport([{ title: 'journey one' }, { title: 'journey two' }])
      const selected = collectReportJourneys(report)
      expect(evaluateGateReport(report, { status: 0 }, selected).problems).toEqual([])
    })
  })

  describe('sanitized run summary', () => {
    it('binds the revision, the selected cases, the outcomes and the exit code', () => {
      // A uniquely created directory, never a predictably named file directly
      // in the system temporary directory (CodeQL js/insecure-temporary-file).
      const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gfs-dnd-summary-'))
      const filePath = path.join(dir, 'playwright-summary.json')
      try {
        const report = journeyReport([{ title: 'journey one' }, { title: 'journey two' }])
        const summary = writeSanitizedRunSummary({
          lane: LANE,
          revision: 'abc123',
          branch: 'feat/example',
          command: COMMAND,
          report,
          run: { status: 0 },
          selectedJourneys: collectReportJourneys(report),
          verdict: 'pass',
          filePath,
        })

        expect(summary).toMatchObject({
          lane: LANE,
          revision: 'abc123',
          branch: 'feat/example',
          command: COMMAND,
          processExit: 0,
          verdict: 'pass',
        })
        expect(summary.selected).toEqual(['journey one', 'journey two'])
        expect(summary.journeys).toEqual([
          { title: 'journey one', status: 'expected' },
          { title: 'journey two', status: 'expected' },
        ])
        expect(JSON.parse(fs.readFileSync(filePath, 'utf8'))).toEqual(summary)
      } finally {
        fs.rmSync(dir, { recursive: true, force: true })
      }
    })

    it('does not carry the raw report text into the summary', () => {
      const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gfs-dnd-summary-'))
      const filePath = path.join(dir, 'playwright-summary.json')
      try {
        const report = {
          ...journeyReport([{ title: 'journey one', outcome: 'unexpected' }]),
          errors: [{ message: 'the upstream said: private-token-value' }],
        }
        const summary = writeSanitizedRunSummary({
          lane: LANE,
          revision: 'abc123',
          branch: 'feat/example',
          command: COMMAND,
          report,
          run: { status: 1 },
          selectedJourneys: collectReportJourneys(report),
          verdict: 'fail',
          filePath,
        })

        expect(JSON.stringify(summary)).not.toContain('private-token-value')
        expect(summary.processExit).toBe(1)
        expect(summary.verdict).toBe('fail')
      } finally {
        fs.rmSync(dir, { recursive: true, force: true })
      }
    })
  })

  describe('stale attempt artifacts', () => {
    it('removes a previous PASS summary when the next attempt starts (success → early failure)', () => {
      const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gfs-dnd-artifacts-'))
      try {
        // A previous attempt completed and passed, leaving its artifacts.
        const report = journeyReport([{ title: 'journey one' }])
        writeSanitizedRunSummary({
          lane: LANE,
          revision: 'abc123',
          branch: 'feat/example',
          command: COMMAND,
          report,
          run: { status: 0 },
          selectedJourneys: collectReportJourneys(report),
          verdict: 'pass',
          filePath: path.join(dir, 'playwright-summary.json'),
        })
        fs.writeFileSync(path.join(dir, 'playwright-report.json'), '{}')
        fs.writeFileSync(path.join(dir, 'playwright-selection.json'), '{}')
        expect(fs.existsSync(path.join(dir, 'playwright-summary.json'))).toBe(true)

        // The next attempt fails before it can write its own summary. The
        // previous pass must not survive it as apparent evidence.
        invalidatePreviousAttemptArtifacts(dir)
        for (const name of [
          'playwright-summary.json',
          'playwright-report.json',
          'playwright-selection.json',
        ]) {
          expect(fs.existsSync(path.join(dir, name))).toBe(false)
        }
      } finally {
        fs.rmSync(dir, { recursive: true, force: true })
      }
    })

    it('tolerates an artifact directory with nothing to remove', () => {
      const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gfs-dnd-artifacts-'))
      try {
        expect(() => invalidatePreviousAttemptArtifacts(dir)).not.toThrow()
      } finally {
        fs.rmSync(dir, { recursive: true, force: true })
      }
    })
  })

  describe('report inventory', () => {
    it('identifies a journey by its file and title path', () => {
      const journeys = collectReportJourneys(journeyReport([{ title: 'journey one' }]))
      expect(journeys).toHaveLength(1)
      expect(journeys[0]?.identity).toBe(`${FILE}::journey one`)
    })

    it('tallies the outcomes the way Playwright does', () => {
      const journeys = collectReportJourneys(
        journeyReport([{ title: 'one' }, { title: 'two' }, { title: 'three', outcome: 'skipped' }])
      )
      expect(tallyReportJourneys(journeys)).toEqual({
        expected: 2,
        skipped: 1,
        unexpected: 0,
        flaky: 0,
        incomplete: 0,
      })
    })
  })

  describe('required breadcrumb journey inventory', () => {
    const breadcrumbJourneys = [
      { title: 'file drag onto a breadcrumb crumb moves it up after one confirmation' },
      { title: 'file drag onto the main drive-root crumb moves it to the root' },
      { title: 'drop onto the active breadcrumb crumb explains it is already there' },
      { title: 'folder drag onto an ancestor crumb above its parent commits after confirmation' },
    ]
    const unrelatedJourneys = [
      { title: 'file drag into folder moves it' },
      { title: 'folder drag opens the one confirmation and Cancel changes nothing' },
    ]

    it('accepts an inventory that carries every required lane', () => {
      const journeys = collectReportJourneys(
        journeyReport([...breadcrumbJourneys, ...unrelatedJourneys])
      )
      expect(missingRequiredJourneyInventories(journeys)).toEqual([])
    })

    it('fails an inventory that lost the breadcrumb journeys entirely', () => {
      // The pre-breadcrumb spec: still a valid Playwright inventory, but it
      // can no longer certify the breadcrumb drop-move feature.
      const journeys = collectReportJourneys(journeyReport(unrelatedJourneys))
      const missing = missingRequiredJourneyInventories(journeys)
      expect(missing).toEqual(REQUIRED_JOURNEY_INVENTORY.map(lane => lane.label))
    })

    it.each(
      breadcrumbJourneys.map((lane, index) => ({
        laneTitle: lane.title,
        laneLabel: REQUIRED_JOURNEY_INVENTORY[index].label,
      }))
    )('fails when only the $laneLabel goes missing', ({ laneTitle, laneLabel }) => {
      // Every lane is required INDEPENDENTLY: removing any single journey —
      // including the root move — must fail the gate even while the other
      // three breadcrumb journeys survive.
      const remaining = breadcrumbJourneys.filter(journey => journey.title !== laneTitle)
      expect(remaining).toHaveLength(breadcrumbJourneys.length - 1)
      const journeys = collectReportJourneys(journeyReport([...remaining, ...unrelatedJourneys]))
      expect(missingRequiredJourneyInventories(journeys)).toEqual([laneLabel])
    })

    it('requires the four lanes through disjoint anchored patterns', () => {
      const titles = breadcrumbJourneys.map(journey => journey.title)
      for (const lane of REQUIRED_JOURNEY_INVENTORY) {
        // The lane pattern matches exactly one journey title — its own —
        // so no sibling lane can satisfy it by accident.
        const matches = titles.filter(title => lane.pattern.test(title))
        expect(matches, lane.label).toHaveLength(1)
      }
      // And nothing outside the breadcrumb lanes is captured.
      for (const lane of REQUIRED_JOURNEY_INVENTORY) {
        for (const journey of unrelatedJourneys) {
          expect(lane.pattern.test(journey.title), lane.label).toBe(false)
        }
      }
    })
  })
})