Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
test(e2e): unit-test the mock matcher instead of narrating it, drop dead comments
Sergey Kozyrenko487b82f44ac8
diff --git a/frontend/e2e/helpers/terminal.ts b/frontend/e2e/helpers/terminal.tsindex 37952281..77aa52ea 100644--- a/frontend/e2e/helpers/terminal.ts+++ b/frontend/e2e/helpers/terminal.ts@@ -14,7 +14,6 @@ interface XtermHost extends HTMLElement {};}-/** Reads the visible xterm buffer as plain text — the WebGL canvas has no DOM text to locate. */export const readTerminalBuffer = async (page: Page): Promise<string> =>page.locator('.xterm')diff --git a/frontend/e2e/mocks/cassette.ts b/frontend/e2e/mocks/cassette.tsindex c11a1641..6ac79dfe 100644--- a/frontend/e2e/mocks/cassette.ts+++ b/frontend/e2e/mocks/cassette.ts@@ -20,7 +20,6 @@ export interface CassetteFrame {*/export const entity = <T extends object>(typename: string, value: T): T => ({ __typename: typename, ...value });-/** Section-wise merge: an override replaces the base entry list per operation/path key. */export const mergeCassettes = (base: Cassette, override: Cassette): Cassette => ({mutations: { ...base.mutations, ...override.mutations },queries: { ...base.queries, ...override.queries },diff --git a/frontend/e2e/mocks/cassettes/base.ts b/frontend/e2e/mocks/cassettes/base.tsindex 70ebc132..f7a26753 100644--- a/frontend/e2e/mocks/cassettes/base.ts+++ b/frontend/e2e/mocks/cassettes/base.ts@@ -43,7 +43,6 @@ const settingsUser: ResultOf<typeof SettingsUserDocument> = {}),};-/** Everything the authenticated app shell requests on any page. */export const baseQueries = (): NonNullable<Cassette['queries']> => ({flows: [{ data: flows }],flowTemplates: [{ data: flowTemplates }],diff --git a/frontend/e2e/mocks/cassettes/flows.ts b/frontend/e2e/mocks/cassettes/flows.tsindex c3d8664c..077cc910 100644--- a/frontend/e2e/mocks/cassettes/flows.ts+++ b/frontend/e2e/mocks/cassettes/flows.ts@@ -69,10 +69,6 @@ export const makeMessage = (...overrides,});-// One message per distinct render path in flow-message.tsx (all Answer-typed-// cassettes never exercise these): thinking-toggle needs thinking+message,-// report auto-expands its details, a Terminal resultFormat mounts the xterm-// renderer, and Input right-aligns.export const VARIED_MESSAGES = [makeMessage('501', '5', { message: 'Planning the run', thinking: 'internal reasoning about the plan' }),makeMessage('502', '5', {@@ -81,7 +77,6 @@ export const VARIED_MESSAGES = [resultFormat: ResultFormat.Markdown,type: MessageLogType.Report,}),- // Report type auto-expands, so the Terminal renderer mounts on load.makeMessage('503', '5', {message: '',result: 'e2e-terminal-marker\nexit 0',@@ -143,11 +138,6 @@ const addedFrame = (message: MessageLogFragmentFragment, delayMs: number) => ({payload: { data: { messageLogAdded: message } },});-/**- * Two concurrent flows with disjoint message streams. The second `flow` entry- * for flow 5 is the post-reconnect reconcile response: everything delivered so- * far plus the message "missed" while the socket was down.- */export const flowsCassette = (override: Cassette = {}): Cassette =>mergeCassettes({@@ -188,7 +178,6 @@ export const flowsCassette = (override: Cassette = {}): Cassette =>override,);-/** Flow 5 pre-loaded with one message per render path, no streaming. */export const variedMessagesCassette = (): Cassette =>flowsCassette({queries: { flow: [{ data: flowQueryData(FLOW_A, VARIED_MESSAGES), variables: { id: '5' } }] },@@ -315,7 +304,6 @@ const flowTabsData: ResultOf<typeof FlowDocument> = {vectorStoreLogs: [TABS_VECTOR_LOG],};-/** Flow 5 with one populated entry per detail tab, including the REST-fed screenshot image. */export const flowTabsCassette = (): Cassette =>flowsCassette({queries: {diff --git a/frontend/e2e/mocks/cassettes/smoke.ts b/frontend/e2e/mocks/cassettes/smoke.tsindex f06e3f81..52bbab11 100644--- a/frontend/e2e/mocks/cassettes/smoke.ts+++ b/frontend/e2e/mocks/cassettes/smoke.ts@@ -3,13 +3,11 @@ import type { Cassette } from '../cassette.ts';import { authenticatedInfoEntry, guestInfoEntry, SEEDED_USER } from '../../fixtures/auth.ts';import { baseQueries, baseRest } from './base.ts';-/** Seeded-session smoke: every request answers as an authenticated user. */export const smokeCassette: Cassette = {queries: baseQueries(),rest: baseRest(),};-/** Login-form journey: /info serves guest until the login mutation raises the flag. */export const loginJourneyCassette: Cassette = {queries: baseQueries(),rest: {diff --git a/frontend/e2e/mocks/world.ts b/frontend/e2e/mocks/world.tsindex eaacd916..f6185676 100644--- a/frontend/e2e/mocks/world.ts+++ b/frontend/e2e/mocks/world.ts@@ -8,8 +8,6 @@ import type {WorldFlagged,} from './cassette.ts';-// Key-sorted stringify: the app builds variables objects at runtime (spreads,-// conditional assignment), so property order must not affect matching.const stableStringify = (value: unknown): string =>JSON.stringify(value, (_key, node: unknown) =>node && typeof node === 'object' && !Array.isArray(node)@@ -20,9 +18,6 @@ const stableStringify = (value: unknown): string =>const isPlainObject = (value: unknown): value is Record<string, unknown> =>typeof value === 'object' && value !== null && !Array.isArray(value);-// Recurses so a nested pin stays a subset — `{ input: { name } }` must not demand-// that the whole `input` match, or any sibling the app also sends (a time-derived-// TTL, a default) would make the entry unmatchable.const isSubsetMatch = (expected?: Record<string, unknown>, actual?: Record<string, unknown>): boolean =>!expected ||Object.entries(expected).every(([key, value]) => {@@ -75,7 +70,6 @@ export class MockWorld {}}- /** Resolves once the flag is raised (immediately if it already is). */flagRaised(flag: string): Promise<void> {if (this.flags.has(flag)) {return Promise.resolve();@@ -111,8 +105,6 @@ export class MockWorld {return candidateMethod === method && candidatePath.replace(/\/+$/, '') === normalized;});- // A path hit whose bodySubset does not match stays unmatched (501): a- // broken request payload must not be answered with a canned success.const matching = (key ? this.cassette.rest?.[key] : undefined)?.filter((entry) =>isSubsetMatch(entry.bodySubset, body),);@@ -148,13 +140,6 @@ export class MockWorld {this.unmatchedSubscriptions.push(description);}- /**- * One driver per stream broadcasts each frame once to every current- * subscriber: duplicate subscribers (e.g. flowUpdated is opened by both the- * list and the detail provider) each get the frame, while a re-subscribe- * after a reconnect joins past the cursor — delta-only, like the real- * server. Returns an unsubscribe function.- */subscribeStream(streamKey: string, entry: SubscriptionCassetteEntry, subscriber: StreamSubscriber): () => void {const state = this.streams.get(streamKey) ?? { cursor: 0, isDriving: false, subscribers: new Set() };@@ -194,7 +179,6 @@ export class MockWorld {}}- /** Entries gated on an unraised flag are invisible; raised-flag entries outrank unflagged ones. */private eligible<T extends WorldFlagged>(candidates?: T[]): T[] | undefined {const open = candidates?.filter((entry) => !entry.whenFlag || this.flags.has(entry.whenFlag));diff --git a/frontend/e2e/mocks/world.unit.test.ts b/frontend/e2e/mocks/world.unit.test.tsnew file mode 100644index 00000000..be98b752--- /dev/null+++ b/frontend/e2e/mocks/world.unit.test.ts@@ -0,0 +1,89 @@+import { describe, expect, it, vi } from 'vitest';++import type { SubscriptionCassetteEntry } from './cassette.ts';+import type { StreamSubscriber } from './world.ts';++import { MockWorld } from './world.ts';++const collect = (into: unknown[]): StreamSubscriber => ({ complete: () => {}, next: (payload) => into.push(payload) });++describe('MockWorld matching', () => {+ it('matches variables regardless of property order inside arrays', () => {+ const world = new MockWorld({+ mutations: { save: [{ data: { ok: true }, variables: { items: [{ a: 1, b: 2 }] } }] },+ });++ // Built key-by-key: a reversed object literal would be re-sorted by the linter,+ // hiding that stableStringify must normalise key order inside array values.+ const item: Record<string, number> = {};+ item.b = 2;+ item.a = 1;++ expect(world.matchGraphQL('save', { items: [item] })?.data).toEqual({ ok: true });+ });++ it('treats a pinned object as a subset — extra siblings match, a wrong pinned value does not', () => {+ const world = new MockWorld({+ mutations: { create: [{ data: { ok: true }, variables: { input: { name: 'x' } } }] },+ });++ expect(world.matchGraphQL('create', { input: { name: 'x', ttl: 99 } })?.data).toEqual({ ok: true });+ expect(world.matchGraphQL('create', { input: { name: 'y', ttl: 99 } })).toBeUndefined();+ });++ it('serves same-key entries in order and repeats the last', () => {+ const world = new MockWorld({+ queries: {+ flow: [+ { data: { n: 1 }, variables: { id: '5' } },+ { data: { n: 2 }, variables: { id: '5' } },+ ],+ },+ });++ expect(world.matchGraphQL('flow', { id: '5' })?.data).toEqual({ n: 1 });+ expect(world.matchGraphQL('flow', { id: '5' })?.data).toEqual({ n: 2 });+ expect(world.matchGraphQL('flow', { id: '5' })?.data).toEqual({ n: 2 });+ });++ it('leaves a REST path hit with a mismatched body unmatched instead of answering success', () => {+ const world = new MockWorld({+ rest: { 'POST /api/v1/resources/mkdir': [{ body: { ok: true }, bodySubset: { path: 'new-folder' } }] },+ });++ expect(world.matchRest('POST', '/api/v1/resources/mkdir', { path: 'new-folder' })).toBeDefined();+ expect(world.matchRest('POST', '/api/v1/resources/mkdir', { path: 'wrong' })).toBeUndefined();+ expect(world.matchRest('POST', '/api/v1/resources/mkdir/', { path: 'new-folder' })).toBeDefined();+ });++ it('hides a flag-gated entry until the flag is raised, then lets it outrank the unflagged one', () => {+ const world = new MockWorld({+ mutations: { login: [{ data: { ok: true }, setFlag: 'logged-in' }] },+ queries: { info: [{ data: { role: 'guest' } }, { data: { role: 'user' }, whenFlag: 'logged-in' }] },+ });++ expect(world.matchGraphQL('info', {})?.data).toEqual({ role: 'guest' });+ world.matchGraphQL('login', {});+ expect(world.matchGraphQL('info', {})?.data).toEqual({ role: 'user' });+ });+});++describe('MockWorld streams', () => {+ it('broadcasts each frame once to every current subscriber; a later subscriber joins delta-only', async () => {+ const world = new MockWorld({ mutations: { trigger: [{ data: {}, setFlag: 'go' }] } });+ const entry: SubscriptionCassetteEntry = {+ frames: [{ payload: { data: { n: 1 } } }, { payload: { data: { n: 2 } }, whenFlag: 'go' }],+ };+ const first: unknown[] = [];+ const late: unknown[] = [];++ world.subscribeStream('k', entry, collect(first));+ await vi.waitFor(() => expect(first).toHaveLength(1));++ world.subscribeStream('k', entry, collect(late));+ world.matchGraphQL('trigger', {});+ await vi.waitFor(() => expect(first).toHaveLength(2));++ expect(late).toEqual([{ data: { n: 2 } }]);+ });+});diff --git a/frontend/e2e/mocks/ws-graphql.ts b/frontend/e2e/mocks/ws-graphql.tsindex 47d815b0..679f32fc 100644--- a/frontend/e2e/mocks/ws-graphql.ts+++ b/frontend/e2e/mocks/ws-graphql.ts@@ -20,11 +20,6 @@ interface SubscribePayload {variables?: Record<string, unknown>;}-/**- * Transport-agnostic graphql-ws server half: Playwright routes and the vitest- * protocol-contract suite (ws-graphql.unit.test.ts) both drive this exact- * code, so the protocol invariants are pinned outside full browser runs.- */export const handleWsConnection = (world: MockWorld,transport: WsTransport,diff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.tsindex 9fd0eb25..d656fa9e 100644--- a/frontend/e2e/playwright.config.ts+++ b/frontend/e2e/playwright.config.ts@@ -5,9 +5,6 @@ import type { BackendOptions, BackendTier } from './fixtures/backend.ts';const rawTier = process.env.E2E_TIER ?? 'mock';const isCI = Boolean(process.env.CI);-// Visual snapshots only ever run inside the pinned Playwright Linux container-// (e2e/tools/run-visual.sh) — a darwin run would generate parallel baselines-// that never match CI pixels.const isVisual = process.env.E2E_VISUAL === '1';// Baselines are linux-suffixed, so a host run writes `*-visual-darwin.png` beside@@ -21,8 +18,6 @@ if (isVisual && process.platform !== 'linux') {// `vite preview` listens on VITE_PORT + 100 and reuses the dev proxy config.const PREVIEW_PORT = 8100;-// Reporter paths resolve against the process cwd, not the config file — pin them-// so CI artifact uploads have one deterministic location.const here = (relative: string) => fileURLToPath(new URL(relative, import.meta.url));export const AUTH_STATE_PATH = here('./.auth/user.json');@@ -54,9 +49,6 @@ export default defineConfig<BackendOptions>({fullyParallel: true,globalTimeout: isCI ? 10 * 60_000 : undefined,outputDir: './test-results',- // Cassette specs run only on the mock tier; specs/real/** run only against a- // live backend, which authenticates once in the setup project and reuses- // storageState.projects:tier === 'mock'? [@@ -96,8 +88,6 @@ export default defineConfig<BackendOptions>({['list'],]: [['html', { open: 'never', outputFolder: here('./playwright-report') }], ['list']],- // Retries and trace policy are intentionally identical local vs CI so a red CI- // run reproduces byte-identically with `CI=1 pnpm e2e`.retries: 1,testDir: './specs',use: {diff --git a/frontend/e2e/routes.ts b/frontend/e2e/routes.tsindex bd1a71f4..5abeecbe 100644--- a/frontend/e2e/routes.ts+++ b/frontend/e2e/routes.ts@@ -15,11 +15,9 @@ import { settingsProvidersCassette } from './mocks/cassettes/settings-providers.import { templatesCassette } from './mocks/cassettes/templates.ts';export interface RouteManifestEntry {- /** Known accessibility debt on this route, waived node-by-node by the axe sweep. */a11yWaivers?: A11yWaiver[];cassette: () => Cassette;path: string;- /** The route counts as rendered when this locator is visible. */ready: (page: Page) => Locator;/*** Owning src/ areas — the substrate for changed-files → affected-routes@@ -58,7 +56,6 @@ export const ROUTE_MANIFEST: RouteManifestEntry[] = ['src/features/flows','src/providers/flow-provider.tsx','src/providers/flows-provider.tsx',- // Rendered inside the detail page's tabs alongside their owning routes.'src/components/shared/file-manager','src/components/dashboard','src/features/resources',diff --git a/frontend/e2e/specs/cross/a11y.spec.ts b/frontend/e2e/specs/cross/a11y.spec.tsindex c3d90ae3..c0185297 100644--- a/frontend/e2e/specs/cross/a11y.spec.ts+++ b/frontend/e2e/specs/cross/a11y.spec.ts@@ -3,8 +3,6 @@ import { scanA11y } from '../../helpers/a11y.ts';import { loginJourneyCassette } from '../../mocks/cassettes/smoke.ts';import { ROUTE_MANIFEST } from '../../routes.ts';-// Both themes are scanned: contrast waivers are theme-independent, but dark mode-// recomputes every color, so a token that passes in light can still fail in dark.const THEMES = ['light', 'dark'] as const;for (const theme of THEMES) {@@ -15,7 +13,6 @@ for (const theme of THEMES) {});}- // Outside the manifest: the only unauthenticated route.test.describe('login', () => {test.use({ cassette: loginJourneyCassette, isAuthSeeded: false });@@ -26,8 +23,6 @@ for (const theme of THEMES) {});});- // Driven off the manifest so a newly swept route is scanned without a- // second, hand-maintained route list drifting behind it.for (const entry of ROUTE_MANIFEST) {test.describe(entry.path, () => {test.use({ cassette: entry.cassette() });@@ -35,8 +30,7 @@ for (const theme of THEMES) {test('has no axe violations', async ({ page }) => {await page.goto(entry.path);await expect(entry.ready(page)).toBeVisible();- // A seed that stops taking effect would otherwise re-run the light- // theme under a dark label and pass.+ // Not redundant: a theme seed that silently stops applying reruns light under a dark label.await expect(page.locator('html')).toHaveClass(theme === 'dark' ? /dark/ : /light/);await scanA11y(page, entry.path, entry.a11yWaivers);});diff --git a/frontend/e2e/specs/flows/create.spec.ts b/frontend/e2e/specs/flows/create.spec.tsindex 93f8ef7e..2bf08cb3 100644--- a/frontend/e2e/specs/flows/create.spec.ts+++ b/frontend/e2e/specs/flows/create.spec.ts@@ -38,8 +38,6 @@ test.describe('flow create', { tag: ['@flows', '@smoke'] }, () => {test('creates a flow from the form and lands on its detail page', async ({ page, pageErrorLog }) => {await page.goto('/flows/new');- // The form only becomes submittable once the providers query resolved- // and the default provider is selected.await expect(page.getByText(PROVIDER.name)).toBeVisible();await page.getByPlaceholder(/Describe what you would like PentAGI to test/).fill('Say hello');await page.getByRole('button', { name: 'Submit' }).click();diff --git a/frontend/e2e/specs/flows/lifecycle.spec.ts b/frontend/e2e/specs/flows/lifecycle.spec.tsindex 4354e541..59ea3284 100644--- a/frontend/e2e/specs/flows/lifecycle.spec.ts+++ b/frontend/e2e/specs/flows/lifecycle.spec.ts@@ -146,8 +146,6 @@ test.describe('flow lifecycle', { tag: '@flows' }, () => {await expect(star).toHaveAttribute('aria-pressed', 'false');await star.click();await expect(star).toHaveAttribute('aria-pressed', 'true');- // The sidebar favorites group renders only from the subscription- // frame — the optimistic star alone can never produce it.await expect(page.getByText('Favorite Flows')).toBeVisible();await expect(star).toHaveAttribute('aria-pressed', 'true');expectCleanPage(pageErrorLog);diff --git a/frontend/e2e/specs/flows/messages.spec.ts b/frontend/e2e/specs/flows/messages.spec.tsindex 9b035129..4609da8d 100644--- a/frontend/e2e/specs/flows/messages.spec.ts+++ b/frontend/e2e/specs/flows/messages.spec.ts@@ -7,8 +7,6 @@ type XtermHost = {};type XtermLine = { translateToString: (trim: boolean) => string };-// The page has more than one xterm (the right-panel Terminal tab + the-// message-level Terminal renderer); read every buffer and match any.const anyTerminalContains = (marker: string): boolean => {const hosts = document.querySelectorAll('.xterm');@@ -30,8 +28,6 @@ const anyTerminalContains = (marker: string): boolean => {});};-// The Answer-only cassettes never exercise the type-specific render paths in-// flow-message.tsx. This drives one message per path.test.describe('flow message rendering', { tag: '@flows' }, () => {test.use({ cassette: variedMessagesCassette() });@@ -42,7 +38,6 @@ test.describe('flow message rendering', { tag: '@flows' }, () => {await page.goto('/flows/5?tab=automation');await expect(page.getByTestId('flow-message-id')).toHaveCount(4);- // thoughts message (thinking + message) → a Show/Hide thinking toggleconst thinkingToggle = page.getByText('Show thinking');await expect(thinkingToggle).toBeVisible();@@ -50,19 +45,14 @@ test.describe('flow message rendering', { tag: '@flows' }, () => {await expect(page.getByText('Hide thinking')).toBeVisible();await expect(page.getByText('internal reasoning about the plan')).toBeVisible();- // report message auto-expands its details (starts on "Hide details")await expect(page.getByRole('heading', { name: 'Report' })).toBeVisible();await expect(page.getByText('Hide details').first()).toBeVisible();- // terminal-format report message mounts the xterm renderer with its outputawait expect(page.locator('.xterm')).toHaveCount(2);await expect(async () => {expect(await page.evaluate(anyTerminalContains, 'e2e-terminal-marker')).toBe(true);}).toPass();- // Only the input message right-aligns: pinning the count and the text means- // an inverted alignment ternary fails on both counts instead of finding some- // other right-aligned element on the page.const rightAligned = page.locator('.items-end');await expect(rightAligned).toHaveCount(1);diff --git a/frontend/e2e/specs/visual/settings-providers-populated.spec.ts b/frontend/e2e/specs/visual/settings-providers-populated.spec.tsindex f067e260..ef3bccc1 100644--- a/frontend/e2e/specs/visual/settings-providers-populated.spec.ts+++ b/frontend/e2e/specs/visual/settings-providers-populated.spec.ts@@ -1,8 +1,6 @@import { expect, test } from '../../fixtures/test.ts';import { populatedSettingsProvidersCassette } from '../../mocks/cassettes/settings-providers.ts';-// The route-manifest visual sweep only captures the empty providers state. This-// pins the populated layout (enabled defaults + a user-defined provider row).const THEMES = ['light', 'dark'] as const;test.describe('visual: settings providers (populated)', { tag: '@visual' }, () => {