Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

test(e2e): unit-test the mock matcher instead of narrating it, drop dead comments

Sergey Kozyrenko487b82f44ac8
diff --git a/frontend/e2e/helpers/terminal.ts b/frontend/e2e/helpers/terminal.ts
index 37952281..77aa52ea 100644
--- a/frontend/e2e/helpers/terminal.ts
+++ b/frontend/e2e/helpers/terminal.ts
@@ -14,7 +14,6 @@ interface XtermHost extends HTMLElement {
};
}
-/** Reads the visible xterm buffer as plain text — the WebGL canvas has no DOM text to locate. */
export const readTerminalBuffer = async (page: Page): Promise<string> =>
page
.locator('.xterm')
diff --git a/frontend/e2e/mocks/cassette.ts b/frontend/e2e/mocks/cassette.ts
index c11a1641..6ac79dfe 100644
--- a/frontend/e2e/mocks/cassette.ts
+++ b/frontend/e2e/mocks/cassette.ts
@@ -20,7 +20,6 @@ export interface CassetteFrame {
*/
export const entity = <T extends object>(typename: string, value: T): T => ({ __typename: typename, ...value });
-/** Section-wise merge: an override replaces the base entry list per operation/path key. */
export const mergeCassettes = (base: Cassette, override: Cassette): Cassette => ({
mutations: { ...base.mutations, ...override.mutations },
queries: { ...base.queries, ...override.queries },
diff --git a/frontend/e2e/mocks/cassettes/base.ts b/frontend/e2e/mocks/cassettes/base.ts
index 70ebc132..f7a26753 100644
--- a/frontend/e2e/mocks/cassettes/base.ts
+++ b/frontend/e2e/mocks/cassettes/base.ts
@@ -43,7 +43,6 @@ const settingsUser: ResultOf<typeof SettingsUserDocument> = {
}),
};
-/** Everything the authenticated app shell requests on any page. */
export const baseQueries = (): NonNullable<Cassette['queries']> => ({
flows: [{ data: flows }],
flowTemplates: [{ data: flowTemplates }],
diff --git a/frontend/e2e/mocks/cassettes/flows.ts b/frontend/e2e/mocks/cassettes/flows.ts
index c3d8664c..077cc910 100644
--- a/frontend/e2e/mocks/cassettes/flows.ts
+++ b/frontend/e2e/mocks/cassettes/flows.ts
@@ -69,10 +69,6 @@ export const makeMessage = (
...overrides,
});
-// One message per distinct render path in flow-message.tsx (all Answer-typed
-// cassettes never exercise these): thinking-toggle needs thinking+message,
-// report auto-expands its details, a Terminal resultFormat mounts the xterm
-// renderer, and Input right-aligns.
export const VARIED_MESSAGES = [
makeMessage('501', '5', { message: 'Planning the run', thinking: 'internal reasoning about the plan' }),
makeMessage('502', '5', {
@@ -81,7 +77,6 @@ export const VARIED_MESSAGES = [
resultFormat: ResultFormat.Markdown,
type: MessageLogType.Report,
}),
- // Report type auto-expands, so the Terminal renderer mounts on load.
makeMessage('503', '5', {
message: '',
result: 'e2e-terminal-marker\nexit 0',
@@ -143,11 +138,6 @@ const addedFrame = (message: MessageLogFragmentFragment, delayMs: number) => ({
payload: { data: { messageLogAdded: message } },
});
-/**
- * Two concurrent flows with disjoint message streams. The second `flow` entry
- * for flow 5 is the post-reconnect reconcile response: everything delivered so
- * far plus the message "missed" while the socket was down.
- */
export const flowsCassette = (override: Cassette = {}): Cassette =>
mergeCassettes(
{
@@ -188,7 +178,6 @@ export const flowsCassette = (override: Cassette = {}): Cassette =>
override,
);
-/** Flow 5 pre-loaded with one message per render path, no streaming. */
export const variedMessagesCassette = (): Cassette =>
flowsCassette({
queries: { flow: [{ data: flowQueryData(FLOW_A, VARIED_MESSAGES), variables: { id: '5' } }] },
@@ -315,7 +304,6 @@ const flowTabsData: ResultOf<typeof FlowDocument> = {
vectorStoreLogs: [TABS_VECTOR_LOG],
};
-/** Flow 5 with one populated entry per detail tab, including the REST-fed screenshot image. */
export const flowTabsCassette = (): Cassette =>
flowsCassette({
queries: {
diff --git a/frontend/e2e/mocks/cassettes/smoke.ts b/frontend/e2e/mocks/cassettes/smoke.ts
index f06e3f81..52bbab11 100644
--- a/frontend/e2e/mocks/cassettes/smoke.ts
+++ b/frontend/e2e/mocks/cassettes/smoke.ts
@@ -3,13 +3,11 @@ import type { Cassette } from '../cassette.ts';
import { authenticatedInfoEntry, guestInfoEntry, SEEDED_USER } from '../../fixtures/auth.ts';
import { baseQueries, baseRest } from './base.ts';
-/** Seeded-session smoke: every request answers as an authenticated user. */
export const smokeCassette: Cassette = {
queries: baseQueries(),
rest: baseRest(),
};
-/** Login-form journey: /info serves guest until the login mutation raises the flag. */
export const loginJourneyCassette: Cassette = {
queries: baseQueries(),
rest: {
diff --git a/frontend/e2e/mocks/world.ts b/frontend/e2e/mocks/world.ts
index eaacd916..f6185676 100644
--- a/frontend/e2e/mocks/world.ts
+++ b/frontend/e2e/mocks/world.ts
@@ -8,8 +8,6 @@ import type {
WorldFlagged,
} from './cassette.ts';
-// Key-sorted stringify: the app builds variables objects at runtime (spreads,
-// conditional assignment), so property order must not affect matching.
const stableStringify = (value: unknown): string =>
JSON.stringify(value, (_key, node: unknown) =>
node && typeof node === 'object' && !Array.isArray(node)
@@ -20,9 +18,6 @@ const stableStringify = (value: unknown): string =>
const isPlainObject = (value: unknown): value is Record<string, unknown> =>
typeof value === 'object' && value !== null && !Array.isArray(value);
-// Recurses so a nested pin stays a subset — `{ input: { name } }` must not demand
-// that the whole `input` match, or any sibling the app also sends (a time-derived
-// TTL, a default) would make the entry unmatchable.
const isSubsetMatch = (expected?: Record<string, unknown>, actual?: Record<string, unknown>): boolean =>
!expected ||
Object.entries(expected).every(([key, value]) => {
@@ -75,7 +70,6 @@ export class MockWorld {
}
}
- /** Resolves once the flag is raised (immediately if it already is). */
flagRaised(flag: string): Promise<void> {
if (this.flags.has(flag)) {
return Promise.resolve();
@@ -111,8 +105,6 @@ export class MockWorld {
return candidateMethod === method && candidatePath.replace(/\/+$/, '') === normalized;
});
- // A path hit whose bodySubset does not match stays unmatched (501): a
- // broken request payload must not be answered with a canned success.
const matching = (key ? this.cassette.rest?.[key] : undefined)?.filter((entry) =>
isSubsetMatch(entry.bodySubset, body),
);
@@ -148,13 +140,6 @@ export class MockWorld {
this.unmatchedSubscriptions.push(description);
}
- /**
- * One driver per stream broadcasts each frame once to every current
- * subscriber: duplicate subscribers (e.g. flowUpdated is opened by both the
- * list and the detail provider) each get the frame, while a re-subscribe
- * after a reconnect joins past the cursor — delta-only, like the real
- * server. Returns an unsubscribe function.
- */
subscribeStream(streamKey: string, entry: SubscriptionCassetteEntry, subscriber: StreamSubscriber): () => void {
const state = this.streams.get(streamKey) ?? { cursor: 0, isDriving: false, subscribers: new Set() };
@@ -194,7 +179,6 @@ export class MockWorld {
}
}
- /** Entries gated on an unraised flag are invisible; raised-flag entries outrank unflagged ones. */
private eligible<T extends WorldFlagged>(candidates?: T[]): T[] | undefined {
const open = candidates?.filter((entry) => !entry.whenFlag || this.flags.has(entry.whenFlag));
diff --git a/frontend/e2e/mocks/world.unit.test.ts b/frontend/e2e/mocks/world.unit.test.ts
new file mode 100644
index 00000000..be98b752
--- /dev/null
+++ b/frontend/e2e/mocks/world.unit.test.ts
@@ -0,0 +1,89 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import type { SubscriptionCassetteEntry } from './cassette.ts';
+import type { StreamSubscriber } from './world.ts';
+
+import { MockWorld } from './world.ts';
+
+const collect = (into: unknown[]): StreamSubscriber => ({ complete: () => {}, next: (payload) => into.push(payload) });
+
+describe('MockWorld matching', () => {
+ it('matches variables regardless of property order inside arrays', () => {
+ const world = new MockWorld({
+ mutations: { save: [{ data: { ok: true }, variables: { items: [{ a: 1, b: 2 }] } }] },
+ });
+
+ // Built key-by-key: a reversed object literal would be re-sorted by the linter,
+ // hiding that stableStringify must normalise key order inside array values.
+ const item: Record<string, number> = {};
+ item.b = 2;
+ item.a = 1;
+
+ expect(world.matchGraphQL('save', { items: [item] })?.data).toEqual({ ok: true });
+ });
+
+ it('treats a pinned object as a subset — extra siblings match, a wrong pinned value does not', () => {
+ const world = new MockWorld({
+ mutations: { create: [{ data: { ok: true }, variables: { input: { name: 'x' } } }] },
+ });
+
+ expect(world.matchGraphQL('create', { input: { name: 'x', ttl: 99 } })?.data).toEqual({ ok: true });
+ expect(world.matchGraphQL('create', { input: { name: 'y', ttl: 99 } })).toBeUndefined();
+ });
+
+ it('serves same-key entries in order and repeats the last', () => {
+ const world = new MockWorld({
+ queries: {
+ flow: [
+ { data: { n: 1 }, variables: { id: '5' } },
+ { data: { n: 2 }, variables: { id: '5' } },
+ ],
+ },
+ });
+
+ expect(world.matchGraphQL('flow', { id: '5' })?.data).toEqual({ n: 1 });
+ expect(world.matchGraphQL('flow', { id: '5' })?.data).toEqual({ n: 2 });
+ expect(world.matchGraphQL('flow', { id: '5' })?.data).toEqual({ n: 2 });
+ });
+
+ it('leaves a REST path hit with a mismatched body unmatched instead of answering success', () => {
+ const world = new MockWorld({
+ rest: { 'POST /api/v1/resources/mkdir': [{ body: { ok: true }, bodySubset: { path: 'new-folder' } }] },
+ });
+
+ expect(world.matchRest('POST', '/api/v1/resources/mkdir', { path: 'new-folder' })).toBeDefined();
+ expect(world.matchRest('POST', '/api/v1/resources/mkdir', { path: 'wrong' })).toBeUndefined();
+ expect(world.matchRest('POST', '/api/v1/resources/mkdir/', { path: 'new-folder' })).toBeDefined();
+ });
+
+ it('hides a flag-gated entry until the flag is raised, then lets it outrank the unflagged one', () => {
+ const world = new MockWorld({
+ mutations: { login: [{ data: { ok: true }, setFlag: 'logged-in' }] },
+ queries: { info: [{ data: { role: 'guest' } }, { data: { role: 'user' }, whenFlag: 'logged-in' }] },
+ });
+
+ expect(world.matchGraphQL('info', {})?.data).toEqual({ role: 'guest' });
+ world.matchGraphQL('login', {});
+ expect(world.matchGraphQL('info', {})?.data).toEqual({ role: 'user' });
+ });
+});
+
+describe('MockWorld streams', () => {
+ it('broadcasts each frame once to every current subscriber; a later subscriber joins delta-only', async () => {
+ const world = new MockWorld({ mutations: { trigger: [{ data: {}, setFlag: 'go' }] } });
+ const entry: SubscriptionCassetteEntry = {
+ frames: [{ payload: { data: { n: 1 } } }, { payload: { data: { n: 2 } }, whenFlag: 'go' }],
+ };
+ const first: unknown[] = [];
+ const late: unknown[] = [];
+
+ world.subscribeStream('k', entry, collect(first));
+ await vi.waitFor(() => expect(first).toHaveLength(1));
+
+ world.subscribeStream('k', entry, collect(late));
+ world.matchGraphQL('trigger', {});
+ await vi.waitFor(() => expect(first).toHaveLength(2));
+
+ expect(late).toEqual([{ data: { n: 2 } }]);
+ });
+});
diff --git a/frontend/e2e/mocks/ws-graphql.ts b/frontend/e2e/mocks/ws-graphql.ts
index 47d815b0..679f32fc 100644
--- a/frontend/e2e/mocks/ws-graphql.ts
+++ b/frontend/e2e/mocks/ws-graphql.ts
@@ -20,11 +20,6 @@ interface SubscribePayload {
variables?: Record<string, unknown>;
}
-/**
- * Transport-agnostic graphql-ws server half: Playwright routes and the vitest
- * protocol-contract suite (ws-graphql.unit.test.ts) both drive this exact
- * code, so the protocol invariants are pinned outside full browser runs.
- */
export const handleWsConnection = (
world: MockWorld,
transport: WsTransport,
diff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.ts
index 9fd0eb25..d656fa9e 100644
--- a/frontend/e2e/playwright.config.ts
+++ b/frontend/e2e/playwright.config.ts
@@ -5,9 +5,6 @@ import type { BackendOptions, BackendTier } from './fixtures/backend.ts';
const rawTier = process.env.E2E_TIER ?? 'mock';
const isCI = Boolean(process.env.CI);
-// Visual snapshots only ever run inside the pinned Playwright Linux container
-// (e2e/tools/run-visual.sh) — a darwin run would generate parallel baselines
-// that never match CI pixels.
const isVisual = process.env.E2E_VISUAL === '1';
// Baselines are linux-suffixed, so a host run writes `*-visual-darwin.png` beside
@@ -21,8 +18,6 @@ if (isVisual && process.platform !== 'linux') {
// `vite preview` listens on VITE_PORT + 100 and reuses the dev proxy config.
const PREVIEW_PORT = 8100;
-// Reporter paths resolve against the process cwd, not the config file — pin them
-// so CI artifact uploads have one deterministic location.
const here = (relative: string) => fileURLToPath(new URL(relative, import.meta.url));
export const AUTH_STATE_PATH = here('./.auth/user.json');
@@ -54,9 +49,6 @@ export default defineConfig<BackendOptions>({
fullyParallel: true,
globalTimeout: isCI ? 10 * 60_000 : undefined,
outputDir: './test-results',
- // Cassette specs run only on the mock tier; specs/real/** run only against a
- // live backend, which authenticates once in the setup project and reuses
- // storageState.
projects:
tier === 'mock'
? [
@@ -96,8 +88,6 @@ export default defineConfig<BackendOptions>({
['list'],
]
: [['html', { open: 'never', outputFolder: here('./playwright-report') }], ['list']],
- // Retries and trace policy are intentionally identical local vs CI so a red CI
- // run reproduces byte-identically with `CI=1 pnpm e2e`.
retries: 1,
testDir: './specs',
use: {
diff --git a/frontend/e2e/routes.ts b/frontend/e2e/routes.ts
index bd1a71f4..5abeecbe 100644
--- a/frontend/e2e/routes.ts
+++ b/frontend/e2e/routes.ts
@@ -15,11 +15,9 @@ import { settingsProvidersCassette } from './mocks/cassettes/settings-providers.
import { templatesCassette } from './mocks/cassettes/templates.ts';
export interface RouteManifestEntry {
- /** Known accessibility debt on this route, waived node-by-node by the axe sweep. */
a11yWaivers?: A11yWaiver[];
cassette: () => Cassette;
path: string;
- /** The route counts as rendered when this locator is visible. */
ready: (page: Page) => Locator;
/**
* Owning src/ areas — the substrate for changed-files → affected-routes
@@ -58,7 +56,6 @@ export const ROUTE_MANIFEST: RouteManifestEntry[] = [
'src/features/flows',
'src/providers/flow-provider.tsx',
'src/providers/flows-provider.tsx',
- // Rendered inside the detail page's tabs alongside their owning routes.
'src/components/shared/file-manager',
'src/components/dashboard',
'src/features/resources',
diff --git a/frontend/e2e/specs/cross/a11y.spec.ts b/frontend/e2e/specs/cross/a11y.spec.ts
index c3d90ae3..c0185297 100644
--- a/frontend/e2e/specs/cross/a11y.spec.ts
+++ b/frontend/e2e/specs/cross/a11y.spec.ts
@@ -3,8 +3,6 @@ import { scanA11y } from '../../helpers/a11y.ts';
import { loginJourneyCassette } from '../../mocks/cassettes/smoke.ts';
import { ROUTE_MANIFEST } from '../../routes.ts';
-// Both themes are scanned: contrast waivers are theme-independent, but dark mode
-// recomputes every color, so a token that passes in light can still fail in dark.
const THEMES = ['light', 'dark'] as const;
for (const theme of THEMES) {
@@ -15,7 +13,6 @@ for (const theme of THEMES) {
});
}
- // Outside the manifest: the only unauthenticated route.
test.describe('login', () => {
test.use({ cassette: loginJourneyCassette, isAuthSeeded: false });
@@ -26,8 +23,6 @@ for (const theme of THEMES) {
});
});
- // Driven off the manifest so a newly swept route is scanned without a
- // second, hand-maintained route list drifting behind it.
for (const entry of ROUTE_MANIFEST) {
test.describe(entry.path, () => {
test.use({ cassette: entry.cassette() });
@@ -35,8 +30,7 @@ for (const theme of THEMES) {
test('has no axe violations', async ({ page }) => {
await page.goto(entry.path);
await expect(entry.ready(page)).toBeVisible();
- // A seed that stops taking effect would otherwise re-run the light
- // theme under a dark label and pass.
+ // Not redundant: a theme seed that silently stops applying reruns light under a dark label.
await expect(page.locator('html')).toHaveClass(theme === 'dark' ? /dark/ : /light/);
await scanA11y(page, entry.path, entry.a11yWaivers);
});
diff --git a/frontend/e2e/specs/flows/create.spec.ts b/frontend/e2e/specs/flows/create.spec.ts
index 93f8ef7e..2bf08cb3 100644
--- a/frontend/e2e/specs/flows/create.spec.ts
+++ b/frontend/e2e/specs/flows/create.spec.ts
@@ -38,8 +38,6 @@ test.describe('flow create', { tag: ['@flows', '@smoke'] }, () => {
test('creates a flow from the form and lands on its detail page', async ({ page, pageErrorLog }) => {
await page.goto('/flows/new');
- // The form only becomes submittable once the providers query resolved
- // and the default provider is selected.
await expect(page.getByText(PROVIDER.name)).toBeVisible();
await page.getByPlaceholder(/Describe what you would like PentAGI to test/).fill('Say hello');
await page.getByRole('button', { name: 'Submit' }).click();
diff --git a/frontend/e2e/specs/flows/lifecycle.spec.ts b/frontend/e2e/specs/flows/lifecycle.spec.ts
index 4354e541..59ea3284 100644
--- a/frontend/e2e/specs/flows/lifecycle.spec.ts
+++ b/frontend/e2e/specs/flows/lifecycle.spec.ts
@@ -146,8 +146,6 @@ test.describe('flow lifecycle', { tag: '@flows' }, () => {
await expect(star).toHaveAttribute('aria-pressed', 'false');
await star.click();
await expect(star).toHaveAttribute('aria-pressed', 'true');
- // The sidebar favorites group renders only from the subscription
- // frame — the optimistic star alone can never produce it.
await expect(page.getByText('Favorite Flows')).toBeVisible();
await expect(star).toHaveAttribute('aria-pressed', 'true');
expectCleanPage(pageErrorLog);
diff --git a/frontend/e2e/specs/flows/messages.spec.ts b/frontend/e2e/specs/flows/messages.spec.ts
index 9b035129..4609da8d 100644
--- a/frontend/e2e/specs/flows/messages.spec.ts
+++ b/frontend/e2e/specs/flows/messages.spec.ts
@@ -7,8 +7,6 @@ type XtermHost = {
};
type XtermLine = { translateToString: (trim: boolean) => string };
-// The page has more than one xterm (the right-panel Terminal tab + the
-// message-level Terminal renderer); read every buffer and match any.
const anyTerminalContains = (marker: string): boolean => {
const hosts = document.querySelectorAll('.xterm');
@@ -30,8 +28,6 @@ const anyTerminalContains = (marker: string): boolean => {
});
};
-// The Answer-only cassettes never exercise the type-specific render paths in
-// flow-message.tsx. This drives one message per path.
test.describe('flow message rendering', { tag: '@flows' }, () => {
test.use({ cassette: variedMessagesCassette() });
@@ -42,7 +38,6 @@ test.describe('flow message rendering', { tag: '@flows' }, () => {
await page.goto('/flows/5?tab=automation');
await expect(page.getByTestId('flow-message-id')).toHaveCount(4);
- // thoughts message (thinking + message) → a Show/Hide thinking toggle
const thinkingToggle = page.getByText('Show thinking');
await expect(thinkingToggle).toBeVisible();
@@ -50,19 +45,14 @@ test.describe('flow message rendering', { tag: '@flows' }, () => {
await expect(page.getByText('Hide thinking')).toBeVisible();
await expect(page.getByText('internal reasoning about the plan')).toBeVisible();
- // report message auto-expands its details (starts on "Hide details")
await expect(page.getByRole('heading', { name: 'Report' })).toBeVisible();
await expect(page.getByText('Hide details').first()).toBeVisible();
- // terminal-format report message mounts the xterm renderer with its output
await expect(page.locator('.xterm')).toHaveCount(2);
await expect(async () => {
expect(await page.evaluate(anyTerminalContains, 'e2e-terminal-marker')).toBe(true);
}).toPass();
- // Only the input message right-aligns: pinning the count and the text means
- // an inverted alignment ternary fails on both counts instead of finding some
- // other right-aligned element on the page.
const rightAligned = page.locator('.items-end');
await expect(rightAligned).toHaveCount(1);
diff --git a/frontend/e2e/specs/visual/settings-providers-populated.spec.ts b/frontend/e2e/specs/visual/settings-providers-populated.spec.ts
index f067e260..ef3bccc1 100644
--- a/frontend/e2e/specs/visual/settings-providers-populated.spec.ts
+++ b/frontend/e2e/specs/visual/settings-providers-populated.spec.ts
@@ -1,8 +1,6 @@
import { expect, test } from '../../fixtures/test.ts';
import { populatedSettingsProvidersCassette } from '../../mocks/cassettes/settings-providers.ts';
-// The route-manifest visual sweep only captures the empty providers state. This
-// pins the populated layout (enabled defaults + a user-defined provider row).
const THEMES = ['light', 'dark'] as const;
test.describe('visual: settings providers (populated)', { tag: '@visual' }, () => {