Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

test(e2e): scan the flow tab panels and the populated provider cards

Sergey Kozyrenkof6bcff867233
diff --git a/frontend/e2e/mocks/cassettes/flows.ts b/frontend/e2e/mocks/cassettes/flows.ts
index 077cc910..b4ce34d7 100644
--- a/frontend/e2e/mocks/cassettes/flows.ts
+++ b/frontend/e2e/mocks/cassettes/flows.ts
@@ -5,8 +5,15 @@ import type {
FlowDocument,
FlowFilesDocument,
FlowFragmentFragment,
+ FlowStatsByFlowDocument,
MessageLogFragmentFragment,
TerminalLogFragmentFragment,
+ ToolcallsStatsByFlowDocument,
+ ToolcallsStatsByFunctionForFlowDocument,
+ UsageStatsByAgentTypeForFlowDocument,
+ UsageStatsByFlowDocument,
+ UsageStatsByModelAgentsForFlowDocument,
+ UsageStatsFragmentFragment,
} from '@/graphql/types';
import {
@@ -304,11 +311,63 @@ const flowTabsData: ResultOf<typeof FlowDocument> = {
vectorStoreLogs: [TABS_VECTOR_LOG],
};
+const flowUsage: UsageStatsFragmentFragment = entity('UsageStats', {
+ totalUsageCacheIn: 5,
+ totalUsageCacheOut: 3,
+ totalUsageCostIn: 0.01,
+ totalUsageCostOut: 0.02,
+ totalUsageIn: 100,
+ totalUsageOut: 40,
+});
+
+const usageStatsByFlow: ResultOf<typeof UsageStatsByFlowDocument> = { usageStatsByFlow: flowUsage };
+
+const usageStatsByAgentTypeForFlow: ResultOf<typeof UsageStatsByAgentTypeForFlowDocument> = {
+ usageStatsByAgentTypeForFlow: [entity('AgentTypeUsageStats', { agentType: AgentType.Pentester, stats: flowUsage })],
+};
+
+const usageStatsByModelAgentsForFlow: ResultOf<typeof UsageStatsByModelAgentsForFlowDocument> = {
+ usageStatsByModelAgentsForFlow: [
+ entity('ModelAgentsUsageStats', {
+ agentTypes: [AgentType.Pentester],
+ model: 'e2e-model',
+ provider: 'e2e-provider',
+ stats: flowUsage,
+ }),
+ ],
+};
+
+const toolcallsStatsByFlow: ResultOf<typeof ToolcallsStatsByFlowDocument> = {
+ toolcallsStatsByFlow: entity('ToolcallsStats', { totalCount: 12, totalDurationSeconds: 34 }),
+};
+
+const toolcallsStatsByFunctionForFlow: ResultOf<typeof ToolcallsStatsByFunctionForFlowDocument> = {
+ toolcallsStatsByFunctionForFlow: [
+ entity('FunctionToolcallsStats', {
+ avgDurationSeconds: 2.5,
+ functionName: 'terminal',
+ isAgent: false,
+ totalCount: 8,
+ totalDurationSeconds: 20,
+ }),
+ ],
+};
+
+const flowStatsByFlow: ResultOf<typeof FlowStatsByFlowDocument> = {
+ flowStatsByFlow: entity('FlowStats', { totalAssistantsCount: 1, totalSubtasksCount: 3, totalTasksCount: 1 }),
+};
+
export const flowTabsCassette = (): Cassette =>
flowsCassette({
queries: {
flow: [{ data: flowTabsData, variables: { id: '5' } }],
flowFiles: [{ data: flowFiles, variables: { flowId: '5' } }],
+ flowStatsByFlow: [{ data: flowStatsByFlow, variables: { flowId: '5' } }],
+ toolcallsStatsByFlow: [{ data: toolcallsStatsByFlow, variables: { flowId: '5' } }],
+ toolcallsStatsByFunctionForFlow: [{ data: toolcallsStatsByFunctionForFlow, variables: { flowId: '5' } }],
+ usageStatsByAgentTypeForFlow: [{ data: usageStatsByAgentTypeForFlow, variables: { flowId: '5' } }],
+ usageStatsByFlow: [{ data: usageStatsByFlow, variables: { flowId: '5' } }],
+ usageStatsByModelAgentsForFlow: [{ data: usageStatsByModelAgentsForFlow, variables: { flowId: '5' } }],
},
rest: {
[`GET /api/v1/flows/5/screenshots/${TABS_SCREENSHOT_ID}/file`]: [
diff --git a/frontend/e2e/routes.ts b/frontend/e2e/routes.ts
index 5abeecbe..1209e6a4 100644
--- a/frontend/e2e/routes.ts
+++ b/frontend/e2e/routes.ts
@@ -7,7 +7,7 @@ import type { Cassette } from './mocks/cassette.ts';
import { apiTokensCassette } from './mocks/cassettes/api-tokens.ts';
import { dashboardCassette } from './mocks/cassettes/dashboard.ts';
-import { flowsCassette } from './mocks/cassettes/flows.ts';
+import { flowsCassette, flowTabsCassette } from './mocks/cassettes/flows.ts';
import { knowledgesCassette } from './mocks/cassettes/knowledges.ts';
import { resourcesCassette } from './mocks/cassettes/resources.ts';
import { settingsPromptsCassette } from './mocks/cassettes/settings-prompts.ts';
@@ -26,6 +26,8 @@ export interface RouteManifestEntry {
* single-route listing silently skips the other consumers.
*/
sources: string[];
+ /** Radix unmounts inactive tab panels, so one scan of the default view sees none of them. */
+ tabs?: string[];
}
/**
@@ -45,8 +47,19 @@ export const ROUTE_MANIFEST: RouteManifestEntry[] = [
{
// Message metadata (date + ID) renders at 50% opacity by design; revisit
// with the design pass.
- a11yWaivers: [{ rule: 'color-contrast', target: /text-muted-foreground\\?\/50/ }],
- cassette: flowsCassette,
+ a11yWaivers: [
+ { rule: 'color-contrast', target: /text-muted-foreground\\?\/50/ },
+ // The tab-panel ones below are defects awaiting a fix, not accepted design.
+ { rule: 'aria-progressbar-name', target: /bg-primary/ },
+ { rule: 'button-name', target: /span\[data-slot|button\[aria-label/ },
+ { rule: 'target-size', target: /text-blue-400|button\[aria-label/ },
+ { rule: 'color-contrast', target: /font-semibold\.truncate/ },
+ // The same file-manager row metadata waived on /resources — this tab embeds it.
+ { rule: 'color-contrast', target: /text-muted-foreground\\?\/80/ },
+ { rule: 'scrollable-region-focusable', target: /table-container/ },
+ ],
+ // Must stay the populated cassette: the empty seed renders none of the sources below.
+ cassette: flowTabsCassette,
path: routes.flow('5'),
// The terminal mounts after the header, and the visual spec masks it —
// capturing before it exists compares live pixels against a masked baseline.
@@ -60,6 +73,7 @@ export const ROUTE_MANIFEST: RouteManifestEntry[] = [
'src/components/dashboard',
'src/features/resources',
],
+ tabs: ['Assistant', 'Dashboard', 'Tasks', 'Agents', 'Searches', 'Vector Store', 'Files', 'Screenshots'],
},
{
cassette: templatesCassette,
diff --git a/frontend/e2e/specs/cross/a11y.spec.ts b/frontend/e2e/specs/cross/a11y.spec.ts
index 7224275a..917fffa0 100644
--- a/frontend/e2e/specs/cross/a11y.spec.ts
+++ b/frontend/e2e/specs/cross/a11y.spec.ts
@@ -1,5 +1,6 @@
import { expect, test } from '../../fixtures/test.ts';
import { scanA11y } from '../../helpers/a11y.ts';
+import { populatedSettingsProvidersCassette } from '../../mocks/cassettes/settings-providers.ts';
import { loginJourneyCassette } from '../../mocks/cassettes/smoke.ts';
import { ROUTE_MANIFEST } from '../../routes.ts';
@@ -26,6 +27,19 @@ for (const theme of THEMES) {
});
});
+ test.describe('settings providers (populated)', () => {
+ test.use({ cassette: populatedSettingsProvidersCassette() });
+
+ // Not a duplicate of the manifest scan: that one sweeps the empty state, which
+ // renders no provider cards at all.
+ test('provider cards have no axe violations', async ({ page }) => {
+ await page.goto('/settings/providers');
+ await expect(page.getByText('My Custom Endpoint')).toBeVisible();
+ await expect(page.locator('html')).toHaveClass(theme === 'dark' ? /dark/ : /light/);
+ await scanA11y(page, '/settings/providers (populated)');
+ });
+ });
+
for (const entry of ROUTE_MANIFEST) {
test.describe(entry.path, () => {
test.use({ cassette: entry.cassette() });
@@ -37,6 +51,16 @@ for (const theme of THEMES) {
await expect(page.locator('html')).toHaveClass(theme === 'dark' ? /dark/ : /light/);
await scanA11y(page, entry.path, entry.a11yWaivers);
});
+
+ for (const tab of entry.tabs ?? []) {
+ test(`tab "${tab}" has no axe violations`, async ({ page }) => {
+ await page.goto(entry.path);
+ await expect(entry.ready(page)).toBeVisible();
+ await expect(page.locator('html')).toHaveClass(theme === 'dark' ? /dark/ : /light/);
+ await page.getByRole('tab', { name: tab }).click();
+ await scanA11y(page, `${entry.path} [${tab}]`, entry.a11yWaivers);
+ });
+ }
});
}
});
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png
index d5e1490e..2570ff10 100644
Binary files a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png differ
diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png
index 2cc2f159..dedda0f9 100644
Binary files a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png differ