Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(tests): bind route guard to prepared agent for R15-L1
Josué9b1ee3e0968e
diff --git a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.tsindex 838a166ff..00c4d4215 100644--- a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts+++ b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts@@ -153,8 +153,14 @@ async function receiptApproval(page: Page, scenario: Scenario, previousCalls: nutest('unauthenticated agent route guard prevents connector use', async ({ page }) => {const scenario = cases[0]!const before = await readEvidence(scenario)- // E2E_GUARDIAN_NAVIGATION: negative-route-guard — protected editor must stay unavailable- await page.goto('/agents/e2e-codex-authoring/connectors')+ const protectedRoute = `/agents/${scenario.agentName}/connectors`+ const protectedRouteRequest = page.waitForRequest(+ request => request.isNavigationRequest() && new URL(request.url()).pathname === protectedRoute+ )+ // E2E_GUARDIAN_NAVIGATION: fixture-agent-route-guard — prepared agent route stays protected+ await page.goto(`/agents/${scenario.agentName}/connectors`)+ const visitedRoute = await protectedRouteRequest+ expect(new URL(visitedRoute.url()).pathname).toBe(protectedRoute)await expect(page.getByLabel('Username or email')).toBeVisible()await expect(page.getByRole('button', { name: 'Sign in', exact: true })).toBeVisible()await expect(page.getByRole('button', { name: 'Add connector', exact: true })).toHaveCount(0)diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.jsonindex 99a2d0dd7..4ec250770 100644--- a/tests/e2e/playwright/package-lock.json+++ b/tests/e2e/playwright/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-playwright-tests",- "version": "0.1.77",+ "version": "0.1.78","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-playwright-tests",- "version": "0.1.77",+ "version": "0.1.78","devDependencies": {"@playwright/test": "^1.50.0","@types/node": "^22.0.0",diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.jsonindex 69341f88b..27b9635c6 100644--- a/tests/e2e/playwright/package.json+++ b/tests/e2e/playwright/package.json@@ -1,6 +1,6 @@{"name": "clerum-playwright-tests",- "version": "0.1.77",+ "version": "0.1.78","private": true,"description": "Playwright E2E tests for Control UI and Desktop App","scripts": {diff --git a/tools/e2e_static_audit.py b/tools/e2e_static_audit.pyindex 1a5b64ca0..2ce11d062 100755--- a/tools/e2e_static_audit.py+++ b/tools/e2e_static_audit.py@@ -26,9 +26,114 @@ EMAIL_RESET_ENTRY_PATH = re.compile(r"/invitations/[A-Za-z0-9_-]+")SAFE_STATIC_APP_PATH = re.compile(r"/(?:[A-Za-z0-9_-]+(?:/[A-Za-z0-9_-]+)*)?")NAVIGATION_ANNOTATION = re.compile(r"\s*//\s*E2E_GUARDIAN_NAVIGATION:\s*"- r"(negative-route-guard|root-entry|email-reset-entry)\s+—\s+(\S.*)"+ r"(negative-route-guard|root-entry|email-reset-entry|fixture-agent-route-guard)\s+—\s+(\S.*)")STATIC_GOTO_ARGUMENT = re.compile(r"\s*(['\"])([^'\"\\\r\n]*)\1\s*(?=[,)])")+FIXTURE_AGENT_GOTO_ARGUMENT = re.compile(+ r"\s*`/agents/\$\{scenario\.agentName\}/connectors`\s*(?=[,)])"+)+FIXTURE_AGENT_GUARD_SPEC = Path(+ "tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts"+)+FIXTURE_AGENT_GUARD_TITLE = "unauthenticated agent route guard prevents connector use"+++def _fixture_agent_sources_are_safe() -> bool:+ repository_root = Path(__file__).resolve().parents[1]+ scenarios_path = repository_root / "tests/e2e/playwright/helpers/approved-tools-scenarios.ts"+ preparer_path = repository_root / "scripts/e2e/prepare-codex-approved-tools.mjs"+ if not scenarios_path.is_file() or not preparer_path.is_file():+ return False+ scenarios = scenarios_path.read_text(encoding="utf-8")+ preparer = preparer_path.read_text(encoding="utf-8")+ safe_name_validation = re.search(+ r"for \(const key of \['runId', 'agentName', 'connectorName', 'contextName'\] as const\)"+ r"\s*\{\s*if \(!/\^\[a-zA-Z0-9_-\]\{1,80\}\$/.test\(row\[key\]\)\)",+ scenarios,+ )+ unique_agent_names = re.search(+ r"for \(const key of \['runId', 'agentName', 'contextName', 'connectorName', 'fixtureUrl'\] as const\)"+ r"\s*\{\s*if \(new Set\(rows\.map\(row => row\[key\]\)\)\.size !== 3\)",+ scenarios,+ )+ return bool(+ safe_name_validation+ and unique_agent_names+ and "agentName: `${run}-agent-${label}`" in preparer+ )+++def _is_fixture_agent_guard(text: str, path: Path, match: re.Match[str]) -> bool:+ repository_root = Path(__file__).resolve().parents[1]+ try:+ if path.resolve() != repository_root / FIXTURE_AGENT_GUARD_SPEC:+ return False+ except OSError:+ return False++ imports = re.search(+ r"import\s*\{(?P<bindings>[^}]+)\}\s*from\s*['\"]\.\./helpers/approved-tools-scenarios['\"]",+ text,+ re.DOTALL,+ )+ if (+ not imports+ or not re.search(r"\breadEvidence\b", imports.group("bindings"))+ or not re.search(r"\bscenarios\b", imports.group("bindings"))+ or text.count("const cases = scenarios()") != 1+ or not _fixture_agent_sources_are_safe()+ ):+ return False++ test_pattern = re.compile(+ r"test\(\s*(['\"])" + re.escape(FIXTURE_AGENT_GUARD_TITLE) + r"\1\s*,"+ r"\s*async\s*\(\s*\{\s*page\s*\}\s*\)\s*=>\s*\{"+ r"(?P<body>[\s\S]*?)\n\}\s*\)",+ )+ test_match = test_pattern.search(text)+ if not test_match:+ return False+ body = test_match.group("body")+ body_navigation = list(re.finditer(r"\bpage\.goto\s*\(", body))+ if len(body_navigation) != 1:+ return False+ body_goto = body_navigation[0]+ if not FIXTURE_AGENT_GOTO_ARGUMENT.match(body[body_goto.end() :]):+ return False+ if not re.search(r"const scenario = cases\[0\]!", body):+ return False+ if not re.search(r"const before = await readEvidence\(scenario\)", body):+ return False+ if not re.search(+ r"expect\(\(await readEvidence\(scenario\)\)\.calls\)\.toEqual\(before\.calls\)",+ body,+ ):+ return False+ if not re.search(+ r"expect\(page\.getByLabel\(['\"]Username or email['\"]\)\)\.toBeVisible\(\)",+ body,+ ):+ return False+ if not re.search(+ r"expect\(page\.getByRole\(['\"]button['\"],\s*\{\s*name:\s*['\"]Sign in['\"],"+ r"\s*exact:\s*true\s*\}\)\)\.toBeVisible\(\)",+ body,+ ):+ return False+ if not re.search(+ r"expect\(page\.getByRole\(['\"]button['\"],\s*\{\s*name:\s*['\"]Add connector"+ r"[^}]*\}\)\)\.toHaveCount\(0\)",+ body,+ ):+ return False+ if not re.search(+ r"expect\(page\.getByLabel\(['\"]Main navigation['\"]\)\)\.toHaveCount\(0\)",+ body,+ ):+ return False++ absolute_goto_start = test_match.start("body") + body_goto.start()+ return match.start() == absolute_goto_startdef audit_file(path: Path) -> list[str]:@@ -81,6 +186,13 @@ def audit_text(text: str, path: Path = Path("<source>")) -> list[str]:annotation = NAVIGATION_ANNOTATION.fullmatch(adjacent)destination = root_argument.group(2) if root_argument else None+ if (+ annotation+ and annotation.group(1) == "fixture-agent-route-guard"+ and FIXTURE_AGENT_GOTO_ARGUMENT.match(after)+ and _is_fixture_agent_guard(text, path, match)+ ):+ continueif destination in {"", "/"}:if annotation is None or annotation.group(1) == "root-entry":continuediff --git a/tools/test_e2e_static_audit_scope.py b/tools/test_e2e_static_audit_scope.pyindex 83e816b21..ebd787143 100644--- a/tools/test_e2e_static_audit_scope.py+++ b/tools/test_e2e_static_audit_scope.py@@ -198,7 +198,7 @@ class E2EStaticAuditMarkerScopeTests(unittest.TestCase):"tests/e2e/playwright/profile-ui-recovery/reset-session-hydration.spec.ts").read_text(encoding="utf-8")without_marker = spec.replace(- "// E2E_GUARDIAN_NAVIGATION: email-reset-entry — the emailed password-reset URL\n", ""+ "// E2E_GUARDIAN_NAVIGATION: email-reset-entry — the synthetic emailed reset URL\n", "")self.assertTrue(any(@@ -231,6 +231,115 @@ class E2EStaticAuditMarkerScopeTests(unittest.TestCase):findings = audit_text(source, Path("root.spec.ts"))self.assertTrue(any("does not match" in finding for finding in findings))+ def fixture_agent_guard(self, route: str, *, path: Path | None = None) -> list[str]:+ source = (+ "// E2E_GUARDIAN_IPC_FLOW: Desktop uses real main-process IPC.\n"+ "import { readEvidence, scenarios } from '../helpers/approved-tools-scenarios'\n"+ "const cases = scenarios()\n"+ "test('unauthenticated agent route guard prevents connector use', async ({ page }) => {\n"+ " const scenario = cases[0]!\n"+ " const before = await readEvidence(scenario)\n"+ " // E2E_GUARDIAN_NAVIGATION: fixture-agent-route-guard — prepared agent route stays protected\n"+ f" await page.goto({route})\n"+ " await expect(page.getByLabel('Username or email')).toBeVisible()\n"+ " await expect(page.getByRole('button', { name: 'Sign in', exact: true })).toBeVisible()\n"+ " await expect(page.getByRole('button', { name: 'Add connector', exact: true })).toHaveCount(0)\n"+ " await expect(page.getByLabel('Main navigation')).toHaveCount(0)\n"+ " expect((await readEvidence(scenario)).calls).toEqual(before.calls)\n"+ "})\n"+ )+ return audit_text(+ source,+ path+ or Path("tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts"),+ )++ def test_fixture_created_agent_route_is_allowed_only_for_its_evidence_scenario(self) -> None:+ self.assertFalse(+ any(+ "direct page.goto" in item+ for item in self.fixture_agent_guard("`/agents/${scenario.agentName}/connectors`")+ )+ )++ def test_fixture_agent_route_exception_rejects_other_files_and_destinations(self) -> None:+ cases = [+ (+ "`/agents/${scenario.agentName}/connectors`",+ Path("tests/e2e/playwright/other.spec.ts"),+ ),+ ("destination", None),+ ("`/agents/${scenario.otherName}/connectors`", None),+ ("`/agents/${scenario.agentName}/connectors/extra`", None),+ ("`https://outside.invalid/agents/${scenario.agentName}/connectors`", None),+ ("`/agents/${scenario.agentName}/connectors?next=/`", None),+ ("`/agents/${scenario.agentName}/connectors#fragment`", None),+ ("`/agents/${scenario.agentName.replace('/', '')}/connectors`", None),+ ("`/agents/${scenario.agentName}/../connectors`", None),+ ("`/agents/${scenario.agentName}/%2e%2e/connectors`", None),+ ("`/agents/${scenario.agentName}/connectors` + '/extra'", None),+ ("`/agents/${scenario.agentName}/connectors` + '?next=/'", None),+ ]+ for route, path in cases:+ with self.subTest(route=route, path=path):+ self.assertTrue(+ any("direct page.goto" in item for item in self.fixture_agent_guard(route, path=path))+ )++ def test_fixture_agent_exception_requires_same_scenario_evidence_before_and_after(self) -> None:+ source = (+ "// E2E_GUARDIAN_IPC_FLOW: Desktop uses real main-process IPC.\n"+ "import { readEvidence, scenarios } from '../helpers/approved-tools-scenarios'\n"+ "const cases = scenarios()\n"+ "test('unauthenticated agent route guard prevents connector use', async ({ page }) => {\n"+ " const scenario = cases[0]!\n"+ " const before = await readEvidence(cases[1]!)\n"+ " // E2E_GUARDIAN_NAVIGATION: fixture-agent-route-guard — prepared agent route stays protected\n"+ " await page.goto(`/agents/${scenario.agentName}/connectors`)\n"+ " await expect(page.getByLabel('Username or email')).toBeVisible()\n"+ " await expect(page.getByRole('button', { name: 'Sign in', exact: true })).toBeVisible()\n"+ " await expect(page.getByRole('button', { name: 'Add connector', exact: true })).toHaveCount(0)\n"+ " await expect(page.getByLabel('Main navigation')).toHaveCount(0)\n"+ " expect((await readEvidence(scenario)).calls).toEqual(before.calls)\n"+ "})\n"+ )+ self.assertTrue(+ any(+ "direct page.goto" in item+ for item in audit_text(+ source,+ Path("tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts"),+ )+ )+ )++ def test_fixture_agent_exception_requires_scenarios_fixture_provenance(self) -> None:+ source = (+ "// E2E_GUARDIAN_IPC_FLOW: Desktop uses real main-process IPC.\n"+ "import { readEvidence, scenarios } from '../helpers/approved-tools-scenarios'\n"+ "const cases = loadUntrustedCases()\n"+ "test('unauthenticated agent route guard prevents connector use', async ({ page }) => {\n"+ " const scenario = cases[0]!\n"+ " const before = await readEvidence(scenario)\n"+ " // E2E_GUARDIAN_NAVIGATION: fixture-agent-route-guard — prepared agent route stays protected\n"+ " await page.goto(`/agents/${scenario.agentName}/connectors`)\n"+ " await expect(page.getByLabel('Username or email')).toBeVisible()\n"+ " await expect(page.getByRole('button', { name: 'Sign in', exact: true })).toBeVisible()\n"+ " await expect(page.getByRole('button', { name: 'Add connector', exact: true })).toHaveCount(0)\n"+ " await expect(page.getByLabel('Main navigation')).toHaveCount(0)\n"+ " expect((await readEvidence(scenario)).calls).toEqual(before.calls)\n"+ "})\n"+ )+ self.assertTrue(+ any(+ "direct page.goto" in item+ for item in audit_text(+ source,+ Path("tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts"),+ )+ )+ )+def run_tests() -> int:suite = unittest.defaultTestLoader.loadTestsFromModule(sys.modules[__name__])