AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
ci(platform): add a journeys leg to e2e for hosted-config user journeys (#15397)
Nicholas Tindlee09793064ab4
diff --git a/.github/scripts/hosted_journeys_env.py b/.github/scripts/hosted_journeys_env.pynew file mode 100644index 0000000000..6d23316735--- /dev/null+++ b/.github/scripts/hosted_journeys_env.py@@ -0,0 +1,49 @@+"""Print the backend .env lines that put a compose stack in hosted configuration.++The journeys leg of platform-fullstack-ci appends these to backend/.env. Flag+values come from the same file the Playwright journeys hand the frontend, so+both sides always agree.+"""++import argparse+import json+from pathlib import Path++DEFAULT_FLAGS_FILE = (+ Path(__file__).resolve().parents[2]+ / "autogpt_platform/frontend/src/playwright/journeys/hosted-flags.json"+)++# Settings the hosted product runs with that the compose defaults leave off.+# EXECUTION_COST_COUNT_THRESHOLD is NOT a hosted value (hosted bills one credit+# per 100 node executions): billing every node makes a run's cost visible, so+# journeys can assert it exactly.+HOSTED_SETTINGS = {+ "ENABLE_CREDIT": "true",+ "EXECUTION_COST_COUNT_THRESHOLD": "1",+}+++def flag_env_name(flag_key: str) -> str:+ return "FORCE_FLAG_" + flag_key.upper().replace("-", "_")+++def hosted_env_lines(flags: dict[str, object]) -> list[str]:+ lines = [f"{name}={value}" for name, value in HOSTED_SETTINGS.items()]+ for key, value in flags.items():+ if not isinstance(value, bool):+ raise ValueError(f"hosted flag {key!r} must be true or false")+ lines.append(f"{flag_env_name(key)}={'true' if value else 'false'}")+ return lines+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--flags-file", type=Path, default=DEFAULT_FLAGS_FILE)+ args = parser.parse_args()+ flags = json.loads(args.flags_file.read_text(encoding="utf-8"))+ print("\n".join(hosted_env_lines(flags)))+++if __name__ == "__main__":+ main()diff --git a/.github/scripts/test_e2e_seed_cache_contract.py b/.github/scripts/test_e2e_seed_cache_contract.pyindex 3f2da4b49c..5a440d24a6 100644--- a/.github/scripts/test_e2e_seed_cache_contract.py+++ b/.github/scripts/test_e2e_seed_cache_contract.py@@ -4,6 +4,51 @@ import unittestfrom pathlib import Path+def _matrix_leg(workflow: str, suite: str) -> dict[str, str]:+ e2e_job = workflow.split("\n e2e_test:\n", 1)[1].split("\n steps:\n", 1)[0]+ leg = e2e_job.split(f"- suite: {suite}\n", 1)[1].split("- suite:", 1)[0]+ return dict(re.findall(r"^\s+(\w+): (\S+)$", leg, re.MULTILINE))+++class E2EMatrixContractTests(unittest.TestCase):+ def test_legs_keep_separate_seed_caches(self):+ workflow = (+ Path(__file__).resolve().parents[1] / "workflows/platform-fullstack-ci.yml"+ ).read_text(encoding="utf-8")+ key = next(+ line for line in workflow.splitlines() if "key: e2e-test-data-" in line+ )+ self.assertIn("matrix.embeddings", key)+ self.assertEqual(_matrix_leg(workflow, "smoke")["embeddings"], "true")+ self.assertEqual(_matrix_leg(workflow, "journeys")["embeddings"], "false")++ def test_only_smoke_writes_shared_caches(self):+ workflow = (+ Path(__file__).resolve().parents[1] / "workflows/platform-fullstack-ci.yml"+ ).read_text(encoding="utf-8")+ write_cache = next(+ line for line in workflow.splitlines() if "WRITE_BUILD_CACHE: ${{" in line+ )+ self.assertIn("matrix.suite == 'smoke' &&", write_cache)+ for step in (+ "Export Next.js build cache from BuildKit",+ "Save Next.js build cache",+ "Set up tests - Cache Playwright browsers",+ ):+ with self.subTest(step=step):+ body = workflow.split(f"- name: {step}\n", 1)[1].split(+ "\n - name:", 1+ )[0]+ self.assertIn("matrix.suite == 'smoke'", body)++ def test_journeys_check_is_advisory(self):+ checker = (+ Path(__file__).resolve().parents[1]+ / "workflows/scripts/check_actions_status.py"+ ).read_text(encoding="utf-8")+ self.assertIn('"end-to-end tests (journeys)"', checker)++class E2ESeedCacheContractTests(unittest.TestCase):def test_fresh_and_cached_seed_credentials_are_verified_before_tests(self):workflow = (@@ -14,7 +59,12 @@ class E2ESeedCacheContractTests(unittest.TestCase):"\n - name:", 1)[0]self.assertIn("--refresh-credentials-only", verification)- self.assertNotIn("if:", verification)+ # Runs whenever the leg seeds, cached dump or fresh; only the+ # journeys leg, which seeds nothing, skips it.+ self.assertEqual(+ re.findall(r"^\s+if: (.+)$", verification, re.M), ["matrix.seed"]+ )+ self.assertEqual(_matrix_leg(workflow, "smoke")["seed"], "true")self.assertNotIn("continue-on-error", verification)self.assertLess(workflow.index("- name: Set up tests - Create E2E test data"),@@ -69,7 +119,11 @@ class E2ECoverageContractTests(unittest.TestCase):workflow = (Path(__file__).resolve().parents[1] / "workflows/platform-fullstack-ci.yml").read_text(encoding="utf-8")- self.assertIn("--min-tests 28", workflow)+ self.assertIn("--min-tests ${{ matrix.min_tests }}", workflow)+ self.assertEqual(_matrix_leg(workflow, "smoke")["min_tests"], "28")+ self.assertEqual(_matrix_leg(workflow, "journeys")["min_tests"], "4")+ # Smoke stays strict: no test.fail() known failures.+ self.assertEqual(_matrix_leg(workflow, "smoke")["known_failures"], "false")def test_codecov_requires_validated_coverage(self):workflow = (diff --git a/.github/scripts/test_hosted_journeys_env.py b/.github/scripts/test_hosted_journeys_env.pynew file mode 100644index 0000000000..bd27684073--- /dev/null+++ b/.github/scripts/test_hosted_journeys_env.py@@ -0,0 +1,29 @@+import json+import unittest++from hosted_journeys_env import DEFAULT_FLAGS_FILE, hosted_env_lines+++class HostedJourneysEnvTests(unittest.TestCase):+ def test_flags_become_backend_overrides(self):+ lines = hosted_env_lines(+ {"hire-experts": True, "enable-platform-payment": False}+ )+ self.assertIn("FORCE_FLAG_HIRE_EXPERTS=true", lines)+ self.assertIn("FORCE_FLAG_ENABLE_PLATFORM_PAYMENT=false", lines)+ self.assertIn("ENABLE_CREDIT=true", lines)++ def test_non_boolean_flags_are_rejected(self):+ with self.assertRaises(ValueError):+ hosted_env_lines({"copilot-model-routing": {"default": "x"}})++ def test_checked_in_flags_match_hosted_and_are_boolean(self):+ # Payments are on as in production; journey users get a paid tier+ # from the Playwright provisioning helper instead of Stripe.+ flags = json.loads(DEFAULT_FLAGS_FILE.read_text(encoding="utf-8"))+ self.assertIs(flags["enable-platform-payment"], True)+ self.assertTrue(all(isinstance(v, bool) for v in flags.values()))+++if __name__ == "__main__":+ unittest.main()diff --git a/.github/scripts/test_validate_playwright_json.py b/.github/scripts/test_validate_playwright_json.pyindex a63f3e209c..722ef3df90 100644--- a/.github/scripts/test_validate_playwright_json.py+++ b/.github/scripts/test_validate_playwright_json.py@@ -46,6 +46,24 @@ class ValidatePlaywrightJSONTests(unittest.TestCase):def test_accepts_clean_single_attempt_report(self):self.assertEqual(validate_report(valid_report()), 1)+ def test_known_failures_are_only_accepted_when_allowed(self):+ report = valid_report()+ test = report["suites"][0]["specs"][0]["tests"][0]+ test["expectedStatus"] = "failed"+ test["results"][0]["status"] = "failed"+ test["results"][0]["errors"] = [{"message": "the bug is still there"}]++ with self.assertRaisesRegex(ValueError, "expectedStatus must be passed"):+ validate_report(report)+ self.assertEqual(validate_report(report, allow_known_failures=True), 1)++ def test_a_known_failure_that_passed_is_rejected(self):+ report = valid_report()+ report["suites"][0]["specs"][0]["tests"][0]["expectedStatus"] = "failed"++ with self.assertRaisesRegex(ValueError, "must be failed for a known failure"):+ validate_report(report, allow_known_failures=True)+def test_rejects_configured_retries(self):report = valid_report()report["config"]["projects"][0]["retries"] = 2diff --git a/.github/scripts/validate_playwright_json.py b/.github/scripts/validate_playwright_json.pyindex 95e93fdde5..ae2f9ad007 100644--- a/.github/scripts/validate_playwright_json.py+++ b/.github/scripts/validate_playwright_json.py@@ -45,7 +45,9 @@ def _iter_tests()-def validate_report(report: object, min_tests: int = 1) -> int:+def validate_report(+ report: object, min_tests: int = 1, allow_known_failures: bool = False+) -> int:root = _require_mapping(report, "report")errors = _require_list(root.get("errors"), "errors")if errors:@@ -92,7 +94,9 @@ def validate_report(report: object, min_tests: int = 1) -> int:raise ValueError(f"{test_name}.status must be expected, got {test.get('status')!r}")- if test.get("expectedStatus") != "passed":+ # test.fail(): a known bug the test is pinned to. It still has to fail.+ known_failure = allow_known_failures and test.get("expectedStatus") == "failed"+ if test.get("expectedStatus") != "passed" and not known_failure:raise ValueError(f"{test_name}.expectedStatus must be passed, got "f"{test.get('expectedStatus')!r}"@@ -106,6 +110,13 @@ def validate_report(report: object, min_tests: int = 1) -> int:retry = _require_count(result.get("retry"), f"{test_name}.results[0].retry")if retry != 0:raise ValueError(f"{test_name}.results[0].retry must be 0, got {retry}")+ if known_failure:+ if result.get("status") != "failed":+ raise ValueError(+ f"{test_name}.results[0].status must be failed for a known "+ f"failure, got {result.get('status')!r}"+ )+ continueif result.get("status") != "passed":raise ValueError(f"{test_name}.results[0].status must be passed, got "@@ -161,6 +172,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:default=1,help="minimum number of discovered passing tests required",)+ parser.add_argument(+ "--allow-known-failures",+ action="store_true",+ help="accept tests marked test.fail() that failed as expected",+ )parser.add_argument("report", type=Path)return parser.parse_args(argv)@@ -183,7 +199,11 @@ def main(argv: list[str] | None = None) -> int:return 1try:- test_count = validate_report(report, min_tests=args.min_tests)+ test_count = validate_report(+ report,+ min_tests=args.min_tests,+ allow_known_failures=args.allow_known_failures,+ )except ValueError as exc:print(f"{args.report}: {exc}")return 1diff --git a/.github/workflows/platform-fullstack-ci.yml b/.github/workflows/platform-fullstack-ci.ymlindex bc844abc4d..56c1ec64f8 100644--- a/.github/workflows/platform-fullstack-ci.yml+++ b/.github/workflows/platform-fullstack-ci.yml@@ -10,6 +10,8 @@ on:- ".github/scripts/validate_cobertura.py"- ".github/scripts/test_validate_cobertura.py"- ".github/scripts/test_e2e_seed_cache_contract.py"+ - ".github/scripts/hosted_journeys_env.py"+ - ".github/scripts/test_hosted_journeys_env.py"- ".github/scripts/docker-pull-with-retry.sh"- ".github/workflows/scripts/docker-ci-fix-compose-build-cache.py"- ".github/workflows/scripts/get_package_version_from_lockfile.py"@@ -22,6 +24,8 @@ on:- ".github/scripts/validate_cobertura.py"- ".github/scripts/test_validate_cobertura.py"- ".github/scripts/test_e2e_seed_cache_contract.py"+ - ".github/scripts/hosted_journeys_env.py"+ - ".github/scripts/test_hosted_journeys_env.py"- ".github/scripts/docker-pull-with-retry.sh"- ".github/workflows/scripts/docker-ci-fix-compose-build-cache.py"- ".github/workflows/scripts/get_package_version_from_lockfile.py"@@ -34,6 +38,11 @@ on:required: falsedefault: falsetype: boolean+ journeys_repeat_each:+ description: Run every journey this many times, to measure flakiness+ required: false+ default: 1+ type: numberconcurrency:group: ${{ github.event_name == 'workflow_dispatch' && format('{0}-dispatch-{1}', github.workflow, github.run_id) || format('{0}-{1}', github.workflow, github.event_name == 'merge_group' && format('merge-queue-{0}', github.ref) || github.head_ref && format('pr-{0}', github.event.pull_request.number) || github.sha) }}@@ -91,6 +100,7 @@ jobs:../../.github/scripts/test_validate_playwright_json.py \../../.github/scripts/test_validate_cobertura.py \../../.github/scripts/test_e2e_seed_cache_contract.py \+ ../../.github/scripts/test_hosted_journeys_env.py \--junitxml="$RUNNER_TEMP/fullstack-validator-results/playwright-validator-tests.xml"- name: Set up Backend - Generate Prisma client@@ -165,8 +175,39 @@ jobs:if-no-files-found: errore2e_test:- name: end-to-end tests+ name: end-to-end tests (${{ matrix.suite }})runs-on: ubuntu-latest+ strategy:+ fail-fast: false+ matrix:+ include:+ # Today's smoke suite on the self-hosted configuration. The only leg+ # that writes the shared build caches.+ - suite: smoke+ config: playwright.config.ts+ playwright_args: ""+ # Pinned inventory: 28 tests in 9 files; raise when adding E2E tests.+ min_tests: 28+ embeddings: true+ seed: true+ hosted: false+ known_failures: false+ # Holistic *-journey.spec.ts flows on the hosted configuration:+ # hosted feature flags, credits on. Advisory until proven stable+ # (ADVISORY_CHECK_RUNS in check_actions_status.py). No OpenAI key,+ # so the seed skips embeddings and finishes in ~30 s.+ - suite: journeys+ config: playwright.journeys.config.ts+ # selfhost-journey asserts nothing hosted-only exists; it needs a+ # self-hosted stack (TODO in src/playwright/journeys/README.md).+ playwright_args: --project hosted+ min_tests: 4+ embeddings: false+ # Journeys sign up their own users and need no seeded data.+ seed: false+ hosted: true+ # Journeys may pin a known bug with test.fail(); it must still fail.+ known_failures: truepermissions:contents: read# Only dev pushes and the merge queue write the GHCR build cache. Every@@ -185,11 +226,12 @@ jobs:- name: Set up tests - Cache E2E test dataid: e2e-data-cache+ if: matrix.seeduses: actions/cache@v5with:path: /tmp/e2e_test_data.sql# Hash checkout inputs before fresh secrets and resolved Compose exist.- key: e2e-test-data-v3-${{ hashFiles('autogpt_platform/backend/**', 'autogpt_platform/autogpt_libs/**', 'autogpt_platform/docker-compose*.yml', 'autogpt_platform/.env.default', '.github/workflows/platform-fullstack-ci.yml') }}+ key: e2e-test-data-v3-${{ matrix.embeddings && 'emb' || 'noemb' }}-${{ hashFiles('autogpt_platform/backend/**', 'autogpt_platform/autogpt_libs/**', 'autogpt_platform/docker-compose*.yml', 'autogpt_platform/.env.default', '.github/workflows/platform-fullstack-ci.yml') }}- name: Set up Platform - Pull BuildKit imageid: buildkit-image@@ -208,13 +250,21 @@ jobs:- name: Set up Platform - Configure backend .envrun: |- echo "OPENAI_INTERNAL_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ../backend/.env+ if [[ "${{ matrix.embeddings }}" == "true" ]]; then+ echo "OPENAI_INTERNAL_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ../backend/.env+ fiecho "CHAT_TEST_MODE=true" >> ../backend/.envecho "SCHEDULER_STARTUP_EMBEDDING_BACKFILL=false" >> ../backend/.envenv:# Used by E2E test data script to generate embeddings for approved store agentsOPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}+ # The hosted configuration, as far as one compose stack allows. Gaps are+ # listed in src/playwright/journeys/README.md.+ - name: Set up Platform - Configure hosted backend .env+ if: matrix.hosted+ run: python ../../.github/scripts/hosted_journeys_env.py >> ../backend/.env+# A failed login leaves buildx anonymous, which still reads a public cache.# A cache that can't be read or written only makes the build cold.- name: Set up Platform - Log in to the GHCR build cache@@ -230,7 +280,8 @@ jobs:env:NEXT_PUBLIC_PW_TEST: trueNEXT_PUBLIC_SOURCEMAPS: true- WRITE_BUILD_CACHE: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/dev') || github.event_name == 'merge_group' || (github.event_name == 'workflow_dispatch' && inputs.publish_build_cache) }}+ # Both legs build the same images; only smoke writes them.+ WRITE_BUILD_CACHE: ${{ matrix.suite == 'smoke' && ((github.event_name == 'push' && github.ref == 'refs/heads/dev') || github.event_name == 'merge_group' || (github.event_name == 'workflow_dispatch' && inputs.publish_build_cache)) }}run: |pip install pyyaml@@ -267,6 +318,8 @@ jobs:- name: Set up Platform - Set up Docker Buildxuses: docker/setup-buildx-action@v3with:+ # The runner is thrown away; removing the builder only costs ~30 s.+ cleanup: falsedriver: docker-containerdriver-opts: |image=${{ env.BUILDKIT_IMAGE }}@@ -338,6 +391,7 @@ jobs:cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml- name: Set up tests - Cache Playwright browsers+ if: matrix.suite == 'smoke'uses: actions/cache@v5with:path: ~/.cache/ms-playwright@@ -345,6 +399,15 @@ jobs:restore-keys: |playwright-${{ runner.os }}-+ - name: Set up tests - Restore Playwright browsers+ if: matrix.suite != 'smoke'+ uses: actions/cache/restore@v5+ with:+ path: ~/.cache/ms-playwright+ key: playwright-${{ runner.os }}-${{ hashFiles('autogpt_platform/frontend/pnpm-lock.yaml') }}+ restore-keys: |+ playwright-${{ runner.os }}-+- name: Set up tests - Install dependenciesrun: pnpm install --frozen-lockfile --prefer-offline@@ -398,7 +461,7 @@ jobs:NEXT_PUBLIC_PW_TEST: true- name: Set up tests - Create E2E test data- if: steps.e2e-data-cache.outputs.cache-hit != 'true'+ if: matrix.seed && steps.e2e-data-cache.outputs.cache-hit != 'true'run: |echo "Creating E2E test data..."docker cp ../backend/test/e2e_test_data.py $(docker compose -f ../docker-compose.resolved.yml ps -q rest_server):/tmp/e2e_test_data.py@@ -422,6 +485,7 @@ jobs:echo "✅ Database dump created for caching ($(wc -l < /tmp/e2e_test_data.sql) lines)"- name: Set up tests - Verify seeded credentials with fresh encryption key+ if: matrix.seedrun: |docker cp ../backend/test/e2e_test_data.py $(docker compose -f ../docker-compose.resolved.yml ps -q rest_server):/tmp/e2e_test_data.pydocker compose -f ../docker-compose.resolved.yml exec -T rest_server \@@ -446,17 +510,22 @@ jobs:docker cp "$FRONTEND_CONTAINER":/app/.next/static .next-static-coverage- name: Run Playwright E2E suite- run: pnpm exec playwright test --retries=0 --trace=retain-on-failure+ run: pnpm exec playwright test --config ${{ matrix.config }} ${{ matrix.playwright_args }} ${{ matrix.suite == 'journeys' && format('--repeat-each={0}', inputs.journeys_repeat_each || 1) || '' }} --retries=0 --trace=retain-on-failureenv:PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/results.jsonPLAYWRIGHT_RETRIES: "0"# One worker per vCPU on the 4-vCPU standard runner; the CI default of 8 starves the stack.PLAYWRIGHT_WORKERS: "4"+ # Matches EXECUTION_COST_COUNT_THRESHOLD=1 from hosted_journeys_env.py.+ PLAYWRIGHT_JOURNEY_COST_PER_NODE: ${{ matrix.hosted && '1' || '' }}+ # Payments are on in the hosted leg; journey users get a paid tier+ # without Stripe (src/playwright/journeys/subscription-tier.ts).+ PLAYWRIGHT_JOURNEY_TIER: ${{ matrix.hosted && 'PRO' || '' }}- name: Export Next.js build cache from BuildKitid: next-build-cache-export# Copying ~3.7 GB competes with the browser tests, so it waits for them- if: ${{ !cancelled() && steps.next-build-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }}+ if: ${{ !cancelled() && matrix.suite == 'smoke' && steps.next-build-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }}continue-on-error: trueworking-directory: /tmpbackground: true@@ -475,11 +544,12 @@ jobs:- name: Validate Playwright JSON reportif: ${{ always() }}- # Pinned inventory: 28 tests in 9 files; raise when adding E2E tests.+ # Each leg's floor is pinned in the matrix; raise it when adding tests.run: >-python ../../.github/scripts/validate_playwright_json.py--synthesize-invalid- --min-tests 28+ --min-tests ${{ matrix.min_tests }}+ ${{ matrix.known_failures && '--allow-known-failures' || '' }}test-results/results.json- name: Validate coverage report@@ -495,6 +565,7 @@ jobs:with:token: ${{ secrets.CODECOV_TOKEN }}flags: platform-frontend-e2e+ name: e2e-${{ matrix.suite }}files: ./autogpt_platform/frontend/coverage/e2e/cobertura-coverage.xmldisable_search: true@@ -502,7 +573,7 @@ jobs:if: always()uses: actions/upload-artifact@v7with:- name: playwright-report+ name: playwright-report-${{ matrix.suite }}path: autogpt_platform/frontend/playwright-reportif-no-files-found: ignoreretention-days: 3@@ -511,7 +582,7 @@ jobs:if: ${{ always() }}uses: actions/upload-artifact@v7with:- name: playwright-test-results+ name: playwright-test-results-${{ matrix.suite }}path: |autogpt_platform/frontend/test-resultsautogpt_platform/frontend/coverage/e2e/cobertura-coverage.xml@@ -526,7 +597,7 @@ jobs:wait: [next-build-cache-export]- name: Save Next.js build cache- if: ${{ !cancelled() && steps.next-build-cache-export.outcome == 'success' }}+ if: ${{ !cancelled() && matrix.suite == 'smoke' && steps.next-build-cache-export.outcome == 'success' }}continue-on-error: trueuses: actions/cache/save@v5with:diff --git a/.github/workflows/scripts/check_actions_status.py b/.github/workflows/scripts/check_actions_status.pyindex d9e2710868..17383913fa 100644--- a/.github/workflows/scripts/check_actions_status.py+++ b/.github/workflows/scripts/check_actions_status.py@@ -14,7 +14,12 @@ PASSING_CONCLUSIONS = {"success", "skipped", "neutral"}# through this check. Other review bots report "neutral" in that case, which# PASSING_CONCLUSIONS already accepts; this is the same treatment for one# that does not.-ADVISORY_CHECK_RUNS = {"copilot-pull-request-reviewer"}+# The journeys e2e leg is advisory while it beds in: its failures show on the+# PR but don't block merging. Drop it from here once it has proven stable.+ADVISORY_CHECK_RUNS = {+ "copilot-pull-request-reviewer",+ "end-to-end tests (journeys)",+}def is_advisory(check_run: Dict) -> bool:diff --git a/.github/workflows/scripts/test_check_actions_status.py b/.github/workflows/scripts/test_check_actions_status.pyindex 2699b327c1..4c2b284fdc 100644--- a/.github/workflows/scripts/test_check_actions_status.py+++ b/.github/workflows/scripts/test_check_actions_status.py@@ -150,6 +150,17 @@ class ProcessCheckRunsTests(unittest.TestCase):failed = {"id": 8, "name": "test", "status": "completed", "conclusion": "failure"}self.assertEqual(process_check_runs([copilot, failed]), (False, False))+ def test_a_failing_journeys_leg_is_advisory_but_smoke_still_blocks(self):+ journeys = {+ "id": 9,+ "name": "end-to-end tests (journeys)",+ "status": "completed",+ "conclusion": "failure",+ }+ smoke = {**journeys, "id": 10, "name": "end-to-end tests (smoke)"}+ self.assertEqual(process_check_runs([journeys]), (False, True))+ self.assertEqual(process_check_runs([journeys, smoke]), (False, False))+def test_rejects_real_failure(self):failed_check = {**CANCELED_CHECK, "conclusion": "failure"}diff --git a/autogpt_platform/frontend/playwright.journeys.config.ts b/autogpt_platform/frontend/playwright.journeys.config.tsindex ab2edc5e0f..1cbd5faa78 100644--- a/autogpt_platform/frontend/playwright.journeys.config.ts+++ b/autogpt_platform/frontend/playwright.journeys.config.ts@@ -1,5 +1,6 @@-import { defineConfig } from "@playwright/test";+import { defineConfig, devices } from "@playwright/test";import baseConfig from "./playwright.config";+import { JOURNEY_USER_AGENT } from "./src/playwright/journeys/user-agent";// Journeys sign up their own users (src/playwright/journeys/journey-user.ts),// so unlike the smoke suite they skip the seeded-account global setup and run@@ -8,6 +9,13 @@ const baseURL = String(baseConfig.use?.baseURL);const { hostname } = new URL(baseURL);const isLocalApp = ["localhost", "127.0.0.1", "[::1]"].includes(hostname);+const SELFHOST_JOURNEYS = /selfhost-journey\.spec\.ts/;+const chrome = {+ ...devices["Desktop Chrome"],+ channel: "chromium",+ userAgent: JOURNEY_USER_AGENT,+};+export default defineConfig({...baseConfig,testDir: "./src/playwright/journeys",@@ -16,4 +24,19 @@ export default defineConfig({timeout: 90_000,expect: { timeout: 15_000 },webServer: isLocalApp ? baseConfig.webServer : undefined,+ // `hosted` runs on the hosted configuration (CI's journeys leg);+ // `selfhost` asserts nothing hosted-only exists, so it needs a+ // self-hosted build and must never run against a hosted one.+ projects: [+ {+ name: "hosted",+ testIgnore: SELFHOST_JOURNEYS,+ use: chrome,+ },+ {+ name: "selfhost",+ testMatch: SELFHOST_JOURNEYS,+ use: chrome,+ },+ ],});diff --git a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/sidebar/SidebarRunsList/components/SidebarItemCard.tsx b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/sidebar/SidebarRunsList/components/SidebarItemCard.tsxindex a438568b74..438fc4798d 100644--- a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/sidebar/SidebarRunsList/components/SidebarItemCard.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/sidebar/SidebarRunsList/components/SidebarItemCard.tsx@@ -12,6 +12,7 @@ interface Props {selected?: boolean;onClick?: () => void;actions?: React.ReactNode;+ testID?: string;}export function SidebarItemCard({@@ -22,6 +23,7 @@ export function SidebarItemCard({selected,onClick,actions,+ testID,}: Props) {return (<div@@ -30,6 +32,7 @@ export function SidebarItemCard({selected ? "border-slate-800 ring-slate-800" : undefined,)}onClick={onClick}+ data-testid={testID}><div className="flex min-w-0 items-center justify-start gap-3">{icon}diff --git a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/sidebar/SidebarRunsList/components/TaskListItem.tsx b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/sidebar/SidebarRunsList/components/TaskListItem.tsxindex 14999ad342..3d59e80b5d 100644--- a/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/sidebar/SidebarRunsList/components/TaskListItem.tsx+++ b/autogpt_platform/frontend/src/app/(platform)/library/agents/[id]/components/NewAgentLibraryView/components/sidebar/SidebarRunsList/components/TaskListItem.tsx@@ -96,6 +96,7 @@ export function TaskListItem({}onClick={onClick}selected={selected}+ testID={`library-run-item-${run.id}`}actions={<TaskActionsDropdown agent={agent} run={run} onDeleted={onDeleted} />}diff --git a/autogpt_platform/frontend/src/playwright/journeys/README.md b/autogpt_platform/frontend/src/playwright/journeys/README.mdnew file mode 100644index 0000000000..50e662ed44--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/README.md@@ -0,0 +1,160 @@+# Journeys++A journey is a Playwright spec that walks one real user flow end to end and+asserts what must be true afterwards: the run reached COMPLETED, the Library+shows it, the balance moved by exactly the cost. Smoke specs+(`*-happy-path.spec.ts`) check that pages work; journeys check that the+product does what the user expects.++## Conventions++- **File name:** `src/playwright/journeys/<area>-journey.spec.ts`. Only files+ matching `*-journey.spec.ts` run, and only in the journeys config.+- **Import `test` and `expect` from `./fixtures`**, never from+ `@playwright/test` or `../coverage-fixture`. That gives every journey:+ - **The guard.** The test fails if the browser sees an API response of 400+ or more (`/api/...`), an uncaught page error, or a `console.error`. The+ failure lists each one with its URL, status and the start of the body.+ A journey that provokes an error on purpose declares it:+ ```ts+ test.use({+ expectedResponses: [{ status: 401, url: "/api/proxy/api/auth/user" }],+ });+ ```+ App-wide noise that is reviewed and harmless goes in `KNOWN_BENIGN` in+ `guard.ts`, with its reason. A real bug never goes there: if the guard+ catches one, that's a finding. Report it, or mark the journey+ `test.fail()` with a link to the issue.+ - **Hosted flags.** The frontend serves the values in `hosted-flags.json`+ (see below). Change single flags per journey with+ `test.use({ flagOverrides: { "<flag>": value } })`; the rest keep their+ hosted values.+ - **`journeyUser`.** Ask for it to get a brand-new user who is signed in and+ has finished onboarding. With credits on, finishing onboarding grants the+ onboarding credits. With `PLAYWRIGHT_JOURNEY_TIER` set (CI: `PRO`), the+ user also gets that plan without Stripe (`subscription-tier.ts` writes the+ tier to the stack's database: a paid tier with no Stripe customer, the+ same shape as an admin grant), so the hosted paywall lets them run agents.+ A journey never shares a user with another one and never depends on CI's+ seeded accounts.+- **Make your own data.** Build or import the agents you need inside the+ journey. Name things uniquely (`Date.now()` suffix).+- **Assert invariants, not DOM trivia.** Use web-first assertions+ (`toBeVisible`, `expect.poll`); no fixed sleeps. When the UI and the API+ should agree, check both.+- **Keep each journey under ~60 s.** The test timeout is 90 s.+- **Reuse page objects** from `../pages/` and helpers from `../utils/`. Put+ helpers only journeys need (credits, users) in this folder.+- **Costs:** read them from the ledger (`credits.ts`), not by diffing the+ balance alone. Onboarding rewards (e.g. +100 for a first Library run) land+ mid-journey.++## Run locally++Start a full stack in the journeys configuration, then point Playwright at it.+Mirror CI's backend settings:++```bash+cd autogpt_platform+make init-env+python ../.github/scripts/hosted_journeys_env.py >> backend/.env+echo "CHAT_TEST_MODE=true" >> backend/.env+NEXT_PUBLIC_PW_TEST=true docker compose up -d --build+cd frontend+PLAYWRIGHT_JOURNEY_COST_PER_NODE=1 PLAYWRIGHT_JOURNEY_TIER=PRO \+ pnpm exec playwright test --config playwright.journeys.config.ts --project hosted+# one journey, headed:+pnpm exec playwright test --config playwright.journeys.config.ts --project hosted run-correctness --headed+```++The frontend must be built with `NEXT_PUBLIC_PW_TEST=true`; only that build+reads the hosted flag values the fixtures hand it.+`PLAYWRIGHT_JOURNEY_COST_PER_NODE` tells cost assertions what the stack bills+per node execution: `1` with `EXECUTION_COST_COUNT_THRESHOLD=1`, otherwise+leave it unset for `0`.++The tier grant uses `DATABASE_URL` from `frontend/.env`; if your stack's+Postgres isn't on `localhost:5432`, set `PLAYWRIGHT_JOURNEY_DATABASE_URL`.++### Projects: `hosted` and `selfhost`++`selfhost-journey.spec.ts` asserts that nothing hosted-only exists (no+billing, paywall or third-party hosts), so it fails by design on the hosted+configuration. The config runs it only in the `selfhost` project, against a+self-hosted build (see the spec's header); every other journey is in+`hosted`. CI's journeys leg runs `--project hosted`.++**TODO:** a `journeys-selfhost` CI leg (compose stack with hosted flags off,+no `CHAT_TEST_MODE`, since the spec expects "No AI connection") would run it+on every PR. It's not added yet: it's a third runner per run, and its config+needs settling first.++## Run against a preview (`!deploy`)++```bash+PLAYWRIGHT_BASE_URL=https://autogpt-pr-<N>.vercel.app \+ pnpm exec playwright test --config playwright.journeys.config.ts --project hosted+```++- Journeys present a normal Chrome user agent (`user-agent.ts`; override with+ `PLAYWRIGHT_USER_AGENT`). PostHog's edge returns no flags at all to a+ `HeadlessChrome` user agent, so with the default headless one, dev and+ previews would serve every flag off: no paywall, no experts, old layout.+ A context you create with `browser.newContext()` doesn't inherit the+ config, so pass it `userAgent: JOURNEY_USER_AGENT` yourself.+- Users sign up as `journey-<id>@previews.agpt.co`, which the preview+ allowlist accepts. Override the domain with `PLAYWRIGHT_JOURNEY_EMAIL_DOMAIN`.+- Previews serve real PostHog flags, so `enable-platform-payment` is on there+ and a fresh user is paywalled: running an agent gets a 402. For journeys+ that run agents, sign in as a paid account instead with+ `PLAYWRIGHT_JOURNEY_EMAIL` and `PLAYWRIGHT_JOURNEY_PASSWORD`. Never put a+ real password in a log, commit or PR.+- Sign-ups are capped at 5 per client IP per 10 minutes, so a full run from+ one machine can hit `429`. Reuse an account as above, or run fewer+ journeys at once.++## In CI++Job `end-to-end tests (journeys)` in `.github/workflows/platform-fullstack-ci.yml`+is a matrix leg beside the smoke suite, on its own runner, so it adds no wall+clock. It's **advisory** for now (`ADVISORY_CHECK_RUNS` in+`.github/workflows/scripts/check_actions_status.py`): a failure shows on the+PR but doesn't block merging. It becomes blocking once it has proven stable.++- Raise `min_tests` for the journeys leg in the workflow matrix when you add+ tests; `.github/scripts/test_e2e_seed_cache_contract.py` pins it.+- Reports and traces: artifacts `playwright-report-journeys` and+ `playwright-test-results-journeys`.+- Check a new journey for flakiness before relying on it: dispatch the+ workflow on your branch with `journeys_repeat_each`, e.g.+ `gh workflow run platform-fullstack-ci.yml --ref <branch> -f journeys_repeat_each=10`.+- Keep the leg's Playwright step under ~200 s at 4 workers. Past that, shard+ it into more legs with `--shard=i/N` rather than lengthening it.++## Hosted vs self-hosted configuration++The journeys leg runs as close to the hosted product as one compose stack+allows. Flag values live in `hosted-flags.json`, read by both sides:++- **Backend:** `.github/scripts/hosted_journeys_env.py` turns them into+ `FORCE_FLAG_<NAME>` lines in `backend/.env`, plus `ENABLE_CREDIT=true`.+- **Frontend:** the fixtures write them to local storage before the app+ loads, and a `NEXT_PUBLIC_PW_TEST=true` build serves them+ (`services/feature-flags/pw-flag-overrides.ts`). That lets both legs share+ one cached frontend image; build-time `NEXT_PUBLIC_FORCE_FLAG_*` would mean+ a second, uncached Next.js build on every run.++Differences from hosted that are still open:++| Area | Journeys leg | Hosted |+| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ |+| Payments | `enable-platform-payment` **on**; journey users get a paid tier without Stripe, so checkout, trials and Stripe webhooks go untested | On, with Stripe: NO_TIER users get a 402 until they subscribe or start a trial |+| Node execution fee | 1 credit per node (`EXECUTION_COST_COUNT_THRESHOLD=1`), so a run's cost is visible | 1 credit per 100 node executions |+| Copilot | `CHAT_TEST_MODE` dummy reply; no real turn pipeline, tools or cost | Real models |+| Flag source | Fixed values from `hosted-flags.json`; JSON flags (model routing, tier limits) at code defaults | PostHog/LaunchDarkly with targeting |+| `BEHAVE_AS` | `LOCAL` on both sides; the frontend's is build time | `CLOUD` |+| Pods | One `executor`, one `copilot_executor` | Many. Scaling needs the fixed host ports 8002/8008 removed first |+| Frontend runtime | Next.js standalone server in Docker | Vercel (body size limits, edge routing) |+| External services | No Stripe, OAuth providers, Postmark or E2B; ClamAV, Redis, RabbitMQ and FalkorDB are real | Real |+| LLM keys | None; blocks that need platform credentials fail | Platform keys |+| Origin | `http://localhost:3000` | HTTPS |diff --git a/autogpt_platform/frontend/src/playwright/journeys/credits.ts b/autogpt_platform/frontend/src/playwright/journeys/credits.tsnew file mode 100644index 0000000000..3c1852b7e8--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/credits.ts@@ -0,0 +1,68 @@+import { expect, type Page } from "@playwright/test";++interface CreditTransaction {+ transaction_key: string;+ transaction_type: string;+ amount: number;+ usage_execution_id: string | null;+}++export interface CreditSnapshot {+ balance: number;+ transactionKeys: Set<string>;+}++export async function getCreditBalance(page: Page) {+ const response = await page.request.get("/api/proxy/api/credits");+ expect(response.ok(), `GET credits: ${response.status()}`).toBe(true);+ const body = (await response.json()) as { credits: number };+ return body.credits;+}++export async function getCreditTransactions(page: Page) {+ const response = await page.request.get(+ "/api/proxy/api/credits/transactions",+ { params: { transaction_count_limit: 100 } },+ );+ expect(response.ok(), `GET credit transactions: ${response.status()}`).toBe(+ true,+ );+ const body = (await response.json()) as {+ transactions: CreditTransaction[];+ };+ return body.transactions;+}++export async function snapshotCredits(page: Page): Promise<CreditSnapshot> {+ const [balance, transactions] = await Promise.all([+ getCreditBalance(page),+ getCreditTransactions(page),+ ]);+ return {+ balance,+ transactionKeys: new Set(transactions.map((t) => t.transaction_key)),+ };+}++/**+ * What changed in the ledger since `before`: the new balance, every new+ * transaction, and how much of it was charged to `executionID`.+ */+export async function creditsSince(+ page: Page,+ before: CreditSnapshot,+ executionID: string,+) {+ const [balance, transactions] = await Promise.all([+ getCreditBalance(page),+ getCreditTransactions(page),+ ]);+ const added = transactions.filter(+ (t) => !before.transactionKeys.has(t.transaction_key),+ );+ const chargedToRun = added+ .filter((t) => t.usage_execution_id === executionID)+ .reduce((sum, t) => sum - t.amount, 0);+ const netChange = added.reduce((sum, t) => sum + t.amount, 0);+ return { balance, added, chargedToRun, netChange };+}diff --git a/autogpt_platform/frontend/src/playwright/journeys/fixtures.ts b/autogpt_platform/frontend/src/playwright/journeys/fixtures.tsnew file mode 100644index 0000000000..c039018d46--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/fixtures.ts@@ -0,0 +1,59 @@+import { PW_FLAG_OVERRIDES_STORAGE_KEY } from "../../services/feature-flags/pw-flag-overrides";+import { expect, test as base } from "../coverage-fixture";+import { type ExpectedResponse, watchForUnexpectedErrors } from "./guard";+import hostedFlags from "./hosted-flags.json";+import { type JourneyUser, provisionJourneyUser } from "./journey-user";++interface JourneyOptions {+ /** API responses of 400 or more this journey provokes on purpose. */+ expectedResponses: ExpectedResponse[];+ /** Flag values to change from the hosted defaults; the rest are kept. */+ flagOverrides: Record<string, unknown>;+}++interface JourneyFixtures {+ journeyGuard: void;+ /** A fresh, signed-in user who has finished onboarding. */+ journeyUser: JourneyUser;+}++export const test = base.extend<JourneyOptions & JourneyFixtures>({+ expectedResponses: [[], { option: true }],+ flagOverrides: [{}, { option: true }],++ journeyGuard: [+ async ({ context, expectedResponses, flagOverrides }, use) => {+ await context.addInitScript(+ ({ key, value }) => {+ try {+ window.localStorage.setItem(key, value);+ } catch {+ // Opaque origins (about:blank, data:) have no storage.+ }+ },+ {+ key: PW_FLAG_OVERRIDES_STORAGE_KEY,+ value: JSON.stringify({ ...hostedFlags, ...flagOverrides }),+ },+ );+ const guard = watchForUnexpectedErrors(context, expectedResponses);++ await use();++ const problems = await guard.collect();+ expect(+ problems,+ `the journey hit ${problems.length} unexpected error(s). Declare ` ++ "deliberate ones with test.use({ expectedResponses }):\n\n" ++ problems.join("\n\n"),+ ).toEqual([]);+ },+ { auto: true },+ ],++ journeyUser: async ({ page }, provide) => {+ await provide(await provisionJourneyUser(page));+ },+});++export { expect };diff --git a/autogpt_platform/frontend/src/playwright/journeys/guard.ts b/autogpt_platform/frontend/src/playwright/journeys/guard.tsnew file mode 100644index 0000000000..15b6267162--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/guard.ts@@ -0,0 +1,131 @@+import type {+ BrowserContext,+ ConsoleMessage,+ Response,+} from "@playwright/test";++/**+ * A response a journey provokes on purpose, e.g. the 401 a signed-out+ * session gets. `url` matches as a substring, or as a RegExp.+ */+export interface ExpectedResponse {+ status: number;+ url: string | RegExp;+ method?: string;+}++interface KnownNoise {+ reason: string;+ kind: "response" | "console" | "pageerror";+ status?: number;+ url?: RegExp;+ message?: RegExp;+}++// Reviewed, app-wide noise that is NOT a bug. Every entry needs a reason a+// reviewer can check. A real defect never goes here: fix it, or mark the+// journey with test.fail() and a link to the issue.+const KNOWN_BENIGN: KnownNoise[] = [+ {+ kind: "console",+ message: /^Failed to load resource: /,+ reason:+ "Chrome's echo of a failed request. Failed API calls are reported " ++ "by the response check, with their URL and body.",+ },+];++const BODY_SNIPPET_LENGTH = 300;++/**+ * Watches every page in a journey's browser context and collects what a user+ * would never want to see: a 4xx/5xx from the API, an uncaught page error, or+ * a console.error. The journey fails at teardown if anything was collected.+ */+export function watchForUnexpectedErrors(+ context: BrowserContext,+ expected: ExpectedResponse[],+) {+ const pending: Promise<string>[] = [];++ context.on("response", (response) => {+ if (!isApiResponse(response) || response.status() < 400) return;+ if (isExpected(response, expected)) return;+ if (isKnownNoise({ kind: "response", response })) return;+ pending.push(describeResponse(response));+ });++ context.on("console", (message) => {+ if (message.type() !== "error") return;+ if (isKnownNoise({ kind: "console", text: message.text() })) return;+ pending.push(Promise.resolve(describeConsole(message)));+ });++ context.on("weberror", (webError) => {+ const error = webError.error();+ if (isKnownNoise({ kind: "pageerror", text: error.message })) return;+ pending.push(+ Promise.resolve(+ `pageerror on ${webError.page()?.url() ?? "unknown page"}: ` ++ `${error.message}\n${error.stack ?? ""}`.trim(),+ ),+ );+ });++ return {+ async collect() {+ return Promise.all(pending);+ },+ };+}++function isApiResponse(response: Response) {+ const { pathname } = new URL(response.url());+ return pathname.startsWith("/api/");+}++function isExpected(response: Response, expected: ExpectedResponse[]) {+ const method = response.request().method();+ return expected.some(+ (entry) =>+ entry.status === response.status() &&+ (!entry.method || entry.method.toUpperCase() === method) &&+ (typeof entry.url === "string"+ ? response.url().includes(entry.url)+ : entry.url.test(response.url())),+ );+}++type NoiseCandidate =+ | { kind: "response"; response: Response }+ | { kind: "console" | "pageerror"; text: string };++function isKnownNoise(candidate: NoiseCandidate) {+ return KNOWN_BENIGN.some((noise) => {+ if (noise.kind !== candidate.kind) return false;+ if (candidate.kind === "response") {+ return (+ (noise.status === undefined ||+ noise.status === candidate.response.status()) &&+ (!noise.url || noise.url.test(candidate.response.url()))+ );+ }+ return !noise.message || noise.message.test(candidate.text);+ });+}++async function describeResponse(response: Response) {+ const body = await response+ .text()+ .then((text) => text.slice(0, BODY_SNIPPET_LENGTH))+ .catch(() => "<body unavailable>");+ return (+ `${response.status()} ${response.request().method()} ${response.url()}\n` ++ ` body: ${body}`+ );+}++function describeConsole(message: ConsoleMessage) {+ const { url, lineNumber } = message.location();+ return `console.error at ${url}:${lineNumber}: ${message.text()}`;+}diff --git a/autogpt_platform/frontend/src/playwright/journeys/hosted-flags.json b/autogpt_platform/frontend/src/playwright/journeys/hosted-flags.jsonnew file mode 100644index 0000000000..5fc673adea--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/hosted-flags.json@@ -0,0 +1,12 @@+{+ "hire-experts": true,+ "autogpt-new-layout": true,+ "onboarding-brain-dump": true,+ "onboarding-expert-team": true,+ "artifacts-page": true,+ "chat-search": true,+ "skills-hub": true,+ "graphiti-memory": true,+ "expert-spend-approval": true,+ "enable-platform-payment": true+}diff --git a/autogpt_platform/frontend/src/playwright/journeys/journey-user.ts b/autogpt_platform/frontend/src/playwright/journeys/journey-user.tsnew file mode 100644index 0000000000..d027785c4c--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/journey-user.ts@@ -0,0 +1,139 @@+import { randomBytes, randomInt } from "crypto";+import { expect, type Page } from "@playwright/test";+import { grantSubscriptionTier } from "./subscription-tier";++export interface JourneyUser {+ email: string;+ password: string;+ name: string;+ id?: string;+}++// Onboarding steps a journey user has already done. ONBOARDING_COMPLETE also+// grants the onboarding credits, which is how a fresh user gets a balance+// when credits are on; the rest dismiss first-visit cards that would cover+// the page.+const COMPLETED_ONBOARDING_STEPS = [+ "ONBOARDING_COMPLETE",+ "AGENT_NEW_RUN",+ "CAPABILITY_CARDS",+ "AGENTS_TAB_INTRO",+ "MARKETPLACE_TAB_INTRO",+ "BUILD_TAB_INTRO",+ "WORKFLOWS_MOVED",+];++/**+ * Signs a user into the page's browser context, ready to use the app.+ *+ * By default this signs up a brand-new user, so a journey never shares state+ * with another one and never depends on CI-only seed data. Set+ * PLAYWRIGHT_JOURNEY_EMAIL and PLAYWRIGHT_JOURNEY_PASSWORD to sign in as an+ * existing account instead, e.g. a paid user on a preview where fresh users+ * meet the paywall.+ */+export async function provisionJourneyUser(page: Page): Promise<JourneyUser> {+ const existingEmail = process.env.PLAYWRIGHT_JOURNEY_EMAIL;+ const existingPassword = process.env.PLAYWRIGHT_JOURNEY_PASSWORD;+ const user =+ existingEmail && existingPassword+ ? await signIn(page, {+ email: existingEmail,+ password: existingPassword,+ name: existingEmail.split("@")[0],+ })+ : await signUp(page, newJourneyUser());+ // Before anything reads the tier: the backend caches it for minutes.+ if (user.id) await grantSubscriptionTier(user.id);+ await finishAccountSetup(page);+ await completeOnboarding(page);+ return user;+}++function newJourneyUser(): JourneyUser {+ const id = `${Date.now()}-${randomBytes(3).toString("hex")}`;+ return {+ email: `journey-${id}@${journeyEmailDomain()}`,+ password: `Journey-${randomBytes(12).toString("hex")}`,+ name: `Journey ${id}`,+ };+}++// Previews only accept sign-ups from allowlisted domains; @previews.agpt.co+// keeps password sign-up and needs no inbox.+function journeyEmailDomain() {+ if (process.env.PLAYWRIGHT_JOURNEY_EMAIL_DOMAIN) {+ return process.env.PLAYWRIGHT_JOURNEY_EMAIL_DOMAIN;+ }+ return isLoopback(appOrigin()) ? "example.com" : "previews.agpt.co";+}++async function signUp(page: Page, user: JourneyUser) {+ const response = await page.request.post("/api/auth/sign-up/email", {+ headers: authHeaders(),+ data: user,+ });+ expect(+ response.ok(),+ `sign-up for ${user.email} failed: ${response.status()} ${await response.text()}`,+ ).toBe(true);+ const body = (await response.json()) as { user?: { id?: string } };+ return { ...user, id: body.user?.id };+}++async function signIn(page: Page, user: JourneyUser) {+ const response = await page.request.post("/api/auth/sign-in/email", {+ headers: authHeaders(),+ data: { email: user.email, password: user.password },+ });+ expect(+ response.ok(),+ `sign-in for ${user.email} failed: ${response.status()}`,+ ).toBe(true);+ const body = (await response.json()) as { user?: { id?: string } };+ return { ...user, id: body.user?.id };+}++// The app does this on a user's first page load; among other things it+// creates their marketplace profile. Done here so a journey doesn't race it.+async function finishAccountSetup(page: Page) {+ const response = await page.request.post("/api/proxy/api/auth/user");+ expect(+ response.ok(),+ `account setup failed: ${response.status()} ${await response.text()}`,+ ).toBe(true);+}++async function completeOnboarding(page: Page) {+ for (const step of COMPLETED_ONBOARDING_STEPS) {+ const response = await page.request.post("/api/proxy/api/onboarding/step", {+ params: { step },+ });+ expect(+ response.ok(),+ `onboarding step ${step} failed: ${response.status()} ${await response.text()}`,+ ).toBe(true);+ }+}++// The app caps sign-ups per client IP. On a local or CI stack every journey+// comes from one address, so each sign-up claims its own; a deployed app+// behind a proxy sets the real address and the cap applies as usual.+function authHeaders(): Record<string, string> {+ const origin = appOrigin();+ if (!isLoopback(origin)) return { Origin: origin };+ return {+ Origin: origin,+ "X-Forwarded-For": `10.${randomInt(256)}.${randomInt(256)}.${randomInt(1, 255)}`,+ };+}++function appOrigin() {+ return new URL(process.env.PLAYWRIGHT_BASE_URL ?? "http://localhost:3000")+ .origin;+}++function isLoopback(origin: string) {+ const { hostname } = new URL(origin);+ return ["localhost", "127.0.0.1", "[::1]"].includes(hostname);+}diff --git a/autogpt_platform/frontend/src/playwright/journeys/run-correctness-journey.spec.ts b/autogpt_platform/frontend/src/playwright/journeys/run-correctness-journey.spec.tsnew file mode 100644index 0000000000..7bae0da08a--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/run-correctness-journey.spec.ts@@ -0,0 +1,226 @@+import fs from "fs";+import type { Page, TestInfo } from "@playwright/test";+// importAgentFromFile loads this lazily, which only works once it's loaded.+import "../pages/build.page";+import {+ assertRunOutputValue,+ clickRunButton,+ getActiveItemId,+ importAgentFromFile,+ waitForAgentPageLoad,+} from "../pages/library.page";+import { creditsSince, snapshotCredits } from "./credits";+import { expect, test } from "./fixtures";++const CALCULATOR_BLOCK_ID = "b1ab9b19-67a6-406d-abf5-2dba76d00c79";+const MATCH_TEXT_PATTERN_BLOCK_ID = "3060088f-6ed9-4928-9ba7-9c92823a7ccd";+const AGENT_OUTPUT_BLOCK_ID = "363ae599-353e-4804-937e-b2ee3cef3da4";++// Credits charged per node execution. CI's journeys leg bills every node so a+// run's cost is visible; a hosted stack bills one credit per 100 executions,+// which a fresh user never reaches.+const COST_PER_NODE = Number(process.env.PLAYWRIGHT_JOURNEY_COST_PER_NODE ?? 0);++const TERMINAL_STATUS = /^(COMPLETED|FAILED|TERMINATED)$/;++test.describe("run correctness journey", () => {+ test("a run completes, shows its output in the run view and the Library, and costs exactly its block cost", async ({+ page,+ journeyUser: _user,+ }, testInfo) => {+ const agentName = uniqueName("Journey multiply");+ const outputName = "answer";+ const agent = await importAgent(page, testInfo, {+ name: agentName,+ nodes: [+ node("multiply", CALCULATOR_BLOCK_ID, {+ operation: "Multiply",+ a: 6,+ b: 7,+ round_result: false,+ }),+ node("output", AGENT_OUTPUT_BLOCK_ID, { name: outputName }),+ ],+ links: [link("multiply", "result", "output", "value")],+ });+ const before = await snapshotCredits(page);++ const runID = await startRunFromLibrary(page, agent.url);++ await expect+ .poll(() => getRunStatus(page, agent.graphID, runID), {+ message: "the run should reach COMPLETED",+ timeout: 30_000,+ })+ .toBe("COMPLETED");+ await assertRunOutputValue(page, outputName, "42");++ await page.goto(agent.url);+ await waitForAgentPageLoad(page, agentName);+ await expect(+ runListItem(page, runID),+ "the Library run list should show the run",+ ).toBeVisible();++ await expect+ .poll(+ async () => (await creditsSince(page, before, runID)).chargedToRun,+ {+ message: "the run should be charged its block cost",+ },+ )+ .toBe(COST_PER_NODE * 2);+ const ledger = await creditsSince(page, before, runID);+ expect(+ ledger.balance - before.balance,+ `the balance should move by exactly the new ledger entries: ${JSON.stringify(ledger.added)}`,+ ).toBe(ledger.netChange);+ });++ test("a run whose node fails ends instead of hanging", async ({+ page,+ journeyUser: _user,+ }, testInfo) => {+ const { agent, runID } = await runAgentWithFailingNode(page, testInfo);++ await expect+ .poll(() => getRunStatus(page, agent.graphID, runID), {+ message: "the run should end, not stay QUEUED or RUNNING",+ timeout: 30_000,+ })+ .toMatch(TERMINAL_STATUS);+ });++ test("a run whose node fails is marked FAILED and shows the user the error", async ({+ page,+ journeyUser: _user,+ }, testInfo) => {+ // Today a node error with nothing wired to its error output is non-fatal+ // (backend/executor/manager.py): the run reports COMPLETED and the run+ // view says "No output from this run." with no sign of the error. Whether+ // the run should fail is the open product question of #14275 and+ // SECRT-2840; either way the user needs to see the error. Remove this+ // marker when both hold.+ test.fail(+ true,+ "https://github.com/Significant-Gravitas/AutoGPT/issues/14275",+ );+ const { agent, runID } = await runAgentWithFailingNode(page, testInfo);++ await expect+ .poll(() => getRunStatus(page, agent.graphID, runID), {+ message: "the run should end as FAILED",+ timeout: 30_000,+ })+ .toBe("FAILED");+ await expect(+ page.getByText(/Invalid regex pattern/).first(),+ "the node's error should be shown in the run view",+ ).toBeVisible();+ });+});++async function runAgentWithFailingNode(page: Page, testInfo: TestInfo) {+ const agent = await importAgent(page, testInfo, {+ name: uniqueName("Journey bad pattern"),+ nodes: [+ node("match", MATCH_TEXT_PATTERN_BLOCK_ID, {+ text: "journey",+ match: "(",+ data: "matched",+ }),+ node("output", AGENT_OUTPUT_BLOCK_ID, { name: "matched" }),+ ],+ links: [link("match", "positive", "output", "value")],+ });+ const runID = await startRunFromLibrary(page, agent.url);+ return { agent, runID };+}++interface GraphSpec {+ name: string;+ nodes: ReturnType<typeof node>[];+ links: ReturnType<typeof link>[];+}++function node(id: string, blockID: string, input: Record<string, unknown>) {+ return {+ id,+ block_id: blockID,+ input_default: input,+ metadata: { position: { x: 0, y: 0 } },+ input_links: [],+ output_links: [],+ };+}++function link(source: string, output: string, sink: string, input: string) {+ return {+ source_id: source,+ source_name: output,+ sink_id: sink,+ sink_name: input,+ };+}++function uniqueName(prefix: string) {+ return `${prefix} ${Date.now()}`;+}++async function importAgent(page: Page, testInfo: TestInfo, graph: GraphSpec) {+ const nodes = graph.nodes.map((graphNode, index) => ({+ ...graphNode,+ metadata: { position: { x: 120 + index * 400, y: 160 } },+ }));+ const filePath = testInfo.outputPath(`${graph.nodes[0].id}-agent.json`);+ fs.writeFileSync(+ filePath,+ JSON.stringify({+ name: graph.name,+ description: "Built by the run correctness journey",+ is_active: true,+ nodes,+ links: graph.links,+ }),+ );+ await importAgentFromFile(page, filePath, graph.name);+ return findLibraryAgent(page, graph.name);+}++async function findLibraryAgent(page: Page, name: string) {+ const response = await page.request.get("/api/proxy/api/library/agents", {+ params: { search_term: name },+ });+ expect(response.ok(), `GET library agents: ${response.status()}`).toBe(true);+ const body = (await response.json()) as {+ agents: { id: string; graph_id: string; name: string }[];+ };+ const agent = body.agents.find((candidate) => candidate.name === name);+ if (!agent) throw new Error(`library agent "${name}" should exist`);+ return { graphID: agent.graph_id, url: `/library/agents/${agent.id}` };+}++async function startRunFromLibrary(page: Page, agentURL: string) {+ await page.goto(agentURL);+ await waitForAgentPageLoad(page);+ await clickRunButton(page);+ await expect+ .poll(() => getActiveItemId(page), {+ message: "starting a run should open it in the run view",+ })+ .toBeTruthy();+ return String(getActiveItemId(page));+}++async function getRunStatus(page: Page, graphID: string, runID: string) {+ const response = await page.request.get(+ `/api/proxy/api/graphs/${graphID}/executions/${runID}`,+ );+ expect(response.ok(), `GET execution: ${response.status()}`).toBe(true);+ const body = (await response.json()) as { status: string };+ return body.status;+}++function runListItem(page: Page, runID: string) {+ return page.getByTestId(`library-run-item-${runID}`);+}diff --git a/autogpt_platform/frontend/src/playwright/journeys/schedules-journey.spec.ts b/autogpt_platform/frontend/src/playwright/journeys/schedules-journey.spec.tsindex 6e45ffb03f..d6b49404cc 100644--- a/autogpt_platform/frontend/src/playwright/journeys/schedules-journey.spec.ts+++ b/autogpt_platform/frontend/src/playwright/journeys/schedules-journey.spec.ts@@ -1,13 +1,12 @@-import { randomBytes } from "crypto";import type { Page } from "@playwright/test";-import { expect, test } from "../coverage-fixture";import { createUniqueAgentName } from "../pages/build.page";import { LibraryPage } from "../pages/library.page";+import { expect, test } from "./fixtures";const TIMEZONE = "America/New_York";const STORE_VALUE_BLOCK_ID = "1ff065e9-88e8-4358-9d82-8dc91f622ba9";-test.use({ storageState: { cookies: [], origins: [] }, timezoneId: TIMEZONE });+test.use({ timezoneId: TIMEZONE });interface Schedule {id: string;@@ -19,9 +18,9 @@ interface Schedule {test.describe("schedules journey", () => {test("a weekday schedule can be created, edited, and deleted from the Library, and every change survives a reload", async ({page,+ journeyUser: _user,}) => {- test.setTimeout(90_000);- await signUpJourneyUser(page);+ await setTimezone(page);const agentName = createUniqueAgentName("Journey schedule");const graphId = await createAgent(page, agentName);@@ -123,53 +122,12 @@ function expectWeekdayAt(schedule: Schedule, hour: number, minute: number) {expect([Number(parts.hour), Number(parts.minute)]).toEqual([hour, minute]);}-// Onboarding steps that dismiss the first-visit cards covering the Library.-const COMPLETED_ONBOARDING_STEPS = [- "ONBOARDING_COMPLETE",- "AGENT_NEW_RUN",- "CAPABILITY_CARDS",- "AGENTS_TAB_INTRO",- "MARKETPLACE_TAB_INTRO",- "BUILD_TAB_INTRO",- "WORKFLOWS_MOVED",-];--// A brand-new user by default. Previews paywall fresh users, so there-// PLAYWRIGHT_JOURNEY_EMAIL and PLAYWRIGHT_JOURNEY_PASSWORD sign in as an-// existing subscribed account instead.-async function signUpJourneyUser(page: Page) {- const email = process.env.PLAYWRIGHT_JOURNEY_EMAIL;- const password = process.env.PLAYWRIGHT_JOURNEY_PASSWORD;- if (email && password) {- const signIn = await page.request.post("/api/auth/sign-in/email", {- headers: { Origin: appOrigin() },- data: { email, password },- });- expect(signIn.ok(), `sign-in failed: ${signIn.status()}`).toBe(true);- } else {- const id = `${Date.now()}-${randomBytes(3).toString("hex")}`;- const signUp = await page.request.post("/api/auth/sign-up/email", {- headers: { Origin: appOrigin() },- data: {- email: `journey-${id}@${journeyEmailDomain()}`,- password: `Journey-${randomBytes(12).toString("hex")}`,- name: `Journey ${id}`,- },- });- expect(signUp.ok(), `sign-up failed: ${signUp.status()}`).toBe(true);- }- for (const step of COMPLETED_ONBOARDING_STEPS) {- const onboarding = await page.request.post(- "/api/proxy/api/onboarding/step",- { params: { step } },- );- expect(onboarding.ok(), `onboarding step ${step} failed`).toBe(true);- }- const timezone = await page.request.post(+async function setTimezone(page: Page) {+ const response = await page.request.post("/api/proxy/api/auth/user/timezone",{ data: { timezone: TIMEZONE } },);- expect(timezone.ok(), "setting the user's timezone failed").toBe(true);+ expect(response.ok(), "setting the user's timezone failed").toBe(true);}async function createAgent(page: Page, name: string): Promise<string> {@@ -199,20 +157,3 @@ async function createAgent(page: Page, name: string): Promise<string> {);return (await response.json()).id;}--function appOrigin() {- return new URL(process.env.PLAYWRIGHT_BASE_URL ?? "http://localhost:3000")- .origin;-}--// Previews only accept sign-ups from allowlisted domains; @previews.agpt.co-// keeps password sign-up and needs no inbox.-function journeyEmailDomain() {- if (process.env.PLAYWRIGHT_JOURNEY_EMAIL_DOMAIN) {- return process.env.PLAYWRIGHT_JOURNEY_EMAIL_DOMAIN;- }- const { hostname } = new URL(appOrigin());- return ["localhost", "127.0.0.1", "[::1]"].includes(hostname)- ? "example.com"- : "previews.agpt.co";-}diff --git a/autogpt_platform/frontend/src/playwright/journeys/subscription-tier.ts b/autogpt_platform/frontend/src/playwright/journeys/subscription-tier.tsnew file mode 100644index 0000000000..8f729788bd--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/subscription-tier.ts@@ -0,0 +1,36 @@+import { Client } from "pg";++/**+ * Gives a journey user a paid plan without Stripe, so the hosted paywall lets+ * them run agents. A paid tier with no Stripe customer is how the platform+ * models an admin grant, which the Stripe reconciliation sweep never revokes.+ *+ * Opt in with PLAYWRIGHT_JOURNEY_TIER (e.g. PRO). It needs the stack's+ * database: PLAYWRIGHT_JOURNEY_DATABASE_URL, else DATABASE_URL from+ * frontend/.env. Unset on a preview, where an existing paid account+ * (PLAYWRIGHT_JOURNEY_EMAIL) is the way past the paywall.+ */+export async function grantSubscriptionTier(userID: string) {+ const tier = process.env.PLAYWRIGHT_JOURNEY_TIER;+ if (!tier) return;+ const connectionString =+ process.env.PLAYWRIGHT_JOURNEY_DATABASE_URL ?? process.env.DATABASE_URL;+ if (!connectionString) {+ throw new Error(+ "PLAYWRIGHT_JOURNEY_TIER needs PLAYWRIGHT_JOURNEY_DATABASE_URL or DATABASE_URL",+ );+ }+ const client = new Client({ connectionString });+ await client.connect();+ try {+ const result = await client.query(+ `UPDATE platform."User" SET "subscriptionTier" = $1::platform."SubscriptionTier" WHERE id = $2`,+ [tier, userID],+ );+ if (result.rowCount !== 1) {+ throw new Error(`no platform user ${userID} to grant ${tier} to`);+ }+ } finally {+ await client.end();+ }+}diff --git a/autogpt_platform/frontend/src/playwright/journeys/user-agent.ts b/autogpt_platform/frontend/src/playwright/journeys/user-agent.tsnew file mode 100644index 0000000000..f48f1a6467--- /dev/null+++ b/autogpt_platform/frontend/src/playwright/journeys/user-agent.ts@@ -0,0 +1,7 @@+// PostHog's edge answers any User-Agent containing "HeadlessChrome" with no+// flags at all, so against dev or a preview a headless browser would see every+// flag off. Journeys present a normal Chrome instead. A context made by hand+// with browser.newContext() doesn't inherit the config; pass this to it.+export const JOURNEY_USER_AGENT =+ process.env.PLAYWRIGHT_USER_AGENT ??+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36";diff --git a/autogpt_platform/frontend/src/playwright/pages/library.page.ts b/autogpt_platform/frontend/src/playwright/pages/library.page.tsindex 740aa976f4..a91e033e99 100644--- a/autogpt_platform/frontend/src/playwright/pages/library.page.ts+++ b/autogpt_platform/frontend/src/playwright/pages/library.page.ts@@ -146,14 +146,16 @@ export class LibraryPage extends BasePage {async openUploadDialog(): Promise<void> {console.log(`opening upload dialog`);- // Open the unified Import dialog first- await this.page.getByRole("button", { name: "Import" }).click();-- // Wait for dialog to appear- await this.page.getByRole("dialog", { name: "Import" }).waitFor({- state: "visible",- timeout: 5_000,- });+ // Open the unified Import dialog first. A click that lands before the+ // page hydrates does nothing, so click again until the dialog shows.+ const importButton = this.page.getByRole("button", { name: "Import" });+ const dialog = this.page.getByRole("dialog", { name: "Import" });+ await expect(async () => {+ if (!(await dialog.isVisible())) {+ await importButton.click({ timeout: 2_000 });+ }+ await expect(dialog).toBeVisible({ timeout: 2_000 });+ }).toPass({ timeout: 20_000 });// Click the "AutoGPT agent" tabawait this.page.getByRole("tab", { name: "AutoGPT agent" }).click();diff --git a/autogpt_platform/frontend/src/services/feature-flags/__tests__/pw-flag-overrides.test.tsx b/autogpt_platform/frontend/src/services/feature-flags/__tests__/pw-flag-overrides.test.tsxnew file mode 100644index 0000000000..7c23286bd4--- /dev/null+++ b/autogpt_platform/frontend/src/services/feature-flags/__tests__/pw-flag-overrides.test.tsx@@ -0,0 +1,111 @@+import { renderHook } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++vi.mock("launchdarkly-react-client-sdk", () => ({+ useFlags: () => ({}),+}));++vi.mock("@/app/(platform)/marketplace/components/HeroSection/helpers", () => ({+ DEFAULT_SEARCH_TERMS: [],+}));++vi.mock("@/services/environment", () => ({+ environment: { areFeatureFlagsEnabled: () => false },+}));++const STORAGE_KEY = "autogpt-pw-flag-overrides";++async function loadFlags() {+ vi.resetModules();+ return import("../use-get-flag");+}++describe("Playwright runtime flag overrides", () => {+ beforeEach(() => {+ window.localStorage.clear();+ });++ afterEach(() => {+ vi.unstubAllEnvs();+ window.localStorage.clear();+ });++ it("serves the stored value in a Playwright build", async () => {+ vi.stubEnv("NEXT_PUBLIC_PW_TEST", "true");+ window.localStorage.setItem(+ STORAGE_KEY,+ JSON.stringify({ "hire-experts": true }),+ );+ const { Flag, useGetFlag, useFlagStatus } = await loadFlags();++ expect(renderHook(() => useGetFlag(Flag.HIRE_EXPERTS)).result.current).toBe(+ true,+ );+ expect(+ renderHook(() => useFlagStatus(Flag.HIRE_EXPERTS)).result.current,+ ).toEqual({ enabled: true, ready: true, answered: true });+ expect(renderHook(() => useGetFlag(Flag.SKILLS_HUB)).result.current).toBe(+ false,+ );+ });++ it("keeps route gates waiting until hydration can read the overrides", async () => {+ vi.stubEnv("NEXT_PUBLIC_PW_TEST", "true");+ const { Flag, useFlagStatus } = await loadFlags();+ const { renderToString } = await import("react-dom/server");++ let status: ReturnType<typeof useFlagStatus> | undefined;+ function Probe() {+ status = useFlagStatus(Flag.HIRE_EXPERTS);+ return null;+ }++ renderToString(<Probe />);+ expect(status).toMatchObject({ ready: false, answered: false });+ });++ it("is ignored outside a Playwright build", async () => {+ vi.stubEnv("NEXT_PUBLIC_PW_TEST", "false");+ window.localStorage.setItem(+ STORAGE_KEY,+ JSON.stringify({ "hire-experts": true }),+ );+ const { Flag, useGetFlag } = await loadFlags();++ expect(renderHook(() => useGetFlag(Flag.HIRE_EXPERTS)).result.current).toBe(+ false,+ );+ });++ it("falls back to the default for a value of the wrong type or bad JSON", async () => {+ vi.stubEnv("NEXT_PUBLIC_PW_TEST", "true");+ window.localStorage.setItem(+ STORAGE_KEY,+ JSON.stringify({ "hire-experts": "yes" }),+ );+ const { Flag, useGetFlag } = await loadFlags();++ expect(renderHook(() => useGetFlag(Flag.HIRE_EXPERTS)).result.current).toBe(+ false,+ );++ window.localStorage.setItem(STORAGE_KEY, "{not json");+ expect(renderHook(() => useGetFlag(Flag.HIRE_EXPERTS)).result.current).toBe(+ false,+ );+ });++ it("wins over a build-time env override", async () => {+ vi.stubEnv("NEXT_PUBLIC_PW_TEST", "true");+ vi.stubEnv("NEXT_PUBLIC_FORCE_FLAG_HIRE_EXPERTS", "true");+ window.localStorage.setItem(+ STORAGE_KEY,+ JSON.stringify({ "hire-experts": false }),+ );+ const { Flag, useGetFlag } = await loadFlags();++ expect(renderHook(() => useGetFlag(Flag.HIRE_EXPERTS)).result.current).toBe(+ false,+ );+ });+});diff --git a/autogpt_platform/frontend/src/services/feature-flags/pw-flag-overrides.ts b/autogpt_platform/frontend/src/services/feature-flags/pw-flag-overrides.tsnew file mode 100644index 0000000000..f0cf9a8230--- /dev/null+++ b/autogpt_platform/frontend/src/services/feature-flags/pw-flag-overrides.ts@@ -0,0 +1,68 @@+"use client";++import { useSyncExternalStore } from "react";++// Playwright builds only (``NEXT_PUBLIC_PW_TEST=true``): a journey writes a+// ``{ "<flag-key>": value }`` map under this key before the app loads, so one+// bundle can serve both the self-hosted defaults and the hosted flag values+// without a rebuild. Any other build never reads it.+export const PW_FLAG_OVERRIDES_STORAGE_KEY = "autogpt-pw-flag-overrides";++type PwFlagOverrides = Record<string, unknown>;++const NO_OVERRIDES: PwFlagOverrides = {};+const parsedOverrides = new Map<string, PwFlagOverrides>();++function isPwBuild() {+ return process.env.NEXT_PUBLIC_PW_TEST === "true";+}++function subscribe() {+ return () => {};+}++function readStoredOverrides() {+ if (!isPwBuild()) return "";+ try {+ return window.localStorage.getItem(PW_FLAG_OVERRIDES_STORAGE_KEY) ?? "";+ } catch {+ return "";+ }+}++// Outside a Playwright build both snapshots agree, so hydration doesn't+// re-render every flag consumer for nothing.+function readServerSnapshot() {+ return isPwBuild() ? null : "";+}++/**+ * The Playwright flag overrides, or ``null`` while the page is hydrating and+ * the browser's storage can't be read yet.+ */+export function usePwFlagOverrides(): PwFlagOverrides | null {+ const raw = useSyncExternalStore(+ subscribe,+ readStoredOverrides,+ readServerSnapshot,+ );+ if (raw === null) return null;+ return parseOverrides(raw);+}++function parseOverrides(raw: string): PwFlagOverrides {+ if (!raw) return NO_OVERRIDES;+ const cached = parsedOverrides.get(raw);+ if (cached) return cached;+ let parsed: PwFlagOverrides = NO_OVERRIDES;+ try {+ const value: unknown = JSON.parse(raw);+ if (value && typeof value === "object" && !Array.isArray(value)) {+ parsed = value as PwFlagOverrides;+ }+ } catch {+ parsed = NO_OVERRIDES;+ }+ parsedOverrides.set(raw, parsed);+ return parsed;+}diff --git a/autogpt_platform/frontend/src/services/feature-flags/use-get-flag.ts b/autogpt_platform/frontend/src/services/feature-flags/use-get-flag.tsindex 17d2d30dcb..28a144d5af 100644--- a/autogpt_platform/frontend/src/services/feature-flags/use-get-flag.ts+++ b/autogpt_platform/frontend/src/services/feature-flags/use-get-flag.ts@@ -8,6 +8,7 @@ import { useEffect, useState } from "react";import { FLAG_BACKEND, isPostHogFlagsEnabled } from "./flag-backend";import { useFlagResolutionStarted } from "./flag-resolution";import { useFlagSource } from "./flag-source";+import { usePwFlagOverrides } from "./pw-flag-overrides";export enum Flag {MARKETPLACE_SEARCH_TERMS = "marketplace-search-terms",@@ -219,9 +220,19 @@ export function envFlagOverride<T extends Flag>(return isForceAllFlags ? (true as FlagValues[T]) : undefined;}+// A Playwright journey's runtime override wins over the build-time env one.+function pwFlagOverride<T extends Flag>(+ overrides: Record<string, unknown> | null,+ flag: T,+): FlagValues[T] | undefined {+ if (!overrides || !(flag in overrides)) return undefined;+ return resolveFlagValue(flag, overrides[flag]);+}+export function useGetFlag<T extends Flag>(flag: T): FlagValues[T] {const { value } = useFlagSource(flag);- const override = envFlagOverride(flag);+ const pwOverrides = usePwFlagOverrides();+ const override = pwFlagOverride(pwOverrides, flag) ?? envFlagOverride(flag);const served = override ?? servedFlagValue(flag, value);recordFlagForSentry(flag, override === undefined ? served : undefined);return served;@@ -246,7 +257,8 @@ export function useFlagStatus<T extends Flag>(): { enabled: FlagValues[T]; ready: boolean; answered: boolean } {const { value, resolved } = useFlagSource(flag);const areFlagsEnabled = areFeatureFlagsEnabled();- const override = envFlagOverride(flag);+ const pwOverrides = usePwFlagOverrides();+ const override = pwFlagOverride(pwOverrides, flag) ?? envFlagOverride(flag);const resolutionStarted = useFlagResolutionStarted();const [timedOut, setTimedOut] = useState(false);@@ -262,6 +274,11 @@ export function useFlagStatus<T extends Flag>(const served = override ?? servedFlagValue(flag, value);recordFlagForSentry(flag, override === undefined ? served : undefined);+ // Until hydration ends a Playwright override can't be read yet, so a route+ // gate must not act on the default it would replace.+ if (isPwMockEnabled && pwOverrides === null) {+ return { enabled: served, ready: false, answered: false };+ }if (override !== undefined || !areFlagsEnabled || isPwMockEnabled) {return { enabled: served, ready: true, answered: true };}