AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): seed E2E embeddings before CI dumps the database for its cache (#14454)
Reinier van der Leer92b250c8685c
diff --git a/autogpt_platform/backend/test/e2e_test_data.py b/autogpt_platform/backend/test/e2e_test_data.pyindex f698fd296e..0e830f7e5c 100644--- a/autogpt_platform/backend/test/e2e_test_data.py+++ b/autogpt_platform/backend/test/e2e_test_data.py@@ -14,7 +14,9 @@ Image/Video URL Domains Used:import asyncioimport json+import osimport random+import timeimport uuidfrom pathlib import Pathfrom typing import Any, Dict, List@@ -28,6 +30,10 @@ from pydantic import SecretStr# Import API functions from the backendfrom backend.api.features.library.db import create_library_agent, create_presetfrom backend.api.features.library.model import LibraryAgentPresetCreatable+from backend.api.features.search.embeddings import (+ backfill_all_content_types,+ get_embedding_stats,+)from backend.api.features.store.db import (create_store_submission,review_store_submission,@@ -40,6 +46,7 @@ from backend.data.db import prismafrom backend.data.graph import Graph, Link, Node, create_graph, make_graph_modelfrom backend.data.model import APIKeyCredentialsfrom backend.data.user import get_or_create_user+from backend.util.clients import get_openai_clientfrom backend.util.encryption import JSONCryptorfrom backend.util.json import SafeJson@@ -81,6 +88,13 @@ _DOCKER_TEMPLATE_PATH = Path(E2E_MARKETPLACE_AGENT_TEMPLATE_PATH = (_LOCAL_TEMPLATE_PATH if _LOCAL_TEMPLATE_PATH.exists() else _DOCKER_TEMPLATE_PATH)+# CI dumps this database for its cache after seeding, so anything left unembedded+# here is re-embedded on every cache hit. Batch size is concurrency, not a page size.+EMBEDDING_BACKFILL_BATCH_SIZE = int(os.getenv("E2E_EMBEDDING_BATCH_SIZE", "100"))+EMBEDDING_BACKFILL_TIMEOUT_SECONDS = float(+ os.getenv("E2E_EMBEDDING_TIMEOUT_SECONDS", "900")+)+SEEDED_TEST_EMAILS = [@@ -1266,6 +1280,8 @@ class TestDataCreator:except Exception as e:print(f"Error refreshing materialized views: {e}")+ await self.backfill_content_embeddings()+print("E2E test data creation completed successfully!")# Print summary@@ -1285,6 +1301,68 @@ class TestDataCreator:print(f" • Library agents per user: >= {MIN_AGENTS_PER_USER}")print("\n🚀 Your E2E test database is ready to use!")+ async def backfill_content_embeddings(self):+ """Drive embedding coverage to 100% so the CI cache dump carries it."""+ if not get_openai_client():+ print("⏭️ No embedding backend configured — skipping embedding backfill")+ return++ print("Backfilling content embeddings...")+ deadline = time.monotonic() + EMBEDDING_BACKFILL_TIMEOUT_SECONDS+ while True:+ try:+ # Both awaits reach the network, and one stuck embedding call is+ # 600s x 3 attempts under the OpenAI client's defaults — longer than+ # the whole deadline. Checking the clock between them bounds nothing.+ stats = await asyncio.wait_for(+ get_embedding_stats(), _seconds_left(deadline)+ )+ # On failure get_embedding_stats reports zero missing, which would+ # read as complete coverage and cache a dump with no embeddings.+ if "error" in stats:+ print(+ "::warning title=e2e-embeddings-unknown::Could not read "+ f"embedding stats ({stats['error']}); skipping backfill. "+ "The cached dump will be incomplete."+ )+ return++ totals = stats["totals"]+ missing = totals["without_embeddings"]+ if missing == 0:+ print(+ f"✅ Embeddings complete: {totals['total']} items, "+ f"{totals['coverage_percent']}% coverage"+ )+ return++ print(f" {missing} items without embeddings — backfilling...")+ result = await asyncio.wait_for(+ backfill_all_content_types(EMBEDDING_BACKFILL_BATCH_SIZE),+ _seconds_left(deadline),+ )+ except asyncio.TimeoutError:+ print(+ "::warning title=e2e-embeddings-incomplete::Embedding backfill "+ f"did not reach full coverage within "+ f"{EMBEDDING_BACKFILL_TIMEOUT_SECONDS:.0f}s. The cached dump "+ "will be incomplete and every cache hit will re-run the backfill."+ )+ return++ if result["totals"]["success"] == 0:+ print(+ "::warning title=e2e-embeddings-stalled::Embedding backfill made "+ f"no progress ({result['totals']['message']}); giving up with "+ f"{missing} items missing."+ )+ return+++def _seconds_left(deadline: float) -> float:+ """Remaining budget, floored at 0 so an expired deadline times out at once."""+ return max(0.0, deadline - time.monotonic())+async def main():"""Main function to run the test data creation."""diff --git a/autogpt_platform/backend/test/test_e2e_test_data_embeddings.py b/autogpt_platform/backend/test/test_e2e_test_data_embeddings.pynew file mode 100644index 0000000000..bf0d847433--- /dev/null+++ b/autogpt_platform/backend/test/test_e2e_test_data_embeddings.py@@ -0,0 +1,144 @@+"""+Unit tests for the E2E seeder's embedding backfill phase.+Stubs the search-embedding functions to avoid needing a database or an LLM.+"""++import asyncio+import time+from test import e2e_test_data+from unittest.mock import patch++import pytest+++def _stats(missing: int, total: int = 5584):+ return {+ "totals": {+ "without_embeddings": missing,+ "total": total,+ "coverage_percent": round((total - missing) / total * 100, 1),+ }+ }+++class _Recorder:+ """Serves a stats sequence and records the backfill calls it triggers."""++ def __init__(self, stats_sequence, success=1):+ self._stats = list(stats_sequence)+ self._success = success+ self.batch_sizes: list[int] = []++ async def get_stats(self):+ return self._stats.pop(0) if len(self._stats) > 1 else self._stats[0]++ async def backfill(self, batch_size):+ self.batch_sizes.append(batch_size)+ return {+ "totals": {+ "success": self._success,+ "message": f"Overall: {self._success} succeeded",+ }+ }+++def seeder():+ return e2e_test_data.TestDataCreator.__new__(e2e_test_data.TestDataCreator)+++async def _run(seeder, recorder, *, client=object()):+ with (+ patch.object(e2e_test_data, "get_openai_client", return_value=client),+ patch.object(e2e_test_data, "get_embedding_stats", recorder.get_stats),+ patch.object(e2e_test_data, "backfill_all_content_types", recorder.backfill),+ ):+ await seeder.backfill_content_embeddings()+++async def test_skips_when_no_embedding_backend(seeder):+ """A stack without an embedding backend must not attempt any API call."""+ recorder = _Recorder([_stats(5584)])+ await _run(seeder, recorder, client=None)+ assert recorder.batch_sizes == []+++async def test_skips_when_already_covered(seeder):+ recorder = _Recorder([_stats(0)])+ await _run(seeder, recorder)+ assert recorder.batch_sizes == []+++async def test_does_not_claim_coverage_when_stats_are_unreadable(seeder):+ """get_embedding_stats reports zero missing on failure; that is not 100%."""+ broken = dict(_stats(0, 5584), error="connection refused")+ recorder = _Recorder([broken])+ await _run(seeder, recorder)+ assert recorder.batch_sizes == []+++async def test_backfills_until_coverage_is_complete(seeder):+ recorder = _Recorder([_stats(5584), _stats(2000), _stats(0)])+ await _run(seeder, recorder)+ assert recorder.batch_sizes == [+ e2e_test_data.EMBEDDING_BACKFILL_BATCH_SIZE,+ e2e_test_data.EMBEDDING_BACKFILL_BATCH_SIZE,+ ]+++async def test_gives_up_when_the_backfill_stops_making_progress(seeder):+ """A backfill that only fails must not spin until the deadline."""+ recorder = _Recorder([_stats(500)], success=0)+ await _run(seeder, recorder)+ assert len(recorder.batch_sizes) == 1+++async def test_returns_on_deadline_instead_of_blocking_the_dump(seeder, monkeypatch):+ monkeypatch.setattr(e2e_test_data, "EMBEDDING_BACKFILL_TIMEOUT_SECONDS", 0.0)+ recorder = _Recorder([_stats(500)])+ await _run(seeder, recorder)+ assert recorder.batch_sizes == []+++async def _hang(*_args, **_kwargs):+ """Block until cancelled — a helper that never returns on its own."""+ await asyncio.Event().wait()+++ "stalled_helper", ["get_embedding_stats", "backfill_all_content_types"]+)+async def test_deadline_fires_while_a_helper_hangs(seeder, monkeypatch, stalled_helper):+ """A stuck call must not outlive the deadline: both awaits are bounded.++ One embedding call is 600s x 3 attempts under the OpenAI client's defaults,+ so a clock check between the awaits would never be reached.+ """+ monkeypatch.setattr(e2e_test_data, "EMBEDDING_BACKFILL_TIMEOUT_SECONDS", 0.25)+ recorder = _Recorder([_stats(5584)])+ patches = {+ "get_openai_client": object(),+ "get_embedding_stats": recorder.get_stats,+ "backfill_all_content_types": recorder.backfill,+ stalled_helper: _hang,+ }+ with (+ patch.object(+ e2e_test_data,+ "get_openai_client",+ return_value=patches.pop("get_openai_client"),+ ),+ patch.object(+ e2e_test_data, "get_embedding_stats", patches["get_embedding_stats"]+ ),+ patch.object(+ e2e_test_data,+ "backfill_all_content_types",+ patches["backfill_all_content_types"],+ ),+ ):+ started = time.monotonic()+ # The outer bound is the assertion: without wait_for inside the loop this+ # never returns, and the test fails here instead of hanging the suite.+ await asyncio.wait_for(seeder.backfill_content_embeddings(), timeout=10)+ assert time.monotonic() - started < 5