BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
Add adapter-driven URL ingest from Lumen cleanup JSON.
Christian7a97e9f8e3bc
diff --git a/.env.example b/.env.exampleindex b1b6ee6..e016fd0 100644--- a/.env.example+++ b/.env.example@@ -136,6 +136,14 @@ DOCPARSER_ENDPOINT="https://docparser.lumen-labs.ai"APP_HOST="https://your-app-host.com/ingest"DOCPARSER_TOKEN="your-docparser-token"+# URL crawler (POST /ingest/url)+# Default provider is Lumen Crawler. BrainAPI never fetches the seed URL itself;+# it creates a crawl, waits for cleanup JSON, and ingests title/headings/core/facts.+CRAWLER_PROVIDER="lumen"+LUMEN_CRAWLER_KEY=""+LUMEN_CRAWLER_ENDPOINT="https://crawler.lumen-labs.ai"+CRAWLER_POLL_TIMEOUT_SECONDS="900"+# Spacy# Used to keep the spacy models in memory or reinitialize them on each requestSPACY_KEEP_MODELS_IN_MEMORY="true"diff --git a/README.md b/README.mdindex 9e2be52..8fbbdac 100644--- a/README.md+++ b/README.md@@ -141,6 +141,15 @@ curl -X POST http://localhost:8000/ingest/ \-d '{"data": {"data_type": "text", "text_data": "Emily organized the AI Ethics Meetup in London on 2024-03-08."}, "brain_id": "default"}'```+To ingest a public website, set `LUMEN_CRAWLER_KEY` and POST a seed URL. BrainAPI crawls through Lumen Crawler, waits for cleanup JSON, and ingests title, headings, core content, and facts — not raw page Markdown:++```sh+curl -X POST http://localhost:8000/ingest/url \+ -H "Authorization: Bearer $BRAINPAT_TOKEN" \+ -H "Content-Type: application/json" \+ -d '{"url": "https://example.com", "max_pages": 10}'+```+---## 🖥️ The Consolediff --git a/deploy/env.heavy.example b/deploy/env.heavy.exampleindex bb84cd8..5ef7fbc 100644--- a/deploy/env.heavy.example+++ b/deploy/env.heavy.example@@ -135,6 +135,12 @@ DOCPARSER_ENDPOINT="https://docparser.lumen-labs.ai"APP_HOST="https://your-app-host.com/ingest"DOCPARSER_TOKEN="your-docparser-token"+# URL crawler (POST /ingest/url)+CRAWLER_PROVIDER="lumen"+LUMEN_CRAWLER_KEY=""+LUMEN_CRAWLER_ENDPOINT="https://crawler.lumen-labs.ai"+CRAWLER_POLL_TIMEOUT_SECONDS="900"+# Spacy# Used to keep the spacy models in memory or reinitialize them on each requestSPACY_KEEP_MODELS_IN_MEMORY="true"diff --git a/deploy/env.light.example b/deploy/env.light.exampleindex adfb6dc..68d6beb 100644--- a/deploy/env.light.example+++ b/deploy/env.light.example@@ -133,6 +133,12 @@ DOCPARSER_ENDPOINT="https://docparser.lumen-labs.ai"APP_HOST="https://your-app-host.com/ingest"DOCPARSER_TOKEN="your-docparser-token"+# URL crawler (POST /ingest/url)+CRAWLER_PROVIDER="lumen"+LUMEN_CRAWLER_KEY=""+LUMEN_CRAWLER_ENDPOINT="https://crawler.lumen-labs.ai"+CRAWLER_POLL_TIMEOUT_SECONDS="900"+# Spacy# Used to keep the spacy models in memory or reinitialize them on each requestSPACY_KEEP_MODELS_IN_MEMORY="true"diff --git a/poetry.lock b/poetry.lockindex f1330e0..03237b4 100644--- a/poetry.lock+++ b/poetry.lock@@ -1145,47 +1145,6 @@ tesserocr = ["docling-slim[feat-ocr-tesserocr] (==2.121.0)"]vlm = ["docling-slim[models-vlm-inline] (==2.121.0)"]xbrl = ["docling-slim[format-xml-xbrl] (==2.121.0)"]-[[package]]-name = "docling-core"-version = "2.92.0"-description = "A python library to define and validate data types in Docling."-optional = false-python-versions = "<4.0,>=3.10"-groups = ["main", "docling-ocr"]-markers = "sys_platform != \"darwin\""-files = [- {file = "docling_core-2.92.0-py3-none-any.whl", hash = "sha256:726d89c23197e53be2f7192bb4c00108ff545830cf9ab7b981fb014fa92b5c02"},- {file = "docling_core-2.92.0.tar.gz", hash = "sha256:33fd25e38c199336447a21925374400aca13a6b9316a032f111972c2dc0f085c"},-]--[package.dependencies]-defusedxml = ">=0.7.1,<0.8.0"-doclang = ">=0.7,<0.8"-jsonref = ">=1.1.0,<2.0.0"-jsonschema = ">=4.16.0,<5.0.0"-latex2mathml = ">=3.77.0,<4.0.0"-pandas = ">=2.1.4,<4.0.0"-pillow = ">=10.0.0,<13.0.0"-pydantic = ">=2.6.0,<2.10.0 || >2.10.0,<2.10.1 || >2.10.1,<2.10.2 || >2.10.2,<3.0.0"-pydantic-settings = ">=2.14.0"-pyyaml = ">=5.1,<7.0.0"-semchunk = {version = ">=2.2.0,<4.0.0", optional = true, markers = "extra == \"chunking\""}-tabulate = ">=0.9.0,<0.11.0"-transformers = {version = ">=4.34.0,<5.0.dev0 || >=5.4.dev0,<6.0.0", optional = true, markers = "sys_platform != \"darwin\" and extra == \"chunking\""}-tree-sitter = {version = ">=0.25.0,<0.27.0", optional = true, markers = "extra == \"chunking\""}-tree-sitter-c = {version = ">=0.23.4", optional = true, markers = "extra == \"chunking\""}-tree-sitter-javascript = {version = ">=0.23.1", optional = true, markers = "extra == \"chunking\""}-tree-sitter-python = {version = ">=0.23.6", optional = true, markers = "extra == \"chunking\""}-tree-sitter-typescript = {version = ">=0.23.2", optional = true, markers = "extra == \"chunking\""}-typer = ">=0.12.5,<0.27.0"-typing-extensions = ">=4.12.2,<5.0.0"--[package.extras]-chunking = ["semchunk (>=2.2.0,<4.0.0)", "transformers (>=4.34.0,<5.0.dev0 || >=5.4.dev0,<5.9.0) ; sys_platform == \"darwin\"", "transformers (>=4.34.0,<5.0.dev0 || >=5.4.dev0,<6.0.0) ; sys_platform != \"darwin\"", "tree-sitter (>=0.25.0,<0.27.0)", "tree-sitter-c (>=0.23.4)", "tree-sitter-javascript (>=0.23.1)", "tree-sitter-python (>=0.23.6)", "tree-sitter-typescript (>=0.23.2)"]-chunking-openai = ["semchunk (>=2.2.0,<4.0.0)", "tiktoken (>=0.9.0,<0.13.0)", "tree-sitter (>=0.25.0,<0.27.0)", "tree-sitter-c (>=0.23.4)", "tree-sitter-javascript (>=0.23.1)", "tree-sitter-python (>=0.23.6)", "tree-sitter-typescript (>=0.23.2)"]-dclq = ["click (>=8.0.0)", "doclang (>=0.7,<0.8)", "lxml (>=6.0.2)", "typer (>=0.15.1,<0.25.0)"]-examples = ["datasets (>=4.0.0)", "matplotlib (>=3.7.0)", "openpyxl (>=3.1.5)"]-[[package]]name = "docling-core"version = "2.101.1"@@ -1193,7 +1152,6 @@ description = "A python library to define and validate data types in Docling."optional = falsepython-versions = "<4.0,>=3.10"groups = ["main", "docling-ocr"]-markers = "sys_platform == \"darwin\""files = [{file = "docling_core-2.101.1-py3-none-any.whl", hash = "sha256:0fbadbf21fb1389f75c2ec4c5c979016c0fe532f29d001467d5a82f92410cc6b"},{file = "docling_core-2.101.1.tar.gz", hash = "sha256:4b338a4c69e20145a256809eef3cb9954b5baa0ff51e84d41c25268143f6ce24"},@@ -8915,4 +8873,4 @@ postgresql-backend = ["networkx", "pgvector", "psycopg2-binary"][metadata]lock-version = "2.1"python-versions = ">=3.11,<4.0.0"-content-hash = "fb63e2545a89059e9856a2948a2d0c25c2c7d7306a4155115e77e8cb437665be"+content-hash = "c85f900cb48f00c3157a9de0624ae966edaf68e1149557e8566b18b75082392f"diff --git a/pyproject.toml b/pyproject.tomlindex 04669cf..3fba8dc 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -49,6 +49,7 @@ dependencies = ["watchfiles (>=1.2.0,<2.0.0)","anthropic (>=0.57.1,<0.58.0)","tiktoken (>=0.12.0,<1.0.0)",+ "httpx (>=0.27.0,<1.0.0)",][project.optional-dependencies]@@ -110,6 +111,7 @@ langchain-aws = ">=1.0.0,<2.0.0"watchfiles = ">=1.2.0,<2.0.0"anthropic = ">=0.57.1,<0.58.0"tiktoken = ">=0.12.0,<1.0.0"+httpx = ">=0.27.0,<1.0.0"[tool.poetry.group.docling-ocr]optional = truediff --git a/src/adapters/crawler.py b/src/adapters/crawler.pynew file mode 100644index 0000000..ddd1acd--- /dev/null+++ b/src/adapters/crawler.py@@ -0,0 +1,12 @@+from src.adapters.interfaces.crawler import CrawledPage, CrawlerClient, CrawlRequest+++class CrawlerAdapter:+ def __init__(self):+ self.crawler = None++ def add_client(self, client: CrawlerClient) -> None:+ self.crawler = client++ def crawl_pages(self, request: CrawlRequest) -> list[CrawledPage]:+ return self.crawler.crawl_pages(request)diff --git a/src/adapters/interfaces/crawler.py b/src/adapters/interfaces/crawler.pynew file mode 100644index 0000000..205d01e--- /dev/null+++ b/src/adapters/interfaces/crawler.py@@ -0,0 +1,27 @@+from abc import ABC, abstractmethod+from typing import Optional++from pydantic import BaseModel, Field+++class CrawledPage(BaseModel):+ url: str+ canonical_url: Optional[str] = None+ title: Optional[str] = None+ page_type: Optional[str] = None+ text: str+ content_hash: Optional[str] = None+ status_code: Optional[int] = None+++class CrawlRequest(BaseModel):+ url: str+ max_pages: int = 10+ allowed_domains: list[str] = Field(default_factory=list)+ idempotency_key: str+++class CrawlerClient(ABC):+ @abstractmethod+ def crawl_pages(self, request: CrawlRequest) -> list[CrawledPage]:+ raise NotImplementedError("crawl_pages method not implemented")diff --git a/src/config.py b/src/config.pyindex 78f00cf..d807b34 100644--- a/src/config.py+++ b/src/config.py@@ -695,6 +695,14 @@ class Config:self.docparser_endpoint = os.getenv("DOCPARSER_ENDPOINT")self.docparser_token = os.getenv("DOCPARSER_TOKEN")self.app_host = os.getenv("APP_HOST")+ self.crawler_provider = os.getenv("CRAWLER_PROVIDER", "lumen").strip().lower()+ self.lumen_crawler_key = (os.getenv("LUMEN_CRAWLER_KEY") or "").strip()+ self.lumen_crawler_endpoint = (+ os.getenv("LUMEN_CRAWLER_ENDPOINT") or "https://crawler.lumen-labs.ai"+ ).rstrip("/")+ self.crawler_poll_timeout_seconds = int(+ os.getenv("CRAWLER_POLL_TIMEOUT_SECONDS", "900")+ )self.pipeline_mode: Literal["lightweight", "accurate"] = validate_pipeline_mode(os.getenv("PIPELINE_MODE", "accurate"))@@ -744,6 +752,12 @@ class Config:bm25_b=self.search_bm25_b,)+ @property+ def crawler_is_configured(self) -> bool:+ if self.crawler_provider == "lumen":+ return bool(self.lumen_crawler_key)+ return False+def search_fts_regconfig_for_brain(self, brain_id: str) -> str | None:bid = (brain_id or "").strip()if not bid.lower().startswith("searchbench"):diff --git a/src/core/instances.py b/src/core/instances.pyindex 1d7f12c..63a8e37 100644--- a/src/core/instances.py+++ b/src/core/instances.py@@ -1,4 +1,5 @@from src.adapters.cache import CacheAdapter+from src.adapters.crawler import CrawlerAdapterfrom src.adapters.data import DataAdapterfrom src.adapters.embeddings import EmbeddingsAdapter, VectorStoreAdapterfrom src.adapters.graph import GraphAdapter@@ -125,6 +126,18 @@ def _build_embeddings(provider: str):raise ValueError(f"Unsupported embeddings provider: {provider}")+def _build_crawler(provider: str):+ if provider == "lumen":+ from src.lib.crawler.client_lumen import LumenCrawlerClient++ return LumenCrawlerClient(+ api_key=config.lumen_crawler_key,+ endpoint=config.lumen_crawler_endpoint,+ poll_timeout_seconds=config.crawler_poll_timeout_seconds,+ )+ raise ValueError(f"Unsupported crawler provider: {provider}")++_llm_small = LLMAdapter()_llm_small.add_client(_build_small_llm(config.llm_small_provider))_llm_large = LLMAdapter()@@ -182,3 +195,6 @@ elif config.vector_db == "postgresql":vector_store_adapter = VectorStoreAdapter()vector_store_adapter.add_client(get_postgresql_vector_store_client())++crawler_adapter = CrawlerAdapter()+crawler_adapter.add_client(_build_crawler(config.crawler_provider))diff --git a/src/lib/crawler/__init__.py b/src/lib/crawler/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/src/lib/crawler/cleaned.py b/src/lib/crawler/cleaned.pynew file mode 100644index 0000000..cccd5d1--- /dev/null+++ b/src/lib/crawler/cleaned.py@@ -0,0 +1,84 @@+from typing import Any, Optional+++def _as_dict(value: Any) -> dict:+ return value if isinstance(value, dict) else {}+++def _as_list(value: Any) -> list:+ return value if isinstance(value, list) else []+++def _text(value: Any) -> str:+ if value is None:+ return ""+ return str(value).strip()+++def format_cleaned_page(cleaned: dict) -> str:+ content = _as_dict(cleaned.get("content"))+ facts = _as_dict(cleaned.get("facts"))+ contacts = _as_dict(facts.get("contacts"))+ lines: list[str] = []++ title = _text(content.get("title"))+ if title:+ lines.append(f"Title: {title}")++ page_type = _text(cleaned.get("page_type"))+ if page_type:+ lines.append(f"Type: {page_type}")++ url = _text(cleaned.get("canonical_url")) or _text(cleaned.get("url"))+ if url:+ lines.append(f"URL: {url}")++ heading_lines: list[str] = []+ for heading in _as_list(content.get("headings")):+ if isinstance(heading, dict):+ text = _text(heading.get("text"))+ if not text:+ continue+ level = heading.get("level") or 1+ heading_lines.append(f"- H{level} {text}")+ else:+ text = _text(heading)+ if text:+ heading_lines.append(f"- {text}")+ if heading_lines:+ lines.extend(["", "Headings:", *heading_lines])++ core = _text(content.get("core"))+ if core:+ lines.extend(["", core])++ contact_lines: list[str] = []+ for email in _as_list(contacts.get("emails")):+ text = _text(email)+ if text:+ contact_lines.append(f"- email: {text}")+ for phone in _as_list(contacts.get("phones")):+ text = _text(phone)+ if text:+ contact_lines.append(f"- phone: {text}")+ for address in _as_list(contacts.get("addresses")):+ text = _text(address)+ if text:+ contact_lines.append(f"- address: {text}")+ if contact_lines:+ lines.extend(["", "Contacts:", *contact_lines])++ cta_lines: list[str] = []+ for cta in _as_list(facts.get("ctas")):+ text = _text(cta)+ if text:+ cta_lines.append(f"- {text}")+ if cta_lines:+ lines.extend(["", "CTAs:", *cta_lines])++ return "\n".join(lines).strip()+++def cleaned_title(cleaned: dict, fallback: Optional[str] = None) -> Optional[str]:+ content = _as_dict(cleaned.get("content"))+ return _text(content.get("title")) or _text(fallback) or Nonediff --git a/src/lib/crawler/client_lumen.py b/src/lib/crawler/client_lumen.pynew file mode 100644index 0000000..4be0c2f--- /dev/null+++ b/src/lib/crawler/client_lumen.py@@ -0,0 +1,190 @@+from __future__ import annotations++import time+from typing import Any, Callable, Optional+from urllib.parse import urljoin++import httpx++from src.adapters.interfaces.crawler import CrawledPage, CrawlerClient, CrawlRequest+from src.lib.crawler.cleaned import cleaned_title, format_cleaned_page+from src.lib.crawler.errors import (+ CrawlerConfigError,+ CrawlerFailedError,+ CrawlerQuotaError,+ CrawlerTimeoutError,+)++TERMINAL_CRAWL_STATUSES = {"completed", "cancelled", "failed"}+TERMINAL_CLEANUP_STATUSES = {"completed", "failed", "skipped"}+POLL_INTERVAL_START = 1.0+POLL_INTERVAL_CAP = 5.0+PAGE_LIMIT = 100+++class LumenCrawlerClient(CrawlerClient):+ def __init__(+ self,+ api_key: Optional[str] = None,+ endpoint: str = "https://crawler.lumen-labs.ai",+ poll_timeout_seconds: float = 900,+ timeout: float = 60.0,+ transport: Optional[httpx.BaseTransport] = None,+ sleep: Callable[[float], None] = time.sleep,+ monotonic: Callable[[], float] = time.monotonic,+ ):+ self.api_key = (api_key or "").strip()+ self.endpoint = endpoint.rstrip("/")+ self.poll_timeout_seconds = poll_timeout_seconds+ self.timeout = timeout+ self.transport = transport+ self._sleep = sleep+ self._monotonic = monotonic++ def _client(self) -> httpx.Client:+ return httpx.Client(timeout=self.timeout, transport=self.transport)++ def _headers(self, idempotency_key: Optional[str] = None) -> dict[str, str]:+ headers = {+ "Accept": "application/json",+ "Authorization": f"Bearer {self.api_key}",+ }+ if idempotency_key:+ headers["Idempotency-Key"] = idempotency_key+ headers["Content-Type"] = "application/json"+ return headers++ def _url(self, path: str) -> str:+ return urljoin(f"{self.endpoint}/", path.lstrip("/"))++ def crawl_pages(self, request: CrawlRequest) -> list[CrawledPage]:+ if not self.api_key:+ raise CrawlerConfigError("LUMEN_CRAWLER_KEY is not set")+ with self._client() as client:+ crawl_id = self._create_crawl(client, request)+ self._poll_until_cleaned(client, crawl_id)+ return self._list_cleaned_pages(client, crawl_id)++ def _create_crawl(self, client: httpx.Client, request: CrawlRequest) -> str:+ response = client.post(+ self._url("/v1/crawls"),+ headers=self._headers(request.idempotency_key),+ json={+ "url": request.url,+ "max_pages": request.max_pages,+ "allowed_domains": request.allowed_domains,+ "follow_doors": False,+ "cleanup": True,+ },+ )+ self._raise_for_status(response)+ body = response.json()+ crawl_id = body.get("id")+ if not crawl_id:+ raise CrawlerFailedError("Crawler did not return a crawl id")+ return str(crawl_id)++ def _poll_until_cleaned(self, client: httpx.Client, crawl_id: str) -> None:+ deadline = self._monotonic() + self.poll_timeout_seconds+ interval = POLL_INTERVAL_START+ while True:+ response = client.get(+ self._url(f"/v1/crawls/{crawl_id}"),+ headers=self._headers(),+ )+ self._raise_for_status(response)+ body = response.json()+ status = body.get("status")+ cleanup_status = (body.get("cleanup") or {}).get("status")+ if status in TERMINAL_CRAWL_STATUSES:+ if status != "completed":+ reason = body.get("stop_reason") or status+ raise CrawlerFailedError(f"Crawl ended with status {reason}")+ if cleanup_status == "completed":+ return+ if cleanup_status in {"failed", "skipped"}:+ raise CrawlerFailedError(+ f"Crawl cleanup ended with status {cleanup_status}"+ )+ if self._monotonic() >= deadline:+ self._cancel(client, crawl_id)+ raise CrawlerTimeoutError("Timed out waiting for crawl cleanup")+ self._sleep(interval)+ interval = min(interval * 1.5, POLL_INTERVAL_CAP)++ def _cancel(self, client: httpx.Client, crawl_id: str) -> None:+ try:+ client.post(+ self._url(f"/v1/crawls/{crawl_id}/cancel"),+ headers=self._headers(),+ )+ except httpx.HTTPError:+ return++ def _list_cleaned_pages(+ self, client: httpx.Client, crawl_id: str+ ) -> list[CrawledPage]:+ pages: list[CrawledPage] = []+ cursor: Optional[str] = None+ while True:+ params: dict[str, Any] = {"include": "cleaned", "limit": PAGE_LIMIT}+ if cursor:+ params["cursor"] = cursor+ response = client.get(+ self._url(f"/v1/crawls/{crawl_id}/pages"),+ headers=self._headers(),+ params=params,+ )+ self._raise_for_status(response)+ body = response.json()+ for item in body.get("items") or []:+ page = self._to_crawled_page(item)+ if page is not None:+ pages.append(page)+ cursor = body.get("next_cursor")+ if not cursor:+ return pages++ def _to_crawled_page(self, item: dict) -> Optional[CrawledPage]:+ if not item.get("ok"):+ return None+ cleaned = item.get("cleaned")+ if not isinstance(cleaned, dict):+ return None+ text = format_cleaned_page(cleaned)+ if not text:+ return None+ url = cleaned.get("canonical_url") or cleaned.get("url") or item.get("url")+ if not url:+ return None+ return CrawledPage(+ url=str(url),+ canonical_url=cleaned.get("canonical_url") or item.get("canonical_url"),+ title=cleaned_title(cleaned, item.get("title")),+ page_type=cleaned.get("page_type"),+ text=text,+ content_hash=item.get("content_hash"),+ status_code=item.get("status_code"),+ )++ def _raise_for_status(self, response: httpx.Response) -> None:+ if response.is_success:+ return+ detail = _problem_detail(response)+ if response.status_code == 402:+ raise CrawlerQuotaError(detail)+ if response.status_code == 401:+ raise CrawlerConfigError(detail)+ raise CrawlerFailedError(detail)+++def _problem_detail(response: httpx.Response) -> str:+ try:+ body = response.json()+ except ValueError:+ body = {}+ if isinstance(body, dict):+ detail = body.get("detail") or body.get("title") or body.get("type")+ if detail:+ return str(detail)+ return f"Crawler request failed with HTTP {response.status_code}"diff --git a/src/lib/crawler/errors.py b/src/lib/crawler/errors.pynew file mode 100644index 0000000..5aa16c0--- /dev/null+++ b/src/lib/crawler/errors.py@@ -0,0 +1,18 @@+class CrawlerError(Exception):+ pass+++class CrawlerConfigError(CrawlerError):+ pass+++class CrawlerQuotaError(CrawlerError):+ pass+++class CrawlerFailedError(CrawlerError):+ pass+++class CrawlerTimeoutError(CrawlerError):+ passdiff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.pyindex 855c736..28599e6 100644--- a/src/services/api/constants/requests.py+++ b/src/services/api/constants/requests.py@@ -9,6 +9,7 @@ Modified By: Christian Nonis <[email protected]>"""from typing import Any, Dict, List, Literal, Optional, Tuple, Union+from urllib.parse import urlparsefrom pydantic import BaseModel, Field, field_serializer, field_validator, model_validator@@ -30,6 +31,41 @@ class IngestionRequestBody(IngestionTaskArgs):"""+class IngestionUrlRequestBody(BaseModel):+ url: str = Field(description="Public https URL to crawl.")+ max_pages: int = Field(+ default=10,+ ge=1,+ le=100,+ description="Maximum pages to crawl. Each cleaned page becomes one ingest job.",+ )+ allowed_domains: List[str] = Field(+ default_factory=list,+ description="Extra eTLD+1 hosts to include, max 20.",+ )+ skip_enrichment: bool = Field(+ default=False,+ description="Persist cleaned page text and embeddings only. Skip Scout/Architect.",+ )++ @field_validator("url")+ @classmethod+ def require_https_url(cls, value: str) -> str:+ parsed = urlparse((value or "").strip())+ if parsed.scheme != "https" or not parsed.netloc:+ raise ValueError("url must be a public https URL")+ return (value or "").strip()++ @field_validator("allowed_domains")+ @classmethod+ def validate_allowed_domains(cls, value: List[str]) -> List[str]:+ if value is None:+ return []+ if len(value) > 20:+ raise ValueError("allowed_domains accepts at most 20 hosts")+ return [item.strip() for item in value if item and str(item).strip()]++class IngestionStructuredDataElement(BaseModel):"""Element for the structured ingestion endpoint.diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.pyindex 6eb936c..b419f26 100644--- a/src/services/api/routes/ingest.py+++ b/src/services/api/routes/ingest.py@@ -24,11 +24,13 @@ from src.services.api.dependencies import get_brain_idfrom src.services.api.constants.requests import (IngestionRequestBody,IngestionStructuredRequestBody,+ IngestionUrlRequestBody,)from src.services.api.constants.responses import IngestionAcceptedResponsefrom src.services.input.agents import cache_adapterfrom src.workers.tasks.ingestion import ingest_data as ingest_data_taskfrom src.workers.tasks.ingestion import ingest_file as ingest_file_task+from src.workers.tasks.ingestion import ingest_url as ingest_url_taskfrom src.workers.tasks.ingestion import (ingest_structured_data as ingest_structured_data_task,)@@ -124,6 +126,49 @@ async def ingest_structured_data()+@ingest_router.post(+ path="/url", status_code=202, response_model=IngestionAcceptedResponse+)+async def ingest_url(+ data: IngestionUrlRequestBody,+ brain_id: str = Depends(get_brain_id),+):+ """+ Accept a public https URL for asynchronous crawl-and-ingest.+ """+ if not config.crawler_is_configured:+ raise HTTPException(status_code=503, detail="URL crawler is not configured")++ payload = data.model_dump()+ payload["brain_id"] = brain_id+ task_id = str(uuid4())+ set_ingestion_task_status(task_id, brain_id, "queued", stage="queued")++ for attempt in range(MAX_TASK_RETRIES):+ try:+ ingest_url_task.apply_async(+ args=[payload],+ task_id=task_id,+ )+ break+ except OperationalError:+ if attempt == MAX_TASK_RETRIES - 1:+ set_ingestion_task_status(+ task_id,+ brain_id,+ "failed",+ stage="queue",+ error="Task queue unavailable",+ )+ raise HTTPException(status_code=503, detail="Task queue unavailable")+ await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))++ return JSONResponse(+ status_code=202,+ content={"message": "URL ingestion accepted", "task_id": task_id},+ )++@ingest_router.post(path="/file", status_code=202, response_model=IngestionAcceptedResponse)diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.pyindex c14c864..22c58a4 100644--- a/src/workers/tasks/ingestion.py+++ b/src/workers/tasks/ingestion.py@@ -1646,3 +1646,117 @@ def ingest_file(self, content_b64: str, filename: str, brain_id: str):expires_in=3600 * 24 * 7,)return {"task_id": self.request.id, "task_ids": task_ids}+++def build_url_page_ingest_payload(+ brain_id: str,+ page,+ skip_enrichment: bool = False,+) -> dict:+ meta_keys = {}+ if getattr(page, "url", None):+ meta_keys["source_url"] = page.url+ if getattr(page, "title", None):+ meta_keys["source_title"] = page.title+ if getattr(page, "content_hash", None):+ meta_keys["content_hash"] = page.content_hash+ if getattr(page, "page_type", None):+ meta_keys["page_type"] = page.page_type+ payload = {+ "data": IngestionTaskTextArgs(+ data_type="text", text_data=page.text+ ).model_dump(),+ "brain_id": brain_id,+ "skip_enrichment": skip_enrichment,+ }+ if meta_keys:+ payload["meta_keys"] = meta_keys+ return payload+++@ingestion_app.task(bind=True)+def ingest_url(self, args: dict):+ from celery.exceptions import OperationalError++ from src.adapters.interfaces.crawler import CrawlRequest+ from src.core.instances import crawler_adapter+ from src.lib.crawler.errors import (+ CrawlerConfigError,+ CrawlerFailedError,+ CrawlerQuotaError,+ CrawlerTimeoutError,+ )++ brain_id = args.get("brain_id", "default")+ skip_enrichment = bool(args.get("skip_enrichment", False))+ try:+ set_ingestion_task_status(+ self.request.id,+ brain_id,+ "started",+ stage="crawling",+ )+ pages = crawler_adapter.crawl_pages(+ CrawlRequest(+ url=args["url"],+ max_pages=int(args.get("max_pages") or 10),+ allowed_domains=args.get("allowed_domains") or [],+ idempotency_key=self.request.id,+ )+ )+ ingestible = [page for page in pages if (page.text or "").strip()]+ task_ids = [str(uuid4()) for _ in ingestible]+ for page_task_id in task_ids:+ set_ingestion_task_status(+ page_task_id,+ brain_id,+ "queued",+ stage="queued",+ )+ for page_task_id, page in zip(task_ids, ingestible):+ payload = build_url_page_ingest_payload(+ brain_id,+ page,+ skip_enrichment=skip_enrichment,+ )+ for attempt in range(FILE_INGEST_MAX_RETRIES):+ try:+ ingest_data.apply_async(+ args=[payload],+ task_id=page_task_id,+ )+ break+ except OperationalError:+ if attempt == FILE_INGEST_MAX_RETRIES - 1:+ raise+ time.sleep(FILE_INGEST_RETRY_DELAY * (attempt + 1))+ set_ingestion_task_status(+ self.request.id,+ brain_id,+ "completed",+ counts={"pages": len(task_ids), "task_ids": task_ids},+ )+ return {"task_id": self.request.id, "task_ids": task_ids}+ except (+ CrawlerConfigError,+ CrawlerQuotaError,+ CrawlerFailedError,+ CrawlerTimeoutError,+ ) as exc:+ set_ingestion_task_status(+ self.request.id,+ brain_id,+ "failed",+ stage="failed",+ error=str(exc),+ )+ raise+ except Exception as exc:+ set_ingestion_task_status(+ self.request.id,+ brain_id,+ "failed",+ stage="failed",+ error=str(exc),+ )+ raisediff --git a/tests/test_agent_readiness_api.py b/tests/test_agent_readiness_api.pyindex dd27595..a723367 100644--- a/tests/test_agent_readiness_api.py+++ b/tests/test_agent_readiness_api.py@@ -333,6 +333,11 @@ class OpenApiContractTests(unittest.TestCase):self.assertIn("/system/brains/{brain_id}", schema["paths"])self.assertIn("delete", schema["paths"]["/system/brains/{brain_id}"])self.assertIn("202", schema["paths"]["/ingest/file"]["post"]["responses"])+ self.assertIn("202", schema["paths"]["/ingest/url"]["post"]["responses"])+ self.assertIn(+ "503",+ schema["paths"]["/ingest/url"]["post"]["responses"],+ )generic_refs = [response["content"]["application/json"]["schema"].get("$ref", "")diff --git a/tests/test_crawler_lumen.py b/tests/test_crawler_lumen.pynew file mode 100644index 0000000..d359bc3--- /dev/null+++ b/tests/test_crawler_lumen.py@@ -0,0 +1,279 @@+from __future__ import annotations++import json++import httpx+import pytest++from src.adapters.crawler import CrawlerAdapter+from src.adapters.interfaces.crawler import CrawlRequest, CrawledPage+from src.lib.crawler.cleaned import format_cleaned_page+from src.lib.crawler.client_lumen import LumenCrawlerClient+from src.lib.crawler.errors import (+ CrawlerConfigError,+ CrawlerFailedError,+ CrawlerQuotaError,+ CrawlerTimeoutError,+)++CRAWL_ID = "crawl-1"+CLEANED_PAGE = {+ "url": "https://example.com/",+ "canonical_url": "https://example.com/",+ "page_type": "landing",+ "chrome": {"navigation": ["Home", "Pricing"], "footer": ["Login"], "forms": []},+ "content": {+ "title": "Home",+ "headings": [{"level": 1, "text": "Home"}, {"level": 2, "text": "Features"}],+ "core": "Welcome to Example.",+ "paragraphs": ["Welcome to Example.", "Home Pricing Login"],+ },+ "areas": [+ {+ "id": "main",+ "heading": "Home",+ "category": "intro",+ "markdown": "# Home\n\n" + ("huge markdown " * 200),+ }+ ],+ "facts": {+ "contacts": {+ "emails": ["[email protected]"],+ "phones": [],+ "addresses": [],+ },+ "ctas": ["Sign up"],+ },+ "model": "mock",+ "input_tokens": 1,+ "output_tokens": 1,+}+++def _request() -> CrawlRequest:+ return CrawlRequest(+ url="https://example.com",+ max_pages=10,+ allowed_domains=[],+ idempotency_key="task-1",+ )+++class FakeClock:+ def __init__(self):+ self.now = 0.0+ self.sleeps: list[float] = []++ def monotonic(self) -> float:+ return self.now++ def sleep(self, seconds: float) -> None:+ self.sleeps.append(seconds)+ self.now += seconds+++def _json(request: httpx.Request, status: int, body: dict, headers: dict | None = None) -> httpx.Response:+ return httpx.Response(+ status,+ request=request,+ headers=headers or {"content-type": "application/json"},+ content=json.dumps(body).encode("utf-8"),+ )+++def test_format_cleaned_page_keeps_core_fields_and_drops_chrome_markdown():+ text = format_cleaned_page(CLEANED_PAGE)+ assert "Title: Home" in text+ assert "Type: landing" in text+ assert "URL: https://example.com/" in text+ assert "H1 Home" in text+ assert "Welcome to Example." in text+ assert "email: [email protected]" in text+ assert "Sign up" in text+ assert "Pricing" not in text+ assert "huge markdown" not in text+ assert "Home Pricing Login" not in text+++def test_missing_api_key_raises_before_http():+ calls = []++ def handler(request: httpx.Request) -> httpx.Response:+ calls.append(request)+ return httpx.Response(500, request=request)++ client = LumenCrawlerClient(+ api_key="",+ transport=httpx.MockTransport(handler),+ )+ with pytest.raises(CrawlerConfigError, match="LUMEN_CRAWLER_KEY"):+ client.crawl_pages(_request())+ assert calls == []+++def test_create_poll_and_cleaned_pages():+ polls = {"n": 0}++ def handler(request: httpx.Request) -> httpx.Response:+ path = request.url.path+ if request.method == "POST" and path == "/v1/crawls":+ body = json.loads(request.content.decode("utf-8"))+ assert body["cleanup"] is True+ assert body["follow_doors"] is False+ assert body["max_pages"] == 10+ assert request.headers["idempotency-key"] == "task-1"+ return _json(+ request,+ 202,+ {+ "id": CRAWL_ID,+ "status": "queued",+ "cleanup": {"status": "pending"},+ },+ )+ if request.method == "GET" and path == f"/v1/crawls/{CRAWL_ID}":+ polls["n"] += 1+ if polls["n"] == 1:+ return _json(+ request,+ 200,+ {"id": CRAWL_ID, "status": "running", "cleanup": {"status": "pending"}},+ )+ return _json(+ request,+ 200,+ {+ "id": CRAWL_ID,+ "status": "completed",+ "cleanup": {"status": "completed"},+ },+ )+ if request.method == "GET" and path == f"/v1/crawls/{CRAWL_ID}/pages":+ assert request.url.params.get("include") == "cleaned"+ return _json(+ request,+ 200,+ {+ "items": [+ {+ "url": "https://example.com/",+ "canonical_url": "https://example.com/",+ "title": "Home",+ "status_code": 200,+ "ok": True,+ "content_hash": "abc",+ "cleaned": CLEANED_PAGE,+ },+ {+ "url": "https://example.com/robots",+ "ok": False,+ "cleaned": CLEANED_PAGE,+ },+ ],+ "next_cursor": None,+ },+ )+ return httpx.Response(404, request=request)++ clock = FakeClock()+ client = LumenCrawlerClient(+ api_key="lc_live_test",+ transport=httpx.MockTransport(handler),+ sleep=clock.sleep,+ monotonic=clock.monotonic,+ )+ pages = client.crawl_pages(_request())+ assert len(pages) == 1+ assert pages[0].url == "https://example.com/"+ assert pages[0].title == "Home"+ assert pages[0].page_type == "landing"+ assert "Welcome to Example." in pages[0].text+ assert "huge markdown" not in pages[0].text+ assert polls["n"] == 2+++def test_quota_error_is_not_retried():+ def handler(request: httpx.Request) -> httpx.Response:+ return _json(+ request,+ 402,+ {+ "type": "quota_exceeded",+ "title": "Quota exceeded",+ "status": 402,+ "detail": "max_pages does not fit remaining included pages",+ },+ )++ client = LumenCrawlerClient(+ api_key="lc_live_test",+ transport=httpx.MockTransport(handler),+ )+ with pytest.raises(CrawlerQuotaError, match="remaining included pages"):+ client.crawl_pages(_request())+++def test_failed_crawl_raises():+ def handler(request: httpx.Request) -> httpx.Response:+ if request.method == "POST":+ return _json(request, 202, {"id": CRAWL_ID, "status": "queued"})+ return _json(+ request,+ 200,+ {+ "id": CRAWL_ID,+ "status": "failed",+ "stop_reason": "seed_failed",+ "cleanup": {"status": "skipped"},+ },+ )++ client = LumenCrawlerClient(+ api_key="lc_live_test",+ transport=httpx.MockTransport(handler),+ sleep=lambda _: None,+ )+ with pytest.raises(CrawlerFailedError, match="seed_failed"):+ client.crawl_pages(_request())+++def test_timeout_cancels_running_crawl():+ cancelled = {"n": 0}++ def handler(request: httpx.Request) -> httpx.Response:+ if request.method == "POST" and request.url.path.endswith("/cancel"):+ cancelled["n"] += 1+ return _json(request, 200, {"id": CRAWL_ID, "status": "cancelled"})+ if request.method == "POST":+ return _json(request, 202, {"id": CRAWL_ID, "status": "queued"})+ return _json(+ request,+ 200,+ {"id": CRAWL_ID, "status": "running", "cleanup": {"status": "pending"}},+ )++ clock = FakeClock()+ client = LumenCrawlerClient(+ api_key="lc_live_test",+ transport=httpx.MockTransport(handler),+ poll_timeout_seconds=0,+ sleep=clock.sleep,+ monotonic=clock.monotonic,+ )+ with pytest.raises(CrawlerTimeoutError):+ client.crawl_pages(_request())+ assert cancelled["n"] == 1+++def test_adapter_delegates_crawl_pages():+ adapter = CrawlerAdapter()+ page = CrawledPage(url="https://example.com/", text="Title: Home")++ class FakeClient:+ def crawl_pages(self, request: CrawlRequest) -> list[CrawledPage]:+ assert request.url == "https://example.com"+ return [page]++ adapter.add_client(FakeClient())+ result = adapter.crawl_pages(_request())+ assert result == [page]diff --git a/tests/test_ingest_url.py b/tests/test_ingest_url.pynew file mode 100644index 0000000..b300579--- /dev/null+++ b/tests/test_ingest_url.py@@ -0,0 +1,276 @@+import os+import unittest+from unittest.mock import MagicMock, patch++from fastapi import FastAPI+from fastapi.testclient import TestClient+from pydantic import ValidationError++from src.adapters.interfaces.crawler import CrawledPage+from src.lib.crawler.errors import CrawlerQuotaError, CrawlerTimeoutError+from src.services.api.constants.requests import IngestionUrlRequestBody+++ENV_DEFAULTS = {+ "BRAINPAT_TOKEN": "test-token",+ "MODELS_MODE": "local",+ "EMBEDDINGS_LOCAL_MODEL": "local-model",+ "EMBEDDINGS_SMALL_MODEL": "small-model",+ "EMBEDDING_NODES_DIMENSION": "3",+ "EMBEDDING_TRIPLETS_DIMENSION": "3",+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",+ "EMBEDDING_DATA_DIMENSION": "3",+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",+ "REDIS_HOST": "localhost",+ "REDIS_PORT": "6379",+ "NEO4J_HOST": "localhost",+ "NEO4J_PORT": "7687",+ "NEO4J_USERNAME": "neo4j",+ "NEO4J_PASSWORD": "password",+ "MILVUS_HOST": "localhost",+ "MILVUS_PORT": "19530",+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",+ "CELERY_WORKER_CONCURRENCY": "1",+ "OLLAMA_HOST": "localhost",+ "OLLAMA_PORT": "11434",+ "OLLAMA_LLM_SMALL_MODEL": "small",+ "OLLAMA_LLM_LARGE_MODEL": "large",+ "PIPELINE_MODE": "accurate",+}+for key, value in ENV_DEFAULTS.items():+ os.environ.setdefault(key, value)+++PAGE = CrawledPage(+ url="https://example.com/",+ canonical_url="https://example.com/",+ title="Home",+ page_type="landing",+ text="Title: Home\n\nWelcome to Example.",+ content_hash="abc",+ status_code=200,+)+++class IngestionUrlRequestTests(unittest.TestCase):+ def test_accepts_https_and_defaults(self):+ body = IngestionUrlRequestBody(url="https://example.com/docs")+ self.assertEqual(body.max_pages, 10)+ self.assertEqual(body.allowed_domains, [])+ self.assertFalse(body.skip_enrichment)++ def test_rejects_http_and_invalid_bounds(self):+ with self.assertRaises(ValidationError):+ IngestionUrlRequestBody(url="http://example.com")+ with self.assertRaises(ValidationError):+ IngestionUrlRequestBody(url="https://example.com", max_pages=0)+ with self.assertRaises(ValidationError):+ IngestionUrlRequestBody(url="https://example.com", max_pages=101)+ with self.assertRaises(ValidationError):+ IngestionUrlRequestBody(+ url="https://example.com",+ allowed_domains=[f"host{i}.com" for i in range(21)],+ )+++class IngestUrlRouteTests(unittest.TestCase):+ def _client(self):+ from src.services.api.dependencies import get_brain_id+ from src.services.api.routes.ingest import ingest_router++ app = FastAPI()+ app.dependency_overrides[get_brain_id] = lambda: "brain-a"+ app.include_router(ingest_router)+ return TestClient(app)++ def test_valid_request_enqueues_task(self):+ captured = {}++ def fake_apply_async(args=None, task_id=None):+ captured["args"] = args+ captured["task_id"] = task_id+ return MagicMock()++ client = self._client()+ with patch(+ "src.services.api.routes.ingest.config"+ ) as config, patch(+ "src.services.api.routes.ingest.set_ingestion_task_status",+ return_value={},+ ), patch(+ "src.services.api.routes.ingest.ingest_url_task.apply_async",+ side_effect=fake_apply_async,+ ):+ config.crawler_is_configured = True+ response = client.post(+ "/ingest/url",+ json={"url": "https://example.com", "max_pages": 3},+ )+ self.assertEqual(response.status_code, 202)+ self.assertEqual(response.json()["message"], "URL ingestion accepted")+ self.assertTrue(response.json()["task_id"])+ self.assertEqual(captured["args"][0]["brain_id"], "brain-a")+ self.assertEqual(captured["args"][0]["url"], "https://example.com")+ self.assertEqual(captured["args"][0]["max_pages"], 3)++ def test_missing_crawler_key_is_503_without_enqueue(self):+ apply_async = MagicMock()+ client = self._client()+ with patch(+ "src.services.api.routes.ingest.config"+ ) as config, patch(+ "src.services.api.routes.ingest.ingest_url_task.apply_async",+ apply_async,+ ):+ config.crawler_is_configured = False+ response = client.post(+ "/ingest/url",+ json={"url": "https://example.com"},+ )+ self.assertEqual(response.status_code, 503)+ apply_async.assert_not_called()++ def test_invalid_url_is_422(self):+ client = self._client()+ with patch(+ "src.services.api.routes.ingest.config"+ ) as config:+ config.crawler_is_configured = True+ response = client.post(+ "/ingest/url",+ json={"url": "http://localhost"},+ )+ self.assertEqual(response.status_code, 422)+++class IngestUrlTaskTests(unittest.TestCase):+ def test_fans_out_cleaned_pages_with_source_url(self):+ from src.workers.tasks import ingestion as ingestion_mod++ request = MagicMock()+ request.id = "url-parent"+ enqueued = []+ statuses = []++ def fake_apply_async(args=None, task_id=None):+ enqueued.append({"args": args, "task_id": task_id})+ return MagicMock()++ def fake_set_status(task_id, brain_id, status, **kwargs):+ statuses.append((task_id, status, kwargs.get("stage")))+ return {"status": status, "task_id": task_id}++ crawler = MagicMock()+ crawler.crawl_pages.return_value = [+ PAGE,+ CrawledPage(url="https://example.com/empty", text=" "),+ ]++ with patch(+ "src.core.instances.crawler_adapter", crawler+ ), patch.object(+ ingestion_mod, "ingest_data"+ ) as ingest_data, patch.object(+ ingestion_mod, "set_ingestion_task_status", side_effect=fake_set_status+ ):+ ingest_data.apply_async.side_effect = fake_apply_async+ result = ingestion_mod.ingest_url.run.__func__(+ type("Bound", (), {"request": request})(),+ {+ "url": "https://example.com",+ "max_pages": 10,+ "brain_id": "brain-a",+ "skip_enrichment": True,+ },+ )++ self.assertEqual(result["task_id"], "url-parent")+ self.assertEqual(len(result["task_ids"]), 1)+ payload = enqueued[0]["args"][0]+ self.assertEqual(payload["meta_keys"]["source_url"], "https://example.com/")+ self.assertEqual(payload["meta_keys"]["source_title"], "Home")+ self.assertEqual(payload["skip_enrichment"], True)+ self.assertIn("Welcome to Example.", payload["data"]["text_data"])+ self.assertEqual(statuses[-1][:2], ("url-parent", "completed"))+ crawl_request = crawler.crawl_pages.call_args.args[0]+ self.assertEqual(crawl_request.idempotency_key, "url-parent")++ def test_empty_crawl_completes_with_zero_pages(self):+ from src.workers.tasks import ingestion as ingestion_mod++ request = MagicMock()+ request.id = "url-empty"+ crawler = MagicMock()+ crawler.crawl_pages.return_value = []++ with patch(+ "src.core.instances.crawler_adapter", crawler+ ), patch.object(+ ingestion_mod, "ingest_data"+ ) as ingest_data, patch.object(+ ingestion_mod, "set_ingestion_task_status", return_value={}+ ):+ result = ingestion_mod.ingest_url.run.__func__(+ type("Bound", (), {"request": request})(),+ {"url": "https://example.com", "brain_id": "brain-a"},+ )++ self.assertEqual(result["task_ids"], [])+ ingest_data.apply_async.assert_not_called()++ def test_quota_failure_marks_parent_failed(self):+ from src.workers.tasks import ingestion as ingestion_mod++ request = MagicMock()+ request.id = "url-quota"+ statuses = []+ crawler = MagicMock()+ crawler.crawl_pages.side_effect = CrawlerQuotaError("max_pages does not fit")++ def fake_set_status(task_id, brain_id, status, **kwargs):+ statuses.append((status, kwargs.get("error")))+ return {"status": status, "task_id": task_id}++ with patch(+ "src.core.instances.crawler_adapter", crawler+ ), patch.object(+ ingestion_mod, "set_ingestion_task_status", side_effect=fake_set_status+ ):+ with self.assertRaises(CrawlerQuotaError):+ ingestion_mod.ingest_url.run.__func__(+ type("Bound", (), {"request": request})(),+ {"url": "https://example.com", "brain_id": "brain-a"},+ )++ self.assertEqual(statuses[-1][0], "failed")+ self.assertIn("max_pages", statuses[-1][1])++ def test_timeout_failure_marks_parent_failed(self):+ from src.workers.tasks import ingestion as ingestion_mod++ request = MagicMock()+ request.id = "url-timeout"+ statuses = []+ crawler = MagicMock()+ crawler.crawl_pages.side_effect = CrawlerTimeoutError("Timed out waiting")++ def fake_set_status(task_id, brain_id, status, **kwargs):+ statuses.append(status)+ return {"status": status, "task_id": task_id}++ with patch(+ "src.core.instances.crawler_adapter", crawler+ ), patch.object(+ ingestion_mod, "set_ingestion_task_status", side_effect=fake_set_status+ ):+ with self.assertRaises(CrawlerTimeoutError):+ ingestion_mod.ingest_url.run.__func__(+ type("Bound", (), {"request": request})(),+ {"url": "https://example.com", "brain_id": "brain-a"},+ )++ self.assertEqual(statuses[-1], "failed")+++if __name__ == "__main__":+ unittest.main()