Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

Add adapter-driven URL ingest from Lumen cleanup JSON.

Christian7a97e9f8e3bc
diff --git a/.env.example b/.env.example
index b1b6ee6..e016fd0 100644
--- a/.env.example
+++ b/.env.example
@@ -136,6 +136,14 @@ DOCPARSER_ENDPOINT="https://docparser.lumen-labs.ai"
APP_HOST="https://your-app-host.com/ingest"
DOCPARSER_TOKEN="your-docparser-token"
+# URL crawler (POST /ingest/url)
+# Default provider is Lumen Crawler. BrainAPI never fetches the seed URL itself;
+# it creates a crawl, waits for cleanup JSON, and ingests title/headings/core/facts.
+CRAWLER_PROVIDER="lumen"
+LUMEN_CRAWLER_KEY=""
+LUMEN_CRAWLER_ENDPOINT="https://crawler.lumen-labs.ai"
+CRAWLER_POLL_TIMEOUT_SECONDS="900"
+
# Spacy
# Used to keep the spacy models in memory or reinitialize them on each request
SPACY_KEEP_MODELS_IN_MEMORY="true"
diff --git a/README.md b/README.md
index 9e2be52..8fbbdac 100644
--- a/README.md
+++ b/README.md
@@ -141,6 +141,15 @@ curl -X POST http://localhost:8000/ingest/ \
-d '{"data": {"data_type": "text", "text_data": "Emily organized the AI Ethics Meetup in London on 2024-03-08."}, "brain_id": "default"}'
```
+To ingest a public website, set `LUMEN_CRAWLER_KEY` and POST a seed URL. BrainAPI crawls through Lumen Crawler, waits for cleanup JSON, and ingests title, headings, core content, and facts — not raw page Markdown:
+
+```sh
+curl -X POST http://localhost:8000/ingest/url \
+ -H "Authorization: Bearer $BRAINPAT_TOKEN" \
+ -H "Content-Type: application/json" \
+ -d '{"url": "https://example.com", "max_pages": 10}'
+```
+
---
## 🖥️ The Console
diff --git a/deploy/env.heavy.example b/deploy/env.heavy.example
index bb84cd8..5ef7fbc 100644
--- a/deploy/env.heavy.example
+++ b/deploy/env.heavy.example
@@ -135,6 +135,12 @@ DOCPARSER_ENDPOINT="https://docparser.lumen-labs.ai"
APP_HOST="https://your-app-host.com/ingest"
DOCPARSER_TOKEN="your-docparser-token"
+# URL crawler (POST /ingest/url)
+CRAWLER_PROVIDER="lumen"
+LUMEN_CRAWLER_KEY=""
+LUMEN_CRAWLER_ENDPOINT="https://crawler.lumen-labs.ai"
+CRAWLER_POLL_TIMEOUT_SECONDS="900"
+
# Spacy
# Used to keep the spacy models in memory or reinitialize them on each request
SPACY_KEEP_MODELS_IN_MEMORY="true"
diff --git a/deploy/env.light.example b/deploy/env.light.example
index adfb6dc..68d6beb 100644
--- a/deploy/env.light.example
+++ b/deploy/env.light.example
@@ -133,6 +133,12 @@ DOCPARSER_ENDPOINT="https://docparser.lumen-labs.ai"
APP_HOST="https://your-app-host.com/ingest"
DOCPARSER_TOKEN="your-docparser-token"
+# URL crawler (POST /ingest/url)
+CRAWLER_PROVIDER="lumen"
+LUMEN_CRAWLER_KEY=""
+LUMEN_CRAWLER_ENDPOINT="https://crawler.lumen-labs.ai"
+CRAWLER_POLL_TIMEOUT_SECONDS="900"
+
# Spacy
# Used to keep the spacy models in memory or reinitialize them on each request
SPACY_KEEP_MODELS_IN_MEMORY="true"
diff --git a/poetry.lock b/poetry.lock
index f1330e0..03237b4 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1145,47 +1145,6 @@ tesserocr = ["docling-slim[feat-ocr-tesserocr] (==2.121.0)"]
vlm = ["docling-slim[models-vlm-inline] (==2.121.0)"]
xbrl = ["docling-slim[format-xml-xbrl] (==2.121.0)"]
-[[package]]
-name = "docling-core"
-version = "2.92.0"
-description = "A python library to define and validate data types in Docling."
-optional = false
-python-versions = "<4.0,>=3.10"
-groups = ["main", "docling-ocr"]
-markers = "sys_platform != \"darwin\""
-files = [
- {file = "docling_core-2.92.0-py3-none-any.whl", hash = "sha256:726d89c23197e53be2f7192bb4c00108ff545830cf9ab7b981fb014fa92b5c02"},
- {file = "docling_core-2.92.0.tar.gz", hash = "sha256:33fd25e38c199336447a21925374400aca13a6b9316a032f111972c2dc0f085c"},
-]
-
-[package.dependencies]
-defusedxml = ">=0.7.1,<0.8.0"
-doclang = ">=0.7,<0.8"
-jsonref = ">=1.1.0,<2.0.0"
-jsonschema = ">=4.16.0,<5.0.0"
-latex2mathml = ">=3.77.0,<4.0.0"
-pandas = ">=2.1.4,<4.0.0"
-pillow = ">=10.0.0,<13.0.0"
-pydantic = ">=2.6.0,<2.10.0 || >2.10.0,<2.10.1 || >2.10.1,<2.10.2 || >2.10.2,<3.0.0"
-pydantic-settings = ">=2.14.0"
-pyyaml = ">=5.1,<7.0.0"
-semchunk = {version = ">=2.2.0,<4.0.0", optional = true, markers = "extra == \"chunking\""}
-tabulate = ">=0.9.0,<0.11.0"
-transformers = {version = ">=4.34.0,<5.0.dev0 || >=5.4.dev0,<6.0.0", optional = true, markers = "sys_platform != \"darwin\" and extra == \"chunking\""}
-tree-sitter = {version = ">=0.25.0,<0.27.0", optional = true, markers = "extra == \"chunking\""}
-tree-sitter-c = {version = ">=0.23.4", optional = true, markers = "extra == \"chunking\""}
-tree-sitter-javascript = {version = ">=0.23.1", optional = true, markers = "extra == \"chunking\""}
-tree-sitter-python = {version = ">=0.23.6", optional = true, markers = "extra == \"chunking\""}
-tree-sitter-typescript = {version = ">=0.23.2", optional = true, markers = "extra == \"chunking\""}
-typer = ">=0.12.5,<0.27.0"
-typing-extensions = ">=4.12.2,<5.0.0"
-
-[package.extras]
-chunking = ["semchunk (>=2.2.0,<4.0.0)", "transformers (>=4.34.0,<5.0.dev0 || >=5.4.dev0,<5.9.0) ; sys_platform == \"darwin\"", "transformers (>=4.34.0,<5.0.dev0 || >=5.4.dev0,<6.0.0) ; sys_platform != \"darwin\"", "tree-sitter (>=0.25.0,<0.27.0)", "tree-sitter-c (>=0.23.4)", "tree-sitter-javascript (>=0.23.1)", "tree-sitter-python (>=0.23.6)", "tree-sitter-typescript (>=0.23.2)"]
-chunking-openai = ["semchunk (>=2.2.0,<4.0.0)", "tiktoken (>=0.9.0,<0.13.0)", "tree-sitter (>=0.25.0,<0.27.0)", "tree-sitter-c (>=0.23.4)", "tree-sitter-javascript (>=0.23.1)", "tree-sitter-python (>=0.23.6)", "tree-sitter-typescript (>=0.23.2)"]
-dclq = ["click (>=8.0.0)", "doclang (>=0.7,<0.8)", "lxml (>=6.0.2)", "typer (>=0.15.1,<0.25.0)"]
-examples = ["datasets (>=4.0.0)", "matplotlib (>=3.7.0)", "openpyxl (>=3.1.5)"]
-
[[package]]
name = "docling-core"
version = "2.101.1"
@@ -1193,7 +1152,6 @@ description = "A python library to define and validate data types in Docling."
optional = false
python-versions = "<4.0,>=3.10"
groups = ["main", "docling-ocr"]
-markers = "sys_platform == \"darwin\""
files = [
{file = "docling_core-2.101.1-py3-none-any.whl", hash = "sha256:0fbadbf21fb1389f75c2ec4c5c979016c0fe532f29d001467d5a82f92410cc6b"},
{file = "docling_core-2.101.1.tar.gz", hash = "sha256:4b338a4c69e20145a256809eef3cb9954b5baa0ff51e84d41c25268143f6ce24"},
@@ -8915,4 +8873,4 @@ postgresql-backend = ["networkx", "pgvector", "psycopg2-binary"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<4.0.0"
-content-hash = "fb63e2545a89059e9856a2948a2d0c25c2c7d7306a4155115e77e8cb437665be"
+content-hash = "c85f900cb48f00c3157a9de0624ae966edaf68e1149557e8566b18b75082392f"
diff --git a/pyproject.toml b/pyproject.toml
index 04669cf..3fba8dc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -49,6 +49,7 @@ dependencies = [
"watchfiles (>=1.2.0,<2.0.0)",
"anthropic (>=0.57.1,<0.58.0)",
"tiktoken (>=0.12.0,<1.0.0)",
+ "httpx (>=0.27.0,<1.0.0)",
]
[project.optional-dependencies]
@@ -110,6 +111,7 @@ langchain-aws = ">=1.0.0,<2.0.0"
watchfiles = ">=1.2.0,<2.0.0"
anthropic = ">=0.57.1,<0.58.0"
tiktoken = ">=0.12.0,<1.0.0"
+httpx = ">=0.27.0,<1.0.0"
[tool.poetry.group.docling-ocr]
optional = true
diff --git a/src/adapters/crawler.py b/src/adapters/crawler.py
new file mode 100644
index 0000000..ddd1acd
--- /dev/null
+++ b/src/adapters/crawler.py
@@ -0,0 +1,12 @@
+from src.adapters.interfaces.crawler import CrawledPage, CrawlerClient, CrawlRequest
+
+
+class CrawlerAdapter:
+ def __init__(self):
+ self.crawler = None
+
+ def add_client(self, client: CrawlerClient) -> None:
+ self.crawler = client
+
+ def crawl_pages(self, request: CrawlRequest) -> list[CrawledPage]:
+ return self.crawler.crawl_pages(request)
diff --git a/src/adapters/interfaces/crawler.py b/src/adapters/interfaces/crawler.py
new file mode 100644
index 0000000..205d01e
--- /dev/null
+++ b/src/adapters/interfaces/crawler.py
@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class CrawledPage(BaseModel):
+ url: str
+ canonical_url: Optional[str] = None
+ title: Optional[str] = None
+ page_type: Optional[str] = None
+ text: str
+ content_hash: Optional[str] = None
+ status_code: Optional[int] = None
+
+
+class CrawlRequest(BaseModel):
+ url: str
+ max_pages: int = 10
+ allowed_domains: list[str] = Field(default_factory=list)
+ idempotency_key: str
+
+
+class CrawlerClient(ABC):
+ @abstractmethod
+ def crawl_pages(self, request: CrawlRequest) -> list[CrawledPage]:
+ raise NotImplementedError("crawl_pages method not implemented")
diff --git a/src/config.py b/src/config.py
index 78f00cf..d807b34 100644
--- a/src/config.py
+++ b/src/config.py
@@ -695,6 +695,14 @@ class Config:
self.docparser_endpoint = os.getenv("DOCPARSER_ENDPOINT")
self.docparser_token = os.getenv("DOCPARSER_TOKEN")
self.app_host = os.getenv("APP_HOST")
+ self.crawler_provider = os.getenv("CRAWLER_PROVIDER", "lumen").strip().lower()
+ self.lumen_crawler_key = (os.getenv("LUMEN_CRAWLER_KEY") or "").strip()
+ self.lumen_crawler_endpoint = (
+ os.getenv("LUMEN_CRAWLER_ENDPOINT") or "https://crawler.lumen-labs.ai"
+ ).rstrip("/")
+ self.crawler_poll_timeout_seconds = int(
+ os.getenv("CRAWLER_POLL_TIMEOUT_SECONDS", "900")
+ )
self.pipeline_mode: Literal["lightweight", "accurate"] = validate_pipeline_mode(
os.getenv("PIPELINE_MODE", "accurate")
)
@@ -744,6 +752,12 @@ class Config:
bm25_b=self.search_bm25_b,
)
+ @property
+ def crawler_is_configured(self) -> bool:
+ if self.crawler_provider == "lumen":
+ return bool(self.lumen_crawler_key)
+ return False
+
def search_fts_regconfig_for_brain(self, brain_id: str) -> str | None:
bid = (brain_id or "").strip()
if not bid.lower().startswith("searchbench"):
diff --git a/src/core/instances.py b/src/core/instances.py
index 1d7f12c..63a8e37 100644
--- a/src/core/instances.py
+++ b/src/core/instances.py
@@ -1,4 +1,5 @@
from src.adapters.cache import CacheAdapter
+from src.adapters.crawler import CrawlerAdapter
from src.adapters.data import DataAdapter
from src.adapters.embeddings import EmbeddingsAdapter, VectorStoreAdapter
from src.adapters.graph import GraphAdapter
@@ -125,6 +126,18 @@ def _build_embeddings(provider: str):
raise ValueError(f"Unsupported embeddings provider: {provider}")
+def _build_crawler(provider: str):
+ if provider == "lumen":
+ from src.lib.crawler.client_lumen import LumenCrawlerClient
+
+ return LumenCrawlerClient(
+ api_key=config.lumen_crawler_key,
+ endpoint=config.lumen_crawler_endpoint,
+ poll_timeout_seconds=config.crawler_poll_timeout_seconds,
+ )
+ raise ValueError(f"Unsupported crawler provider: {provider}")
+
+
_llm_small = LLMAdapter()
_llm_small.add_client(_build_small_llm(config.llm_small_provider))
_llm_large = LLMAdapter()
@@ -182,3 +195,6 @@ elif config.vector_db == "postgresql":
vector_store_adapter = VectorStoreAdapter()
vector_store_adapter.add_client(get_postgresql_vector_store_client())
+
+crawler_adapter = CrawlerAdapter()
+crawler_adapter.add_client(_build_crawler(config.crawler_provider))
diff --git a/src/lib/crawler/__init__.py b/src/lib/crawler/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/lib/crawler/cleaned.py b/src/lib/crawler/cleaned.py
new file mode 100644
index 0000000..cccd5d1
--- /dev/null
+++ b/src/lib/crawler/cleaned.py
@@ -0,0 +1,84 @@
+from typing import Any, Optional
+
+
+def _as_dict(value: Any) -> dict:
+ return value if isinstance(value, dict) else {}
+
+
+def _as_list(value: Any) -> list:
+ return value if isinstance(value, list) else []
+
+
+def _text(value: Any) -> str:
+ if value is None:
+ return ""
+ return str(value).strip()
+
+
+def format_cleaned_page(cleaned: dict) -> str:
+ content = _as_dict(cleaned.get("content"))
+ facts = _as_dict(cleaned.get("facts"))
+ contacts = _as_dict(facts.get("contacts"))
+ lines: list[str] = []
+
+ title = _text(content.get("title"))
+ if title:
+ lines.append(f"Title: {title}")
+
+ page_type = _text(cleaned.get("page_type"))
+ if page_type:
+ lines.append(f"Type: {page_type}")
+
+ url = _text(cleaned.get("canonical_url")) or _text(cleaned.get("url"))
+ if url:
+ lines.append(f"URL: {url}")
+
+ heading_lines: list[str] = []
+ for heading in _as_list(content.get("headings")):
+ if isinstance(heading, dict):
+ text = _text(heading.get("text"))
+ if not text:
+ continue
+ level = heading.get("level") or 1
+ heading_lines.append(f"- H{level} {text}")
+ else:
+ text = _text(heading)
+ if text:
+ heading_lines.append(f"- {text}")
+ if heading_lines:
+ lines.extend(["", "Headings:", *heading_lines])
+
+ core = _text(content.get("core"))
+ if core:
+ lines.extend(["", core])
+
+ contact_lines: list[str] = []
+ for email in _as_list(contacts.get("emails")):
+ text = _text(email)
+ if text:
+ contact_lines.append(f"- email: {text}")
+ for phone in _as_list(contacts.get("phones")):
+ text = _text(phone)
+ if text:
+ contact_lines.append(f"- phone: {text}")
+ for address in _as_list(contacts.get("addresses")):
+ text = _text(address)
+ if text:
+ contact_lines.append(f"- address: {text}")
+ if contact_lines:
+ lines.extend(["", "Contacts:", *contact_lines])
+
+ cta_lines: list[str] = []
+ for cta in _as_list(facts.get("ctas")):
+ text = _text(cta)
+ if text:
+ cta_lines.append(f"- {text}")
+ if cta_lines:
+ lines.extend(["", "CTAs:", *cta_lines])
+
+ return "\n".join(lines).strip()
+
+
+def cleaned_title(cleaned: dict, fallback: Optional[str] = None) -> Optional[str]:
+ content = _as_dict(cleaned.get("content"))
+ return _text(content.get("title")) or _text(fallback) or None
diff --git a/src/lib/crawler/client_lumen.py b/src/lib/crawler/client_lumen.py
new file mode 100644
index 0000000..4be0c2f
--- /dev/null
+++ b/src/lib/crawler/client_lumen.py
@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+import time
+from typing import Any, Callable, Optional
+from urllib.parse import urljoin
+
+import httpx
+
+from src.adapters.interfaces.crawler import CrawledPage, CrawlerClient, CrawlRequest
+from src.lib.crawler.cleaned import cleaned_title, format_cleaned_page
+from src.lib.crawler.errors import (
+ CrawlerConfigError,
+ CrawlerFailedError,
+ CrawlerQuotaError,
+ CrawlerTimeoutError,
+)
+
+TERMINAL_CRAWL_STATUSES = {"completed", "cancelled", "failed"}
+TERMINAL_CLEANUP_STATUSES = {"completed", "failed", "skipped"}
+POLL_INTERVAL_START = 1.0
+POLL_INTERVAL_CAP = 5.0
+PAGE_LIMIT = 100
+
+
+class LumenCrawlerClient(CrawlerClient):
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ endpoint: str = "https://crawler.lumen-labs.ai",
+ poll_timeout_seconds: float = 900,
+ timeout: float = 60.0,
+ transport: Optional[httpx.BaseTransport] = None,
+ sleep: Callable[[float], None] = time.sleep,
+ monotonic: Callable[[], float] = time.monotonic,
+ ):
+ self.api_key = (api_key or "").strip()
+ self.endpoint = endpoint.rstrip("/")
+ self.poll_timeout_seconds = poll_timeout_seconds
+ self.timeout = timeout
+ self.transport = transport
+ self._sleep = sleep
+ self._monotonic = monotonic
+
+ def _client(self) -> httpx.Client:
+ return httpx.Client(timeout=self.timeout, transport=self.transport)
+
+ def _headers(self, idempotency_key: Optional[str] = None) -> dict[str, str]:
+ headers = {
+ "Accept": "application/json",
+ "Authorization": f"Bearer {self.api_key}",
+ }
+ if idempotency_key:
+ headers["Idempotency-Key"] = idempotency_key
+ headers["Content-Type"] = "application/json"
+ return headers
+
+ def _url(self, path: str) -> str:
+ return urljoin(f"{self.endpoint}/", path.lstrip("/"))
+
+ def crawl_pages(self, request: CrawlRequest) -> list[CrawledPage]:
+ if not self.api_key:
+ raise CrawlerConfigError("LUMEN_CRAWLER_KEY is not set")
+ with self._client() as client:
+ crawl_id = self._create_crawl(client, request)
+ self._poll_until_cleaned(client, crawl_id)
+ return self._list_cleaned_pages(client, crawl_id)
+
+ def _create_crawl(self, client: httpx.Client, request: CrawlRequest) -> str:
+ response = client.post(
+ self._url("/v1/crawls"),
+ headers=self._headers(request.idempotency_key),
+ json={
+ "url": request.url,
+ "max_pages": request.max_pages,
+ "allowed_domains": request.allowed_domains,
+ "follow_doors": False,
+ "cleanup": True,
+ },
+ )
+ self._raise_for_status(response)
+ body = response.json()
+ crawl_id = body.get("id")
+ if not crawl_id:
+ raise CrawlerFailedError("Crawler did not return a crawl id")
+ return str(crawl_id)
+
+ def _poll_until_cleaned(self, client: httpx.Client, crawl_id: str) -> None:
+ deadline = self._monotonic() + self.poll_timeout_seconds
+ interval = POLL_INTERVAL_START
+ while True:
+ response = client.get(
+ self._url(f"/v1/crawls/{crawl_id}"),
+ headers=self._headers(),
+ )
+ self._raise_for_status(response)
+ body = response.json()
+ status = body.get("status")
+ cleanup_status = (body.get("cleanup") or {}).get("status")
+ if status in TERMINAL_CRAWL_STATUSES:
+ if status != "completed":
+ reason = body.get("stop_reason") or status
+ raise CrawlerFailedError(f"Crawl ended with status {reason}")
+ if cleanup_status == "completed":
+ return
+ if cleanup_status in {"failed", "skipped"}:
+ raise CrawlerFailedError(
+ f"Crawl cleanup ended with status {cleanup_status}"
+ )
+ if self._monotonic() >= deadline:
+ self._cancel(client, crawl_id)
+ raise CrawlerTimeoutError("Timed out waiting for crawl cleanup")
+ self._sleep(interval)
+ interval = min(interval * 1.5, POLL_INTERVAL_CAP)
+
+ def _cancel(self, client: httpx.Client, crawl_id: str) -> None:
+ try:
+ client.post(
+ self._url(f"/v1/crawls/{crawl_id}/cancel"),
+ headers=self._headers(),
+ )
+ except httpx.HTTPError:
+ return
+
+ def _list_cleaned_pages(
+ self, client: httpx.Client, crawl_id: str
+ ) -> list[CrawledPage]:
+ pages: list[CrawledPage] = []
+ cursor: Optional[str] = None
+ while True:
+ params: dict[str, Any] = {"include": "cleaned", "limit": PAGE_LIMIT}
+ if cursor:
+ params["cursor"] = cursor
+ response = client.get(
+ self._url(f"/v1/crawls/{crawl_id}/pages"),
+ headers=self._headers(),
+ params=params,
+ )
+ self._raise_for_status(response)
+ body = response.json()
+ for item in body.get("items") or []:
+ page = self._to_crawled_page(item)
+ if page is not None:
+ pages.append(page)
+ cursor = body.get("next_cursor")
+ if not cursor:
+ return pages
+
+ def _to_crawled_page(self, item: dict) -> Optional[CrawledPage]:
+ if not item.get("ok"):
+ return None
+ cleaned = item.get("cleaned")
+ if not isinstance(cleaned, dict):
+ return None
+ text = format_cleaned_page(cleaned)
+ if not text:
+ return None
+ url = cleaned.get("canonical_url") or cleaned.get("url") or item.get("url")
+ if not url:
+ return None
+ return CrawledPage(
+ url=str(url),
+ canonical_url=cleaned.get("canonical_url") or item.get("canonical_url"),
+ title=cleaned_title(cleaned, item.get("title")),
+ page_type=cleaned.get("page_type"),
+ text=text,
+ content_hash=item.get("content_hash"),
+ status_code=item.get("status_code"),
+ )
+
+ def _raise_for_status(self, response: httpx.Response) -> None:
+ if response.is_success:
+ return
+ detail = _problem_detail(response)
+ if response.status_code == 402:
+ raise CrawlerQuotaError(detail)
+ if response.status_code == 401:
+ raise CrawlerConfigError(detail)
+ raise CrawlerFailedError(detail)
+
+
+def _problem_detail(response: httpx.Response) -> str:
+ try:
+ body = response.json()
+ except ValueError:
+ body = {}
+ if isinstance(body, dict):
+ detail = body.get("detail") or body.get("title") or body.get("type")
+ if detail:
+ return str(detail)
+ return f"Crawler request failed with HTTP {response.status_code}"
diff --git a/src/lib/crawler/errors.py b/src/lib/crawler/errors.py
new file mode 100644
index 0000000..5aa16c0
--- /dev/null
+++ b/src/lib/crawler/errors.py
@@ -0,0 +1,18 @@
+class CrawlerError(Exception):
+ pass
+
+
+class CrawlerConfigError(CrawlerError):
+ pass
+
+
+class CrawlerQuotaError(CrawlerError):
+ pass
+
+
+class CrawlerFailedError(CrawlerError):
+ pass
+
+
+class CrawlerTimeoutError(CrawlerError):
+ pass
diff --git a/src/services/api/constants/requests.py b/src/services/api/constants/requests.py
index 855c736..28599e6 100644
--- a/src/services/api/constants/requests.py
+++ b/src/services/api/constants/requests.py
@@ -9,6 +9,7 @@ Modified By: Christian Nonis <[email protected]>
"""
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+from urllib.parse import urlparse
from pydantic import BaseModel, Field, field_serializer, field_validator, model_validator
@@ -30,6 +31,41 @@ class IngestionRequestBody(IngestionTaskArgs):
"""
+class IngestionUrlRequestBody(BaseModel):
+ url: str = Field(description="Public https URL to crawl.")
+ max_pages: int = Field(
+ default=10,
+ ge=1,
+ le=100,
+ description="Maximum pages to crawl. Each cleaned page becomes one ingest job.",
+ )
+ allowed_domains: List[str] = Field(
+ default_factory=list,
+ description="Extra eTLD+1 hosts to include, max 20.",
+ )
+ skip_enrichment: bool = Field(
+ default=False,
+ description="Persist cleaned page text and embeddings only. Skip Scout/Architect.",
+ )
+
+ @field_validator("url")
+ @classmethod
+ def require_https_url(cls, value: str) -> str:
+ parsed = urlparse((value or "").strip())
+ if parsed.scheme != "https" or not parsed.netloc:
+ raise ValueError("url must be a public https URL")
+ return (value or "").strip()
+
+ @field_validator("allowed_domains")
+ @classmethod
+ def validate_allowed_domains(cls, value: List[str]) -> List[str]:
+ if value is None:
+ return []
+ if len(value) > 20:
+ raise ValueError("allowed_domains accepts at most 20 hosts")
+ return [item.strip() for item in value if item and str(item).strip()]
+
+
class IngestionStructuredDataElement(BaseModel):
"""
Element for the structured ingestion endpoint.
diff --git a/src/services/api/routes/ingest.py b/src/services/api/routes/ingest.py
index 6eb936c..b419f26 100644
--- a/src/services/api/routes/ingest.py
+++ b/src/services/api/routes/ingest.py
@@ -24,11 +24,13 @@ from src.services.api.dependencies import get_brain_id
from src.services.api.constants.requests import (
IngestionRequestBody,
IngestionStructuredRequestBody,
+ IngestionUrlRequestBody,
)
from src.services.api.constants.responses import IngestionAcceptedResponse
from src.services.input.agents import cache_adapter
from src.workers.tasks.ingestion import ingest_data as ingest_data_task
from src.workers.tasks.ingestion import ingest_file as ingest_file_task
+from src.workers.tasks.ingestion import ingest_url as ingest_url_task
from src.workers.tasks.ingestion import (
ingest_structured_data as ingest_structured_data_task,
)
@@ -124,6 +126,49 @@ async def ingest_structured_data(
)
+@ingest_router.post(
+ path="/url", status_code=202, response_model=IngestionAcceptedResponse
+)
+async def ingest_url(
+ data: IngestionUrlRequestBody,
+ brain_id: str = Depends(get_brain_id),
+):
+ """
+ Accept a public https URL for asynchronous crawl-and-ingest.
+ """
+ if not config.crawler_is_configured:
+ raise HTTPException(status_code=503, detail="URL crawler is not configured")
+
+ payload = data.model_dump()
+ payload["brain_id"] = brain_id
+ task_id = str(uuid4())
+ set_ingestion_task_status(task_id, brain_id, "queued", stage="queued")
+
+ for attempt in range(MAX_TASK_RETRIES):
+ try:
+ ingest_url_task.apply_async(
+ args=[payload],
+ task_id=task_id,
+ )
+ break
+ except OperationalError:
+ if attempt == MAX_TASK_RETRIES - 1:
+ set_ingestion_task_status(
+ task_id,
+ brain_id,
+ "failed",
+ stage="queue",
+ error="Task queue unavailable",
+ )
+ raise HTTPException(status_code=503, detail="Task queue unavailable")
+ await asyncio.sleep(RETRY_DELAY_BASE * (attempt + 1))
+
+ return JSONResponse(
+ status_code=202,
+ content={"message": "URL ingestion accepted", "task_id": task_id},
+ )
+
+
@ingest_router.post(
path="/file", status_code=202, response_model=IngestionAcceptedResponse
)
diff --git a/src/workers/tasks/ingestion.py b/src/workers/tasks/ingestion.py
index c14c864..22c58a4 100644
--- a/src/workers/tasks/ingestion.py
+++ b/src/workers/tasks/ingestion.py
@@ -1646,3 +1646,117 @@ def ingest_file(self, content_b64: str, filename: str, brain_id: str):
expires_in=3600 * 24 * 7,
)
return {"task_id": self.request.id, "task_ids": task_ids}
+
+
+def build_url_page_ingest_payload(
+ brain_id: str,
+ page,
+ skip_enrichment: bool = False,
+) -> dict:
+ meta_keys = {}
+ if getattr(page, "url", None):
+ meta_keys["source_url"] = page.url
+ if getattr(page, "title", None):
+ meta_keys["source_title"] = page.title
+ if getattr(page, "content_hash", None):
+ meta_keys["content_hash"] = page.content_hash
+ if getattr(page, "page_type", None):
+ meta_keys["page_type"] = page.page_type
+ payload = {
+ "data": IngestionTaskTextArgs(
+ data_type="text", text_data=page.text
+ ).model_dump(),
+ "brain_id": brain_id,
+ "skip_enrichment": skip_enrichment,
+ }
+ if meta_keys:
+ payload["meta_keys"] = meta_keys
+ return payload
+
+
+@ingestion_app.task(bind=True)
+def ingest_url(self, args: dict):
+ from celery.exceptions import OperationalError
+
+ from src.adapters.interfaces.crawler import CrawlRequest
+ from src.core.instances import crawler_adapter
+ from src.lib.crawler.errors import (
+ CrawlerConfigError,
+ CrawlerFailedError,
+ CrawlerQuotaError,
+ CrawlerTimeoutError,
+ )
+
+ brain_id = args.get("brain_id", "default")
+ skip_enrichment = bool(args.get("skip_enrichment", False))
+ try:
+ set_ingestion_task_status(
+ self.request.id,
+ brain_id,
+ "started",
+ stage="crawling",
+ )
+ pages = crawler_adapter.crawl_pages(
+ CrawlRequest(
+ url=args["url"],
+ max_pages=int(args.get("max_pages") or 10),
+ allowed_domains=args.get("allowed_domains") or [],
+ idempotency_key=self.request.id,
+ )
+ )
+ ingestible = [page for page in pages if (page.text or "").strip()]
+ task_ids = [str(uuid4()) for _ in ingestible]
+ for page_task_id in task_ids:
+ set_ingestion_task_status(
+ page_task_id,
+ brain_id,
+ "queued",
+ stage="queued",
+ )
+ for page_task_id, page in zip(task_ids, ingestible):
+ payload = build_url_page_ingest_payload(
+ brain_id,
+ page,
+ skip_enrichment=skip_enrichment,
+ )
+ for attempt in range(FILE_INGEST_MAX_RETRIES):
+ try:
+ ingest_data.apply_async(
+ args=[payload],
+ task_id=page_task_id,
+ )
+ break
+ except OperationalError:
+ if attempt == FILE_INGEST_MAX_RETRIES - 1:
+ raise
+ time.sleep(FILE_INGEST_RETRY_DELAY * (attempt + 1))
+ set_ingestion_task_status(
+ self.request.id,
+ brain_id,
+ "completed",
+ counts={"pages": len(task_ids), "task_ids": task_ids},
+ )
+ return {"task_id": self.request.id, "task_ids": task_ids}
+ except (
+ CrawlerConfigError,
+ CrawlerQuotaError,
+ CrawlerFailedError,
+ CrawlerTimeoutError,
+ ) as exc:
+ set_ingestion_task_status(
+ self.request.id,
+ brain_id,
+ "failed",
+ stage="failed",
+ error=str(exc),
+ )
+ raise
+ except Exception as exc:
+ set_ingestion_task_status(
+ self.request.id,
+ brain_id,
+ "failed",
+ stage="failed",
+ error=str(exc),
+ )
+ raise
diff --git a/tests/test_agent_readiness_api.py b/tests/test_agent_readiness_api.py
index dd27595..a723367 100644
--- a/tests/test_agent_readiness_api.py
+++ b/tests/test_agent_readiness_api.py
@@ -333,6 +333,11 @@ class OpenApiContractTests(unittest.TestCase):
self.assertIn("/system/brains/{brain_id}", schema["paths"])
self.assertIn("delete", schema["paths"]["/system/brains/{brain_id}"])
self.assertIn("202", schema["paths"]["/ingest/file"]["post"]["responses"])
+ self.assertIn("202", schema["paths"]["/ingest/url"]["post"]["responses"])
+ self.assertIn(
+ "503",
+ schema["paths"]["/ingest/url"]["post"]["responses"],
+ )
generic_refs = [
response["content"]["application/json"]["schema"].get("$ref", "")
diff --git a/tests/test_crawler_lumen.py b/tests/test_crawler_lumen.py
new file mode 100644
index 0000000..d359bc3
--- /dev/null
+++ b/tests/test_crawler_lumen.py
@@ -0,0 +1,279 @@
+from __future__ import annotations
+
+import json
+
+import httpx
+import pytest
+
+from src.adapters.crawler import CrawlerAdapter
+from src.adapters.interfaces.crawler import CrawlRequest, CrawledPage
+from src.lib.crawler.cleaned import format_cleaned_page
+from src.lib.crawler.client_lumen import LumenCrawlerClient
+from src.lib.crawler.errors import (
+ CrawlerConfigError,
+ CrawlerFailedError,
+ CrawlerQuotaError,
+ CrawlerTimeoutError,
+)
+
+CRAWL_ID = "crawl-1"
+CLEANED_PAGE = {
+ "url": "https://example.com/",
+ "canonical_url": "https://example.com/",
+ "page_type": "landing",
+ "chrome": {"navigation": ["Home", "Pricing"], "footer": ["Login"], "forms": []},
+ "content": {
+ "title": "Home",
+ "headings": [{"level": 1, "text": "Home"}, {"level": 2, "text": "Features"}],
+ "core": "Welcome to Example.",
+ "paragraphs": ["Welcome to Example.", "Home Pricing Login"],
+ },
+ "areas": [
+ {
+ "id": "main",
+ "heading": "Home",
+ "category": "intro",
+ "markdown": "# Home\n\n" + ("huge markdown " * 200),
+ }
+ ],
+ "facts": {
+ "contacts": {
+ "emails": ["[email protected]"],
+ "phones": [],
+ "addresses": [],
+ },
+ "ctas": ["Sign up"],
+ },
+ "model": "mock",
+ "input_tokens": 1,
+ "output_tokens": 1,
+}
+
+
+def _request() -> CrawlRequest:
+ return CrawlRequest(
+ url="https://example.com",
+ max_pages=10,
+ allowed_domains=[],
+ idempotency_key="task-1",
+ )
+
+
+class FakeClock:
+ def __init__(self):
+ self.now = 0.0
+ self.sleeps: list[float] = []
+
+ def monotonic(self) -> float:
+ return self.now
+
+ def sleep(self, seconds: float) -> None:
+ self.sleeps.append(seconds)
+ self.now += seconds
+
+
+def _json(request: httpx.Request, status: int, body: dict, headers: dict | None = None) -> httpx.Response:
+ return httpx.Response(
+ status,
+ request=request,
+ headers=headers or {"content-type": "application/json"},
+ content=json.dumps(body).encode("utf-8"),
+ )
+
+
+def test_format_cleaned_page_keeps_core_fields_and_drops_chrome_markdown():
+ text = format_cleaned_page(CLEANED_PAGE)
+ assert "Title: Home" in text
+ assert "Type: landing" in text
+ assert "URL: https://example.com/" in text
+ assert "H1 Home" in text
+ assert "Welcome to Example." in text
+ assert "email: [email protected]" in text
+ assert "Sign up" in text
+ assert "Pricing" not in text
+ assert "huge markdown" not in text
+ assert "Home Pricing Login" not in text
+
+
+def test_missing_api_key_raises_before_http():
+ calls = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request)
+ return httpx.Response(500, request=request)
+
+ client = LumenCrawlerClient(
+ api_key="",
+ transport=httpx.MockTransport(handler),
+ )
+ with pytest.raises(CrawlerConfigError, match="LUMEN_CRAWLER_KEY"):
+ client.crawl_pages(_request())
+ assert calls == []
+
+
+def test_create_poll_and_cleaned_pages():
+ polls = {"n": 0}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ path = request.url.path
+ if request.method == "POST" and path == "/v1/crawls":
+ body = json.loads(request.content.decode("utf-8"))
+ assert body["cleanup"] is True
+ assert body["follow_doors"] is False
+ assert body["max_pages"] == 10
+ assert request.headers["idempotency-key"] == "task-1"
+ return _json(
+ request,
+ 202,
+ {
+ "id": CRAWL_ID,
+ "status": "queued",
+ "cleanup": {"status": "pending"},
+ },
+ )
+ if request.method == "GET" and path == f"/v1/crawls/{CRAWL_ID}":
+ polls["n"] += 1
+ if polls["n"] == 1:
+ return _json(
+ request,
+ 200,
+ {"id": CRAWL_ID, "status": "running", "cleanup": {"status": "pending"}},
+ )
+ return _json(
+ request,
+ 200,
+ {
+ "id": CRAWL_ID,
+ "status": "completed",
+ "cleanup": {"status": "completed"},
+ },
+ )
+ if request.method == "GET" and path == f"/v1/crawls/{CRAWL_ID}/pages":
+ assert request.url.params.get("include") == "cleaned"
+ return _json(
+ request,
+ 200,
+ {
+ "items": [
+ {
+ "url": "https://example.com/",
+ "canonical_url": "https://example.com/",
+ "title": "Home",
+ "status_code": 200,
+ "ok": True,
+ "content_hash": "abc",
+ "cleaned": CLEANED_PAGE,
+ },
+ {
+ "url": "https://example.com/robots",
+ "ok": False,
+ "cleaned": CLEANED_PAGE,
+ },
+ ],
+ "next_cursor": None,
+ },
+ )
+ return httpx.Response(404, request=request)
+
+ clock = FakeClock()
+ client = LumenCrawlerClient(
+ api_key="lc_live_test",
+ transport=httpx.MockTransport(handler),
+ sleep=clock.sleep,
+ monotonic=clock.monotonic,
+ )
+ pages = client.crawl_pages(_request())
+ assert len(pages) == 1
+ assert pages[0].url == "https://example.com/"
+ assert pages[0].title == "Home"
+ assert pages[0].page_type == "landing"
+ assert "Welcome to Example." in pages[0].text
+ assert "huge markdown" not in pages[0].text
+ assert polls["n"] == 2
+
+
+def test_quota_error_is_not_retried():
+ def handler(request: httpx.Request) -> httpx.Response:
+ return _json(
+ request,
+ 402,
+ {
+ "type": "quota_exceeded",
+ "title": "Quota exceeded",
+ "status": 402,
+ "detail": "max_pages does not fit remaining included pages",
+ },
+ )
+
+ client = LumenCrawlerClient(
+ api_key="lc_live_test",
+ transport=httpx.MockTransport(handler),
+ )
+ with pytest.raises(CrawlerQuotaError, match="remaining included pages"):
+ client.crawl_pages(_request())
+
+
+def test_failed_crawl_raises():
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "POST":
+ return _json(request, 202, {"id": CRAWL_ID, "status": "queued"})
+ return _json(
+ request,
+ 200,
+ {
+ "id": CRAWL_ID,
+ "status": "failed",
+ "stop_reason": "seed_failed",
+ "cleanup": {"status": "skipped"},
+ },
+ )
+
+ client = LumenCrawlerClient(
+ api_key="lc_live_test",
+ transport=httpx.MockTransport(handler),
+ sleep=lambda _: None,
+ )
+ with pytest.raises(CrawlerFailedError, match="seed_failed"):
+ client.crawl_pages(_request())
+
+
+def test_timeout_cancels_running_crawl():
+ cancelled = {"n": 0}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.method == "POST" and request.url.path.endswith("/cancel"):
+ cancelled["n"] += 1
+ return _json(request, 200, {"id": CRAWL_ID, "status": "cancelled"})
+ if request.method == "POST":
+ return _json(request, 202, {"id": CRAWL_ID, "status": "queued"})
+ return _json(
+ request,
+ 200,
+ {"id": CRAWL_ID, "status": "running", "cleanup": {"status": "pending"}},
+ )
+
+ clock = FakeClock()
+ client = LumenCrawlerClient(
+ api_key="lc_live_test",
+ transport=httpx.MockTransport(handler),
+ poll_timeout_seconds=0,
+ sleep=clock.sleep,
+ monotonic=clock.monotonic,
+ )
+ with pytest.raises(CrawlerTimeoutError):
+ client.crawl_pages(_request())
+ assert cancelled["n"] == 1
+
+
+def test_adapter_delegates_crawl_pages():
+ adapter = CrawlerAdapter()
+ page = CrawledPage(url="https://example.com/", text="Title: Home")
+
+ class FakeClient:
+ def crawl_pages(self, request: CrawlRequest) -> list[CrawledPage]:
+ assert request.url == "https://example.com"
+ return [page]
+
+ adapter.add_client(FakeClient())
+ result = adapter.crawl_pages(_request())
+ assert result == [page]
diff --git a/tests/test_ingest_url.py b/tests/test_ingest_url.py
new file mode 100644
index 0000000..b300579
--- /dev/null
+++ b/tests/test_ingest_url.py
@@ -0,0 +1,276 @@
+import os
+import unittest
+from unittest.mock import MagicMock, patch
+
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from pydantic import ValidationError
+
+from src.adapters.interfaces.crawler import CrawledPage
+from src.lib.crawler.errors import CrawlerQuotaError, CrawlerTimeoutError
+from src.services.api.constants.requests import IngestionUrlRequestBody
+
+
+ENV_DEFAULTS = {
+ "BRAINPAT_TOKEN": "test-token",
+ "MODELS_MODE": "local",
+ "EMBEDDINGS_LOCAL_MODEL": "local-model",
+ "EMBEDDINGS_SMALL_MODEL": "small-model",
+ "EMBEDDING_NODES_DIMENSION": "3",
+ "EMBEDDING_TRIPLETS_DIMENSION": "3",
+ "EMBEDDING_OBSERVATIONS_DIMENSION": "3",
+ "EMBEDDING_DATA_DIMENSION": "3",
+ "EMBEDDING_RELATIONSHIPS_DIMENSION": "3",
+ "REDIS_HOST": "localhost",
+ "REDIS_PORT": "6379",
+ "NEO4J_HOST": "localhost",
+ "NEO4J_PORT": "7687",
+ "NEO4J_USERNAME": "neo4j",
+ "NEO4J_PASSWORD": "password",
+ "MILVUS_HOST": "localhost",
+ "MILVUS_PORT": "19530",
+ "MONGO_CONNECTION_STRING": "mongodb://localhost:27017",
+ "CELERY_WORKER_CONCURRENCY": "1",
+ "OLLAMA_HOST": "localhost",
+ "OLLAMA_PORT": "11434",
+ "OLLAMA_LLM_SMALL_MODEL": "small",
+ "OLLAMA_LLM_LARGE_MODEL": "large",
+ "PIPELINE_MODE": "accurate",
+}
+for key, value in ENV_DEFAULTS.items():
+ os.environ.setdefault(key, value)
+
+
+PAGE = CrawledPage(
+ url="https://example.com/",
+ canonical_url="https://example.com/",
+ title="Home",
+ page_type="landing",
+ text="Title: Home\n\nWelcome to Example.",
+ content_hash="abc",
+ status_code=200,
+)
+
+
+class IngestionUrlRequestTests(unittest.TestCase):
+ def test_accepts_https_and_defaults(self):
+ body = IngestionUrlRequestBody(url="https://example.com/docs")
+ self.assertEqual(body.max_pages, 10)
+ self.assertEqual(body.allowed_domains, [])
+ self.assertFalse(body.skip_enrichment)
+
+ def test_rejects_http_and_invalid_bounds(self):
+ with self.assertRaises(ValidationError):
+ IngestionUrlRequestBody(url="http://example.com")
+ with self.assertRaises(ValidationError):
+ IngestionUrlRequestBody(url="https://example.com", max_pages=0)
+ with self.assertRaises(ValidationError):
+ IngestionUrlRequestBody(url="https://example.com", max_pages=101)
+ with self.assertRaises(ValidationError):
+ IngestionUrlRequestBody(
+ url="https://example.com",
+ allowed_domains=[f"host{i}.com" for i in range(21)],
+ )
+
+
+class IngestUrlRouteTests(unittest.TestCase):
+ def _client(self):
+ from src.services.api.dependencies import get_brain_id
+ from src.services.api.routes.ingest import ingest_router
+
+ app = FastAPI()
+ app.dependency_overrides[get_brain_id] = lambda: "brain-a"
+ app.include_router(ingest_router)
+ return TestClient(app)
+
+ def test_valid_request_enqueues_task(self):
+ captured = {}
+
+ def fake_apply_async(args=None, task_id=None):
+ captured["args"] = args
+ captured["task_id"] = task_id
+ return MagicMock()
+
+ client = self._client()
+ with patch(
+ "src.services.api.routes.ingest.config"
+ ) as config, patch(
+ "src.services.api.routes.ingest.set_ingestion_task_status",
+ return_value={},
+ ), patch(
+ "src.services.api.routes.ingest.ingest_url_task.apply_async",
+ side_effect=fake_apply_async,
+ ):
+ config.crawler_is_configured = True
+ response = client.post(
+ "/ingest/url",
+ json={"url": "https://example.com", "max_pages": 3},
+ )
+ self.assertEqual(response.status_code, 202)
+ self.assertEqual(response.json()["message"], "URL ingestion accepted")
+ self.assertTrue(response.json()["task_id"])
+ self.assertEqual(captured["args"][0]["brain_id"], "brain-a")
+ self.assertEqual(captured["args"][0]["url"], "https://example.com")
+ self.assertEqual(captured["args"][0]["max_pages"], 3)
+
+ def test_missing_crawler_key_is_503_without_enqueue(self):
+ apply_async = MagicMock()
+ client = self._client()
+ with patch(
+ "src.services.api.routes.ingest.config"
+ ) as config, patch(
+ "src.services.api.routes.ingest.ingest_url_task.apply_async",
+ apply_async,
+ ):
+ config.crawler_is_configured = False
+ response = client.post(
+ "/ingest/url",
+ json={"url": "https://example.com"},
+ )
+ self.assertEqual(response.status_code, 503)
+ apply_async.assert_not_called()
+
+ def test_invalid_url_is_422(self):
+ client = self._client()
+ with patch(
+ "src.services.api.routes.ingest.config"
+ ) as config:
+ config.crawler_is_configured = True
+ response = client.post(
+ "/ingest/url",
+ json={"url": "http://localhost"},
+ )
+ self.assertEqual(response.status_code, 422)
+
+
+class IngestUrlTaskTests(unittest.TestCase):
+ def test_fans_out_cleaned_pages_with_source_url(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "url-parent"
+ enqueued = []
+ statuses = []
+
+ def fake_apply_async(args=None, task_id=None):
+ enqueued.append({"args": args, "task_id": task_id})
+ return MagicMock()
+
+ def fake_set_status(task_id, brain_id, status, **kwargs):
+ statuses.append((task_id, status, kwargs.get("stage")))
+ return {"status": status, "task_id": task_id}
+
+ crawler = MagicMock()
+ crawler.crawl_pages.return_value = [
+ PAGE,
+ CrawledPage(url="https://example.com/empty", text=" "),
+ ]
+
+ with patch(
+ "src.core.instances.crawler_adapter", crawler
+ ), patch.object(
+ ingestion_mod, "ingest_data"
+ ) as ingest_data, patch.object(
+ ingestion_mod, "set_ingestion_task_status", side_effect=fake_set_status
+ ):
+ ingest_data.apply_async.side_effect = fake_apply_async
+ result = ingestion_mod.ingest_url.run.__func__(
+ type("Bound", (), {"request": request})(),
+ {
+ "url": "https://example.com",
+ "max_pages": 10,
+ "brain_id": "brain-a",
+ "skip_enrichment": True,
+ },
+ )
+
+ self.assertEqual(result["task_id"], "url-parent")
+ self.assertEqual(len(result["task_ids"]), 1)
+ payload = enqueued[0]["args"][0]
+ self.assertEqual(payload["meta_keys"]["source_url"], "https://example.com/")
+ self.assertEqual(payload["meta_keys"]["source_title"], "Home")
+ self.assertEqual(payload["skip_enrichment"], True)
+ self.assertIn("Welcome to Example.", payload["data"]["text_data"])
+ self.assertEqual(statuses[-1][:2], ("url-parent", "completed"))
+ crawl_request = crawler.crawl_pages.call_args.args[0]
+ self.assertEqual(crawl_request.idempotency_key, "url-parent")
+
+ def test_empty_crawl_completes_with_zero_pages(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "url-empty"
+ crawler = MagicMock()
+ crawler.crawl_pages.return_value = []
+
+ with patch(
+ "src.core.instances.crawler_adapter", crawler
+ ), patch.object(
+ ingestion_mod, "ingest_data"
+ ) as ingest_data, patch.object(
+ ingestion_mod, "set_ingestion_task_status", return_value={}
+ ):
+ result = ingestion_mod.ingest_url.run.__func__(
+ type("Bound", (), {"request": request})(),
+ {"url": "https://example.com", "brain_id": "brain-a"},
+ )
+
+ self.assertEqual(result["task_ids"], [])
+ ingest_data.apply_async.assert_not_called()
+
+ def test_quota_failure_marks_parent_failed(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "url-quota"
+ statuses = []
+ crawler = MagicMock()
+ crawler.crawl_pages.side_effect = CrawlerQuotaError("max_pages does not fit")
+
+ def fake_set_status(task_id, brain_id, status, **kwargs):
+ statuses.append((status, kwargs.get("error")))
+ return {"status": status, "task_id": task_id}
+
+ with patch(
+ "src.core.instances.crawler_adapter", crawler
+ ), patch.object(
+ ingestion_mod, "set_ingestion_task_status", side_effect=fake_set_status
+ ):
+ with self.assertRaises(CrawlerQuotaError):
+ ingestion_mod.ingest_url.run.__func__(
+ type("Bound", (), {"request": request})(),
+ {"url": "https://example.com", "brain_id": "brain-a"},
+ )
+
+ self.assertEqual(statuses[-1][0], "failed")
+ self.assertIn("max_pages", statuses[-1][1])
+
+ def test_timeout_failure_marks_parent_failed(self):
+ from src.workers.tasks import ingestion as ingestion_mod
+
+ request = MagicMock()
+ request.id = "url-timeout"
+ statuses = []
+ crawler = MagicMock()
+ crawler.crawl_pages.side_effect = CrawlerTimeoutError("Timed out waiting")
+
+ def fake_set_status(task_id, brain_id, status, **kwargs):
+ statuses.append(status)
+ return {"status": status, "task_id": task_id}
+
+ with patch(
+ "src.core.instances.crawler_adapter", crawler
+ ), patch.object(
+ ingestion_mod, "set_ingestion_task_status", side_effect=fake_set_status
+ ):
+ with self.assertRaises(CrawlerTimeoutError):
+ ingestion_mod.ingest_url.run.__func__(
+ type("Bound", (), {"request": request})(),
+ {"url": "https://example.com", "brain_id": "brain-a"},
+ )
+
+ self.assertEqual(statuses[-1], "failed")
+
+
+if __name__ == "__main__":
+ unittest.main()