OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
"""Task processor for one managed URL source crawl."""
from __future__ import annotations
import asyncio
import hashlib
import os
import tempfile
import time
import uuid
from datetime import UTC, datetime
from typing import Any
from models.processors import TaskProcessor
from models.tasks import FileTask, TaskStatus, UploadTask
from .crawler import CrawlResult, crawl
from .policy import CrawlSpec
from .projection import (
delete_page_chunks,
delete_source_chunks,
get_page_index_metadata,
get_page_manifest,
get_source_manifest,
list_page_manifests,
upsert_page_manifest,
upsert_source_manifest,
)
CRAWL_INGESTION_HEADROOM_SECONDS = 5 * 60
CRAWL_SHUTDOWN_GRACE_SECONDS = 30
LEGACY_DEFAULT_CRAWL_LIMITS = {
"max_pages": 250,
"max_depth": 4,
"max_downloaded_mb": 128,
"max_crawl_minutes": 15,
}
def _document_id(source_id: str, canonical_url: str) -> str:
return hashlib.sha256(f"{source_id}\0{canonical_url}".encode()).hexdigest()
def _crawl_settings(source: dict[str, Any]) -> dict[str, Any]:
"""Use the higher page cap for sources created with the old untouched defaults."""
settings = dict(source["crawl_settings"])
if all(settings.get(key) == value for key, value in LEGACY_DEFAULT_CRAWL_LIMITS.items()):
settings["max_pages"] = CrawlSpec.max_pages
source["crawl_settings"] = settings
return settings
class WebsiteSourceProcessor(TaskProcessor):
def __init__(
self,
*,
source_id: str,
owner_id: str,
jwt_token: str | None,
owner_name: str | None,
owner_email: str | None,
document_service,
models_service,
page_id: str | None = None,
source_manifest: dict[str, Any] | None = None,
ingestion_timeout: int | None = None,
):
super().__init__(document_service=document_service, models_service=models_service)
self.source_id = source_id
self.owner_id = owner_id
self.jwt_token = jwt_token
self.owner_name = owner_name
self.owner_email = owner_email
self.page_id = page_id
self.source_manifest = dict(source_manifest) if source_manifest is not None else None
self._active_source: dict[str, Any] | None = None
self.ingestion_timeout = ingestion_timeout
async def _source_is_active(self) -> dict[str, Any]:
source = await get_source_manifest(self.source_id)
if source is None or source.get("deleting"):
raise ValueError("Website source no longer exists")
self._active_source = source
return source
async def _finalize_unhandled_failure(
self,
reason: str,
upload_task: UploadTask,
file_task: FileTask,
) -> None:
"""Make a task failure retryable even when startup itself failed.
Normal crawl failures are finalised below. This covers failures before
that flow is established (manifest reads/writes and initial projection
writes), which TaskService cannot reconcile on its own.
"""
source = await get_source_manifest(self.source_id)
if source is None:
source = self._active_source or self.source_manifest
if source is not None and not source.get("deleting"):
source_is_established = bool(source.get("last_successful_sync_at"))
source.update(
status="active" if source_is_established else "failed",
last_error=reason,
updated_at=datetime.now(UTC).isoformat(),
)
try:
pages = await list_page_manifests(source["id"])
await upsert_source_manifest(source, pages=pages)
except Exception:
# OpenSearch may be the reason this finalisation path ran. A
# later sync request detects the terminal task and recovers it.
pass
file_task.status, file_task.error, file_task.result, file_task.updated_at = (
TaskStatus.FAILED,
reason,
{"indexed": 0, "pages": 0},
time.time(),
)
upload_task.failed_files += 1
async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
try:
await self._process_item(upload_task, item, file_task)
except asyncio.CancelledError:
await asyncio.shield(
self._finalize_unhandled_failure(
"Website crawl was cancelled.", upload_task, file_task
)
)
raise
except Exception as exc:
await self._finalize_unhandled_failure(
str(exc) or "Website crawl failed before it could start.",
upload_task,
file_task,
)
@staticmethod
def _new_page(source: dict[str, Any], page_id: str, canonical_url: str) -> dict[str, Any]:
now = datetime.now(UTC).isoformat()
return {
"web_source_id": source["id"],
"web_page_id": page_id,
"content_document_id": _document_id(source["id"], canonical_url),
"canonical_url": canonical_url,
"source_url": canonical_url,
"filename": canonical_url,
"mimetype": "text/html",
"file_size": 0,
"chunk_count": 0,
"status": "unavailable",
"suppressed_by_user": False,
"last_seen_at": now,
"updated_at": now,
}
async def _process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
source = await self._source_is_active()
target_page = (
await get_page_manifest(source["id"], self.page_id)
if self.page_id is not None
else None
)
if self.page_id is not None and target_page is None:
raise ValueError("Website page no longer exists")
pages = await list_page_manifests(source["id"])
pages_by_url = {str(page["canonical_url"]): page for page in pages}
source_is_established = bool(source.get("last_successful_sync_at")) or bool(pages)
started_at = datetime.now(UTC)
source.update(status="processing", last_error=None, updated_at=started_at.isoformat())
await upsert_source_manifest(source, pages=pages)
spec_values = _crawl_settings(source)
if target_page is not None:
spec_values.update(
seed_url=target_page["canonical_url"], scope="page", max_pages=1, max_depth=0
)
elif source_is_established and source.get("resync_behavior") == "root":
spec_values.update(scope="page", max_pages=1, max_depth=0)
if self.ingestion_timeout is not None:
# TaskService applies one timeout to crawling and ingestion. Keep a
# bounded portion for page ingestion after Scrapy returns, plus
# the crawler subprocess's shutdown grace period. On short task
# timeouts, a fixed five-minute reserve leaves an impractically
# small crawl window, so reserve at most one quarter instead.
crawl_ingestion_headroom = min(
CRAWL_INGESTION_HEADROOM_SECONDS,
max(60, self.ingestion_timeout // 4),
)
crawl_budget_seconds = (
self.ingestion_timeout - crawl_ingestion_headroom - CRAWL_SHUTDOWN_GRACE_SECONDS
)
if crawl_budget_seconds < 60:
raise ValueError("Task timeout is too short for a website crawl")
crawl_minutes = crawl_budget_seconds // 60
spec_values["max_crawl_minutes"] = min(
int(spec_values["max_crawl_minutes"]), crawl_minutes
)
try:
result = await crawl(CrawlSpec(**spec_values))
if not result.pages and not result.reason and not result.capped:
result = await crawl(CrawlSpec(**spec_values))
except Exception as exc:
result = CrawlResult(
(), complete=False, capped=False, reason=str(exc) or "Website crawl failed"
)
indexed = 0
successful_pages = 0
page_errors: list[str] = []
for outcome in result.pages:
page = pages_by_url.get(outcome.canonical_url)
if page is None:
page = self._new_page(source, str(uuid.uuid4()), outcome.canonical_url)
pages_by_url[outcome.canonical_url] = page
page_depth = outcome.depth
if (
target_page is not None
and page.get("web_page_id") == target_page.get("web_page_id")
and page.get("web_page_depth") is not None
):
# A page-scoped crawl starts at the selected page, so Scrapy
# reports depth 0. Keep the source-relative depth already
# recorded for that page instead of replacing it with 0.
page_depth = int(page["web_page_depth"])
page.setdefault(
"content_document_id", _document_id(source["id"], outcome.canonical_url)
)
now = datetime.now(UTC).isoformat()
page.update(
final_url=outcome.final_url,
source_url=outcome.final_url,
web_page_depth=page_depth,
canonical_url=outcome.canonical_url,
last_seen_at=now,
updated_at=now,
)
# Task failures are ephemeral, just like ordinary document uploads.
# Do not overwrite an existing page manifest with a transient failure.
if outcome.error:
page_errors.append(outcome.error)
continue
if outcome.document is None or outcome.noindex:
if outcome.noindex:
await delete_page_chunks(str(page["content_document_id"]))
page["chunk_count"] = 0
page["status"] = "unavailable"
await upsert_page_manifest(page)
continue
document = outcome.document
page.update(
filename=document.title,
file_size=document.byte_size,
mimetype="text/html",
)
if page.get("suppressed_by_user"):
page["status"] = "disabled"
await upsert_page_manifest(page)
successful_pages += 1
continue
if (
source.get("change_detection") == "normalized_content_hash"
and page.get("content_hash") == document.content_hash
and page.get("status") == "active"
and int(page.get("chunk_count") or 0) > 0
):
await upsert_page_manifest(page)
successful_pages += 1
continue
handle = tempfile.NamedTemporaryFile(
mode="w", suffix=".md", delete=False, encoding="utf-8"
)
try:
handle.write(document.markdown)
handle.close()
try:
processed = await self.process_document_standard(
file_path=handle.name,
file_hash=str(page["content_document_id"]),
document_id=str(page["content_document_id"]),
replace_existing=True,
owner_user_id=self.owner_id,
jwt_token=self.jwt_token,
owner_name=self.owner_name,
owner_email=self.owner_email,
file_size=document.byte_size,
original_filename=document.title,
connector_type="url",
source_url=outcome.final_url,
web_source_id=source["id"],
web_page_id=str(page["web_page_id"]),
web_page_depth=page_depth,
root_source_url=source["starting_url"],
canonical_url=outcome.canonical_url,
before_index_write=self._source_is_active,
)
except Exception as exc:
page_errors.append(str(exc) or "Website page ingestion failed")
continue
if processed.get("status") == "error":
page_errors.append(processed.get("error") or "Failed to ingest website page")
continue
embedding_metadata = await get_page_index_metadata(str(page["content_document_id"]))
page.update(
content_hash=document.content_hash,
chunk_count=processed.get("chunk_count", 0),
embedding_model=embedding_metadata.get("embedding_model", ""),
embedding_provider=embedding_metadata.get("embedding_provider", ""),
embedding_space_id=embedding_metadata.get("embedding_space_id", ""),
embedding_dimensions=embedding_metadata.get("embedding_dimensions"),
status="active",
last_ingested_at=datetime.now(UTC).isoformat(),
last_error=None,
)
await upsert_page_manifest(page)
indexed, successful_pages = indexed + 1, successful_pages + 1
finally:
try:
os.unlink(handle.name)
except FileNotFoundError:
pass
crawl_authoritative = result.complete and not result.capped and not page_errors
if target_page is None and source.get("resync_behavior") == "full" and crawl_authoritative:
for page in pages_by_url.values():
if (
page.get("suppressed_by_user")
or page.get("last_seen_at", "") >= started_at.isoformat()
):
continue
page["status"] = "unavailable"
page["updated_at"] = datetime.now(UTC).isoformat()
if source.get("removed_page_behavior") == "delete":
await delete_page_chunks(str(page["content_document_id"]))
page["chunk_count"] = 0
await upsert_page_manifest(page)
failure_reason = (
result.reason
or next(iter(page_errors), None)
or "No indexable website pages were found."
)
partial_failure = False
try:
source = await self._source_is_active()
except ValueError:
failure_reason = "Website source was deleted"
succeeded = False
else:
succeeded = successful_pages > 0
capped_with_progress = succeeded and result.capped and not page_errors
partial_failure = succeeded and not crawl_authoritative and not capped_with_progress
if not succeeded and target_page is not None:
final_page = pages_by_url.get(str(target_page["canonical_url"]), target_page)
if final_page.get("status") == "processing":
final_page.update(
status="failed",
last_error=failure_reason,
updated_at=datetime.now(UTC).isoformat(),
)
await upsert_page_manifest(final_page)
source["status"] = "active" if succeeded or source_is_established else "failed"
source["last_error"] = (
failure_reason
if partial_failure
else (None if source_is_established else (None if succeeded else failure_reason))
)
if capped_with_progress:
source["last_crawl_warning"] = failure_reason
else:
source.pop("last_crawl_warning", None)
if succeeded:
source["last_successful_sync_at"] = datetime.now(UTC).isoformat()
source["updated_at"] = datetime.now(UTC).isoformat()
if not succeeded and not source_is_established:
await delete_source_chunks(source["id"])
else:
await upsert_source_manifest(source, pages=await list_page_manifests(source["id"]))
if not succeeded or partial_failure:
file_task.status, file_task.error, file_task.result, file_task.updated_at = (
TaskStatus.FAILED,
failure_reason,
{"indexed": indexed, "pages": successful_pages},
time.time(),
)
upload_task.failed_files += 1
return
result_summary: dict[str, Any] = {"indexed": indexed, "pages": successful_pages}
if result.capped:
result_summary["warning"] = failure_reason
file_task.status, file_task.error, file_task.result, file_task.updated_at = (
TaskStatus.COMPLETED,
None,
result_summary,
time.time(),
)
upload_task.successful_files += 1