Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
13.9 KB
Raw Blame
"""Isolated Scrapy runner for a single managed URL source crawl.

This module is intentionally executed as a subprocess by ``crawler.py``. It
keeps Twisted out of the FastAPI process while retaining the existing host-side
scope and public-network policy for every request and redirect.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit

import scrapy
from scrapy import Request
from scrapy.crawler import CrawlerProcess
from scrapy.exceptions import IgnoreRequest
from scrapy.linkextractors import LinkExtractor
from scrapy.resolver import CachingThreadedResolver
from twisted.internet import defer
from twisted.internet.error import DNSLookupError

from .policy import (
    CrawlPolicyError,
    CrawlSpec,
    canonicalize_url,
    normalize_host,
    resolve_public_addresses,
)

USER_AGENT = "OpenRAG URL connector (+https://openrag.ai)"

# One runner process owns one crawl. Bind each hostname to the public addresses
# approved by the downloader middleware so Scrapy cannot re-resolve it later.
_validated_public_addresses: dict[str, tuple[str, ...]] = {}


class ValidatedAddressResolver(CachingThreadedResolver):
    """Resolve a host only to addresses approved by the policy middleware."""

    def getHostByName(self, name: str, timeout=()):  # noqa: N802 - Twisted API
        addresses = _validated_public_addresses.get(normalize_host(name))
        if not addresses:
            return defer.fail(DNSLookupError(f"no validated public address for {name}"))
        return defer.succeed(addresses[0])


class CrawlPolicyDownloaderMiddleware:
    """Reject private, out-of-scope, or redirected requests before download."""

    def process_request(self, request: Request, spider: ManagedWebsiteSpider) -> None:
        try:
            url = canonicalize_url(request.url)
            if not spider.allows_request(
                url, is_robots=request.meta.get("dont_obey_robotstxt", False)
            ):
                raise CrawlPolicyError("URL is not within the allowed crawl scope")
            parsed = urlsplit(url)
            addresses = resolve_public_addresses(
                parsed.hostname or "", parsed.port or (443 if parsed.scheme == "https" else 80)
            )
            _validated_public_addresses[normalize_host(parsed.hostname or "")] = addresses
        except CrawlPolicyError as exc:
            raise IgnoreRequest(str(exc)) from exc

    def process_response(
        self, request: Request, response: scrapy.http.Response, spider: ManagedWebsiteSpider
    ) -> scrapy.http.Response:
        if not spider.count_downloaded_response(response):
            raise IgnoreRequest("aggregate download limit reached")
        return response

    def process_exception(
        self,
        request: Request,
        exception: Exception,
        spider: ManagedWebsiteSpider,
    ) -> None:
        """Persist request rejections that Scrapy otherwise drops silently.

        Both this middleware and Scrapy's robots middleware use ``IgnoreRequest``.
        Those failures bypass a request errback, so record the source request here
        to preserve the actual failure reason for the task and source projection.
        """
        if isinstance(exception, IgnoreRequest):
            if request.meta.get("dont_obey_robotstxt", False):
                spider.mark_incomplete(str(exception))
                return
            spider.request_rejected(request, str(exception))


class ManagedWebsiteSpider(scrapy.Spider):
    name = "openrag_managed_website"

    def __init__(self, *, spec: dict[str, Any], output_dir: str, **kwargs: Any) -> None:
        super().__init__(**kwargs)
        self.spec = CrawlSpec(**spec)
        self.output_dir = Path(output_dir)
        self.pages: list[dict[str, Any]] = []
        self._recorded: set[str] = set()
        self._scheduled: set[str] = set()
        self.downloaded_bytes = 0
        self.capped = False
        self.incomplete = False
        self.reason: str | None = None
        self._links = LinkExtractor(tags=("a",), attrs=("href",), canonicalize=False)

    def allows_request(self, url: str, *, is_robots: bool) -> bool:
        if self.spec.allows(url):
            return True
        if not is_robots or urlsplit(url).path != "/robots.txt":
            return False
        host = normalize_host(urlsplit(url).hostname or "")
        seed_host = normalize_host(urlsplit(self.spec.seed_url).hostname or "")
        return (
            host == seed_host
            or host in self.spec.additional_hosts
            or (self.spec.allow_subdomains and host.endswith("." + seed_host))
        )

    def _stop_for_limit(self, reason: str) -> None:
        self.capped = True
        self.reason = self._limit_reason(reason)
        self.crawler.engine.close_spider(self, reason)

    def _mark_frontier_capped(self) -> None:
        """Stop growing the frontier while allowing accepted requests to finish."""
        self.capped = True
        self.reason = self._limit_reason("openrag_page_limit")

    def _limit_reason(self, reason: str) -> str:
        if reason == "openrag_page_limit":
            return f"Website crawl reached its configured limit of {self.spec.max_pages} pages."
        if reason == "openrag_download_limit":
            return (
                "Website crawl reached its configured download limit of "
                f"{self.spec.max_downloaded_mb} MB."
            )
        if reason == "closespider_timeout":
            return "Website crawl reached its configured time limit."
        return "Website crawl reached a configured safety limit."

    def mark_incomplete(self, reason: str) -> None:
        self.incomplete = True
        self.reason = self.reason or reason

    def count_downloaded_response(self, response: scrapy.http.Response) -> bool:
        self.downloaded_bytes += len(response.body)
        if self.downloaded_bytes > self.spec.max_downloaded_mb * 1024 * 1024:
            self._stop_for_limit("openrag_download_limit")
            return False
        return True

    def _record(
        self,
        canonical_url: str,
        final_url: str,
        depth: int,
        *,
        noindex: bool = False,
        error: str | None = None,
        content: bytes | None = None,
        encoding: str | None = None,
    ) -> None:
        if canonical_url in self._recorded:
            return
        self._recorded.add(canonical_url)
        record: dict[str, Any] = {
            "canonical_url": canonical_url,
            "final_url": final_url,
            "depth": depth,
            "noindex": noindex,
            "error": error,
        }
        if content is not None:
            pages_dir = self.output_dir / "pages"
            pages_dir.mkdir(parents=True, exist_ok=True)
            relative_path = Path("pages") / f"{len(self.pages):06d}.html"
            (self.output_dir / relative_path).write_bytes(content)
            record["content_path"] = str(relative_path)
            if encoding:
                record["encoding"] = encoding
        self.pages.append(record)

    def request_rejected(self, request: Request, error: str) -> None:
        self.mark_incomplete(error)
        canonical_url = str(request.meta.get("canonical_url", request.url))
        depth = int(request.meta.get("crawl_depth", 0))
        self._record(canonical_url, canonical_url, depth, error=error)

    def _request(self, value: str, depth: int) -> Request | None:
        if depth > self.spec.max_depth:
            return None
        try:
            canonical_url = canonicalize_url(value)
        except CrawlPolicyError:
            return None
        if canonical_url in self._scheduled or not self.spec.allows(canonical_url):
            return None
        if len(self._scheduled) >= self.spec.max_pages:
            self._mark_frontier_capped()
            return None
        self._scheduled.add(canonical_url)
        return Request(
            canonical_url,
            callback=self.parse_page,
            errback=self.request_failed,
            dont_filter=True,
            meta={"canonical_url": canonical_url, "crawl_depth": depth},
        )

    async def start(self):
        request = self._request(self.spec.seed_url, 0)
        if request is not None:
            yield request

    def parse_page(self, response: scrapy.http.Response):
        canonical_url = str(response.meta["canonical_url"])
        depth = int(response.meta["crawl_depth"])
        try:
            final_url = canonicalize_url(response.url)
        except CrawlPolicyError as exc:
            self.mark_incomplete(str(exc))
            self._record(canonical_url, canonical_url, depth, error=str(exc))
            return
        if not self.spec.allows(final_url):
            self.mark_incomplete("redirect left the approved crawl scope")
            self._record(
                canonical_url, final_url, depth, error="redirect left the approved crawl scope"
            )
            return
        if len(self._recorded) >= self.spec.max_pages:
            self._stop_for_limit("openrag_page_limit")
            return

        content_type = response.headers.get(b"Content-Type", b"").decode("latin1").lower()
        if not 200 <= response.status < 300:
            self.mark_incomplete(f"HTTP {response.status}")
            self._record(canonical_url, final_url, depth, error=f"HTTP {response.status}")
            return
        if not content_type.startswith(("text/html", "application/xhtml+xml")):
            self.mark_incomplete("Unsupported content type")
            self._record(canonical_url, final_url, depth, error="Unsupported content type")
            return
        if not isinstance(response, scrapy.http.TextResponse):
            self._record(canonical_url, final_url, depth, content=response.body)
            return

        directives = self._robot_directives(response)
        noindex, nofollow = "noindex" in directives, "nofollow" in directives
        self._record(
            canonical_url,
            final_url,
            depth,
            noindex=noindex,
            content=response.body,
            encoding=response.encoding,
        )

        if nofollow:
            # A full crawl that intentionally stops discovery cannot prove
            # previously known children disappeared. Keep reconciliation
            # non-destructive for this run.
            self.mark_incomplete("nofollow directive prevented link discovery")
            return
        if self.spec.scope == "page":
            return
        for link in self._links.extract_links(response):
            request = self._request(link.url, depth + 1)
            if request is not None:
                yield request
            if self.capped:
                return

    @staticmethod
    def _robot_directives(response: scrapy.http.TextResponse) -> set[str]:
        meta_values = response.xpath(
            "//meta[translate(@name, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', "
            "'abcdefghijklmnopqrstuvwxyz')='robots']/@content"
        ).getall()
        header_values = [
            value.decode("latin1") for value in response.headers.getlist(b"X-Robots-Tag")
        ]
        directives: set[str] = set()
        for value in (*meta_values, *header_values):
            # A header can be agent-targeted (for example, "googlebot: noindex").
            # For OpenRAG, honor the directives that follow the optional target.
            directives.update(re.findall(r"[a-z-]+", value.lower().split(":")[-1]))
        if "none" in directives:
            directives.update({"noindex", "nofollow"})
        return directives

    def request_failed(self, failure: Any) -> None:
        request = failure.request
        if request.meta.get("dont_obey_robotstxt", False):
            self.mark_incomplete(failure.getErrorMessage())
            return
        self.request_rejected(request, failure.getErrorMessage())

    def closed(self, reason: str) -> None:
        if reason in {"closespider_timeout", "openrag_page_limit", "openrag_download_limit"}:
            self.capped = True
            self.reason = self.reason or self._limit_reason(reason)
        manifest = {
            "pages": self.pages,
            "complete": not self.capped and not self.incomplete,
            "capped": self.capped,
            "reason": self.reason,
        }
        temporary = self.output_dir / "result.tmp"
        temporary.write_text(json.dumps(manifest), encoding="utf-8")
        temporary.replace(self.output_dir / "result.json")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--output-dir", required=True)
    args = parser.parse_args()
    spec = CrawlSpec(**json.loads(sys.stdin.read())).as_dict()
    _validated_public_addresses.clear()
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)
    max_bytes = int(spec["max_downloaded_mb"]) * 1024 * 1024
    settings = {
        "BOT_NAME": "openrag_url_connector",
        "COOKIES_ENABLED": False,
        "CLOSESPIDER_TIMEOUT": int(spec["max_crawl_minutes"]) * 60,
        "CONCURRENT_REQUESTS": 1,
        "CONCURRENT_REQUESTS_PER_DOMAIN": 1,
        "DEPTH_LIMIT": int(spec["max_depth"]),
        "DOWNLOAD_MAXSIZE": max_bytes,
        "DOWNLOAD_TIMEOUT": 20,
        "DNS_RESOLVER": "connectors.url.scrapy_runner.ValidatedAddressResolver",
        "HTTPERROR_ALLOW_ALL": True,
        "HTTPPROXY_ENABLED": False,
        "LOG_ENABLED": False,
        "REDIRECT_MAX_TIMES": 5,
        "RETRY_ENABLED": False,
        "ROBOTSTXT_OBEY": True,
        "TELNETCONSOLE_ENABLED": False,
        "USER_AGENT": USER_AGENT,
        "DOWNLOADER_MIDDLEWARES": {
            # Run before RedirectMiddleware on the response path so redirects
            # count towards the aggregate download limit too.
            "connectors.url.scrapy_runner.CrawlPolicyDownloaderMiddleware": 700,
        },
    }
    process = CrawlerProcess(settings=settings)
    process.crawl(ManagedWebsiteSpider, spec=spec, output_dir=str(output_dir))
    process.start(stop_after_crawl=True)


if __name__ == "__main__":
    main()