Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

6.7 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Scrapy-backed recursive web crawler."""

import asyncio
import multiprocessing
import os
import queue
from contextlib import contextmanager
from typing import Iterator, Mapping

from openviking.parse.accessors.web_crawler.config import CrawlConfig
from openviking.parse.accessors.web_crawler.models import (
    CrawledDownload,
    CrawledPage,
    CrawlResult,
)
from openviking.parse.network import (
    ParserNetworkContext,
    ParserNetworkOperation,
    overlay_parser_subprocess_env,
)


@contextmanager
def _temporary_environment(overlay: Mapping[str, str]) -> Iterator[None]:
    """Install child-only network variables without leaking across worker reuse."""

    previous = {name: os.environ.get(name) for name in overlay}
    os.environ.update(overlay)
    try:
        yield
    finally:
        for name, value in previous.items():
            if value is None:
                os.environ.pop(name, None)
            else:
                os.environ[name] = value


def _run_crawl_worker(
    root_url: str,
    config: CrawlConfig,
    result_queue,
    network_env: Mapping[str, str] | None = None,
) -> None:
    with _temporary_environment(network_env or {}):
        _run_crawl_worker_with_environment(root_url, config, result_queue, network_env or {})


def _run_crawl_worker_with_environment(
    root_url: str,
    config: CrawlConfig,
    result_queue,
    network_env: Mapping[str, str],
) -> None:
    from scrapy.utils.reactor import install_reactor

    install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

    from scrapy.crawler import CrawlerProcess

    from openviking.parse.accessors.web_crawler.scrapy_spider import OpenVikingWebSpider

    pages: list[CrawledPage] = []
    downloads: list[CrawledDownload] = []
    process = CrawlerProcess(_build_settings(config))
    crawler = process.create_crawler(OpenVikingWebSpider)
    process.crawl(
        crawler,
        root_url=root_url,
        config=config,
        collector=pages,
        download_collector=downloads,
        network_env=network_env,
    )
    try:
        process.start()
    except Exception as exc:
        result_queue.put((pages, downloads, str(exc)))
        return
    result_queue.put((pages, downloads, None))


def _build_settings(config: CrawlConfig):
    from scrapy.settings import Settings

    depth_limit = 0 if config.depth < 0 else config.depth
    return Settings(
        {
            "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
            "DEPTH_LIMIT": depth_limit,
            # max_pages is enforced inside the spider by successful-resource count
            # (OpenVikingWebSpider._success_at_limit). Scrapy's
            # CLOSESPIDER_PAGECOUNT counts every response, including failed and
            # skipped ones, so enabling it would stop the crawl on a different
            # metric and yield fewer successful pages than requested. Keep it
            # disabled so the spider is the single source of truth.
            "CLOSESPIDER_PAGECOUNT": 0,
            "CONCURRENT_REQUESTS": config.concurrency,
            "DOWNLOAD_TIMEOUT": config.timeout,
            "DOWNLOAD_DELAY": config.download_delay,
            "RETRY_ENABLED": True,
            "RETRY_TIMES": config.retry_times,
            "ROBOTSTXT_OBEY": True,
            "LOG_ENABLED": False,
            "TELNETCONSOLE_ENABLED": False,
            "USER_AGENT": "OpenViking/0.4 (+recursive-web-crawler)",
            "DOWNLOADER_MIDDLEWARES": {
                "openviking.parse.accessors.web_crawler.middlewares.RequestValidatorMiddleware": 50,
                "openviking.parse.accessors.web_crawler.middlewares.ParserNetworkProxyMiddleware": 100,
                "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,
            },
        }
    )


class ScrapyWebCrawler:
    def __init__(self, config: CrawlConfig) -> None:
        self.config = config

    async def crawl(self, root_url: str) -> CrawlResult:
        context = multiprocessing.get_context("spawn")
        result_queue = context.Queue()
        network_env = overlay_parser_subprocess_env(
            ParserNetworkContext(ParserNetworkOperation.WEB_CRAWL, root_url),
            {},
        )
        process = context.Process(
            target=_run_crawl_worker,
            args=(root_url, self.config, result_queue, network_env),
        )
        process.start()
        try:
            pages, downloads, error = await self._wait_worker_result(process, result_queue)
            if error:
                raise RuntimeError(error)
            if process.exitcode not in (0, None):
                raise RuntimeError(f"Scrapy crawler exited with code {process.exitcode}.")
            return self._build_result(pages, downloads)
        finally:
            await self._cleanup_worker(process, result_queue)

    @staticmethod
    async def _wait_worker_result(
        process,
        result_queue,
    ) -> tuple[list[CrawledPage], list[CrawledDownload], str | None]:
        while True:
            try:
                pages, downloads, error = await asyncio.to_thread(result_queue.get, True, 0.2)
                await asyncio.to_thread(process.join)
                return pages, downloads, error
            except queue.Empty:
                if process.is_alive():
                    continue
                await asyncio.to_thread(process.join)
                return ScrapyWebCrawler._read_worker_result(result_queue)

    @staticmethod
    def _read_worker_result(
        result_queue,
    ) -> tuple[list[CrawledPage], list[CrawledDownload], str | None]:
        try:
            return result_queue.get(True, 0.2)
        except queue.Empty:
            return [], [], "Scrapy crawler did not return a result."

    @staticmethod
    async def _cleanup_worker(process, result_queue) -> None:
        if process.is_alive():
            process.terminate()
            await asyncio.to_thread(process.join, 5.0)
            if process.is_alive():
                process.kill()
                await asyncio.to_thread(process.join, 5.0)
        try:
            result_queue.close()
            result_queue.join_thread()
        except Exception:
            pass

    @staticmethod
    def _build_result(pages: list[CrawledPage], downloads: list[CrawledDownload]) -> CrawlResult:
        result = CrawlResult(pages=pages, downloads=downloads)
        result.total_downloads = len(downloads)
        for page in pages:
            if page.status == "success":
                result.total_crawled += 1
            elif page.status == "skipped":
                result.total_skipped += 1
            else:
                result.total_failed += 1
        return result