Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(web-crawler): remove Playwright rendering andoptimize the robots.txt entry-failure message (#3040)

zgy8d861fabfe5e
diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md
index 44e3e8c71..8a8f2bb4f 100644
--- a/docs/en/api/02-resources.md
+++ b/docs/en/api/02-resources.md
@@ -53,7 +53,7 @@ OpenViking supports various resource types, categorized by functionality:
| Type | Resource Name | Description |
|------|---------------|-------------|
-| Single page / recursive crawl | `https://host/path` | By default only the entry page is fetched. Set `args.depth > 0` to crawl same-host links breadth-first; `args.max_pages` only bounds how many pages are collected. Each page is extracted to Markdown with trafilatura, with an automatic Playwright fallback for SPAs whose static HTML carries no body text. Supported `args`: `depth`, `max_pages`, `include_paths`, `exclude_paths`, `allow_external_links`, `skip_download_links`. Download links discovered on pages are skipped by default (`skip_download_links=true`) to avoid importing sidecar files such as `llms.txt`; set it to `false` to download same-host file links and count them toward `max_pages`. `include_paths`/`exclude_paths` use **path-prefix** matching (e.g. `/docs/` matches only paths starting with `/docs/`, never substrings like `/blog/docs-tips`). |
+| Single page / recursive crawl | `https://host/path` | By default only the entry page is fetched. Set `args.depth > 0` to crawl same-host links breadth-first; `args.max_pages` only bounds how many pages are collected. Each page is extracted to Markdown with trafilatura. Supported `args`: `depth`, `max_pages`, `include_paths`, `exclude_paths`, `allow_external_links`, `skip_download_links`. Download links discovered on pages are skipped by default (`skip_download_links=true`) to avoid importing sidecar files such as `llms.txt`; set it to `false` to download same-host file links and count them toward `max_pages`. `include_paths`/`exclude_paths` use **path-prefix** matching (e.g. `/docs/` matches only paths starting with `/docs/`, never substrings like `/blog/docs-tips`). |
> Routing: sitemap-looking URLs (`https://host/sitemap.xml`, `https://host/feed.xml`, `*.atom`, ...) and explicit `args.site=true` are delegated to the whole-website ingestion below; Git hosting URLs such as `https://github.com/{org}/{repo}` are delegated to the Code section above.
diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md
index f5b66de84..5f70e8331 100644
--- a/docs/zh/api/02-resources.md
+++ b/docs/zh/api/02-resources.md
@@ -47,7 +47,7 @@ OpenViking 支持多种资源类型,按照功能分类如下:
网页类(递归网页爬虫)
| 类型 | 资源名 | 说明 |
|------|--------|------|
-| 单页 / 递归抓取 | `https://host/path` | 默认仅抓入口页;设置 `args.depth > 0` 后,沿同域链接 BFS 递归展开,`args.max_pages` 只限制最多收集的页面数。每页用 trafilatura 抽成 Markdown,对 SPA 等静态 HTML 拿不到正文的站点自动降级到 Playwright 渲染。可选 `args`:`depth`、`max_pages`、`include_paths`、`exclude_paths`、`allow_external_links`、`skip_download_links`。页面中发现的下载链接默认跳过(`skip_download_links=true`),避免导入 `llms.txt` 等 sidecar 文件造成重复;设为 `false` 时会下载同域文件链接,并计入 `max_pages`。`include_paths`/`exclude_paths` 按**路径前缀**匹配(例如 `/docs/` 仅匹配以 `/docs/` 开头的路径,不会误命中 `/blog/docs-tips`)。|
+| 单页 / 递归抓取 | `https://host/path` | 默认仅抓入口页;设置 `args.depth > 0` 后,沿同域链接 BFS 递归展开,`args.max_pages` 只限制最多收集的页面数。每页用 trafilatura 抽成 Markdown。可选 `args`:`depth`、`max_pages`、`include_paths`、`exclude_paths`、`allow_external_links`、`skip_download_links`。页面中发现的下载链接默认跳过(`skip_download_links=true`),避免导入 `llms.txt` 等 sidecar 文件造成重复;设为 `false` 时会下载同域文件链接,并计入 `max_pages`。`include_paths`/`exclude_paths` 按**路径前缀**匹配(例如 `/docs/` 仅匹配以 `/docs/` 开头的路径,不会误命中 `/blog/docs-tips`)。|
> 路由说明:`https://host/sitemap.xml`、`https://host/feed.xml`、`*.atom` 等 sitemap-looking URL 和显式 `args.site=true` 让出给下表的整站导入;`https://github.com/{org}/{repo}` 等 Git 托管平台 URL 让出给上文的代码导入。
diff --git a/openviking/parse/accessors/web_crawler/config.py b/openviking/parse/accessors/web_crawler/config.py
index bcab21bab..d32f9e989 100644
--- a/openviking/parse/accessors/web_crawler/config.py
+++ b/openviking/parse/accessors/web_crawler/config.py
@@ -22,8 +22,6 @@ class CrawlConfig:
retry_times: int = 2
max_links_per_page: int = 500
max_html_bytes: int = 10 * 1024 * 1024
- fallback_playwright: bool = True
- playwright_timeout: float = 30.0
request_validator: Optional[Callable[[str], None]] = None
def __post_init__(self) -> None:
@@ -43,5 +41,3 @@ class CrawlConfig:
raise ValueError("max_links_per_page must be >= 1.")
if self.max_html_bytes < 1:
raise ValueError("max_html_bytes must be >= 1.")
- if self.playwright_timeout <= 0:
- raise ValueError("playwright_timeout must be > 0 for recursive web crawling.")
diff --git a/openviking/parse/accessors/web_crawler/models.py b/openviking/parse/accessors/web_crawler/models.py
index 771bfb877..863a472d4 100644
--- a/openviking/parse/accessors/web_crawler/models.py
+++ b/openviking/parse/accessors/web_crawler/models.py
@@ -31,4 +31,3 @@ class CrawlResult:
total_downloads: int = 0
total_skipped: int = 0
total_failed: int = 0
- fallback_rendered: int = 0
diff --git a/openviking/parse/accessors/web_crawler/playwright_renderer.py b/openviking/parse/accessors/web_crawler/playwright_renderer.py
deleted file mode 100644
index b3063e02f..000000000
--- a/openviking/parse/accessors/web_crawler/playwright_renderer.py
+++ /dev/null
@@ -1,191 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Playwright renderer used only as low-content fallback."""
-
-import asyncio
-from collections.abc import Callable
-from dataclasses import dataclass
-from typing import Optional
-
-from openviking.parse.accessors.web_crawler.render_heuristics import (
- CHALLENGE_MARKERS,
- SHELL_VISIBLE_TEXT_CHARS,
-)
-
-
-PLAYWRIGHT_PACKAGE_INSTALL_HINT = (
- "Playwright fallback was needed, but the Python package is not installed. "
- "Install it with `pip install playwright` and install Chromium with "
- "`python -m playwright install chromium`."
-)
-PLAYWRIGHT_CHROMIUM_INSTALL_HINT = (
- "Playwright fallback was needed, but Chromium is not installed or cannot be "
- "launched. Run `python -m playwright install chromium` and retry."
-)
-
-# Cap the networkidle wait: pages with continuous background activity (e.g. the
-# GraphiQL playground, polling/websocket apps) never go idle and would otherwise
-# block until the full render timeout. Content is usually ready right after
-# domcontentloaded, and ``_wait_past_challenge`` handles late-arriving text.
-_NETWORKIDLE_TIMEOUT_MS = 8000
-
-
-@dataclass
-class RenderResult:
- html: str = ""
- status_code: int = 0
- final_url: str = ""
- content_type: str = ""
- error: Optional[str] = None
-
- @property
- def is_success(self) -> bool:
- return 200 <= self.status_code < 400 and self.error is None
-
-
-class PlaywrightRenderer:
- def __init__(self, request_validator: Optional[Callable[[str], None]] = None) -> None:
- self._request_validator = request_validator
- self._playwright_manager = None
- self._browser = None
- self._browser_lock = asyncio.Lock()
-
- async def render(self, url: str, timeout: float) -> RenderResult:
- page = None
- try:
- if self._request_validator:
- self._request_validator(url)
- browser = await self._get_browser()
- page = await browser.new_page(accept_downloads=False)
-
- if self._request_validator:
-
- async def _validate_route(route):
- await self._guard_route(route, self._request_validator)
-
- await page.route("**/*", _validate_route)
-
- response = await page.goto(
- url,
- wait_until="domcontentloaded",
- timeout=timeout * 1000,
- )
- try:
- await page.wait_for_load_state("networkidle", timeout=_NETWORKIDLE_TIMEOUT_MS)
- except Exception:
- pass
- await self._wait_past_challenge(page, timeout * 1000)
- html = await self._read_content(page)
- final_url = page.url
- if self._request_validator:
- self._request_validator(final_url)
- return RenderResult(
- html=html,
- status_code=response.status if response else 200,
- final_url=final_url,
- content_type=response.headers.get("content-type", "") if response else "",
- )
- except ImportError:
- return RenderResult(final_url=url, error=PLAYWRIGHT_PACKAGE_INSTALL_HINT)
- except Exception as exc:
- error = str(exc)
- if (
- "Executable doesn't exist" in error
- or "playwright install" in error
- or "BrowserType.launch" in error
- ):
- error = PLAYWRIGHT_CHROMIUM_INSTALL_HINT
- return RenderResult(final_url=url, error=error)
- finally:
- if page:
- try:
- await page.close()
- except Exception:
- pass
-
- @staticmethod
- async def _guard_route(route, request_validator) -> None:
- """SSRF guard for sub-resource requests.
-
- Block requests to disallowed hosts (e.g. private-network probe
- endpoints) by aborting them, but never let a blocked sub-resource
- fail the whole page render. Only the main document URL and the final
- URL (validated in ``render``) gate the overall result.
- """
- try:
- request_validator(route.request.url)
- except Exception:
- await route.abort()
- return
- await route.continue_()
-
- @staticmethod
- async def _wait_past_challenge(page, timeout_ms: float, poll_ms: int = 500) -> None:
- """Wait out JS anti-bot interstitials (e.g. "Please wait...").
-
- These challenge pages run a CPU-bound JS proof-of-work and then
- auto-redirect to the real content without further network activity,
- so ``networkidle`` returns while the interstitial is still showing.
- Poll the body text until real content appears or we run out of time.
- """
- import time
-
- deadline = time.monotonic() + max(timeout_ms, 0) / 1000
- while True:
- try:
- body = (await page.inner_text("body")).strip()
- except Exception:
- body = ""
- lowered = body.lower()
- looks_like_challenge = (
- not body
- or len(body) < SHELL_VISIBLE_TEXT_CHARS
- or any(marker in lowered for marker in CHALLENGE_MARKERS)
- )
- if not looks_like_challenge or time.monotonic() >= deadline:
- return
- await page.wait_for_timeout(poll_ms)
-
- @staticmethod
- async def _read_content(page) -> str:
- """Read page HTML, retrying through in-flight client-side navigation.
-
- Heavy SPAs (client-side redirects, late hydration) can still be
- navigating when we first ask for content, which raises "page is
- navigating and changing the content". Retry until the page settles.
- """
- last_exc = None
- for _ in range(6):
- try:
- return await page.content()
- except Exception as exc:
- if "navigating and changing the content" not in str(exc):
- raise
- last_exc = exc
- try:
- await page.wait_for_load_state("load", timeout=5000)
- except Exception:
- pass
- await page.wait_for_timeout(400)
- if last_exc:
- raise last_exc
- return await page.content()
-
- async def close(self) -> None:
- async with self._browser_lock:
- if self._browser:
- await self._browser.close()
- self._browser = None
- if self._playwright_manager:
- await self._playwright_manager.stop()
- self._playwright_manager = None
-
- async def _get_browser(self):
- async with self._browser_lock:
- if self._browser is None or not self._browser.is_connected():
- from playwright.async_api import async_playwright
-
- if self._playwright_manager is None:
- self._playwright_manager = await async_playwright().start()
- self._browser = await self._playwright_manager.chromium.launch(headless=True)
- return self._browser
diff --git a/openviking/parse/accessors/web_crawler/render_heuristics.py b/openviking/parse/accessors/web_crawler/render_heuristics.py
deleted file mode 100644
index 91283e54f..000000000
--- a/openviking/parse/accessors/web_crawler/render_heuristics.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Heuristics for deciding whether a page likely needs browser rendering."""
-
-import re
-
-from bs4 import BeautifulSoup
-
-
-_SPA_EMPTY_PATTERNS = (
- "You need to enable JavaScript to run this app.",
- "This app works best with JavaScript enabled.",
- "Please enable JavaScript to continue.",
- "JavaScript is required to use this application.",
- "Enable JavaScript to view this page.",
-)
-
-_MIN_VISIBLE_TEXT_CHARS = 200
-SHELL_VISIBLE_TEXT_CHARS = 40
-
-# Text markers of JS anti-bot interstitials that briefly show before the real
-# page loads (e.g. volcengine's proof-of-work gate renders only "Please wait...").
-CHALLENGE_MARKERS = (
- "please wait",
- "checking your browser",
- "verifying you are human",
- "just a moment",
- "attention required",
-)
-
-
-def should_render_with_playwright(html: str) -> bool:
- """Return True when static HTML looks like a client-rendered shell."""
- html = html or ""
- html_lower = html.lower()
- if any(pattern.lower() in html_lower for pattern in _SPA_EMPTY_PATTERNS):
- return True
- visible_len = visible_body_text_len(html)
- if visible_len < SHELL_VISIBLE_TEXT_CHARS:
- return True
- # __NEXT_DATA__ only signals a Next.js app; most such pages are SSR/SSG and
- # already ship full body text in the static HTML. Only render when the
- # static body is also too thin to be the real content.
- if "__next_data__" in html_lower and visible_len < _MIN_VISIBLE_TEXT_CHARS:
- return True
- if re.search(r'id=["\'](?:root|app)["\']', html_lower) and visible_len < _MIN_VISIBLE_TEXT_CHARS:
- return True
- if html_lower.count("<script") >= 5 and visible_len < _MIN_VISIBLE_TEXT_CHARS:
- return True
- return False
-
-
-def looks_like_unrendered_page(html: str) -> bool:
- """Return True when HTML is an empty shell or an anti-bot challenge page.
-
- Used to reject content that must not be stored as real page text: the
- static SPA shell or a JS interstitial such as "Please wait...".
- """
- try:
- html = html or ""
- soup = BeautifulSoup(html, "html.parser")
- body = soup.body or soup
- for el in body(["script", "style", "noscript"]):
- el.decompose()
- text = body.get_text(strip=True)
- lowered = text.lower()
- if any(marker in lowered for marker in CHALLENGE_MARKERS):
- return True
- return len(text) < SHELL_VISIBLE_TEXT_CHARS
- except Exception:
- return True
-
-
-def visible_body_text_len(html: str) -> int:
- """Approximate visible text length without running page JavaScript."""
- try:
- soup = BeautifulSoup(html or "", "html.parser")
- body = soup.body or soup
- for el in body(["script", "style", "noscript"]):
- el.decompose()
- return len(body.get_text(strip=True))
- except Exception:
- return 0
diff --git a/openviking/parse/accessors/web_crawler/scrapy_spider.py b/openviking/parse/accessors/web_crawler/scrapy_spider.py
index 2b72e70e9..9cf344242 100644
--- a/openviking/parse/accessors/web_crawler/scrapy_spider.py
+++ b/openviking/parse/accessors/web_crawler/scrapy_spider.py
@@ -2,24 +2,17 @@
# SPDX-License-Identifier: AGPL-3.0
"""Scrapy spider for recursive web resource import."""
-import asyncio
import os
from collections.abc import Iterator
from urllib.parse import urljoin, urlparse
import scrapy
from parsel import Selector
-from scrapy import signals
from scrapy.exceptions import CloseSpider
from openviking.parse.accessors.http_accessor import URLType, URLTypeDetector
from openviking.parse.accessors.web_crawler.config import CrawlConfig
from openviking.parse.accessors.web_crawler.models import CrawledDownload, CrawledPage
-from openviking.parse.accessors.web_crawler.playwright_renderer import PlaywrightRenderer
-from openviking.parse.accessors.web_crawler.render_heuristics import (
- looks_like_unrendered_page,
- should_render_with_playwright,
-)
_DOWNLOAD_URL_TYPES = frozenset(
@@ -78,23 +71,9 @@ class OpenVikingWebSpider(scrapy.Spider):
self.config = config
self.collector = collector
self.download_collector = download_collector
- self.renderer = PlaywrightRenderer(config.request_validator)
self.root_host = urlparse(root_url).netloc.lower()
self._success_count = 0
self._seen_download_urls: set[str] = set()
- self._render_semaphore: asyncio.Semaphore | None = None
-
- @classmethod
- def from_crawler(cls, crawler, *args, **kwargs):
- spider = super().from_crawler(crawler, *args, **kwargs)
- crawler.signals.connect(spider._on_spider_closed, signal=signals.spider_closed)
- return spider
-
- async def _on_spider_closed(self, spider, reason):
- try:
- await asyncio.wait_for(self.renderer.close(), timeout=5.0)
- except Exception:
- pass
def _success_at_limit(self) -> bool:
return 0 < self.config.max_pages <= self._success_count
@@ -128,39 +107,6 @@ class OpenVikingWebSpider(scrapy.Spider):
return
page_html = response.text
- page_source = "scrapy_static"
- needs_render = self.config.fallback_playwright and should_render_with_playwright(
- response.text
- )
- try:
- if needs_render:
- if self._render_semaphore is None:
- self._render_semaphore = asyncio.Semaphore(self.config.concurrency)
- async with self._render_semaphore:
- if self._success_at_limit():
- return
- rendered = await self.renderer.render(
- final_url, self.config.playwright_timeout
- )
- if rendered.is_success and rendered.html:
- final_url = rendered.final_url or final_url
- page_html = rendered.html
- page_source = "playwright"
- elif rendered.error:
- self._add_failed(response.url, final_url, depth, rendered.error)
- return
- except Exception as exc:
- self._add_failed(response.url, final_url, depth, str(exc))
- return
-
- if needs_render and looks_like_unrendered_page(page_html):
- self._add_failed(
- response.url,
- final_url,
- depth,
- "page did not render real content (empty shell or anti-bot challenge page)",
- )
- return
if self._stop_if_success_at_limit():
return
@@ -170,7 +116,7 @@ class OpenVikingWebSpider(scrapy.Spider):
final_url=final_url,
depth=depth,
html=page_html,
- source=page_source,
+ source="scrapy_static",
)
)
self._success_count += 1
diff --git a/openviking/parse/accessors/web_crawler/web_crawler.py b/openviking/parse/accessors/web_crawler/web_crawler.py
index 39ee7345d..05bb2e46e 100644
--- a/openviking/parse/accessors/web_crawler/web_crawler.py
+++ b/openviking/parse/accessors/web_crawler/web_crawler.py
@@ -146,6 +146,4 @@ class ScrapyWebCrawler:
result.total_skipped += 1
else:
result.total_failed += 1
- if page.source == "playwright":
- result.fallback_rendered += 1
return result
diff --git a/openviking/parse/accessors/web_importer.py b/openviking/parse/accessors/web_importer.py
index ea3e61acd..06a099587 100644
--- a/openviking/parse/accessors/web_importer.py
+++ b/openviking/parse/accessors/web_importer.py
@@ -13,10 +13,6 @@ from urllib.parse import unquote, urlsplit, urlunsplit
from openviking.parse.accessors.http_accessor import HTTPAccessor
from openviking.parse.accessors.web_crawler import CrawlConfig, ScrapyWebCrawler
-from openviking.parse.accessors.web_crawler.playwright_renderer import (
- PLAYWRIGHT_CHROMIUM_INSTALL_HINT,
- PLAYWRIGHT_PACKAGE_INSTALL_HINT,
-)
from openviking.parse.accessors.web_feed_accessor import (
FeedEntry,
_dedup_relpath,
@@ -25,24 +21,10 @@ from openviking.parse.accessors.web_feed_accessor import (
url_to_relpath,
)
from openviking_cli.exceptions import InvalidArgumentError
-from openviking_cli.utils.logger import get_logger
-
-logger = get_logger(__name__)
DEPTH_UNLIMITED = -1
MAX_PAGES_UNLIMITED = -1
-# Actionable install hints the renderer records on CrawledPage.error when the
-# Playwright fallback is needed but unavailable. These are surfaced to the user
-# even for non-entry pages, which would otherwise only show up as a failure
-# count.
-_RENDER_INSTALL_HINTS = frozenset(
- {
- PLAYWRIGHT_PACKAGE_INSTALL_HINT,
- PLAYWRIGHT_CHROMIUM_INSTALL_HINT,
- }
-)
-
@dataclass(frozen=True)
class WebImportOptions:
@@ -111,19 +93,7 @@ class WebImporter:
success_pages = self._dedupe_success_pages(crawl_result.pages)
if not any(page.depth == 0 for page in success_pages):
detail = self._entry_failure_detail(crawl_result.pages)
- message = f"Failed to fetch entry page: {root_url}"
- if detail:
- message = f"{message} ({detail})"
- raise RuntimeError(message)
-
- render_hints = self._render_install_hints(crawl_result.pages)
- if render_hints:
- logger.warning(
- "Some pages could not be rendered during web import of %s and were "
- "skipped: %s",
- root_url,
- " ".join(render_hints),
- )
+ raise RuntimeError(_entry_failure_message(root_url, detail))
temp_root = Path(tempfile.mkdtemp(prefix="ov_web_"))
temp_dir = temp_root / _host_name(root_url)
@@ -155,35 +125,16 @@ class WebImporter:
"page_count": len(success_pages),
"download_count": len(downloaded_files),
"crawl_result": _crawl_summary(crawl_result),
- "render_hints": render_hints,
"original_filename": _host_name(root_url),
},
)
- @staticmethod
- def _render_install_hints(pages) -> list[str]:
- """Collect actionable render-install hints from failed pages (any depth).
-
- Child SPA pages that fail because Playwright is unavailable only record
- the hint on ``CrawledPage.error``; the entry page still succeeds, so the
- caller would otherwise see the hint nowhere. Surface each distinct hint
- once so the user can act on it.
- """
- hints: list[str] = []
- for page in pages:
- if page.status == "success" or not page.error:
- continue
- if page.error in _RENDER_INSTALL_HINTS and page.error not in hints:
- hints.append(page.error)
- return hints
-
@staticmethod
def _entry_failure_detail(pages) -> str:
"""Return the failure reason of the entry page, if one was recorded.
- The renderer surfaces actionable hints (e.g. missing Playwright
- install) via ``CrawledPage.error``; without this the caller only sees
- the generic "Failed to fetch entry page" message.
+ Without this the caller only sees the generic "Failed to fetch entry
+ page" message instead of the underlying error (e.g. an SSRF rejection).
"""
for page in pages:
if page.depth == 0 and page.status != "success" and page.error:
@@ -269,10 +220,28 @@ def _crawl_summary(crawl_result: Any) -> Dict[str, Any]:
"total_downloads": getattr(crawl_result, "total_downloads", 0),
"total_failed": crawl_result.total_failed,
"total_skipped": crawl_result.total_skipped,
- "fallback_rendered": crawl_result.fallback_rendered,
}
+def _entry_failure_message(root_url: str, detail: str) -> str:
+ """Build a human-readable error for an entry page that could not be fetched.
+
+ The most common cause is the site's robots.txt disallowing crawlers, which
+ Scrapy reports with the terse "Forbidden by robots.txt". Turn that into a
+ short compliance-oriented hint instead.
+ """
+ if detail and "robots.txt" in detail.lower():
+ return (
+ "This URL cannot be imported for compliance reasons "
+ "(the site disallows crawlers). Please save the page and import it "
+ "as a local file instead."
+ )
+ message = f"Failed to fetch entry page: {root_url}"
+ if detail:
+ message = f"{message} ({detail})"
+ return message
+
+
def _normalize_page_url(url: str) -> str:
parts = urlsplit(url or "")
scheme = parts.scheme.lower()
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py
index a169cd276..452067842 100644
--- a/openviking/parse/parsers/html.py
+++ b/openviking/parse/parsers/html.py
@@ -24,14 +24,6 @@ from openviking_cli.utils.config.parser_config import HTMLConfig
logger = __import__("openviking_cli.utils.logger").utils.logger.get_logger(__name__)
-_SPA_EMPTY_PATTERNS = (
- "You need to enable JavaScript to run this app.",
- "This app works best with JavaScript enabled.",
- "Please enable JavaScript to continue.",
- "JavaScript is required to use this application.",
- "Enable JavaScript to view this page.",
-)
-
class HTMLParser(BaseParser):
"""
@@ -133,10 +125,33 @@ class HTMLParser(BaseParser):
content = self._extract_markdown(html, base_url or "")
title = self._extract_title(html, base_url or "")
content = self._clean_markdown(content)
+ if not content:
+ content = self._extract_noscript_notice(html)
if title and title not in content:
content = f"# {title}\n\n{content}" if content else f"# {title}"
return content
+ @staticmethod
+ def _extract_noscript_notice(html: str) -> str:
+ """Return the <noscript> text when the page has no extractable body.
+
+ SPA shells (e.g. React apps) render their real content via JavaScript
+ and only ship a static ``<noscript>You need to enable JavaScript...``
+ placeholder. trafilatura ignores <noscript>, so such pages otherwise
+ extract to an empty string. Surfacing the notice keeps at least a hint
+ of why the page looks empty instead of writing a blank file.
+ """
+ try:
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(html, "html.parser")
+ texts = [
+ node.get_text(" ", strip=True) for node in soup.find_all("noscript")
+ ]
+ return "\n\n".join(text for text in texts if text)
+ except Exception:
+ return ""
+
@staticmethod
def _extract_markdown(html: str, url: str) -> str:
"""Extract clean Markdown body from HTML."""
@@ -182,8 +197,6 @@ class HTMLParser(BaseParser):
flags=re.IGNORECASE,
)
markdown = re.sub(r"<(span|a)\s+(?:id|name)=['\"][^'\"]*['\"]\s*>\s*</\1>", "", markdown)
- for pattern in _SPA_EMPTY_PATTERNS:
- markdown = markdown.replace(pattern, "")
markdown = re.sub(r"\n{3,}", "\n\n", markdown)
return markdown.strip()
diff --git a/tests/parse/accessors/web_crawler/test_config.py b/tests/parse/accessors/web_crawler/test_config.py
index 501aeb0d5..cb5a2e519 100644
--- a/tests/parse/accessors/web_crawler/test_config.py
+++ b/tests/parse/accessors/web_crawler/test_config.py
@@ -47,7 +47,3 @@ class TestCrawlConfigValidation:
def test_max_html_bytes_zero_rejected(self):
with pytest.raises(ValueError):
CrawlConfig(max_html_bytes=0)
-
- def test_playwright_timeout_non_positive_rejected(self):
- with pytest.raises(ValueError):
- CrawlConfig(playwright_timeout=0)
diff --git a/tests/parse/accessors/web_crawler/test_playwright_renderer.py b/tests/parse/accessors/web_crawler/test_playwright_renderer.py
deleted file mode 100644
index bc1cc8250..000000000
--- a/tests/parse/accessors/web_crawler/test_playwright_renderer.py
+++ /dev/null
@@ -1,87 +0,0 @@
-from openviking.parse.accessors.web_crawler.playwright_renderer import PlaywrightRenderer
-
-
-class _FakePage:
- """Minimal page stub returning a scripted sequence of body texts."""
-
- def __init__(self, body_sequence):
- self._bodies = list(body_sequence)
- self.inner_text_calls = 0
- self.slept_ms = 0
-
- async def inner_text(self, _selector):
- self.inner_text_calls += 1
- idx = min(self.inner_text_calls - 1, len(self._bodies) - 1)
- return self._bodies[idx]
-
- async def wait_for_timeout(self, ms):
- self.slept_ms += ms
-
-
-class TestRenderer:
- async def test_close_is_idempotent_when_browser_never_started(self):
- renderer = PlaywrightRenderer()
- await renderer.close()
- await renderer.close()
-
- def test_request_validator_stored(self):
- called = []
- renderer = PlaywrightRenderer(request_validator=lambda u: called.append(u))
- assert renderer._request_validator is not None
- renderer._request_validator("http://x.com/")
- assert called == ["http://x.com/"]
-
-
-class _FakeRoute:
- def __init__(self, url):
- self.request = type("Req", (), {"url": url})()
- self.aborted = False
- self.continued = False
-
- async def abort(self):
- self.aborted = True
-
- async def continue_(self):
- self.continued = True
-
-
-class TestGuardRoute:
- async def test_blocks_disallowed_subresource_without_raising(self):
- def validator(url):
- if "private" in url:
- raise ValueError("private network blocked")
-
- route = _FakeRoute("http://private.internal/probe")
- # Must not raise: a blocked sub-resource cannot fail the whole render.
- await PlaywrightRenderer._guard_route(route, validator)
- assert route.aborted is True
- assert route.continued is False
-
- async def test_allows_valid_subresource(self):
- route = _FakeRoute("https://cdn.example.com/app.js")
- await PlaywrightRenderer._guard_route(route, lambda _u: None)
- assert route.continued is True
- assert route.aborted is False
-
-
-class TestWaitPastChallenge:
- async def test_waits_until_challenge_redirects_to_content(self):
- real = "Real documentation body with plenty of visible content here." * 3
- page = _FakePage(["Please wait...", "Please wait...", real])
- await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=10_000, poll_ms=10)
- assert page.inner_text_calls == 3
- assert page.slept_ms == 20
-
- async def test_returns_immediately_when_content_present(self):
- real = "Real documentation body with plenty of visible content here." * 3
- page = _FakePage([real])
- await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=10_000, poll_ms=10)
- assert page.inner_text_calls == 1
- assert page.slept_ms == 0
-
- async def test_gives_up_after_timeout_on_persistent_challenge(self):
- page = _FakePage(["Please wait..."])
- await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=0, poll_ms=10)
- assert page.inner_text_calls == 1
- assert page.slept_ms == 0
-
diff --git a/tests/parse/accessors/web_crawler/test_render_heuristics.py b/tests/parse/accessors/web_crawler/test_render_heuristics.py
deleted file mode 100644
index a1bcff0b1..000000000
--- a/tests/parse/accessors/web_crawler/test_render_heuristics.py
+++ /dev/null
@@ -1,63 +0,0 @@
-from openviking.parse.accessors.web_crawler.render_heuristics import (
- looks_like_unrendered_page,
- should_render_with_playwright,
-)
-
-
-def test_js_required_pattern_triggers_render():
- html = "<html>You need to enable JavaScript to run this app.</html>"
- assert should_render_with_playwright(html) is True
-
-
-def test_react_root_triggers_render():
- html = "<html><body><div id='root'></div></body></html>"
- assert should_render_with_playwright(html) is True
-
-
-def test_ssr_app_root_with_rich_content_skips_render():
- body = "OpenViking 快速开始文档,包含安装、配置、添加资源、创建集合和检索示例。" * 8
- html = f"<html><body><div id='app'><main><p>{body}</p></main></div></body></html>"
- assert should_render_with_playwright(html) is False
-
-
-def test_next_data_triggers_render():
- html = "<html><script>__NEXT_DATA__</script></html>"
- assert should_render_with_playwright(html) is True
-
-
-def test_many_scripts_with_tiny_body_triggers_render():
- html = "<html><body>x</body>" + "<script></script>" * 6 + "</html>"
- assert should_render_with_playwright(html) is True
-
-
-def test_content_rich_static_page_skips_render():
- body = "This is a fully server-rendered article with plenty of readable text. " * 3
- html = f"<html><body><p>{body}</p></body></html>"
- assert should_render_with_playwright(html) is False
-
-
-def test_challenge_page_is_unrendered():
- html = "<html><body>Please wait...</body></html>"
- assert looks_like_unrendered_page(html) is True
-
-
-def test_empty_shell_is_unrendered():
- html = "<html><body><div id='root'></div></body></html>"
- assert looks_like_unrendered_page(html) is True
-
-
-def test_real_content_is_not_unrendered():
- body = "Collection 数据过滤删除任务的完整说明文档,包含请求参数、示例与返回值。" * 3
- html = f"<html><body><article>{body}</article></body></html>"
- assert looks_like_unrendered_page(html) is False
-
-
-def test_unrendered_check_fails_closed_on_parser_error(monkeypatch):
- from openviking.parse.accessors.web_crawler import render_heuristics
-
- def raise_parser_error(*_args, **_kwargs):
- raise RuntimeError("parser failed")
-
- monkeypatch.setattr(render_heuristics, "BeautifulSoup", raise_parser_error)
-
- assert render_heuristics.looks_like_unrendered_page("<html></html>") is True
diff --git a/tests/parse/accessors/web_crawler/test_scrapy_spider.py b/tests/parse/accessors/web_crawler/test_scrapy_spider.py
index 66dccb4e9..9e3977566 100644
--- a/tests/parse/accessors/web_crawler/test_scrapy_spider.py
+++ b/tests/parse/accessors/web_crawler/test_scrapy_spider.py
@@ -183,7 +183,7 @@ _RICH_HTML = (
class TestParseGate:
async def test_early_return_when_limit_reached(self):
- spider = _make_spider(CrawlConfig(max_pages=10, fallback_playwright=False))
+ spider = _make_spider(CrawlConfig(max_pages=10))
spider._success_count = 10
resp = _make_response(body=_RICH_HTML)
items = await _drive_parse(spider, resp)
@@ -192,7 +192,7 @@ class TestParseGate:
assert spider.collector == []
async def test_appends_success_and_increments_count(self):
- spider = _make_spider(CrawlConfig(max_pages=10, fallback_playwright=False))
+ spider = _make_spider(CrawlConfig(max_pages=10))
resp = _make_response(body=_RICH_HTML)
await _drive_parse(spider, resp)
assert spider._success_count == 1
@@ -207,7 +207,6 @@ class TestParseGate:
depth=1,
max_pages=2,
skip_download_links=False,
- fallback_playwright=False,
)
)
body = (
@@ -225,7 +224,7 @@ class TestParseGate:
assert items == []
async def test_second_call_blocked_after_reaching_max(self):
- spider = _make_spider(CrawlConfig(max_pages=1, fallback_playwright=False))
+ spider = _make_spider(CrawlConfig(max_pages=1))
resp = _make_response(body=_RICH_HTML)
await _drive_parse(spider, resp)
assert spider._success_count == 1
@@ -235,7 +234,7 @@ class TestParseGate:
assert len(success) == 1
async def test_reaching_limit_closes_scrapy_engine(self):
- spider = _make_spider(CrawlConfig(max_pages=1, fallback_playwright=False))
+ spider = _make_spider(CrawlConfig(max_pages=1))
spider.crawler = MagicMock()
spider.crawler.engine = MagicMock()
resp = _make_response(body=_RICH_HTML)
@@ -248,7 +247,7 @@ class TestParseGate:
assert len(spider.collector) == 1
async def test_skipped_on_non_html_response(self):
- spider = _make_spider(CrawlConfig(fallback_playwright=False))
+ spider = _make_spider(CrawlConfig())
resp = _make_response(
body=b"binary",
headers={"content-type": b"application/octet-stream"},
@@ -261,7 +260,7 @@ class TestParseGate:
async def test_failed_on_validator_rejecting_entry(self):
def blocker(_):
raise ValueError("blocked by SSRF")
- spider = _make_spider(CrawlConfig(request_validator=blocker, fallback_playwright=False))
+ spider = _make_spider(CrawlConfig(request_validator=blocker))
resp = _make_response(body=_RICH_HTML)
await _drive_parse(spider, resp)
assert spider._success_count == 0
diff --git a/tests/parse/parsers/test_html.py b/tests/parse/parsers/test_html.py
index ef87b8ac9..213c2697c 100644
--- a/tests/parse/parsers/test_html.py
+++ b/tests/parse/parsers/test_html.py
@@ -19,9 +19,9 @@ class TestHTMLParserMarkdownCleaning:
assert "<span" not in cleaned
assert "<a name" not in cleaned
- def test_strips_spa_empty_pattern(self):
- md = "real content. You need to enable JavaScript to run this app."
- assert "JavaScript" not in self.parser._clean_markdown(md)
+ def test_keeps_spa_notice_text(self):
+ md = "You need to enable JavaScript to run this app."
+ assert self.parser._clean_markdown(md) == md
def test_collapses_blank_lines(self):
cleaned = self.parser._clean_markdown("a\n\n\n\n\nb")
@@ -31,6 +31,31 @@ class TestHTMLParserMarkdownCleaning:
assert self.parser._clean_markdown(" \n\nhello\n ") == "hello"
+class TestHTMLParserNoscriptFallback:
+ def setup_method(self):
+ self.parser = HTMLParser()
+
+ def test_spa_shell_keeps_noscript_notice(self):
+ html = (
+ "<html><head><title></title></head><body>"
+ "<noscript>You need to enable JavaScript to run this app.</noscript>"
+ '<div id="root"></div></body></html>'
+ )
+ md = self.parser._html_to_markdown(html, base_url="https://example.com/")
+ assert "You need to enable JavaScript to run this app." in md
+
+ def test_noscript_fallback_skipped_when_body_extractable(self):
+ html = (
+ "<html><head><title>Doc</title></head><body><article>"
+ "<p>" + ("Real body content that trafilatura can extract. " * 20) + "</p>"
+ "</article>"
+ "<noscript>enable javascript</noscript></body></html>"
+ )
+ md = self.parser._html_to_markdown(html, base_url="https://example.com/")
+ assert "Real body content" in md
+ assert "enable javascript" not in md
+
+
class TestHTMLParserTitleExtraction:
def setup_method(self):
self.parser = HTMLParser()
diff --git a/tests/parse/test_html_parser_utils.py b/tests/parse/test_html_parser_utils.py
deleted file mode 100644
index ca2373e7b..000000000
--- a/tests/parse/test_html_parser_utils.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import pytest
-from openviking.parse.parsers.html import HTMLParser
-
-
-class TestHTMLParserRawUrlConversion:
- """Test suite for HTMLParser._convert_to_raw_url method."""
-
- def setup_method(self):
- self.parser = HTMLParser()
-
- def test_github_blob_conversion(self):
- blob_url = "https://github.com/volcengine/OpenViking/blob/main/docs/design.md"
- expected = "https://raw.githubusercontent.com/volcengine/OpenViking/main/docs/design.md"
- assert self.parser._convert_to_raw_url(blob_url) == expected
-
- blob_deep = "https://github.com/user/repo/blob/feature/branch/src/components/Button.tsx"
- expected_deep = (
- "https://raw.githubusercontent.com/user/repo/feature/branch/src/components/Button.tsx"
- )
- assert self.parser._convert_to_raw_url(blob_deep) == expected_deep
-
- def test_github_non_blob_urls(self):
- repo_root = "https://github.com/volcengine/OpenViking"
- assert self.parser._convert_to_raw_url(repo_root) == repo_root
-
- issue_url = "https://github.com/volcengine/OpenViking/issues/1"
- assert self.parser._convert_to_raw_url(issue_url) == issue_url
-
- raw_url = "https://raw.githubusercontent.com/volcengine/OpenViking/main/README.md"
- assert self.parser._convert_to_raw_url(raw_url) == raw_url
-
- def test_gitlab_blob_conversion(self):
- blob_url = "https://gitlab.com/gitlab-org/gitlab/-/blob/master/README.md"
- expected = "https://gitlab.com/gitlab-org/gitlab/-/raw/master/README.md"
- assert self.parser._convert_to_raw_url(blob_url) == expected
-
- blob_deep = "https://gitlab.com/group/project/-/blob/dev/src/main.rs"
- expected_deep = "https://gitlab.com/group/project/-/raw/dev/src/main.rs"
- assert self.parser._convert_to_raw_url(blob_deep) == expected_deep
-
- def test_gitlab_non_blob_urls(self):
- root = "https://gitlab.com/gitlab-org/gitlab"
- assert self.parser._convert_to_raw_url(root) == root
-
- issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/123"
- assert self.parser._convert_to_raw_url(issue) == issue
-
- def test_other_domains(self):
- url = "https://example.com/blob/main/file.txt"
- assert self.parser._convert_to_raw_url(url) == url
-
- bitbucket = "https://bitbucket.org/user/repo/src/master/README.md"
- assert self.parser._convert_to_raw_url(bitbucket) == bitbucket
diff --git a/tests/unit/test_accessors_http.py b/tests/unit/test_accessors_http.py
index 47ae0e27c..329ab312e 100644
--- a/tests/unit/test_accessors_http.py
+++ b/tests/unit/test_accessors_http.py
@@ -16,6 +16,7 @@ def _mock_config():
code=SimpleNamespace(
github_domains=["github.com", "www.github.com"],
gitlab_domains=["gitlab.com", "www.gitlab.com"],
+ github_raw_domain="raw.githubusercontent.com",
azure_devops_domains=[
"dev.azure.com",
"ssh.dev.azure.com",
@@ -288,3 +289,63 @@ class TestHTTPAccessorPriorityRouting:
assert len(accessors) == 2
assert accessors[0].__class__.__name__ == "GitAccessor"
assert accessors[1].__class__.__name__ == "HTTPAccessor"
+
+
+class TestHTTPAccessorRawUrlConversion:
+ """Tests for HTTPAccessor._convert_to_raw_url (GitHub/GitLab blob -> raw)."""
+
+ @pytest.fixture(autouse=True)
+ def _patch_config(self):
+ with patch(
+ "openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",
+ side_effect=_mock_config,
+ ):
+ yield
+
+ @pytest.fixture
+ def accessor(self) -> HTTPAccessor:
+ return HTTPAccessor()
+
+ def test_github_blob_conversion(self, accessor: HTTPAccessor) -> None:
+ blob_url = "https://github.com/volcengine/OpenViking/blob/main/docs/design.md"
+ expected = "https://raw.githubusercontent.com/volcengine/OpenViking/main/docs/design.md"
+ assert accessor._convert_to_raw_url(blob_url) == expected
+
+ blob_deep = "https://github.com/user/repo/blob/feature/branch/src/components/Button.tsx"
+ expected_deep = (
+ "https://raw.githubusercontent.com/user/repo/feature/branch/src/components/Button.tsx"
+ )
+ assert accessor._convert_to_raw_url(blob_deep) == expected_deep
+
+ def test_github_non_blob_urls(self, accessor: HTTPAccessor) -> None:
+ repo_root = "https://github.com/volcengine/OpenViking"
+ assert accessor._convert_to_raw_url(repo_root) == repo_root
+
+ issue_url = "https://github.com/volcengine/OpenViking/issues/1"
+ assert accessor._convert_to_raw_url(issue_url) == issue_url
+
+ raw_url = "https://raw.githubusercontent.com/volcengine/OpenViking/main/README.md"
+ assert accessor._convert_to_raw_url(raw_url) == raw_url
+
+ def test_gitlab_blob_conversion(self, accessor: HTTPAccessor) -> None:
+ blob_url = "https://gitlab.com/gitlab-org/gitlab/-/blob/master/README.md"
+ expected = "https://gitlab.com/gitlab-org/gitlab/-/raw/master/README.md"
+ assert accessor._convert_to_raw_url(blob_url) == expected
+
+ blob_deep = "https://gitlab.com/group/project/-/blob/dev/src/main.rs"
+ expected_deep = "https://gitlab.com/group/project/-/raw/dev/src/main.rs"
+ assert accessor._convert_to_raw_url(blob_deep) == expected_deep
+
+ def test_gitlab_non_blob_urls(self, accessor: HTTPAccessor) -> None:
+ root = "https://gitlab.com/gitlab-org/gitlab"
+ assert accessor._convert_to_raw_url(root) == root
+
+ issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/123"
+ assert accessor._convert_to_raw_url(issue) == issue
+
+ def test_other_domains(self, accessor: HTTPAccessor) -> None:
+ url = "https://example.com/blob/main/file.txt"
+ assert accessor._convert_to_raw_url(url) == url
+
+ bitbucket = "https://bitbucket.org/user/repo/src/master/README.md"
+ assert accessor._convert_to_raw_url(bitbucket) == bitbucket
diff --git a/tests/unit/test_web_importer.py b/tests/unit/test_web_importer.py
index 421077108..4fb20579b 100644
--- a/tests/unit/test_web_importer.py
+++ b/tests/unit/test_web_importer.py
@@ -133,7 +133,6 @@ class TestWebImporter:
total_downloads=0,
total_failed=0,
total_skipped=0,
- fallback_rendered=0,
)
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
@@ -177,7 +176,6 @@ class TestWebImporter:
total_downloads=0,
total_failed=0,
total_skipped=0,
- fallback_rendered=0,
)
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
@@ -188,7 +186,7 @@ class TestWebImporter:
options=WebImportOptions(),
)
- async def test_entry_failure_surfaces_renderer_hint(self, monkeypatch):
+ async def test_entry_failure_surfaces_error_detail(self, monkeypatch):
class FakeCrawler:
def __init__(self, config):
self.config = config
@@ -202,7 +200,7 @@ class TestWebImporter:
depth=0,
status="failed",
html=None,
- error="Playwright fallback was needed, but the Python package is not installed.",
+ error="blocked by SSRF",
)
],
downloads=[],
@@ -210,22 +208,17 @@ class TestWebImporter:
total_downloads=0,
total_failed=1,
total_skipped=0,
- fallback_rendered=0,
)
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
- with pytest.raises(RuntimeError, match="Playwright fallback was needed"):
+ with pytest.raises(RuntimeError, match="blocked by SSRF"):
await WebImporter().import_to_directory(
root_url="https://example.com",
options=WebImportOptions(),
)
- async def test_child_render_hint_surfaces_in_meta(self, monkeypatch):
- from openviking.parse.accessors.web_crawler.playwright_renderer import (
- PLAYWRIGHT_PACKAGE_INSTALL_HINT,
- )
-
+ async def test_entry_failure_robots_txt_is_humanized(self, monkeypatch):
class FakeCrawler:
def __init__(self, config):
self.config = config
@@ -237,40 +230,29 @@ class TestWebImporter:
url=root_url,
final_url=root_url,
depth=0,
- status="success",
- html="<html><body><h1>Home</h1></body></html>",
- error=None,
- ),
- SimpleNamespace(
- url=f"{root_url}/spa",
- final_url=f"{root_url}/spa",
- depth=1,
status="failed",
html=None,
- error=PLAYWRIGHT_PACKAGE_INSTALL_HINT,
- ),
+ error="Forbidden by robots.txt",
+ )
],
downloads=[],
- total_crawled=1,
+ total_crawled=0,
total_downloads=0,
total_failed=1,
total_skipped=0,
- fallback_rendered=0,
)
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
- result = await WebImporter().import_to_directory(
- root_url="https://example.com",
- options=WebImportOptions(depth=1, max_pages=5),
- )
- try:
- assert result.meta["render_hints"] == [PLAYWRIGHT_PACKAGE_INSTALL_HINT]
- assert result.meta["page_count"] == 1
- finally:
- import shutil
-
- shutil.rmtree(result.path.parent, ignore_errors=True)
+ with pytest.raises(RuntimeError) as exc_info:
+ await WebImporter().import_to_directory(
+ root_url="https://mp.weixin.qq.com/s/abc",
+ options=WebImportOptions(),
+ )
+ message = str(exc_info.value)
+ assert "Forbidden by robots.txt" not in message
+ assert "compliance" in message
+ assert "local file" in message
async def test_import_to_directory_downloads_child_links(self, monkeypatch, tmp_path):
class FakeCrawler:
@@ -295,7 +277,6 @@ class TestWebImporter:
total_downloads=1,
total_failed=0,
total_skipped=0,
- fallback_rendered=0,
)
download_file = tmp_path / "download.txt"