Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: add recursive web crawler based on Scrapy (#2836)

zgya50e9fd67789
diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md
index ad286681c..44e3e8c71 100644
--- a/docs/en/api/02-resources.md
+++ b/docs/en/api/02-resources.md
@@ -49,6 +49,14 @@ OpenViking supports various resource types, categorized by functionality:
|------|-------------|
| Feishu/Lark | URL-based, supports docx, wiki, sheets, bitable. By default uses app credentials from FEISHU_APP_ID and FEISHU_APP_SECRET; user-token imports can pass `args.feishu_access_token`, and user-token watches also pass `args.feishu_refresh_token` |
+**Web Pages (recursive web crawler)**
+
+| Type | Resource Name | Description |
+|------|---------------|-------------|
+| Single page / recursive crawl | `https://host/path` | By default only the entry page is fetched. Set `args.depth > 0` to crawl same-host links breadth-first; `args.max_pages` only bounds how many pages are collected. Each page is extracted to Markdown with trafilatura, with an automatic Playwright fallback for SPAs whose static HTML carries no body text. Supported `args`: `depth`, `max_pages`, `include_paths`, `exclude_paths`, `allow_external_links`, `skip_download_links`. Download links discovered on pages are skipped by default (`skip_download_links=true`) to avoid importing sidecar files such as `llms.txt`; set it to `false` to download same-host file links and count them toward `max_pages`. `include_paths`/`exclude_paths` use **path-prefix** matching (e.g. `/docs/` matches only paths starting with `/docs/`, never substrings like `/blog/docs-tips`). |
+
+> Routing: sitemap-looking URLs (`https://host/sitemap.xml`, `https://host/feed.xml`, `*.atom`, ...) and explicit `args.site=true` are delegated to the whole-website ingestion below; Git hosting URLs such as `https://github.com/{org}/{repo}` are delegated to the Code section above.
+
**Whole Website (sitemap / RSS / Atom)**
| Type | Resource Name | Description |
@@ -168,7 +176,7 @@ This endpoint is the core entry point for resource management, supporting adding
| exclude | string | No | None | File patterns to exclude (glob) |
| directly_upload_media | bool | No | True | Whether to directly upload media files |
| preserve_structure | bool | No | None | Whether to preserve directory structure |
-| args | object | No | `{}` | Parser-specific import options forwarded to the source parser/accessor. E.g. `args.site=true/false` forces/opts out of whole-site (sitemap/RSS) ingestion, `args.max_pages` etc. override the `webfeed` config, and Feishu user-token imports pass `args.feishu_access_token`. Core `add_resource` fields such as `path`, `to`, `watch_interval`, `include`, and `exclude` are not allowed inside `args` |
+| args | object | No | `{}` | Parser-specific import options forwarded to the source parser/accessor. E.g. `args.site=true/false` forces/opts out of whole-site (sitemap/RSS) ingestion, `args.max_pages` etc. override the `webfeed` config; the recursive web crawler accepts `args.depth`, `args.max_pages`, `args.include_paths`, `args.exclude_paths`, `args.allow_external_links`, `args.skip_download_links`; Feishu user-token imports pass `args.feishu_access_token`. Core `add_resource` fields such as `path`, `to`, `watch_interval`, `include`, and `exclude` are not allowed inside `args` |
| watch_interval | float | No | 0 | Scheduled update interval (minutes). >0 creates task; <=0 cancels task; explicit `to` wins, otherwise binds to the imported `root_uri` |
| telemetry | TelemetryRequest | No | False | Whether to return telemetry data |
@@ -213,6 +221,18 @@ curl -X POST http://localhost:1933/api/v1/resources \
"wait": true
}'
+# Recursively crawl a site: expand along same-host links; depth bounds
+# how many levels, max_pages bounds how many pages are collected
+curl -X POST http://localhost:1933/api/v1/resources \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "path": "https://docs.openviking.ai/getting-started/01-introduction",
+ "wait": true,
+ "timeout": 60,
+ "args": { "depth": 1, "max_pages": 10 }
+ }'
+
# Add from local file (requires temp_upload first)
TEMP_FILE_ID=$(
curl -s -X POST http://localhost:1933/api/v1/resources/temp_upload \
@@ -293,6 +313,26 @@ result = client.add_resource(
reason="External API documentation"
)
+# Recursively crawl a site (same-host BFS; depth levels, max_pages cap)
+result = client.add_resource(
+ "https://docs.openviking.ai/getting-started/01-introduction",
+ wait=True,
+ timeout=180,
+ args={"depth": 1, "max_pages": 10},
+)
+
+# Recursive crawl with path-prefix filters, also downloading file links
+result = client.add_resource(
+ "https://docs.openviking.ai/",
+ args={
+ "depth": 2,
+ "max_pages": 50,
+ "include_paths": ["/docs/"],
+ "exclude_paths": ["/changelog"],
+ "skip_download_links": False,
+ },
+)
+
# Add to the current user's private resource root
result = client.add_resource(
"./documents/guide.md",
@@ -350,6 +390,18 @@ ov add-resource ./documents/guide.md --reason "User guide"
# Add from URL
ov add-resource https://example.com/guide.md --to viking://resources/guide.md
+# Recursively crawl a site: only the entry page is fetched unless depth>0
+ov add-resource "https://docs.openviking.ai/getting-started/01-introduction" \
+ --args="depth:1,max_pages:10"
+
+# Recursive crawl with path-prefix filters (only /docs/, exclude changelog)
+ov add-resource "https://docs.openviking.ai/" \
+ --args='{"depth":2,"max_pages":50,"include_paths":["/docs/"],"exclude_paths":["/changelog"]}'
+
+# Download links on pages are skipped by default; opt in to fetch PDF/TXT/MD etc.
+ov add-resource "https://example.com/docs" \
+ --args="depth:1,max_pages:20,skip_download_links:false"
+
# Wait for processing to complete
ov add-resource ./documents/guide.md --wait
diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md
index b6adeb0ff..f5b66de84 100644
--- a/docs/zh/api/02-resources.md
+++ b/docs/zh/api/02-resources.md
@@ -44,6 +44,13 @@ OpenViking 支持多种资源类型,按照功能分类如下:
|------|------|
| 飞书/Lark | URL 方式,支持 docx, wiki, sheets, bitable。默认使用 FEISHU_APP_ID 和 FEISHU_APP_SECRET 应用凭证;用户 token 导入可传 `args.feishu_access_token`,用户 token watch 还需传 `args.feishu_refresh_token` |
+网页类(递归网页爬虫)
+| 类型 | 资源名 | 说明 |
+|------|--------|------|
+| 单页 / 递归抓取 | `https://host/path` | 默认仅抓入口页;设置 `args.depth > 0` 后,沿同域链接 BFS 递归展开,`args.max_pages` 只限制最多收集的页面数。每页用 trafilatura 抽成 Markdown,对 SPA 等静态 HTML 拿不到正文的站点自动降级到 Playwright 渲染。可选 `args`:`depth`、`max_pages`、`include_paths`、`exclude_paths`、`allow_external_links`、`skip_download_links`。页面中发现的下载链接默认跳过(`skip_download_links=true`),避免导入 `llms.txt` 等 sidecar 文件造成重复;设为 `false` 时会下载同域文件链接,并计入 `max_pages`。`include_paths`/`exclude_paths` 按**路径前缀**匹配(例如 `/docs/` 仅匹配以 `/docs/` 开头的路径,不会误命中 `/blog/docs-tips`)。|
+
+> 路由说明:`https://host/sitemap.xml`、`https://host/feed.xml`、`*.atom` 等 sitemap-looking URL 和显式 `args.site=true` 让出给下表的整站导入;`https://github.com/{org}/{repo}` 等 Git 托管平台 URL 让出给上文的代码导入。
+
网站类(sitemap / RSS / Atom 整站导入)
| 类型 | 资源名 | 说明 |
|------|--------|------|
@@ -162,7 +169,7 @@ URL/文件 Parser TreeBuilder AGFS Summarizer/Vector
| exclude | string | 否 | None | 排除的文件模式(glob) |
| directly_upload_media | bool | 否 | True | 是否直接上传媒体文件 |
| preserve_structure | bool | 否 | None | 是否保留目录结构 |
-| args | object | 否 | `{}` | 传给特定 parser/accessor 的导入参数。例如 `args.site=true/false` 强制/禁用整站(sitemap/RSS)导入,`args.max_pages` 等可覆盖 `webfeed` 配置,飞书用户 token 导入传 `args.feishu_access_token`。`path`、`to`、`watch_interval`、`include`、`exclude` 等 `add_resource` 核心字段不能放入 `args` |
+| args | object | 否 | `{}` | 传给特定 parser/accessor 的导入参数。例如 `args.site=true/false` 强制/禁用整站(sitemap/RSS)导入,`args.max_pages` 等可覆盖 `webfeed` 配置;递归网页爬虫支持 `args.depth`、`args.max_pages`、`args.include_paths`、`args.exclude_paths`、`args.allow_external_links`、`args.skip_download_links`;飞书用户 token 导入传 `args.feishu_access_token`。`path`、`to`、`watch_interval`、`include`、`exclude` 等 `add_resource` 核心字段不能放入 `args` |
| watch_interval | float | 否 | 0 | 定时更新间隔(分钟)。>0 创建任务;≤0 取消任务;显式 `to` 优先,否则绑定本次导入的 `root_uri` |
| telemetry | TelemetryRequest | 否 | False | 是否返回遥测数据 |
@@ -205,6 +212,17 @@ curl -X POST http://localhost:1933/api/v1/resources \
"wait": true
}'
+# 递归抓取网页:从入口页沿同域链接展开,depth 控制层数,max_pages 限制页数
+curl -X POST http://localhost:1933/api/v1/resources \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "path": "https://docs.openviking.ai/zh/getting-started/01-introduction",
+ "wait": true,
+ "timeout": 60,
+ "args": { "depth": 1, "max_pages": 10 }
+ }'
+
# 从本地文件添加(需先使用 temp_upload 上传)
TEMP_FILE_ID=$(
curl -s -X POST http://localhost:1933/api/v1/resources/temp_upload \
@@ -285,6 +303,26 @@ result = client.add_resource(
reason="External API docs"
)
+## 递归抓取网页(同域 BFS,depth 层数、max_pages 页数上限)
+result = client.add_resource(
+ "https://docs.openviking.ai/zh/getting-started/01-introduction",
+ wait=True,
+ timeout=180,
+ args={"depth": 1, "max_pages": 10},
+)
+
+## 递归抓取并按路径前缀过滤,同时下载页面中的文件链接
+result = client.add_resource(
+ "https://docs.openviking.ai/",
+ args={
+ "depth": 2,
+ "max_pages": 50,
+ "include_paths": ["/zh/"],
+ "exclude_paths": ["/changelog"],
+ "skip_download_links": False,
+ },
+)
+
## 添加到当前用户私有资源根
result = client.add_resource(
"./documents/guide.md",
@@ -342,6 +380,18 @@ ov add-resource ./documents/guide.md --reason "User guide"
# 从 URL 添加
ov add-resource https://example.com/guide.md --to viking://resources/guide.md
+# 递归抓取网页:默认只抓入口页,depth>0 才沿同域链接展开
+ov add-resource "https://docs.openviking.ai/zh/getting-started/01-introduction" \
+ --args="depth:1,max_pages:10"
+
+# 递归抓取并按路径前缀过滤(只抓 /zh/,排除 changelog)
+ov add-resource "https://docs.openviking.ai/" \
+ --args='{"depth":2,"max_pages":50,"include_paths":["/zh/"],"exclude_paths":["/changelog"]}'
+
+# 默认跳过页面里的下载链接;如需一并下载 PDF/TXT/MD 等,显式关闭跳过
+ov add-resource "https://example.com/docs" \
+ --args="depth:1,max_pages:20,skip_download_links:false"
+
# 等待处理完成
ov add-resource ./documents/guide.md --wait
diff --git a/openviking/parse/accessors/http_accessor.py b/openviking/parse/accessors/http_accessor.py
index 3265d9c7e..6b5f442a4 100644
--- a/openviking/parse/accessors/http_accessor.py
+++ b/openviking/parse/accessors/http_accessor.py
@@ -29,6 +29,7 @@ from openviking.parse.parsers.media.constants import (
IMAGE_EXTENSIONS,
VIDEO_EXTENSIONS,
)
+from openviking.utils import is_code_hosting_blob_url
from openviking.utils.network_guard import build_httpx_request_validation_hooks
from openviking_cli.exceptions import PermissionDeniedError
from openviking_cli.utils.logger import get_logger
@@ -429,14 +430,14 @@ class HTTPAccessor(DataAccessor):
async def access(self, source: Union[str, Path], **kwargs) -> LocalResource:
"""
- Fetch the HTTP URL to a local file.
+ Fetch the HTTP URL to a local file or directory.
Args:
source: HTTP/HTTPS URL
**kwargs: Additional arguments (request_validator, etc.)
Returns:
- LocalResource pointing to the downloaded file
+ LocalResource pointing to the downloaded file or crawled web directory
"""
source_str = str(source)
request_validator = kwargs.get("request_validator")
@@ -447,6 +448,49 @@ class HTTPAccessor(DataAccessor):
request_validator=request_validator,
)
+ # Both an extensionless text/html page (WEBPAGE) and an explicit
+ # ``.html``/``.htm`` URL (DOWNLOAD_HTML) are webpages the user may want
+ # to crawl: route both through WebImporter so ``depth``/``max_pages``
+ # apply. A plain single-page import is just the ``depth=0`` case.
+ #
+ # Exception: a code-hosting single-file URL (GitHub/GitLab ``blob`` or
+ # GitHub ``raw``) is semantically one file, not a site. ``_download_url``
+ # already rewrote it to raw and fetched the file, so keep it on the
+ # single-file path instead of crawling the hosting UI shell.
+ if url_type in (URLType.WEBPAGE, URLType.DOWNLOAD_HTML) and not is_code_hosting_blob_url(
+ source_str
+ ):
+ from openviking.parse.accessors.web_importer import (
+ WebImporter,
+ parse_web_import_options,
+ )
+
+ try:
+ options = parse_web_import_options(kwargs)
+ result = await WebImporter().import_to_directory(
+ root_url=source_str,
+ options=options,
+ request_validator=request_validator,
+ )
+ finally:
+ Path(temp_path).unlink(missing_ok=True)
+
+ meta.update(result.meta)
+ meta.update(
+ {
+ "url": source_str,
+ "downloaded": True,
+ "url_type": URLType.WEBPAGE.value,
+ }
+ )
+ return LocalResource(
+ path=result.path,
+ source_type=SourceType.HTTP,
+ original_source=source_str,
+ meta=meta,
+ is_temporary=True,
+ )
+
# Build metadata
meta.update(
{
diff --git a/openviking/parse/accessors/web_crawler/__init__.py b/openviking/parse/accessors/web_crawler/__init__.py
new file mode 100644
index 000000000..f2ece8192
--- /dev/null
+++ b/openviking/parse/accessors/web_crawler/__init__.py
@@ -0,0 +1,15 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""HTML crawler internals used by HTTP web imports."""
+
+from openviking.parse.accessors.web_crawler.config import CrawlConfig
+from openviking.parse.accessors.web_crawler.models import CrawledDownload, CrawledPage, CrawlResult
+from openviking.parse.accessors.web_crawler.web_crawler import ScrapyWebCrawler
+
+__all__ = [
+ "CrawlConfig",
+ "CrawledDownload",
+ "CrawledPage",
+ "CrawlResult",
+ "ScrapyWebCrawler",
+]
diff --git a/openviking/parse/accessors/web_crawler/config.py b/openviking/parse/accessors/web_crawler/config.py
new file mode 100644
index 000000000..bcab21bab
--- /dev/null
+++ b/openviking/parse/accessors/web_crawler/config.py
@@ -0,0 +1,47 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Configuration for recursive HTML crawling."""
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class CrawlConfig:
+ depth: int = 0
+ max_pages: int = 50
+ include_paths: Optional[list[str]] = None
+ exclude_paths: Optional[list[str]] = None
+ allow_external_links: bool = False
+ skip_download_links: bool = True
+
+ concurrency: int = 10
+ timeout: float = 10.0
+ download_delay: float = 0.2
+ retry_times: int = 2
+ max_links_per_page: int = 500
+ max_html_bytes: int = 10 * 1024 * 1024
+ fallback_playwright: bool = True
+ playwright_timeout: float = 30.0
+ request_validator: Optional[Callable[[str], None]] = None
+
+ def __post_init__(self) -> None:
+ if self.depth < -1:
+ raise ValueError("depth must be >= -1 (use -1 for unlimited) for recursive web crawling.")
+ if self.max_pages < 1 and self.max_pages != -1:
+ raise ValueError("max_pages must be -1 (unlimited) or >= 1 for recursive web crawling.")
+ if self.concurrency < 1:
+ raise ValueError("concurrency must be >= 1 for recursive web crawling.")
+ if self.timeout <= 0:
+ raise ValueError("timeout must be > 0 for recursive web crawling.")
+ if self.download_delay < 0:
+ raise ValueError("download_delay must be >= 0 for recursive web crawling.")
+ if self.retry_times < 0:
+ raise ValueError("retry_times must be >= 0 for recursive web crawling.")
+ if self.max_links_per_page < 1:
+ raise ValueError("max_links_per_page must be >= 1.")
+ if self.max_html_bytes < 1:
+ raise ValueError("max_html_bytes must be >= 1.")
+ if self.playwright_timeout <= 0:
+ raise ValueError("playwright_timeout must be > 0 for recursive web crawling.")
diff --git a/openviking/parse/accessors/web_crawler/middlewares.py b/openviking/parse/accessors/web_crawler/middlewares.py
new file mode 100644
index 000000000..b9ae22998
--- /dev/null
+++ b/openviking/parse/accessors/web_crawler/middlewares.py
@@ -0,0 +1,19 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Scrapy downloader middlewares for the recursive web crawler."""
+
+from scrapy.exceptions import IgnoreRequest
+
+
+class RequestValidatorMiddleware:
+ """Pre-flight SSRF guard: validate every outbound URL, including redirects/retries."""
+
+ def process_request(self, request, spider):
+ validator = getattr(spider.config, "request_validator", None)
+ if validator is None:
+ return None
+ try:
+ validator(request.url)
+ except Exception as exc:
+ raise IgnoreRequest(f"Blocked by request_validator: {exc}") from exc
+ return None
diff --git a/openviking/parse/accessors/web_crawler/models.py b/openviking/parse/accessors/web_crawler/models.py
new file mode 100644
index 000000000..771bfb877
--- /dev/null
+++ b/openviking/parse/accessors/web_crawler/models.py
@@ -0,0 +1,34 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Shared data models for recursive HTML crawling."""
+
+from dataclasses import dataclass, field
+from typing import Optional
+
+
+@dataclass
+class CrawledPage:
+ url: str = ""
+ final_url: str = ""
+ depth: int = 0
+ status: str = "success"
+ html: Optional[str] = None
+ source: str = "scrapy_static"
+ error: Optional[str] = None
+
+
+@dataclass
+class CrawledDownload:
+ url: str = ""
+ depth: int = 0
+
+
+@dataclass
+class CrawlResult:
+ pages: list[CrawledPage] = field(default_factory=list)
+ downloads: list[CrawledDownload] = field(default_factory=list)
+ total_crawled: int = 0
+ total_downloads: int = 0
+ total_skipped: int = 0
+ total_failed: int = 0
+ fallback_rendered: int = 0
diff --git a/openviking/parse/accessors/web_crawler/playwright_renderer.py b/openviking/parse/accessors/web_crawler/playwright_renderer.py
new file mode 100644
index 000000000..b3063e02f
--- /dev/null
+++ b/openviking/parse/accessors/web_crawler/playwright_renderer.py
@@ -0,0 +1,191 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Playwright renderer used only as low-content fallback."""
+
+import asyncio
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Optional
+
+from openviking.parse.accessors.web_crawler.render_heuristics import (
+ CHALLENGE_MARKERS,
+ SHELL_VISIBLE_TEXT_CHARS,
+)
+
+
+PLAYWRIGHT_PACKAGE_INSTALL_HINT = (
+ "Playwright fallback was needed, but the Python package is not installed. "
+ "Install it with `pip install playwright` and install Chromium with "
+ "`python -m playwright install chromium`."
+)
+PLAYWRIGHT_CHROMIUM_INSTALL_HINT = (
+ "Playwright fallback was needed, but Chromium is not installed or cannot be "
+ "launched. Run `python -m playwright install chromium` and retry."
+)
+
+# Cap the networkidle wait: pages with continuous background activity (e.g. the
+# GraphiQL playground, polling/websocket apps) never go idle and would otherwise
+# block until the full render timeout. Content is usually ready right after
+# domcontentloaded, and ``_wait_past_challenge`` handles late-arriving text.
+_NETWORKIDLE_TIMEOUT_MS = 8000
+
+
+@dataclass
+class RenderResult:
+ html: str = ""
+ status_code: int = 0
+ final_url: str = ""
+ content_type: str = ""
+ error: Optional[str] = None
+
+ @property
+ def is_success(self) -> bool:
+ return 200 <= self.status_code < 400 and self.error is None
+
+
+class PlaywrightRenderer:
+ def __init__(self, request_validator: Optional[Callable[[str], None]] = None) -> None:
+ self._request_validator = request_validator
+ self._playwright_manager = None
+ self._browser = None
+ self._browser_lock = asyncio.Lock()
+
+ async def render(self, url: str, timeout: float) -> RenderResult:
+ page = None
+ try:
+ if self._request_validator:
+ self._request_validator(url)
+ browser = await self._get_browser()
+ page = await browser.new_page(accept_downloads=False)
+
+ if self._request_validator:
+
+ async def _validate_route(route):
+ await self._guard_route(route, self._request_validator)
+
+ await page.route("**/*", _validate_route)
+
+ response = await page.goto(
+ url,
+ wait_until="domcontentloaded",
+ timeout=timeout * 1000,
+ )
+ try:
+ await page.wait_for_load_state("networkidle", timeout=_NETWORKIDLE_TIMEOUT_MS)
+ except Exception:
+ pass
+ await self._wait_past_challenge(page, timeout * 1000)
+ html = await self._read_content(page)
+ final_url = page.url
+ if self._request_validator:
+ self._request_validator(final_url)
+ return RenderResult(
+ html=html,
+ status_code=response.status if response else 200,
+ final_url=final_url,
+ content_type=response.headers.get("content-type", "") if response else "",
+ )
+ except ImportError:
+ return RenderResult(final_url=url, error=PLAYWRIGHT_PACKAGE_INSTALL_HINT)
+ except Exception as exc:
+ error = str(exc)
+ if (
+ "Executable doesn't exist" in error
+ or "playwright install" in error
+ or "BrowserType.launch" in error
+ ):
+ error = PLAYWRIGHT_CHROMIUM_INSTALL_HINT
+ return RenderResult(final_url=url, error=error)
+ finally:
+ if page:
+ try:
+ await page.close()
+ except Exception:
+ pass
+
+ @staticmethod
+ async def _guard_route(route, request_validator) -> None:
+ """SSRF guard for sub-resource requests.
+
+ Block requests to disallowed hosts (e.g. private-network probe
+ endpoints) by aborting them, but never let a blocked sub-resource
+ fail the whole page render. Only the main document URL and the final
+ URL (validated in ``render``) gate the overall result.
+ """
+ try:
+ request_validator(route.request.url)
+ except Exception:
+ await route.abort()
+ return
+ await route.continue_()
+
+ @staticmethod
+ async def _wait_past_challenge(page, timeout_ms: float, poll_ms: int = 500) -> None:
+ """Wait out JS anti-bot interstitials (e.g. "Please wait...").
+
+ These challenge pages run a CPU-bound JS proof-of-work and then
+ auto-redirect to the real content without further network activity,
+ so ``networkidle`` returns while the interstitial is still showing.
+ Poll the body text until real content appears or we run out of time.
+ """
+ import time
+
+ deadline = time.monotonic() + max(timeout_ms, 0) / 1000
+ while True:
+ try:
+ body = (await page.inner_text("body")).strip()
+ except Exception:
+ body = ""
+ lowered = body.lower()
+ looks_like_challenge = (
+ not body
+ or len(body) < SHELL_VISIBLE_TEXT_CHARS
+ or any(marker in lowered for marker in CHALLENGE_MARKERS)
+ )
+ if not looks_like_challenge or time.monotonic() >= deadline:
+ return
+ await page.wait_for_timeout(poll_ms)
+
+ @staticmethod
+ async def _read_content(page) -> str:
+ """Read page HTML, retrying through in-flight client-side navigation.
+
+ Heavy SPAs (client-side redirects, late hydration) can still be
+ navigating when we first ask for content, which raises "page is
+ navigating and changing the content". Retry until the page settles.
+ """
+ last_exc = None
+ for _ in range(6):
+ try:
+ return await page.content()
+ except Exception as exc:
+ if "navigating and changing the content" not in str(exc):
+ raise
+ last_exc = exc
+ try:
+ await page.wait_for_load_state("load", timeout=5000)
+ except Exception:
+ pass
+ await page.wait_for_timeout(400)
+ if last_exc:
+ raise last_exc
+ return await page.content()
+
+ async def close(self) -> None:
+ async with self._browser_lock:
+ if self._browser:
+ await self._browser.close()
+ self._browser = None
+ if self._playwright_manager:
+ await self._playwright_manager.stop()
+ self._playwright_manager = None
+
+ async def _get_browser(self):
+ async with self._browser_lock:
+ if self._browser is None or not self._browser.is_connected():
+ from playwright.async_api import async_playwright
+
+ if self._playwright_manager is None:
+ self._playwright_manager = await async_playwright().start()
+ self._browser = await self._playwright_manager.chromium.launch(headless=True)
+ return self._browser
diff --git a/openviking/parse/accessors/web_crawler/render_heuristics.py b/openviking/parse/accessors/web_crawler/render_heuristics.py
new file mode 100644
index 000000000..91283e54f
--- /dev/null
+++ b/openviking/parse/accessors/web_crawler/render_heuristics.py
@@ -0,0 +1,83 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Heuristics for deciding whether a page likely needs browser rendering."""
+
+import re
+
+from bs4 import BeautifulSoup
+
+
+_SPA_EMPTY_PATTERNS = (
+ "You need to enable JavaScript to run this app.",
+ "This app works best with JavaScript enabled.",
+ "Please enable JavaScript to continue.",
+ "JavaScript is required to use this application.",
+ "Enable JavaScript to view this page.",
+)
+
+_MIN_VISIBLE_TEXT_CHARS = 200
+SHELL_VISIBLE_TEXT_CHARS = 40
+
+# Text markers of JS anti-bot interstitials that briefly show before the real
+# page loads (e.g. volcengine's proof-of-work gate renders only "Please wait...").
+CHALLENGE_MARKERS = (
+ "please wait",
+ "checking your browser",
+ "verifying you are human",
+ "just a moment",
+ "attention required",
+)
+
+
+def should_render_with_playwright(html: str) -> bool:
+ """Return True when static HTML looks like a client-rendered shell."""
+ html = html or ""
+ html_lower = html.lower()
+ if any(pattern.lower() in html_lower for pattern in _SPA_EMPTY_PATTERNS):
+ return True
+ visible_len = visible_body_text_len(html)
+ if visible_len < SHELL_VISIBLE_TEXT_CHARS:
+ return True
+ # __NEXT_DATA__ only signals a Next.js app; most such pages are SSR/SSG and
+ # already ship full body text in the static HTML. Only render when the
+ # static body is also too thin to be the real content.
+ if "__next_data__" in html_lower and visible_len < _MIN_VISIBLE_TEXT_CHARS:
+ return True
+ if re.search(r'id=["\'](?:root|app)["\']', html_lower) and visible_len < _MIN_VISIBLE_TEXT_CHARS:
+ return True
+ if html_lower.count("<script") >= 5 and visible_len < _MIN_VISIBLE_TEXT_CHARS:
+ return True
+ return False
+
+
+def looks_like_unrendered_page(html: str) -> bool:
+ """Return True when HTML is an empty shell or an anti-bot challenge page.
+
+ Used to reject content that must not be stored as real page text: the
+ static SPA shell or a JS interstitial such as "Please wait...".
+ """
+ try:
+ html = html or ""
+ soup = BeautifulSoup(html, "html.parser")
+ body = soup.body or soup
+ for el in body(["script", "style", "noscript"]):
+ el.decompose()
+ text = body.get_text(strip=True)
+ lowered = text.lower()
+ if any(marker in lowered for marker in CHALLENGE_MARKERS):
+ return True
+ return len(text) < SHELL_VISIBLE_TEXT_CHARS
+ except Exception:
+ return True
+
+
+def visible_body_text_len(html: str) -> int:
+ """Approximate visible text length without running page JavaScript."""
+ try:
+ soup = BeautifulSoup(html or "", "html.parser")
+ body = soup.body or soup
+ for el in body(["script", "style", "noscript"]):
+ el.decompose()
+ return len(body.get_text(strip=True))
+ except Exception:
+ return 0
diff --git a/openviking/parse/accessors/web_crawler/scrapy_spider.py b/openviking/parse/accessors/web_crawler/scrapy_spider.py
new file mode 100644
index 000000000..2b72e70e9
--- /dev/null
+++ b/openviking/parse/accessors/web_crawler/scrapy_spider.py
@@ -0,0 +1,297 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Scrapy spider for recursive web resource import."""
+
+import asyncio
+import os
+from collections.abc import Iterator
+from urllib.parse import urljoin, urlparse
+
+import scrapy
+from parsel import Selector
+from scrapy import signals
+from scrapy.exceptions import CloseSpider
+
+from openviking.parse.accessors.http_accessor import URLType, URLTypeDetector
+from openviking.parse.accessors.web_crawler.config import CrawlConfig
+from openviking.parse.accessors.web_crawler.models import CrawledDownload, CrawledPage
+from openviking.parse.accessors.web_crawler.playwright_renderer import PlaywrightRenderer
+from openviking.parse.accessors.web_crawler.render_heuristics import (
+ looks_like_unrendered_page,
+ should_render_with_playwright,
+)
+
+
+_DOWNLOAD_URL_TYPES = frozenset(
+ {
+ URLType.DOWNLOAD_PDF,
+ URLType.DOWNLOAD_MD,
+ URLType.DOWNLOAD_TXT,
+ URLType.DOWNLOAD_DOCUMENT,
+ }
+)
+_DOWNLOAD_EXTENSIONS = frozenset(
+ ext
+ for ext, url_type in URLTypeDetector.EXTENSION_MAP.items()
+ if url_type in _DOWNLOAD_URL_TYPES
+)
+
+
+class OpenVikingWebSpider(scrapy.Spider):
+ name = "openviking_web"
+
+ ASSET_EXTENSIONS = frozenset(
+ {
+ ".css",
+ ".js",
+ ".map",
+ ".mjs",
+ ".png",
+ ".jpg",
+ ".jpeg",
+ ".gif",
+ ".svg",
+ ".ico",
+ ".webp",
+ ".avif",
+ ".bmp",
+ ".woff",
+ ".woff2",
+ ".ttf",
+ ".eot",
+ ".mp3",
+ ".mp4",
+ }
+ )
+ DOWNLOAD_EXTENSIONS = _DOWNLOAD_EXTENSIONS
+
+ def __init__(
+ self,
+ root_url: str,
+ config: CrawlConfig,
+ collector: list[CrawledPage],
+ download_collector: list[CrawledDownload],
+ **kwargs,
+ ):
+ super().__init__(**kwargs)
+ self.root_url = root_url
+ self.config = config
+ self.collector = collector
+ self.download_collector = download_collector
+ self.renderer = PlaywrightRenderer(config.request_validator)
+ self.root_host = urlparse(root_url).netloc.lower()
+ self._success_count = 0
+ self._seen_download_urls: set[str] = set()
+ self._render_semaphore: asyncio.Semaphore | None = None
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ crawler.signals.connect(spider._on_spider_closed, signal=signals.spider_closed)
+ return spider
+
+ async def _on_spider_closed(self, spider, reason):
+ try:
+ await asyncio.wait_for(self.renderer.close(), timeout=5.0)
+ except Exception:
+ pass
+
+ def _success_at_limit(self) -> bool:
+ return 0 < self.config.max_pages <= self._success_count
+
+ def _stop_if_success_at_limit(self) -> bool:
+ if not self._success_at_limit():
+ return False
+ if getattr(self, "crawler", None) and getattr(self.crawler, "engine", None):
+ raise CloseSpider("max_pages_reached")
+ return True
+
+ async def start(self):
+ yield scrapy.Request(self.root_url, callback=self.parse, errback=self.errback)
+
+ async def parse(self, response):
+ if self._stop_if_success_at_limit():
+ return
+ final_url = response.url
+ depth = int(response.meta.get("depth", 0) or 0)
+ try:
+ self._validate_url(final_url)
+ except Exception as exc:
+ self._add_failed(response.url, final_url, depth, str(exc))
+ return
+
+ if not self._is_html_response(response):
+ self._add_skipped(final_url, depth, "non-html response")
+ return
+ if len(response.body or b"") > self.config.max_html_bytes:
+ self._add_skipped(final_url, depth, "html too large")
+ return
+
+ page_html = response.text
+ page_source = "scrapy_static"
+ needs_render = self.config.fallback_playwright and should_render_with_playwright(
+ response.text
+ )
+ try:
+ if needs_render:
+ if self._render_semaphore is None:
+ self._render_semaphore = asyncio.Semaphore(self.config.concurrency)
+ async with self._render_semaphore:
+ if self._success_at_limit():
+ return
+ rendered = await self.renderer.render(
+ final_url, self.config.playwright_timeout
+ )
+ if rendered.is_success and rendered.html:
+ final_url = rendered.final_url or final_url
+ page_html = rendered.html
+ page_source = "playwright"
+ elif rendered.error:
+ self._add_failed(response.url, final_url, depth, rendered.error)
+ return
+ except Exception as exc:
+ self._add_failed(response.url, final_url, depth, str(exc))
+ return
+
+ if needs_render and looks_like_unrendered_page(page_html):
+ self._add_failed(
+ response.url,
+ final_url,
+ depth,
+ "page did not render real content (empty shell or anti-bot challenge page)",
+ )
+ return
+
+ if self._stop_if_success_at_limit():
+ return
+ self.collector.append(
+ CrawledPage(
+ url=response.url,
+ final_url=final_url,
+ depth=depth,
+ html=page_html,
+ source=page_source,
+ )
+ )
+ self._success_count += 1
+
+ if self._stop_if_success_at_limit():
+ return
+ if self.config.depth >= 0 and depth >= self.config.depth:
+ return
+ for url in self._extract_download_urls(page_html, final_url, depth + 1):
+ self.download_collector.append(CrawledDownload(url=url, depth=depth + 1))
+ self._success_count += 1
+ if self._stop_if_success_at_limit():
+ return
+ for url in self._extract_child_urls(page_html, final_url):
+ yield response.follow(
+ url,
+ callback=self.parse,
+ errback=self.errback,
+ )
+
+ def errback(self, failure):
+ request = failure.request
+ depth = int(request.meta.get("depth", 0) or 0)
+ self._add_failed(request.url, request.url, depth, failure.getErrorMessage())
+
+ def _iter_unique_links(self, html: str, base_url: str) -> Iterator[str]:
+ try:
+ hrefs = Selector(text=html or "").css("a::attr(href)").getall()
+ except Exception:
+ hrefs = []
+ seen: set[str] = set()
+ for href in hrefs:
+ href = (href or "").strip()
+ if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
+ continue
+ url = urljoin(base_url, href).split("#", 1)[0]
+ if url in seen:
+ continue
+ seen.add(url)
+ yield url
+
+ def _extract_child_urls(self, html: str, base_url: str) -> list[str]:
+ accepted: list[str] = []
+ for url in self._iter_unique_links(html, base_url):
+ if self._accept_child_url(url):
+ accepted.append(url)
+ if len(accepted) >= self.config.max_links_per_page:
+ break
+ return accepted
+
+ def _extract_download_urls(self, html: str, base_url: str, depth: int) -> list[str]:
+ if self.config.skip_download_links:
+ return []
+ accepted: list[str] = []
+ for url in self._iter_unique_links(html, base_url):
+ if self._success_at_limit():
+ break
+ if url in self._seen_download_urls or not self._accept_download_url(url):
+ continue
+ self._seen_download_urls.add(url)
+ accepted.append(url)
+ if len(accepted) >= self.config.max_links_per_page:
+ break
+ return accepted
+
+ def _accept_child_url(self, url: str) -> bool:
+ parsed = urlparse(url)
+ ext = os.path.splitext(parsed.path.lower())[1]
+ if ext in self.ASSET_EXTENSIONS or ext in self.DOWNLOAD_EXTENSIONS:
+ return False
+ return self._passes_url_filters(url, parsed)
+
+ def _accept_download_url(self, url: str) -> bool:
+ parsed = urlparse(url)
+ ext = os.path.splitext(parsed.path.lower())[1]
+ if ext not in self.DOWNLOAD_EXTENSIONS:
+ return False
+ return self._passes_url_filters(url, parsed)
+
+ def _passes_url_filters(self, url: str, parsed) -> bool:
+ if parsed.scheme not in ("http", "https"):
+ return False
+ if not self.config.allow_external_links and parsed.netloc.lower() != self.root_host:
+ return False
+ if self.config.include_paths and not self._matches_any(
+ parsed.path, self.config.include_paths
+ ):
+ return False
+ if self.config.exclude_paths and self._matches_any(parsed.path, self.config.exclude_paths):
+ return False
+ try:
+ self._validate_url(url)
+ except Exception:
+ return False
+ return True
+
+ def _validate_url(self, url: str) -> None:
+ if self.config.request_validator:
+ self.config.request_validator(url)
+
+ @staticmethod
+ def _matches_any(path: str, patterns: list[str]) -> bool:
+ return any(pattern and path.startswith(pattern) for pattern in patterns)
+
+ @staticmethod
+ def _is_html_response(response) -> bool:
+ content_type = response.headers.get("content-type", b"").decode("latin1").lower()
+ return not content_type or "html" in content_type
+
+ def _add_skipped(self, url: str, depth: int, reason: str) -> None:
+ self.collector.append(
+ CrawledPage(url=url, final_url=url, depth=depth, status="skipped", error=reason)
+ )
+
+ def _add_failed(self, url: str, final_url: str, depth: int, error: str) -> None:
+ self.collector.append(
+ CrawledPage(
+ url=url,
+ final_url=final_url,
+ depth=depth,
+ status="failed",
+ error=error,
+ )
+ )
diff --git a/openviking/parse/accessors/web_crawler/web_crawler.py b/openviking/parse/accessors/web_crawler/web_crawler.py
new file mode 100644
index 000000000..39ee7345d
--- /dev/null
+++ b/openviking/parse/accessors/web_crawler/web_crawler.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Scrapy-backed recursive web crawler."""
+
+import asyncio
+import multiprocessing
+import queue
+
+from openviking.parse.accessors.web_crawler.config import CrawlConfig
+from openviking.parse.accessors.web_crawler.models import (
+ CrawledDownload,
+ CrawledPage,
+ CrawlResult,
+)
+
+
+def _run_crawl_worker(root_url: str, config: CrawlConfig, result_queue) -> None:
+ from scrapy.utils.reactor import install_reactor
+
+ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+
+ from scrapy.crawler import CrawlerProcess
+
+ from openviking.parse.accessors.web_crawler.scrapy_spider import OpenVikingWebSpider
+
+ pages: list[CrawledPage] = []
+ downloads: list[CrawledDownload] = []
+ process = CrawlerProcess(_build_settings(config))
+ crawler = process.create_crawler(OpenVikingWebSpider)
+ process.crawl(
+ crawler,
+ root_url=root_url,
+ config=config,
+ collector=pages,
+ download_collector=downloads,
+ )
+ try:
+ process.start()
+ except Exception as exc:
+ result_queue.put((pages, downloads, str(exc)))
+ return
+ result_queue.put((pages, downloads, None))
+
+
+def _build_settings(config: CrawlConfig):
+ from scrapy.settings import Settings
+
+ depth_limit = 0 if config.depth < 0 else config.depth
+ return Settings(
+ {
+ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+ "DEPTH_LIMIT": depth_limit,
+ # max_pages is enforced inside the spider by successful-resource count
+ # (OpenVikingWebSpider._success_at_limit). Scrapy's
+ # CLOSESPIDER_PAGECOUNT counts every response, including failed and
+ # skipped ones, so enabling it would stop the crawl on a different
+ # metric and yield fewer successful pages than requested. Keep it
+ # disabled so the spider is the single source of truth.
+ "CLOSESPIDER_PAGECOUNT": 0,
+ "CONCURRENT_REQUESTS": config.concurrency,
+ "DOWNLOAD_TIMEOUT": config.timeout,
+ "DOWNLOAD_DELAY": config.download_delay,
+ "RETRY_ENABLED": True,
+ "RETRY_TIMES": config.retry_times,
+ "ROBOTSTXT_OBEY": True,
+ "LOG_ENABLED": False,
+ "TELNETCONSOLE_ENABLED": False,
+ "USER_AGENT": "OpenViking/0.4 (+recursive-web-crawler)",
+ "DOWNLOADER_MIDDLEWARES": {
+ "openviking.parse.accessors.web_crawler.middlewares."
+ "RequestValidatorMiddleware": 50,
+ },
+ }
+ )
+
+
+class ScrapyWebCrawler:
+ def __init__(self, config: CrawlConfig) -> None:
+ self.config = config
+
+ async def crawl(self, root_url: str) -> CrawlResult:
+ context = multiprocessing.get_context("spawn")
+ result_queue = context.Queue()
+ process = context.Process(
+ target=_run_crawl_worker,
+ args=(root_url, self.config, result_queue),
+ )
+ process.start()
+ try:
+ pages, downloads, error = await self._wait_worker_result(process, result_queue)
+ if error:
+ raise RuntimeError(error)
+ if process.exitcode not in (0, None):
+ raise RuntimeError(f"Scrapy crawler exited with code {process.exitcode}.")
+ return self._build_result(pages, downloads)
+ finally:
+ await self._cleanup_worker(process, result_queue)
+
+ @staticmethod
+ async def _wait_worker_result(
+ process,
+ result_queue,
+ ) -> tuple[list[CrawledPage], list[CrawledDownload], str | None]:
+ while True:
+ try:
+ pages, downloads, error = await asyncio.to_thread(result_queue.get, True, 0.2)
+ await asyncio.to_thread(process.join)
+ return pages, downloads, error
+ except queue.Empty:
+ if process.is_alive():
+ continue
+ await asyncio.to_thread(process.join)
+ return ScrapyWebCrawler._read_worker_result(result_queue)
+
+ @staticmethod
+ def _read_worker_result(
+ result_queue,
+ ) -> tuple[list[CrawledPage], list[CrawledDownload], str | None]:
+ try:
+ return result_queue.get(True, 0.2)
+ except queue.Empty:
+ return [], [], "Scrapy crawler did not return a result."
+
+ @staticmethod
+ async def _cleanup_worker(process, result_queue) -> None:
+ if process.is_alive():
+ process.terminate()
+ await asyncio.to_thread(process.join, 5.0)
+ if process.is_alive():
+ process.kill()
+ await asyncio.to_thread(process.join, 5.0)
+ try:
+ result_queue.close()
+ result_queue.join_thread()
+ except Exception:
+ pass
+
+ @staticmethod
+ def _build_result(pages: list[CrawledPage], downloads: list[CrawledDownload]) -> CrawlResult:
+ result = CrawlResult(pages=pages, downloads=downloads)
+ result.total_downloads = len(downloads)
+ for page in pages:
+ if page.status == "success":
+ result.total_crawled += 1
+ elif page.status == "skipped":
+ result.total_skipped += 1
+ else:
+ result.total_failed += 1
+ if page.source == "playwright":
+ result.fallback_rendered += 1
+ return result
diff --git a/openviking/parse/accessors/web_importer.py b/openviking/parse/accessors/web_importer.py
new file mode 100644
index 000000000..ea3e61acd
--- /dev/null
+++ b/openviking/parse/accessors/web_importer.py
@@ -0,0 +1,374 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Import ordinary web pages as a temporary local HTML directory."""
+
+import hashlib
+import re
+import shutil
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, Optional
+from urllib.parse import unquote, urlsplit, urlunsplit
+
+from openviking.parse.accessors.http_accessor import HTTPAccessor
+from openviking.parse.accessors.web_crawler import CrawlConfig, ScrapyWebCrawler
+from openviking.parse.accessors.web_crawler.playwright_renderer import (
+ PLAYWRIGHT_CHROMIUM_INSTALL_HINT,
+ PLAYWRIGHT_PACKAGE_INSTALL_HINT,
+)
+from openviking.parse.accessors.web_feed_accessor import (
+ FeedEntry,
+ _dedup_relpath,
+ _entry_to_relpath,
+ _sanitize_filename,
+ url_to_relpath,
+)
+from openviking_cli.exceptions import InvalidArgumentError
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+DEPTH_UNLIMITED = -1
+MAX_PAGES_UNLIMITED = -1
+
+# Actionable install hints the renderer records on CrawledPage.error when the
+# Playwright fallback is needed but unavailable. These are surfaced to the user
+# even for non-entry pages, which would otherwise only show up as a failure
+# count.
+_RENDER_INSTALL_HINTS = frozenset(
+ {
+ PLAYWRIGHT_PACKAGE_INSTALL_HINT,
+ PLAYWRIGHT_CHROMIUM_INSTALL_HINT,
+ }
+)
+
+
+@dataclass(frozen=True)
+class WebImportOptions:
+ depth: int = 0
+ max_pages: int = 50
+ include_paths: Optional[list[str]] = None
+ exclude_paths: Optional[list[str]] = None
+ allow_external_links: bool = False
+ skip_download_links: bool = True
+
+
+@dataclass(frozen=True)
+class WebImportResult:
+ path: Path
+ meta: Dict[str, Any]
+
+
+def parse_web_import_options(kwargs: Dict[str, Any]) -> WebImportOptions:
+ """Pop web importer options from accessor kwargs."""
+ depth = _pop_int_arg(kwargs, "depth", 0, min_value=-1)
+ max_pages = _pop_int_arg(kwargs, "max_pages", 50, min_value=-1)
+ if max_pages == 0:
+ raise InvalidArgumentError("args.max_pages must be -1 (unlimited) or >= 1.")
+ allow_external_links = _pop_bool_arg(kwargs, "allow_external_links", False)
+ skip_download_links = _pop_bool_arg(kwargs, "skip_download_links", True)
+ if (
+ depth == DEPTH_UNLIMITED
+ and max_pages == MAX_PAGES_UNLIMITED
+ and allow_external_links
+ ):
+ raise InvalidArgumentError(
+ "args.depth=-1 and args.max_pages=-1 cannot be combined with "
+ "allow_external_links=true; at least one bound is required when "
+ "crawling external sites."
+ )
+ return WebImportOptions(
+ depth=depth,
+ max_pages=max_pages,
+ include_paths=_pop_optional_patterns(kwargs, "include_paths"),
+ exclude_paths=_pop_optional_patterns(kwargs, "exclude_paths"),
+ allow_external_links=allow_external_links,
+ skip_download_links=skip_download_links,
+ )
+
+
+class WebImporter:
+ """Crawl ordinary web pages and materialize successful pages as HTML files."""
+
+ async def import_to_directory(
+ self,
+ *,
+ root_url: str,
+ options: WebImportOptions,
+ request_validator=None,
+ ) -> WebImportResult:
+ config = CrawlConfig(
+ depth=options.depth,
+ max_pages=options.max_pages,
+ include_paths=options.include_paths,
+ exclude_paths=options.exclude_paths,
+ allow_external_links=options.allow_external_links,
+ skip_download_links=options.skip_download_links,
+ request_validator=request_validator,
+ )
+ crawl_result = await ScrapyWebCrawler(config).crawl(root_url)
+ success_pages = self._dedupe_success_pages(crawl_result.pages)
+ if not any(page.depth == 0 for page in success_pages):
+ detail = self._entry_failure_detail(crawl_result.pages)
+ message = f"Failed to fetch entry page: {root_url}"
+ if detail:
+ message = f"{message} ({detail})"
+ raise RuntimeError(message)
+
+ render_hints = self._render_install_hints(crawl_result.pages)
+ if render_hints:
+ logger.warning(
+ "Some pages could not be rendered during web import of %s and were "
+ "skipped: %s",
+ root_url,
+ " ".join(render_hints),
+ )
+
+ temp_root = Path(tempfile.mkdtemp(prefix="ov_web_"))
+ temp_dir = temp_root / _host_name(root_url)
+ try:
+ temp_dir.mkdir(parents=True, exist_ok=True)
+ used_relpaths: set[str] = set()
+ for page in success_pages:
+ page_url = page.final_url or page.url
+ relpath = self._page_relpath(page)
+ relpath = _dedup_relpath(relpath, _normalize_page_url(page_url), used_relpaths)
+ used_relpaths.add(relpath)
+ dest = self._safe_dest(temp_dir, relpath)
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ dest.write_text(page.html or "", encoding="utf-8")
+ downloaded_files = await self._write_downloads(
+ crawl_result.downloads,
+ temp_dir,
+ used_relpaths,
+ request_validator=request_validator,
+ )
+ except Exception:
+ shutil.rmtree(temp_root, ignore_errors=True)
+ raise
+
+ return WebImportResult(
+ path=temp_dir,
+ meta={
+ "web_import": True,
+ "page_count": len(success_pages),
+ "download_count": len(downloaded_files),
+ "crawl_result": _crawl_summary(crawl_result),
+ "render_hints": render_hints,
+ "original_filename": _host_name(root_url),
+ },
+ )
+
+ @staticmethod
+ def _render_install_hints(pages) -> list[str]:
+ """Collect actionable render-install hints from failed pages (any depth).
+
+ Child SPA pages that fail because Playwright is unavailable only record
+ the hint on ``CrawledPage.error``; the entry page still succeeds, so the
+ caller would otherwise see the hint nowhere. Surface each distinct hint
+ once so the user can act on it.
+ """
+ hints: list[str] = []
+ for page in pages:
+ if page.status == "success" or not page.error:
+ continue
+ if page.error in _RENDER_INSTALL_HINTS and page.error not in hints:
+ hints.append(page.error)
+ return hints
+
+ @staticmethod
+ def _entry_failure_detail(pages) -> str:
+ """Return the failure reason of the entry page, if one was recorded.
+
+ The renderer surfaces actionable hints (e.g. missing Playwright
+ install) via ``CrawledPage.error``; without this the caller only sees
+ the generic "Failed to fetch entry page" message.
+ """
+ for page in pages:
+ if page.depth == 0 and page.status != "success" and page.error:
+ return page.error
+ return ""
+
+ @staticmethod
+ def _dedupe_success_pages(pages) -> list[Any]:
+ seen: set[str] = set()
+ deduped: list[Any] = []
+ for page in pages:
+ if page.status != "success":
+ continue
+ key = _normalize_page_url(page.final_url or page.url)
+ if key in seen:
+ continue
+ seen.add(key)
+ deduped.append(page)
+ return deduped
+
+ @staticmethod
+ def _safe_dest(root: Path, relpath: str) -> Path:
+ dest = (root / relpath).resolve()
+ root_resolved = root.resolve()
+ try:
+ dest.relative_to(root_resolved)
+ except ValueError as exc:
+ digest = hashlib.sha1(relpath.encode("utf-8")).hexdigest()[:8]
+ dest = root_resolved / f"page-{digest}.html"
+ return dest
+
+ @staticmethod
+ def _page_relpath(page: Any) -> str:
+ page_url = page.final_url or page.url
+ title = _extract_page_title(page.html or "")
+ if title:
+ return _entry_to_relpath(FeedEntry(url=page_url, title=title))
+ return url_to_relpath(page_url)
+
+ async def _write_downloads(
+ self,
+ downloads: list[Any],
+ temp_dir: Path,
+ used_relpaths: set[str],
+ *,
+ request_validator=None,
+ ) -> list[str]:
+ http_accessor = HTTPAccessor()
+ written: list[str] = []
+ for download in self._dedupe_downloads(downloads):
+ temp_path, _url_type, meta = await http_accessor._download_url(
+ download.url,
+ request_validator=request_validator,
+ )
+ try:
+ relpath = _download_relpath(download.url, meta)
+ relpath = _dedup_relpath(relpath, _normalize_page_url(download.url), used_relpaths)
+ used_relpaths.add(relpath)
+ dest = self._safe_dest(temp_dir, relpath)
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ shutil.move(temp_path, dest)
+ written.append(relpath)
+ finally:
+ Path(temp_path).unlink(missing_ok=True)
+ return written
+
+ @staticmethod
+ def _dedupe_downloads(downloads: list[Any]) -> list[Any]:
+ seen: set[str] = set()
+ deduped: list[Any] = []
+ for download in downloads:
+ key = _normalize_page_url(download.url)
+ if key in seen:
+ continue
+ seen.add(key)
+ deduped.append(download)
+ return deduped
+
+
+def _crawl_summary(crawl_result: Any) -> Dict[str, Any]:
+ return {
+ "total_crawled": crawl_result.total_crawled,
+ "total_downloads": getattr(crawl_result, "total_downloads", 0),
+ "total_failed": crawl_result.total_failed,
+ "total_skipped": crawl_result.total_skipped,
+ "fallback_rendered": crawl_result.fallback_rendered,
+ }
+
+
+def _normalize_page_url(url: str) -> str:
+ parts = urlsplit(url or "")
+ scheme = parts.scheme.lower()
+ hostname = (parts.hostname or "").lower().rstrip(".")
+ port = parts.port
+ if (scheme == "http" and port == 80) or (scheme == "https" and port == 443):
+ port = None
+ netloc = hostname if port is None else f"{hostname}:{port}"
+ return urlunsplit((scheme, netloc, parts.path or "/", parts.query, ""))
+
+
+def _host_name(url: str) -> str:
+ host = urlsplit(url).hostname or "web"
+ return host.rstrip(".").lower() or "web"
+
+
+def _download_relpath(url: str, meta: Dict[str, Any]) -> str:
+ parts = urlsplit(url)
+ segments = [unquote(segment) for segment in parts.path.split("/") if segment]
+ original_filename = meta.get("original_filename") or "download"
+ filename = _sanitize_filename(str(original_filename)) or "download"
+ if segments:
+ parents = [_sanitize_filename(segment) for segment in segments[:-1]]
+ parents = [segment for segment in parents if segment]
+ return "/".join(parents + [filename])
+ return filename
+
+
+def _extract_page_title(html: str) -> Optional[str]:
+ """Extract a readable page title for local filename selection.
+
+ This deliberately stays lightweight and does not use trafilatura; full
+ content/title extraction remains the HTMLParser's responsibility.
+ """
+ if not html:
+ return None
+ try:
+ from bs4 import BeautifulSoup
+
+ soup = BeautifulSoup(html, "html.parser")
+ for selector in ("h1", "title"):
+ node = soup.select_one(selector)
+ if node:
+ title = _clean_title(node.get_text(" ", strip=True))
+ if title:
+ return title
+ except Exception:
+ pass
+ match = re.search(r"<title[^>]*>(.*?)</title>", html, flags=re.IGNORECASE | re.DOTALL)
+ if match:
+ title = re.sub(r"<[^>]+>", "", match.group(1))
+ return _clean_title(title)
+ return None
+
+
+def _clean_title(title: str) -> Optional[str]:
+ title = re.sub(r"\s+", " ", title or "").strip()
+ return title or None
+
+
+def _pop_int_arg(
+ args: Dict[str, Any],
+ name: str,
+ default: int,
+ *,
+ min_value: Optional[int] = None,
+) -> int:
+ value = args.pop(name, None)
+ if value in (None, ""):
+ value = default
+ try:
+ value = int(value)
+ except (TypeError, ValueError) as exc:
+ raise InvalidArgumentError(f"args.{name} must be an integer.") from exc
+ if min_value is not None and value < min_value:
+ raise InvalidArgumentError(f"args.{name} must be >= {min_value}.")
+ return value
+
+
+def _pop_bool_arg(args: Dict[str, Any], name: str, default: bool) -> bool:
+ value = args.pop(name, default)
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, str) and value.strip().lower() in {"true", "false"}:
+ return value.strip().lower() == "true"
+ raise InvalidArgumentError(f"args.{name} must be a boolean.")
+
+
+def _pop_optional_patterns(args: Dict[str, Any], name: str) -> Optional[list[str]]:
+ value = args.pop(name, None)
+ if value is None or value == "":
+ return None
+ if isinstance(value, str):
+ patterns = [part.strip() for part in value.split(",")]
+ elif isinstance(value, list):
+ patterns = [str(part).strip() for part in value]
+ else:
+ raise InvalidArgumentError(f"args.{name} must be a string or list of strings.")
+ return [pattern for pattern in patterns if pattern] or None
diff --git a/openviking/parse/parsers/README.md b/openviking/parse/parsers/README.md
index 9fb5637fb..1508cdad7 100644
--- a/openviking/parse/parsers/README.md
+++ b/openviking/parse/parsers/README.md
@@ -131,7 +131,7 @@ L1: """
### 3. HTMLParser (`html.py`)
**支持格式**: `.html`, `.htm`
-使用 `readabilipy` 库提取可读内容,去除广告、导航等噪音元素,转换为 Markdown 后通过 MarkdownParser 处理。
+使用 `trafilatura` 抽取正文,去除广告、导航等噪音元素,转换为 Markdown 后通过 MarkdownParser 处理。
### 4. TextParser (`text.py`)
**支持格式**: `.txt`, `.text`, `.log`, `.csv`, `.tsv`, `.json`, `.yaml`, `.yml`, `.xml`, `.ini`, `.cfg`, `.conf`
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py
index 4988b5b3f..a169cd276 100644
--- a/openviking/parse/parsers/html.py
+++ b/openviking/parse/parsers/html.py
@@ -8,6 +8,7 @@ Parses local HTML files.
For URL downloading, use HTTPAccessor in the new two-layer architecture.
"""
+import re
import time
from pathlib import Path
from typing import List, Optional, Union
@@ -23,13 +24,13 @@ from openviking_cli.utils.config.parser_config import HTMLConfig
logger = __import__("openviking_cli.utils.logger").utils.logger.get_logger(__name__)
-
-# Backward compatibility: Re-export URLType and URLTypeDetector from http_accessor
-try:
- from openviking.parse.accessors.http_accessor import URLType, URLTypeDetector
-except ImportError:
- URLType = None
- URLTypeDetector = None
+_SPA_EMPTY_PATTERNS = (
+ "You need to enable JavaScript to run this app.",
+ "This app works best with JavaScript enabled.",
+ "Please enable JavaScript to continue.",
+ "JavaScript is required to use this application.",
+ "Enable JavaScript to view this page.",
+)
class HTMLParser(BaseParser):
@@ -63,55 +64,6 @@ class HTMLParser(BaseParser):
self.config = config or HTMLConfig()
self._markdown_parser = None
- @staticmethod
- def _extract_filename_from_url(url: str) -> str:
- """
- Extract and URL-decode the original filename from a URL.
-
- Args:
- url: URL to extract filename from
-
- Returns:
- Decoded filename (e.g., "schemas.py" from ".../schemas.py")
- Falls back to "download" if no filename can be extracted.
- """
- from pathlib import Path
- from urllib.parse import unquote, urlparse
-
- parsed = urlparse(url)
- # URL-decode path to handle encoded characters (e.g., %E7%99%BE -> Chinese chars)
- decoded_path = unquote(parsed.path)
- basename = Path(decoded_path).name
- return basename if basename else "download"
-
- def _get_readabilipy(self):
- """Lazy import of readabilipy."""
- if not hasattr(self, "_readabilipy") or self._readabilipy is None:
- try:
- from readabilipy import simple_json
-
- self._readabilipy = simple_json
- except ImportError:
- raise ImportError(
- "readabilipy is required for HTML parsing. "
- "Install it with: pip install readabilipy"
- )
- return self._readabilipy
-
- def _get_markdownify(self):
- """Lazy import of markdownify."""
- if not hasattr(self, "_markdownify") or self._markdownify is None:
- try:
- import markdownify
-
- self._markdownify = markdownify
- except ImportError:
- raise ImportError(
- "markdownify is required for HTML parsing. "
- "Install it with: pip install markdownify"
- )
- return self._markdownify
-
def _get_markdown_parser(self):
"""Lazy import and create MarkdownParser with the HTML parser config."""
if self._markdown_parser is None:
@@ -176,26 +128,63 @@ class HTMLParser(BaseParser):
)
def _html_to_markdown(self, html: str, base_url: str = "") -> str:
- """
- Convert HTML to Markdown using readabilipy + markdownify (Anthropic approach).
- """
- markdownify = self._get_markdownify()
-
- # Preprocess: extract hidden content areas (e.g., WeChat public account's js_content)
+ """Convert HTML to Markdown using trafilatura."""
html = self._preprocess_html(html)
+ content = self._extract_markdown(html, base_url or "")
+ title = self._extract_title(html, base_url or "")
+ content = self._clean_markdown(content)
+ if title and title not in content:
+ content = f"# {title}\n\n{content}" if content else f"# {title}"
+ return content
+
+ @staticmethod
+ def _extract_markdown(html: str, url: str) -> str:
+ """Extract clean Markdown body from HTML."""
+ try:
+ import trafilatura
+
+ return (
+ trafilatura.extract(
+ html,
+ url=url,
+ output_format="markdown",
+ include_links=True,
+ include_tables=True,
+ )
+ or ""
+ )
+ except Exception as exc:
+ logger.debug("[HTMLParser] trafilatura extraction failed: %s", exc)
+ return ""
+
+ @staticmethod
+ def _extract_title(html: str, url: str) -> str:
+ """Extract page title from HTML metadata."""
+ try:
+ import trafilatura
- # Use readabilipy to extract main content (based on Mozilla Readability)
- readabilipy = self._get_readabilipy()
- result = readabilipy.simple_json_from_html_string(html, use_readability=True)
- content_html = result.get("content") or html
+ metadata = trafilatura.extract_metadata(html, default_url=url)
+ if metadata and metadata.title:
+ return str(metadata.title).strip()
+ except Exception as exc:
+ logger.debug("[HTMLParser] trafilatura title extraction failed: %s", exc)
+ return ""
- # Convert to markdown using markdownify
- markdown = markdownify.markdownify(
- content_html,
- heading_style=markdownify.ATX,
- strip=["script", "style"],
+ @staticmethod
+ def _clean_markdown(markdown: str) -> str:
+ """Remove noisy artifacts from extracted Markdown."""
+ markdown = markdown or ""
+ markdown = re.sub(r"!\[[^\]]*\]\(data:image/[^)]+\)", "", markdown)
+ markdown = re.sub(
+ r"<img[^>]*src\s*=\s*['\"]data:image/[^'\"]*['\"][^>]*/?>",
+ "",
+ markdown,
+ flags=re.IGNORECASE,
)
-
+ markdown = re.sub(r"<(span|a)\s+(?:id|name)=['\"][^'\"]*['\"]\s*>\s*</\1>", "", markdown)
+ for pattern in _SPA_EMPTY_PATTERNS:
+ markdown = markdown.replace(pattern, "")
+ markdown = re.sub(r"\n{3,}", "\n\n", markdown)
return markdown.strip()
def _preprocess_html(self, html: str) -> str:
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index f71503a5e..ac8dae028 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -630,7 +630,7 @@ class ResourceService:
avoid recursive watch task creation during scheduled execution)
enforce_public_remote_targets: When True, reject non-public remote hosts and
validate each outbound HTTP request URL during fetch.
- args: Parser-specific options forwarded to the parser chain.
+ args: Parser/accessor-specific options forwarded to the processing chain.
**kwargs: Extra options forwarded to the parser chain
Returns:
@@ -875,7 +875,7 @@ class ResourceService:
if result.get("status") == "error":
return result
- elif wait:
+ if wait:
if stage_callback is not None:
stage_result = stage_callback("processing_queue")
if inspect.isawaitable(stage_result):
diff --git a/openviking/utils/__init__.py b/openviking/utils/__init__.py
index 990f86bd3..8213db8a7 100644
--- a/openviking/utils/__init__.py
+++ b/openviking/utils/__init__.py
@@ -3,6 +3,7 @@
"""Utility functions and helpers."""
from openviking.utils.code_hosting_utils import (
+ is_code_hosting_blob_url,
is_code_hosting_url,
is_git_repo_url,
is_github_url,
@@ -29,6 +30,7 @@ __all__ = [
"is_github_url",
"is_gitlab_url",
"is_code_hosting_url",
+ "is_code_hosting_blob_url",
"validate_git_ssh_uri",
"is_git_repo_url",
]
diff --git a/openviking/utils/code_hosting_utils.py b/openviking/utils/code_hosting_utils.py
index a9470787c..1cb00d826 100644
--- a/openviking/utils/code_hosting_utils.py
+++ b/openviking/utils/code_hosting_utils.py
@@ -220,6 +220,36 @@ def is_code_hosting_url(url: str) -> bool:
return _domain_matches(urlparse(url), all_domains)
+def is_code_hosting_blob_url(url: str) -> bool:
+ """Check whether a URL points to a single file on a code hosting platform.
+
+ Recognizes GitHub/GitLab ``blob`` URLs and GitHub ``raw`` URLs, e.g.
+ ``https://github.com/org/repo/blob/main/README.md`` or
+ ``https://raw.githubusercontent.com/org/repo/main/README.md``. These
+ historically go through HTTPAccessor (with optional blob->raw rewriting),
+ not the recursive web crawler.
+ """
+ if not url.startswith(("http://", "https://")):
+ return False
+
+ config = get_openviking_config()
+ parsed = urlparse(url)
+ host = (parsed.hostname or "").lower()
+ if not host:
+ return False
+
+ raw_hosts = {"raw.githubusercontent.com"}
+ if host in raw_hosts:
+ return True
+
+ code_hosts = set(config.code.github_domains) | set(config.code.gitlab_domains)
+ if host not in code_hosts:
+ return False
+
+ path_parts = [p for p in parsed.path.split("/") if p]
+ return "blob" in path_parts or "raw" in path_parts
+
+
def validate_git_ssh_uri(url: str) -> None:
"""Validate a git@ SSH URI format.
diff --git a/pyproject.toml b/pyproject.toml
index d3b1310be..c5504b455 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -36,8 +36,8 @@ dependencies = [
"pyyaml>=6.0",
"httpx>=0.25.0",
"pdfplumber>=0.10.0",
- "readabilipy>=0.2.0",
- "markdownify>=0.11.0",
+ "scrapy>=2.11.0",
+ "trafilatura>=1.12.0",
"feedparser>=6.0.0",
"defusedxml>=0.7.1",
"openai>=1.0.0",
diff --git a/tests/parse/accessors/web_crawler/__init__.py b/tests/parse/accessors/web_crawler/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/parse/accessors/web_crawler/test_config.py b/tests/parse/accessors/web_crawler/test_config.py
new file mode 100644
index 000000000..501aeb0d5
--- /dev/null
+++ b/tests/parse/accessors/web_crawler/test_config.py
@@ -0,0 +1,53 @@
+import pytest
+
+from openviking.parse.accessors.web_crawler.config import CrawlConfig
+
+
+class TestCrawlConfigValidation:
+ def test_defaults_are_valid(self):
+ config = CrawlConfig()
+ assert config.skip_download_links is True
+
+ def test_depth_minus_one_unlimited_ok(self):
+ CrawlConfig(depth=-1)
+
+ def test_depth_minus_two_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(depth=-2)
+
+ def test_max_pages_zero_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(max_pages=0)
+
+ def test_max_pages_minus_one_unlimited_ok(self):
+ CrawlConfig(max_pages=-1)
+
+ def test_concurrency_zero_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(concurrency=0)
+
+ def test_timeout_non_positive_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(timeout=0)
+ with pytest.raises(ValueError):
+ CrawlConfig(timeout=-1.0)
+
+ def test_download_delay_negative_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(download_delay=-0.1)
+
+ def test_retry_times_negative_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(retry_times=-1)
+
+ def test_max_links_per_page_zero_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(max_links_per_page=0)
+
+ def test_max_html_bytes_zero_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(max_html_bytes=0)
+
+ def test_playwright_timeout_non_positive_rejected(self):
+ with pytest.raises(ValueError):
+ CrawlConfig(playwright_timeout=0)
diff --git a/tests/parse/accessors/web_crawler/test_middlewares.py b/tests/parse/accessors/web_crawler/test_middlewares.py
new file mode 100644
index 000000000..efd7d0265
--- /dev/null
+++ b/tests/parse/accessors/web_crawler/test_middlewares.py
@@ -0,0 +1,53 @@
+from unittest.mock import MagicMock
+
+import pytest
+from scrapy.exceptions import IgnoreRequest
+
+from openviking.parse.accessors.web_crawler.config import CrawlConfig
+from openviking.parse.accessors.web_crawler.middlewares import RequestValidatorMiddleware
+
+
+def _make_spider(validator):
+ spider = MagicMock()
+ spider.config = CrawlConfig(request_validator=validator)
+ return spider
+
+
+def _make_request(url):
+ request = MagicMock()
+ request.url = url
+ return request
+
+
+class TestRequestValidatorMiddleware:
+ def test_passes_through_when_validator_accepts(self):
+ mw = RequestValidatorMiddleware()
+ spider = _make_spider(validator=lambda _url: None)
+ request = _make_request("https://example.com/")
+ assert mw.process_request(request, spider) is None
+
+ def test_raises_ignore_when_validator_rejects(self):
+ mw = RequestValidatorMiddleware()
+
+ def reject(url):
+ raise ValueError(f"private address: {url}")
+
+ spider = _make_spider(validator=reject)
+ request = _make_request("http://169.254.169.254/latest/meta-data/")
+ with pytest.raises(IgnoreRequest) as exc_info:
+ mw.process_request(request, spider)
+ assert "private address" in str(exc_info.value)
+
+ def test_no_op_when_validator_unset(self):
+ mw = RequestValidatorMiddleware()
+ spider = _make_spider(validator=None)
+ request = _make_request("https://example.com/")
+ assert mw.process_request(request, spider) is None
+
+ def test_validator_receives_request_url(self):
+ mw = RequestValidatorMiddleware()
+ seen: list[str] = []
+ spider = _make_spider(validator=lambda url: seen.append(url))
+ request = _make_request("https://example.com/redirected-target")
+ mw.process_request(request, spider)
+ assert seen == ["https://example.com/redirected-target"]
diff --git a/tests/parse/accessors/web_crawler/test_playwright_renderer.py b/tests/parse/accessors/web_crawler/test_playwright_renderer.py
new file mode 100644
index 000000000..bc1cc8250
--- /dev/null
+++ b/tests/parse/accessors/web_crawler/test_playwright_renderer.py
@@ -0,0 +1,87 @@
+from openviking.parse.accessors.web_crawler.playwright_renderer import PlaywrightRenderer
+
+
+class _FakePage:
+ """Minimal page stub returning a scripted sequence of body texts."""
+
+ def __init__(self, body_sequence):
+ self._bodies = list(body_sequence)
+ self.inner_text_calls = 0
+ self.slept_ms = 0
+
+ async def inner_text(self, _selector):
+ self.inner_text_calls += 1
+ idx = min(self.inner_text_calls - 1, len(self._bodies) - 1)
+ return self._bodies[idx]
+
+ async def wait_for_timeout(self, ms):
+ self.slept_ms += ms
+
+
+class TestRenderer:
+ async def test_close_is_idempotent_when_browser_never_started(self):
+ renderer = PlaywrightRenderer()
+ await renderer.close()
+ await renderer.close()
+
+ def test_request_validator_stored(self):
+ called = []
+ renderer = PlaywrightRenderer(request_validator=lambda u: called.append(u))
+ assert renderer._request_validator is not None
+ renderer._request_validator("http://x.com/")
+ assert called == ["http://x.com/"]
+
+
+class _FakeRoute:
+ def __init__(self, url):
+ self.request = type("Req", (), {"url": url})()
+ self.aborted = False
+ self.continued = False
+
+ async def abort(self):
+ self.aborted = True
+
+ async def continue_(self):
+ self.continued = True
+
+
+class TestGuardRoute:
+ async def test_blocks_disallowed_subresource_without_raising(self):
+ def validator(url):
+ if "private" in url:
+ raise ValueError("private network blocked")
+
+ route = _FakeRoute("http://private.internal/probe")
+ # Must not raise: a blocked sub-resource cannot fail the whole render.
+ await PlaywrightRenderer._guard_route(route, validator)
+ assert route.aborted is True
+ assert route.continued is False
+
+ async def test_allows_valid_subresource(self):
+ route = _FakeRoute("https://cdn.example.com/app.js")
+ await PlaywrightRenderer._guard_route(route, lambda _u: None)
+ assert route.continued is True
+ assert route.aborted is False
+
+
+class TestWaitPastChallenge:
+ async def test_waits_until_challenge_redirects_to_content(self):
+ real = "Real documentation body with plenty of visible content here." * 3
+ page = _FakePage(["Please wait...", "Please wait...", real])
+ await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=10_000, poll_ms=10)
+ assert page.inner_text_calls == 3
+ assert page.slept_ms == 20
+
+ async def test_returns_immediately_when_content_present(self):
+ real = "Real documentation body with plenty of visible content here." * 3
+ page = _FakePage([real])
+ await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=10_000, poll_ms=10)
+ assert page.inner_text_calls == 1
+ assert page.slept_ms == 0
+
+ async def test_gives_up_after_timeout_on_persistent_challenge(self):
+ page = _FakePage(["Please wait..."])
+ await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=0, poll_ms=10)
+ assert page.inner_text_calls == 1
+ assert page.slept_ms == 0
+
diff --git a/tests/parse/accessors/web_crawler/test_render_heuristics.py b/tests/parse/accessors/web_crawler/test_render_heuristics.py
new file mode 100644
index 000000000..a1bcff0b1
--- /dev/null
+++ b/tests/parse/accessors/web_crawler/test_render_heuristics.py
@@ -0,0 +1,63 @@
+from openviking.parse.accessors.web_crawler.render_heuristics import (
+ looks_like_unrendered_page,
+ should_render_with_playwright,
+)
+
+
+def test_js_required_pattern_triggers_render():
+ html = "<html>You need to enable JavaScript to run this app.</html>"
+ assert should_render_with_playwright(html) is True
+
+
+def test_react_root_triggers_render():
+ html = "<html><body><div id='root'></div></body></html>"
+ assert should_render_with_playwright(html) is True
+
+
+def test_ssr_app_root_with_rich_content_skips_render():
+ body = "OpenViking 快速开始文档,包含安装、配置、添加资源、创建集合和检索示例。" * 8
+ html = f"<html><body><div id='app'><main><p>{body}</p></main></div></body></html>"
+ assert should_render_with_playwright(html) is False
+
+
+def test_next_data_triggers_render():
+ html = "<html><script>__NEXT_DATA__</script></html>"
+ assert should_render_with_playwright(html) is True
+
+
+def test_many_scripts_with_tiny_body_triggers_render():
+ html = "<html><body>x</body>" + "<script></script>" * 6 + "</html>"
+ assert should_render_with_playwright(html) is True
+
+
+def test_content_rich_static_page_skips_render():
+ body = "This is a fully server-rendered article with plenty of readable text. " * 3
+ html = f"<html><body><p>{body}</p></body></html>"
+ assert should_render_with_playwright(html) is False
+
+
+def test_challenge_page_is_unrendered():
+ html = "<html><body>Please wait...</body></html>"
+ assert looks_like_unrendered_page(html) is True
+
+
+def test_empty_shell_is_unrendered():
+ html = "<html><body><div id='root'></div></body></html>"
+ assert looks_like_unrendered_page(html) is True
+
+
+def test_real_content_is_not_unrendered():
+ body = "Collection 数据过滤删除任务的完整说明文档,包含请求参数、示例与返回值。" * 3
+ html = f"<html><body><article>{body}</article></body></html>"
+ assert looks_like_unrendered_page(html) is False
+
+
+def test_unrendered_check_fails_closed_on_parser_error(monkeypatch):
+ from openviking.parse.accessors.web_crawler import render_heuristics
+
+ def raise_parser_error(*_args, **_kwargs):
+ raise RuntimeError("parser failed")
+
+ monkeypatch.setattr(render_heuristics, "BeautifulSoup", raise_parser_error)
+
+ assert render_heuristics.looks_like_unrendered_page("<html></html>") is True
diff --git a/tests/parse/accessors/web_crawler/test_scrapy_spider.py b/tests/parse/accessors/web_crawler/test_scrapy_spider.py
new file mode 100644
index 000000000..66dccb4e9
--- /dev/null
+++ b/tests/parse/accessors/web_crawler/test_scrapy_spider.py
@@ -0,0 +1,269 @@
+import pytest
+from unittest.mock import MagicMock
+
+from scrapy.exceptions import CloseSpider
+
+from openviking.parse.accessors.web_crawler.config import CrawlConfig
+from openviking.parse.accessors.web_crawler.scrapy_spider import OpenVikingWebSpider
+
+
+def _make_spider(config=None, root_url="http://example.com/"):
+ return OpenVikingWebSpider(
+ root_url=root_url,
+ config=config or CrawlConfig(),
+ collector=[],
+ download_collector=[],
+ )
+
+
+def _make_response(url="http://example.com/", body=b"", headers=None, depth=0):
+ resp = MagicMock()
+ resp.url = url
+ resp.body = body
+ resp.text = body.decode("utf-8", errors="ignore")
+ headers_dict = headers or {"content-type": b"text/html"}
+ resp.headers.get = lambda k, default=b"": headers_dict.get(k, default)
+ resp.meta = {"depth": depth}
+ resp.follow = MagicMock(return_value=MagicMock())
+ return resp
+
+
+async def _drive_parse(spider, response):
+ items = []
+ async for item in spider.parse(response):
+ items.append(item)
+ return items
+
+
+class TestAcceptChildUrl:
+ def test_accepts_same_host(self):
+ spider = _make_spider()
+ assert spider._accept_child_url("http://example.com/foo") is True
+
+ def test_rejects_external_when_disallowed(self):
+ spider = _make_spider()
+ assert spider._accept_child_url("http://other.com/x") is False
+
+ def test_accepts_external_when_allowed(self):
+ spider = _make_spider(CrawlConfig(allow_external_links=True, max_pages=10))
+ assert spider._accept_child_url("http://other.com/x") is True
+
+ def test_rejects_non_http_scheme(self):
+ spider = _make_spider()
+ assert spider._accept_child_url("ftp://example.com/x") is False
+ assert spider._accept_child_url("file:///etc/passwd") is False
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "http://example.com/a.css",
+ "http://example.com/a.js",
+ "http://example.com/a.png",
+ "http://example.com/a.pdf",
+ "http://example.com/a.zip",
+ "http://example.com/a.txt",
+ ],
+ )
+ def test_rejects_non_page_extensions(self, url):
+ assert _make_spider()._accept_child_url(url) is False
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "http://example.com/a.pdf",
+ "http://example.com/a.zip",
+ "http://example.com/a.txt",
+ "http://example.com/a.md",
+ "http://example.com/a.docx",
+ ],
+ )
+ def test_accepts_download_extensions_as_downloads(self, url):
+ assert _make_spider()._accept_download_url(url) is True
+
+ def test_rejects_download_when_skip_download_links_enabled(self):
+ spider = _make_spider(CrawlConfig(skip_download_links=True))
+ html = '<a href="/llms.txt">txt</a>'
+ assert spider._extract_download_urls(html, "http://example.com/", 1) == []
+
+ def test_rejects_external_download_when_disallowed(self):
+ spider = _make_spider()
+ assert spider._accept_download_url("http://other.com/a.pdf") is False
+
+ def test_accepts_external_download_when_allowed(self):
+ spider = _make_spider(CrawlConfig(allow_external_links=True))
+ assert spider._accept_download_url("http://other.com/a.pdf") is True
+
+ def test_rejects_when_include_paths_miss(self):
+ spider = _make_spider(CrawlConfig(include_paths=["/docs/"]))
+ assert spider._accept_child_url("http://example.com/blog/x") is False
+
+ def test_accepts_when_include_paths_hit(self):
+ spider = _make_spider(CrawlConfig(include_paths=["/docs/"]))
+ assert spider._accept_child_url("http://example.com/docs/intro") is True
+
+ def test_rejects_when_exclude_paths_hit(self):
+ spider = _make_spider(CrawlConfig(exclude_paths=["/admin/"]))
+ assert spider._accept_child_url("http://example.com/admin/x") is False
+
+ def test_include_paths_is_prefix_not_substring(self):
+ spider = _make_spider(CrawlConfig(include_paths=["/docs/"]))
+ assert spider._accept_child_url("http://example.com/blog/docs-tips") is False
+ assert spider._accept_child_url("http://example.com/api/v1/getdocs") is False
+
+ def test_exclude_paths_is_prefix_not_substring(self):
+ spider = _make_spider(CrawlConfig(exclude_paths=["/admin/"]))
+ assert spider._accept_child_url("http://example.com/blog/admin-tips") is True
+
+ def test_rejects_when_validator_raises(self):
+ def blocker(_):
+ raise ValueError("blocked")
+ spider = _make_spider(CrawlConfig(request_validator=blocker))
+ assert spider._accept_child_url("http://example.com/anywhere") is False
+
+
+class TestExtractChildUrls:
+ def test_strips_anchor_and_dedup(self):
+ spider = _make_spider()
+ html = '<a href="/a#x">x</a><a href="/a">y</a>'
+ assert spider._extract_child_urls(html, "http://example.com/") == [
+ "http://example.com/a"
+ ]
+
+ def test_skips_non_http_schemes(self):
+ spider = _make_spider()
+ html = (
+ '<a href="javascript:void(0)">x</a>'
+ '<a href="mailto:[email protected]">y</a>'
+ '<a href="tel:123">z</a>'
+ '<a href="#frag">w</a>'
+ )
+ assert spider._extract_child_urls(html, "http://example.com/") == []
+
+ def test_resolves_relative_with_urljoin(self):
+ spider = _make_spider()
+ html = '<a href="docs/intro">x</a>'
+ assert spider._extract_child_urls(html, "http://example.com/") == [
+ "http://example.com/docs/intro"
+ ]
+
+ def test_caps_at_max_links_per_page(self):
+ spider = _make_spider(CrawlConfig(max_links_per_page=2))
+ html = "".join(f'<a href="/p{i}">x</a>' for i in range(5))
+ assert len(spider._extract_child_urls(html, "http://example.com/")) == 2
+
+ def test_returns_empty_on_blank_html(self):
+ spider = _make_spider()
+ assert spider._extract_child_urls("", "http://example.com/") == []
+
+
+class TestSuccessAtLimit:
+ def test_unlimited_never_at_limit(self):
+ spider = _make_spider(CrawlConfig(max_pages=-1))
+ spider._success_count = 100
+ assert spider._success_at_limit() is False
+
+ def test_at_limit_when_count_equals_max(self):
+ spider = _make_spider(CrawlConfig(max_pages=10))
+ spider._success_count = 10
+ assert spider._success_at_limit() is True
+
+ def test_below_limit(self):
+ spider = _make_spider(CrawlConfig(max_pages=10))
+ spider._success_count = 9
+ assert spider._success_at_limit() is False
+
+
+_RICH_HTML = (
+ "<html><head><title>Example</title></head><body><article>"
+ + "<p>" + ("This is a meaningful paragraph for the crawler. " * 20) + "</p>"
+ + "<p>" + ("Another paragraph with enough characters to extract. " * 20) + "</p>"
+ + "</article></body></html>"
+).encode("utf-8")
+
+
+class TestParseGate:
+ async def test_early_return_when_limit_reached(self):
+ spider = _make_spider(CrawlConfig(max_pages=10, fallback_playwright=False))
+ spider._success_count = 10
+ resp = _make_response(body=_RICH_HTML)
+ items = await _drive_parse(spider, resp)
+ assert items == []
+ assert spider._success_count == 10
+ assert spider.collector == []
+
+ async def test_appends_success_and_increments_count(self):
+ spider = _make_spider(CrawlConfig(max_pages=10, fallback_playwright=False))
+ resp = _make_response(body=_RICH_HTML)
+ await _drive_parse(spider, resp)
+ assert spider._success_count == 1
+ success = [p for p in spider.collector if p.status == "success"]
+ assert len(success) == 1
+ assert success[0].html == _RICH_HTML.decode("utf-8")
+ assert success[0].source == "scrapy_static"
+
+ async def test_download_links_count_toward_max_pages(self):
+ spider = _make_spider(
+ CrawlConfig(
+ depth=1,
+ max_pages=2,
+ skip_download_links=False,
+ fallback_playwright=False,
+ )
+ )
+ body = (
+ b"<html><body><h1>Home</h1>"
+ b'<a href="/llms.txt">txt</a>'
+ b'<a href="/guide">guide</a>'
+ b"</body></html>"
+ )
+ resp = _make_response(body=body)
+ items = await _drive_parse(spider, resp)
+ assert spider._success_count == 2
+ assert [download.url for download in spider.download_collector] == [
+ "http://example.com/llms.txt"
+ ]
+ assert items == []
+
+ async def test_second_call_blocked_after_reaching_max(self):
+ spider = _make_spider(CrawlConfig(max_pages=1, fallback_playwright=False))
+ resp = _make_response(body=_RICH_HTML)
+ await _drive_parse(spider, resp)
+ assert spider._success_count == 1
+ await _drive_parse(spider, resp)
+ assert spider._success_count == 1
+ success = [p for p in spider.collector if p.status == "success"]
+ assert len(success) == 1
+
+ async def test_reaching_limit_closes_scrapy_engine(self):
+ spider = _make_spider(CrawlConfig(max_pages=1, fallback_playwright=False))
+ spider.crawler = MagicMock()
+ spider.crawler.engine = MagicMock()
+ resp = _make_response(body=_RICH_HTML)
+
+ with pytest.raises(CloseSpider) as exc_info:
+ await _drive_parse(spider, resp)
+
+ assert exc_info.value.reason == "max_pages_reached"
+ assert spider._success_count == 1
+ assert len(spider.collector) == 1
+
+ async def test_skipped_on_non_html_response(self):
+ spider = _make_spider(CrawlConfig(fallback_playwright=False))
+ resp = _make_response(
+ body=b"binary",
+ headers={"content-type": b"application/octet-stream"},
+ )
+ await _drive_parse(spider, resp)
+ assert spider._success_count == 0
+ statuses = [p.status for p in spider.collector]
+ assert statuses == ["skipped"]
+
+ async def test_failed_on_validator_rejecting_entry(self):
+ def blocker(_):
+ raise ValueError("blocked by SSRF")
+ spider = _make_spider(CrawlConfig(request_validator=blocker, fallback_playwright=False))
+ resp = _make_response(body=_RICH_HTML)
+ await _drive_parse(spider, resp)
+ assert spider._success_count == 0
+ statuses = [p.status for p in spider.collector]
+ assert statuses == ["failed"]
diff --git a/tests/parse/parsers/__init__.py b/tests/parse/parsers/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/parse/parsers/test_html.py b/tests/parse/parsers/test_html.py
new file mode 100644
index 000000000..ef87b8ac9
--- /dev/null
+++ b/tests/parse/parsers/test_html.py
@@ -0,0 +1,53 @@
+from openviking.parse.parsers.html import HTMLParser
+
+
+class TestHTMLParserMarkdownCleaning:
+ def setup_method(self):
+ self.parser = HTMLParser()
+
+ def test_strips_data_image_markdown(self):
+ md = "before ![alt](data:image/png;base64,AAA) after"
+ assert "data:image" not in self.parser._clean_markdown(md)
+
+ def test_strips_data_image_html_tag(self):
+ md = "<img src='data:image/png;base64,AAA' alt='x'/> tail"
+ assert "data:image" not in self.parser._clean_markdown(md)
+
+ def test_strips_empty_anchor_spans(self):
+ md = "head <span id='foo'></span><a name='bar'></a> tail"
+ cleaned = self.parser._clean_markdown(md)
+ assert "<span" not in cleaned
+ assert "<a name" not in cleaned
+
+ def test_strips_spa_empty_pattern(self):
+ md = "real content. You need to enable JavaScript to run this app."
+ assert "JavaScript" not in self.parser._clean_markdown(md)
+
+ def test_collapses_blank_lines(self):
+ cleaned = self.parser._clean_markdown("a\n\n\n\n\nb")
+ assert cleaned == "a\n\nb"
+
+ def test_trims_leading_trailing_whitespace(self):
+ assert self.parser._clean_markdown(" \n\nhello\n ") == "hello"
+
+
+class TestHTMLParserTitleExtraction:
+ def setup_method(self):
+ self.parser = HTMLParser()
+
+ def test_title_from_og_meta(self):
+ html = '<html><head><meta property="og:title" content="HelloOG"></head><body>x</body></html>'
+ title = self.parser._extract_title(html, "http://example.com/")
+ assert title and "HelloOG" in title
+
+ def test_title_from_title_tag(self):
+ html = "<html><head><title>HelloTitle</title></head><body>x</body></html>"
+ title = self.parser._extract_title(html, "http://example.com/")
+ assert title and "HelloTitle" in title
+
+ def test_returns_empty_on_blank_html(self):
+ assert self.parser._extract_title("", "http://example.com/") == ""
+
+ def test_returns_empty_on_no_title(self):
+ html = "<html><body>no title</body></html>"
+ assert self.parser._extract_title(html, "http://example.com/") == ""
diff --git a/tests/parse/test_html_parser_extraction.py b/tests/parse/test_html_parser_extraction.py
new file mode 100644
index 000000000..300f28ed5
--- /dev/null
+++ b/tests/parse/test_html_parser_extraction.py
@@ -0,0 +1,52 @@
+"""Regression tests for HTMLParser._html_to_markdown after switching to trafilatura.
+
+Focus: verify that the WeChat-public-account preprocessing path still produces
+non-empty Markdown after the underlying extractor changed from
+readabilipy + markdownify to trafilatura.
+"""
+
+from openviking.parse.parsers.html import HTMLParser
+
+
+WECHAT_HTML = """
+<!doctype html>
+<html>
+<head><title>WeChat Article</title></head>
+<body>
+ <div id="page-content">
+ <div id="js_content" style="visibility: hidden; opacity: 0;">
+ <h1>OpenViking 周报</h1>
+ <p>这是一篇微信公众号文章的正文段落,至少包含两百字以上的有效内容,
+ 用于让基于文本密度的抽取器能够稳定识别出主体区域。</p>
+ <p>第二段同样是足够长的正文,避免被启发式规则误判为噪声。我们再加一些
+ 内容以保证抽取器有充分的密度信号去命中这块隐藏 div 区域。</p>
+ <p>第三段是为了进一步增加正文密度。OpenViking 的 HTMLParser 在切换
+ 抽取器后仍然需要正确地从 #js_content 这个被 CSS 隐藏的容器里取出文本。</p>
+ <img src="" data-src="https://example.com/cover.jpg" alt="cover" />
+ </div>
+ </div>
+</body>
+</html>
+"""
+
+
+def test_preprocess_strips_hidden_style_and_keeps_content():
+ parser = HTMLParser()
+ cleaned = parser._preprocess_html(WECHAT_HTML)
+ assert "visibility: hidden" not in cleaned
+ assert "OpenViking 周报" in cleaned
+ assert 'src="https://example.com/cover.jpg"' in cleaned
+
+
+def test_html_to_markdown_extracts_wechat_body():
+ parser = HTMLParser()
+ md = parser._html_to_markdown(WECHAT_HTML)
+ assert md, "trafilatura should extract non-empty markdown from a WeChat-style article"
+ assert "OpenViking 周报" in md
+ assert "微信公众号" in md
+
+
+def test_html_to_markdown_returns_empty_string_on_garbage_input():
+ parser = HTMLParser()
+ md = parser._html_to_markdown("<html><body></body></html>")
+ assert md == ""
diff --git a/tests/parse/test_url_filename_preservation.py b/tests/parse/test_url_filename_preservation.py
index e7cc16dc1..e2e539d6b 100644
--- a/tests/parse/test_url_filename_preservation.py
+++ b/tests/parse/test_url_filename_preservation.py
@@ -9,47 +9,46 @@ Verifies fix for https://github.com/volcengine/OpenViking/issues/251:
import pytest
-from openviking.parse.accessors.http_accessor import HTTPAccessor
-from openviking.parse.parsers.html import HTMLParser, URLType, URLTypeDetector
+from openviking.parse.accessors.http_accessor import HTTPAccessor, URLType, URLTypeDetector
class TestExtractFilenameFromUrl:
- """Test HTMLParser._extract_filename_from_url."""
+ """Test HTTPAccessor._extract_filename_from_url."""
def test_simple_filename(self):
url = "https://example.com/path/to/schemas.py"
- assert HTMLParser._extract_filename_from_url(url) == "schemas.py"
+ assert HTTPAccessor._extract_filename_from_url(url) == "schemas.py"
def test_url_encoded_path(self):
url = "https://example.com/%E7%99%BE%E5%BA%A64/src/baidu_search/schemas.py"
- assert HTMLParser._extract_filename_from_url(url) == "schemas.py"
+ assert HTTPAccessor._extract_filename_from_url(url) == "schemas.py"
def test_url_encoded_filename(self):
url = "https://example.com/path/%E6%96%87%E4%BB%B6.py"
- assert HTMLParser._extract_filename_from_url(url) == "\u6587\u4ef6.py"
+ assert HTTPAccessor._extract_filename_from_url(url) == "\u6587\u4ef6.py"
def test_query_params_ignored(self):
url = "https://example.com/file.py?version=2&token=abc"
- assert HTMLParser._extract_filename_from_url(url) == "file.py"
+ assert HTTPAccessor._extract_filename_from_url(url) == "file.py"
def test_no_filename_fallback(self):
url = "https://example.com/"
- assert HTMLParser._extract_filename_from_url(url) == "download"
+ assert HTTPAccessor._extract_filename_from_url(url) == "download"
def test_cos_url(self):
url = (
"https://cos.ap-beijing.myqcloud.com/bucket/"
"%E7%99%BE%E5%BA%A64/src/baidu_search/schemas.py"
)
- assert HTMLParser._extract_filename_from_url(url) == "schemas.py"
+ assert HTTPAccessor._extract_filename_from_url(url) == "schemas.py"
def test_markdown_extension(self):
url = "https://example.com/docs/README.md"
- assert HTMLParser._extract_filename_from_url(url) == "README.md"
+ assert HTTPAccessor._extract_filename_from_url(url) == "README.md"
def test_no_extension(self):
url = "https://example.com/path/Makefile"
- assert HTMLParser._extract_filename_from_url(url) == "Makefile"
+ assert HTTPAccessor._extract_filename_from_url(url) == "Makefile"
class TestURLTypeDetectorCodeExtensions:
diff --git a/tests/server/test_api_local_input_security.py b/tests/server/test_api_local_input_security.py
index 189c941c5..ecccbea84 100644
--- a/tests/server/test_api_local_input_security.py
+++ b/tests/server/test_api_local_input_security.py
@@ -10,7 +10,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
import httpx
import pytest
-from openviking.parse.parsers.html import URLTypeDetector
+from openviking.parse.accessors.http_accessor import URLTypeDetector
from openviking.utils.network_guard import ensure_public_remote_target
from openviking_cli.exceptions import PermissionDeniedError
from tests.server.ovpack_test_helpers import build_ovpack_bytes
diff --git a/tests/unit/test_accessors_http.py b/tests/unit/test_accessors_http.py
index 8c172bbfa..47ae0e27c 100644
--- a/tests/unit/test_accessors_http.py
+++ b/tests/unit/test_accessors_http.py
@@ -8,6 +8,7 @@ from unittest.mock import patch
import pytest
from openviking.parse.accessors import AccessorRegistry, GitAccessor, HTTPAccessor
+from openviking.parse.accessors.http_accessor import URLType
def _mock_config():
@@ -28,6 +29,14 @@ def _mock_config():
class TestHTTPAccessor:
"""Tests for HTTPAccessor."""
+ @pytest.fixture(autouse=True)
+ def _patch_config(self):
+ with patch(
+ "openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",
+ side_effect=_mock_config,
+ ):
+ yield
+
@pytest.fixture
def accessor(self) -> HTTPAccessor:
"""Create a HTTPAccessor instance."""
@@ -74,6 +83,111 @@ class TestHTTPAccessor:
"""Test filename extraction from URLs."""
assert HTTPAccessor._extract_filename_from_url(url) == expected
+ @pytest.mark.parametrize("entry_url_type", [URLType.WEBPAGE, URLType.DOWNLOAD_HTML])
+ async def test_webpage_uses_web_importer_directory(
+ self, accessor, tmp_path, monkeypatch, entry_url_type
+ ):
+ downloaded = tmp_path / "entry.html"
+ downloaded.write_text("<html>entry</html>", encoding="utf-8")
+ imported_dir = tmp_path / "web"
+ imported_dir.mkdir()
+
+ async def fake_download(url, request_validator=None):
+ return str(downloaded), entry_url_type, {"extension": ".html"}
+
+ class FakeImporter:
+ async def import_to_directory(self, *, root_url, options, request_validator=None):
+ assert root_url == "https://example.com/page"
+ assert options.depth == 1
+ return SimpleNamespace(
+ path=imported_dir,
+ meta={
+ "web_import": True,
+ "crawl_result": {"total_crawled": 1},
+ "original_filename": "example.com",
+ },
+ )
+
+ monkeypatch.setattr(accessor, "_download_url", fake_download)
+ monkeypatch.setattr(
+ "openviking.parse.accessors.web_importer.WebImporter",
+ lambda: FakeImporter(),
+ )
+
+ resource = await accessor.access("https://example.com/page", depth=1)
+
+ assert resource.path == imported_dir
+ assert resource.path.is_dir()
+ assert resource.meta["url_type"] == "webpage"
+ assert resource.meta["web_import"] is True
+ assert not downloaded.exists()
+
+ async def test_download_file_does_not_use_web_importer(self, accessor, tmp_path, monkeypatch):
+ downloaded = tmp_path / "file.pdf"
+ downloaded.write_bytes(b"%PDF-test")
+ called = False
+
+ async def fake_download(url, request_validator=None):
+ return str(downloaded), URLType.DOWNLOAD_PDF, {"extension": ".pdf"}
+
+ class FakeImporter:
+ async def import_to_directory(self, **kwargs):
+ nonlocal called
+ called = True
+
+ monkeypatch.setattr(accessor, "_download_url", fake_download)
+ monkeypatch.setattr(
+ "openviking.parse.accessors.web_importer.WebImporter",
+ lambda: FakeImporter(),
+ )
+
+ resource = await accessor.access("https://example.com/file.pdf", depth=1)
+
+ assert resource.path == downloaded
+ assert resource.meta["url_type"] == "download_pdf"
+ assert called is False
+
+ @pytest.mark.parametrize(
+ "url",
+ [
+ "https://github.com/org/repo/blob/main/x.html",
+ "https://raw.githubusercontent.com/org/repo/main/x.html",
+ "https://gitlab.com/org/repo/blob/main/x.html",
+ ],
+ )
+ async def test_code_hosting_single_file_does_not_use_web_importer(
+ self, accessor, tmp_path, monkeypatch, url
+ ):
+ """Code-hosting single-file URLs (blob/raw) stay on the single-file path.
+
+ Even though the download resolves to an ``.html`` file (DOWNLOAD_HTML),
+ these URLs are semantically one file, not a crawlable site, so they must
+ NOT be routed through WebImporter.
+ """
+ downloaded = tmp_path / "x.html"
+ downloaded.write_text("<html>file</html>", encoding="utf-8")
+ called = False
+
+ async def fake_download(u, request_validator=None):
+ return str(downloaded), URLType.DOWNLOAD_HTML, {"extension": ".html"}
+
+ class FakeImporter:
+ async def import_to_directory(self, **kwargs):
+ nonlocal called
+ called = True
+
+ monkeypatch.setattr(accessor, "_download_url", fake_download)
+ monkeypatch.setattr(
+ "openviking.parse.accessors.web_importer.WebImporter",
+ lambda: FakeImporter(),
+ )
+
+ resource = await accessor.access(url, depth=1)
+
+ assert resource.path == downloaded
+ assert resource.meta["url_type"] == "download_html"
+ assert called is False
+
class TestHTTPAccessorPriorityRouting:
"""Tests that verify HTTPAccessor works correctly with priority-based routing."""
diff --git a/tests/unit/test_web_importer.py b/tests/unit/test_web_importer.py
new file mode 100644
index 000000000..421077108
--- /dev/null
+++ b/tests/unit/test_web_importer.py
@@ -0,0 +1,328 @@
+from dataclasses import FrozenInstanceError
+from types import SimpleNamespace
+
+import pytest
+
+from openviking.parse.accessors.web_importer import (
+ WebImportOptions,
+ WebImporter,
+ parse_web_import_options,
+)
+from openviking_cli.exceptions import InvalidArgumentError
+
+
+class TestParseWebImportOptions:
+ def test_defaults(self):
+ opts = parse_web_import_options({})
+ assert opts.depth == 0
+ assert opts.max_pages == 50
+ assert opts.allow_external_links is False
+ assert opts.skip_download_links is True
+ assert opts.include_paths is None
+ assert opts.exclude_paths is None
+
+ def test_depth_and_max_pages_parsed(self):
+ opts = parse_web_import_options({"depth": 2, "max_pages": 30})
+ assert opts.depth == 2
+ assert opts.max_pages == 30
+
+ def test_max_pages_zero_rejected(self):
+ with pytest.raises(InvalidArgumentError):
+ parse_web_import_options({"max_pages": 0})
+
+ def test_depth_minus_two_rejected(self):
+ with pytest.raises(InvalidArgumentError):
+ parse_web_import_options({"depth": -2})
+
+ def test_unlimited_combo_with_external_rejected(self):
+ with pytest.raises(InvalidArgumentError):
+ parse_web_import_options(
+ {"depth": -1, "max_pages": -1, "allow_external_links": True}
+ )
+
+ def test_unlimited_depth_with_bounded_pages_ok(self):
+ opts = parse_web_import_options(
+ {"depth": -1, "max_pages": 100, "allow_external_links": True}
+ )
+ assert opts.depth == -1
+ assert opts.max_pages == 100
+ assert opts.allow_external_links is True
+
+ def test_string_int_coerced(self):
+ opts = parse_web_import_options({"depth": "3"})
+ assert opts.depth == 3
+
+ def test_invalid_int_rejected(self):
+ with pytest.raises(InvalidArgumentError):
+ parse_web_import_options({"depth": "abc"})
+
+ def test_bool_parsed_from_string(self):
+ opts = parse_web_import_options({"allow_external_links": "true"})
+ assert opts.allow_external_links is True
+
+ def test_skip_download_links_parsed_from_string(self):
+ opts = parse_web_import_options({"skip_download_links": "true"})
+ assert opts.skip_download_links is True
+
+ def test_skip_download_links_can_be_disabled(self):
+ opts = parse_web_import_options({"skip_download_links": "false"})
+ assert opts.skip_download_links is False
+
+ def test_invalid_bool_rejected(self):
+ with pytest.raises(InvalidArgumentError):
+ parse_web_import_options({"allow_external_links": "maybe"})
+
+ def test_include_paths_csv(self):
+ opts = parse_web_import_options({"include_paths": "/docs/,/api/"})
+ assert opts.include_paths == ["/docs/", "/api/"]
+
+ def test_include_paths_list(self):
+ opts = parse_web_import_options({"include_paths": ["/a", "/b"]})
+ assert opts.include_paths == ["/a", "/b"]
+
+ def test_include_paths_empty_drops_to_none(self):
+ opts = parse_web_import_options({"include_paths": ""})
+ assert opts.include_paths is None
+
+ def test_processor_args_popped_after_parse(self):
+ args = {"depth": 2, "max_pages": 30, "other": "keep"}
+ parse_web_import_options(args)
+ assert args == {"other": "keep"}
+
+
+class TestWebImportOptionsDataclass:
+ def test_frozen(self):
+ opts = WebImportOptions()
+ with pytest.raises(FrozenInstanceError):
+ opts.depth = 5 # type: ignore[misc]
+
+
+class TestWebImporter:
+ async def test_import_to_directory_writes_html_and_dedupes(self, monkeypatch):
+ class FakeCrawler:
+ def __init__(self, config):
+ self.config = config
+
+ async def crawl(self, root_url):
+ return SimpleNamespace(
+ pages=[
+ SimpleNamespace(
+ url=root_url,
+ final_url=root_url,
+ depth=0,
+ status="success",
+ html="<html><body><h1>Home</h1></body></html>",
+ ),
+ SimpleNamespace(
+ url=f"{root_url}/guide",
+ final_url=f"{root_url}/guide",
+ depth=1,
+ status="success",
+ html="<html><body><h1>Guide</h1></body></html>",
+ ),
+ SimpleNamespace(
+ url=f"{root_url}/guide#frag",
+ final_url=f"{root_url}/guide#frag",
+ depth=1,
+ status="success",
+ html="<html><body><h1>Duplicate</h1></body></html>",
+ ),
+ ],
+ downloads=[],
+ total_crawled=3,
+ total_downloads=0,
+ total_failed=0,
+ total_skipped=0,
+ fallback_rendered=0,
+ )
+
+ monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
+
+ result = await WebImporter().import_to_directory(
+ root_url="https://example.com",
+ options=WebImportOptions(depth=1, max_pages=3),
+ )
+ try:
+ files = sorted(path.relative_to(result.path).as_posix() for path in result.path.rglob("*.html"))
+ assert files == ["Guide.html", "Home.html"]
+ assert result.path.name == "example.com"
+ assert (result.path / "Home.html").read_text(encoding="utf-8") == (
+ "<html><body><h1>Home</h1></body></html>"
+ )
+ assert result.meta["page_count"] == 2
+ assert result.meta["original_filename"] == "example.com"
+ finally:
+ import shutil
+
+ shutil.rmtree(result.path.parent, ignore_errors=True)
+
+ async def test_import_to_directory_rejects_missing_entry(self, monkeypatch):
+ class FakeCrawler:
+ def __init__(self, config):
+ self.config = config
+
+ async def crawl(self, root_url):
+ return SimpleNamespace(
+ pages=[
+ SimpleNamespace(
+ url=f"{root_url}/child",
+ final_url=f"{root_url}/child",
+ depth=1,
+ status="success",
+ html="<html>child</html>",
+ )
+ ],
+ downloads=[],
+ total_crawled=1,
+ total_downloads=0,
+ total_failed=0,
+ total_skipped=0,
+ fallback_rendered=0,
+ )
+
+ monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
+
+ with pytest.raises(RuntimeError, match="Failed to fetch entry page"):
+ await WebImporter().import_to_directory(
+ root_url="https://example.com",
+ options=WebImportOptions(),
+ )
+
+ async def test_entry_failure_surfaces_renderer_hint(self, monkeypatch):
+ class FakeCrawler:
+ def __init__(self, config):
+ self.config = config
+
+ async def crawl(self, root_url):
+ return SimpleNamespace(
+ pages=[
+ SimpleNamespace(
+ url=root_url,
+ final_url=root_url,
+ depth=0,
+ status="failed",
+ html=None,
+ error="Playwright fallback was needed, but the Python package is not installed.",
+ )
+ ],
+ downloads=[],
+ total_crawled=0,
+ total_downloads=0,
+ total_failed=1,
+ total_skipped=0,
+ fallback_rendered=0,
+ )
+
+ monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
+
+ with pytest.raises(RuntimeError, match="Playwright fallback was needed"):
+ await WebImporter().import_to_directory(
+ root_url="https://example.com",
+ options=WebImportOptions(),
+ )
+
+ async def test_child_render_hint_surfaces_in_meta(self, monkeypatch):
+ from openviking.parse.accessors.web_crawler.playwright_renderer import (
+ PLAYWRIGHT_PACKAGE_INSTALL_HINT,
+ )
+
+ class FakeCrawler:
+ def __init__(self, config):
+ self.config = config
+
+ async def crawl(self, root_url):
+ return SimpleNamespace(
+ pages=[
+ SimpleNamespace(
+ url=root_url,
+ final_url=root_url,
+ depth=0,
+ status="success",
+ html="<html><body><h1>Home</h1></body></html>",
+ error=None,
+ ),
+ SimpleNamespace(
+ url=f"{root_url}/spa",
+ final_url=f"{root_url}/spa",
+ depth=1,
+ status="failed",
+ html=None,
+ error=PLAYWRIGHT_PACKAGE_INSTALL_HINT,
+ ),
+ ],
+ downloads=[],
+ total_crawled=1,
+ total_downloads=0,
+ total_failed=1,
+ total_skipped=0,
+ fallback_rendered=0,
+ )
+
+ monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
+
+ result = await WebImporter().import_to_directory(
+ root_url="https://example.com",
+ options=WebImportOptions(depth=1, max_pages=5),
+ )
+ try:
+ assert result.meta["render_hints"] == [PLAYWRIGHT_PACKAGE_INSTALL_HINT]
+ assert result.meta["page_count"] == 1
+ finally:
+ import shutil
+
+ shutil.rmtree(result.path.parent, ignore_errors=True)
+
+ async def test_import_to_directory_downloads_child_links(self, monkeypatch, tmp_path):
+ class FakeCrawler:
+ def __init__(self, config):
+ self.config = config
+
+ async def crawl(self, root_url):
+ return SimpleNamespace(
+ pages=[
+ SimpleNamespace(
+ url=root_url,
+ final_url=root_url,
+ depth=0,
+ status="success",
+ html="<html><body><h1>Home</h1></body></html>",
+ )
+ ],
+ downloads=[
+ SimpleNamespace(url=f"{root_url}/docs/llms.txt", depth=1),
+ ],
+ total_crawled=1,
+ total_downloads=1,
+ total_failed=0,
+ total_skipped=0,
+ fallback_rendered=0,
+ )
+
+ download_file = tmp_path / "download.txt"
+ download_file.write_text("plain docs", encoding="utf-8")
+
+ async def fake_download_url(self, url, request_validator=None):
+ return str(download_file), SimpleNamespace(value="download_txt"), {
+ "original_filename": "llms.txt",
+ }
+
+ monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
+ monkeypatch.setattr(
+ "openviking.parse.accessors.http_accessor.HTTPAccessor._download_url",
+ fake_download_url,
+ )
+
+ result = await WebImporter().import_to_directory(
+ root_url="https://example.com",
+ options=WebImportOptions(depth=1, max_pages=2),
+ )
+ try:
+ assert (result.path / "docs" / "llms.txt").read_text(encoding="utf-8") == (
+ "plain docs"
+ )
+ assert result.meta["download_count"] == 1
+ assert result.meta["crawl_result"]["total_downloads"] == 1
+ finally:
+ import shutil
+
+ shutil.rmtree(result.path.parent, ignore_errors=True)
diff --git a/uv.lock b/uv.lock
index ecf327a3a..f888c0a18 100644
--- a/uv.lock
+++ b/uv.lock
@@ -366,6 +366,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f6/22/91616fe707a5c5510de2cac9b046a30defe7007ba8a0c04f9c08f27df312/audioop_lts-0.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:b492c3b040153e68b9fdaff5913305aaaba5bb433d8a7f73d5cf6a64ed3cc1dd", size = 25206, upload-time = "2025-08-05T16:43:16.444Z" },
]
+[[package]]
+name = "automat"
+version = "25.4.16"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e3/0f/d40bbe294bbf004d436a8bcbcfaadca8b5140d39ad0ad3d73d1a8ba15f14/automat-25.4.16.tar.gz", hash = "sha256:0017591a5477066e90d26b0e696ddc143baafd87b588cfac8100bc6be9634de0", size = 129977, upload-time = "2025-04-16T20:12:16.002Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/02/ff/1175b0b7371e46244032d43a56862d0af455823b5280a50c63d99cc50f18/automat-25.4.16-py3-none-any.whl", hash = "sha256:04e9bce696a8d5671ee698005af6e5a9fa15354140a87f4870744604dcdd3ba1", size = 42842, upload-time = "2025-04-16T20:12:14.447Z" },
+]
+
[[package]]
name = "babel"
version = "2.18.0"
@@ -778,6 +787,29 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
]
+[[package]]
+name = "constantly"
+version = "23.10.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/4d/6f/cb2a94494ff74aa9528a36c5b1422756330a75a8367bf20bd63171fc324d/constantly-23.10.4.tar.gz", hash = "sha256:aa92b70a33e2ac0bb33cd745eb61776594dc48764b06c35e0efd050b7f1c7cbd", size = 13300, upload-time = "2023-10-28T23:18:24.316Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b8/40/c199d095151addf69efdb4b9ca3a4f20f70e20508d6222bffb9b76f58573/constantly-23.10.4-py3-none-any.whl", hash = "sha256:3fd9b4d1c3dc1ec9757f3c52aef7e53ad9323dbe39f51dfd4c43853b68dfa3f9", size = 13547, upload-time = "2023-10-28T23:18:23.038Z" },
+]
+
+[[package]]
+name = "courlan"
+version = "1.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "babel" },
+ { name = "tld" },
+ { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/bb/16/2a771612ee0b3acaa95ac21cc7e8a3319e815d6360f8ffc5987d1ce28499/courlan-1.4.0.tar.gz", hash = "sha256:fbbac7b7fcde2195ea08e707609503c81cf39c891e8d26cdb1fed4585782d63d", size = 208997, upload-time = "2026-06-01T17:30:17.306Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1f/38/ce65091ff20a16e06d17418c4353af5f56d3190821b1a06983c79ae79274/courlan-1.4.0-py3-none-any.whl", hash = "sha256:ad1dbdefd912ca7238d4607dc855df5df097f56bac175dd662c84eed3802f49e", size = 34193, upload-time = "2026-06-01T17:30:14.984Z" },
+]
+
[[package]]
name = "coverage"
version = "7.13.4"
@@ -1015,6 +1047,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/37/f0/99fe6eb530c7ee9ee1faee48059eb8a6437f80c893a496b98a78864e0fc6/datasets-4.6.1-py3-none-any.whl", hash = "sha256:f53228e6dadc9f837037b1bf3051d7d8c054abbb3eb29f1f022926e08090e0da", size = 520667, upload-time = "2026-02-27T23:26:46.855Z" },
]
+[[package]]
+name = "dateparser"
+version = "1.4.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "python-dateutil" },
+ { name = "pytz" },
+ { name = "regex" },
+ { name = "tzlocal" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d3/f4/561c49bca97af561d34eed27e3e831135eb5cb88e754c1150be41820f5c6/dateparser-1.4.1.tar.gz", hash = "sha256:f265df13c0380e2e07543ba74b67c0681aaa1096981ffcd35227e1aa0cb81c7c", size = 314734, upload-time = "2026-06-15T08:45:47.659Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/8b/7c/2e5dcf53909deddd0bf38cbe277ad9806be038276b1c6c436561b4d9b2e2/dateparser-1.4.1-py3-none-any.whl", hash = "sha256:f25d4e051a84be27a35bd297e3e1dc59ff78373701b89be352ba80372d22d0d0", size = 300503, upload-time = "2026-06-15T08:45:45.951Z" },
+]
+
[[package]]
name = "ddgs"
version = "9.11.3"
@@ -1743,16 +1790,19 @@ wheels = [
]
[[package]]
-name = "html5lib"
-version = "1.1"
+name = "htmldate"
+version = "1.10.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
- { name = "six" },
- { name = "webencodings" },
+ { name = "charset-normalizer" },
+ { name = "dateparser" },
+ { name = "lxml" },
+ { name = "python-dateutil" },
+ { name = "urllib3" },
]
-sdist = { url = "https://files.pythonhosted.org/packages/ac/b6/b55c3f49042f1df3dcd422b7f224f939892ee94f22abcf503a9b7339eaf2/html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f", size = 272215, upload-time = "2020-06-22T23:32:38.834Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ad/1f/e7cf83e23d7b68105de8b874a8b36ba23b450d6f71388583e4ca3ce475ca/htmldate-1.10.0.tar.gz", hash = "sha256:a38df10772ab5d7dbb11896e3f6a852a8491fb1b0965465bc174e23fc2baae58", size = 44455, upload-time = "2026-06-01T17:43:53.437Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/6c/dd/a834df6482147d48e225a49515aabc28974ad5a4ca3215c18a882565b028/html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", size = 112173, upload-time = "2020-06-22T23:32:36.781Z" },
+ { url = "https://files.pythonhosted.org/packages/f7/17/d3356233c826c641f940983d9479eab27faec59d49f4070bc58e80fcc021/htmldate-1.10.0-py3-none-any.whl", hash = "sha256:9211dae35ab94147c8ed9e5fc2c9287a5cf31d2394cb7857e7f5dd814eb2aad6", size = 31561, upload-time = "2026-06-01T17:43:51.797Z" },
]
[[package]]
@@ -1829,6 +1879,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/55/33/71e45a6bd6875f44a26f99da31c63b6840123e88bedf2c0b1ce429b8be12/hvac-2.4.0-py3-none-any.whl", hash = "sha256:008db5efd8c2f77bd37d2368ea5f713edceae1c65f11fd608393179478649e0f", size = 155921, upload-time = "2025-10-30T12:57:46.253Z" },
]
+[[package]]
+name = "hyperlink"
+version = "21.0.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "idna" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3a/51/1947bd81d75af87e3bb9e34593a4cf118115a8feb451ce7a69044ef1412e/hyperlink-21.0.0.tar.gz", hash = "sha256:427af957daa58bc909471c6c40f74c5450fa123dd093fc53efd2e91d2705a56b", size = 140743, upload-time = "2021-01-08T05:51:20.972Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/6e/aa/8caf6a0a3e62863cbb9dab27135660acba46903b703e224f14f447e57934/hyperlink-21.0.0-py2.py3-none-any.whl", hash = "sha256:e6b14c37ecb73e89c77d78cdb4c2cc8f3fb59a885c5b3f819ff4ed80f25af1b4", size = 74638, upload-time = "2021-01-08T05:51:22.906Z" },
+]
+
[[package]]
name = "idna"
version = "3.11"
@@ -1859,6 +1921,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514, upload-time = "2024-09-11T14:56:07.019Z" },
]
+[[package]]
+name = "incremental"
+version = "24.11.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "packaging" },
+ { name = "tomli", marker = "python_full_version < '3.11'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ef/3c/82e84109e02c492f382c711c58a3dd91badda6d746def81a1465f74dc9f5/incremental-24.11.0.tar.gz", hash = "sha256:87d3480dbb083c1d736222511a8cf380012a8176c2456d01ef483242abbbcf8c", size = 24000, upload-time = "2025-11-28T02:30:17.861Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1d/55/0f4df2a44053867ea9cbea73fc588b03c55605cd695cee0a3d86f0029cb2/incremental-24.11.0-py3-none-any.whl", hash = "sha256:a34450716b1c4341fe6676a0598e88a39e04189f4dce5dc96f656e040baa10b3", size = 21109, upload-time = "2025-11-28T02:30:16.442Z" },
+]
+
[[package]]
name = "iniconfig"
version = "2.3.0"
@@ -1868,6 +1943,29 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
]
+[[package]]
+name = "itemadapter"
+version = "0.13.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/52/47/4c75c5396941e653d5f864389964da6951e8f338c6739602dd778f62333e/itemadapter-0.13.1.tar.gz", hash = "sha256:fa139c7be2aa80f8874b2f23d165d5d4aa47c4b85c54ab530b567fd5f684f1b4", size = 32343, upload-time = "2026-01-08T17:56:38.863Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/db/a6/48805cef65b13644f1c23545dc525a7051581c84f5227efb1cd9a8ac9b02/itemadapter-0.13.1-py3-none-any.whl", hash = "sha256:f3c6b1babb4fb6cca4aa9061ef0b0c25c783c24a571c30e3667e7bcfea41815b", size = 18540, upload-time = "2026-01-08T17:56:37.29Z" },
+]
+
+[[package]]
+name = "itemloaders"
+version = "1.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "itemadapter" },
+ { name = "jmespath" },
+ { name = "parsel" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/05/bd/916f4fd26e14e6ad292b69693ccca4f192bcaf9f817ba7d6f7162dbbd835/itemloaders-1.4.0.tar.gz", hash = "sha256:b5338308a819098f43525b7afc5f7d46ba338ba4710f5ebe7a21b3b47bb29929", size = 29740, upload-time = "2026-01-29T12:50:38.04Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ac/71/d9cd0e4c6a4aace991009fc47362ce9251be0fbcf2b6c533f918b31854d5/itemloaders-1.4.0-py3-none-any.whl", hash = "sha256:202b6f855299b4cadfdf78bb93a6cf977899e3c40c4c54524e120a444e65b5ac", size = 12188, upload-time = "2026-01-29T12:50:36.148Z" },
+]
+
[[package]]
name = "jinja2"
version = "3.1.6"
@@ -2043,6 +2141,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" },
]
+[[package]]
+name = "justext"
+version = "3.0.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "lxml", extra = ["html-clean"] },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/49/f3/45890c1b314f0d04e19c1c83d534e611513150939a7cf039664d9ab1e649/justext-3.0.2.tar.gz", hash = "sha256:13496a450c44c4cd5b5a75a5efcd9996066d2a189794ea99a49949685a0beb05", size = 828521, upload-time = "2025-02-25T20:21:49.934Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f2/ac/52f4e86d1924a7fc05af3aeb34488570eccc39b4af90530dd6acecdf16b5/justext-3.0.2-py2.py3-none-any.whl", hash = "sha256:62b1c562b15c3c6265e121cc070874243a443bfd53060e869393f09d6b6cc9a7", size = 837940, upload-time = "2025-02-25T20:21:44.179Z" },
+]
+
[[package]]
name = "kubernetes"
version = "35.0.0"
@@ -2451,126 +2561,120 @@ wheels = [
[[package]]
name = "lxml"
-version = "6.0.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/aa/88/262177de60548e5a2bfc46ad28232c9e9cbde697bd94132aeb80364675cb/lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62", size = 4073426, upload-time = "2025-09-22T04:04:59.287Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/db/8a/f8192a08237ef2fb1b19733f709db88a4c43bc8ab8357f01cb41a27e7f6a/lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388", size = 8590589, upload-time = "2025-09-22T04:00:10.51Z" },
- { url = "https://files.pythonhosted.org/packages/12/64/27bcd07ae17ff5e5536e8d88f4c7d581b48963817a13de11f3ac3329bfa2/lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153", size = 4629671, upload-time = "2025-09-22T04:00:15.411Z" },
- { url = "https://files.pythonhosted.org/packages/02/5a/a7d53b3291c324e0b6e48f3c797be63836cc52156ddf8f33cd72aac78866/lxml-6.0.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f952dacaa552f3bb8834908dddd500ba7d508e6ea6eb8c52eb2d28f48ca06a31", size = 4999961, upload-time = "2025-09-22T04:00:17.619Z" },
- { url = "https://files.pythonhosted.org/packages/f5/55/d465e9b89df1761674d8672bb3e4ae2c47033b01ec243964b6e334c6743f/lxml-6.0.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71695772df6acea9f3c0e59e44ba8ac50c4f125217e84aab21074a1a55e7e5c9", size = 5157087, upload-time = "2025-09-22T04:00:19.868Z" },
- { url = "https://files.pythonhosted.org/packages/62/38/3073cd7e3e8dfc3ba3c3a139e33bee3a82de2bfb0925714351ad3d255c13/lxml-6.0.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f68764f35fd78d7c4cc4ef209a184c38b65440378013d24b8aecd327c3e0c8", size = 5067620, upload-time = "2025-09-22T04:00:21.877Z" },
- { url = "https://files.pythonhosted.org/packages/4a/d3/1e001588c5e2205637b08985597827d3827dbaaece16348c8822bfe61c29/lxml-6.0.2-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:058027e261afed589eddcfe530fcc6f3402d7fd7e89bfd0532df82ebc1563dba", size = 5406664, upload-time = "2025-09-22T04:00:23.714Z" },
- { url = "https://files.pythonhosted.org/packages/20/cf/cab09478699b003857ed6ebfe95e9fb9fa3d3c25f1353b905c9b73cfb624/lxml-6.0.2-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8ffaeec5dfea5881d4c9d8913a32d10cfe3923495386106e4a24d45300ef79c", size = 5289397, upload-time = "2025-09-22T04:00:25.544Z" },
- { url = "https://files.pythonhosted.org/packages/a3/84/02a2d0c38ac9a8b9f9e5e1bbd3f24b3f426044ad618b552e9549ee91bd63/lxml-6.0.2-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:f2e3b1a6bb38de0bc713edd4d612969dd250ca8b724be8d460001a387507021c", size = 4772178, upload-time = "2025-09-22T04:00:27.602Z" },
- { url = "https://files.pythonhosted.org/packages/56/87/e1ceadcc031ec4aa605fe95476892d0b0ba3b7f8c7dcdf88fdeff59a9c86/lxml-6.0.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d6690ec5ec1cce0385cb20896b16be35247ac8c2046e493d03232f1c2414d321", size = 5358148, upload-time = "2025-09-22T04:00:29.323Z" },
- { url = "https://files.pythonhosted.org/packages/fe/13/5bb6cf42bb228353fd4ac5f162c6a84fd68a4d6f67c1031c8cf97e131fc6/lxml-6.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2a50c3c1d11cad0ebebbac357a97b26aa79d2bcaf46f256551152aa85d3a4d1", size = 5112035, upload-time = "2025-09-22T04:00:31.061Z" },
- { url = "https://files.pythonhosted.org/packages/e4/e2/ea0498552102e59834e297c5c6dff8d8ded3db72ed5e8aad77871476f073/lxml-6.0.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:3efe1b21c7801ffa29a1112fab3b0f643628c30472d507f39544fd48e9549e34", size = 4799111, upload-time = "2025-09-22T04:00:33.11Z" },
- { url = "https://files.pythonhosted.org/packages/6a/9e/8de42b52a73abb8af86c66c969b3b4c2a96567b6ac74637c037d2e3baa60/lxml-6.0.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:59c45e125140b2c4b33920d21d83681940ca29f0b83f8629ea1a2196dc8cfe6a", size = 5351662, upload-time = "2025-09-22T04:00:35.237Z" },
- { url = "https://files.pythonhosted.org/packages/28/a2/de776a573dfb15114509a37351937c367530865edb10a90189d0b4b9b70a/lxml-6.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:452b899faa64f1805943ec1c0c9ebeaece01a1af83e130b69cdefeda180bb42c", size = 5314973, upload-time = "2025-09-22T04:00:37.086Z" },
- { url = "https://files.pythonhosted.org/packages/50/a0/3ae1b1f8964c271b5eec91db2043cf8c6c0bce101ebb2a633b51b044db6c/lxml-6.0.2-cp310-cp310-win32.whl", hash = "sha256:1e786a464c191ca43b133906c6903a7e4d56bef376b75d97ccbb8ec5cf1f0a4b", size = 3611953, upload-time = "2025-09-22T04:00:39.224Z" },
- { url = "https://files.pythonhosted.org/packages/d1/70/bd42491f0634aad41bdfc1e46f5cff98825fb6185688dc82baa35d509f1a/lxml-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:dacf3c64ef3f7440e3167aa4b49aa9e0fb99e0aa4f9ff03795640bf94531bcb0", size = 4032695, upload-time = "2025-09-22T04:00:41.402Z" },
- { url = "https://files.pythonhosted.org/packages/d2/d0/05c6a72299f54c2c561a6c6cbb2f512e047fca20ea97a05e57931f194ac4/lxml-6.0.2-cp310-cp310-win_arm64.whl", hash = "sha256:45f93e6f75123f88d7f0cfd90f2d05f441b808562bf0bc01070a00f53f5028b5", size = 3680051, upload-time = "2025-09-22T04:00:43.525Z" },
- { url = "https://files.pythonhosted.org/packages/77/d5/becbe1e2569b474a23f0c672ead8a29ac50b2dc1d5b9de184831bda8d14c/lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607", size = 8634365, upload-time = "2025-09-22T04:00:45.672Z" },
- { url = "https://files.pythonhosted.org/packages/28/66/1ced58f12e804644426b85d0bb8a4478ca77bc1761455da310505f1a3526/lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938", size = 4650793, upload-time = "2025-09-22T04:00:47.783Z" },
- { url = "https://files.pythonhosted.org/packages/11/84/549098ffea39dfd167e3f174b4ce983d0eed61f9d8d25b7bf2a57c3247fc/lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d", size = 4944362, upload-time = "2025-09-22T04:00:49.845Z" },
- { url = "https://files.pythonhosted.org/packages/ac/bd/f207f16abf9749d2037453d56b643a7471d8fde855a231a12d1e095c4f01/lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438", size = 5083152, upload-time = "2025-09-22T04:00:51.709Z" },
- { url = "https://files.pythonhosted.org/packages/15/ae/bd813e87d8941d52ad5b65071b1affb48da01c4ed3c9c99e40abb266fbff/lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964", size = 5023539, upload-time = "2025-09-22T04:00:53.593Z" },
- { url = "https://files.pythonhosted.org/packages/02/cd/9bfef16bd1d874fbe0cb51afb00329540f30a3283beb9f0780adbb7eec03/lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d", size = 5344853, upload-time = "2025-09-22T04:00:55.524Z" },
- { url = "https://files.pythonhosted.org/packages/b8/89/ea8f91594bc5dbb879734d35a6f2b0ad50605d7fb419de2b63d4211765cc/lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7", size = 5225133, upload-time = "2025-09-22T04:00:57.269Z" },
- { url = "https://files.pythonhosted.org/packages/b9/37/9c735274f5dbec726b2db99b98a43950395ba3d4a1043083dba2ad814170/lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178", size = 4677944, upload-time = "2025-09-22T04:00:59.052Z" },
- { url = "https://files.pythonhosted.org/packages/20/28/7dfe1ba3475d8bfca3878365075abe002e05d40dfaaeb7ec01b4c587d533/lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553", size = 5284535, upload-time = "2025-09-22T04:01:01.335Z" },
- { url = "https://files.pythonhosted.org/packages/e7/cf/5f14bc0de763498fc29510e3532bf2b4b3a1c1d5d0dff2e900c16ba021ef/lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb", size = 5067343, upload-time = "2025-09-22T04:01:03.13Z" },
- { url = "https://files.pythonhosted.org/packages/1c/b0/bb8275ab5472f32b28cfbbcc6db7c9d092482d3439ca279d8d6fa02f7025/lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a", size = 4725419, upload-time = "2025-09-22T04:01:05.013Z" },
- { url = "https://files.pythonhosted.org/packages/25/4c/7c222753bc72edca3b99dbadba1b064209bc8ed4ad448af990e60dcce462/lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c", size = 5275008, upload-time = "2025-09-22T04:01:07.327Z" },
- { url = "https://files.pythonhosted.org/packages/6c/8c/478a0dc6b6ed661451379447cdbec77c05741a75736d97e5b2b729687828/lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7", size = 5248906, upload-time = "2025-09-22T04:01:09.452Z" },
- { url = "https://files.pythonhosted.org/packages/2d/d9/5be3a6ab2784cdf9accb0703b65e1b64fcdd9311c9f007630c7db0cfcce1/lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46", size = 3610357, upload-time = "2025-09-22T04:01:11.102Z" },
- { url = "https://files.pythonhosted.org/packages/e2/7d/ca6fb13349b473d5732fb0ee3eec8f6c80fc0688e76b7d79c1008481bf1f/lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078", size = 4036583, upload-time = "2025-09-22T04:01:12.766Z" },
- { url = "https://files.pythonhosted.org/packages/ab/a2/51363b5ecd3eab46563645f3a2c3836a2fc67d01a1b87c5017040f39f567/lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285", size = 3680591, upload-time = "2025-09-22T04:01:14.874Z" },
- { url = "https://files.pythonhosted.org/packages/f3/c8/8ff2bc6b920c84355146cd1ab7d181bc543b89241cfb1ebee824a7c81457/lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456", size = 8661887, upload-time = "2025-09-22T04:01:17.265Z" },
- { url = "https://files.pythonhosted.org/packages/37/6f/9aae1008083bb501ef63284220ce81638332f9ccbfa53765b2b7502203cf/lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924", size = 4667818, upload-time = "2025-09-22T04:01:19.688Z" },
- { url = "https://files.pythonhosted.org/packages/f1/ca/31fb37f99f37f1536c133476674c10b577e409c0a624384147653e38baf2/lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f", size = 4950807, upload-time = "2025-09-22T04:01:21.487Z" },
- { url = "https://files.pythonhosted.org/packages/da/87/f6cb9442e4bada8aab5ae7e1046264f62fdbeaa6e3f6211b93f4c0dd97f1/lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534", size = 5109179, upload-time = "2025-09-22T04:01:23.32Z" },
- { url = "https://files.pythonhosted.org/packages/c8/20/a7760713e65888db79bbae4f6146a6ae5c04e4a204a3c48896c408cd6ed2/lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564", size = 5023044, upload-time = "2025-09-22T04:01:25.118Z" },
- { url = "https://files.pythonhosted.org/packages/a2/b0/7e64e0460fcb36471899f75831509098f3fd7cd02a3833ac517433cb4f8f/lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f", size = 5359685, upload-time = "2025-09-22T04:01:27.398Z" },
- { url = "https://files.pythonhosted.org/packages/b9/e1/e5df362e9ca4e2f48ed6411bd4b3a0ae737cc842e96877f5bf9428055ab4/lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0", size = 5654127, upload-time = "2025-09-22T04:01:29.629Z" },
- { url = "https://files.pythonhosted.org/packages/c6/d1/232b3309a02d60f11e71857778bfcd4acbdb86c07db8260caf7d008b08f8/lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192", size = 5253958, upload-time = "2025-09-22T04:01:31.535Z" },
- { url = "https://files.pythonhosted.org/packages/35/35/d955a070994725c4f7d80583a96cab9c107c57a125b20bb5f708fe941011/lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0", size = 4711541, upload-time = "2025-09-22T04:01:33.801Z" },
- { url = "https://files.pythonhosted.org/packages/1e/be/667d17363b38a78c4bd63cfd4b4632029fd68d2c2dc81f25ce9eb5224dd5/lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092", size = 5267426, upload-time = "2025-09-22T04:01:35.639Z" },
- { url = "https://files.pythonhosted.org/packages/ea/47/62c70aa4a1c26569bc958c9ca86af2bb4e1f614e8c04fb2989833874f7ae/lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f", size = 5064917, upload-time = "2025-09-22T04:01:37.448Z" },
- { url = "https://files.pythonhosted.org/packages/bd/55/6ceddaca353ebd0f1908ef712c597f8570cc9c58130dbb89903198e441fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8", size = 4788795, upload-time = "2025-09-22T04:01:39.165Z" },
- { url = "https://files.pythonhosted.org/packages/cf/e8/fd63e15da5e3fd4c2146f8bbb3c14e94ab850589beab88e547b2dbce22e1/lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f", size = 5676759, upload-time = "2025-09-22T04:01:41.506Z" },
- { url = "https://files.pythonhosted.org/packages/76/47/b3ec58dc5c374697f5ba37412cd2728f427d056315d124dd4b61da381877/lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6", size = 5255666, upload-time = "2025-09-22T04:01:43.363Z" },
- { url = "https://files.pythonhosted.org/packages/19/93/03ba725df4c3d72afd9596eef4a37a837ce8e4806010569bedfcd2cb68fd/lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322", size = 5277989, upload-time = "2025-09-22T04:01:45.215Z" },
- { url = "https://files.pythonhosted.org/packages/c6/80/c06de80bfce881d0ad738576f243911fccf992687ae09fd80b734712b39c/lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849", size = 3611456, upload-time = "2025-09-22T04:01:48.243Z" },
- { url = "https://files.pythonhosted.org/packages/f7/d7/0cdfb6c3e30893463fb3d1e52bc5f5f99684a03c29a0b6b605cfae879cd5/lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f", size = 4011793, upload-time = "2025-09-22T04:01:50.042Z" },
- { url = "https://files.pythonhosted.org/packages/ea/7b/93c73c67db235931527301ed3785f849c78991e2e34f3fd9a6663ffda4c5/lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6", size = 3672836, upload-time = "2025-09-22T04:01:52.145Z" },
- { url = "https://files.pythonhosted.org/packages/53/fd/4e8f0540608977aea078bf6d79f128e0e2c2bba8af1acf775c30baa70460/lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77", size = 8648494, upload-time = "2025-09-22T04:01:54.242Z" },
- { url = "https://files.pythonhosted.org/packages/5d/f4/2a94a3d3dfd6c6b433501b8d470a1960a20ecce93245cf2db1706adf6c19/lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f", size = 4661146, upload-time = "2025-09-22T04:01:56.282Z" },
- { url = "https://files.pythonhosted.org/packages/25/2e/4efa677fa6b322013035d38016f6ae859d06cac67437ca7dc708a6af7028/lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452", size = 4946932, upload-time = "2025-09-22T04:01:58.989Z" },
- { url = "https://files.pythonhosted.org/packages/ce/0f/526e78a6d38d109fdbaa5049c62e1d32fdd70c75fb61c4eadf3045d3d124/lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048", size = 5100060, upload-time = "2025-09-22T04:02:00.812Z" },
- { url = "https://files.pythonhosted.org/packages/81/76/99de58d81fa702cc0ea7edae4f4640416c2062813a00ff24bd70ac1d9c9b/lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df", size = 5019000, upload-time = "2025-09-22T04:02:02.671Z" },
- { url = "https://files.pythonhosted.org/packages/b5/35/9e57d25482bc9a9882cb0037fdb9cc18f4b79d85df94fa9d2a89562f1d25/lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1", size = 5348496, upload-time = "2025-09-22T04:02:04.904Z" },
- { url = "https://files.pythonhosted.org/packages/a6/8e/cb99bd0b83ccc3e8f0f528e9aa1f7a9965dfec08c617070c5db8d63a87ce/lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916", size = 5643779, upload-time = "2025-09-22T04:02:06.689Z" },
- { url = "https://files.pythonhosted.org/packages/d0/34/9e591954939276bb679b73773836c6684c22e56d05980e31d52a9a8deb18/lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd", size = 5244072, upload-time = "2025-09-22T04:02:08.587Z" },
- { url = "https://files.pythonhosted.org/packages/8d/27/b29ff065f9aaca443ee377aff699714fcbffb371b4fce5ac4ca759e436d5/lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6", size = 4718675, upload-time = "2025-09-22T04:02:10.783Z" },
- { url = "https://files.pythonhosted.org/packages/2b/9f/f756f9c2cd27caa1a6ef8c32ae47aadea697f5c2c6d07b0dae133c244fbe/lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a", size = 5255171, upload-time = "2025-09-22T04:02:12.631Z" },
- { url = "https://files.pythonhosted.org/packages/61/46/bb85ea42d2cb1bd8395484fd72f38e3389611aa496ac7772da9205bbda0e/lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679", size = 5057175, upload-time = "2025-09-22T04:02:14.718Z" },
- { url = "https://files.pythonhosted.org/packages/95/0c/443fc476dcc8e41577f0af70458c50fe299a97bb6b7505bb1ae09aa7f9ac/lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659", size = 4785688, upload-time = "2025-09-22T04:02:16.957Z" },
- { url = "https://files.pythonhosted.org/packages/48/78/6ef0b359d45bb9697bc5a626e1992fa5d27aa3f8004b137b2314793b50a0/lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484", size = 5660655, upload-time = "2025-09-22T04:02:18.815Z" },
- { url = "https://files.pythonhosted.org/packages/ff/ea/e1d33808f386bc1339d08c0dcada6e4712d4ed8e93fcad5f057070b7988a/lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2", size = 5247695, upload-time = "2025-09-22T04:02:20.593Z" },
- { url = "https://files.pythonhosted.org/packages/4f/47/eba75dfd8183673725255247a603b4ad606f4ae657b60c6c145b381697da/lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314", size = 5269841, upload-time = "2025-09-22T04:02:22.489Z" },
- { url = "https://files.pythonhosted.org/packages/76/04/5c5e2b8577bc936e219becb2e98cdb1aca14a4921a12995b9d0c523502ae/lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2", size = 3610700, upload-time = "2025-09-22T04:02:24.465Z" },
- { url = "https://files.pythonhosted.org/packages/fe/0a/4643ccc6bb8b143e9f9640aa54e38255f9d3b45feb2cbe7ae2ca47e8782e/lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7", size = 4010347, upload-time = "2025-09-22T04:02:26.286Z" },
- { url = "https://files.pythonhosted.org/packages/31/ef/dcf1d29c3f530577f61e5fe2f1bd72929acf779953668a8a47a479ae6f26/lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf", size = 3671248, upload-time = "2025-09-22T04:02:27.918Z" },
- { url = "https://files.pythonhosted.org/packages/03/15/d4a377b385ab693ce97b472fe0c77c2b16ec79590e688b3ccc71fba19884/lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe", size = 8659801, upload-time = "2025-09-22T04:02:30.113Z" },
- { url = "https://files.pythonhosted.org/packages/c8/e8/c128e37589463668794d503afaeb003987373c5f94d667124ffd8078bbd9/lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d", size = 4659403, upload-time = "2025-09-22T04:02:32.119Z" },
- { url = "https://files.pythonhosted.org/packages/00/ce/74903904339decdf7da7847bb5741fc98a5451b42fc419a86c0c13d26fe2/lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d", size = 4966974, upload-time = "2025-09-22T04:02:34.155Z" },
- { url = "https://files.pythonhosted.org/packages/1f/d3/131dec79ce61c5567fecf82515bd9bc36395df42501b50f7f7f3bd065df0/lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5", size = 5102953, upload-time = "2025-09-22T04:02:36.054Z" },
- { url = "https://files.pythonhosted.org/packages/3a/ea/a43ba9bb750d4ffdd885f2cd333572f5bb900cd2408b67fdda07e85978a0/lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0", size = 5055054, upload-time = "2025-09-22T04:02:38.154Z" },
- { url = "https://files.pythonhosted.org/packages/60/23/6885b451636ae286c34628f70a7ed1fcc759f8d9ad382d132e1c8d3d9bfd/lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba", size = 5352421, upload-time = "2025-09-22T04:02:40.413Z" },
- { url = "https://files.pythonhosted.org/packages/48/5b/fc2ddfc94ddbe3eebb8e9af6e3fd65e2feba4967f6a4e9683875c394c2d8/lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0", size = 5673684, upload-time = "2025-09-22T04:02:42.288Z" },
- { url = "https://files.pythonhosted.org/packages/29/9c/47293c58cc91769130fbf85531280e8cc7868f7fbb6d92f4670071b9cb3e/lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d", size = 5252463, upload-time = "2025-09-22T04:02:44.165Z" },
- { url = "https://files.pythonhosted.org/packages/9b/da/ba6eceb830c762b48e711ded880d7e3e89fc6c7323e587c36540b6b23c6b/lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37", size = 4698437, upload-time = "2025-09-22T04:02:46.524Z" },
- { url = "https://files.pythonhosted.org/packages/a5/24/7be3f82cb7990b89118d944b619e53c656c97dc89c28cfb143fdb7cd6f4d/lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9", size = 5269890, upload-time = "2025-09-22T04:02:48.812Z" },
- { url = "https://files.pythonhosted.org/packages/1b/bd/dcfb9ea1e16c665efd7538fc5d5c34071276ce9220e234217682e7d2c4a5/lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917", size = 5097185, upload-time = "2025-09-22T04:02:50.746Z" },
- { url = "https://files.pythonhosted.org/packages/21/04/a60b0ff9314736316f28316b694bccbbabe100f8483ad83852d77fc7468e/lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f", size = 4745895, upload-time = "2025-09-22T04:02:52.968Z" },
- { url = "https://files.pythonhosted.org/packages/d6/bd/7d54bd1846e5a310d9c715921c5faa71cf5c0853372adf78aee70c8d7aa2/lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8", size = 5695246, upload-time = "2025-09-22T04:02:54.798Z" },
- { url = "https://files.pythonhosted.org/packages/fd/32/5643d6ab947bc371da21323acb2a6e603cedbe71cb4c99c8254289ab6f4e/lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a", size = 5260797, upload-time = "2025-09-22T04:02:57.058Z" },
- { url = "https://files.pythonhosted.org/packages/33/da/34c1ec4cff1eea7d0b4cd44af8411806ed943141804ac9c5d565302afb78/lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c", size = 5277404, upload-time = "2025-09-22T04:02:58.966Z" },
- { url = "https://files.pythonhosted.org/packages/82/57/4eca3e31e54dc89e2c3507e1cd411074a17565fa5ffc437c4ae0a00d439e/lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b", size = 3670072, upload-time = "2025-09-22T04:03:38.05Z" },
- { url = "https://files.pythonhosted.org/packages/e3/e0/c96cf13eccd20c9421ba910304dae0f619724dcf1702864fd59dd386404d/lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed", size = 4080617, upload-time = "2025-09-22T04:03:39.835Z" },
- { url = "https://files.pythonhosted.org/packages/d5/5d/b3f03e22b3d38d6f188ef044900a9b29b2fe0aebb94625ce9fe244011d34/lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8", size = 3754930, upload-time = "2025-09-22T04:03:41.565Z" },
- { url = "https://files.pythonhosted.org/packages/5e/5c/42c2c4c03554580708fc738d13414801f340c04c3eff90d8d2d227145275/lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d", size = 8910380, upload-time = "2025-09-22T04:03:01.645Z" },
- { url = "https://files.pythonhosted.org/packages/bf/4f/12df843e3e10d18d468a7557058f8d3733e8b6e12401f30b1ef29360740f/lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba", size = 4775632, upload-time = "2025-09-22T04:03:03.814Z" },
- { url = "https://files.pythonhosted.org/packages/e4/0c/9dc31e6c2d0d418483cbcb469d1f5a582a1cd00a1f4081953d44051f3c50/lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601", size = 4975171, upload-time = "2025-09-22T04:03:05.651Z" },
- { url = "https://files.pythonhosted.org/packages/e7/2b/9b870c6ca24c841bdd887504808f0417aa9d8d564114689266f19ddf29c8/lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed", size = 5110109, upload-time = "2025-09-22T04:03:07.452Z" },
- { url = "https://files.pythonhosted.org/packages/bf/0c/4f5f2a4dd319a178912751564471355d9019e220c20d7db3fb8307ed8582/lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37", size = 5041061, upload-time = "2025-09-22T04:03:09.297Z" },
- { url = "https://files.pythonhosted.org/packages/12/64/554eed290365267671fe001a20d72d14f468ae4e6acef1e179b039436967/lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338", size = 5306233, upload-time = "2025-09-22T04:03:11.651Z" },
- { url = "https://files.pythonhosted.org/packages/7a/31/1d748aa275e71802ad9722df32a7a35034246b42c0ecdd8235412c3396ef/lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9", size = 5604739, upload-time = "2025-09-22T04:03:13.592Z" },
- { url = "https://files.pythonhosted.org/packages/8f/41/2c11916bcac09ed561adccacceaedd2bf0e0b25b297ea92aab99fd03d0fa/lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd", size = 5225119, upload-time = "2025-09-22T04:03:15.408Z" },
- { url = "https://files.pythonhosted.org/packages/99/05/4e5c2873d8f17aa018e6afde417c80cc5d0c33be4854cce3ef5670c49367/lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d", size = 4633665, upload-time = "2025-09-22T04:03:17.262Z" },
- { url = "https://files.pythonhosted.org/packages/0f/c9/dcc2da1bebd6275cdc723b515f93edf548b82f36a5458cca3578bc899332/lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9", size = 5234997, upload-time = "2025-09-22T04:03:19.14Z" },
- { url = "https://files.pythonhosted.org/packages/9c/e2/5172e4e7468afca64a37b81dba152fc5d90e30f9c83c7c3213d6a02a5ce4/lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e", size = 5090957, upload-time = "2025-09-22T04:03:21.436Z" },
- { url = "https://files.pythonhosted.org/packages/a5/b3/15461fd3e5cd4ddcb7938b87fc20b14ab113b92312fc97afe65cd7c85de1/lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d", size = 4764372, upload-time = "2025-09-22T04:03:23.27Z" },
- { url = "https://files.pythonhosted.org/packages/05/33/f310b987c8bf9e61c4dd8e8035c416bd3230098f5e3cfa69fc4232de7059/lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec", size = 5634653, upload-time = "2025-09-22T04:03:25.767Z" },
- { url = "https://files.pythonhosted.org/packages/70/ff/51c80e75e0bc9382158133bdcf4e339b5886c6ee2418b5199b3f1a61ed6d/lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272", size = 5233795, upload-time = "2025-09-22T04:03:27.62Z" },
- { url = "https://files.pythonhosted.org/packages/56/4d/4856e897df0d588789dd844dbed9d91782c4ef0b327f96ce53c807e13128/lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f", size = 5257023, upload-time = "2025-09-22T04:03:30.056Z" },
- { url = "https://files.pythonhosted.org/packages/0f/85/86766dfebfa87bea0ab78e9ff7a4b4b45225df4b4d3b8cc3c03c5cd68464/lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312", size = 3911420, upload-time = "2025-09-22T04:03:32.198Z" },
- { url = "https://files.pythonhosted.org/packages/fe/1a/b248b355834c8e32614650b8008c69ffeb0ceb149c793961dd8c0b991bb3/lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca", size = 4406837, upload-time = "2025-09-22T04:03:34.027Z" },
- { url = "https://files.pythonhosted.org/packages/92/aa/df863bcc39c5e0946263454aba394de8a9084dbaff8ad143846b0d844739/lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c", size = 3822205, upload-time = "2025-09-22T04:03:36.249Z" },
- { url = "https://files.pythonhosted.org/packages/e7/9c/780c9a8fce3f04690b374f72f41306866b0400b9d0fdf3e17aaa37887eed/lxml-6.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e748d4cf8fef2526bb2a589a417eba0c8674e29ffcb570ce2ceca44f1e567bf6", size = 3939264, upload-time = "2025-09-22T04:04:32.892Z" },
- { url = "https://files.pythonhosted.org/packages/f5/5a/1ab260c00adf645d8bf7dec7f920f744b032f69130c681302821d5debea6/lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ddb1049fa0579d0cbd00503ad8c58b9ab34d1254c77bc6a5576d96ec7853dba", size = 4216435, upload-time = "2025-09-22T04:04:34.907Z" },
- { url = "https://files.pythonhosted.org/packages/f2/37/565f3b3d7ffede22874b6d86be1a1763d00f4ea9fc5b9b6ccb11e4ec8612/lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb233f9c95f83707dae461b12b720c1af9c28c2d19208e1be03387222151daf5", size = 4325913, upload-time = "2025-09-22T04:04:37.205Z" },
- { url = "https://files.pythonhosted.org/packages/22/ec/f3a1b169b2fb9d03467e2e3c0c752ea30e993be440a068b125fc7dd248b0/lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc456d04db0515ce3320d714a1eac7a97774ff0849e7718b492d957da4631dd4", size = 4269357, upload-time = "2025-09-22T04:04:39.322Z" },
- { url = "https://files.pythonhosted.org/packages/77/a2/585a28fe3e67daa1cf2f06f34490d556d121c25d500b10082a7db96e3bcd/lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2613e67de13d619fd283d58bda40bff0ee07739f624ffee8b13b631abf33083d", size = 4412295, upload-time = "2025-09-22T04:04:41.647Z" },
- { url = "https://files.pythonhosted.org/packages/7b/d9/a57dd8bcebd7c69386c20263830d4fa72d27e6b72a229ef7a48e88952d9a/lxml-6.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:24a8e756c982c001ca8d59e87c80c4d9dcd4d9b44a4cbeb8d9be4482c514d41d", size = 3516913, upload-time = "2025-09-22T04:04:43.602Z" },
- { url = "https://files.pythonhosted.org/packages/0b/11/29d08bc103a62c0eba8016e7ed5aeebbf1e4312e83b0b1648dd203b0e87d/lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700", size = 3949829, upload-time = "2025-09-22T04:04:45.608Z" },
- { url = "https://files.pythonhosted.org/packages/12/b3/52ab9a3b31e5ab8238da241baa19eec44d2ab426532441ee607165aebb52/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee", size = 4226277, upload-time = "2025-09-22T04:04:47.754Z" },
- { url = "https://files.pythonhosted.org/packages/a0/33/1eaf780c1baad88224611df13b1c2a9dfa460b526cacfe769103ff50d845/lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f", size = 4330433, upload-time = "2025-09-22T04:04:49.907Z" },
- { url = "https://files.pythonhosted.org/packages/7a/c1/27428a2ff348e994ab4f8777d3a0ad510b6b92d37718e5887d2da99952a2/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9", size = 4272119, upload-time = "2025-09-22T04:04:51.801Z" },
- { url = "https://files.pythonhosted.org/packages/f0/d0/3020fa12bcec4ab62f97aab026d57c2f0cfd480a558758d9ca233bb6a79d/lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a", size = 4417314, upload-time = "2025-09-22T04:04:55.024Z" },
- { url = "https://files.pythonhosted.org/packages/6c/77/d7f491cbc05303ac6801651aabeb262d43f319288c1ea96c66b1d2692ff3/lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e", size = 3518768, upload-time = "2025-09-22T04:04:57.097Z" },
+version = "6.1.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/05/3b/aab6728cae887456f409b4d75e8a01856e4f04bd510de38052a47768b680/lxml-6.1.1.tar.gz", hash = "sha256:ba96ae44888e0185281e937633a743ea90d5a196c6000f82565ebb0580012d40", size = 4197430, upload-time = "2026-05-18T19:19:06.424Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/12/da/dbe4dfc01ac226fb0504fad035f4d69f3202f3502e20e68537631daddd96/lxml-6.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:09dd5b7075dc2f7709654a46543ba1ea3c2e217b2ed8fbd413a8a945a0f40f60", size = 8541124, upload-time = "2026-05-18T19:17:11.589Z" },
+ { url = "https://files.pythonhosted.org/packages/78/20/f7095ed9fc2c025f9cfe71cc6ec9f1feb05624edc1812423b5f1aecf3d4b/lxml-6.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f6ac4ef4d82dff54670227a69c67782ae0b811b5cf6b17954f1e8f7502fc0d1d", size = 4602783, upload-time = "2026-05-18T19:17:20.888Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/a4/65c63ca98bd129f6cff7b8c2fa48953ab058cc6005b541354e7dd54d8000/lxml-6.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:556e94a63c9b04716f8e4de2abb65775061f846e89331b6c5be79183a24f98ea", size = 5002687, upload-time = "2026-05-18T19:17:01.738Z" },
+ { url = "https://files.pythonhosted.org/packages/96/1d/ab7a5c4b5a394d98a94e2d0fc67bab8297597426770dd4978370fbdaf531/lxml-6.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5c6bf403fbb3b3e348a561a5f4f0b9961835657981c802a1df03653eef8a9074", size = 5155099, upload-time = "2026-05-18T19:17:05.159Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/b1/07603bfeeb891a2596d5c2a68f7d2f70f7d11c841ebe391412c69c2857b0/lxml-6.1.1-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1dde6131244bba38a17c745836ba190bc753fd73c9291666287fd0a3fa3dcf30", size = 5057225, upload-time = "2026-05-18T19:17:08.117Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/16/cb391ee4b90186fa16d9ebcbe3ea96c71b8da3b0686386c8dcbcc3c67d44/lxml-6.1.1-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98fc784c2c1440667aeedf8465bdfe10208acf0ead656a2c68627299f546b315", size = 5287643, upload-time = "2026-05-18T19:17:11.507Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/d6/b619717f918fd76747448fdbaee0e769edbc70e659b5b5d0112b7020b7a3/lxml-6.1.1-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:add8cf6ddf9a65116119a28ece0f7886e30af27ba724a7594305f1d1b58a92a1", size = 5412445, upload-time = "2026-05-18T19:17:22.182Z" },
+ { url = "https://files.pythonhosted.org/packages/c6/80/12bc5390ac0a3edeb579d9535e5049a5dda663438728e179d52fb319c33a/lxml-6.1.1-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:cf9d57306d848218f3601fee7601fab1a327c942d56e2e97610583cb4dd74206", size = 4770864, upload-time = "2026-05-18T19:17:26.851Z" },
+ { url = "https://files.pythonhosted.org/packages/0b/59/6500c09da3137f54f020e908d81cfc5ee3e8888e908fd380207afad7c2e6/lxml-6.1.1-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88136950da4d13c318bde414ce10219931937851327f44328f2df4d2c4614067", size = 5359594, upload-time = "2026-05-18T19:17:32.527Z" },
+ { url = "https://files.pythonhosted.org/packages/f2/9b/f64b4cc6b7ebcf75d95af3cde934d254b5f2f10d4163928d838d86b6eb48/lxml-6.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cecdd5dfdc87b1fd87dbf81d4b037a544f47f4c744200a67013771682d67686a", size = 5107713, upload-time = "2026-05-18T19:17:04.402Z" },
+ { url = "https://files.pythonhosted.org/packages/16/19/c7388ad5d3a72315d2832dc1458cbf4f2af7f2b990b606ff4876efd04511/lxml-6.1.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:cd312b9692e831d2ffcad61eab31d91d4b4655a962e61de8fb410472cbcd37aa", size = 4803973, upload-time = "2026-05-18T19:17:06.545Z" },
+ { url = "https://files.pythonhosted.org/packages/3f/22/76197f0bbf165f0b9e75be59be4997e5259cde973f12f098c1b54c7f5d60/lxml-6.1.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:5b7328b46d49fc9477d91ae8f6d55340347d827b7734ba3ea33faae0efef1383", size = 5349925, upload-time = "2026-05-18T19:17:09.743Z" },
+ { url = "https://files.pythonhosted.org/packages/24/52/d2a0cfeccb9bcdc47c7ee05cdae5d69b48c9acf20997790a6338bb0d0b3b/lxml-6.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37a58976370f36d9329d118ad0b953c5aeb9119ac9c6a4e258942a225d0573a1", size = 5309825, upload-time = "2026-05-18T19:17:13.831Z" },
+ { url = "https://files.pythonhosted.org/packages/19/4a/b30944266776c2f49749ef2445aa7e78898194134b80ad776386f61b56ae/lxml-6.1.1-cp310-cp310-win32.whl", hash = "sha256:cea3f4c1af79af13cdb2da0c028111d8f8522d4f22a000c82385535f24e5cf3a", size = 3598402, upload-time = "2026-05-18T19:17:08.21Z" },
+ { url = "https://files.pythonhosted.org/packages/9e/97/33691c66a4d7ec1a5a98e7c909a5b83ee45c7f7ba4cf92b1c4cf26e98079/lxml-6.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:3abf332af33a74288675d936fe861fd4344da0dd6622193fbc4f2bfbb35536b5", size = 4021295, upload-time = "2026-05-18T19:17:28.638Z" },
+ { url = "https://files.pythonhosted.org/packages/d0/5f/26a4dd0e12b9456ff7b12a21af5b491eb6629680d1edd73f4140fd386bcf/lxml-6.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:8dadbe5b217ff35b6a8d16610dd710219b59b76d13f0e3f0d9f36786206e4485", size = 3667717, upload-time = "2026-05-19T19:22:44.474Z" },
+ { url = "https://files.pythonhosted.org/packages/62/b0/83f481780d1548750b8ce2ec824073deef2f452d9cd1a6faff8507e3d16d/lxml-6.1.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:53b7d2b7a10b1c35c0a5e21e9224accf60c1bbfba523990732e521b2b73adef2", size = 8526461, upload-time = "2026-05-18T19:17:25.862Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/d5/30fa0f808002c7329397bfbb24e306789c0b29f04aa5842c07b174b4216f/lxml-6.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ff3f333630ab480244a1bff72043e511a91eb22e7595dead8653ee5612dd8f3d", size = 4595375, upload-time = "2026-05-18T19:17:34.555Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/d2/edb71cf0e561581a7c5eb2626244320eb04e9f8ce6d563184fd668b45073/lxml-6.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a4bbea04c97f6d78a48e3fbc1cb9116d2780b1b39e03a23f6eb9b603fd61f510", size = 4923654, upload-time = "2026-05-18T19:17:42.917Z" },
+ { url = "https://files.pythonhosted.org/packages/4c/77/1bc7eeb0de4577d783fb625aa092cc9357883bba35845a3666bf1259f3dc/lxml-6.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db1d75f6617a49c1c01bc7023713e0ff59ab32c9579ae62a7674c0e34f3b0b0a", size = 5067921, upload-time = "2026-05-18T19:17:49.175Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/3c/c0690d74bd2bc17bc03b5b0d093569ead597dd0bfa088bf99eef8c24e19c/lxml-6.1.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a12689be69a28ddaa0ab99a5a1137da2afd5f8f16df7b5680b66f616d3eda1d", size = 5002456, upload-time = "2026-05-18T19:17:59.715Z" },
+ { url = "https://files.pythonhosted.org/packages/66/8d/d1b3271af0c0f1e27e8472a849e4d2c65bc7766884b9ad2da9e76e145c88/lxml-6.1.1-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b73c339ae29b90fd2d06e58ebd555a751bde9cd6bbd36cc0281b9a2c94e9d8", size = 5202776, upload-time = "2026-05-18T19:18:08.924Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/45/689824ffb237fd10125ad273f32b28ff04dc6203c2822c85ff65a93df65e/lxml-6.1.1-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:752d3bbfe874715ccd0aec7f88d7fc623c0f1fd7aa7b3238a084e017bad2a009", size = 5329945, upload-time = "2026-05-18T19:18:13.673Z" },
+ { url = "https://files.pythonhosted.org/packages/5d/c0/ef73af53767e958fd87d437c170f272e2f6e6c0f854939f133a895f1e711/lxml-6.1.1-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:6b1761fbf9ec984e2e9d9c589ef5f5fd684b7c19f92aadd567a26c5224958db6", size = 4659237, upload-time = "2026-05-18T19:18:18.657Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/5e/e1158e40397585e91cb0472374a1f63d0926a1ddeaa92f13d1a1ffe306d5/lxml-6.1.1-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d680fbcb768404c601ecb43519ecd8461f6954cb11c06a78962f666832ccfca8", size = 5265904, upload-time = "2026-05-18T19:18:24.883Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/16/8687e5d1400ed1c0bc41dace232ebb7553952b618ea1f2e5fb6e2cfbbe23/lxml-6.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:162af1091cd785f2f27e62d3547ae9bc58ec5c86dd314d67021fd02463708d83", size = 5045225, upload-time = "2026-05-18T19:17:20.073Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/18/d877bd1ae2e5ffdfd4836565aba350db31feb2f2656d6ce70316ed66a05e/lxml-6.1.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e9308ff8241c532df3f3e570f9a5aeed6c853f888512ba4b75638d7c11c95ef6", size = 4712721, upload-time = "2026-05-18T19:17:40.512Z" },
+ { url = "https://files.pythonhosted.org/packages/44/4d/1f44fd1d770b10dacbf6b5c6e520f4d6e0708744930f719dc04e67cab981/lxml-6.1.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:5f6994074ebae6ffb04447268e37dc16edc304f9859cf91acb86e0af6c1b395c", size = 5252549, upload-time = "2026-05-18T19:17:51.236Z" },
+ { url = "https://files.pythonhosted.org/packages/64/5d/1d66b84f850089254c230ef6ea6b267a5a54e2e179a5d960036a05d501d7/lxml-6.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:80c2dfadb855da477cf73373ad29a333535dedb9b12bad02c9814c8e2b43bf08", size = 5226877, upload-time = "2026-05-18T19:18:00.875Z" },
+ { url = "https://files.pythonhosted.org/packages/ad/00/84c4b5302d42a2d0184f38d538c8a197f33b52a50bd4f7bcfe990bce3036/lxml-6.1.1-cp311-cp311-win32.whl", hash = "sha256:30a89d3ac8faec007453fb541f3f46807eeec88edd5826f6e3fe001752a2c621", size = 3594072, upload-time = "2026-05-18T19:17:12.714Z" },
+ { url = "https://files.pythonhosted.org/packages/61/9d/2e2f7d876349f45e0f3e29f72da311668853d59b58d473a2dea4f0160135/lxml-6.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:abbefa31eee84842140f67acef1c828e28bba8bbf0c3bc6e5492a9af88152c28", size = 4025469, upload-time = "2026-05-18T19:17:50.566Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/d5/570e6390e4110331e6208b2ba83d1482cc9146808ee118b22824a34c1070/lxml-6.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:dcb292aa7fe485ceff7af4f92e46c5af397daec5dff64871a528f0fc47a3cc5b", size = 3667640, upload-time = "2026-05-19T19:22:48.293Z" },
+ { url = "https://files.pythonhosted.org/packages/6a/6e/c4add832b6fc1e887125b96f880d7b9b70aae5248718e046b1704bcac4b9/lxml-6.1.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:104c09bda8d2a562824c0e319d0768ce26a779b7601e0931d33b09b53c392ef7", size = 8570821, upload-time = "2026-05-18T19:17:42.068Z" },
+ { url = "https://files.pythonhosted.org/packages/22/00/ff3009c88e65de8011630acf8ab5a09cb2becd2aaf47fba2f3449f6224e9/lxml-6.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:25c6997a9a534e016695a0ba06b2f07945de682731ff01065b6d5a4474179da1", size = 4624252, upload-time = "2026-05-18T19:17:47.897Z" },
+ { url = "https://files.pythonhosted.org/packages/42/95/bb63f0fd62e554fe078e1fb3c8fe9083c14ddc7ad7fa178d10e57e071ac7/lxml-6.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c921ba5c51e4e9f63b8b00267d06566e1f63407408a0496da2d1d0bfc819c7fc", size = 4930746, upload-time = "2026-05-18T19:18:29.637Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/99/0013e8d9b5960f4f041cf0b73e2f80c23eb5205b1f7bfb20203243651359/lxml-6.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:54a7f95e4de5fb94e2f9f4b9055c6ba33bf3d628fd77a1d647c5923caa2cdcdc", size = 5093723, upload-time = "2026-05-18T19:18:34.168Z" },
+ { url = "https://files.pythonhosted.org/packages/29/91/317b332636bfc7bddcff828d41b3307f50043f4b237e40849c333d80fa1a/lxml-6.1.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f2ec43df44b1f76249ee0a615334f9b5b060e1c8bd90e706dad2d14d02f383", size = 5005557, upload-time = "2026-05-18T19:18:39.798Z" },
+ { url = "https://files.pythonhosted.org/packages/42/2f/cc9bf06afe70f9c9093ae60855d9759da9db601ec4080f7473319666ffd7/lxml-6.1.1-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:70ef8a7e102a1508f8121aae5b0867abd663f72c14f0a9c937e6554cb4587b7b", size = 5631036, upload-time = "2026-05-18T19:18:44.858Z" },
+ { url = "https://files.pythonhosted.org/packages/08/f6/af32e23e563971ffb0fb86be52bc5be5c2c118858ffc119bf6a9039b173d/lxml-6.1.1-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ebe6af670449830d6d9b752c256a983291c766a1365ba5d5460048f9e33a7818", size = 5240367, upload-time = "2026-05-18T19:18:49.217Z" },
+ { url = "https://files.pythonhosted.org/packages/78/83/8555d40948b09ce86f1bd0c68a7ac31d07b1929f92cc1b074006c97ef2d2/lxml-6.1.1-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:27acc820660aaffa4f7c087f29120e12980f7779d56d8492d263170111284740", size = 5350171, upload-time = "2026-05-18T19:18:52.779Z" },
+ { url = "https://files.pythonhosted.org/packages/63/75/5d92da93729b7bad783689e6496049fa40927b45bec7bf183c981de3ca70/lxml-6.1.1-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:1db753c9115ec7100d073b744d17e25e88a8f90f5c39b2f5dd878149af59671f", size = 4694874, upload-time = "2026-05-18T19:18:55.139Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/b5/3aad415a9a25b822e783f15deeb4dffccf5113030f1afa2222dd929313d9/lxml-6.1.1-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c4f469aebd783bb741c2ecb2a681008fd26bfe5c16a9a72ed5467f834e810df2", size = 5244492, upload-time = "2026-05-18T19:19:01.28Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/a1/5fcf7eb9904b80086aa47dcf0027de07b1bb990afad2e6823144c368ae04/lxml-6.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:766b010012d59470072c1816b5b6c69f1d243e5db36ea5968e94accf430a4635", size = 5048232, upload-time = "2026-05-18T19:18:12.67Z" },
+ { url = "https://files.pythonhosted.org/packages/77/74/1f601b63c7a69fcdf10fa9b148c81da8442204194f6c55509cc485c786b9/lxml-6.1.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b8d812c6011c08b8111a15e54dd990b8923692d80adf35488bee34026c35accf", size = 4777023, upload-time = "2026-05-18T19:18:15.928Z" },
+ { url = "https://files.pythonhosted.org/packages/a2/b9/7a78f51aec95b1bf780d78e12705a9f6533284f8693dc5c0e6724fa53d3f/lxml-6.1.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:fe0306bd29505a9177aac19f1877174b0e7422c222a59f70b2cd41633448c3dc", size = 5645773, upload-time = "2026-05-18T19:18:23.223Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/6e/98a7b7ad54e4e74fa1f20fff776913980619d0ebe5558232d7da6580bdd8/lxml-6.1.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5ba186ad207446c65d3bb3d3e0412b032b1d9f595e59861e2354798c5703d955", size = 5233088, upload-time = "2026-05-18T19:18:31.433Z" },
+ { url = "https://files.pythonhosted.org/packages/65/d1/bc0ed2427bf609f2ee10da303a6a226f9c8bce94f945dc29a32ce55de6e4/lxml-6.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aa366a1e55b8ebfe8ca8ddc3cfe75c8ebade181aeb0f661d0cb05986b647f72a", size = 5260995, upload-time = "2026-05-18T19:18:37.091Z" },
+ { url = "https://files.pythonhosted.org/packages/69/8b/6772e1a4b513fc50a8d931f19edde0e13ae6918510a1e13ff67864f3e5ed/lxml-6.1.1-cp312-cp312-win32.whl", hash = "sha256:126c93f7f56f0eda92f6d8c619edc463a4f23d9252f1c9d0405a76f25fa9f11a", size = 3596382, upload-time = "2026-05-18T19:17:18.37Z" },
+ { url = "https://files.pythonhosted.org/packages/1b/89/45198e9624762af2dfd2cb8782598477ceb29f6e59caab560388ae1f4ec1/lxml-6.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:26e6eda8d38c1fcab1090dd196ee87cbd13788e531937610e2589085de074e77", size = 3997255, upload-time = "2026-05-18T19:17:56.781Z" },
+ { url = "https://files.pythonhosted.org/packages/90/a9/7a54b6834088d9ae528a7b780584ba6a39a9457b0ac330479f20ffbc9449/lxml-6.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:6540377fbd53fe1b629172288c464fb18db11ce1fa7dc15891da10aa9dcc3e7f", size = 3659610, upload-time = "2026-05-19T19:22:50.843Z" },
+ { url = "https://files.pythonhosted.org/packages/a5/eb/7e6f37c5584ccbb2ff267f56fd0339016938c1c8684cfefab9b33ffc2f36/lxml-6.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:68a9198d0fc122d14bb76837de9aa80cf84caed990b5b237f532ed87d3706736", size = 8559780, upload-time = "2026-05-18T19:17:57.661Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/36/587c2521cf23a2cd6c9c22108aa7528f683a1f195ed7ccd23a4b1786ad36/lxml-6.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7d47866cb32fb503450b6edc9df355d10dc49836af2e89901bd6ac6b0896d9d9", size = 4618006, upload-time = "2026-05-18T19:18:04.452Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/ca/ab7bfe2bf4c972af5e7878262845ead3a24a929a9b04bc11c7c1ece6c82a/lxml-6.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb7c9811bfaa8b1ed5ed319f5d370dfbcaa59d52ea64be2a5a85e18195930354", size = 4924139, upload-time = "2026-05-18T19:19:04.873Z" },
+ { url = "https://files.pythonhosted.org/packages/6b/55/a0c72851dfee5ecc689f949723a73dea457758912542cb955b108eaf0d8f/lxml-6.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:762ff394d5bd56da0cf034a23dcce4e13923f15321a2adfa2ac00201dc6d3fca", size = 5082329, upload-time = "2026-05-18T19:19:09.728Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/b6/0608f7d61a3b96cc67e5648a3d906e31a5082093e10e7be65b3886289938/lxml-6.1.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a088f287f7d8275a33c07f2cac6c50b9319309a0200a39e7e75d80c707723099", size = 4993564, upload-time = "2026-05-18T19:19:13.608Z" },
+ { url = "https://files.pythonhosted.org/packages/4c/66/ae227524b066d29d55bf0b453d93d2d793c40218657d643dcbbca13b8faf/lxml-6.1.1-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e902da4b04e6b52e5893900d4b8ab46068f75f3561f01bf1080957f9fd932ed6", size = 5613467, upload-time = "2026-05-18T19:19:16.228Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/76/dbe4a00b50385e40194231dcfe5a12c059de7cf90e89c83407d2b085b719/lxml-6.1.1-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1d4962d4c66bf830a7e59ed6cfc17d148149898a3aefa8ec6e59763e6e3ed085", size = 5228304, upload-time = "2026-05-18T19:19:19.354Z" },
+ { url = "https://files.pythonhosted.org/packages/1c/01/00b1b8442ed2041793336868ba0b9ea4b13d7da7c085c6404c207a63bf79/lxml-6.1.1-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:581d4c8ae690a6609e64862dd6b7c2489635c2d13907fc2b20f2bc200ff1d21e", size = 5341607, upload-time = "2026-05-18T19:19:22.297Z" },
+ { url = "https://files.pythonhosted.org/packages/63/36/1ad29931e9a4638bb707869f01d423a6c815f82152138d1a40dfcfde2b95/lxml-6.1.1-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:876e1ff5930ed8bf295ec5ef9a8155e9b6b1876bbf1deed8b3a8069311875a8f", size = 4700168, upload-time = "2026-05-18T19:19:25.133Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/d1/a9536cecf9be18a0dc72d32bead283a2332d1ffebd2dd3ac70ce444686e5/lxml-6.1.1-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9eb9b5a968f6e0f6d640092a567e14529ff8cea2e29d00da6f78a79fa49f013c", size = 5232487, upload-time = "2026-05-18T19:19:28.603Z" },
+ { url = "https://files.pythonhosted.org/packages/0e/77/b4fb1e03bf5d130e879214d3100092e386418807fb74dd0adc4b0a48f351/lxml-6.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:aa49e06d94aba782c6a02eecb7e507969e7e7a41b267f1b359bb35585f295d5b", size = 5044231, upload-time = "2026-05-18T19:18:42.246Z" },
+ { url = "https://files.pythonhosted.org/packages/26/4c/d00daeeb0a5530c4028a9232aa1b93db3ef4ed2158c116ea73c79a9765b3/lxml-6.1.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:70cdfd80589d59e43e18005dd7244e8895e93db8ab6a620b7e23df5445a4e3d2", size = 4769450, upload-time = "2026-05-18T19:18:48.013Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/6a/715a3a8d156ce42f29cf014706f5410c2ff3b02267774110fc23266409fe/lxml-6.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:aad9aa39483ed8ec44d6d2e59e5b98a0d80676ef0d92f44bfc374836111f62f5", size = 5635874, upload-time = "2026-05-18T19:18:51.914Z" },
+ { url = "https://files.pythonhosted.org/packages/45/37/0544bc21dde2a88f3a17b504e6fc79c0e01d25a33c2f6079724e9e72b9c7/lxml-6.1.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d49514be2f28d895c38cf9d2b72d7b9a07d00314519f456c0b50b53cfcf4c785", size = 5223987, upload-time = "2026-05-18T19:18:59.715Z" },
+ { url = "https://files.pythonhosted.org/packages/4d/f8/f6a5e8185bcb28c2befae3d31f8e3df3b811cb0f47746517a81279fcafe1/lxml-6.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:47402e62c52ff5988c1e8c6c63177f5708bccf48e366dea4e3dcf1e645e04947", size = 5250276, upload-time = "2026-05-18T19:19:03.834Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/f2/1a2b9f1b7a49d45495369be7ef9ad05b262930f2eab3e3145706fca8083f/lxml-6.1.1-cp313-cp313-win32.whl", hash = "sha256:3483644525531e1d5762b0c44a8e18b6efba321b6dcf8a8952de10b037618bca", size = 3596903, upload-time = "2026-05-18T19:17:29.863Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/99/f4ffb024f238eec2131aaa09f3278fb6129cf892741bf68e1fc1afb8c100/lxml-6.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:a10bd2fd62e8ce916ececb342f348f190724a098c1faa056fdfb2a22ad5e8660", size = 3995869, upload-time = "2026-05-18T19:18:02.596Z" },
+ { url = "https://files.pythonhosted.org/packages/d1/53/70eb8c5c6037f27448f1e3c54ebede9545a801ae63f0a7254afca4fe8e45/lxml-6.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:424aa57aca0897eb922aef34395bd1289b3b6f04e6bae20ea123c0c7e333cffc", size = 3658490, upload-time = "2026-05-19T19:22:53.846Z" },
+ { url = "https://files.pythonhosted.org/packages/13/e2/2e325795566de01d0d7c3bb57d3c370616b2d07b01214e84eec5d3b10963/lxml-6.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:19b7ab10b210b0b3ad7985d9ac4eb66ab09a90b20fe6e2f7ba55d01a234345d0", size = 8577146, upload-time = "2026-05-18T19:18:17.765Z" },
+ { url = "https://files.pythonhosted.org/packages/93/cf/5630b5e4be7d2e6bee8efe83865c925221103cf0221303b104ce134b01e2/lxml-6.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c08e5c694306507275f2290073350c4f32e383db15213b2c69e7ff39c1193840", size = 4623866, upload-time = "2026-05-18T19:18:30.669Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/51/3904907c063451cf8d4a5c9fe0cad95fa1f4ec57f4e3884fa0731bd7a305/lxml-6.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:74a9717fd0d82effef5c2854f0d917231d5324b5a3eb7275c43ac9fa32f97a14", size = 4950022, upload-time = "2026-05-18T19:19:31.958Z" },
+ { url = "https://files.pythonhosted.org/packages/94/cd/9c7611a51c37a2830928405817cc5d56a97f64fab83cc3f628748b135749/lxml-6.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:efe0374196335f93b53269acd811b944f2e6bdc88e8894f214bd636455484909", size = 5086695, upload-time = "2026-05-18T19:19:34.764Z" },
+ { url = "https://files.pythonhosted.org/packages/da/d6/24e3b5906abb0b674ff2ae195bc3ce59708df2bcd17cf17703b2d7dd643a/lxml-6.1.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac931cdc9442c1763b8a8f6cd62c0c938737eafc5be75eff88df55fc73bc0d00", size = 5031642, upload-time = "2026-05-18T19:19:37.771Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/db/6ec54f99019838bff54785c51da07f189eb4676861c5f2730962b0d8d665/lxml-6.1.1-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:aee395f5d0927f947758b4ec119fd5fc8ec71f07a1c5c52077b30b04c0fa6955", size = 5647338, upload-time = "2026-05-18T19:19:40.553Z" },
+ { url = "https://files.pythonhosted.org/packages/42/3d/ef4dcfffd22d27a61805d8ed9f7fb888495bc6aa88648fa07c1eaa5586b6/lxml-6.1.1-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9395002973c827b3ed67db77e6ec09f092919a587022174554096a269378fb13", size = 5239528, upload-time = "2026-05-18T19:19:43.657Z" },
+ { url = "https://files.pythonhosted.org/packages/62/bb/37fb3f0dff146bdcfa78eec47879273820b2a0bf350ec236ce14bd0b1c26/lxml-6.1.1-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:73bc2086f141224ebddb7fc5c6a36ca58b31b94b561e1dfe8e073e3270fad1e7", size = 5350730, upload-time = "2026-05-18T19:19:46.307Z" },
+ { url = "https://files.pythonhosted.org/packages/90/42/43253f168388df4fae1f38c01df36ddb9bee39e2048167b54cdcbae85ea3/lxml-6.1.1-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:3779def59032b81e44a5f70096ef6bf2082f8d901937dca354474ba09782e245", size = 4697530, upload-time = "2026-05-18T19:19:49.889Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/a8/c5a8504f81bbdfc8e7094c2c850cdb4ed6777fc4d5ddd9e5ab819f3b0d54/lxml-6.1.1-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:86c89b9d55ebf820ad7c90bc533410f0d098054f293351f10603c0c46ff598f5", size = 5250670, upload-time = "2026-05-18T19:19:53.199Z" },
+ { url = "https://files.pythonhosted.org/packages/77/b7/c7e76ab18744d75e21f320ebf9ff9d1ceae2b54dd431ea5a64caf26c9672/lxml-6.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19607c6bbff2a44cf3fe8250abccd20942d3462473e0a721d01d379ed017e462", size = 5084485, upload-time = "2026-05-18T19:19:08.422Z" },
+ { url = "https://files.pythonhosted.org/packages/31/31/b35c53f8ef7b7c31cacd23d3638652fff7bcd1deb6eedb709ab43b685908/lxml-6.1.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:c6ed5141a5c7507cf3ee76bd363b0d6f801e3321adc35b5d825a23115faa5465", size = 4737635, upload-time = "2026-05-18T19:19:12.321Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/06/31f23c813a7fe8e0cb1b175e915b08c9bf4e86d225b210feadbdbe519667/lxml-6.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:62aeb7e85b5d60320b9d77eef2e773994e2c0ce10121b277e0a19804e1654a5a", size = 5670681, upload-time = "2026-05-18T19:19:15.001Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/bc/ce619bccc89b1fd9ad8a8e1330ee3f3beff9f2ff95b712d7bbcdd6e22fc3/lxml-6.1.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b1b963fd8f5caa68e99dfae060d54de1fe9cba899b8718b44a00cdca53c3e590", size = 5238229, upload-time = "2026-05-18T19:19:18.131Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/5d/b329acbbedc0b619ebc2be6cf7ee9ed07e80892c88d4dfd612c33805789a/lxml-6.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:63876be28efefa04a1df615b46770e82042cce445cfdce55160522f57b231ccb", size = 5264191, upload-time = "2026-05-18T19:19:21.118Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/85/be36fb1425b30db3c3f9df75fe86343ebffb79e6320bd7f588e25bfeac39/lxml-6.1.1-cp314-cp314-win32.whl", hash = "sha256:7f7a92e8583f06b1fd49d01158143b8461cfcd135dcb10ec807270a3051bd603", size = 3657202, upload-time = "2026-05-18T19:17:39.509Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/ce/3cf9a827342269f54d405a6202397de63f07c69cbd6ce7d183a3f0cba1e9/lxml-6.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:b2d444f2e66624d68e9c6b211e28a76e22fff5fcabcfff4deac18b529b7d4137", size = 4064497, upload-time = "2026-05-18T19:18:14.662Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/3e/1a957bde8f0760039e627f94699f82caa782c9d838d86c3d28245ee67212/lxml-6.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3fd9728a2735fda14f4e8235830c86b539e9661e849665bf926d3f867943b4bf", size = 3741991, upload-time = "2026-05-19T19:22:59.111Z" },
+ { url = "https://files.pythonhosted.org/packages/78/b2/00ed55b3a2efa4658fb795c38d1090ec9b3e8a6c3683d4441fa517f09c3b/lxml-6.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:787b2496d0dbe8cd180984e8d29e3a6f76e7ea34db781cb3bd55e4ba1ef8b4ee", size = 8827545, upload-time = "2026-05-18T19:18:41.193Z" },
+ { url = "https://files.pythonhosted.org/packages/c0/73/74573db19baa618d5f266f2407898b087ff6927115b00b71e5fc1b700847/lxml-6.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:2c8daa471358dc2d6fcf02165e80ec68f77871a286df95bc5cc3816153b0fd2c", size = 4735736, upload-time = "2026-05-18T19:18:46.761Z" },
+ { url = "https://files.pythonhosted.org/packages/16/02/6f7061f4f95f51e545d48e87647c54791d204a4e881be4156e7a26ba5338/lxml-6.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:acd7d70b64c0aae0c7922cca83d288a16f5f6da523637697872253415269baef", size = 4970291, upload-time = "2026-05-18T19:19:56.215Z" },
+ { url = "https://files.pythonhosted.org/packages/b0/02/55fc057d8283427dea7d6edb102e7a840239c77a64a983d92f62a304c0e9/lxml-6.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4f0dd2f01f9f8a89f565d000e03abcf0a13d692a346c8d22f628d49af098777a", size = 5102822, upload-time = "2026-05-18T19:19:59.223Z" },
+ { url = "https://files.pythonhosted.org/packages/e4/48/8e1cf78d89d66850121d9255a2a24414c98f775da93b90cf976956c24b14/lxml-6.1.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b7e8a14c8634bf6f7a568634cb395305a6d964aeb5b7ee32248094bed3a7e2c", size = 5027923, upload-time = "2026-05-18T19:20:01.549Z" },
+ { url = "https://files.pythonhosted.org/packages/ed/00/0632a0647612c8af24d26997b3b961397daa9d5b2581444805933629a4cb/lxml-6.1.1-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:86281fbdd6a8162756f8d603f37e3435bfa38043adb79c6dc6a2dfee065e7525", size = 5595843, upload-time = "2026-05-18T19:20:03.93Z" },
+ { url = "https://files.pythonhosted.org/packages/bc/86/ab008a7dc360711b66858d61c80a5979a70a09f2aa2b05d9698df80b803d/lxml-6.1.1-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5d7152ec39ca7c402d8fb9bad86140a15b9503bd0c54484e3f1bbe3dd37ceca", size = 5224515, upload-time = "2026-05-18T19:20:06.381Z" },
+ { url = "https://files.pythonhosted.org/packages/75/c6/2702ff375e728e34f56d9a45339a9cf7e4427e917f542225242d63a05afa/lxml-6.1.1-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:88d8cb75b9d82858497a5393e3c63cfbf03035225e4b35a49ed7ccb151e4dc0e", size = 5312511, upload-time = "2026-05-18T19:20:09.308Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/57/a5807c98f87a86f10ef9ffab35516df7c0f0c4b6d5d33e9f608ab9c04a31/lxml-6.1.1-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f64ec5397ea6a41fc1b4af0380d79b44a755b5531dcaccd9940fb260dca93038", size = 4639206, upload-time = "2026-05-18T19:20:11.704Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/e1/8a0a2c35734812395f4da4eaf33748a7e5705bfb2a58b128da764339d5ec/lxml-6.1.1-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d34bbf07dbc7ca5970671b1512e928991fb5e9d95365636c9b2d8b4f53af405e", size = 5232404, upload-time = "2026-05-18T19:20:14.064Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/e2/0e6a4dd5ad84d01d99aa7bae7cfefd4a760a0e0f8176818241de17d9b6c0/lxml-6.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:17e0e18d4ad8adbd0399291bc44845b69d9dd68439a3cdebdf35ff902ec05072", size = 5083769, upload-time = "2026-05-18T19:19:23.758Z" },
+ { url = "https://files.pythonhosted.org/packages/a0/7e/161f33d463f6ffc1c7679104b65086dea120080d49dde4d238f015aaee2f/lxml-6.1.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:3ab541146f1f6968c462d6c2ac495148e8cdba2f8347700b2141b6ec5a75bf52", size = 4758936, upload-time = "2026-05-18T19:19:27.256Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/fb/2369825e3f6ca99305bf9f7b7085fda91c8b0922a89e54d900974aa3ef85/lxml-6.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2a0217714657e023ef4293500f65aa20fce6164c8fd6b08fa5bd4a859fb14b9b", size = 5620296, upload-time = "2026-05-18T19:19:29.993Z" },
+ { url = "https://files.pythonhosted.org/packages/30/90/d61e383146f74c5ab683947ea14dc7b82778838ab9b95ea73a23b60d0191/lxml-6.1.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:05a82eb6e1530a64f26225b55cbd178113bd0b5af1c2b625f25e5296742c26d2", size = 5228598, upload-time = "2026-05-18T19:19:33.523Z" },
+ { url = "https://files.pythonhosted.org/packages/76/2d/2dafd8149e94b05bb070690efd5bb2680720681e03ff03fc57d2b70a1105/lxml-6.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9e36f163528fc50cbef305f02a5fd66d404edf7049cdaff211dbc2cba5a7013e", size = 5247845, upload-time = "2026-05-18T19:19:36.649Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/68/b30e913340c380ddac9580c6e6230991fc37240ec4f64704833e4f3e2769/lxml-6.1.1-cp314-cp314t-win32.whl", hash = "sha256:649dda677cf3bd6ac9ae14007ba0c824ded8ce5808b53fc7431d9140399118c1", size = 3897345, upload-time = "2026-05-18T19:17:33.562Z" },
+ { url = "https://files.pythonhosted.org/packages/3c/4e/9eb2af5335545f9fbcd7af57bcf87c6025d31eaa31b14ec184a6c8675328/lxml-6.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:793033d6c5cdf33a573f910d9bea14ef8f5771820411d118da8e1182edb53d5e", size = 4393350, upload-time = "2026-05-18T19:18:10.076Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/2c/0f1e93c636720e8a3eb59af2bfda99d98b55891e1c53bc30c2e0e865f01b/lxml-6.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:58bb955caba94e467d2a96da17660d2d704e0675894cba21ab8a775b8621fd1c", size = 3817223, upload-time = "2026-05-19T19:22:56.823Z" },
+ { url = "https://files.pythonhosted.org/packages/b5/32/86a3f0f724a3a402d4627937a7fc27b160e45e7012b4adf47f6e1e844511/lxml-6.1.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:31033dc34636ea6b7d5cc11b1ddbda78a14de858ba9d3e1ed4b69a3085bc521e", size = 3930127, upload-time = "2026-05-18T19:19:02.27Z" },
+ { url = "https://files.pythonhosted.org/packages/40/44/d832e82af08723761556d004b1d04d281c09f9a8cecd7d3148548c9941a3/lxml-6.1.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3893c14c4b6ac5b2d54ba8cf03e99fe5104e592de491f19bd6b82756c09f8004", size = 4210769, upload-time = "2026-05-18T19:20:41.427Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/39/0dc5949f759ed7d951e0bb8c2f2d9d7aca1908d22352fa84a8afd2ea54af/lxml-6.1.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c07da4cebf6889f03ebac8d238f62318e29f495de0aa18a51ea14e61ae907e2e", size = 4318163, upload-time = "2026-05-18T19:20:44.702Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/fb/8ab3845fe046ba4cbf74536bcf6801a774b7caf4350de1c5d37f1f0a9e90/lxml-6.1.1-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6f0ce10945fab9c4c06ce14e22af9059d1a87493a9af4501a5b0b9187e21cf2", size = 4250945, upload-time = "2026-05-18T19:20:47.385Z" },
+ { url = "https://files.pythonhosted.org/packages/68/1b/7553ab136894374ffae8851ec06f98f511cd8e66246e41b6be059d0a7289/lxml-6.1.1-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f8844cd288697c6425c9beba919302241e3278871dc6519515e72b04e987abcf", size = 4401664, upload-time = "2026-05-18T19:20:50.489Z" },
+ { url = "https://files.pythonhosted.org/packages/db/a4/441aee36c6f6b249823d20fd91f9be9ab89d7c5a8ae542a4a4ca6d342d56/lxml-6.1.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:ed21202aec73cda4d55d1ce57b389aadb90ffb044e6cd1080b8347efe1b1ec84", size = 3508989, upload-time = "2026-05-18T19:18:38.158Z" },
]
[package.optional-dependencies]
@@ -2631,19 +2735,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" },
]
-[[package]]
-name = "markdownify"
-version = "1.2.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "beautifulsoup4" },
- { name = "six" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/43/ce/f1e3e9d959db134cedf06825fae8d5b294bd368aacdd0831a3975b7c4d55/markdownify-1.2.2-py3-none-any.whl", hash = "sha256:3f02d3cc52714084d6e589f70397b6fc9f2f3a8531481bf35e8cc39f975e186a", size = 15724, upload-time = "2025-11-16T19:21:17.622Z" },
-]
-
[[package]]
name = "markupsafe"
version = "3.0.3"
@@ -3533,7 +3624,6 @@ dependencies = [
{ name = "lark-oapi" },
{ name = "litellm" },
{ name = "loguru" },
- { name = "markdownify" },
{ name = "mcp" },
{ name = "olefile" },
{ name = "openai" },
@@ -3553,9 +3643,10 @@ dependencies = [
{ name = "python-multipart" },
{ name = "python-pptx" },
{ name = "pyyaml" },
- { name = "readabilipy" },
{ name = "requests" },
+ { name = "scrapy" },
{ name = "tabulate" },
+ { name = "trafilatura" },
{ name = "tree-sitter" },
{ name = "tree-sitter-c-sharp" },
{ name = "tree-sitter-cpp" },
@@ -3786,7 +3877,6 @@ requires-dist = [
{ name = "litellm", specifier = ">=1.83.7,<1.89.3" },
{ name = "llama-cpp-python", marker = "extra == 'local-embed'", specifier = ">=0.3.0" },
{ name = "loguru", specifier = ">=0.7.3" },
- { name = "markdownify", specifier = ">=0.11.0" },
{ name = "mcp", specifier = ">=1.27.0" },
{ name = "mcp", marker = "extra == 'bot'", specifier = ">=1.0.0" },
{ name = "msgpack", marker = "extra == 'bot'", specifier = ">=1.0.8" },
@@ -3833,11 +3923,11 @@ requires-dist = [
{ name = "qq-botpy", marker = "extra == 'bot-qq'", specifier = ">=1.0.0" },
{ name = "ragas", marker = "extra == 'eval'", specifier = ">=0.1.0" },
{ name = "ragas", marker = "extra == 'test'", specifier = ">=0.1.0" },
- { name = "readabilipy", specifier = ">=0.2.0" },
{ name = "readability-lxml", marker = "extra == 'bot'", specifier = ">=0.8.0" },
{ name = "requests", specifier = ">=2.31.0" },
{ name = "rich", marker = "extra == 'bot'", specifier = ">=13.0.0" },
{ name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" },
+ { name = "scrapy", specifier = ">=2.11.0" },
{ name = "setuptools", marker = "extra == 'build'", specifier = ">=61.0" },
{ name = "setuptools-scm", marker = "extra == 'build'", specifier = ">=8.0" },
{ name = "setuptools-scm", marker = "extra == 'dev'", specifier = ">=10.0.0" },
@@ -3848,6 +3938,7 @@ requires-dist = [
{ name = "tabulate", specifier = ">=0.9.0" },
{ name = "tavily-python", marker = "extra == 'bot'", specifier = ">=0.5.0" },
{ name = "tiktoken", marker = "extra == 'benchmark'", specifier = ">=0.5.0" },
+ { name = "trafilatura", specifier = ">=1.12.0" },
{ name = "tree-sitter", specifier = ">=0.23.0" },
{ name = "tree-sitter-c-sharp", specifier = ">=0.23.0" },
{ name = "tree-sitter-cpp", specifier = ">=0.23.0" },
@@ -4172,6 +4263,22 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/68/b0/34937815889fa982613775e4b97fddd13250f11012d769949c5465af2150/pandas-3.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:108dd1790337a494aa80e38def654ca3f0968cf4f362c85f44c15e471667102d", size = 9452085, upload-time = "2026-02-17T22:20:14.331Z" },
]
+[[package]]
+name = "parsel"
+version = "1.11.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cssselect" },
+ { name = "jmespath" },
+ { name = "lxml" },
+ { name = "packaging" },
+ { name = "w3lib" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/91/c8/4ace3a5c61e39ca21734a5715d0e076eea6200dd8daea2a5b99452f5a0d6/parsel-1.11.0.tar.gz", hash = "sha256:5925fe087eb16fc404a7ed91e31e2c1e2a9b230da4b64f34d81358c0d0e27e88", size = 106849, upload-time = "2026-01-29T07:19:23.388Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/08/23/4e0dae5e5bee14aea26dba003a682e621563451a20f751ed985810f818b6/parsel-1.11.0-py3-none-any.whl", hash = "sha256:bda82575df1774dd64e1c1396163f3cadb3e383e0f8080d43d45fa6705355daa", size = 14176, upload-time = "2026-01-29T07:19:22.255Z" },
+]
+
[[package]]
name = "pathspec"
version = "1.1.1"
@@ -4479,6 +4586,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" },
]
+[[package]]
+name = "protego"
+version = "0.6.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/f3/5d/b06b682f6a0a722fd5b90577cc8762894758478e1b92a1439ec9f23ebc9e/protego-0.6.1.tar.gz", hash = "sha256:b2b740edd5c691d882a198a66b1d463bf0d574ec915c29f44d2da2bac8efe139", size = 3137534, upload-time = "2026-06-11T10:21:08.294Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/73/9c/a94c0617a71bcf0b8f2c9d0c8f281482dbfebb165929e7712f27d4a0a229/protego-0.6.1-py3-none-any.whl", hash = "sha256:cc8e328c7698b352e8967e5feae770aa597e4965c60bbf7e8200ba8c5017d77b", size = 10345, upload-time = "2026-06-11T10:21:07.212Z" },
+]
+
[[package]]
name = "protobuf"
version = "6.33.5"
@@ -4847,6 +4963,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/00/4b/ccc026168948fec4f7555b9164c724cf4125eac006e176541483d2c959be/pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237", size = 58929, upload-time = "2026-02-19T13:45:06.034Z" },
]
+[[package]]
+name = "pydispatcher"
+version = "2.0.7"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/21/db/030d0700ae90d2f9d52c2f3c1f864881e19cef8cba3b0a08759c8494c19c/PyDispatcher-2.0.7.tar.gz", hash = "sha256:b777c6ad080dc1bad74a4c29d6a46914fa6701ac70f94b0d66fbcfde62f5be31", size = 38891, upload-time = "2023-02-17T20:11:13.106Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/66/0e/9ee7bc0b48ec45d93b302fa2d787830dca4dc454d31a237faa5815995988/PyDispatcher-2.0.7-py3-none-any.whl", hash = "sha256:96543bea04115ffde08f851e1d45cacbfd1ee866ac42127d9b476dc5aefa7de0", size = 12040, upload-time = "2023-02-17T20:11:11.991Z" },
+]
+
[[package]]
name = "pydub"
version = "0.25.1"
@@ -4882,6 +5007,19 @@ crypto = [
{ name = "cryptography" },
]
+[[package]]
+name = "pyopenssl"
+version = "26.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cryptography" },
+ { name = "typing-extensions", marker = "python_full_version < '3.13'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/74/b7/da07bae88f5a9506b4def6f2f4903cf4c3b8831e560dba8fa18ca08f758f/pyopenssl-26.3.0.tar.gz", hash = "sha256:589de7fae1c9ea670d18422ed00fc04da787bbde8e1454aea872aa57b49ad341", size = 182024, upload-time = "2026-06-12T20:28:07.458Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/54/18/1dd71c9b43192ab83f1d531ad6002dc81108ac36c475f79fb7a295abe2f4/pyopenssl-26.3.0-py3-none-any.whl", hash = "sha256:46367f8f66b92271e6d218da9c87607e1ef5a0bc5c8dea5bb3db82f395c385a3", size = 56008, upload-time = "2026-06-12T20:28:05.999Z" },
+]
+
[[package]]
name = "pypdfium2"
version = "5.6.0"
@@ -4920,6 +5058,12 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" },
]
+[[package]]
+name = "pypydispatcher"
+version = "2.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/d5/7b/65f55513d3c769fd677f90032d8d8703e3dc17e88a41b6074d2177548bca/PyPyDispatcher-2.1.2.tar.gz", hash = "sha256:b6bec5dfcff9d2535bca2b23c80eae367b1ac250a645106948d315fcfa9130f2", size = 23224, upload-time = "2017-07-03T14:20:51.806Z" }
+
[[package]]
name = "pytesseract"
version = "0.3.13"
@@ -5216,6 +5360,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/94/2e/cf662566627f1c3508924ef5a0f8277ffc4ac033d6c3a05d1ead6e76f60b/qq_botpy-1.2.1-py3-none-any.whl", hash = "sha256:18b215690dfed88f711322136ec54b6760040b9b1608eb5db7a44e00f59e4f01", size = 51356, upload-time = "2024-03-22T10:57:24.695Z" },
]
+[[package]]
+name = "queuelib"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/76/f3/d80ab8c7c91b8c42d9a2aa4dd97a8be1321e7b26000c2675b75e641d958c/queuelib-1.9.0.tar.gz", hash = "sha256:b12fea79fd8c1dd23e212b1f3db58003b773949801d4f4e6f34d882467d4a192", size = 11729, upload-time = "2026-01-29T11:19:37.065Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/4a/1c/8df7b461497b42fcc1e7c44529201975ec77b0e1ebecd00df4b1f096c1d4/queuelib-1.9.0-py3-none-any.whl", hash = "sha256:c5fd3bebf2c924446fa94fca6b72e81168f79cf4c2a9143b8b26f266a423fcf3", size = 13585, upload-time = "2026-01-29T11:19:35.616Z" },
+]
+
[[package]]
name = "ragas"
version = "0.3.1"
@@ -5241,21 +5394,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/ff/2e/1e06552b2c2e2b979664ac50492a89f84da51e4b1c16e39364c0ecdc09ae/ragas-0.3.1-py3-none-any.whl", hash = "sha256:b61ba63c56d460e43edb0f99a65559766961ac64e628bc5ccaa61da7e8687fa5", size = 190654, upload-time = "2025-08-11T11:15:30.442Z" },
]
-[[package]]
-name = "readabilipy"
-version = "0.3.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
- { name = "beautifulsoup4" },
- { name = "html5lib" },
- { name = "lxml" },
- { name = "regex" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/b8/e4/260a202516886c2e0cc6e6ae96d1f491792d829098886d9529a2439fbe8e/readabilipy-0.3.0.tar.gz", hash = "sha256:e13313771216953935ac031db4234bdb9725413534bfb3c19dbd6caab0887ae0", size = 35491, upload-time = "2024-12-02T23:03:02.311Z" }
-wheels = [
- { url = "https://files.pythonhosted.org/packages/dd/46/8a640c6de1a6c6af971f858b2fb178ca5e1db91f223d8ba5f40efe1491e5/readabilipy-0.3.0-py3-none-any.whl", hash = "sha256:d106da0fad11d5fdfcde21f5c5385556bfa8ff0258483037d39ea6b1d6db3943", size = 22158, upload-time = "2024-12-02T23:03:00.438Z" },
-]
-
[[package]]
name = "readability-lxml"
version = "0.8.4.1"
@@ -5421,6 +5559,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" },
]
+[[package]]
+name = "requests-file"
+version = "3.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/3c/f8/5dc70102e4d337063452c82e1f0d95e39abfe67aa222ed8a5ddeb9df8de8/requests_file-3.0.1.tar.gz", hash = "sha256:f14243d7796c588f3521bd423c5dea2ee4cc730e54a3cac9574d78aca1272576", size = 6967, upload-time = "2025-10-20T18:56:42.279Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/e1/d5/de8f089119205a09da657ed4784c584ede8381a0ce6821212a6d4ca47054/requests_file-3.0.1-py2.py3-none-any.whl", hash = "sha256:d0f5eb94353986d998f80ac63c7f146a307728be051d4d1cd390dbdb59c10fa2", size = 4514, upload-time = "2025-10-20T18:56:41.184Z" },
+]
+
[[package]]
name = "requests-oauthlib"
version = "2.0.0"
@@ -5653,6 +5803,35 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2e/a3/0f0b7d78e2f1eb9e8e1afbff1d2bff8d60144aee17aca51c065b516743dd/safehttpx-0.1.7-py3-none-any.whl", hash = "sha256:c4f4a162db6993464d7ca3d7cc4af0ffc6515a606dfd220b9f82c6945d869cde", size = 8959, upload-time = "2025-10-24T18:30:08.733Z" },
]
+[[package]]
+name = "scrapy"
+version = "2.16.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cryptography" },
+ { name = "cssselect" },
+ { name = "defusedxml" },
+ { name = "itemadapter" },
+ { name = "itemloaders" },
+ { name = "lxml" },
+ { name = "packaging" },
+ { name = "parsel" },
+ { name = "protego" },
+ { name = "pydispatcher", marker = "platform_python_implementation == 'CPython'" },
+ { name = "pyopenssl" },
+ { name = "pypydispatcher", marker = "platform_python_implementation == 'PyPy'" },
+ { name = "queuelib" },
+ { name = "service-identity" },
+ { name = "tldextract" },
+ { name = "twisted" },
+ { name = "w3lib" },
+ { name = "zope-interface" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cd/a0/f710c21e87d64686f1b6d153eff7d1a2f7167bd94dd6c19aada07629fb38/scrapy-2.16.0.tar.gz", hash = "sha256:7287952a81fa302797ec0c3370a0096a8d81d4f9fe25a608a76df2a164511714", size = 1280834, upload-time = "2026-05-19T12:29:19.202Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1d/c9/7cc7631ca85c08b7e1162e1e08ecb0c139f8f640cc403257ac500a8e50fe/scrapy-2.16.0-py3-none-any.whl", hash = "sha256:fcb83db5500743503b1fea4d55c81aac056b15481d0aad46b6f929d324d9bcf5", size = 344254, upload-time = "2026-05-19T12:29:17.206Z" },
+]
+
[[package]]
name = "semantic-version"
version = "2.10.0"
@@ -5662,6 +5841,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177", size = 15552, upload-time = "2022-05-26T13:35:21.206Z" },
]
+[[package]]
+name = "service-identity"
+version = "26.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "attrs" },
+ { name = "cryptography" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/61/87/ad52e2c582c0f0e7f0a1b86950494c38d67422dc0f5ed9044a5fb9569a49/service_identity-26.1.0.tar.gz", hash = "sha256:6358c52882c96e66ac4a55eb3a72c7dd4a70763f8cc6fa4e70abde2656f4bf3b", size = 42898, upload-time = "2026-05-30T12:04:55.184Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/93/eb/2433e1af4ff903499144de4846569fb3300b816179ae99a03c2f011b666a/service_identity-26.1.0-py3-none-any.whl", hash = "sha256:68c32dadbb69135fb951077677e07cd7f6031020f3a8c8f47a28cda8a0742118", size = 11370, upload-time = "2026-05-30T12:04:53.911Z" },
+]
+
[[package]]
name = "setuptools"
version = "82.0.1"
@@ -6135,6 +6327,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" },
]
+[[package]]
+name = "tld"
+version = "0.13.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/5c/5d/76b4383ac4e5b5e254e50c09807b3e13820bed6d6c11cd540264988d6802/tld-0.13.2.tar.gz", hash = "sha256:d983fa92b9d717400742fca844e29d5e18271079c7bcfabf66d01b39b4a14345", size = 467175, upload-time = "2026-03-06T23:50:34.498Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9e/90/39a85a4b63c84213e78b3c17d22e1bf45328acf8ebb33ef93be30d0a3911/tld-0.13.2-py2.py3-none-any.whl", hash = "sha256:9b8fdbdb880e7ba65b216a4937f2c94c49a7226723783d5838fc958ac76f4e0c", size = 296743, upload-time = "2026-03-06T23:50:32.465Z" },
+]
+
+[[package]]
+name = "tldextract"
+version = "5.3.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "filelock" },
+ { name = "idna" },
+ { name = "requests" },
+ { name = "requests-file" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/65/7b/644fbbb49564a6cb124a8582013315a41148dba2f72209bba14a84242bf0/tldextract-5.3.1.tar.gz", hash = "sha256:a72756ca170b2510315076383ea2993478f7da6f897eef1f4a5400735d5057fb", size = 126105, upload-time = "2025-12-28T23:58:05.532Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/6d/42/0e49d6d0aac449ca71952ec5bae764af009754fcb2e76a5cc097543747b3/tldextract-5.3.1-py3-none-any.whl", hash = "sha256:6bfe36d518de569c572062b788e16a659ccaceffc486d243af0484e8ecf432d9", size = 105886, upload-time = "2025-12-28T23:58:04.071Z" },
+]
+
[[package]]
name = "tokenizers"
version = "0.22.2"
@@ -6240,6 +6456,24 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" },
]
+[[package]]
+name = "trafilatura"
+version = "2.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "certifi" },
+ { name = "charset-normalizer" },
+ { name = "courlan" },
+ { name = "htmldate" },
+ { name = "justext" },
+ { name = "lxml" },
+ { name = "urllib3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e8/19/24833e905df2d80e3bb67424f95febcc17709a1f61a522120bc438afca70/trafilatura-2.1.0.tar.gz", hash = "sha256:f689e2116fc89c7bc0b9a296d01dcfe2eb0b5455f8c371a77dc0db1f06a05643", size = 263876, upload-time = "2026-06-07T17:43:31.829Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0e/78/4ad99d79aee2784f49f20fd0a29058ce4c032fe4439047924c43521cd211/trafilatura-2.1.0-py3-none-any.whl", hash = "sha256:0eded5207a806445ddebbe36eae30b9035fe6a2f233c36f6fe82663fca8b9d30", size = 134600, upload-time = "2026-06-07T17:43:28.404Z" },
+]
+
[[package]]
name = "tree-sitter"
version = "0.25.2"
@@ -6444,6 +6678,24 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9f/e4/81f9a935789233cf412a0ed5fe04c883841d2c8fb0b7e075958a35c65032/tree_sitter_typescript-0.23.2-cp39-abi3-win_arm64.whl", hash = "sha256:05db58f70b95ef0ea126db5560f3775692f609589ed6f8dd0af84b7f19f1cbb7", size = 274052, upload-time = "2024-11-11T02:36:09.514Z" },
]
+[[package]]
+name = "twisted"
+version = "26.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "attrs" },
+ { name = "automat" },
+ { name = "constantly" },
+ { name = "hyperlink" },
+ { name = "incremental" },
+ { name = "typing-extensions" },
+ { name = "zope-interface" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/db/97/6e9beb1e78247ae6dc34114f27d538cf2cb183c4afcd3609dfdf2b0439c8/twisted-26.4.0.tar.gz", hash = "sha256:dbfd0fe1ee409d0243fdd7a6a6ff14f4948cec1fd78e0376291f805e1501fae9", size = 3575095, upload-time = "2026-05-11T11:24:51.861Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a6/57/bcf4e2370dd218c9aa68a9140a65d86729c73f1d529f7e94786c2766fc72/twisted-26.4.0-py3-none-any.whl", hash = "sha256:dc25ea0ebf6511c24f03232ee9f4afa54b291c5d897990e3a39cc4d14a1ef4c0", size = 3230362, upload-time = "2026-05-11T11:24:49.5Z" },
+]
+
[[package]]
name = "typer"
version = "0.23.1"
@@ -6622,21 +6874,21 @@ ark = [
]
[[package]]
-name = "wcwidth"
-version = "0.6.0"
+name = "w3lib"
+version = "2.4.1"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/c0/91/b2eb59c2cf243de5de1e91c963655df78c015509f51297685a8c86a27b8c/w3lib-2.4.1.tar.gz", hash = "sha256:8dd69ee39ff6398d708c793abc779c334a69bac7cee1cdf71736c669ed6be864", size = 48494, upload-time = "2026-03-20T09:50:27.477Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" },
+ { url = "https://files.pythonhosted.org/packages/66/c3/f8b216cbd742e5b84c40f045204c764ccb7524d2aeab021054ec69446b0a/w3lib-2.4.1-py3-none-any.whl", hash = "sha256:40930132907e68de906a5b89331ab8c8ff4f01bd35b5539ef7896017d814138d", size = 21695, upload-time = "2026-03-20T09:50:26.187Z" },
]
[[package]]
-name = "webencodings"
-version = "0.5.1"
+name = "wcwidth"
+version = "0.6.0"
source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47/webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923", size = 9721, upload-time = "2017-04-05T20:21:34.189Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/35/a2/8e3becb46433538a38726c948d3399905a4c7cabd0df578ede5dc51f0ec2/wcwidth-0.6.0.tar.gz", hash = "sha256:cdc4e4262d6ef9a1a57e018384cbeb1208d8abbc64176027e2c2455c81313159", size = 159684, upload-time = "2026-02-06T19:19:40.919Z" }
wheels = [
- { url = "https://files.pythonhosted.org/packages/f4/24/2a3e3df732393fed8b3ebf2ec078f05546de641fe1b667ee316ec1dcf3b7/webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", size = 11774, upload-time = "2017-04-05T20:21:32.581Z" },
+ { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" },
]
[[package]]
@@ -7112,6 +7364,54 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" },
]
+[[package]]
+name = "zope-interface"
+version = "8.5"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/08/dc/50550cfcbb2ea3cbca5f1d7ed05c8aa840f831a0f2d63aec0a953f7c590e/zope_interface-8.5.tar.gz", hash = "sha256:7a3ba1c5877f0f3e3906b02ddf793abed2becc2948116414ce0e1dd820b68d6d", size = 257957, upload-time = "2026-05-26T06:50:14.574Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b6/43/9cd98bee951d23848de690ba2809f87e3b22c67c370987acc960da15ad37/zope_interface-8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0c8aa2bf8f3911ef37b87deb1bbe225a310e6eb6522a16d77f5d8330c4f6fbe", size = 210951, upload-time = "2026-05-26T06:49:00.178Z" },
+ { url = "https://files.pythonhosted.org/packages/17/0f/8f1a29966bcf863e3a2121edcafb81c55715de7886bcc9544749cc79e7da/zope_interface-8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:efe234a0fafb4b6b1602e9be9245b97c2bf06d67c07af5a4bc3c0438978b555c", size = 211309, upload-time = "2026-05-26T06:49:02.732Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/9f/37e564eaaf85e3abc1ada40a79fa43f2ab45bdb67431b0ec0fe29e4763e2/zope_interface-8.5-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:dabeb6fe1228d411994f300811edc6866fff0cdcbc9cef98a78f05ea0da42e37", size = 254881, upload-time = "2026-05-26T06:49:04.303Z" },
+ { url = "https://files.pythonhosted.org/packages/06/61/e6501d8ea7a2cac3217e03f404e1f98c1df7191d83cfe86b1895fbba5dac/zope_interface-8.5-cp310-cp310-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:147a9442dcc2b7339ecdb1be2b3cdb098e90462e39425054053ebfb50d99125a", size = 259811, upload-time = "2026-05-26T06:49:06.373Z" },
+ { url = "https://files.pythonhosted.org/packages/91/15/bfa25ef480b02af6e9452c478483fec75e87c9e2b60c407fd0b1f6054b9c/zope_interface-8.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a17e681224267880707c9ec9e730ad9a1ad2d65c371256843efba6cf48711b58", size = 260358, upload-time = "2026-05-26T06:49:08.317Z" },
+ { url = "https://files.pythonhosted.org/packages/64/51/2b518072fea76242da64451d501c69b7b5ccdef9b57fead584ccf1c180d5/zope_interface-8.5-cp310-cp310-win_amd64.whl", hash = "sha256:d178968a1a611df30549a717d1624cb38ca810347339e3e37b7baa6f6781a170", size = 214822, upload-time = "2026-05-26T06:49:10.441Z" },
+ { url = "https://files.pythonhosted.org/packages/ce/f1/83ad110fb847413affe71609bb50e59e1aa082e1236030122227c7c283d3/zope_interface-8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:afc66ccaef2a3c0bef6ca02aad40d29a39276389dad16a8eac36f9f385e4d057", size = 211426, upload-time = "2026-05-26T06:49:12.595Z" },
+ { url = "https://files.pythonhosted.org/packages/bb/a7/6b6e0c31ac240cb9fc015ae9ed45ca54be886c18fcf7bfa2377a4d7a8785/zope_interface-8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c28044972187245d7a309e4699319bfdbd2ffcbf7176d1d4ddf5adffb2dea80f", size = 211850, upload-time = "2026-05-26T06:49:14.474Z" },
+ { url = "https://files.pythonhosted.org/packages/37/36/7599ecabcf80ce4fef2e1ef3c5ac0d4696b61f03f724cc44022f4d226af9/zope_interface-8.5-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:03bbecc7982af713d7499d4084bc03916413d17ffd45f89009348cc0c1d9e376", size = 260711, upload-time = "2026-05-26T06:49:16.568Z" },
+ { url = "https://files.pythonhosted.org/packages/03/3e/1774b0ee46ccbb5498ee3c33ece40315b6ef58bc71957be94bd345340bc1/zope_interface-8.5-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bf917009a4a7457c7290225a019f4a0aa706d96accd2cfdba2418d3bc1fcde2f", size = 265277, upload-time = "2026-05-26T06:49:18.656Z" },
+ { url = "https://files.pythonhosted.org/packages/b6/09/e533b2ffabaae4e5d5730d6768a591cf335defe8e37bec2ad905d09be656/zope_interface-8.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:31cff25b2aaedb5267e6e77b1e9be6b0ec4f622032de8a069202b8ffacda7dc2", size = 266369, upload-time = "2026-05-26T06:49:20.174Z" },
+ { url = "https://files.pythonhosted.org/packages/49/4a/3ebe6a4c122b2d5340db45cbe7e490663d3228b172710ec71060cd5d541e/zope_interface-8.5-cp311-cp311-win_amd64.whl", hash = "sha256:17a3114bbdddb5e75e5784cdf318944636190cbbc72d357ef9fb1a8b0351f955", size = 215161, upload-time = "2026-05-26T06:49:21.799Z" },
+ { url = "https://files.pythonhosted.org/packages/d2/59/056ad97af5b16db1975ee98ec7ab03d2ce3f3355efad904ced1dbce0e39f/zope_interface-8.5-cp311-cp311-win_arm64.whl", hash = "sha256:aab6bb5bee10f38ea688b95ba054396b67f613552d2c8378be7fcb2d2fba7646", size = 213481, upload-time = "2026-05-26T06:49:25.085Z" },
+ { url = "https://files.pythonhosted.org/packages/97/cc/b84123a948f3162a34623e188922827cd845244fdd043ed20f8d02228caa/zope_interface-8.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8e6ee90c2e6de7c37058d5fa41f123c8b13a312db8d1e0fb5840d7f4bcdff9c9", size = 212165, upload-time = "2026-05-26T06:49:26.566Z" },
+ { url = "https://files.pythonhosted.org/packages/4e/78/cbceec44f1b27208a76c1a688c131302685852406a23df5aab68324109cc/zope_interface-8.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c1adc90d3576b3b4c4de4953e6002c37bef28b78d7fa54c1bbfd0c50f022fe7c", size = 212341, upload-time = "2026-05-26T06:49:28.182Z" },
+ { url = "https://files.pythonhosted.org/packages/e1/c3/005032195ff3b210c139b7c560ed5c534e844b0907d8e44d2b3d8919305e/zope_interface-8.5-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:e6347b8d8d12c5eca6502450a92be30079b7acfade2c4f693efa0deb8871b06e", size = 265296, upload-time = "2026-05-26T06:49:29.741Z" },
+ { url = "https://files.pythonhosted.org/packages/c5/66/1036543d6a66bc04c19df3cf650f3ad938a002ab0a443c24e23e8de5e8b9/zope_interface-8.5-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5e970dabea777a24b0b0bbf9dae3ab75ce8b2d8e948edf4875627034b21f3560", size = 270689, upload-time = "2026-05-26T06:49:31.767Z" },
+ { url = "https://files.pythonhosted.org/packages/30/4c/8b56259558cace4414e753ca6740396a1f59d4a95ddb55b4658600408670/zope_interface-8.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f0b48ccadaa9839e09ff81e969703cecb3f402c813bfe8b958652e699bea69f5", size = 270280, upload-time = "2026-05-26T06:49:33.489Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/ea/649908c83aa8fdb7faf2ddca4d3cf6fb8f2157121267dc56e8f72681e26c/zope_interface-8.5-cp312-cp312-win_amd64.whl", hash = "sha256:e0e311f1277468c08fd59a2b41f71b43d25dff639789d364747acd1705c0df6e", size = 215019, upload-time = "2026-05-26T06:49:35.607Z" },
+ { url = "https://files.pythonhosted.org/packages/9f/97/da13037b4c563e4df32eedbc819f8c00b754af494f68211e3dffd48d52da/zope_interface-8.5-cp312-cp312-win_arm64.whl", hash = "sha256:652b73107a04159ec6c020db6c1543d4f1e8f4d069bd2aac88a947820923517b", size = 213569, upload-time = "2026-05-26T06:49:37.317Z" },
+ { url = "https://files.pythonhosted.org/packages/f4/8c/4c15755d701f2ec0e80d64a18e1ebaf5be2c584c0ec153fd516f5d13eada/zope_interface-8.5-cp313-cp313-macosx_10_9_x86_64.whl", hash = "sha256:28e80457c134d1fa57a7d758004dece348654e1b1467ac22dcdc20fc1d127c52", size = 212512, upload-time = "2026-05-26T06:49:38.996Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/2e/4360c54c465db042cc8fbeeec92abac28b4cedbf6ba63c1f092fd08a190f/zope_interface-8.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:09495ce9d559c06b70f2d4855b3e4f48a822a9ddc8be1d30c5b4e5be14ae1ace", size = 212541, upload-time = "2026-05-26T06:49:41.186Z" },
+ { url = "https://files.pythonhosted.org/packages/aa/a5/692a2b8d70f78e848793231d5fae5fecbf8d0cccd73430fdc34802a6d3c1/zope_interface-8.5-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:7849ad8fa90763cc1087f4dda78ca3a233e950b3e08fac7079297c9cafbbd7bb", size = 265191, upload-time = "2026-05-26T06:49:43.449Z" },
+ { url = "https://files.pythonhosted.org/packages/70/8d/454a9cfc7a050c394ab4f11b3371f7897828b7415e096afff724637e65e0/zope_interface-8.5-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5578c9421ca409a1f39f153d6f7803e4cde01da592ec75a9ac5e1b777d18d33b", size = 270626, upload-time = "2026-05-26T06:49:45.425Z" },
+ { url = "https://files.pythonhosted.org/packages/51/8c/db8409cfa3575b8e9b4800babd7d49f8228433cd1f0c56814bd0ada49c33/zope_interface-8.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e1bd7d96b4ca5fa311f54c9eac16dce4886b428c1531dbe06067763ccdf123b4", size = 270444, upload-time = "2026-05-26T06:49:47.025Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/df/a386940e41469ef615e100a216d8b386521e9e598817147f87932ca203c4/zope_interface-8.5-cp313-cp313-win_amd64.whl", hash = "sha256:0c8123d2a4dfde2a613c7cb772605477724782c20bc2e0ad1d9435376a6a44a3", size = 215021, upload-time = "2026-05-26T06:49:48.478Z" },
+ { url = "https://files.pythonhosted.org/packages/89/75/477eb5669b6b2a7a843decd1a075e9b1971a8720017654143a7183abd3d9/zope_interface-8.5-cp313-cp313-win_arm64.whl", hash = "sha256:6d02be14f3173c6c7288bc2fdf530090c01c3cf8764ad46c68024686f364278e", size = 213610, upload-time = "2026-05-26T06:49:50.01Z" },
+ { url = "https://files.pythonhosted.org/packages/d4/19/5032e954827fdf02db2d2f49737ac4378bb9cfc2cd95a8f2e2a5ae2ec01a/zope_interface-8.5-cp314-cp314-macosx_10_9_x86_64.whl", hash = "sha256:ffaecf013251a89d0de6feb49a46eba48ad8cbbf8a40aeb6045e459e7bec6784", size = 212597, upload-time = "2026-05-26T06:49:51.63Z" },
+ { url = "https://files.pythonhosted.org/packages/f1/53/3ef644012cf8a6a234a2d6134aab5a5c65ac5467c86296865501d4fbc406/zope_interface-8.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:126fa9d1c52295ae076d4cf968634f0a1826afa408a20808b57ff72877b8f69f", size = 212626, upload-time = "2026-05-26T06:49:53.236Z" },
+ { url = "https://files.pythonhosted.org/packages/32/67/bc8b4f465d388039255003e230c284a175cedf1203c692f23cb7bff64efe/zope_interface-8.5-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:3090e3a663d20194756a59a272e0c8508b889341e31d5894223331fe6b4f9b21", size = 266827, upload-time = "2026-05-26T06:49:54.873Z" },
+ { url = "https://files.pythonhosted.org/packages/a7/eb/37d05b935ede53d79690fecc8d201440084418e590bcfc05f384451c7593/zope_interface-8.5-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9342fb74e2afefdb081bf1df727d209ea56995c6e13f5a0540e6d7aff4beafb8", size = 270139, upload-time = "2026-05-26T06:49:57.116Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/0b/fd0c54579e2ce8dc6cf1a757903f3374bc6fbda929a46af9e0f53cb0e5f0/zope_interface-8.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6c54725d818f1b57a7efb8b16528326e1f3c257b602b32393fd255c45af8799d", size = 270338, upload-time = "2026-05-26T06:49:58.698Z" },
+ { url = "https://files.pythonhosted.org/packages/c1/1d/c420dcd777bb761067ea92879ac766694a5ca78608185f1aecea64cbfc11/zope_interface-8.5-cp314-cp314-win_amd64.whl", hash = "sha256:29d74febbae1afeb6834c4ccbf42e242a673c860060f09e53142825270456140", size = 215789, upload-time = "2026-05-26T06:50:00.405Z" },
+ { url = "https://files.pythonhosted.org/packages/62/94/50b5eb8f94e527edceac14f9955e58917424ea79bb572ddc18548561cbc2/zope_interface-8.5-cp314-cp314-win_arm64.whl", hash = "sha256:633c8c49396f38df030340797c533e9fe460d1b5d1e42d88e55e938e525f548c", size = 213757, upload-time = "2026-05-26T06:50:01.973Z" },
+ { url = "https://files.pythonhosted.org/packages/17/6f/5d5f32c4dfcdb16ce2ec5363da686840f13c13e1a1214cb70b49e1cd6d9f/zope_interface-8.5-cp314-cp314t-macosx_10_9_x86_64.whl", hash = "sha256:133999820fdbae513c36c03d6f29ef87317aaa3edef39112222b155083664714", size = 213591, upload-time = "2026-05-26T06:50:03.529Z" },
+ { url = "https://files.pythonhosted.org/packages/f3/55/de0c3459ff717fce3342f9a29464c281fdeb0d36c3171ee88d119d5f0650/zope_interface-8.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8bd75c96966e573232f0599deaff717564828031c7f05563ccc1ac35c5ee0304", size = 213733, upload-time = "2026-05-26T06:50:05.101Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/95/d97430abd5ae9677e8b9295b58720c0064a5b557dbb6b8bf5928484cf0d8/zope_interface-8.5-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:14b0e9799351d4c34fe99afd67f0cdd76e55ba15c66a98699d5fc22ea8241e08", size = 294905, upload-time = "2026-05-26T06:50:07.384Z" },
+ { url = "https://files.pythonhosted.org/packages/41/ec/a0f8f3dad6e74992f4654bdd94802be0929eabca7b871cac3b6fbb5e961b/zope_interface-8.5-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cd6a732ac84b94eb1ef9222a117347a27efd294ee16810ffdf7ecd307677ed5", size = 300885, upload-time = "2026-05-26T06:50:08.997Z" },
+ { url = "https://files.pythonhosted.org/packages/0f/da/6881b48803a0ee8d23eb5efa30fce3ed218a2bd9de5758ce489d224fee81/zope_interface-8.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:798b7c87d0e59a7d5d086d642208d0d8700ff0d55c4029134b3c479c3bfb110f", size = 304672, upload-time = "2026-05-26T06:50:10.563Z" },
+ { url = "https://files.pythonhosted.org/packages/2e/0e/b4c01320859ff1d585438bc231fd60bd258d096359bccf6654fecdf0cffb/zope_interface-8.5-cp314-cp314t-win_amd64.whl", hash = "sha256:0fc3a9d45f114d27eaa1e53beeb144533689edca8a9f66505b1e8e8b3f075e42", size = 217241, upload-time = "2026-05-26T06:50:12.171Z" },
+]
+
[[package]]
name = "zstandard"
version = "0.25.0"