Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(feishu): unify folder and file imports with Understanding (#4806)

ShaoZegangBytef3afef11637f
diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md
index 398641a7a..ebaef342c 100644
--- a/docs/zh/api/02-resources.md
+++ b/docs/zh/api/02-resources.md
@@ -42,7 +42,7 @@ OpenViking 支持多种资源类型,按照功能分类如下:
云文档类
| 类型 | 说明 |
|------|------|
-| 飞书/Lark | URL 方式,支持 doc/docx, wiki, sheets, bitable。默认使用 FEISHU_APP_ID 和 FEISHU_APP_SECRET 应用凭证;用户 token 导入可传 `args.feishu_access_token`,用户 token watch 还需传 `args.feishu_refresh_token`,并可选传入 `args.feishu_app_id` / `args.feishu_app_secret` |
+| 飞书/Lark | URL 方式,支持 doc/docx, wiki, sheets, bitable、Drive 文件和目录集。Wiki 默认仅导入入口文档,设置 `args.feishu_recursive=true` 可递归导入子节点。默认使用 FEISHU_APP_ID 和 FEISHU_APP_SECRET 应用凭证;用户 token 导入可传 `args.feishu_access_token`,用户 token watch 还需传 `args.feishu_refresh_token`,并可选传入 `args.feishu_app_id` / `args.feishu_app_secret` |
网页类(递归网页爬虫)
| 类型 | 资源名 | 说明 |
diff --git a/openviking/parse/accessors/base.py b/openviking/parse/accessors/base.py
index dbf30f640..5a85706a6 100644
--- a/openviking/parse/accessors/base.py
+++ b/openviking/parse/accessors/base.py
@@ -11,7 +11,10 @@ import shutil
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from pathlib import Path
-from typing import Any, Dict, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+
+if TYPE_CHECKING:
+ from openviking.parse.feishu_import import FeishuImportPlan
class SourceType:
@@ -58,6 +61,8 @@ class LocalResource:
is_temporary: bool = True
"""Whether this is a temporary resource that can be cleaned up after parsing."""
+ feishu_plan: Optional["FeishuImportPlan"] = None
+
def cleanup(self) -> None:
"""
Clean up the local resource if it's temporary.
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.py
index 431492b45..e2484552d 100644
--- a/openviking/parse/accessors/feishu_accessor.py
+++ b/openviking/parse/accessors/feishu_accessor.py
@@ -19,11 +19,12 @@ import tempfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union
-from urllib.parse import parse_qs, unquote, urlparse
+from urllib.parse import parse_qs, unquote, urlparse, urlunparse
from openviking.parse.base import format_table_to_markdown
+from openviking.parse.feishu_import import FeishuImportPlan, recursive_wiki
from openviking.utils.exceptions import error_code_from_http_status
-from openviking_cli.exceptions import OpenVikingError
+from openviking_cli.exceptions import InvalidArgumentError, OpenVikingError
from openviking_cli.utils.logger import get_logger
from .base import DataAccessor, LocalResource, SourceType
@@ -130,11 +131,14 @@ def _safe_path_segment(
max_chars=max_len - len(suffix),
max_bytes=max_bytes - suffix_bytes,
).rstrip(" ._")
- return _fit_path_segment(
- f"{stem or 'untitled'}{suffix}",
- max_chars=max_len,
- max_bytes=max_bytes,
- ) or "untitled"
+ return (
+ _fit_path_segment(
+ f"{stem or 'untitled'}{suffix}",
+ max_chars=max_len,
+ max_bytes=max_bytes,
+ )
+ or "untitled"
+ )
def _numbered_path_segment(stem: str, suffix: str, index: int) -> str:
@@ -148,11 +152,14 @@ def _numbered_path_segment(stem: str, suffix: str, index: int) -> str:
max_chars=max_stem_chars,
max_bytes=max_stem_bytes,
).rstrip(" ._")
- return _fit_path_segment(
- f"{safe_stem or 'untitled'}{marker}{suffix}",
- max_chars=_MAX_PATH_SEGMENT_CHARS,
- max_bytes=_MAX_PATH_SEGMENT_BYTES,
- ) or "untitled"
+ return (
+ _fit_path_segment(
+ f"{safe_stem or 'untitled'}{marker}{suffix}",
+ max_chars=_MAX_PATH_SEGMENT_CHARS,
+ max_bytes=_MAX_PATH_SEGMENT_BYTES,
+ )
+ or "untitled"
+ )
def _getattr_safe(obj, key: str, default=None):
@@ -393,9 +400,15 @@ class FeishuAccessor(DataAccessor):
"""
source_str = str(source)
feishu_access_token = kwargs.get("feishu_access_token")
-
try:
doc_type, token = self._parse_feishu_url(source_str)
+ if (
+ doc_type in {"folder", "file"} or (doc_type == "wiki" and recursive_wiki(kwargs))
+ ) and kwargs.get("lark_file") is not None:
+ raise InvalidArgumentError(
+ "Feishu source preparation requires feishu_access_token or configured "
+ "Feishu application credentials, not lark_file."
+ )
if doc_type == "file":
content, content_type, filename = await asyncio.to_thread(
self._download_drive_file,
@@ -414,6 +427,7 @@ class FeishuAccessor(DataAccessor):
original_source=source_str,
meta={
"feishu_doc_type": doc_type,
+ "feishu_content_kind": "file",
"feishu_token": token,
"original_filename": local_path.name,
"_cleanup_path": str(local_path.parent),
@@ -423,6 +437,8 @@ class FeishuAccessor(DataAccessor):
if doc_type == "folder":
temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_folder_"))
+ plan = self._new_import_plan(temp_dir, kwargs)
+ plan.source_url = source_str
skipped_items: list[dict[str, Any]] = []
folder_name = await asyncio.to_thread(
self._drive_folder_display_name,
@@ -436,12 +452,14 @@ class FeishuAccessor(DataAccessor):
feishu_access_token=feishu_access_token,
skipped_items=skipped_items,
strict=bool(kwargs.get("strict", False)),
+ plan=plan,
)
except Exception:
shutil.rmtree(temp_dir, ignore_errors=True)
raise
return LocalResource(
path=temp_dir,
+ feishu_plan=plan,
source_type=SourceType.FEISHU,
original_source=source_str,
meta={
@@ -453,8 +471,10 @@ class FeishuAccessor(DataAccessor):
is_temporary=True,
)
- if doc_type == "wiki" and bool(kwargs.get("feishu_recursive", False)):
+ if doc_type == "wiki" and recursive_wiki(kwargs):
temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_wiki_"))
+ plan = self._new_import_plan(temp_dir, kwargs)
+ plan.source_url = source_str
skipped_items: list[dict[str, Any]] = []
try:
root = await asyncio.to_thread(
@@ -468,6 +488,7 @@ class FeishuAccessor(DataAccessor):
feishu_access_token=feishu_access_token,
skipped_items=skipped_items,
strict=bool(kwargs.get("strict", False)),
+ plan=plan,
visited=set(),
depth=0,
max_depth=self._positive_int_option(
@@ -483,12 +504,19 @@ class FeishuAccessor(DataAccessor):
except Exception:
shutil.rmtree(temp_dir, ignore_errors=True)
raise
+ if not root_dir.exists():
+ root_dir = temp_dir
+ plan.root = root_dir if root_dir.is_dir() else root_dir.parent
return LocalResource(
path=root_dir,
+ feishu_plan=plan,
source_type=SourceType.FEISHU,
original_source=source_str,
meta={
"feishu_doc_type": "wiki",
+ "feishu_content_kind": "file"
+ if root.obj_type == "file" and root_dir.is_file()
+ else "markdown",
"feishu_token": token,
"feishu_title": root.title,
"original_filename": _title_as_filename(root.title),
@@ -563,24 +591,35 @@ class FeishuAccessor(DataAccessor):
source: Union[str, Path],
*,
feishu_access_token: Optional[str] = None,
+ feishu_recursive: bool = False,
) -> FeishuSourcePreflight:
"""Resolve lightweight source identity and root permission before enqueueing."""
return await asyncio.to_thread(
self._preflight_source_sync,
str(source),
feishu_access_token,
+ feishu_recursive,
)
def _preflight_source_sync(
self,
url: str,
feishu_access_token: Optional[str] = None,
+ feishu_recursive: bool = False,
) -> FeishuSourcePreflight:
doc_type, token = self._parse_feishu_url(url)
query = parse_qs(urlparse(url).query)
table_id = (query.get("table") or [None])[0]
view_id = (query.get("view") or [None])[0]
+ if doc_type == "wiki" and recursive_wiki({"feishu_recursive": feishu_recursive}):
+ node = self._resolve_wiki_tree_root(url, feishu_access_token=feishu_access_token)
+ return FeishuSourcePreflight(
+ doc_type="wiki",
+ token=token,
+ source_name=_title_as_filename(node.title),
+ source_format="directory",
+ )
if doc_type == "wiki":
real_type, real_token, title = self._resolve_wiki_node(
token,
@@ -814,6 +853,17 @@ class FeishuAccessor(DataAccessor):
token = path_parts[1]
return doc_type, token
+ def _new_import_plan(self, root: Path, options: Dict[str, Any]) -> FeishuImportPlan:
+ return FeishuImportPlan(
+ root=root,
+ use_understanding=bool(options.get("_feishu_use_understanding", False)),
+ max_nodes=self._positive_int_option(options.get("feishu_max_nodes"), default=5000),
+ max_depth=self._positive_int_option(options.get("feishu_max_depth"), default=20),
+ max_bytes=self._positive_int_option(
+ options.get("feishu_max_download_bytes"), default=1024 * 1024 * 1024
+ ),
+ )
+
async def _materialize_drive_folder(
self,
folder_token: str,
@@ -823,137 +873,148 @@ class FeishuAccessor(DataAccessor):
_seen: Optional[set[str]] = None,
skipped_items: Optional[list[dict[str, Any]]] = None,
strict: bool = False,
+ plan: Optional[FeishuImportPlan] = None,
+ depth: int = 0,
) -> None:
- """Expand a Feishu Drive folder into a local directory tree."""
- target_dir.mkdir(parents=True, exist_ok=True)
- seen = _seen or set()
+ plan = plan if plan is not None else FeishuImportPlan(target_dir)
+ seen = _seen if _seen is not None else set()
if folder_token in seen:
- logger.warning("[FeishuAccessor] Skipping recursive Drive folder %s", folder_token)
- return
+ raise ValueError("Cyclic Feishu Drive folder reference")
+ plan.visit(depth)
+ target_dir.mkdir(parents=True, exist_ok=True)
seen.add(folder_token)
-
try:
children = await asyncio.to_thread(
self._list_drive_folder_children,
folder_token,
+ max_items=max(0, plan.max_nodes - plan.nodes),
feishu_access_token=feishu_access_token,
)
- for item in children:
- item_type, item_token, item_name, item_url = self._normalize_drive_item(item)
- if not item_token:
- logger.warning("[FeishuAccessor] Skipping Drive item without token: %s", item)
- continue
-
- if item_type == "folder":
- folder_name = _safe_path_segment(item_name or item_token, fallback=item_token)
- child_dir = self._unique_child_path(target_dir, folder_name)
- try:
+ # Stable order makes collision allocation independent of API page ordering.
+ for item in sorted(children, key=lambda item: self._normalize_drive_item(item)[1]):
+ item_type, token, name, url = self._normalize_drive_item(item)
+ try:
+ if not token:
+ raise ValueError("Drive item has no token")
+ if item_type == "folder":
+ child_dir = self._unique_child_path(
+ target_dir, name or token, plan.reserved
+ )
await self._materialize_drive_folder(
- item_token,
+ token,
child_dir,
feishu_access_token=feishu_access_token,
_seen=seen,
skipped_items=skipped_items,
strict=strict,
+ plan=plan,
+ depth=depth + 1,
)
- except Exception as exc:
- shutil.rmtree(child_dir, ignore_errors=True)
- self._record_skipped_drive_item(
- skipped_items,
- item_type=item_type,
- token=item_token,
- name=item_name,
- target_dir=target_dir,
- error=exc,
- )
- if strict:
- raise
- continue
-
- doc_path_type = _FEISHU_DRIVE_DOC_TYPES.get(item_type)
- if doc_path_type:
- try:
- url = item_url or self._build_feishu_doc_url(doc_path_type, item_token)
- doc = await self._fetch_document(
+ else:
+ plan.visit(depth + 1)
+ path_type = _FEISHU_DRIVE_DOC_TYPES.get(item_type)
+ if item_type != "file" and not path_type:
+ raise ValueError(f"Unsupported Feishu Drive item type: {item_type}")
+ await self._write_import_content(
+ item_type if item_type == "file" else path_type,
+ token,
+ name,
url,
- feishu_access_token=feishu_access_token,
- )
- markdown_content, downloaded_images = await asyncio.to_thread(
- self._resolve_image_refs,
- doc.markdown_content,
- feishu_access_token=feishu_access_token,
- media_download_extras=doc.media_download_extras,
- )
- doc_name = _safe_path_segment(
- item_name or doc.title or item_token,
- fallback=item_token,
- )
- markdown_path = self._unique_child_path(
target_dir,
- self._markdown_file_name(doc_name),
- )
- markdown_path.write_text(markdown_content, encoding="utf-8")
- for rel_path, image_bytes in downloaded_images.items():
- image_path = markdown_path.parent / rel_path
- image_path.parent.mkdir(parents=True, exist_ok=True)
- image_path.write_bytes(image_bytes)
- except Exception as exc:
- self._record_skipped_drive_item(
- skipped_items,
- item_type=item_type,
- token=item_token,
- name=item_name,
- target_dir=target_dir,
- error=exc,
- )
- if strict:
- raise
- continue
-
- if item_type == "file":
- try:
- content, content_type, downloaded_name = await asyncio.to_thread(
- self._download_drive_file,
- item_token,
+ plan,
feishu_access_token=feishu_access_token,
- filename_hint=item_name,
- )
- file_name = self._drive_file_name(
- item_token,
- content,
- content_type,
- filename_hint=downloaded_name or item_name,
)
- file_path = self._unique_child_path(target_dir, file_name)
- file_path.write_bytes(content)
- except Exception as exc:
- self._record_skipped_drive_item(
- skipped_items,
- item_type=item_type,
- token=item_token,
- name=item_name,
- target_dir=target_dir,
- error=exc,
- )
- if strict:
- raise
- continue
-
- error = ValueError(f"Unsupported Feishu Drive item type: {item_type}")
- self._record_skipped_drive_item(
- skipped_items,
- item_type=item_type,
- token=item_token,
- name=item_name,
- target_dir=target_dir,
- error=error,
- )
- if strict:
- raise error
-
+ except Exception as exc:
+ self._record_skipped_drive_item(
+ skipped_items,
+ item_type=item_type,
+ token=token,
+ name=name,
+ target_dir=target_dir,
+ error=exc,
+ )
+ if strict:
+ raise
+ if plan.nodes >= plan.max_nodes:
+ break
finally:
seen.discard(folder_token)
+ @classmethod
+ def _import_doc_url(cls, plan: FeishuImportPlan, doc_type: str, token: str) -> str:
+ if not plan.source_url:
+ return cls._build_feishu_doc_url(doc_type, token)
+ source = urlparse(plan.source_url)
+ path_type = "docs" if doc_type == "doc" else doc_type
+ query = source.query if source.path.rstrip("/").endswith(f"/{token}") else ""
+ return urlunparse((source.scheme, source.netloc, f"/{path_type}/{token}", "", query, ""))
+
+ async def _write_import_content(
+ self,
+ doc_type: str,
+ token: str,
+ title: str,
+ url: str,
+ target_dir: Path,
+ plan: FeishuImportPlan,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> Path:
+ doc_type = "doc" if doc_type == "docs" else self._WIKI_TYPE_MAP.get(doc_type, doc_type)
+ url = url or self._import_doc_url(plan, doc_type, token)
+ if doc_type == "wiki":
+ doc_type, token, _ = await asyncio.to_thread(
+ self._resolve_wiki_node, token, feishu_access_token
+ )
+ if plan.download_exhausted:
+ raise ValueError("Feishu import download byte limit exceeded")
+ if doc_type == "file":
+ content, content_type, filename = await asyncio.to_thread(
+ self._download_drive_file,
+ token,
+ feishu_access_token=feishu_access_token,
+ filename_hint=title,
+ )
+ plan.account_download(len(content))
+ path = self._unique_child_path(
+ target_dir,
+ self._drive_file_name(
+ token, content, content_type, filename_hint=filename or title
+ ),
+ plan.reserved,
+ )
+ path.write_bytes(content)
+ plan.add(path, url, token, "file")
+ return path
+ if doc_type not in self._DOC_TYPE_HANDLERS:
+ raise ValueError(f"Unsupported Feishu document type: {doc_type}")
+ name = self._markdown_file_name(_safe_path_segment(title or token, fallback=token))
+ path = self._unique_child_path(target_dir, name, plan.reserved)
+ if plan.use_understanding and doc_type in {"docx", "sheets", "base"}:
+ plan.add(path, url, token, "url")
+ return path
+ content_url = (
+ self._import_doc_url(plan, doc_type, token)
+ if self._parse_feishu_url(url)[0] == "wiki"
+ else url
+ )
+ content_url = urlparse(content_url)._replace(query=urlparse(url).query).geturl()
+ doc = await self._fetch_document(content_url, feishu_access_token=feishu_access_token)
+ markdown, images = await asyncio.to_thread(
+ self._resolve_image_refs,
+ doc.markdown_content,
+ feishu_access_token=feishu_access_token,
+ media_download_extras=doc.media_download_extras,
+ )
+ plan.account_download(len(markdown.encode("utf-8")) + sum(map(len, images.values())))
+ path.write_text(markdown, encoding="utf-8")
+ for rel_path, content in images.items():
+ image_path = target_dir / rel_path
+ image_path.parent.mkdir(parents=True, exist_ok=True)
+ image_path.write_bytes(content)
+ plan.add(path, url, token, "markdown")
+ return path
+
async def _materialize_wiki_tree_node(
self,
node: _FeishuWikiTreeNode,
@@ -967,15 +1028,23 @@ class FeishuAccessor(DataAccessor):
max_depth: int = 20,
max_nodes_state: Optional[list[int]] = None,
max_nodes: int = 5000,
+ plan: Optional[FeishuImportPlan] = None,
) -> Path:
"""Expand a Feishu Wiki node tree into a local directory tree."""
+ plan = (
+ plan
+ if plan is not None
+ else FeishuImportPlan(target_dir, max_nodes=max_nodes, max_depth=max_depth)
+ )
target_dir.mkdir(parents=True, exist_ok=True)
seen = visited if visited is not None else set()
if node.wiki_node_token in seen:
- logger.warning(
- "[FeishuAccessor] Skipping recursive Wiki node %s",
- node.wiki_node_token,
+ error = ValueError("Cyclic Feishu Wiki reference")
+ self._record_skipped_wiki_node(
+ skipped_items, node=node, target_dir=target_dir, error=error
)
+ if strict:
+ raise error
return target_dir
seen.add(node.wiki_node_token)
try:
@@ -995,30 +1064,42 @@ class FeishuAccessor(DataAccessor):
children: List[_FeishuWikiTreeNode] = []
children_known = False
- if depth < max_depth:
- try:
- children = await asyncio.to_thread(
- self._list_wiki_node_children,
- node.space_id,
- node.wiki_node_token,
- feishu_access_token=feishu_access_token,
- )
- children_known = True
- except Exception as exc:
- self._record_skipped_wiki_node(
- skipped_items,
- node=node,
- target_dir=target_dir,
- error=exc,
- )
- if strict:
- raise
+ try:
+ children = await asyncio.to_thread(
+ self._list_wiki_node_children,
+ node.space_id,
+ node.wiki_node_token,
+ # At the boundary, probe for a child without enumerating its subtree.
+ max_items=0 if depth >= max_depth else max(0, max_nodes - state[0]),
+ feishu_access_token=feishu_access_token,
+ )
+ children_known = True
+ except Exception as exc:
+ self._record_skipped_wiki_node(
+ skipped_items,
+ node=node,
+ target_dir=target_dir,
+ error=exc,
+ )
+ if strict or depth == 0:
+ raise
+
+ needs_directory = bool(children) or not children_known
+ if children and depth >= max_depth:
+ error = ValueError(f"Feishu Wiki depth limit reached: {max_depth}")
+ self._record_skipped_wiki_node(
+ skipped_items, node=node, target_dir=target_dir, error=error
+ )
+ if strict:
+ raise error
+ children = []
content_dir = target_dir
- if children or not children_known:
+ if needs_directory:
content_dir = self._unique_child_path(
target_dir,
self._wiki_node_dir_name(node),
+ plan.reserved,
)
content_dir.mkdir(parents=True, exist_ok=True)
@@ -1026,6 +1107,7 @@ class FeishuAccessor(DataAccessor):
content_path = await self._write_wiki_node_content(
node,
content_dir,
+ plan=plan,
feishu_access_token=feishu_access_token,
)
except Exception as exc:
@@ -1039,10 +1121,11 @@ class FeishuAccessor(DataAccessor):
raise
content_path = None
- for child in children:
+ for child in sorted(children, key=lambda child: child.wiki_node_token):
await self._materialize_wiki_tree_node(
child,
content_dir,
+ plan=plan,
feishu_access_token=feishu_access_token,
skipped_items=skipped_items,
strict=strict,
@@ -1052,7 +1135,9 @@ class FeishuAccessor(DataAccessor):
max_nodes_state=state,
max_nodes=max_nodes,
)
- return content_dir if children or not children_known else content_path or target_dir
+ if state[0] > max_nodes:
+ break
+ return content_dir if needs_directory else content_path or target_dir
finally:
seen.discard(node.wiki_node_token)
@@ -1062,53 +1147,20 @@ class FeishuAccessor(DataAccessor):
target_dir: Path,
*,
feishu_access_token: Optional[str] = None,
+ plan: Optional[FeishuImportPlan] = None,
) -> Optional[Path]:
- doc_type = self._normalize_wiki_obj_type(node.obj_type)
- if not doc_type or not node.obj_token:
+ if not node.obj_type or not node.obj_token:
return None
-
- if doc_type == "file":
- content, content_type, downloaded_name = await asyncio.to_thread(
- self._download_drive_file,
- node.obj_token,
- feishu_access_token=feishu_access_token,
- filename_hint=node.title,
- )
- file_path = self._unique_child_path(
- target_dir,
- self._drive_file_name(
- node.obj_token,
- content,
- content_type,
- filename_hint=downloaded_name or node.title,
- ),
- )
- file_path.write_bytes(content)
- return file_path
-
- if doc_type not in self._DOC_TYPE_HANDLERS:
- raise ValueError(f"Unsupported Feishu Wiki node object type: {node.obj_type}")
-
- doc = await self._fetch_document(
- self._build_feishu_doc_url(doc_type, node.obj_token),
- feishu_access_token=feishu_access_token,
- )
- markdown_content, downloaded_images = await asyncio.to_thread(
- self._resolve_image_refs,
- doc.markdown_content,
- feishu_access_token=feishu_access_token,
- media_download_extras=doc.media_download_extras,
- )
- markdown_path = self._unique_child_path(
+ plan = plan if plan is not None else FeishuImportPlan(target_dir)
+ return await self._write_import_content(
+ self._normalize_wiki_obj_type(node.obj_type),
+ node.obj_token,
+ node.title,
+ self._import_doc_url(plan, "wiki", node.wiki_node_token),
target_dir,
- self._markdown_file_name(node.title),
+ plan,
+ feishu_access_token=feishu_access_token,
)
- markdown_path.write_text(markdown_content, encoding="utf-8")
- for rel_path, image_bytes in downloaded_images.items():
- image_path = markdown_path.parent / rel_path
- image_path.parent.mkdir(parents=True, exist_ok=True)
- image_path.write_bytes(image_bytes)
- return markdown_path
@staticmethod
def _record_skipped_wiki_node(
@@ -1232,10 +1284,12 @@ class FeishuAccessor(DataAccessor):
folder_token: str,
*,
feishu_access_token: Optional[str] = None,
+ max_items: Optional[int] = None,
) -> List[Any]:
"""List direct children under a Feishu Drive folder token."""
all_children: List[Any] = []
page_token = None
+ pages = set()
while True:
items, has_more, page_token = self._fetch_drive_folder_children_page(
folder_token,
@@ -1244,8 +1298,13 @@ class FeishuAccessor(DataAccessor):
)
all_children.extend(items)
+ if max_items is not None and len(all_children) > max_items:
+ return all_children[: max_items + 1]
if not has_more:
break
+ if page_token in pages:
+ raise RuntimeError("Feishu returned a repeated page token")
+ pages.add(page_token)
if not page_token:
raise RuntimeError(
f"Feishu returned more Drive folder items for {folder_token} "
@@ -1418,15 +1477,22 @@ class FeishuAccessor(DataAccessor):
return f"https://open.feishu.cn/{doc_type}/{token}"
@staticmethod
- def _unique_child_path(parent: Path, name: str) -> Path:
+ def _unique_child_path(parent: Path, name: str, reserved: Optional[set[Path]] = None) -> Path:
path = parent / _safe_path_segment(name)
- if not path.exists():
+ reserved = reserved if reserved is not None else set()
+ if not path.exists() and path not in reserved and path.with_suffix("") not in reserved:
+ reserved.update({path, path.with_suffix("")})
return path
suffix = path.suffix
stem = path.stem
for index in range(2, 10000):
candidate = parent / _numbered_path_segment(stem, suffix, index)
- if not candidate.exists():
+ if (
+ not candidate.exists()
+ and candidate not in reserved
+ and candidate.with_suffix("") not in reserved
+ ):
+ reserved.update({candidate, candidate.with_suffix("")})
return candidate
raise RuntimeError(f"Unable to allocate unique path under {parent}")
@@ -1549,6 +1615,13 @@ class FeishuAccessor(DataAccessor):
@classmethod
def _raw_feishu_error(cls, raw_resp: Any) -> Optional[Tuple[int, str]]:
+ disposition = cls._response_header(raw_resp, "content-disposition") or ""
+ if 200 <= raw_resp.status_code < 300 and (
+ disposition.split(";", 1)[0].strip().lower() == "attachment"
+ or cls._filename_from_content_disposition(disposition)
+ ):
+ # Downloaded JSON can legitimately contain code/msg fields.
+ return None
content_type = cls._response_content_type(raw_resp)
if not content_type or "application/json" not in content_type.lower():
return None
@@ -1710,9 +1783,11 @@ class FeishuAccessor(DataAccessor):
wiki_node_token: str,
*,
feishu_access_token: Optional[str] = None,
+ max_items: Optional[int] = None,
) -> List[_FeishuWikiTreeNode]:
children: List[_FeishuWikiTreeNode] = []
page_token = None
+ pages = set()
while True:
items, has_more, page_token = self._fetch_wiki_node_children_page(
space_id,
@@ -1724,16 +1799,20 @@ class FeishuAccessor(DataAccessor):
children.append(
self._wiki_tree_node_from_api_node(
item,
- fallback_token=wiki_node_token,
+ fallback_token="",
fallback_space_id=space_id,
)
)
+ if max_items is not None and len(children) > max_items:
+ return children[: max_items + 1]
if not has_more:
break
+ if page_token in pages:
+ raise RuntimeError("Feishu returned a repeated page token")
+ pages.add(page_token)
if not page_token:
raise RuntimeError(
- f"Feishu returned more Wiki children for {wiki_node_token} "
- "without a page token"
+ f"Feishu returned more Wiki children for {wiki_node_token} without a page token"
)
return children
@@ -2814,10 +2893,7 @@ class FeishuAccessor(DataAccessor):
)
record_builder = (
- ListAppTableRecordRequest.builder()
- .app_token(app_token)
- .table_id(table_id)
- .page_size(1)
+ ListAppTableRecordRequest.builder().app_token(app_token).table_id(table_id).page_size(1)
)
if view_id:
record_builder = record_builder.view_id(view_id)
diff --git a/openviking/parse/feishu_import.py b/openviking/parse/feishu_import.py
new file mode 100644
index 000000000..eee958e0f
--- /dev/null
+++ b/openviking/parse/feishu_import.py
@@ -0,0 +1,59 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Runtime-only Feishu content references and directory download budget."""
+
+import hashlib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from openviking_cli.exceptions import InvalidArgumentError
+
+
+def recursive_wiki(options: dict[str, Any]) -> bool:
+ value = options.get("feishu_recursive", False)
+ if not isinstance(value, bool):
+ raise InvalidArgumentError("feishu_recursive must be a boolean")
+ return value
+
+
+@dataclass(frozen=True)
+class FeishuContent:
+ path: Path
+ url: str
+ token: str
+ kind: str # url, markdown, file
+
+ def checkpoint_key(self, root: Path) -> str:
+ relative = self.path.resolve().relative_to(root.resolve()).as_posix()
+ identity = f"{relative}\n{self.url}\n{self.token}"
+ return hashlib.sha256(identity.encode("utf-8")).hexdigest()
+
+
+@dataclass
+class FeishuImportPlan:
+ root: Path
+ use_understanding: bool = False
+ source_url: str = ""
+ entries: list[FeishuContent] = field(default_factory=list)
+ reserved: set[Path] = field(default_factory=set)
+ max_nodes: int = 5000
+ max_depth: int = 20
+ max_bytes: int = 1024 * 1024 * 1024
+ nodes: int = 0
+ downloaded_bytes: int = 0
+ download_exhausted: bool = False
+
+ def visit(self, depth: int) -> None:
+ if depth > self.max_depth or self.nodes >= self.max_nodes:
+ raise ValueError("Feishu import node/depth limit exceeded")
+ self.nodes += 1
+
+ def account_download(self, size: int) -> None:
+ if self.downloaded_bytes + size > self.max_bytes:
+ self.download_exhausted = True
+ raise ValueError("Feishu import download byte limit exceeded")
+ self.downloaded_bytes += size
+
+ def add(self, path: Path, url: str, token: str, kind: str) -> None:
+ self.entries.append(FeishuContent(path, url, token, kind))
diff --git a/openviking/parse/parser_router.py b/openviking/parse/parser_router.py
index 21a8d50fe..d9b889084 100644
--- a/openviking/parse/parser_router.py
+++ b/openviking/parse/parser_router.py
@@ -53,7 +53,11 @@ class ParserRouter:
Decide whether to use UnderstandingAPI.
"""
# FeishuAccessor has already normalized proprietary content to Markdown.
- if isinstance(source, LocalResource) and source.source_type == SourceType.FEISHU:
+ if (
+ isinstance(source, LocalResource)
+ and source.source_type == SourceType.FEISHU
+ and source.meta.get("feishu_content_kind") != "file"
+ ):
return False
try:
@@ -110,7 +114,9 @@ class ParserRouter:
parser_backend = normalize_parser_backend(kwargs.pop("parser_backend", None))
normalized_feishu = (
- isinstance(source, LocalResource) and source.source_type == SourceType.FEISHU
+ isinstance(source, LocalResource)
+ and source.source_type == SourceType.FEISHU
+ and source.meta.get("feishu_content_kind") != "file"
)
use_understanding = not normalized_feishu and (
parser_backend is ParserBackend.UNDERSTANDING
diff --git a/openviking/parse/parsers/directory.py b/openviking/parse/parsers/directory.py
index 2c7f5d4e2..8602d1eea 100644
--- a/openviking/parse/parsers/directory.py
+++ b/openviking/parse/parsers/directory.py
@@ -23,12 +23,14 @@ from pathlib import Path, PurePosixPath
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
from weakref import WeakKeyDictionary
+from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.base import (
NodeType,
ParseResult,
ResourceNode,
create_parse_result,
)
+from openviking.parse.gitignore import GitignoreMatcher
from openviking.parse.image_rewrite import IMAGE_MAPPINGS_FILENAME
from openviking.parse.parsers.base_parser import BaseParser
from openviking.parse.parsers.media.constants import MEDIA_EXTENSIONS
@@ -142,6 +144,9 @@ class DirectoryParser(BaseParser):
dir_name = kwargs.get("source_name") or source_path.name
warnings: List[str] = []
+ temp_uri: Optional[str] = None
+ keep_temp = False
+ pending_temp_uris: set[str] = set()
try:
# ── Phase 1: scan directory ───────────────────────────────
@@ -157,11 +162,22 @@ class DirectoryParser(BaseParser):
directory_config = getattr(ov_config, "directory", None) or DirectoryConfig()
split_content = kwargs.get("split_content", True)
+ plan = kwargs.get("_feishu_import_plan")
+ # Ordinary source jobs use INTERNAL for directory orchestration;
+ # their files still select a parser independently by extension.
+ backend = normalize_parser_backend(kwargs.get("parser_backend")) if plan else None
parser_api_config = getattr(ov_config, "parser_api", None)
understanding_limits_enabled = bool(
split_content
- and parser_router.understanding_api_enabled()
- and getattr(parser_api_config, "extensions", None)
+ and backend is not ParserBackend.INTERNAL
+ and (
+ (plan and plan.use_understanding)
+ or backend is ParserBackend.UNDERSTANDING
+ or (
+ parser_router.understanding_api_enabled()
+ and getattr(parser_api_config, "extensions", None)
+ )
+ )
)
scan_result = scan_directory(
@@ -172,14 +188,37 @@ class DirectoryParser(BaseParser):
include=kwargs.get("include"),
exclude=kwargs.get("exclude"),
additional_can_process=parser_router.should_use_understanding_api,
- max_files=directory_config.max_files if understanding_limits_enabled else None,
+ max_files=directory_config.max_files
+ if plan or understanding_limits_enabled
+ else None,
max_depth=directory_config.max_depth if understanding_limits_enabled else None,
)
directly_upload_media = kwargs.get("directly_upload_media", True)
preserve_structure = kwargs.get("preserve_structure")
if preserve_structure is None:
preserve_structure = directory_config.preserve_structure
- processable_files = scan_result.all_processable_files()
+ processable_files = list(scan_result.all_processable_files())
+ entry_map = {entry.path.resolve(): entry for entry in plan.entries} if plan else {}
+ feishu_gitignore = GitignoreMatcher(source_path) if plan else None
+ if plan:
+ from openviking.parse.directory_scan import CLASS_PROCESSABLE, ClassifiedFile
+
+ for entry in plan.entries:
+ if entry.kind != "url":
+ continue
+ entry_path = entry.path.resolve()
+ relative = entry_path.relative_to(source_path).as_posix()
+ if not self._include_feishu_path(
+ entry_path, source_path, kwargs, gitignore=feishu_gitignore
+ ):
+ scan_result.skipped.append(f"{relative} (excluded by source filter)")
+ continue
+ processable_files.append(
+ ClassifiedFile(entry_path, relative, CLASS_PROCESSABLE)
+ )
+ if len(processable_files) > directory_config.max_files:
+ raise InvalidArgumentError("Feishu directory file limit exceeded")
+ processable_files.sort(key=lambda item: item.rel_path)
warnings.extend(scan_result.warnings)
source_skipped_items = self._source_skipped_items(
kwargs.get("_source_meta"),
@@ -193,15 +232,23 @@ class DirectoryParser(BaseParser):
file_jobs: List[Dict[str, Any]] = []
understanding_jobs: List[Dict[str, Any]] = []
for index, cf in enumerate(processable_files):
- configured_for_understanding = parser_router.should_use_understanding_api(cf.path)
+ entry = entry_map.get(cf.path)
+ remote = entry is not None and entry.kind == "url"
+ normalized = entry is not None and entry.kind == "markdown"
+ configured_for_understanding = bool(
+ backend is not ParserBackend.INTERNAL
+ and not normalized
+ and (
+ remote
+ or backend is ParserBackend.UNDERSTANDING
+ or parser_router.should_use_understanding_api(cf.path)
+ )
+ )
use_understanding = bool(split_content and configured_for_understanding)
native_parser = None if use_understanding else self._assign_parser(cf, registry)
file_parser = parser_router if use_understanding else native_parser
native_parser_unavailable = bool(
- not split_content
- and configured_for_understanding
- and native_parser is None
- and not is_text_file(cf.path)
+ not split_content and native_parser is None and not is_text_file(cf.path)
)
parser_name = (
"UnderstandingAPI"
@@ -233,6 +280,28 @@ class DirectoryParser(BaseParser):
and is_media_file
),
}
+ if use_understanding:
+ parse_options = {"parser_backend": ParserBackend.UNDERSTANDING}
+ if remote:
+ parse_options.update(
+ _source=entry.url,
+ resource_name=cf.path.name,
+ feishu_access_token=kwargs.get("feishu_access_token"),
+ )
+ if kwargs.get("lark_file"):
+ parse_options["lark_file"] = kwargs["lark_file"]
+ checkpoint = kwargs.get("_feishu_checkpoint")
+ if plan and checkpoint is not None:
+ saved, save = checkpoint
+ key = entry.checkpoint_key(plan.root) if entry else cf.rel_path
+ if key in saved:
+ parse_options["understanding_response_id"] = saved[key]
+
+ async def record(response_id, key=key, save=save):
+ await save(key, response_id)
+
+ parse_options["_response_checkpoint"] = record
+ job["parse_options"] = parse_options
file_jobs.append(job)
if use_understanding:
understanding_jobs.append(job)
@@ -265,6 +334,7 @@ class DirectoryParser(BaseParser):
result.meta["failed_files"] = source_skipped_items
result.meta["unsupported_files"] = []
result.meta["skipped_files"] = self._parse_skipped(scan_result.skipped)
+ keep_temp = True
return result
# ── Phase 2: process each file ────────────────────────────
@@ -291,6 +361,11 @@ class DirectoryParser(BaseParser):
max_concurrent=directory_config.max_concurrent,
job_timeout=job_timeout,
)
+ pending_temp_uris.update(
+ parsed["result"].temp_dir_path
+ for parsed in understanding_results.values()
+ if parsed.get("result") and parsed["result"].temp_dir_path
+ )
for job in file_jobs:
cf = job["classified_file"]
@@ -312,6 +387,8 @@ class DirectoryParser(BaseParser):
}
else:
try:
+ # The merge helper owns this artifact once merging starts.
+ pending_temp_uris.discard(sub_result.temp_dir_path)
await self._merge_parser_result(
cf,
sub_result,
@@ -361,6 +438,14 @@ class DirectoryParser(BaseParser):
)
file_entry = self._file_status_entry(cf, parser_name, detail)
+ entry = entry_map.get(cf.path)
+ if entry:
+ file_entry["source_url"] = entry.url
+ file_entry["source_token"] = entry.token
+ if plan and not detail["ok"] and kwargs.get("strict", False):
+ raise InvalidArgumentError(
+ f"Failed to import {cf.rel_path}: {detail.get('error')}"
+ )
if detail["ok"]:
file_count += 1
processed_files.append(file_entry)
@@ -408,6 +493,7 @@ class DirectoryParser(BaseParser):
result.meta["unsupported_files"] = unsupported_files
result.meta["skipped_files"] = skipped_files
+ keep_temp = True
return result
except InvalidArgumentError:
@@ -425,6 +511,56 @@ class DirectoryParser(BaseParser):
parse_time=time.time() - start_time,
warnings=[f"Failed to parse directory: {exc}"],
)
+ finally:
+ if temp_uri and not keep_temp:
+ pending_temp_uris.add(temp_uri)
+ for pending_uri in pending_temp_uris:
+ try:
+ await viking_fs.delete_temp(pending_uri)
+ except Exception as exc:
+ logger.warning(
+ "[DirectoryParser] Failed to clean temporary artifact %s: %s",
+ pending_uri,
+ exc,
+ )
+
+ @staticmethod
+ def _include_feishu_path(
+ path: Path,
+ root: Path,
+ options: Dict[str, Any],
+ *,
+ gitignore: Optional[GitignoreMatcher] = None,
+ ) -> bool:
+ from openviking.parse.directory_scan import (
+ _matches_exclude,
+ _matches_include,
+ _parse_patterns,
+ _should_skip_directory,
+ )
+
+ relative = path.relative_to(root).as_posix()
+ ignored = options.get("ignore_dirs") or set()
+ if isinstance(ignored, str):
+ ignored = set(_parse_patterns(ignored))
+ for parent in path.parents:
+ if parent == root:
+ break
+ if _should_skip_directory(parent, root, ignored)[0]:
+ return False
+ if gitignore and gitignore.is_ignored_dir(
+ parent, gitignore.spec_for_dir(parent.parent)
+ ):
+ return False
+ if path.name.startswith("."):
+ return False
+ if gitignore and gitignore.is_ignored_file(path, gitignore.spec_for_dir(path.parent)):
+ return False
+ includes = _parse_patterns(options.get("include"))
+ excludes = _parse_patterns(options.get("exclude"))
+ return (not includes or _matches_include(path.name, includes)) and not _matches_exclude(
+ relative, path.name, excludes
+ )
# ------------------------------------------------------------------
# parse_content – not applicable for directories
@@ -630,6 +766,7 @@ class DirectoryParser(BaseParser):
preserve_structure=preserve_structure,
import_root=import_root,
split_content=split_content,
+ parse_options=job.get("parse_options"),
)
sub_result = await asyncio.wait_for(parse_coro, timeout=job_timeout)
results[job["index"]] = {"result": sub_result, "error": None}
@@ -665,16 +802,19 @@ class DirectoryParser(BaseParser):
preserve_structure: bool,
import_root: Optional[str],
split_content: bool,
+ parse_options: Optional[Dict[str, Any]] = None,
) -> ParseResult:
"""Run one parser without mutating the directory destination tree."""
- return await parser.parse(
- str(classified_file.path),
+ options = dict(parse_options or {})
+ source = options.pop("_source", str(classified_file.path))
+ options.update(
enable_link_rewrite=preserve_structure,
link_rewrite_root=import_root,
allowed_media_dirs=[Path(import_root)] if import_root else None,
split_content=split_content,
flatten_single_output=bool(not split_content and preserve_structure),
)
+ return await parser.parse(source, **options)
@staticmethod
async def _merge_parser_result(
@@ -698,12 +838,23 @@ class DirectoryParser(BaseParser):
dest = f"{target_uri}/{parent}" if parent != "." else target_uri
else:
dest = target_uri
- merged = await DirectoryParser._merge_temp(
- viking_fs,
- sub_result.temp_dir_path,
- dest,
- flatten_single_output=bool(not split_content and preserve_structure),
- )
+ try:
+ merged = await DirectoryParser._merge_temp(
+ viking_fs,
+ sub_result.temp_dir_path,
+ dest,
+ flatten_single_output=bool(not split_content and preserve_structure),
+ )
+ except BaseException:
+ try:
+ await viking_fs.delete_temp(sub_result.temp_dir_path)
+ except Exception as exc:
+ logger.warning(
+ "[DirectoryParser] Failed to clean temporary artifact %s: %s",
+ sub_result.temp_dir_path,
+ exc,
+ )
+ raise
if not merged:
raise ValueError(no_content_error)
diff --git a/openviking/parse/understanding_api.py b/openviking/parse/understanding_api.py
index 26c11cab4..06ebc9e7e 100644
--- a/openviking/parse/understanding_api.py
+++ b/openviking/parse/understanding_api.py
@@ -206,6 +206,9 @@ class UnderstandingAPI(BaseParser):
)
response_id = str(response_id_value)
task_meta["response_id"] = response_id
+ checkpoint = kwargs.get("_response_checkpoint")
+ if checkpoint is not None:
+ await checkpoint(response_id)
response_obj = await self._poll_response(response_id=response_id)
zip_url = self._extract_zip_url(response_obj)
@@ -332,6 +335,12 @@ class UnderstandingAPI(BaseParser):
"""Return whether this URL can bypass source materialization."""
if not source.startswith(("http://", "https://")) or not self._is_feishu_url(source):
return False
+ from openviking.parse.accessors.feishu_accessor import FeishuAccessor
+ from openviking.parse.feishu_import import recursive_wiki
+
+ doc_type, _ = FeishuAccessor._parse_feishu_url(source)
+ if doc_type in {"folder", "file"} or (doc_type == "wiki" and recursive_wiki(kwargs)):
+ return False
if self._normalize_lark_file(kwargs):
return True
try:
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index 0a942599b..7cca717e2 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -282,6 +282,7 @@ class ResourceService:
for key, value in processor_kwargs.items():
if key in {
"auth_config",
+ "lark_file",
FEISHU_ACCESS_TOKEN_ARG,
FEISHU_REFRESH_TOKEN_ARG,
"parser_backend",
@@ -709,10 +710,13 @@ class ResourceService:
parent_uri = None
queued_args = dict(msg.args)
legacy_backend = normalize_parser_backend(queued_args.pop("parser_backend", None))
+ feishu_prepared = queued_args.pop("_feishu_prepared_source", False)
parser_backend = legacy_backend or (
ParserBackend.UNDERSTANDING
if msg.understanding_response_id is not None
or msg.understanding_file_id is not None
+ else None
+ if feishu_prepared
else ParserBackend.INTERNAL
)
internal_kwargs: Dict[str, Any] = {"parser_backend": parser_backend}
@@ -744,6 +748,20 @@ class ResourceService:
task_auth or {},
)
internal_kwargs.update(auth_kwargs)
+ if feishu_prepared:
+ from openviking.service.task_tracker import get_task_tracker
+
+ tracker = get_task_tracker()
+ task = await tracker.get(msg.task_id, ctx.account_id, ctx.user.user_id)
+ saved = dict(task.meta.get("feishu_responses", {})) if task else {}
+
+ async def save_response(entry: str, response_id: str) -> None:
+ await tracker.record_feishu_response(
+ msg.task_id, entry, response_id, ctx.account_id, ctx.user.user_id
+ )
+ saved[entry] = response_id
+
+ internal_kwargs["_feishu_checkpoint"] = (saved, save_response)
prepared_resource = None
if msg.staged_source is not None:
prepared_resource = await materialize_source(
@@ -934,6 +952,11 @@ class ResourceService:
source_name = source_name or source_info.source_name
source_info.source_name = source_name
elif feishu_source:
+ from openviking.parse.feishu_import import recursive_wiki
+
+ recursive = FeishuAccessor._parse_feishu_url(path)[0] == "wiki" and recursive_wiki(
+ processor_kwargs
+ )
token = processor_kwargs.get(FEISHU_ACCESS_TOKEN_ARG)
if isinstance(token, str) and token.strip():
task_auth = dict(
@@ -946,6 +969,7 @@ class ResourceService:
preflight = await FeishuAccessor().preflight_source(
path,
feishu_access_token=token.strip() if isinstance(token, str) else None,
+ **({"feishu_recursive": True} if recursive else {}),
)
source_name = source_name or preflight.source_name
source_info = _ResourceSourceInfo(
@@ -968,6 +992,18 @@ class ResourceService:
)
if watch_auth_state is None:
task_auth = {}
+ else:
+ source_type, _ = FeishuAccessor._parse_feishu_url(path)
+ if source_type in {"folder", "file"} or recursive:
+ if processor_kwargs.get("lark_file") is not None:
+ raise InvalidArgumentError(
+ "Feishu sources requiring preparation use args.feishu_access_token "
+ "or configured Feishu application credentials; args.lark_file "
+ "is only supported for direct single-document Understanding imports."
+ )
+ # These sources choose a content backend after source preparation.
+ queued_args["parser_backend"] = processor_kwargs.get("parser_backend")
+ queued_args["_feishu_prepared_source"] = True
else:
prepared = await self._resource_processor.prepare_durable_source(
path,
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index 6ca78af30..130229fc5 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -556,6 +556,28 @@ class TaskTracker:
updated.updated_at = self._next_updated_at(task)
await self._persist_and_publish("update", updated)
+ async def record_feishu_response(
+ self,
+ task_id: str,
+ entry: str,
+ response_id: str,
+ account_id: str,
+ user_id: str,
+ ) -> None:
+ """Persist a child submission before polling so a source retry can resume it."""
+
+ async def record() -> None:
+ async with self._task_locks.acquire(task_id):
+ task = await self._load_for_update(task_id, account_id, user_id)
+ if task is None or task.status not in (TaskStatus.PENDING, TaskStatus.RUNNING):
+ raise ValueError("Feishu source task is no longer active")
+ updated = deepcopy(task)
+ updated.meta.setdefault("feishu_responses", {})[entry] = response_id
+ updated.updated_at = self._next_updated_at(task)
+ await self._persist_and_publish("update", updated)
+
+ await self._dispatcher.run(record)
+
async def complete(
self,
task_id: str,
diff --git a/openviking/utils/media_processor.py b/openviking/utils/media_processor.py
index ad9db8f24..509620bad 100644
--- a/openviking/utils/media_processor.py
+++ b/openviking/utils/media_processor.py
@@ -200,6 +200,26 @@ class UnifiedResourceProcessor:
"direct host filesystem paths are not allowed."
)
+ from openviking.parse.accessors.feishu_accessor import FeishuAccessor
+ from openviking.parse.feishu_import import recursive_wiki
+
+ if FeishuAccessor._is_feishu_url(str(source)) and (
+ FeishuAccessor._parse_feishu_url(str(source))[0] == "folder"
+ or (
+ FeishuAccessor._parse_feishu_url(str(source))[0] == "wiki"
+ and recursive_wiki(kwargs)
+ )
+ ):
+ backend = normalize_parser_backend(kwargs.get("parser_backend"))
+ mode = normalize_parse_mode(kwargs.get("parse_mode", ParseMode.DEFAULT))
+ kwargs["_feishu_use_understanding"] = bool(
+ mode is ParseMode.DEFAULT
+ and backend is not ParserBackend.INTERNAL
+ and (
+ backend is ParserBackend.UNDERSTANDING
+ or self._get_parser_router().should_use_understanding_api(source)
+ )
+ )
resource = await self._get_accessor_registry().access(source, **kwargs)
try:
self._set_resolved_identity(resource, kwargs.get("source_name"))
@@ -331,6 +351,7 @@ class UnifiedResourceProcessor:
local_resource = prepared_resource or await self.prepare(
source,
allow_local_path_resolution=allow_local_path_resolution,
+ **({"parse_mode": mode} if mode is ParseMode.NO_SPLIT else {}),
**kwargs,
)
@@ -385,6 +406,7 @@ class UnifiedResourceProcessor:
from openviking.parse.parsers.directory import DirectoryParser
parser = DirectoryParser()
+ parse_kwargs["_feishu_import_plan"] = local_resource.feishu_plan
result = await parser.parse(str(local_resource.path), **parse_kwargs)
# Preserve temporary directory for TreeBuilder
@@ -396,6 +418,24 @@ class UnifiedResourceProcessor:
# For files, use ParserRouter to decide which parser to use
parser_router = self._get_parser_router()
+ if (
+ mode is ParseMode.NO_SPLIT
+ and local_resource.source_type == SourceType.FEISHU
+ and local_resource.meta.get("feishu_content_kind") == "file"
+ ):
+ parse_kwargs["parser_backend"] = ParserBackend.INTERNAL
+ parse_kwargs.pop("feishu_access_token", None)
+ parse_kwargs.pop("lark_file", None)
+ checkpoint = parse_kwargs.pop("_feishu_checkpoint", None)
+ if checkpoint is not None and local_resource.meta.get("feishu_content_kind") == "file":
+ saved, save = checkpoint
+ if "file" in saved:
+ parse_kwargs["understanding_response_id"] = saved["file"]
+
+ async def record(response_id):
+ await save("file", response_id)
+
+ parse_kwargs["_response_checkpoint"] = record
return await parser_router.parse(local_resource, **parse_kwargs)
finally:
# Clean up temporary resources unless they need to be preserved
diff --git a/tests/parse/test_directory_understanding_routing.py b/tests/parse/test_directory_understanding_routing.py
index 50e59ba8e..17365c91f 100644
--- a/tests/parse/test_directory_understanding_routing.py
+++ b/tests/parse/test_directory_understanding_routing.py
@@ -747,10 +747,17 @@ async def test_no_split_directory_falls_back_to_native_parser(
@pytest.mark.asyncio
[email protected]("backend", [None, "internal"])
[email protected]("feishu_collection", [False, True])
async def test_no_split_directory_records_missing_native_parser_per_file(
monkeypatch,
tmp_path: Path,
+ backend,
+ feishu_collection,
):
+ from openviking.parse.feishu_import import FeishuImportPlan
+
+ plan = FeishuImportPlan(tmp_path) if feishu_collection else None
_configure_understanding(monkeypatch, ["bin"])
(tmp_path / "README").write_text("plain text", encoding="utf-8")
(tmp_path / "payload.bin").write_bytes(b"\x00\x01")
@@ -762,7 +769,13 @@ async def test_no_split_directory_records_missing_native_parser_per_file(
patch.object(BaseParser, "_get_viking_fs", return_value=_FakeVikingFS()),
patch.object(ParserRouter, "parse", new=understanding_parse),
):
- result = await DirectoryParser().parse(str(tmp_path), split_content=False, strict=True)
+ result = await DirectoryParser().parse(
+ str(tmp_path),
+ split_content=False,
+ strict=not feishu_collection,
+ parser_backend=backend,
+ _feishu_import_plan=plan,
+ )
understanding_parse.assert_not_awaited()
assert result.meta["file_count"] == 1
diff --git a/tests/parse/test_feishu_accessor.py b/tests/parse/test_feishu_accessor.py
index 2d5dc29a9..a1c6553da 100644
--- a/tests/parse/test_feishu_accessor.py
+++ b/tests/parse/test_feishu_accessor.py
@@ -546,7 +546,33 @@ def test_access_downloads_json_drive_file_as_raw_bytes(monkeypatch):
resource.cleanup()
-def test_access_rejects_raw_feishu_error_envelope(monkeypatch):
+ "disposition",
+ ['attachment; filename="record.json"', 'inline; filename="record.json"', "attachment"],
+)
+def test_access_downloads_json_attachment_with_business_error_fields(monkeypatch, disposition):
+ _install_fake_lark_modules(monkeypatch)
+ payload = b'{"code":404,"message":"Example application record"}'
+ _FakeTransport.response = _FakeRawResponse(
+ payload,
+ headers={"content-type": "application/json", "content-disposition": disposition},
+ )
+ accessor = FeishuAccessor()
+ accessor._user_token_client = SimpleNamespace(_config=SimpleNamespace())
+ resource = asyncio.run(
+ accessor.access("https://example.feishu.cn/file/json_file", feishu_access_token="u-test")
+ )
+ try:
+ assert resource.path.read_bytes() == payload
+ finally:
+ resource.cleanup()
+
+
+ "status_code,disposition",
+ [(200, ""), (403, 'attachment; filename="error.json"')],
+)
+def test_access_rejects_raw_feishu_error_envelope(monkeypatch, status_code, disposition):
_install_fake_lark_modules(monkeypatch)
_FakeTransport.response = _FakeRawResponse(
json.dumps(
@@ -556,7 +582,11 @@ def test_access_rejects_raw_feishu_error_envelope(monkeypatch):
"data": {},
}
).encode("utf-8"),
- headers={"content-type": "application/json; charset=utf-8"},
+ status_code=status_code,
+ headers={
+ "content-type": "application/json; charset=utf-8",
+ "content-disposition": disposition,
+ },
)
accessor = FeishuAccessor()
accessor._user_token_client = SimpleNamespace(_config=SimpleNamespace())
@@ -760,6 +790,37 @@ def test_access_wiki_without_recursive_keeps_single_document_behavior(monkeypatc
resource.cleanup()
+def test_access_wiki_keeps_existing_single_document_behavior(monkeypatch):
+ from openviking.parse.accessors.feishu_accessor import FeishuDocument
+
+ accessor = FeishuAccessor()
+ monkeypatch.setattr(accessor, "_resolve_wiki_node", lambda *_args: ("docx", "doc", "Wiki"))
+
+ async def fake_fetch_document(*_args, **_kwargs):
+ return FeishuDocument(
+ doc_type="docx",
+ token="doc_token",
+ markdown_content="# single",
+ title="Single Wiki",
+ meta={"wiki_resolved": True},
+ )
+
+ monkeypatch.setattr(accessor, "_fetch_document", fake_fetch_document)
+ monkeypatch.setattr(
+ accessor,
+ "_resolve_image_refs",
+ lambda markdown, **_kwargs: (markdown, {}),
+ )
+
+ resource = asyncio.run(accessor.access("https://example.feishu.cn/wiki/wiki_token"))
+ try:
+ assert resource.path.is_file()
+ assert resource.path.read_text(encoding="utf-8") == "# single"
+ assert resource.meta["original_filename"] == "Single Wiki"
+ finally:
+ resource.cleanup()
+
+
def test_access_offloads_synchronous_download_to_thread(monkeypatch):
"""access() must not run the synchronous _resolve_image_refs on the event loop."""
import threading
@@ -974,9 +1035,7 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):
legacy_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/doc/doccn"))
docx_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/docx/doc"))
- sheets_identity = asyncio.run(
- accessor.preflight_source("https://example.feishu.cn/sheets/sht")
- )
+ sheets_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/sheets/sht"))
base_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/base/app"))
table_identity = asyncio.run(
accessor.preflight_source("https://example.feishu.cn/base/app?table=tbl_one")
diff --git a/tests/parse/test_feishu_unified_import.py b/tests/parse/test_feishu_unified_import.py
new file mode 100644
index 000000000..7884608e8
--- /dev/null
+++ b/tests/parse/test_feishu_unified_import.py
@@ -0,0 +1,1014 @@
+"""Regression coverage for Feishu source preparation and directory execution."""
+
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from openviking.parse.accessors.base import LocalResource, SourceType
+from openviking.parse.accessors.feishu_accessor import FeishuAccessor, _FeishuWikiTreeNode
+from openviking.parse.base import NodeType, ResourceNode, create_parse_result
+from openviking.parse.feishu_import import FeishuImportPlan
+from openviking.parse.parser_router import ParserRouter
+from openviking.parse.parsers.directory import DirectoryParser
+from openviking.parse.understanding_api import UnderstandingAPI
+from openviking.utils.media_processor import UnifiedResourceProcessor
+from openviking_cli.exceptions import InvalidArgumentError
+from tests.parse.test_add_directory import FakeVikingFS
+from tests.parse.test_directory_understanding_routing import _configure_understanding
+
+
+ "path,options",
+ [
+ ("drive/folder/folder", {}),
+ ("file/pdf", {}),
+ ("wiki/root", {"feishu_recursive": True}),
+ ],
+)
+def test_unsupported_direct_sources_require_preparation(path, options):
+ api = UnderstandingAPI.__new__(UnderstandingAPI)
+ assert not api.can_submit_url_directly(
+ f"https://example.feishu.cn/{path}", feishu_access_token="user-token", **options
+ )
+
+
+async def test_downloaded_file_routes_by_extension_but_normalized_markdown_does_not(
+ monkeypatch, tmp_path
+):
+ _configure_understanding(monkeypatch, ["pdf", "md"])
+ api = SimpleNamespace(parse=AsyncMock(return_value=object()))
+ registry = SimpleNamespace(parse=AsyncMock(return_value=object()))
+ router = ParserRouter(registry)
+ router._understanding_api = api
+ for name, kind in [("doc.pdf", "file"), ("doc.md", "markdown")]:
+ resource = LocalResource(
+ tmp_path / name,
+ SourceType.FEISHU,
+ "https://example.feishu.cn/file/t",
+ meta={"feishu_content_kind": kind},
+ )
+ await router.parse(resource)
+ api.parse.assert_awaited_once()
+ assert api.parse.await_args.args[0].endswith("doc.pdf")
+ registry.parse.assert_awaited_once()
+
+
+async def test_folder_remote_doc_and_empty_directory(monkeypatch):
+ accessor = FeishuAccessor()
+ monkeypatch.setattr(accessor, "_drive_folder_display_name", lambda *a, **k: "Folder")
+ monkeypatch.setattr(
+ accessor,
+ "_list_drive_folder_children",
+ lambda token, **k: (
+ [
+ SimpleNamespace(
+ type="docx",
+ token="doc",
+ name="Document",
+ url="https://example.feishu.cn/docx/doc",
+ ),
+ SimpleNamespace(type="folder", token="sub", name="Empty", url=""),
+ ]
+ if token == "root"
+ else []
+ ),
+ )
+ monkeypatch.setattr(
+ accessor, "_fetch_document", AsyncMock(side_effect=AssertionError("no Markdown conversion"))
+ )
+ resource = await accessor.access(
+ "https://example.feishu.cn/drive/folder/root", _feishu_use_understanding=True
+ )
+ try:
+ assert len(resource.feishu_plan.entries) == 1
+ assert resource.feishu_plan.entries[0].kind == "url"
+ assert (resource.path / "Empty").is_dir()
+ assert not (resource.path / "Document.md").exists()
+ finally:
+ resource.cleanup()
+
+
[email protected]("resume", [False, True])
+async def test_directory_executes_url_with_auth_and_preserves_output_parent(
+ monkeypatch, tmp_path, resume
+):
+ _configure_understanding(monkeypatch, ["pdf"])
+ (tmp_path / "nested").mkdir()
+ (tmp_path / "empty").mkdir()
+ plan = FeishuImportPlan(tmp_path, use_understanding=True)
+ path = tmp_path / "nested/Cloud.md"
+ url = "https://example.feishu.cn/base/app?table=t&view=v"
+ plan.add(path, url, "app", "url")
+ fs = FakeVikingFS()
+ parser = DirectoryParser()
+ monkeypatch.setattr(parser, "_get_viking_fs", lambda: fs)
+ monkeypatch.setattr(parser, "_create_temp_uri", lambda: "viking://temp/import")
+ calls = []
+
+ async def parse_api(self, source, **options):
+ calls.append((source, options))
+ await options["_response_checkpoint"]("response-1")
+ await fs.mkdir("viking://temp/doc/Cloud", exist_ok=True)
+ await fs.write("viking://temp/doc/Cloud/0.md", "body")
+ result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT, title="Cloud"),
+ source_path=source,
+ source_format="docx",
+ parser_name="UnderstandingAPI",
+ )
+ result.temp_dir_path = "viking://temp/doc"
+ result.meta["response_id"] = "response-1"
+ return result
+
+ monkeypatch.setattr(UnderstandingAPI, "parse", parse_api)
+ save = AsyncMock()
+ result = await parser.parse(
+ tmp_path,
+ source_name="Folder",
+ _feishu_import_plan=plan,
+ feishu_access_token="secret",
+ _feishu_checkpoint=(
+ {plan.entries[0].checkpoint_key(plan.root): "old"} if resume else {},
+ save,
+ ),
+ )
+ assert calls[0][0] == url
+ assert calls[0][1]["feishu_access_token"] == "secret"
+ assert calls[0][1]["resource_name"] == "Cloud.md"
+ assert calls[0][1].get("understanding_response_id") == ("old" if resume else None)
+ save.assert_awaited_once_with(plan.entries[0].checkpoint_key(plan.root), "response-1")
+ assert result.meta["failed_files"] == []
+ assert "viking://temp/import/Folder/nested/Cloud/0.md" in fs.files
+ assert "viking://temp/import/Folder/empty" not in fs.dirs
+ assert "secret" not in str(result.meta)
+
+
[email protected]("mode", ["legacy", "native", "understanding"])
+async def test_directory_omits_source_directories_without_imported_content(
+ monkeypatch, tmp_path, mode
+):
+ _configure_understanding(monkeypatch, ["pdf"])
+ plan = FeishuImportPlan(tmp_path, use_understanding=mode == "understanding")
+ (tmp_path / "empty/deep").mkdir(parents=True)
+ (tmp_path / "kept/nested").mkdir(parents=True)
+ (tmp_path / "kept/nested/data.json").write_text('{"body": "keep"}')
+ for directory, name in [("failed", "Broken.md"), ("filtered", "Private.md")]:
+ path = tmp_path / directory / name
+ path.parent.mkdir()
+ if mode != "understanding":
+ path.write_text("document")
+ plan.add(
+ path,
+ f"https://example.feishu.cn/docx/{directory}",
+ directory,
+ "url" if mode == "understanding" else "markdown",
+ )
+ fs = FakeVikingFS()
+ parser = DirectoryParser()
+ monkeypatch.setattr(parser, "_get_viking_fs", lambda: fs)
+ monkeypatch.setattr(parser, "_create_temp_uri", lambda: "viking://temp/empty-filter")
+ parse = AsyncMock(side_effect=ValueError("document parse failed"))
+ monkeypatch.setattr(DirectoryParser, "_parse_file_with_parser", parse)
+ result = await parser.parse(
+ tmp_path,
+ source_name="Folder",
+ preserve_structure=True,
+ exclude="Private.md",
+ **({"_feishu_import_plan": plan} if mode != "legacy" else {}),
+ )
+ parse.assert_awaited_once()
+ assert result.meta["file_count"] == 1
+ assert len(result.meta["failed_files"]) == 1
+ assert result.meta["failed_files"][0]["path"] == "failed/Broken.md"
+ target = "viking://temp/empty-filter/Folder"
+ assert fs.files[f"{target}/kept/nested/data.json"] == b'{"body": "keep"}'
+ assert [entry["name"] for entry in await fs.ls(target)] == ["kept"]
+
+
+async def test_directory_virtual_url_obeys_include_filter(monkeypatch, tmp_path):
+ _configure_understanding(monkeypatch, [])
+ plan = FeishuImportPlan(tmp_path, use_understanding=True)
+ plan.add(tmp_path / "Private.md", "https://example.feishu.cn/docx/private", "private", "url")
+ parser = DirectoryParser()
+ fs = FakeVikingFS()
+ monkeypatch.setattr(parser, "_get_viking_fs", lambda: fs)
+ monkeypatch.setattr(parser, "_create_temp_uri", lambda: "viking://temp/filter")
+ parse = AsyncMock(side_effect=AssertionError("excluded URL was submitted"))
+ monkeypatch.setattr(UnderstandingAPI, "parse", parse)
+ result = await parser.parse(tmp_path, _feishu_import_plan=plan, include="*.pdf")
+ parse.assert_not_called()
+ assert result.meta["file_count"] == 0
+
+
+async def test_media_prepare_enables_url_plan_only_for_correct_backend(monkeypatch, tmp_path):
+ _configure_understanding(monkeypatch, ["pdf"])
+ processor = UnifiedResourceProcessor(vlm_processor=object())
+ resource = LocalResource(
+ tmp_path, SourceType.FEISHU, "https://example.feishu.cn/drive/folder/f", is_temporary=False
+ )
+ access = AsyncMock(return_value=resource)
+ processor._accessor_registry = SimpleNamespace(access=access)
+ await processor.prepare(resource.original_source, parser_backend="understanding")
+ assert access.await_args.kwargs["_feishu_use_understanding"]
+ await processor.prepare(resource.original_source, parser_backend="internal")
+ assert not access.await_args.kwargs["_feishu_use_understanding"]
+ await processor.prepare(
+ resource.original_source, parser_backend="understanding", parse_mode="no_split"
+ )
+ assert not access.await_args.kwargs["_feishu_use_understanding"]
+
+
[email protected]("kind,recursive", [("folder", False), ("wiki", True), ("file", False)])
+async def test_async_source_plan_does_not_submit_collection_or_file_url(
+ monkeypatch, kind, recursive
+):
+ from openviking.parse.mode import ParseMode
+ from openviking.server.identity import RequestContext, Role
+ from openviking.service.resource_service import ResourceService
+ from openviking_cli.session.user_id import UserIdentifier
+
+ _configure_understanding(monkeypatch, ["pdf"])
+ router = ParserRouter(SimpleNamespace())
+ router._understanding_api = UnderstandingAPI.__new__(UnderstandingAPI)
+ submit = AsyncMock(side_effect=AssertionError("root URL must not be submitted"))
+ service = ResourceService(
+ viking_fs=SimpleNamespace(),
+ skill_processor=SimpleNamespace(),
+ resource_processor=SimpleNamespace(
+ should_use_understanding_directly=router.should_use_understanding_directly,
+ submit_understanding=submit,
+ ),
+ )
+ preflight = AsyncMock(
+ return_value=SimpleNamespace(
+ doc_type=kind,
+ source_format="file" if kind == "file" else "directory",
+ source_name="Root",
+ )
+ )
+ monkeypatch.setattr(FeishuAccessor, "preflight_source", preflight)
+ path = "drive/folder/root" if kind == "folder" else f"{kind}/root"
+ plan = await service._prepare_standard_source_plan(
+ path=f"https://example.feishu.cn/{path}",
+ ctx=RequestContext(user=UserIdentifier("account", "user"), role=Role.USER),
+ mode=ParseMode.DEFAULT,
+ allow_local_path_resolution=False,
+ processor_kwargs={"feishu_recursive": recursive, "feishu_access_token": "secret"},
+ watch_auth_state=None,
+ )
+ submit.assert_not_awaited()
+ assert plan.understanding_response_id is None
+ assert plan.processor_args["_feishu_prepared_source"]
+ assert "secret" not in str(plan.processor_args)
+ assert plan.task_auth["access_token"] == "secret"
+ if recursive:
+ assert preflight.await_args.kwargs["feishu_recursive"]
+
+
+async def test_checkpoint_is_written_before_polling():
+ from openviking.parse.understanding_api import UnderstandingAPIError
+ from tests.parse.test_feishu_parser_api import _understanding_api_for_parse
+
+ api = _understanding_api_for_parse()
+ api._create_response_for_url = AsyncMock(return_value={"id": "response-1"})
+ saved = []
+
+ async def record(response_id):
+ saved.append(response_id)
+
+ async def poll(**kwargs):
+ assert saved == ["response-1"]
+ raise TimeoutError("waiting")
+
+ api._poll_response = poll
+ with pytest.raises(UnderstandingAPIError) as error:
+ await api.parse(
+ "https://example.feishu.cn/docx/doc",
+ feishu_access_token="secret",
+ _response_checkpoint=record,
+ )
+ assert error.value.meta["response_id"] == "response-1"
+
+
+async def test_different_extensions_cannot_share_an_output_name(monkeypatch, tmp_path):
+ accessor = FeishuAccessor()
+ monkeypatch.setattr(
+ accessor,
+ "_download_drive_file",
+ lambda *a, **k: (b"%PDF-1.7", "application/pdf", "Report.pdf"),
+ )
+ plan = FeishuImportPlan(tmp_path, use_understanding=True)
+ await accessor._write_import_content("docx", "doc", "Report", "", tmp_path, plan)
+ await accessor._write_import_content("file", "pdf", "Report.pdf", "", tmp_path, plan)
+ assert len({entry.path.stem for entry in plan.entries}) == 2
+
+
+def test_pagination_budget_stops_before_next_page(monkeypatch):
+ accessor = FeishuAccessor()
+ page = Mock(
+ return_value=([SimpleNamespace(token="a"), SimpleNamespace(token="b")], True, "next")
+ )
+ monkeypatch.setattr(accessor, "_fetch_drive_folder_children_page", page)
+ assert len(accessor._list_drive_folder_children("root", max_items=1)) == 2
+ page.assert_called_once()
+
+
[email protected]("backend", [None, "understanding"])
+async def test_folder_processor_routes_each_content_and_keeps_tree(monkeypatch, backend):
+ _configure_understanding(monkeypatch, ["pdf"])
+ from openviking_cli.utils.config.open_viking_config import get_openviking_config
+
+ get_openviking_config().parser_api.enable_feishu_url = True
+ accessor = FeishuAccessor()
+ monkeypatch.setattr(accessor, "_drive_folder_display_name", lambda *a, **k: "Root")
+ monkeypatch.setattr(
+ accessor,
+ "_list_drive_folder_children",
+ lambda token, **k: (
+ [
+ SimpleNamespace(type="docx", token="rootdoc", name="Root", url=""),
+ SimpleNamespace(type="folder", token="nested", name="Report", url=""),
+ ]
+ if token == "root"
+ else [
+ SimpleNamespace(type="file", token="pdf", name="Report.pdf", url=""),
+ SimpleNamespace(
+ type="bitable",
+ token="leaf",
+ name="Leaf",
+ url="https://example.larksuite.com/base/leaf?table=t&view=v",
+ ),
+ ]
+ ),
+ )
+ monkeypatch.setattr(
+ accessor,
+ "_download_drive_file",
+ lambda *a, **k: (b"%PDF-1.7", "application/pdf", "Report.pdf"),
+ )
+ monkeypatch.setattr(
+ accessor,
+ "_fetch_document",
+ AsyncMock(side_effect=AssertionError("cloud docs must remain URLs")),
+ )
+ resources = []
+
+ async def access(source, **options):
+ resource = await accessor.access(source, **options)
+ resources.append(resource)
+ return resource
+
+ processor = UnifiedResourceProcessor(vlm_processor=object())
+ processor._accessor_registry = SimpleNamespace(access=access)
+ fs = FakeVikingFS()
+ monkeypatch.setattr(DirectoryParser, "_get_viking_fs", lambda self: fs)
+ monkeypatch.setattr(DirectoryParser, "_create_temp_uri", lambda self: "viking://temp/tree")
+ calls = []
+
+ async def parse_api(self, source, **options):
+ calls.append((source, options))
+ name = Path(options.get("resource_name") or source).stem
+ temporary = f"viking://temp/content_{len(calls)}"
+ await fs.mkdir(f"{temporary}/{name}", exist_ok=True)
+ await fs.write(f"{temporary}/{name}/0.md", "body")
+ result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT, title=name),
+ source_path=source,
+ source_format="docx" if source.startswith("https:") else "pdf",
+ parser_name="UnderstandingAPI",
+ )
+ result.temp_dir_path = temporary
+ return result
+
+ monkeypatch.setattr(UnderstandingAPI, "parse", parse_api)
+ try:
+ result = await processor.process(
+ "https://example.larksuite.com/drive/folder/root",
+ feishu_access_token="secret",
+ parser_backend=backend,
+ )
+ assert result.meta["failed_files"] == []
+ assert result.meta["file_count"] == 3
+ cloud_calls = [
+ (source, options) for source, options in calls if source.startswith("https:")
+ ]
+ assert {source for source, _ in cloud_calls} == {
+ "https://example.larksuite.com/docx/rootdoc",
+ "https://example.larksuite.com/base/leaf?table=t&view=v",
+ }
+ assert all(options["feishu_access_token"] == "secret" for _, options in cloud_calls)
+ binary_options = next(
+ options for source, options in calls if not source.startswith("https:")
+ )
+ assert "feishu_access_token" not in binary_options
+ assert "lark_file" not in binary_options
+ assert "viking://temp/tree/Root/Root/0.md" in fs.files
+ assert "viking://temp/tree/Root/Report/Leaf/0.md" in fs.files
+ assert "viking://temp/tree/Root/Report/Report/0.md" in fs.files
+ finally:
+ for resource in resources:
+ resource.is_temporary = True
+ resource.cleanup()
+
+
+async def test_background_collection_restores_auth_and_child_checkpoint(monkeypatch):
+ from openviking.server.identity import RequestContext, Role
+ from openviking.service.resource_service import ResourceService
+ from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
+ from openviking_cli.session.user_id import UserIdentifier
+
+ tracker = SimpleNamespace(
+ get=AsyncMock(return_value=SimpleNamespace(meta={"feishu_responses": {"entry": "old"}})),
+ record_feishu_response=AsyncMock(),
+ )
+ monkeypatch.setattr("openviking.service.task_tracker.get_task_tracker", lambda: tracker)
+ service = ResourceService()
+ service._execute_resource_ingestion = AsyncMock(return_value={"status": "success"})
+ msg = AddResourceMsg(
+ task_id="task",
+ path="https://example.feishu.cn/drive/folder/root",
+ root_uri="viking://resources/Root",
+ account_id="account",
+ user_id="user",
+ role="user",
+ args={"_feishu_prepared_source": True},
+ )
+ service._restore_source_task_auth = Mock(return_value=({"feishu_access_token": "secret"}, None))
+ await service.execute_add_resource_job(
+ msg,
+ ctx=RequestContext(user=UserIdentifier("account", "user"), role=Role.USER),
+ resource_lock=None,
+ stage_callback=AsyncMock(),
+ )
+ options = service._execute_resource_ingestion.await_args.kwargs
+ assert options["parser_backend"] is None
+ assert options["feishu_access_token"] == "secret"
+ saved, record = options["_feishu_checkpoint"]
+ assert saved == {"entry": "old"}
+ await record("next", "new")
+ tracker.record_feishu_response.assert_awaited_once_with(
+ "task", "next", "new", "account", "user"
+ )
+ assert saved["next"] == "new"
+
+
+async def test_resume_polls_original_response_without_resubmitting():
+ from openviking.parse.understanding_api import UnderstandingAPIError
+ from tests.parse.test_feishu_parser_api import _understanding_api_for_parse
+
+ api = _understanding_api_for_parse()
+ api._create_response_for_url = AsyncMock(side_effect=AssertionError("duplicate submission"))
+ api._resolve_lark_file = AsyncMock(side_effect=AssertionError("unnecessary auth resolution"))
+ api._poll_response = AsyncMock(side_effect=TimeoutError("still waiting"))
+ with pytest.raises(UnderstandingAPIError):
+ await api.parse("https://example.feishu.cn/docx/doc", understanding_response_id="old")
+ api._poll_response.assert_awaited_once_with(response_id="old")
+ api._create_response_for_url.assert_not_awaited()
+ api._resolve_lark_file.assert_not_awaited()
+
+
+async def test_drive_wiki_reference_resolves_backing_content(monkeypatch, tmp_path):
+ accessor = FeishuAccessor()
+ monkeypatch.setattr(accessor, "_resolve_wiki_node", lambda *a: ("base", "app", "Table"))
+ plan = FeishuImportPlan(tmp_path, use_understanding=True)
+ url = "https://example.feishu.cn/wiki/ref?table=t&view=v"
+ await accessor._write_import_content("wiki", "ref", "Table", url, tmp_path, plan)
+ assert plan.entries[0].url == url
+ assert plan.entries[0].token == "app"
+ assert plan.entries[0].kind == "url"
+
+
+async def test_source_preparation_does_not_silently_ignore_explicit_lark_auth():
+ with pytest.raises(InvalidArgumentError, match="feishu_access_token"):
+ await FeishuAccessor().access(
+ "https://example.feishu.cn/drive/folder/root",
+ lark_file={"tenant_access_token": "secret"},
+ )
+ from openviking.service.resource_service import ResourceService
+
+ assert "lark_file" not in ResourceService()._sanitize_watch_processor_kwargs(
+ {"lark_file": {"tenant_access_token": "secret"}}
+ )
+
+
[email protected]("filter_kind", ["gitignore", "ignore_dirs"])
+def test_virtual_content_uses_directory_filters(tmp_path, filter_kind):
+ from openviking.parse.gitignore import GitignoreMatcher
+
+ (tmp_path / "private").mkdir()
+ options = {}
+ if filter_kind == "gitignore":
+ (tmp_path / ".gitignore").write_text("private/\n")
+ else:
+ options["ignore_dirs"] = {"private"}
+ assert not DirectoryParser._include_feishu_path(
+ tmp_path / "private/Document.md",
+ tmp_path,
+ options,
+ gitignore=GitignoreMatcher(tmp_path),
+ )
+
+
[email protected]("mode", ["default", "no_split"])
+async def test_single_feishu_file_no_split_uses_native_parser(monkeypatch, tmp_path, mode):
+ _configure_understanding(monkeypatch, ["pdf"])
+ pdf = tmp_path / "Report.pdf"
+ pdf.write_bytes(b"%PDF-1.7")
+ source = "https://example.feishu.cn/file/pdf"
+ resource = LocalResource(
+ pdf,
+ SourceType.FEISHU,
+ source,
+ meta={"feishu_content_kind": "file"},
+ is_temporary=False,
+ )
+ registry = SimpleNamespace(parse=AsyncMock(return_value=object()))
+ api = SimpleNamespace(parse=AsyncMock(return_value=object()))
+ processor = UnifiedResourceProcessor(vlm_processor=object())
+ processor._accessor_registry = SimpleNamespace(access=AsyncMock(return_value=resource))
+ processor._parser_router = ParserRouter(registry)
+ processor._parser_router._understanding_api = api
+ await processor.process(source, parse_mode=mode)
+ if mode == "no_split":
+ registry.parse.assert_awaited_once()
+ assert registry.parse.await_args.kwargs["split_content"] is False
+ api.parse.assert_not_awaited()
+ else:
+ api.parse.assert_awaited_once()
+ registry.parse.assert_not_awaited()
+
+
[email protected]("strict", [False, True])
[email protected]("failure_stage", ["parse", "merge"])
+async def test_directory_failure_cleans_unmerged_artifacts(
+ monkeypatch, tmp_path, strict, failure_stage
+):
+ _configure_understanding(monkeypatch, ["pdf"])
+ plan = FeishuImportPlan(tmp_path, use_understanding=True)
+ for name in ("A", "B"):
+ plan.add(tmp_path / f"{name}.md", f"https://example.feishu.cn/docx/{name}", name, "url")
+ fs = FakeVikingFS()
+ parser = DirectoryParser()
+ monkeypatch.setattr(parser, "_get_viking_fs", lambda: fs)
+ monkeypatch.setattr(parser, "_create_temp_uri", lambda: "viking://temp/import")
+
+ async def parse_api(self, source, **options):
+ name = source.rsplit("/", 1)[-1]
+ if name == "A" and failure_stage == "parse":
+ raise ValueError("denied")
+ temporary = f"viking://temp/doc_{name}"
+ await fs.mkdir(f"{temporary}/{name}", exist_ok=True)
+ await fs.write(f"{temporary}/{name}/0.md", "body")
+ result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT, title=name),
+ source_path=source,
+ source_format="docx",
+ parser_name="UnderstandingAPI",
+ )
+ result.temp_dir_path = temporary
+ return result
+
+ move_file = fs.move_file
+
+ async def move_with_failure(source, target):
+ if "/A/" in source:
+ raise OSError("merge denied")
+ await move_file(source, target)
+
+ monkeypatch.setattr(fs, "move_file", move_with_failure)
+ monkeypatch.setattr(UnderstandingAPI, "parse", parse_api)
+ if strict:
+ with pytest.raises(InvalidArgumentError, match="denied"):
+ await parser.parse(tmp_path, _feishu_import_plan=plan, strict=True)
+ assert not fs.files
+ assert not fs.dirs
+ else:
+ result = await parser.parse(tmp_path, _feishu_import_plan=plan)
+ assert result.meta["file_count"] == 1
+ assert len(result.meta["failed_files"]) == 1
+ assert any(path.endswith("/B/0.md") for path in fs.files)
+ assert not any(path.startswith("viking://temp/doc_") for path in fs.files)
+
+
[email protected]("kind,extensions", [("url", ["pdf"]), ("url", []), ("file", ["pdf"])])
[email protected]("depth", [1, 2])
+async def test_feishu_collection_obeys_understanding_depth(
+ monkeypatch, tmp_path, kind, extensions, depth
+):
+ from openviking_cli.utils.config.open_viking_config import get_openviking_config
+
+ _configure_understanding(monkeypatch, extensions, max_depth=1)
+ get_openviking_config().parser_api.enable_feishu_url = True
+ parent = tmp_path.joinpath(*[f"level-{i}" for i in range(depth)])
+ parent.mkdir(parents=True)
+ plan = FeishuImportPlan(tmp_path, use_understanding=kind == "url")
+ url = f"https://example.feishu.cn/{'docx' if kind == 'url' else 'file'}/cloud"
+ path = parent / ("Cloud.md" if kind == "url" else "Cloud.pdf")
+ if kind == "file":
+ path.write_bytes(b"%PDF-1.7")
+ plan.add(path, url, "cloud", kind)
+ fs = FakeVikingFS()
+ result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT, title="Cloud"),
+ source_path=url,
+ source_format="docx",
+ parser_name="UnderstandingAPI",
+ )
+ result.temp_dir_path = "viking://temp/remote"
+ await fs.mkdir("viking://temp/remote/Cloud", exist_ok=True)
+ await fs.write("viking://temp/remote/Cloud/0.md", "cloud body")
+ api_parse = AsyncMock(return_value=result)
+ parser = DirectoryParser()
+ monkeypatch.setattr(parser, "_get_viking_fs", lambda: fs)
+ monkeypatch.setattr(UnderstandingAPI, "parse", api_parse)
+ if depth > 1:
+ with pytest.raises(InvalidArgumentError, match="depth"):
+ await parser.parse(tmp_path, _feishu_import_plan=plan, strict=True)
+ api_parse.assert_not_awaited()
+ else:
+ result = await parser.parse(tmp_path, _feishu_import_plan=plan, strict=True)
+ api_parse.assert_awaited_once()
+ assert result.meta["file_count"] == 1
+ assert result.meta["failed_files"] == []
+
+
[email protected]("mode", ["disabled", "no_split", "internal"])
+async def test_native_feishu_collection_keeps_source_depth_limit(monkeypatch, tmp_path, mode):
+ _configure_understanding(monkeypatch, ["md"], enabled=mode != "disabled", max_depth=1)
+ parent = tmp_path / "a" / "b"
+ parent.mkdir(parents=True)
+ path = parent / "Document.md"
+ path.write_text("body", encoding="utf-8")
+ plan = FeishuImportPlan(tmp_path, use_understanding=False, max_depth=20)
+ plan.add(path, "https://example.feishu.cn/docx/doc", "doc", "markdown")
+ parser = DirectoryParser()
+ fs = FakeVikingFS()
+ native_parse = AsyncMock(return_value={"ok": True, "meta": {}, "error": None})
+ api_parse = AsyncMock(side_effect=AssertionError("Understanding must not be submitted"))
+ monkeypatch.setattr(parser, "_get_viking_fs", lambda: fs)
+ monkeypatch.setattr(parser, "_process_single_file", native_parse)
+ monkeypatch.setattr(UnderstandingAPI, "parse", api_parse)
+
+ result = await parser.parse(
+ tmp_path,
+ _feishu_import_plan=plan,
+ split_content=mode != "no_split",
+ parser_backend="internal" if mode == "internal" else None,
+ )
+
+ native_parse.assert_awaited_once()
+ api_parse.assert_not_awaited()
+ assert result.meta["file_count"] == 1
+ assert result.meta["failed_files"] == []
+
+
+async def test_native_feishu_collection_keeps_binary_on_native_parser(monkeypatch, tmp_path):
+ _configure_understanding(monkeypatch, ["pdf"])
+ path = tmp_path / "Report.pdf"
+ path.write_bytes(b"%PDF-1.7")
+ plan = FeishuImportPlan(tmp_path)
+ plan.add(path, "https://example.feishu.cn/file/report", "report", "file")
+ parser = DirectoryParser()
+ fs = FakeVikingFS()
+ native_parse = AsyncMock(return_value={"ok": True, "meta": {}, "error": None})
+ api_parse = AsyncMock(side_effect=AssertionError("Understanding must not be submitted"))
+ monkeypatch.setattr(parser, "_get_viking_fs", lambda: fs)
+ monkeypatch.setattr(parser, "_process_single_file", native_parse)
+ monkeypatch.setattr(UnderstandingAPI, "parse", api_parse)
+
+ result = await parser.parse(tmp_path, _feishu_import_plan=plan, parser_backend="internal")
+
+ api_parse.assert_not_awaited()
+ native_parse.assert_awaited_once()
+ assert result.meta["processed_files"] == [
+ {
+ "path": "Report.pdf",
+ "parser": "PDFParser",
+ "source_url": "https://example.feishu.cn/file/report",
+ "source_token": "report",
+ }
+ ]
+ assert result.meta["failed_files"] == []
+
+
+def wiki_accessor(monkeypatch):
+ accessor = FeishuAccessor()
+ root = _FeishuWikiTreeNode("root", "space", "Root", "docx", "rootdoc")
+ pdf = _FeishuWikiTreeNode("pdf", "space", "Report.pdf", "file", "pdffile")
+ leaf = _FeishuWikiTreeNode("leaf", "space", "Leaf", "docx", "leafdoc")
+ nodes = {"root": [pdf], "pdf": [leaf], "leaf": []}
+ monkeypatch.setattr(accessor, "_resolve_wiki_tree_root", lambda *a, **k: root)
+ monkeypatch.setattr(
+ accessor, "_list_wiki_node_children", lambda space, token, **k: nodes[token]
+ )
+ monkeypatch.setattr(
+ accessor,
+ "_download_drive_file",
+ lambda *a, **k: (b"%PDF-1.7", "application/pdf", "Report.pdf"),
+ )
+ monkeypatch.setattr(
+ accessor,
+ "_fetch_document",
+ AsyncMock(side_effect=AssertionError("cloud docs must remain URLs")),
+ )
+ return accessor, nodes
+
+
+async def test_recursive_wiki_plan_keeps_body_and_pdf_children_without_placeholder_files(
+ monkeypatch,
+):
+ accessor, _ = wiki_accessor(monkeypatch)
+ resource = await accessor.access(
+ "https://example.feishu.cn/wiki/root", feishu_recursive=True, _feishu_use_understanding=True
+ )
+ try:
+ entries = resource.feishu_plan.entries
+ assert {e.path.relative_to(resource.path).as_posix(): e.kind for e in entries} == {
+ "Root.md": "url",
+ "Report/Report.pdf": "file",
+ "Report/Leaf.md": "url",
+ }
+ assert not list(resource.path.rglob("*.md"))
+ assert (resource.path / "Report/Report.pdf").read_bytes() == b"%PDF-1.7"
+ assert next(e for e in entries if e.token == "rootdoc").url.endswith("wiki/root")
+ assert resource.meta["feishu_folder_skipped_items"] == []
+ finally:
+ resource.cleanup()
+
+
+async def test_wiki_content_failure_does_not_stop_children_and_strict_fails(monkeypatch):
+ accessor, _ = wiki_accessor(monkeypatch)
+ monkeypatch.setattr(accessor, "_download_drive_file", Mock(side_effect=ValueError("denied")))
+ resource = await accessor.access(
+ "https://example.feishu.cn/wiki/root", feishu_recursive=True, _feishu_use_understanding=True
+ )
+ try:
+ assert any(e.token == "leafdoc" for e in resource.feishu_plan.entries)
+ assert "denied" in resource.meta["feishu_folder_skipped_items"][0]["reason"]
+ finally:
+ resource.cleanup()
+ with pytest.raises(ValueError, match="denied"):
+ await accessor.access(
+ "https://example.feishu.cn/wiki/root",
+ feishu_recursive=True,
+ _feishu_use_understanding=True,
+ strict=True,
+ )
+
+
+async def test_wiki_unknown_root_type_still_imports_children(monkeypatch):
+ accessor, _ = wiki_accessor(monkeypatch)
+ monkeypatch.setattr(
+ accessor,
+ "_resolve_wiki_tree_root",
+ lambda *a, **k: _FeishuWikiTreeNode("root", "space", "Root", "unknown", "unknown"),
+ )
+ resource = await accessor.access(
+ "https://example.feishu.cn/wiki/root", feishu_recursive=True, _feishu_use_understanding=True
+ )
+ try:
+ assert any(entry.token == "leafdoc" for entry in resource.feishu_plan.entries)
+ assert resource.meta["feishu_folder_skipped_items"]
+ finally:
+ resource.cleanup()
+
+
+async def test_wiki_failed_root_enumeration_is_not_an_empty_success(monkeypatch):
+ accessor, _ = wiki_accessor(monkeypatch)
+ monkeypatch.setattr(
+ accessor, "_list_wiki_node_children", Mock(side_effect=ValueError("list denied"))
+ )
+ with pytest.raises(ValueError, match="list denied"):
+ await accessor.access(
+ "https://example.feishu.cn/wiki/root",
+ feishu_recursive=True,
+ _feishu_use_understanding=True,
+ )
+
+
[email protected]("backend", [None, "understanding"])
+async def test_recursive_wiki_processor_routes_each_content_and_keeps_tree(monkeypatch, backend):
+ _configure_understanding(monkeypatch, ["pdf"])
+ from openviking_cli.utils.config.open_viking_config import get_openviking_config
+
+ get_openviking_config().parser_api.enable_feishu_url = True
+ accessor, _ = wiki_accessor(monkeypatch)
+ resources = []
+
+ async def access(source, **options):
+ resource = await accessor.access(source, **options)
+ resources.append(resource)
+ return resource
+
+ processor = UnifiedResourceProcessor(vlm_processor=object())
+ processor._accessor_registry = SimpleNamespace(access=access)
+ fs = FakeVikingFS()
+ monkeypatch.setattr(DirectoryParser, "_get_viking_fs", lambda self: fs)
+ monkeypatch.setattr(DirectoryParser, "_create_temp_uri", lambda self: "viking://temp/tree")
+ calls = []
+
+ async def parse_api(self, source, **options):
+ calls.append((source, options))
+ name = Path(options.get("resource_name") or source).stem
+ temporary = f"viking://temp/content_{len(calls)}"
+ await fs.mkdir(f"{temporary}/{name}", exist_ok=True)
+ await fs.write(f"{temporary}/{name}/0.md", "body")
+ result = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT, title=name),
+ source_path=source,
+ source_format="docx" if source.startswith("https:") else "pdf",
+ parser_name="UnderstandingAPI",
+ )
+ result.temp_dir_path = temporary
+ return result
+
+ monkeypatch.setattr(UnderstandingAPI, "parse", parse_api)
+ try:
+ result = await processor.process(
+ "https://example.larksuite.com/wiki/root?table=t&view=v",
+ feishu_recursive=True,
+ feishu_access_token="secret",
+ parser_backend=backend,
+ )
+ assert result.meta["failed_files"] == []
+ assert result.meta["file_count"] == 3
+ cloud_calls = [
+ (source, options) for source, options in calls if source.startswith("https:")
+ ]
+ assert {source for source, _ in cloud_calls} == {
+ "https://example.larksuite.com/wiki/root?table=t&view=v",
+ "https://example.larksuite.com/wiki/leaf",
+ }
+ assert all(options["feishu_access_token"] == "secret" for _, options in cloud_calls)
+ binary_options = next(
+ options for source, options in calls if not source.startswith("https:")
+ )
+ assert "feishu_access_token" not in binary_options
+ assert "lark_file" not in binary_options
+ assert "viking://temp/tree/Root/Root/0.md" in fs.files
+ assert "viking://temp/tree/Root/Report/Leaf/0.md" in fs.files
+ assert "viking://temp/tree/Root/Report/Report/0.md" in fs.files
+ finally:
+ for resource in resources:
+ resource.is_temporary = True
+ resource.cleanup()
+
+
+async def test_wiki_cycle_is_reported_and_sibling_content_survives(monkeypatch):
+ accessor, nodes = wiki_accessor(monkeypatch)
+ nodes["leaf"] = [_FeishuWikiTreeNode("root", "space", "Root", "docx", "rootdoc")]
+ resource = await accessor.access(
+ "https://example.feishu.cn/wiki/root", feishu_recursive=True, _feishu_use_understanding=True
+ )
+ try:
+ assert len(resource.feishu_plan.entries) == 3
+ assert any(
+ "Cyclic" in item["reason"] for item in resource.meta["feishu_folder_skipped_items"]
+ )
+ finally:
+ resource.cleanup()
+
+
[email protected]("strict", [False, True])
+async def test_wiki_leaf_at_depth_limit_keeps_leaf_layout(monkeypatch, strict):
+ accessor, nodes = wiki_accessor(monkeypatch)
+ nodes["root"] = nodes["pdf"]
+ resource = await accessor.access(
+ "https://example.feishu.cn/wiki/root",
+ feishu_recursive=True,
+ _feishu_use_understanding=True,
+ feishu_max_depth=1,
+ strict=strict,
+ )
+ try:
+ assert {
+ entry.path.relative_to(resource.path).as_posix()
+ for entry in resource.feishu_plan.entries
+ } == {"Root.md", "Leaf.md"}
+ assert not resource.meta["feishu_folder_skipped_items"]
+ finally:
+ resource.cleanup()
+
+
+async def test_wiki_depth_limit_reports_actual_truncation(monkeypatch):
+ accessor, _ = wiki_accessor(monkeypatch)
+ resource = await accessor.access(
+ "https://example.feishu.cn/wiki/root",
+ feishu_recursive=True,
+ _feishu_use_understanding=True,
+ feishu_max_depth=1,
+ )
+ try:
+ assert {
+ entry.path.relative_to(resource.path).as_posix()
+ for entry in resource.feishu_plan.entries
+ } == {"Root.md", "Report/Report.pdf"}
+ failures = resource.meta["feishu_folder_skipped_items"]
+ assert len(failures) == 1 and "depth" in failures[0]["reason"]
+ finally:
+ resource.cleanup()
+ with pytest.raises(ValueError, match="depth limit"):
+ await accessor.access(
+ "https://example.feishu.cn/wiki/root",
+ feishu_recursive=True,
+ _feishu_use_understanding=True,
+ feishu_max_depth=1,
+ strict=True,
+ )
+
+
[email protected]("value", ["false", "true", 1, None])
+def test_recursive_wiki_flag_requires_boolean(value):
+ api = UnderstandingAPI.__new__(UnderstandingAPI)
+ with pytest.raises(InvalidArgumentError, match="boolean"):
+ api.can_submit_url_directly(
+ "https://example.feishu.cn/wiki/root",
+ feishu_recursive=value,
+ feishu_access_token="user-token",
+ )
+
+
+async def test_recursive_wiki_preflight_keeps_unknown_root_as_directory(monkeypatch):
+ accessor, _ = wiki_accessor(monkeypatch)
+ monkeypatch.setattr(
+ accessor,
+ "_resolve_wiki_tree_root",
+ lambda *a, **k: _FeishuWikiTreeNode("root", "space", "Root", "unknown", "unknown"),
+ )
+ monkeypatch.setattr(accessor, "_probe_document_permission", Mock(side_effect=AssertionError))
+ result = await accessor.preflight_source(
+ "https://example.feishu.cn/wiki/root", feishu_recursive=True
+ )
+ assert result.source_format == "directory"
+ assert result.source_name == "Root"
+
+
[email protected]("content_kind", ["url", "file"])
+async def test_recursive_leaf_wiki_keeps_routable_resource(monkeypatch, content_kind):
+ accessor, nodes = wiki_accessor(monkeypatch)
+ node = (
+ _FeishuWikiTreeNode("leaf", "space", "Leaf", "docx", "leafdoc")
+ if content_kind == "url"
+ else _FeishuWikiTreeNode("leaf", "space", "Report.pdf", "file", "pdffile")
+ )
+ monkeypatch.setattr(accessor, "_resolve_wiki_tree_root", lambda *a, **k: node)
+ resource = await accessor.access(
+ "https://example.feishu.cn/wiki/leaf?table=t&view=v",
+ feishu_recursive=True,
+ _feishu_use_understanding=True,
+ )
+ try:
+ assert resource.path.exists()
+ assert len(resource.feishu_plan.entries) == 1
+ entry = resource.feishu_plan.entries[0]
+ assert entry.kind == content_kind
+ if content_kind == "url":
+ assert resource.path.is_dir()
+ assert not entry.path.exists()
+ assert entry.url.endswith("/wiki/leaf?table=t&view=v")
+ else:
+ assert resource.path.is_file()
+ assert resource.meta["feishu_content_kind"] == "file"
+ finally:
+ resource.cleanup()
+
+
+async def test_recursive_wiki_rejects_direct_only_lark_auth():
+ with pytest.raises(InvalidArgumentError, match="not lark_file"):
+ await FeishuAccessor().access(
+ "https://example.feishu.cn/wiki/root",
+ feishu_recursive=True,
+ lark_file={"user_access_token": "secret"},
+ )
diff --git a/tests/service/test_resource_service_understanding_routing.py b/tests/service/test_resource_service_understanding_routing.py
index 7e7282232..65b94b366 100644
--- a/tests/service/test_resource_service_understanding_routing.py
+++ b/tests/service/test_resource_service_understanding_routing.py
@@ -313,3 +313,69 @@ async def test_temp_uploaded_file_queues_external_parse_with_file_id(
processor.process_resource.assert_not_awaited()
agfs.pathlock_to_handoff.assert_awaited_once_with(lock)
agfs.pathlock_handoff.assert_awaited_once_with(lock)
+
+
+async def test_background_directory_keeps_extension_routing(monkeypatch, tmp_path):
+ from openviking.parse.base import NodeType, ResourceNode, create_parse_result
+ from openviking.parse.parsers.base_parser import BaseParser
+ from openviking.parse.parsers.directory import DirectoryParser
+ from openviking.parse.parsers.pdf import PDFParser
+ from openviking.parse.understanding_api import UnderstandingAPI
+ from openviking.utils.media_processor import UnifiedResourceProcessor
+ from tests.parse.test_add_directory import FakeVikingFS
+ from tests.parse.test_directory_understanding_routing import _configure_understanding
+
+ _configure_understanding(monkeypatch, ["pdf"])
+ (tmp_path / "paper.pdf").write_bytes(b"%PDF-1.7")
+ prepared = LocalResource(tmp_path, SourceType.LOCAL, str(tmp_path), is_temporary=False)
+ fs = FakeVikingFS()
+ service = ResourceService(viking_fs=fs)
+ parsed = create_parse_result(
+ root=ResourceNode(type=NodeType.ROOT, title="paper"),
+ source_path=str(tmp_path / "paper.pdf"),
+ source_format="pdf",
+ parser_name="test",
+ )
+ parsed.temp_dir_path = "viking://temp/parsed"
+ native_parse = AsyncMock(return_value=parsed)
+ api_parse = AsyncMock(return_value=parsed)
+ processor = UnifiedResourceProcessor(vlm_processor=object())
+
+ async def execute(**kwargs):
+ await processor.process(
+ str(tmp_path),
+ prepared_resource=kwargs["prepared_resource"],
+ parser_backend=kwargs["parser_backend"],
+ )
+ return {"status": "success"}
+
+ monkeypatch.setattr(service, "_execute_resource_ingestion", execute)
+ monkeypatch.setattr(BaseParser, "_get_viking_fs", lambda self: fs)
+ monkeypatch.setattr(
+ "openviking.resource.staged_source.materialize_source", AsyncMock(return_value=prepared)
+ )
+ monkeypatch.setattr(PDFParser, "parse", native_parse)
+ monkeypatch.setattr(UnderstandingAPI, "parse", api_parse)
+ monkeypatch.setattr(DirectoryParser, "_merge_parser_result", AsyncMock())
+ msg = AddResourceMsg(
+ task_id="task-1",
+ path=str(tmp_path),
+ root_uri="viking://resources/directory",
+ account_id="acct",
+ user_id="alice",
+ role="user",
+ staged_source={
+ "temp_uri": "viking://temp/staged",
+ "source_uri": "viking://temp/staged/source/directory",
+ "source_type": SourceType.LOCAL,
+ "original_source": str(tmp_path),
+ "meta": {},
+ },
+ )
+ ctx = RequestContext(user=UserIdentifier("acct", "alice"), role=Role.USER)
+ await service.execute_add_resource_job(
+ msg, ctx=ctx, resource_lock=None, stage_callback=AsyncMock()
+ )
+ api_parse.assert_awaited_once()
+ native_parse.assert_not_awaited()
diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.py
index 9b1320d49..3e71e774b 100644
--- a/tests/test_task_tracker.py
+++ b/tests/test_task_tracker.py
@@ -709,3 +709,11 @@ async def test_session_service_get_commit_task_also_filters_account():
)
assert other_account_result is None
+
+
+async def test_feishu_response_checkpoint_survives_reload(tracker):
+ task = await tracker.create("add_resource", **_owner_kwargs())
+ await tracker.record_feishu_response(task.task_id, "nested/doc", "response-1", "acme", "alice")
+ restored = TaskTracker(store=tracker._store)
+ record = await restored.get(task.task_id, **_owner_kwargs())
+ assert record.meta["feishu_responses"] == {"nested/doc": "response-1"}