Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(feishu):Support Feishu Drive folder and file imports (#3937)

zgyc1345a1f7e13
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.py
index 81a7fc87a..e3dec074c 100644
--- a/openviking/parse/accessors/feishu_accessor.py
+++ b/openviking/parse/accessors/feishu_accessor.py
@@ -14,11 +14,12 @@ import json
import mimetypes
import os
import re
+import shutil
import tempfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import parse_qs, unquote, urlparse
from openviking.parse.base import format_table_to_markdown
from openviking.utils.exceptions import error_code_from_http_status
@@ -34,6 +35,18 @@ _FEISHU_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(feishu://image/([^)]+)\)")
_FEISHU_DOCUMENT_FORBIDDEN = 1770032
_FEISHU_BITABLE_PERMISSION_REQUIRED = 99991672
_MAX_MEDIA_DOWNLOAD_CONTEXTS = 8
+_FEISHU_DOC_PATH_TYPES = {"docx", "wiki", "sheets", "base"}
+_FEISHU_DRIVE_DOC_TYPES = {
+ "doc": "docx",
+ "docx": "docx",
+ "sheet": "sheets",
+ "sheets": "sheets",
+ "bitable": "base",
+ "base": "base",
+ "wiki": "wiki",
+}
+_MAX_PATH_SEGMENT_CHARS = 120
+_MAX_PATH_SEGMENT_BYTES = 240
_MediaDownloadExtras = Dict[str, List[Optional[str]]]
@@ -48,6 +61,90 @@ def _title_as_filename(title: str) -> str:
return title.replace("/", "_").replace("\\", "_")
+def _truncate_text_for_path_segment(text: str, *, max_chars: int, max_bytes: int) -> str:
+ """Trim text so its UTF-8 representation fits a single path segment budget."""
+ if max_chars <= 0 or max_bytes <= 0:
+ return ""
+ result: list[str] = []
+ used_bytes = 0
+ for char in text[:max_chars]:
+ char_bytes = len(char.encode("utf-8"))
+ if used_bytes + char_bytes > max_bytes:
+ break
+ result.append(char)
+ used_bytes += char_bytes
+ return "".join(result)
+
+
+def _fit_path_segment(text: str, *, max_chars: int, max_bytes: int) -> str:
+ """Ensure a complete path segment fits the configured character and byte budgets."""
+ if len(text) <= max_chars and len(text.encode("utf-8")) <= max_bytes:
+ return text
+ return _truncate_text_for_path_segment(
+ text,
+ max_chars=max_chars,
+ max_bytes=max_bytes,
+ ).rstrip(" ._")
+
+
+def _safe_path_segment(
+ name: str,
+ *,
+ fallback: str = "untitled",
+ max_len: int = _MAX_PATH_SEGMENT_CHARS,
+ max_bytes: int = _MAX_PATH_SEGMENT_BYTES,
+) -> str:
+ """Return one portable path segment while preserving readable names."""
+ safe_name = re.sub(r"[\x00-\x1f/\\:*?\"<>|]+", "_", str(name or "")).strip(" ._")
+ safe_name = re.sub(r"\s+", " ", safe_name)
+ if not safe_name:
+ safe_name = fallback
+ if len(safe_name) <= max_len and len(safe_name.encode("utf-8")) <= max_bytes:
+ return safe_name
+
+ stem = Path(safe_name).stem
+ suffix = Path(safe_name).suffix
+ suffix_bytes = len(suffix.encode("utf-8"))
+ if suffix_bytes >= max_bytes:
+ suffix = ""
+ suffix_bytes = 0
+
+ stem = _truncate_text_for_path_segment(
+ stem,
+ max_chars=max_len - len(suffix),
+ max_bytes=max_bytes - suffix_bytes,
+ ).rstrip(" ._")
+ if not stem:
+ stem = _truncate_text_for_path_segment(
+ fallback,
+ max_chars=max_len - len(suffix),
+ max_bytes=max_bytes - suffix_bytes,
+ ).rstrip(" ._")
+ return _fit_path_segment(
+ f"{stem or 'untitled'}{suffix}",
+ max_chars=max_len,
+ max_bytes=max_bytes,
+ ) or "untitled"
+
+
+def _numbered_path_segment(stem: str, suffix: str, index: int) -> str:
+ marker = f" ({index})"
+ marker_bytes = len(marker.encode("utf-8"))
+ suffix_bytes = len(suffix.encode("utf-8"))
+ max_stem_bytes = _MAX_PATH_SEGMENT_BYTES - marker_bytes - suffix_bytes
+ max_stem_chars = _MAX_PATH_SEGMENT_CHARS - len(marker) - len(suffix)
+ safe_stem = _truncate_text_for_path_segment(
+ stem,
+ max_chars=max_stem_chars,
+ max_bytes=max_stem_bytes,
+ ).rstrip(" ._")
+ return _fit_path_segment(
+ f"{safe_stem or 'untitled'}{marker}{suffix}",
+ max_chars=_MAX_PATH_SEGMENT_CHARS,
+ max_bytes=_MAX_PATH_SEGMENT_BYTES,
+ ) or "untitled"
+
+
def _getattr_safe(obj, key: str, default=None):
"""Get attribute from SDK object or dict, with safe fallback."""
if isinstance(obj, dict):
@@ -122,6 +219,8 @@ class FeishuAccessor(DataAccessor):
- Wiki pages: https://*.feishu.cn/wiki/{token}
- Spreadsheets: https://*.feishu.cn/sheets/{token}
- Bitable: https://*.feishu.cn/base/{app_token}
+ - Drive files: https://*.feishu.cn/file/{file_token}
+ - Drive folders: https://*.feishu.cn/drive/folder/{folder_token}
Requires:
- lark-oapi package
@@ -260,6 +359,64 @@ class FeishuAccessor(DataAccessor):
feishu_access_token = kwargs.get("feishu_access_token")
try:
+ doc_type, token = self._parse_feishu_url(source_str)
+ if doc_type == "file":
+ content, content_type, filename = await asyncio.to_thread(
+ self._download_drive_file,
+ token,
+ feishu_access_token=feishu_access_token,
+ )
+ local_path = self._write_temp_drive_file(
+ token,
+ content,
+ content_type,
+ filename_hint=filename,
+ )
+ return LocalResource(
+ path=local_path,
+ source_type=SourceType.FEISHU,
+ original_source=source_str,
+ meta={
+ "feishu_doc_type": doc_type,
+ "feishu_token": token,
+ "original_filename": local_path.name,
+ "_cleanup_path": str(local_path.parent),
+ },
+ is_temporary=True,
+ )
+
+ if doc_type == "folder":
+ temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_folder_"))
+ skipped_items: list[dict[str, Any]] = []
+ folder_name = await asyncio.to_thread(
+ self._drive_folder_display_name,
+ token,
+ feishu_access_token=feishu_access_token,
+ )
+ try:
+ await self._materialize_drive_folder(
+ token,
+ temp_dir,
+ feishu_access_token=feishu_access_token,
+ skipped_items=skipped_items,
+ strict=bool(kwargs.get("strict", False)),
+ )
+ except Exception:
+ shutil.rmtree(temp_dir, ignore_errors=True)
+ raise
+ return LocalResource(
+ path=temp_dir,
+ source_type=SourceType.FEISHU,
+ original_source=source_str,
+ meta={
+ "feishu_doc_type": doc_type,
+ "feishu_token": token,
+ "original_filename": _safe_path_segment(folder_name, fallback=token),
+ "feishu_folder_skipped_items": skipped_items,
+ },
+ is_temporary=True,
+ )
+
# Fetch the document and convert to Markdown
doc = await self._fetch_document(
source_str,
@@ -400,15 +557,17 @@ class FeishuAccessor(DataAccessor):
"""Check if URL is a Feishu/Lark cloud document."""
parsed = urlparse(url)
host = (parsed.hostname or "").lower().rstrip(".")
- path = parsed.path
+ path_parts = [p for p in parsed.path.split("/") if p]
is_feishu_domain = any(
host == allowed_host or host.endswith(f".{allowed_host}")
for allowed_host in ("feishu.cn", "larksuite.com", "larkoffice.com")
)
- has_doc_path = any(
- path == f"/{t}" or path.startswith(f"/{t}/") for t in ("docx", "wiki", "sheets", "base")
- )
- return is_feishu_domain and has_doc_path
+ if not is_feishu_domain or len(path_parts) < 2:
+ return False
+ has_doc_path = path_parts[0] in _FEISHU_DOC_PATH_TYPES
+ has_drive_folder_path = len(path_parts) >= 3 and path_parts[:2] == ["drive", "folder"]
+ has_file_path = path_parts[0] == "file"
+ return is_feishu_domain and (has_doc_path or has_drive_folder_path or has_file_path)
@staticmethod
def _parse_feishu_url(url: str) -> Tuple[str, str]:
@@ -422,10 +581,466 @@ class FeishuAccessor(DataAccessor):
path_parts = [p for p in parsed.path.split("/") if p]
if len(path_parts) < 2:
raise ValueError(f"Cannot parse Feishu URL: {url}")
+ if len(path_parts) >= 3 and path_parts[:2] == ["drive", "folder"]:
+ return "folder", path_parts[2]
+ if path_parts[0] == "file":
+ return "file", path_parts[1]
doc_type = path_parts[0] # docx, wiki, sheets, base
token = path_parts[1]
return doc_type, token
+ async def _materialize_drive_folder(
+ self,
+ folder_token: str,
+ target_dir: Path,
+ *,
+ feishu_access_token: Optional[str] = None,
+ _seen: Optional[set[str]] = None,
+ skipped_items: Optional[list[dict[str, Any]]] = None,
+ strict: bool = False,
+ ) -> None:
+ """Expand a Feishu Drive folder into a local directory tree."""
+ target_dir.mkdir(parents=True, exist_ok=True)
+ seen = _seen or set()
+ if folder_token in seen:
+ logger.warning("[FeishuAccessor] Skipping recursive Drive folder %s", folder_token)
+ return
+ seen.add(folder_token)
+
+ try:
+ children = await asyncio.to_thread(
+ self._list_drive_folder_children,
+ folder_token,
+ feishu_access_token=feishu_access_token,
+ )
+ for item in children:
+ item_type, item_token, item_name, item_url = self._normalize_drive_item(item)
+ if not item_token:
+ logger.warning("[FeishuAccessor] Skipping Drive item without token: %s", item)
+ continue
+
+ if item_type == "folder":
+ folder_name = _safe_path_segment(item_name or item_token, fallback=item_token)
+ child_dir = self._unique_child_path(target_dir, folder_name)
+ try:
+ await self._materialize_drive_folder(
+ item_token,
+ child_dir,
+ feishu_access_token=feishu_access_token,
+ _seen=seen,
+ skipped_items=skipped_items,
+ strict=strict,
+ )
+ except Exception as exc:
+ shutil.rmtree(child_dir, ignore_errors=True)
+ self._record_skipped_drive_item(
+ skipped_items,
+ item_type=item_type,
+ token=item_token,
+ name=item_name,
+ target_dir=target_dir,
+ error=exc,
+ )
+ if strict:
+ raise
+ continue
+
+ doc_path_type = _FEISHU_DRIVE_DOC_TYPES.get(item_type)
+ if doc_path_type:
+ try:
+ url = item_url or self._build_feishu_doc_url(doc_path_type, item_token)
+ doc = await self._fetch_document(
+ url,
+ feishu_access_token=feishu_access_token,
+ )
+ markdown_content, downloaded_images = await asyncio.to_thread(
+ self._resolve_image_refs,
+ doc.markdown_content,
+ feishu_access_token=feishu_access_token,
+ media_download_extras=doc.media_download_extras,
+ )
+ doc_name = _safe_path_segment(
+ item_name or doc.title or item_token,
+ fallback=item_token,
+ )
+ markdown_path = self._unique_child_path(
+ target_dir,
+ self._markdown_file_name(doc_name),
+ )
+ markdown_path.write_text(markdown_content, encoding="utf-8")
+ for rel_path, image_bytes in downloaded_images.items():
+ image_path = markdown_path.parent / rel_path
+ image_path.parent.mkdir(parents=True, exist_ok=True)
+ image_path.write_bytes(image_bytes)
+ except Exception as exc:
+ self._record_skipped_drive_item(
+ skipped_items,
+ item_type=item_type,
+ token=item_token,
+ name=item_name,
+ target_dir=target_dir,
+ error=exc,
+ )
+ if strict:
+ raise
+ continue
+
+ if item_type == "file":
+ try:
+ content, content_type, downloaded_name = await asyncio.to_thread(
+ self._download_drive_file,
+ item_token,
+ feishu_access_token=feishu_access_token,
+ filename_hint=item_name,
+ )
+ file_name = self._drive_file_name(
+ item_token,
+ content,
+ content_type,
+ filename_hint=downloaded_name or item_name,
+ )
+ file_path = self._unique_child_path(target_dir, file_name)
+ file_path.write_bytes(content)
+ except Exception as exc:
+ self._record_skipped_drive_item(
+ skipped_items,
+ item_type=item_type,
+ token=item_token,
+ name=item_name,
+ target_dir=target_dir,
+ error=exc,
+ )
+ if strict:
+ raise
+ continue
+
+ error = ValueError(f"Unsupported Feishu Drive item type: {item_type}")
+ self._record_skipped_drive_item(
+ skipped_items,
+ item_type=item_type,
+ token=item_token,
+ name=item_name,
+ target_dir=target_dir,
+ error=error,
+ )
+ if strict:
+ raise error
+
+ finally:
+ seen.discard(folder_token)
+
+ @staticmethod
+ def _record_skipped_drive_item(
+ skipped_items: Optional[list[dict[str, Any]]],
+ *,
+ item_type: str,
+ token: str,
+ name: str,
+ target_dir: Path,
+ error: Exception,
+ ) -> None:
+ message = str(error).replace("\n", " ")
+ logger.warning(
+ "[FeishuAccessor] Skipping Drive %s %s under %s: %s",
+ item_type,
+ token,
+ target_dir,
+ message,
+ )
+ if skipped_items is None:
+ return
+ skipped_items.append(
+ {
+ "path": str(target_dir / _safe_path_segment(name or token, fallback=token)),
+ "name": name or token,
+ "type": item_type,
+ "token": token,
+ "reason": message,
+ }
+ )
+
+ def _list_drive_folder_children(
+ self,
+ folder_token: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> List[Any]:
+ """List direct children under a Feishu Drive folder token."""
+ import lark_oapi as lark
+
+ client = self._get_client(use_user_token=bool(feishu_access_token))
+ token_type = (
+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT
+ )
+ all_children: List[Any] = []
+ page_token = None
+
+ while True:
+ raw_req = (
+ lark.BaseRequest.builder()
+ .http_method(lark.HttpMethod.GET)
+ .uri("/open-apis/drive/v1/files")
+ .token_types({token_type})
+ .build()
+ )
+ raw_req.add_query("folder_token", folder_token)
+ raw_req.add_query("page_size", 200)
+ if page_token:
+ raw_req.add_query("page_token", page_token)
+
+ response = self._call_api(client.request, raw_req, feishu_access_token)
+ if not response.success():
+ _raise_from_lark_response(
+ response,
+ operation=f"list Drive folder {folder_token}",
+ resource=folder_token,
+ )
+
+ data = self._drive_response_data(response)
+ items = _getattr_safe(data, "files", None) or _getattr_safe(data, "items", None) or []
+ all_children.extend(items)
+
+ has_more = bool(_getattr_safe(data, "has_more", False))
+ if not has_more:
+ break
+ page_token = _getattr_safe(data, "next_page_token", None) or _getattr_safe(
+ data, "page_token", None
+ )
+ if not page_token:
+ raise RuntimeError(
+ f"Feishu returned more Drive folder items for {folder_token} "
+ "without a page token"
+ )
+
+ return all_children
+
+ def _drive_folder_display_name(
+ self,
+ folder_token: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> str:
+ """Best-effort readable folder name for resource roots."""
+ try:
+ name = self._get_drive_folder_name(
+ folder_token,
+ feishu_access_token=feishu_access_token,
+ )
+ except Exception as exc:
+ logger.warning(
+ "[FeishuAccessor] Falling back to Drive folder token %s as name: %s",
+ folder_token,
+ exc,
+ )
+ return folder_token
+ return name or folder_token
+
+ def _get_drive_folder_name(
+ self,
+ folder_token: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> Optional[str]:
+ """Fetch a Feishu Drive folder display name by folder token."""
+ import lark_oapi as lark
+
+ client = self._get_client(use_user_token=bool(feishu_access_token))
+ token_type = (
+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT
+ )
+ raw_req = (
+ lark.BaseRequest.builder()
+ .http_method(lark.HttpMethod.GET)
+ .uri(f"/open-apis/drive/explorer/v2/folder/{folder_token}/meta")
+ .token_types({token_type})
+ .build()
+ )
+ response = self._call_api(client.request, raw_req, feishu_access_token)
+ if not response.success():
+ _raise_from_lark_response(
+ response,
+ operation=f"fetch Drive folder metadata {folder_token}",
+ resource=folder_token,
+ )
+
+ data = self._drive_response_data(response)
+ folder_meta = _getattr_safe(data, "folder", None) or _getattr_safe(data, "meta", None)
+ name = _getattr_safe(data, "name", None) or _getattr_safe(folder_meta, "name", None)
+ return str(name) if name else None
+
+ def _download_drive_file(
+ self,
+ file_token: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ filename_hint: Optional[str] = None,
+ ) -> Tuple[bytes, Optional[str], Optional[str]]:
+ """Download a Feishu Drive binary file by file token."""
+ import lark_oapi as lark
+
+ client = self._get_client(use_user_token=bool(feishu_access_token))
+ token_type = (
+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT
+ )
+ raw_req = (
+ lark.BaseRequest.builder()
+ .http_method(lark.HttpMethod.GET)
+ .uri(f"/open-apis/drive/v1/files/{file_token}/download")
+ .token_types({token_type})
+ .build()
+ )
+ response = self._call_api(client.request, raw_req, feishu_access_token)
+ if not response.success():
+ _raise_from_lark_response(
+ response,
+ operation=f"download Drive file {file_token}",
+ resource=file_token,
+ )
+
+ raw = getattr(response, "raw", None)
+ content = getattr(raw, "content", None)
+ if content is None:
+ raise OpenVikingError(
+ f"Feishu Drive file download returned empty content: {file_token}",
+ code="NOT_FOUND",
+ details={"operation": "download Drive file", "resource": file_token},
+ )
+ if isinstance(content, str):
+ content = content.encode("utf-8")
+
+ content_type = self._response_content_type(raw)
+ filename = self._filename_from_content_disposition(
+ self._response_header(raw, "content-disposition")
+ )
+ return content, content_type, filename or filename_hint
+
+ def _write_temp_drive_file(
+ self,
+ file_token: str,
+ content: bytes,
+ content_type: Optional[str],
+ *,
+ filename_hint: Optional[str] = None,
+ ) -> Path:
+ filename = self._drive_file_name(
+ file_token,
+ content,
+ content_type,
+ filename_hint=filename_hint,
+ )
+ temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_file_"))
+ path = temp_dir / filename
+ path.write_bytes(content)
+ return path
+
+ @staticmethod
+ def _drive_response_data(response: Any) -> Any:
+ data = getattr(response, "data", None)
+ if data is not None:
+ return data
+ raw_content = getattr(getattr(response, "raw", None), "content", None)
+ if not raw_content:
+ return {}
+ if isinstance(raw_content, bytes):
+ raw_content = raw_content.decode("utf-8")
+ return json.loads(raw_content).get("data", {})
+
+ @classmethod
+ def _normalize_drive_item(cls, item: Any) -> Tuple[str, str, str, str]:
+ item_type = str(_getattr_safe(item, "type", "") or "").lower()
+ token = str(_getattr_safe(item, "token", "") or "")
+ name = str(_getattr_safe(item, "name", "") or "")
+ url = str(_getattr_safe(item, "url", "") or "")
+ shortcut_info = _getattr_safe(item, "shortcut_info", None)
+ if shortcut_info:
+ item_type = str(
+ _getattr_safe(shortcut_info, "target_type", item_type) or item_type
+ ).lower()
+ token = str(_getattr_safe(shortcut_info, "target_token", token) or token)
+ return item_type, token, name, url
+
+ @staticmethod
+ def _build_feishu_doc_url(doc_type: str, token: str) -> str:
+ return f"https://open.feishu.cn/{doc_type}/{token}"
+
+ @staticmethod
+ def _unique_child_path(parent: Path, name: str) -> Path:
+ path = parent / _safe_path_segment(name)
+ if not path.exists():
+ return path
+ suffix = path.suffix
+ stem = path.stem
+ for index in range(2, 10000):
+ candidate = parent / _numbered_path_segment(stem, suffix, index)
+ if not candidate.exists():
+ return candidate
+ raise RuntimeError(f"Unable to allocate unique path under {parent}")
+
+ @classmethod
+ def _drive_file_name(
+ cls,
+ file_token: str,
+ content: bytes,
+ content_type: Optional[str],
+ *,
+ filename_hint: Optional[str] = None,
+ ) -> str:
+ raw_name = filename_hint or file_token
+ if Path(raw_name).suffix:
+ return _safe_path_segment(raw_name, fallback=file_token)
+ ext = cls._guess_drive_file_ext(content, content_type)
+ return _safe_path_segment(f"{raw_name}{ext}", fallback=f"{file_token}{ext}")
+
+ @staticmethod
+ def _markdown_file_name(name: str) -> str:
+ if str(name).lower().endswith((".md", ".markdown")):
+ return _safe_path_segment(name)
+ return _safe_path_segment(f"{name}.md")
+
+ @staticmethod
+ def _guess_drive_file_ext(content: bytes, content_type: Optional[str]) -> str:
+ if content.startswith(b"%PDF-"):
+ return ".pdf"
+ if (
+ content.startswith(b"PK\x03\x04")
+ or content.startswith(b"PK\x05\x06")
+ or content.startswith(b"PK\x07\x08")
+ ):
+ return ".zip"
+ if content.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
+ return ".doc"
+ if content_type:
+ ext = get_preferred_extension(content_type)
+ if ext:
+ return ext
+ return ".bin"
+
+ @staticmethod
+ def _response_header(raw: Any, name: str) -> Optional[str]:
+ headers = getattr(raw, "headers", None)
+ if not headers:
+ return None
+ try:
+ get = headers.get
+ except AttributeError:
+ return None
+ return get(name) or get(name.title()) or get(name.lower())
+
+ @staticmethod
+ def _filename_from_content_disposition(content_disposition: Optional[str]) -> Optional[str]:
+ if not content_disposition:
+ return None
+ utf8_match = re.search(r"filename\*=UTF-8''([^;]+)", content_disposition, re.I)
+ if utf8_match:
+ return unquote(utf8_match.group(1))
+ quoted_match = re.search(r'filename="([^"]+)"', content_disposition, re.I)
+ if quoted_match:
+ return quoted_match.group(1)
+ simple_match = re.search(r"filename=([^;]+)", content_disposition, re.I)
+ if simple_match:
+ return simple_match.group(1).strip()
+ return None
+
# ========== Configuration & Client ==========
def _get_config(self):
diff --git a/openviking/parse/parsers/directory.py b/openviking/parse/parsers/directory.py
index f01291703..2c526d5df 100644
--- a/openviking/parse/parsers/directory.py
+++ b/openviking/parse/parsers/directory.py
@@ -142,6 +142,14 @@ class DirectoryParser(BaseParser):
preserve_structure = True
processable_files = scan_result.all_processable_files()
warnings.extend(scan_result.warnings)
+ source_skipped_items = self._source_skipped_items(
+ kwargs.get("_source_meta"),
+ source_path,
+ )
+ warnings.extend(
+ f"Skipped Feishu Drive item {item['path']}: {item.get('reason', 'unknown error')}"
+ for item in source_skipped_items
+ )
viking_fs = self._get_viking_fs()
temp_uri = self._create_temp_uri()
@@ -164,6 +172,13 @@ class DirectoryParser(BaseParser):
warnings=warnings,
)
result.temp_dir_path = temp_uri
+ result.meta["file_count"] = 0
+ result.meta["dir_name"] = dir_name
+ result.meta["total_processable"] = 0
+ result.meta["processed_files"] = []
+ result.meta["failed_files"] = source_skipped_items
+ result.meta["unsupported_files"] = []
+ result.meta["skipped_files"] = []
return result
# ── Phase 2: process each file ────────────────────────────
@@ -259,7 +274,7 @@ class DirectoryParser(BaseParser):
result.meta["dir_name"] = dir_name
result.meta["total_processable"] = len(processable_files)
result.meta["processed_files"] = processed_files
- result.meta["failed_files"] = failed_files
+ result.meta["failed_files"] = failed_files + source_skipped_items
result.meta["unsupported_files"] = unsupported_files
result.meta["skipped_files"] = skipped_files
@@ -329,6 +344,41 @@ class DirectoryParser(BaseParser):
result.append({"path": path, "status": status})
return result
+ @staticmethod
+ def _source_skipped_items(source_meta: Any, source_path: Path) -> List[Dict[str, str]]:
+ """Normalize skipped items reported by a remote source accessor."""
+ if not isinstance(source_meta, dict):
+ return []
+ items = source_meta.get("feishu_folder_skipped_items") or []
+ if not isinstance(items, list):
+ return []
+
+ normalized: List[Dict[str, str]] = []
+ for item in items:
+ if not isinstance(item, dict):
+ continue
+ raw_path = str(item.get("path") or item.get("name") or item.get("token") or "unknown")
+ display_path = raw_path
+ try:
+ path = Path(raw_path).resolve(strict=False)
+ if path.is_absolute():
+ display_path = str(path.relative_to(source_path.resolve(strict=False)))
+ except Exception:
+ pass
+ display_path = display_path.replace("\\", "/")
+
+ normalized.append(
+ {
+ "path": display_path,
+ "parser": "feishu",
+ "status": "failed",
+ "type": str(item.get("type") or ""),
+ "token": str(item.get("token") or ""),
+ "reason": str(item.get("reason") or "unknown error"),
+ }
+ )
+ return normalized
+
# ------------------------------------------------------------------
# Parser assignment
# ------------------------------------------------------------------
diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.py
index 70269b62d..3325e5332 100644
--- a/tests/parse/test_add_directory.py
+++ b/tests/parse/test_add_directory.py
@@ -270,12 +270,38 @@ class TestEmptyDirectory:
@pytest.mark.asyncio
async def test_empty_dir_returns_zero_files(self, tmp_empty: Path, parser, fake_fs) -> None:
- result = await parser.parse(str(tmp_empty))
+ blocked_path = tmp_empty / "blocked.pdf"
+ result = await parser.parse(
+ str(tmp_empty),
+ _source_meta={
+ "feishu_folder_skipped_items": [
+ {
+ "path": str(blocked_path),
+ "name": blocked_path.name,
+ "type": "file",
+ "token": "file-token",
+ "reason": "HTTP 403",
+ }
+ ]
+ },
+ )
assert result.parser_name == "DirectoryParser"
assert result.source_format == "directory"
assert result.temp_dir_path is not None
- assert result.meta.get("file_count", 0) == 0 or len(fake_fs.files) == 0
+ assert result.meta["file_count"] == 0
+ assert result.meta["total_processable"] == 0
+ assert result.meta["failed_files"] == [
+ {
+ "path": "blocked.pdf",
+ "parser": "feishu",
+ "status": "failed",
+ "type": "file",
+ "token": "file-token",
+ "reason": "HTTP 403",
+ }
+ ]
+ assert any("Skipped Feishu Drive item blocked.pdf: HTTP 403" in w for w in result.warnings)
# ---------------------------------------------------------------------------
diff --git a/tests/parse/test_feishu_accessor.py b/tests/parse/test_feishu_accessor.py
index b92eb114c..b022aba57 100644
--- a/tests/parse/test_feishu_accessor.py
+++ b/tests/parse/test_feishu_accessor.py
@@ -13,7 +13,6 @@ import pytest
from openviking.parse.accessors.feishu_accessor import (
_MAX_MEDIA_DOWNLOAD_CONTEXTS,
FeishuAccessor,
- _title_as_filename,
)
@@ -167,24 +166,50 @@ def _install_fake_lark_modules(monkeypatch):
monkeypatch.setitem(sys.modules, "lark_oapi.core.model", core_model)
-def test_fetch_all_blocks_uses_user_access_token_option(monkeypatch):
+def test_feishu_api_lists_paginated_content_with_user_token(monkeypatch):
_install_fake_lark_modules(monkeypatch)
list_blocks = MagicMock(
return_value=_SuccessResponse(
SimpleNamespace(items=[], has_more=False, page_token=None),
)
)
+ list_drive = MagicMock(
+ side_effect=[
+ _SuccessResponse(
+ SimpleNamespace(
+ files=[SimpleNamespace(token="doc_token")],
+ has_more=True,
+ next_page_token="page-2",
+ )
+ ),
+ _SuccessResponse(
+ SimpleNamespace(
+ files=[SimpleNamespace(token="file_token")],
+ has_more=False,
+ next_page_token=None,
+ )
+ ),
+ ]
+ )
accessor = FeishuAccessor()
accessor._user_token_client = SimpleNamespace(
- docx=SimpleNamespace(v1=SimpleNamespace(document_block=SimpleNamespace(list=list_blocks)))
+ docx=SimpleNamespace(v1=SimpleNamespace(document_block=SimpleNamespace(list=list_blocks))),
+ request=list_drive,
)
blocks = accessor._fetch_all_blocks("doc_token", feishu_access_token="u-test")
+ children = accessor._list_drive_folder_children(
+ "folder_token",
+ feishu_access_token="u-test",
+ )
assert blocks == []
+ assert [child.token for child in children] == ["doc_token", "file_token"]
request, option = list_blocks.call_args.args
assert request.document_id == "doc_token"
assert option.user_access_token == "u-test"
+ assert list_drive.call_args_list[1].args[0].queries["page_token"] == "page-2"
+ assert all(call.args[1].user_access_token == "u-test" for call in list_drive.call_args_list)
def test_resolve_image_refs_respects_download_images_disabled():
@@ -375,15 +400,96 @@ def test_download_image_advertises_user_token_when_provided(monkeypatch):
assert args[1].user_access_token == "u-test"
-def test_guess_image_ext_defaults_to_png_when_unknown():
+def test_access_downloads_drive_file_with_user_token(monkeypatch):
+ _install_fake_lark_modules(monkeypatch)
+ request = MagicMock(
+ return_value=_FakeMediaResponse(
+ b"%PDF-1.7",
+ headers={"content-type": "application/pdf"},
+ )
+ )
+ accessor = FeishuAccessor()
+ accessor._user_token_client = SimpleNamespace(request=request)
+ url = "https://bytedance.larkoffice.com/file/file_token"
+
+ assert accessor.can_handle(url)
+ resource = asyncio.run(accessor.access(url, feishu_access_token="u-test"))
+ try:
+ assert resource.path.read_bytes() == b"%PDF-1.7"
+ assert resource.path.name == "file_token.pdf"
+ assert resource.meta["feishu_doc_type"] == "file"
+ assert resource.meta["feishu_token"] == "file_token"
+ raw_request, option = request.call_args.args
+ assert raw_request.uri == "/open-apis/drive/v1/files/file_token/download"
+ assert option.user_access_token == "u-test"
+ finally:
+ resource.cleanup()
+
+
+def test_access_materializes_drive_folder_contract(monkeypatch):
+ from openviking.parse.accessors.feishu_accessor import FeishuDocument
+
accessor = FeishuAccessor()
- assert accessor._guess_image_ext(b"not-an-image", None) == ".png"
- assert accessor._guess_image_ext(b"\xff\xd8\xff", None) == ".jpg"
- assert accessor._guess_image_ext(b"anything", "image/gif") == ".gif"
+ accessor._config = SimpleNamespace(download_images=False)
+ long_name = "文" * 100
+ children = {
+ "root_folder": [
+ SimpleNamespace(token="doc_one", name=long_name, type="docx", url=""),
+ SimpleNamespace(token="doc_two", name=long_name, type="docx", url=""),
+ SimpleNamespace(token="nested", name="Nested", type="folder", url=""),
+ SimpleNamespace(token="design", name="Design.pdf", type="file", url=""),
+ SimpleNamespace(token="blocked", name="Blocked.pptx", type="file", url=""),
+ ],
+ "nested": [SimpleNamespace(token="sheet", name="Metrics", type="sheet", url="")],
+ }
+
+ async def fake_fetch_document(url, **_kwargs):
+ doc_type, token = accessor._parse_feishu_url(url)
+ return FeishuDocument(
+ doc_type=doc_type,
+ token=token,
+ markdown_content=f"# {token}",
+ title=token,
+ meta={},
+ )
+ def fake_download(file_token, **_kwargs):
+ if file_token == "blocked":
+ raise RuntimeError("HTTP 403")
+ return b"%PDF-1.7", "application/pdf", "Design.pdf"
-def test_title_as_filename_preserves_prefix_around_path_separators():
- assert _title_as_filename("API Docs/Overview\\v2") == "API Docs_Overview_v2"
+ monkeypatch.setattr(
+ accessor, "_get_drive_folder_name", lambda *_args, **_kwargs: "Product Docs"
+ )
+ monkeypatch.setattr(
+ accessor,
+ "_list_drive_folder_children",
+ lambda folder_token, **_kwargs: children[folder_token],
+ )
+ monkeypatch.setattr(accessor, "_fetch_document", fake_fetch_document)
+ monkeypatch.setattr(accessor, "_download_drive_file", fake_download)
+ url = "https://bytedance.larkoffice.com/drive/folder/root_folder"
+
+ assert accessor.can_handle(url)
+ resource = asyncio.run(accessor.access(url, feishu_access_token="u-test"))
+ try:
+ markdown_files = sorted(resource.path.glob("*.md"))
+ assert {path.read_text(encoding="utf-8") for path in markdown_files} == {
+ "# doc_one",
+ "# doc_two",
+ }
+ assert all(len(path.name.encode("utf-8")) <= 240 for path in markdown_files)
+ assert any(" (2).md" in path.name for path in markdown_files)
+ assert (resource.path / "Nested" / "Metrics.md").read_text(encoding="utf-8") == "# sheet"
+ assert (resource.path / "Design.pdf").read_bytes() == b"%PDF-1.7"
+ assert resource.meta["original_filename"] == "Product Docs"
+ assert resource.meta["feishu_doc_type"] == "folder"
+ skipped = resource.meta["feishu_folder_skipped_items"]
+ assert [(item["name"], item["token"], item["reason"]) for item in skipped] == [
+ ("Blocked.pptx", "blocked", "HTTP 403")
+ ]
+ finally:
+ resource.cleanup()
def test_access_offloads_synchronous_download_to_thread(monkeypatch):