Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(feishu):Support Feishu Drive folder and file imports (#3937)
zgyc1345a1f7e13
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.pyindex 81a7fc87a..e3dec074c 100644--- a/openviking/parse/accessors/feishu_accessor.py+++ b/openviking/parse/accessors/feishu_accessor.py@@ -14,11 +14,12 @@ import jsonimport mimetypesimport osimport re+import shutilimport tempfilefrom dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Dict, List, NoReturn, Optional, Tuple, Union-from urllib.parse import parse_qs, urlparse+from urllib.parse import parse_qs, unquote, urlparsefrom openviking.parse.base import format_table_to_markdownfrom openviking.utils.exceptions import error_code_from_http_status@@ -34,6 +35,18 @@ _FEISHU_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(feishu://image/([^)]+)\)")_FEISHU_DOCUMENT_FORBIDDEN = 1770032_FEISHU_BITABLE_PERMISSION_REQUIRED = 99991672_MAX_MEDIA_DOWNLOAD_CONTEXTS = 8+_FEISHU_DOC_PATH_TYPES = {"docx", "wiki", "sheets", "base"}+_FEISHU_DRIVE_DOC_TYPES = {+ "doc": "docx",+ "docx": "docx",+ "sheet": "sheets",+ "sheets": "sheets",+ "bitable": "base",+ "base": "base",+ "wiki": "wiki",+}+_MAX_PATH_SEGMENT_CHARS = 120+_MAX_PATH_SEGMENT_BYTES = 240_MediaDownloadExtras = Dict[str, List[Optional[str]]]@@ -48,6 +61,90 @@ def _title_as_filename(title: str) -> str:return title.replace("/", "_").replace("\\", "_")+def _truncate_text_for_path_segment(text: str, *, max_chars: int, max_bytes: int) -> str:+ """Trim text so its UTF-8 representation fits a single path segment budget."""+ if max_chars <= 0 or max_bytes <= 0:+ return ""+ result: list[str] = []+ used_bytes = 0+ for char in text[:max_chars]:+ char_bytes = len(char.encode("utf-8"))+ if used_bytes + char_bytes > max_bytes:+ break+ result.append(char)+ used_bytes += char_bytes+ return "".join(result)+++def _fit_path_segment(text: str, *, max_chars: int, max_bytes: int) -> str:+ """Ensure a complete path segment fits the configured character and byte budgets."""+ if len(text) <= max_chars and len(text.encode("utf-8")) <= max_bytes:+ return text+ return _truncate_text_for_path_segment(+ text,+ max_chars=max_chars,+ max_bytes=max_bytes,+ ).rstrip(" ._")+++def _safe_path_segment(+ name: str,+ *,+ fallback: str = "untitled",+ max_len: int = _MAX_PATH_SEGMENT_CHARS,+ max_bytes: int = _MAX_PATH_SEGMENT_BYTES,+) -> str:+ """Return one portable path segment while preserving readable names."""+ safe_name = re.sub(r"[\x00-\x1f/\\:*?\"<>|]+", "_", str(name or "")).strip(" ._")+ safe_name = re.sub(r"\s+", " ", safe_name)+ if not safe_name:+ safe_name = fallback+ if len(safe_name) <= max_len and len(safe_name.encode("utf-8")) <= max_bytes:+ return safe_name++ stem = Path(safe_name).stem+ suffix = Path(safe_name).suffix+ suffix_bytes = len(suffix.encode("utf-8"))+ if suffix_bytes >= max_bytes:+ suffix = ""+ suffix_bytes = 0++ stem = _truncate_text_for_path_segment(+ stem,+ max_chars=max_len - len(suffix),+ max_bytes=max_bytes - suffix_bytes,+ ).rstrip(" ._")+ if not stem:+ stem = _truncate_text_for_path_segment(+ fallback,+ max_chars=max_len - len(suffix),+ max_bytes=max_bytes - suffix_bytes,+ ).rstrip(" ._")+ return _fit_path_segment(+ f"{stem or 'untitled'}{suffix}",+ max_chars=max_len,+ max_bytes=max_bytes,+ ) or "untitled"+++def _numbered_path_segment(stem: str, suffix: str, index: int) -> str:+ marker = f" ({index})"+ marker_bytes = len(marker.encode("utf-8"))+ suffix_bytes = len(suffix.encode("utf-8"))+ max_stem_bytes = _MAX_PATH_SEGMENT_BYTES - marker_bytes - suffix_bytes+ max_stem_chars = _MAX_PATH_SEGMENT_CHARS - len(marker) - len(suffix)+ safe_stem = _truncate_text_for_path_segment(+ stem,+ max_chars=max_stem_chars,+ max_bytes=max_stem_bytes,+ ).rstrip(" ._")+ return _fit_path_segment(+ f"{safe_stem or 'untitled'}{marker}{suffix}",+ max_chars=_MAX_PATH_SEGMENT_CHARS,+ max_bytes=_MAX_PATH_SEGMENT_BYTES,+ ) or "untitled"++def _getattr_safe(obj, key: str, default=None):"""Get attribute from SDK object or dict, with safe fallback."""if isinstance(obj, dict):@@ -122,6 +219,8 @@ class FeishuAccessor(DataAccessor):- Wiki pages: https://*.feishu.cn/wiki/{token}- Spreadsheets: https://*.feishu.cn/sheets/{token}- Bitable: https://*.feishu.cn/base/{app_token}+ - Drive files: https://*.feishu.cn/file/{file_token}+ - Drive folders: https://*.feishu.cn/drive/folder/{folder_token}Requires:- lark-oapi package@@ -260,6 +359,64 @@ class FeishuAccessor(DataAccessor):feishu_access_token = kwargs.get("feishu_access_token")try:+ doc_type, token = self._parse_feishu_url(source_str)+ if doc_type == "file":+ content, content_type, filename = await asyncio.to_thread(+ self._download_drive_file,+ token,+ feishu_access_token=feishu_access_token,+ )+ local_path = self._write_temp_drive_file(+ token,+ content,+ content_type,+ filename_hint=filename,+ )+ return LocalResource(+ path=local_path,+ source_type=SourceType.FEISHU,+ original_source=source_str,+ meta={+ "feishu_doc_type": doc_type,+ "feishu_token": token,+ "original_filename": local_path.name,+ "_cleanup_path": str(local_path.parent),+ },+ is_temporary=True,+ )++ if doc_type == "folder":+ temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_folder_"))+ skipped_items: list[dict[str, Any]] = []+ folder_name = await asyncio.to_thread(+ self._drive_folder_display_name,+ token,+ feishu_access_token=feishu_access_token,+ )+ try:+ await self._materialize_drive_folder(+ token,+ temp_dir,+ feishu_access_token=feishu_access_token,+ skipped_items=skipped_items,+ strict=bool(kwargs.get("strict", False)),+ )+ except Exception:+ shutil.rmtree(temp_dir, ignore_errors=True)+ raise+ return LocalResource(+ path=temp_dir,+ source_type=SourceType.FEISHU,+ original_source=source_str,+ meta={+ "feishu_doc_type": doc_type,+ "feishu_token": token,+ "original_filename": _safe_path_segment(folder_name, fallback=token),+ "feishu_folder_skipped_items": skipped_items,+ },+ is_temporary=True,+ )+# Fetch the document and convert to Markdowndoc = await self._fetch_document(source_str,@@ -400,15 +557,17 @@ class FeishuAccessor(DataAccessor):"""Check if URL is a Feishu/Lark cloud document."""parsed = urlparse(url)host = (parsed.hostname or "").lower().rstrip(".")- path = parsed.path+ path_parts = [p for p in parsed.path.split("/") if p]is_feishu_domain = any(host == allowed_host or host.endswith(f".{allowed_host}")for allowed_host in ("feishu.cn", "larksuite.com", "larkoffice.com"))- has_doc_path = any(- path == f"/{t}" or path.startswith(f"/{t}/") for t in ("docx", "wiki", "sheets", "base")- )- return is_feishu_domain and has_doc_path+ if not is_feishu_domain or len(path_parts) < 2:+ return False+ has_doc_path = path_parts[0] in _FEISHU_DOC_PATH_TYPES+ has_drive_folder_path = len(path_parts) >= 3 and path_parts[:2] == ["drive", "folder"]+ has_file_path = path_parts[0] == "file"+ return is_feishu_domain and (has_doc_path or has_drive_folder_path or has_file_path)@staticmethoddef _parse_feishu_url(url: str) -> Tuple[str, str]:@@ -422,10 +581,466 @@ class FeishuAccessor(DataAccessor):path_parts = [p for p in parsed.path.split("/") if p]if len(path_parts) < 2:raise ValueError(f"Cannot parse Feishu URL: {url}")+ if len(path_parts) >= 3 and path_parts[:2] == ["drive", "folder"]:+ return "folder", path_parts[2]+ if path_parts[0] == "file":+ return "file", path_parts[1]doc_type = path_parts[0] # docx, wiki, sheets, basetoken = path_parts[1]return doc_type, token+ async def _materialize_drive_folder(+ self,+ folder_token: str,+ target_dir: Path,+ *,+ feishu_access_token: Optional[str] = None,+ _seen: Optional[set[str]] = None,+ skipped_items: Optional[list[dict[str, Any]]] = None,+ strict: bool = False,+ ) -> None:+ """Expand a Feishu Drive folder into a local directory tree."""+ target_dir.mkdir(parents=True, exist_ok=True)+ seen = _seen or set()+ if folder_token in seen:+ logger.warning("[FeishuAccessor] Skipping recursive Drive folder %s", folder_token)+ return+ seen.add(folder_token)++ try:+ children = await asyncio.to_thread(+ self._list_drive_folder_children,+ folder_token,+ feishu_access_token=feishu_access_token,+ )+ for item in children:+ item_type, item_token, item_name, item_url = self._normalize_drive_item(item)+ if not item_token:+ logger.warning("[FeishuAccessor] Skipping Drive item without token: %s", item)+ continue++ if item_type == "folder":+ folder_name = _safe_path_segment(item_name or item_token, fallback=item_token)+ child_dir = self._unique_child_path(target_dir, folder_name)+ try:+ await self._materialize_drive_folder(+ item_token,+ child_dir,+ feishu_access_token=feishu_access_token,+ _seen=seen,+ skipped_items=skipped_items,+ strict=strict,+ )+ except Exception as exc:+ shutil.rmtree(child_dir, ignore_errors=True)+ self._record_skipped_drive_item(+ skipped_items,+ item_type=item_type,+ token=item_token,+ name=item_name,+ target_dir=target_dir,+ error=exc,+ )+ if strict:+ raise+ continue++ doc_path_type = _FEISHU_DRIVE_DOC_TYPES.get(item_type)+ if doc_path_type:+ try:+ url = item_url or self._build_feishu_doc_url(doc_path_type, item_token)+ doc = await self._fetch_document(+ url,+ feishu_access_token=feishu_access_token,+ )+ markdown_content, downloaded_images = await asyncio.to_thread(+ self._resolve_image_refs,+ doc.markdown_content,+ feishu_access_token=feishu_access_token,+ media_download_extras=doc.media_download_extras,+ )+ doc_name = _safe_path_segment(+ item_name or doc.title or item_token,+ fallback=item_token,+ )+ markdown_path = self._unique_child_path(+ target_dir,+ self._markdown_file_name(doc_name),+ )+ markdown_path.write_text(markdown_content, encoding="utf-8")+ for rel_path, image_bytes in downloaded_images.items():+ image_path = markdown_path.parent / rel_path+ image_path.parent.mkdir(parents=True, exist_ok=True)+ image_path.write_bytes(image_bytes)+ except Exception as exc:+ self._record_skipped_drive_item(+ skipped_items,+ item_type=item_type,+ token=item_token,+ name=item_name,+ target_dir=target_dir,+ error=exc,+ )+ if strict:+ raise+ continue++ if item_type == "file":+ try:+ content, content_type, downloaded_name = await asyncio.to_thread(+ self._download_drive_file,+ item_token,+ feishu_access_token=feishu_access_token,+ filename_hint=item_name,+ )+ file_name = self._drive_file_name(+ item_token,+ content,+ content_type,+ filename_hint=downloaded_name or item_name,+ )+ file_path = self._unique_child_path(target_dir, file_name)+ file_path.write_bytes(content)+ except Exception as exc:+ self._record_skipped_drive_item(+ skipped_items,+ item_type=item_type,+ token=item_token,+ name=item_name,+ target_dir=target_dir,+ error=exc,+ )+ if strict:+ raise+ continue++ error = ValueError(f"Unsupported Feishu Drive item type: {item_type}")+ self._record_skipped_drive_item(+ skipped_items,+ item_type=item_type,+ token=item_token,+ name=item_name,+ target_dir=target_dir,+ error=error,+ )+ if strict:+ raise error++ finally:+ seen.discard(folder_token)++ @staticmethod+ def _record_skipped_drive_item(+ skipped_items: Optional[list[dict[str, Any]]],+ *,+ item_type: str,+ token: str,+ name: str,+ target_dir: Path,+ error: Exception,+ ) -> None:+ message = str(error).replace("\n", " ")+ logger.warning(+ "[FeishuAccessor] Skipping Drive %s %s under %s: %s",+ item_type,+ token,+ target_dir,+ message,+ )+ if skipped_items is None:+ return+ skipped_items.append(+ {+ "path": str(target_dir / _safe_path_segment(name or token, fallback=token)),+ "name": name or token,+ "type": item_type,+ "token": token,+ "reason": message,+ }+ )++ def _list_drive_folder_children(+ self,+ folder_token: str,+ *,+ feishu_access_token: Optional[str] = None,+ ) -> List[Any]:+ """List direct children under a Feishu Drive folder token."""+ import lark_oapi as lark++ client = self._get_client(use_user_token=bool(feishu_access_token))+ token_type = (+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT+ )+ all_children: List[Any] = []+ page_token = None++ while True:+ raw_req = (+ lark.BaseRequest.builder()+ .http_method(lark.HttpMethod.GET)+ .uri("/open-apis/drive/v1/files")+ .token_types({token_type})+ .build()+ )+ raw_req.add_query("folder_token", folder_token)+ raw_req.add_query("page_size", 200)+ if page_token:+ raw_req.add_query("page_token", page_token)++ response = self._call_api(client.request, raw_req, feishu_access_token)+ if not response.success():+ _raise_from_lark_response(+ response,+ operation=f"list Drive folder {folder_token}",+ resource=folder_token,+ )++ data = self._drive_response_data(response)+ items = _getattr_safe(data, "files", None) or _getattr_safe(data, "items", None) or []+ all_children.extend(items)++ has_more = bool(_getattr_safe(data, "has_more", False))+ if not has_more:+ break+ page_token = _getattr_safe(data, "next_page_token", None) or _getattr_safe(+ data, "page_token", None+ )+ if not page_token:+ raise RuntimeError(+ f"Feishu returned more Drive folder items for {folder_token} "+ "without a page token"+ )++ return all_children++ def _drive_folder_display_name(+ self,+ folder_token: str,+ *,+ feishu_access_token: Optional[str] = None,+ ) -> str:+ """Best-effort readable folder name for resource roots."""+ try:+ name = self._get_drive_folder_name(+ folder_token,+ feishu_access_token=feishu_access_token,+ )+ except Exception as exc:+ logger.warning(+ "[FeishuAccessor] Falling back to Drive folder token %s as name: %s",+ folder_token,+ exc,+ )+ return folder_token+ return name or folder_token++ def _get_drive_folder_name(+ self,+ folder_token: str,+ *,+ feishu_access_token: Optional[str] = None,+ ) -> Optional[str]:+ """Fetch a Feishu Drive folder display name by folder token."""+ import lark_oapi as lark++ client = self._get_client(use_user_token=bool(feishu_access_token))+ token_type = (+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT+ )+ raw_req = (+ lark.BaseRequest.builder()+ .http_method(lark.HttpMethod.GET)+ .uri(f"/open-apis/drive/explorer/v2/folder/{folder_token}/meta")+ .token_types({token_type})+ .build()+ )+ response = self._call_api(client.request, raw_req, feishu_access_token)+ if not response.success():+ _raise_from_lark_response(+ response,+ operation=f"fetch Drive folder metadata {folder_token}",+ resource=folder_token,+ )++ data = self._drive_response_data(response)+ folder_meta = _getattr_safe(data, "folder", None) or _getattr_safe(data, "meta", None)+ name = _getattr_safe(data, "name", None) or _getattr_safe(folder_meta, "name", None)+ return str(name) if name else None++ def _download_drive_file(+ self,+ file_token: str,+ *,+ feishu_access_token: Optional[str] = None,+ filename_hint: Optional[str] = None,+ ) -> Tuple[bytes, Optional[str], Optional[str]]:+ """Download a Feishu Drive binary file by file token."""+ import lark_oapi as lark++ client = self._get_client(use_user_token=bool(feishu_access_token))+ token_type = (+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT+ )+ raw_req = (+ lark.BaseRequest.builder()+ .http_method(lark.HttpMethod.GET)+ .uri(f"/open-apis/drive/v1/files/{file_token}/download")+ .token_types({token_type})+ .build()+ )+ response = self._call_api(client.request, raw_req, feishu_access_token)+ if not response.success():+ _raise_from_lark_response(+ response,+ operation=f"download Drive file {file_token}",+ resource=file_token,+ )++ raw = getattr(response, "raw", None)+ content = getattr(raw, "content", None)+ if content is None:+ raise OpenVikingError(+ f"Feishu Drive file download returned empty content: {file_token}",+ code="NOT_FOUND",+ details={"operation": "download Drive file", "resource": file_token},+ )+ if isinstance(content, str):+ content = content.encode("utf-8")++ content_type = self._response_content_type(raw)+ filename = self._filename_from_content_disposition(+ self._response_header(raw, "content-disposition")+ )+ return content, content_type, filename or filename_hint++ def _write_temp_drive_file(+ self,+ file_token: str,+ content: bytes,+ content_type: Optional[str],+ *,+ filename_hint: Optional[str] = None,+ ) -> Path:+ filename = self._drive_file_name(+ file_token,+ content,+ content_type,+ filename_hint=filename_hint,+ )+ temp_dir = Path(tempfile.mkdtemp(prefix="ov_feishu_file_"))+ path = temp_dir / filename+ path.write_bytes(content)+ return path++ @staticmethod+ def _drive_response_data(response: Any) -> Any:+ data = getattr(response, "data", None)+ if data is not None:+ return data+ raw_content = getattr(getattr(response, "raw", None), "content", None)+ if not raw_content:+ return {}+ if isinstance(raw_content, bytes):+ raw_content = raw_content.decode("utf-8")+ return json.loads(raw_content).get("data", {})++ @classmethod+ def _normalize_drive_item(cls, item: Any) -> Tuple[str, str, str, str]:+ item_type = str(_getattr_safe(item, "type", "") or "").lower()+ token = str(_getattr_safe(item, "token", "") or "")+ name = str(_getattr_safe(item, "name", "") or "")+ url = str(_getattr_safe(item, "url", "") or "")+ shortcut_info = _getattr_safe(item, "shortcut_info", None)+ if shortcut_info:+ item_type = str(+ _getattr_safe(shortcut_info, "target_type", item_type) or item_type+ ).lower()+ token = str(_getattr_safe(shortcut_info, "target_token", token) or token)+ return item_type, token, name, url++ @staticmethod+ def _build_feishu_doc_url(doc_type: str, token: str) -> str:+ return f"https://open.feishu.cn/{doc_type}/{token}"++ @staticmethod+ def _unique_child_path(parent: Path, name: str) -> Path:+ path = parent / _safe_path_segment(name)+ if not path.exists():+ return path+ suffix = path.suffix+ stem = path.stem+ for index in range(2, 10000):+ candidate = parent / _numbered_path_segment(stem, suffix, index)+ if not candidate.exists():+ return candidate+ raise RuntimeError(f"Unable to allocate unique path under {parent}")++ @classmethod+ def _drive_file_name(+ cls,+ file_token: str,+ content: bytes,+ content_type: Optional[str],+ *,+ filename_hint: Optional[str] = None,+ ) -> str:+ raw_name = filename_hint or file_token+ if Path(raw_name).suffix:+ return _safe_path_segment(raw_name, fallback=file_token)+ ext = cls._guess_drive_file_ext(content, content_type)+ return _safe_path_segment(f"{raw_name}{ext}", fallback=f"{file_token}{ext}")++ @staticmethod+ def _markdown_file_name(name: str) -> str:+ if str(name).lower().endswith((".md", ".markdown")):+ return _safe_path_segment(name)+ return _safe_path_segment(f"{name}.md")++ @staticmethod+ def _guess_drive_file_ext(content: bytes, content_type: Optional[str]) -> str:+ if content.startswith(b"%PDF-"):+ return ".pdf"+ if (+ content.startswith(b"PK\x03\x04")+ or content.startswith(b"PK\x05\x06")+ or content.startswith(b"PK\x07\x08")+ ):+ return ".zip"+ if content.startswith(b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):+ return ".doc"+ if content_type:+ ext = get_preferred_extension(content_type)+ if ext:+ return ext+ return ".bin"++ @staticmethod+ def _response_header(raw: Any, name: str) -> Optional[str]:+ headers = getattr(raw, "headers", None)+ if not headers:+ return None+ try:+ get = headers.get+ except AttributeError:+ return None+ return get(name) or get(name.title()) or get(name.lower())++ @staticmethod+ def _filename_from_content_disposition(content_disposition: Optional[str]) -> Optional[str]:+ if not content_disposition:+ return None+ utf8_match = re.search(r"filename\*=UTF-8''([^;]+)", content_disposition, re.I)+ if utf8_match:+ return unquote(utf8_match.group(1))+ quoted_match = re.search(r'filename="([^"]+)"', content_disposition, re.I)+ if quoted_match:+ return quoted_match.group(1)+ simple_match = re.search(r"filename=([^;]+)", content_disposition, re.I)+ if simple_match:+ return simple_match.group(1).strip()+ return None+# ========== Configuration & Client ==========def _get_config(self):diff --git a/openviking/parse/parsers/directory.py b/openviking/parse/parsers/directory.pyindex f01291703..2c526d5df 100644--- a/openviking/parse/parsers/directory.py+++ b/openviking/parse/parsers/directory.py@@ -142,6 +142,14 @@ class DirectoryParser(BaseParser):preserve_structure = Trueprocessable_files = scan_result.all_processable_files()warnings.extend(scan_result.warnings)+ source_skipped_items = self._source_skipped_items(+ kwargs.get("_source_meta"),+ source_path,+ )+ warnings.extend(+ f"Skipped Feishu Drive item {item['path']}: {item.get('reason', 'unknown error')}"+ for item in source_skipped_items+ )viking_fs = self._get_viking_fs()temp_uri = self._create_temp_uri()@@ -164,6 +172,13 @@ class DirectoryParser(BaseParser):warnings=warnings,)result.temp_dir_path = temp_uri+ result.meta["file_count"] = 0+ result.meta["dir_name"] = dir_name+ result.meta["total_processable"] = 0+ result.meta["processed_files"] = []+ result.meta["failed_files"] = source_skipped_items+ result.meta["unsupported_files"] = []+ result.meta["skipped_files"] = []return result# ── Phase 2: process each file ────────────────────────────@@ -259,7 +274,7 @@ class DirectoryParser(BaseParser):result.meta["dir_name"] = dir_nameresult.meta["total_processable"] = len(processable_files)result.meta["processed_files"] = processed_files- result.meta["failed_files"] = failed_files+ result.meta["failed_files"] = failed_files + source_skipped_itemsresult.meta["unsupported_files"] = unsupported_filesresult.meta["skipped_files"] = skipped_files@@ -329,6 +344,41 @@ class DirectoryParser(BaseParser):result.append({"path": path, "status": status})return result+ @staticmethod+ def _source_skipped_items(source_meta: Any, source_path: Path) -> List[Dict[str, str]]:+ """Normalize skipped items reported by a remote source accessor."""+ if not isinstance(source_meta, dict):+ return []+ items = source_meta.get("feishu_folder_skipped_items") or []+ if not isinstance(items, list):+ return []++ normalized: List[Dict[str, str]] = []+ for item in items:+ if not isinstance(item, dict):+ continue+ raw_path = str(item.get("path") or item.get("name") or item.get("token") or "unknown")+ display_path = raw_path+ try:+ path = Path(raw_path).resolve(strict=False)+ if path.is_absolute():+ display_path = str(path.relative_to(source_path.resolve(strict=False)))+ except Exception:+ pass+ display_path = display_path.replace("\\", "/")++ normalized.append(+ {+ "path": display_path,+ "parser": "feishu",+ "status": "failed",+ "type": str(item.get("type") or ""),+ "token": str(item.get("token") or ""),+ "reason": str(item.get("reason") or "unknown error"),+ }+ )+ return normalized+# ------------------------------------------------------------------# Parser assignment# ------------------------------------------------------------------diff --git a/tests/parse/test_add_directory.py b/tests/parse/test_add_directory.pyindex 70269b62d..3325e5332 100644--- a/tests/parse/test_add_directory.py+++ b/tests/parse/test_add_directory.py@@ -270,12 +270,38 @@ class TestEmptyDirectory:@pytest.mark.asyncioasync def test_empty_dir_returns_zero_files(self, tmp_empty: Path, parser, fake_fs) -> None:- result = await parser.parse(str(tmp_empty))+ blocked_path = tmp_empty / "blocked.pdf"+ result = await parser.parse(+ str(tmp_empty),+ _source_meta={+ "feishu_folder_skipped_items": [+ {+ "path": str(blocked_path),+ "name": blocked_path.name,+ "type": "file",+ "token": "file-token",+ "reason": "HTTP 403",+ }+ ]+ },+ )assert result.parser_name == "DirectoryParser"assert result.source_format == "directory"assert result.temp_dir_path is not None- assert result.meta.get("file_count", 0) == 0 or len(fake_fs.files) == 0+ assert result.meta["file_count"] == 0+ assert result.meta["total_processable"] == 0+ assert result.meta["failed_files"] == [+ {+ "path": "blocked.pdf",+ "parser": "feishu",+ "status": "failed",+ "type": "file",+ "token": "file-token",+ "reason": "HTTP 403",+ }+ ]+ assert any("Skipped Feishu Drive item blocked.pdf: HTTP 403" in w for w in result.warnings)# ---------------------------------------------------------------------------diff --git a/tests/parse/test_feishu_accessor.py b/tests/parse/test_feishu_accessor.pyindex b92eb114c..b022aba57 100644--- a/tests/parse/test_feishu_accessor.py+++ b/tests/parse/test_feishu_accessor.py@@ -13,7 +13,6 @@ import pytestfrom openviking.parse.accessors.feishu_accessor import (_MAX_MEDIA_DOWNLOAD_CONTEXTS,FeishuAccessor,- _title_as_filename,)@@ -167,24 +166,50 @@ def _install_fake_lark_modules(monkeypatch):monkeypatch.setitem(sys.modules, "lark_oapi.core.model", core_model)-def test_fetch_all_blocks_uses_user_access_token_option(monkeypatch):+def test_feishu_api_lists_paginated_content_with_user_token(monkeypatch):_install_fake_lark_modules(monkeypatch)list_blocks = MagicMock(return_value=_SuccessResponse(SimpleNamespace(items=[], has_more=False, page_token=None),))+ list_drive = MagicMock(+ side_effect=[+ _SuccessResponse(+ SimpleNamespace(+ files=[SimpleNamespace(token="doc_token")],+ has_more=True,+ next_page_token="page-2",+ )+ ),+ _SuccessResponse(+ SimpleNamespace(+ files=[SimpleNamespace(token="file_token")],+ has_more=False,+ next_page_token=None,+ )+ ),+ ]+ )accessor = FeishuAccessor()accessor._user_token_client = SimpleNamespace(- docx=SimpleNamespace(v1=SimpleNamespace(document_block=SimpleNamespace(list=list_blocks)))+ docx=SimpleNamespace(v1=SimpleNamespace(document_block=SimpleNamespace(list=list_blocks))),+ request=list_drive,)blocks = accessor._fetch_all_blocks("doc_token", feishu_access_token="u-test")+ children = accessor._list_drive_folder_children(+ "folder_token",+ feishu_access_token="u-test",+ )assert blocks == []+ assert [child.token for child in children] == ["doc_token", "file_token"]request, option = list_blocks.call_args.argsassert request.document_id == "doc_token"assert option.user_access_token == "u-test"+ assert list_drive.call_args_list[1].args[0].queries["page_token"] == "page-2"+ assert all(call.args[1].user_access_token == "u-test" for call in list_drive.call_args_list)def test_resolve_image_refs_respects_download_images_disabled():@@ -375,15 +400,96 @@ def test_download_image_advertises_user_token_when_provided(monkeypatch):assert args[1].user_access_token == "u-test"-def test_guess_image_ext_defaults_to_png_when_unknown():+def test_access_downloads_drive_file_with_user_token(monkeypatch):+ _install_fake_lark_modules(monkeypatch)+ request = MagicMock(+ return_value=_FakeMediaResponse(+ b"%PDF-1.7",+ headers={"content-type": "application/pdf"},+ )+ )+ accessor = FeishuAccessor()+ accessor._user_token_client = SimpleNamespace(request=request)+ url = "https://bytedance.larkoffice.com/file/file_token"++ assert accessor.can_handle(url)+ resource = asyncio.run(accessor.access(url, feishu_access_token="u-test"))+ try:+ assert resource.path.read_bytes() == b"%PDF-1.7"+ assert resource.path.name == "file_token.pdf"+ assert resource.meta["feishu_doc_type"] == "file"+ assert resource.meta["feishu_token"] == "file_token"+ raw_request, option = request.call_args.args+ assert raw_request.uri == "/open-apis/drive/v1/files/file_token/download"+ assert option.user_access_token == "u-test"+ finally:+ resource.cleanup()+++def test_access_materializes_drive_folder_contract(monkeypatch):+ from openviking.parse.accessors.feishu_accessor import FeishuDocument+accessor = FeishuAccessor()- assert accessor._guess_image_ext(b"not-an-image", None) == ".png"- assert accessor._guess_image_ext(b"\xff\xd8\xff", None) == ".jpg"- assert accessor._guess_image_ext(b"anything", "image/gif") == ".gif"+ accessor._config = SimpleNamespace(download_images=False)+ long_name = "文" * 100+ children = {+ "root_folder": [+ SimpleNamespace(token="doc_one", name=long_name, type="docx", url=""),+ SimpleNamespace(token="doc_two", name=long_name, type="docx", url=""),+ SimpleNamespace(token="nested", name="Nested", type="folder", url=""),+ SimpleNamespace(token="design", name="Design.pdf", type="file", url=""),+ SimpleNamespace(token="blocked", name="Blocked.pptx", type="file", url=""),+ ],+ "nested": [SimpleNamespace(token="sheet", name="Metrics", type="sheet", url="")],+ }++ async def fake_fetch_document(url, **_kwargs):+ doc_type, token = accessor._parse_feishu_url(url)+ return FeishuDocument(+ doc_type=doc_type,+ token=token,+ markdown_content=f"# {token}",+ title=token,+ meta={},+ )+ def fake_download(file_token, **_kwargs):+ if file_token == "blocked":+ raise RuntimeError("HTTP 403")+ return b"%PDF-1.7", "application/pdf", "Design.pdf"-def test_title_as_filename_preserves_prefix_around_path_separators():- assert _title_as_filename("API Docs/Overview\\v2") == "API Docs_Overview_v2"+ monkeypatch.setattr(+ accessor, "_get_drive_folder_name", lambda *_args, **_kwargs: "Product Docs"+ )+ monkeypatch.setattr(+ accessor,+ "_list_drive_folder_children",+ lambda folder_token, **_kwargs: children[folder_token],+ )+ monkeypatch.setattr(accessor, "_fetch_document", fake_fetch_document)+ monkeypatch.setattr(accessor, "_download_drive_file", fake_download)+ url = "https://bytedance.larkoffice.com/drive/folder/root_folder"++ assert accessor.can_handle(url)+ resource = asyncio.run(accessor.access(url, feishu_access_token="u-test"))+ try:+ markdown_files = sorted(resource.path.glob("*.md"))+ assert {path.read_text(encoding="utf-8") for path in markdown_files} == {+ "# doc_one",+ "# doc_two",+ }+ assert all(len(path.name.encode("utf-8")) <= 240 for path in markdown_files)+ assert any(" (2).md" in path.name for path in markdown_files)+ assert (resource.path / "Nested" / "Metrics.md").read_text(encoding="utf-8") == "# sheet"+ assert (resource.path / "Design.pdf").read_bytes() == b"%PDF-1.7"+ assert resource.meta["original_filename"] == "Product Docs"+ assert resource.meta["feishu_doc_type"] == "folder"+ skipped = resource.meta["feishu_folder_skipped_items"]+ assert [(item["name"], item["token"], item["reason"]) for item in skipped] == [+ ("Blocked.pptx", "blocked", "HTTP 403")+ ]+ finally:+ resource.cleanup()def test_access_offloads_synchronous_download_to_thread(monkeypatch):