Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(feishu): support legacy doc imports (#4130)
Qin Haojie592c0fe0369a
diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.mdindex 4275eb95a..3904a196c 100644--- a/docs/en/api/02-resources.md+++ b/docs/en/api/02-resources.md@@ -47,7 +47,7 @@ OpenViking supports various resource types, categorized by functionality:| Type | Description ||------|-------------|-| Feishu/Lark | URL-based, supports docx, wiki, sheets, bitable. By default uses app credentials from FEISHU_APP_ID and FEISHU_APP_SECRET; user-token imports can pass `args.feishu_access_token`, and user-token watches also pass `args.feishu_refresh_token` |+| Feishu/Lark | URL-based, supports doc/docx, wiki, sheets, bitable. By default uses app credentials from FEISHU_APP_ID and FEISHU_APP_SECRET; user-token imports can pass `args.feishu_access_token`, and user-token watches also pass `args.feishu_refresh_token` |**Web Pages (recursive web crawler)**diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.mdindex 83437c9d0..3ca4925e4 100644--- a/docs/zh/api/02-resources.md+++ b/docs/zh/api/02-resources.md@@ -42,7 +42,7 @@ OpenViking 支持多种资源类型,按照功能分类如下:云文档类| 类型 | 说明 ||------|------|-| 飞书/Lark | URL 方式,支持 docx, wiki, sheets, bitable。默认使用 FEISHU_APP_ID 和 FEISHU_APP_SECRET 应用凭证;用户 token 导入可传 `args.feishu_access_token`,用户 token watch 还需传 `args.feishu_refresh_token` |+| 飞书/Lark | URL 方式,支持 doc/docx, wiki, sheets, bitable。默认使用 FEISHU_APP_ID 和 FEISHU_APP_SECRET 应用凭证;用户 token 导入可传 `args.feishu_access_token`,用户 token watch 还需传 `args.feishu_refresh_token` |网页类(递归网页爬虫)| 类型 | 资源名 | 说明 |diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.pyindex 73ec41ace..2324a2377 100644--- a/openviking/parse/accessors/feishu_accessor.py+++ b/openviking/parse/accessors/feishu_accessor.py@@ -35,10 +35,19 @@ _FEISHU_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(feishu://image/([^)]+)\)")_FEISHU_DOCUMENT_FORBIDDEN = 1770032_FEISHU_WIKI_NODE_PERMISSION_DENIED = 131006_FEISHU_BITABLE_PERMISSION_REQUIRED = 99991672+_FEISHU_LEGACY_DOC_LOGIN_REQUIRED = 91404+_FEISHU_PERMISSION_DENIED_CODES = {+ _FEISHU_DOCUMENT_FORBIDDEN,+ _FEISHU_WIKI_NODE_PERMISSION_DENIED,+ 91403,+ 95008,+ 95009,+}+_FEISHU_NOT_FOUND_CODES = {91402, 95006, 95007}_MAX_MEDIA_DOWNLOAD_CONTEXTS = 8-_FEISHU_DOC_PATH_TYPES = {"docx", "wiki", "sheets", "base"}+_FEISHU_DOC_PATH_TYPES = {"doc", "docs", "docx", "wiki", "sheets", "base"}_FEISHU_DRIVE_DOC_TYPES = {- "doc": "docx",+ "doc": "docs","docx": "docx","sheet": "sheets","sheets": "sheets",@@ -189,11 +198,14 @@ def _raise_from_lark_response(f"Feishu application is missing required Bitable permissions: code={code}, msg={msg}")else:- public_code = (- "PERMISSION_DENIED"- if code in {_FEISHU_DOCUMENT_FORBIDDEN, _FEISHU_WIKI_NODE_PERMISSION_DENIED}- else error_code_from_http_status(http_status)- )+ if code in _FEISHU_PERMISSION_DENIED_CODES:+ public_code = "PERMISSION_DENIED"+ elif code in _FEISHU_NOT_FOUND_CODES:+ public_code = "NOT_FOUND"+ elif code == _FEISHU_LEGACY_DOC_LOGIN_REQUIRED:+ public_code = "UNAUTHENTICATED"+ else:+ public_code = error_code_from_http_status(http_status)message = f"Feishu {operation} failed: code={code}, msg={msg}"raise OpenVikingError(message, code=public_code, details=details)@@ -227,6 +239,7 @@ class FeishuAccessor(DataAccessor):Supports:- Documents: https://*.feishu.cn/docx/{document_id}+ - Legacy documents: https://*.feishu.cn/docs/{doc_token}- Wiki pages: https://*.feishu.cn/wiki/{token}- Spreadsheets: https://*.feishu.cn/sheets/{token}- Bitable: https://*.feishu.cn/base/{app_token}@@ -243,8 +256,9 @@ class FeishuAccessor(DataAccessor):PRIORITY = 100 # Higher than Git/HTTP, very specific# Wiki obj_type normalization (API returns short names)- _WIKI_TYPE_MAP = {"doc": "docx", "sheet": "sheets", "bitable": "base"}+ _WIKI_TYPE_MAP = {"doc": "doc", "sheet": "sheets", "bitable": "base"}_DOC_TYPE_HANDLERS = {+ "doc": "_parse_legacy_doc","docx": "_parse_docx","sheets": "_parse_sheets","base": "_parse_bitable",@@ -573,6 +587,13 @@ class FeishuAccessor(DataAccessor):table_id: Optional[str],view_id: Optional[str],) -> Optional[str]:+ if doc_type == "doc":+ metadata = self._fetch_legacy_doc_metadata(+ token,+ feishu_access_token=feishu_access_token,+ )+ title = metadata.get("title") or None+ return _title_as_filename(str(title)) if title else Noneif doc_type == "docx":self._probe_docx_document(token, feishu_access_token=feishu_access_token)return None@@ -732,7 +753,7 @@ class FeishuAccessor(DataAccessor):return "folder", path_parts[2]if path_parts[0] == "file":return "file", path_parts[1]- doc_type = path_parts[0] # docx, wiki, sheets, base+ doc_type = "doc" if path_parts[0] in {"doc", "docs"} else path_parts[0]token = path_parts[1]return doc_type, token@@ -940,7 +961,7 @@ class FeishuAccessor(DataAccessor):resource=folder_token,)- data = self._drive_response_data(response)+ data = self._raw_response_data(response)items = _getattr_safe(data, "files", None) or _getattr_safe(data, "items", None) or []has_more = bool(_getattr_safe(data, "has_more", False))next_page_token = _getattr_safe(data, "next_page_token", None) or _getattr_safe(@@ -1038,7 +1059,7 @@ class FeishuAccessor(DataAccessor):resource=folder_token,)- data = self._drive_response_data(response)+ data = self._raw_response_data(response)folder_meta = _getattr_safe(data, "folder", None) or _getattr_safe(data, "meta", None)name = _getattr_safe(data, "name", None) or _getattr_safe(folder_meta, "name", None)return str(name) if name else None@@ -1109,7 +1130,7 @@ class FeishuAccessor(DataAccessor):return path@staticmethod- def _drive_response_data(response: Any) -> Any:+ def _raw_response_data(response: Any) -> Any:data = getattr(response, "data", None)if data is not None:return data@@ -1136,6 +1157,8 @@ class FeishuAccessor(DataAccessor):@staticmethoddef _build_feishu_doc_url(doc_type: str, token: str) -> str:+ if doc_type == "doc":+ doc_type = "docs"return f"https://open.feishu.cn/{doc_type}/{token}"@staticmethod@@ -1306,6 +1329,88 @@ class FeishuAccessor(DataAccessor):return doc_type, obj_token, title+ # ========== Legacy Doc Parsing ==========++ def _parse_legacy_doc(+ self,+ doc_token: str,+ feishu_access_token: Optional[str] = None,+ ) -> Tuple[str, str]:+ metadata = self._fetch_legacy_doc_metadata(+ doc_token,+ feishu_access_token=feishu_access_token,+ )+ title = str(metadata.get("title") or "Untitled")+ content = self._fetch_legacy_doc_raw_content(+ doc_token,+ feishu_access_token=feishu_access_token,+ ).strip()++ if title and title != "Untitled":+ markdown = f"# {title}\n\n{content}" if content else f"# {title}"+ else:+ markdown = content+ return markdown, title++ def _fetch_legacy_doc_metadata(+ self,+ doc_token: str,+ *,+ feishu_access_token: Optional[str] = None,+ ) -> Dict[str, Any]:+ import lark_oapi as lark++ client = self._get_client(use_user_token=bool(feishu_access_token))+ token_type = (+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT+ )+ request = (+ lark.BaseRequest.builder()+ .http_method(lark.HttpMethod.GET)+ .uri(f"/open-apis/doc/v2/meta/{doc_token}")+ .token_types({token_type})+ .build()+ )+ response = self._call_api(client.request, request, feishu_access_token)+ if not response.success():+ _raise_from_lark_response(+ response,+ operation=f"fetch legacy document metadata for {doc_token}",+ resource=doc_token,+ )+ data = self._raw_response_data(response)+ return data if isinstance(data, dict) else {}++ def _fetch_legacy_doc_raw_content(+ self,+ doc_token: str,+ *,+ feishu_access_token: Optional[str] = None,+ ) -> str:+ import lark_oapi as lark++ client = self._get_client(use_user_token=bool(feishu_access_token))+ token_type = (+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT+ )+ request = (+ lark.BaseRequest.builder()+ .http_method(lark.HttpMethod.GET)+ .uri(f"/open-apis/doc/v2/{doc_token}/raw_content")+ .token_types({token_type})+ .build()+ )+ response = self._call_api(client.request, request, feishu_access_token)+ if not response.success():+ _raise_from_lark_response(+ response,+ operation=f"fetch legacy document content for {doc_token}",+ resource=doc_token,+ )+ data = self._raw_response_data(response)+ content = _getattr_safe(data, "content", "")+ return str(content or "")+# ========== Docx Parsing ==========def _parse_docx(diff --git a/tests/parse/test_feishu_accessor.py b/tests/parse/test_feishu_accessor.pyindex 033f8c2a7..9d79e1ea2 100644--- a/tests/parse/test_feishu_accessor.py+++ b/tests/parse/test_feishu_accessor.py@@ -154,6 +154,8 @@ def _install_fake_lark_modules(monkeypatch):lark.AccessTokenType = SimpleNamespace(TENANT="tenant", USER="user")docx_v1 = ModuleType("lark_oapi.api.docx.v1")docx_v1.ListDocumentBlockRequest = _FakeListDocumentBlockRequest+ wiki_v2 = ModuleType("lark_oapi.api.wiki.v2")+ wiki_v2.GetNodeSpaceRequest = _FakeTypedRequestbitable_v1 = ModuleType("lark_oapi.api.bitable.v1")bitable_v1.ListAppTableRequest = _FakeTypedRequestbitable_v1.ListAppTableFieldRequest = _FakeTypedRequest@@ -162,6 +164,7 @@ def _install_fake_lark_modules(monkeypatch):core_model.RequestOption = _FakeRequestOptionmonkeypatch.setitem(sys.modules, "lark_oapi", lark)monkeypatch.setitem(sys.modules, "lark_oapi.api.docx.v1", docx_v1)+ monkeypatch.setitem(sys.modules, "lark_oapi.api.wiki.v2", wiki_v2)monkeypatch.setitem(sys.modules, "lark_oapi.api.bitable.v1", bitable_v1)monkeypatch.setitem(sys.modules, "lark_oapi.core.model", core_model)@@ -435,6 +438,7 @@ def test_access_materializes_drive_folder_contract(monkeypatch):children = {"root_folder": [SimpleNamespace(token="doc_one", name=long_name, type="docx", url=""),+ SimpleNamespace(token="doc_legacy", name="Legacy", type="doc", url=""),SimpleNamespace(token="doc_two", name=long_name, type="docx", url=""),SimpleNamespace(token="nested", name="Nested", type="folder", url=""),SimpleNamespace(token="design", name="Design.pdf", type="file", url=""),@@ -445,6 +449,8 @@ def test_access_materializes_drive_folder_contract(monkeypatch):async def fake_fetch_document(url, **_kwargs):doc_type, token = accessor._parse_feishu_url(url)+ if token == "doc_legacy":+ assert doc_type == "doc"return FeishuDocument(doc_type=doc_type,token=token,@@ -476,6 +482,7 @@ def test_access_materializes_drive_folder_contract(monkeypatch):markdown_files = sorted(resource.path.glob("*.md"))assert {path.read_text(encoding="utf-8") for path in markdown_files} == {"# doc_one",+ "# doc_legacy","# doc_two",}assert all(len(path.name.encode("utf-8")) <= 240 for path in markdown_files)@@ -582,6 +589,7 @@ def test_access_writes_downloaded_images_next_to_markdown(monkeypatch):def test_fetch_document_dispatches_all_supported_types(monkeypatch):+ _install_fake_lark_modules(monkeypatch)accessor = FeishuAccessor()handlers = {"_parse_docx": MagicMock(return_value=("docx body", "Doc")),@@ -591,6 +599,49 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):for name, handler in handlers.items():monkeypatch.setattr(accessor, name, handler)+ def raw_doc_response(data):+ return _FakeMediaResponse(content=json.dumps({"data": data}))++ legacy_responses = {+ "/open-apis/doc/v2/meta/doccn_token": raw_doc_response({"title": "Legacy Doc"}),+ "/open-apis/doc/v2/doccn_token/raw_content": raw_doc_response({"content": "doc body"}),+ }+ raw_request = MagicMock(side_effect=lambda request, _option: legacy_responses[request.uri])+ get_wiki_node = MagicMock(+ return_value=_SuccessResponse(+ SimpleNamespace(+ node=SimpleNamespace(+ obj_type="doc",+ obj_token="doccn_from_wiki",+ title="Wiki Legacy",+ )+ )+ )+ )+ accessor._user_token_client = SimpleNamespace(+ request=raw_request,+ wiki=SimpleNamespace(v2=SimpleNamespace(space=SimpleNamespace(get_node=get_wiki_node))),+ )++ assert accessor._resolve_wiki_node("wiki_token", "u-test") == (+ "doc",+ "doccn_from_wiki",+ "Wiki Legacy",+ )++ assert accessor.can_handle("https://example.feishu.cn/doc/doccn_token")+ assert accessor.can_handle("https://example.feishu.cn/docs/doccn_token")+ assert accessor._parse_feishu_url("https://example.feishu.cn/docs/doccn_token") == (+ "doc",+ "doccn_token",+ )++ legacy_doc = asyncio.run(+ accessor._fetch_document(+ "https://example.feishu.cn/docs/doccn_token",+ feishu_access_token="u-test",+ )+ )docx = asyncio.run(accessor._fetch_document("https://example.feishu.cn/docx/doc_token",@@ -606,12 +657,21 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):)wiki = asyncio.run(accessor._fetch_document("https://example.feishu.cn/wiki/wiki_token"))- assert (docx.doc_type, sheets.doc_type, base.doc_type, wiki.doc_type) == (+ assert (+ legacy_doc.doc_type,+ docx.doc_type,+ sheets.doc_type,+ base.doc_type,+ wiki.doc_type,+ ) == (+ "doc","docx","sheets","base","base",)+ assert legacy_doc.title == "Legacy Doc"+ assert legacy_doc.markdown_content == "# Legacy Doc\n\ndoc body"assert wiki.title == "Wiki Base"handlers["_parse_docx"].assert_called_once_with("doc_token", "u-test")handlers["_parse_sheets"].assert_called_once_with(@@ -622,6 +682,11 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):assert handlers["_parse_bitable"].call_args_list[-1].args == ("wiki_app_token", None)monkeypatch.setattr(accessor, "_probe_docx_document", MagicMock())+ monkeypatch.setattr(+ accessor,+ "_fetch_legacy_doc_metadata",+ MagicMock(return_value={"title": "Legacy/Title"}),+ )monkeypatch.setattr(accessor,"_fetch_spreadsheet_metadata",@@ -640,7 +705,13 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):monkeypatch.setattr(accessor, "_probe_bitable_table", MagicMock())monkeypatch.setattr(accessor, "_get_drive_folder_name", MagicMock(return_value="Docs/Root"))monkeypatch.setattr(accessor, "_probe_drive_folder_children", MagicMock())+ monkeypatch.setattr(+ accessor,+ "_resolve_wiki_node",+ MagicMock(return_value=("base", "wiki_app_token", "Wiki Base")),+ )+ legacy_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/doc/doccn"))docx_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/docx/doc"))sheets_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/sheets/sht")@@ -654,6 +725,8 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):)wiki_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/wiki/wiki"))+ assert legacy_identity.doc_type == "doc"+ assert legacy_identity.source_name == "Legacy_Title"assert docx_identity.source_name is Noneassert sheets_identity.source_name == "Sheet_Title"assert base_identity.source_name == "Bitable (2 tables)"