Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(feishu): preserve Bitable media permission context during import (#3558)
Evo16e7c33f1f03
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.pyindex df8a30db9..81a7fc87a 100644--- a/openviking/parse/accessors/feishu_accessor.py+++ b/openviking/parse/accessors/feishu_accessor.py@@ -15,7 +15,7 @@ import mimetypesimport osimport reimport tempfile-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Dict, List, NoReturn, Optional, Tuple, Unionfrom urllib.parse import parse_qs, urlparse@@ -33,6 +33,9 @@ logger = get_logger(__name__)_FEISHU_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(feishu://image/([^)]+)\)")_FEISHU_DOCUMENT_FORBIDDEN = 1770032_FEISHU_BITABLE_PERMISSION_REQUIRED = 99991672+_MAX_MEDIA_DOWNLOAD_CONTEXTS = 8++_MediaDownloadExtras = Dict[str, List[Optional[str]]]def _title_as_filename(title: str) -> str:@@ -107,6 +110,7 @@ class FeishuDocument:markdown_content: strtitle: strmeta: Dict[str, Any]+ media_download_extras: _MediaDownloadExtras = field(default_factory=dict)class FeishuAccessor(DataAccessor):@@ -268,6 +272,7 @@ class FeishuAccessor(DataAccessor):self._resolve_image_refs,doc.markdown_content,feishu_access_token=feishu_access_token,+ media_download_extras=doc.media_download_extras,)# Build metadata@@ -331,6 +336,7 @@ class FeishuAccessor(DataAccessor):view_id = (query.get("view") or [None])[0]title = Nonemeta = {}+ media_download_extras: _MediaDownloadExtras = {}if doc_type == "wiki":# Resolve wiki node to actual document type@@ -354,7 +360,13 @@ class FeishuAccessor(DataAccessor):handler_kwargs = {}if doc_type == "base":- handler_kwargs = {"table_id": table_id, "view_id": view_id}+ handler_kwargs = {+ "table_id": table_id,+ "view_id": view_id,+ "media_download_extras": media_download_extras,+ }+ elif doc_type == "sheets":+ handler_kwargs = {"media_download_extras": media_download_extras}# Feishu's SDK is synchronous; keep it off the event loop.markdown, doc_title = await asyncio.to_thread(@@ -380,6 +392,7 @@ class FeishuAccessor(DataAccessor):markdown_content=markdown,title=doc_title,meta=meta,+ media_download_extras=media_download_extras,)@staticmethod@@ -770,6 +783,7 @@ class FeishuAccessor(DataAccessor):file_token: str,*,feishu_access_token: Optional[str] = None,+ extra: Optional[str] = None,) -> Optional[Tuple[bytes, Optional[str]]]:"""Download an image from Feishu Drive API by file token.@@ -792,6 +806,8 @@ class FeishuAccessor(DataAccessor):.token_types({token_type}).build())+ if extra:+ raw_req.add_query("extra", extra)try:raw_resp = self._call_api(client.request, raw_req, feishu_access_token)except Exception as exc:@@ -838,6 +854,7 @@ class FeishuAccessor(DataAccessor):markdown: str,*,feishu_access_token: Optional[str] = None,+ media_download_extras: Optional[_MediaDownloadExtras] = None,) -> Tuple[str, Dict[str, bytes]]:"""Download Feishu image refs and rewrite them to local relative paths."""config = self._get_config()@@ -855,10 +872,25 @@ class FeishuAccessor(DataAccessor):if file_token in token_to_rel_path:continue- downloaded = self._download_image(- file_token,- feishu_access_token=feishu_access_token,- )+ configured_extras = (media_download_extras or {}).get(file_token)+ if configured_extras:+ # Try protected contexts before the legacy token-only fallback.+ extras: List[Optional[str]] = list(+ dict.fromkeys(extra for extra in configured_extras if extra)+ )[:_MAX_MEDIA_DOWNLOAD_CONTEXTS]+ extras.append(None)+ else:+ extras = [None]++ downloaded = None+ for extra in extras:+ downloaded = self._download_image(+ file_token,+ feishu_access_token=feishu_access_token,+ extra=extra,+ )+ if downloaded is not None:+ breakif downloaded is None:continueimage_bytes, content_type = downloaded@@ -1073,6 +1105,8 @@ class FeishuAccessor(DataAccessor):self,token: str,feishu_access_token: Optional[str] = None,+ *,+ media_download_extras: Optional[_MediaDownloadExtras] = None,) -> Tuple[str, str]:"""Fetch a Feishu spreadsheet and convert it to Markdown."""import lark_oapi as lark@@ -1125,6 +1159,7 @@ class FeishuAccessor(DataAccessor):feishu_access_token,table_id=tokens[1],table_name=sheet_title,+ media_download_extras=media_download_extras,)parts.append(bitable or "*Empty bitable*")markdown_parts.append("\n\n".join(parts))@@ -1208,6 +1243,7 @@ class FeishuAccessor(DataAccessor):table_id: Optional[str] = None,table_name: Optional[str] = None,view_id: Optional[str] = None,+ media_download_extras: Optional[_MediaDownloadExtras] = None,) -> Tuple[str, str]:"""Fetch a Feishu bitable app and convert it to Markdown."""if view_id and not table_id:@@ -1291,6 +1327,11 @@ class FeishuAccessor(DataAccessor):"without a page token")field_names = [field.field_name for field in fields]+ field_ids = {+ field.field_name: field.field_id+ for field in fields+ if getattr(field, "field_name", None) and getattr(field, "field_id", None)+ }records = []page_token = None@@ -1340,10 +1381,20 @@ class FeishuAccessor(DataAccessor):if field_names and records:rows = [field_names]for record in records:- fields = record.fields or {}- rows.append(- [self._format_bitable_field(fields.get(name, "")) for name in field_names]- )+ record_fields = record.fields or {}+ row = []+ for name in field_names:+ value = record_fields.get(name, "")+ row.append(self._format_bitable_field(value))+ if media_download_extras is not None:+ self._collect_bitable_media_extras(+ value,+ table_id=current_table_id,+ field_id=field_ids.get(name),+ record_id=getattr(record, "record_id", None),+ media_download_extras=media_download_extras,+ )+ rows.append(row)parts.append(format_table_to_markdown(rows, has_header=True))if records_truncated:parts.append(f"\n*... records truncated at {config.max_records_per_table} ...*")@@ -1351,6 +1402,53 @@ class FeishuAccessor(DataAccessor):return "\n\n".join(markdown_parts), title+ @classmethod+ def _collect_bitable_media_extras(+ cls,+ value: Any,+ *,+ table_id: str,+ field_id: Optional[str],+ record_id: Optional[str],+ media_download_extras: _MediaDownloadExtras,+ ) -> None:+ """Collect transient permission contexts for image refs emitted from a cell."""+ if isinstance(value, list):+ for item in value:+ cls._collect_bitable_media_extras(+ item,+ table_id=table_id,+ field_id=field_id,+ record_id=record_id,+ media_download_extras=media_download_extras,+ )+ return+ if not isinstance(value, dict):+ return++ file_token = value.get("file_token")+ name = str(value.get("name") or "image")+ media_type = value.get("type") or mimetypes.guess_type(name)[0]+ if not file_token or not str(media_type).lower().startswith("image/"):+ return++ contexts = media_download_extras.setdefault(str(file_token), [])+ if field_id and record_id:+ extra = json.dumps(+ {+ "bitablePerm": {+ "tableId": table_id,+ "attachments": {field_id: {record_id: [str(file_token)]}},+ }+ },+ separators=(",", ":"),+ )+ context_count = sum(context is not None for context in contexts)+ if extra not in contexts and context_count < _MAX_MEDIA_DOWNLOAD_CONTEXTS:+ contexts.append(extra)+ elif None not in contexts:+ contexts.append(None)+@classmethoddef _format_bitable_field(cls, value: Any) -> str:"""Render the common structured values returned by bitable fields."""diff --git a/tests/parse/test_feishu_accessor.py b/tests/parse/test_feishu_accessor.pyindex 505f932cf..b92eb114c 100644--- a/tests/parse/test_feishu_accessor.py+++ b/tests/parse/test_feishu_accessor.py@@ -3,13 +3,18 @@"""Tests for FeishuAccessor supported types, user token, and image handling."""import asyncio+import jsonimport sysfrom types import ModuleType, SimpleNamespacefrom unittest.mock import MagicMockimport pytest-from openviking.parse.accessors.feishu_accessor import FeishuAccessor, _title_as_filename+from openviking.parse.accessors.feishu_accessor import (+ _MAX_MEDIA_DOWNLOAD_CONTEXTS,+ FeishuAccessor,+ _title_as_filename,+)class _SuccessResponse:@@ -53,6 +58,8 @@ class _FakeBaseRequest:class _FakeBaseRequestBuilder:def __init__(self):self._request = SimpleNamespace(http_method=None, uri=None, token_types=None)+ self._request.queries = {}+ self._request.add_query = self._request.queries.__setitem__def http_method(self, method):self._request.http_method = method@@ -78,8 +85,16 @@ class _FakeRawResponse:class _FakeMediaResponse:- def __init__(self, content=b"image-bytes", success=True, code=0, msg="", headers=None):- self.raw = _FakeRawResponse(content, headers=headers)+ def __init__(+ self,+ content=b"image-bytes",+ success=True,+ code=0,+ msg="",+ headers=None,+ status_code=200,+ ):+ self.raw = _FakeRawResponse(content, status_code=status_code, headers=headers)self.code = codeself.msg = msgself._success = success@@ -234,6 +249,100 @@ def test_resolve_image_refs_falls_back_to_byte_magic_extension(monkeypatch):assert images == {"images/img_token_webp.webp": webp_bytes}+def test_resolve_image_refs_tries_distinct_permission_contexts(monkeypatch):+ _install_fake_lark_modules(monkeypatch)+ request_media = MagicMock(+ side_effect=[+ _FakeMediaResponse(success=False, code=400, status_code=400),+ _FakeMediaResponse(b"\x89PNG\r\n"),+ ]+ )+ accessor = FeishuAccessor()+ accessor._config = SimpleNamespace(download_images=True)+ accessor._client = SimpleNamespace(request=request_media)++ updated, images = accessor._resolve_image_refs(+ "",+ media_download_extras={"shared-token": ["context-one", "context-two", None]},+ )++ assert updated == ""+ assert images == {"images/shared-token.png": b"\x89PNG\r\n"}+ requests = [call.args[0] for call in request_media.call_args_list]+ assert [request.queries["extra"] for request in requests] == [+ "context-one",+ "context-two",+ ]+++ ("feishu_access_token", "token_type"),+ [(None, "tenant"), ("u-test", "user")],+)+def test_resolve_image_refs_falls_back_to_token_only(+ monkeypatch,+ feishu_access_token,+ token_type,+):+ _install_fake_lark_modules(monkeypatch)+ request_media = MagicMock(+ side_effect=[+ _FakeMediaResponse(success=False, code=400, status_code=400),+ _FakeMediaResponse(b"\x89PNG\r\n"),+ ]+ )+ accessor = FeishuAccessor()+ accessor._config = SimpleNamespace(download_images=True)+ accessor._client = SimpleNamespace(request=request_media)+ accessor._user_token_client = SimpleNamespace(request=request_media)++ updated, images = accessor._resolve_image_refs(+ "",+ feishu_access_token=feishu_access_token,+ media_download_extras={"shared-token": ["permission-context"]},+ )++ assert updated == ""+ assert images == {"images/shared-token.png": b"\x89PNG\r\n"}+ requests = [call.args[0] for call in request_media.call_args_list]+ assert [request.queries for request in requests] == [+ {"extra": "permission-context"},+ {},+ ]+ assert all(request.token_types == {token_type} for request in requests)+ if feishu_access_token:+ assert all(+ call.args[1].user_access_token == feishu_access_token+ for call in request_media.call_args_list+ )+ else:+ assert all(len(call.args) == 1 for call in request_media.call_args_list)+++def test_resolve_image_refs_bounds_permission_context_attempts(monkeypatch):+ _install_fake_lark_modules(monkeypatch)+ request_media = MagicMock(+ return_value=_FakeMediaResponse(success=False, code=400, status_code=400)+ )+ accessor = FeishuAccessor()+ accessor._config = SimpleNamespace(download_images=True)+ accessor._client = SimpleNamespace(request=request_media)+ contexts = [f"context-{index}" for index in range(_MAX_MEDIA_DOWNLOAD_CONTEXTS + 2)]++ updated, images = accessor._resolve_image_refs(+ "",+ media_download_extras={"shared-token": contexts},+ )++ assert updated == ""+ assert images == {}+ requests = [call.args[0] for call in request_media.call_args_list]+ assert [request.queries.get("extra") for request in requests] == [+ *contexts[:_MAX_MEDIA_DOWNLOAD_CONTEXTS],+ None,+ ]++def test_download_image_uses_tenant_token_without_user_token(monkeypatch):_install_fake_lark_modules(monkeypatch)request_media = MagicMock(return_value=_FakeMediaResponse(b"\x89PNG\r\n"))@@ -294,6 +403,7 @@ def test_access_offloads_synchronous_download_to_thread(monkeypatch):markdown_content="",title="Test Doc",meta={},+ media_download_extras={"img_token_123": ["permission-context"]},)monkeypatch.setattr(accessor, "_fetch_document", fake_fetch_document)@@ -303,6 +413,7 @@ def test_access_offloads_synchronous_download_to_thread(monkeypatch):def fake_resolve(markdown, **_):ran_on["thread"] = threading.get_ident()+ ran_on["extras"] = _["media_download_extras"]return ("",{"images/img_token_123.png": b"\x89PNG\r\n"},@@ -317,6 +428,8 @@ def test_access_offloads_synchronous_download_to_thread(monkeypatch):"_resolve_image_refs ran on the event-loop thread; ""it must be offloaded via asyncio.to_thread")+ assert ran_on["extras"] == {"img_token_123": ["permission-context"]}+ assert "permission-context" not in str(resource.meta)finally:resource.cleanup()@@ -395,7 +508,11 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):)assert wiki.title == "Wiki Base"handlers["_parse_docx"].assert_called_once_with("doc_token", "u-test")- handlers["_parse_sheets"].assert_called_once_with("sht_token", None)+ handlers["_parse_sheets"].assert_called_once_with(+ "sht_token",+ None,+ media_download_extras=sheets.media_download_extras,+ )assert handlers["_parse_bitable"].call_args_list[-1].args == ("wiki_app_token", None)@@ -475,9 +592,9 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):return_value=_SuccessResponse(SimpleNamespace(items=[- SimpleNamespace(field_name="Topic"),- SimpleNamespace(field_name="Cover"),- SimpleNamespace(field_name="Brief"),+ SimpleNamespace(field_name="Topic", field_id="fld-topic"),+ SimpleNamespace(field_name="Cover", field_id="fld-cover"),+ SimpleNamespace(field_name="Brief", field_id="fld-brief"),],has_more=False,page_token=None,@@ -489,6 +606,7 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):SimpleNamespace(items=[SimpleNamespace(+ record_id="rec-1",fields={"Topic": "Welcome","Cover": [@@ -505,7 +623,7 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):"type": "application/pdf",}],- }+ },)],has_more=False,@@ -530,7 +648,12 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):),)- markdown, title = accessor._parse_sheets("sht_token", "u-test")+ media_download_extras = {}+ markdown, title = accessor._parse_sheets(+ "sht_token",+ "u-test",+ media_download_extras=media_download_extras,+ )assert title == "Budget"assert "| name | amount |" in markdown@@ -542,12 +665,26 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):assert "brief.pdf" in markdownassert "feishu://image/brief-token" not in markdownassert "Empty sheet" not in markdown+ assert media_download_extras == {+ "cover-token": [+ json.dumps(+ {+ "bitablePerm": {+ "tableId": "table-1",+ "attachments": {"fld-cover": {"rec-1": ["cover-token"]}},+ }+ },+ separators=(",", ":"),+ )+ ]+ }assert list_tables.call_count == 0assert list_fields.call_args.args[0].table_id == "table-1"resolved, images = accessor._resolve_image_refs(markdown,feishu_access_token="u-test",+ media_download_extras=media_download_extras,)assert "" in resolved@@ -555,10 +692,45 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):assert request.call_args_list[-1].args[0].uri == ("/open-apis/drive/v1/medias/cover-token/download")+ assert json.loads(request.call_args_list[-1].args[0].queries["extra"]) == {+ "bitablePerm": {+ "tableId": "table-1",+ "attachments": {"fld-cover": {"rec-1": ["cover-token"]}},+ }+ }assert all(call.args[0].token_types == {"user"} for call in request.call_args_list)assert all(call.args[1].user_access_token == "u-test" for call in request.call_args_list)+def test_bitable_media_context_falls_back_without_sdk_ids():+ extras = {}++ FeishuAccessor._collect_bitable_media_extras(+ {"file_token": "cover-token", "name": "cover.png", "type": "image/png"},+ table_id="table-1",+ field_id=None,+ record_id="rec-1",+ media_download_extras=extras,+ )++ assert extras == {"cover-token": [None]}+++def test_bitable_media_context_collection_is_bounded():+ extras = {}++ for index in range(_MAX_MEDIA_DOWNLOAD_CONTEXTS + 2):+ FeishuAccessor._collect_bitable_media_extras(+ {"file_token": "cover-token", "name": "cover.png", "type": "image/png"},+ table_id="table-1",+ field_id="field-1",+ record_id=f"record-{index}",+ media_download_extras=extras,+ )++ assert len(extras["cover-token"]) == _MAX_MEDIA_DOWNLOAD_CONTEXTS++def test_parse_bitable_uses_user_token_and_formats_records(monkeypatch):_install_fake_lark_modules(monkeypatch)list_tables = MagicMock(