Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(feishu): preserve Bitable media permission context during import (#3558)

Evo16e7c33f1f03
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.py
index df8a30db9..81a7fc87a 100644
--- a/openviking/parse/accessors/feishu_accessor.py
+++ b/openviking/parse/accessors/feishu_accessor.py
@@ -15,7 +15,7 @@ import mimetypes
import os
import re
import tempfile
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Dict, List, NoReturn, Optional, Tuple, Union
from urllib.parse import parse_qs, urlparse
@@ -33,6 +33,9 @@ logger = get_logger(__name__)
_FEISHU_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(feishu://image/([^)]+)\)")
_FEISHU_DOCUMENT_FORBIDDEN = 1770032
_FEISHU_BITABLE_PERMISSION_REQUIRED = 99991672
+_MAX_MEDIA_DOWNLOAD_CONTEXTS = 8
+
+_MediaDownloadExtras = Dict[str, List[Optional[str]]]
def _title_as_filename(title: str) -> str:
@@ -107,6 +110,7 @@ class FeishuDocument:
markdown_content: str
title: str
meta: Dict[str, Any]
+ media_download_extras: _MediaDownloadExtras = field(default_factory=dict)
class FeishuAccessor(DataAccessor):
@@ -268,6 +272,7 @@ class FeishuAccessor(DataAccessor):
self._resolve_image_refs,
doc.markdown_content,
feishu_access_token=feishu_access_token,
+ media_download_extras=doc.media_download_extras,
)
# Build metadata
@@ -331,6 +336,7 @@ class FeishuAccessor(DataAccessor):
view_id = (query.get("view") or [None])[0]
title = None
meta = {}
+ media_download_extras: _MediaDownloadExtras = {}
if doc_type == "wiki":
# Resolve wiki node to actual document type
@@ -354,7 +360,13 @@ class FeishuAccessor(DataAccessor):
handler_kwargs = {}
if doc_type == "base":
- handler_kwargs = {"table_id": table_id, "view_id": view_id}
+ handler_kwargs = {
+ "table_id": table_id,
+ "view_id": view_id,
+ "media_download_extras": media_download_extras,
+ }
+ elif doc_type == "sheets":
+ handler_kwargs = {"media_download_extras": media_download_extras}
# Feishu's SDK is synchronous; keep it off the event loop.
markdown, doc_title = await asyncio.to_thread(
@@ -380,6 +392,7 @@ class FeishuAccessor(DataAccessor):
markdown_content=markdown,
title=doc_title,
meta=meta,
+ media_download_extras=media_download_extras,
)
@staticmethod
@@ -770,6 +783,7 @@ class FeishuAccessor(DataAccessor):
file_token: str,
*,
feishu_access_token: Optional[str] = None,
+ extra: Optional[str] = None,
) -> Optional[Tuple[bytes, Optional[str]]]:
"""Download an image from Feishu Drive API by file token.
@@ -792,6 +806,8 @@ class FeishuAccessor(DataAccessor):
.token_types({token_type})
.build()
)
+ if extra:
+ raw_req.add_query("extra", extra)
try:
raw_resp = self._call_api(client.request, raw_req, feishu_access_token)
except Exception as exc:
@@ -838,6 +854,7 @@ class FeishuAccessor(DataAccessor):
markdown: str,
*,
feishu_access_token: Optional[str] = None,
+ media_download_extras: Optional[_MediaDownloadExtras] = None,
) -> Tuple[str, Dict[str, bytes]]:
"""Download Feishu image refs and rewrite them to local relative paths."""
config = self._get_config()
@@ -855,10 +872,25 @@ class FeishuAccessor(DataAccessor):
if file_token in token_to_rel_path:
continue
- downloaded = self._download_image(
- file_token,
- feishu_access_token=feishu_access_token,
- )
+ configured_extras = (media_download_extras or {}).get(file_token)
+ if configured_extras:
+ # Try protected contexts before the legacy token-only fallback.
+ extras: List[Optional[str]] = list(
+ dict.fromkeys(extra for extra in configured_extras if extra)
+ )[:_MAX_MEDIA_DOWNLOAD_CONTEXTS]
+ extras.append(None)
+ else:
+ extras = [None]
+
+ downloaded = None
+ for extra in extras:
+ downloaded = self._download_image(
+ file_token,
+ feishu_access_token=feishu_access_token,
+ extra=extra,
+ )
+ if downloaded is not None:
+ break
if downloaded is None:
continue
image_bytes, content_type = downloaded
@@ -1073,6 +1105,8 @@ class FeishuAccessor(DataAccessor):
self,
token: str,
feishu_access_token: Optional[str] = None,
+ *,
+ media_download_extras: Optional[_MediaDownloadExtras] = None,
) -> Tuple[str, str]:
"""Fetch a Feishu spreadsheet and convert it to Markdown."""
import lark_oapi as lark
@@ -1125,6 +1159,7 @@ class FeishuAccessor(DataAccessor):
feishu_access_token,
table_id=tokens[1],
table_name=sheet_title,
+ media_download_extras=media_download_extras,
)
parts.append(bitable or "*Empty bitable*")
markdown_parts.append("\n\n".join(parts))
@@ -1208,6 +1243,7 @@ class FeishuAccessor(DataAccessor):
table_id: Optional[str] = None,
table_name: Optional[str] = None,
view_id: Optional[str] = None,
+ media_download_extras: Optional[_MediaDownloadExtras] = None,
) -> Tuple[str, str]:
"""Fetch a Feishu bitable app and convert it to Markdown."""
if view_id and not table_id:
@@ -1291,6 +1327,11 @@ class FeishuAccessor(DataAccessor):
"without a page token"
)
field_names = [field.field_name for field in fields]
+ field_ids = {
+ field.field_name: field.field_id
+ for field in fields
+ if getattr(field, "field_name", None) and getattr(field, "field_id", None)
+ }
records = []
page_token = None
@@ -1340,10 +1381,20 @@ class FeishuAccessor(DataAccessor):
if field_names and records:
rows = [field_names]
for record in records:
- fields = record.fields or {}
- rows.append(
- [self._format_bitable_field(fields.get(name, "")) for name in field_names]
- )
+ record_fields = record.fields or {}
+ row = []
+ for name in field_names:
+ value = record_fields.get(name, "")
+ row.append(self._format_bitable_field(value))
+ if media_download_extras is not None:
+ self._collect_bitable_media_extras(
+ value,
+ table_id=current_table_id,
+ field_id=field_ids.get(name),
+ record_id=getattr(record, "record_id", None),
+ media_download_extras=media_download_extras,
+ )
+ rows.append(row)
parts.append(format_table_to_markdown(rows, has_header=True))
if records_truncated:
parts.append(f"\n*... records truncated at {config.max_records_per_table} ...*")
@@ -1351,6 +1402,53 @@ class FeishuAccessor(DataAccessor):
return "\n\n".join(markdown_parts), title
+ @classmethod
+ def _collect_bitable_media_extras(
+ cls,
+ value: Any,
+ *,
+ table_id: str,
+ field_id: Optional[str],
+ record_id: Optional[str],
+ media_download_extras: _MediaDownloadExtras,
+ ) -> None:
+ """Collect transient permission contexts for image refs emitted from a cell."""
+ if isinstance(value, list):
+ for item in value:
+ cls._collect_bitable_media_extras(
+ item,
+ table_id=table_id,
+ field_id=field_id,
+ record_id=record_id,
+ media_download_extras=media_download_extras,
+ )
+ return
+ if not isinstance(value, dict):
+ return
+
+ file_token = value.get("file_token")
+ name = str(value.get("name") or "image")
+ media_type = value.get("type") or mimetypes.guess_type(name)[0]
+ if not file_token or not str(media_type).lower().startswith("image/"):
+ return
+
+ contexts = media_download_extras.setdefault(str(file_token), [])
+ if field_id and record_id:
+ extra = json.dumps(
+ {
+ "bitablePerm": {
+ "tableId": table_id,
+ "attachments": {field_id: {record_id: [str(file_token)]}},
+ }
+ },
+ separators=(",", ":"),
+ )
+ context_count = sum(context is not None for context in contexts)
+ if extra not in contexts and context_count < _MAX_MEDIA_DOWNLOAD_CONTEXTS:
+ contexts.append(extra)
+ elif None not in contexts:
+ contexts.append(None)
+
@classmethod
def _format_bitable_field(cls, value: Any) -> str:
"""Render the common structured values returned by bitable fields."""
diff --git a/tests/parse/test_feishu_accessor.py b/tests/parse/test_feishu_accessor.py
index 505f932cf..b92eb114c 100644
--- a/tests/parse/test_feishu_accessor.py
+++ b/tests/parse/test_feishu_accessor.py
@@ -3,13 +3,18 @@
"""Tests for FeishuAccessor supported types, user token, and image handling."""
import asyncio
+import json
import sys
from types import ModuleType, SimpleNamespace
from unittest.mock import MagicMock
import pytest
-from openviking.parse.accessors.feishu_accessor import FeishuAccessor, _title_as_filename
+from openviking.parse.accessors.feishu_accessor import (
+ _MAX_MEDIA_DOWNLOAD_CONTEXTS,
+ FeishuAccessor,
+ _title_as_filename,
+)
class _SuccessResponse:
@@ -53,6 +58,8 @@ class _FakeBaseRequest:
class _FakeBaseRequestBuilder:
def __init__(self):
self._request = SimpleNamespace(http_method=None, uri=None, token_types=None)
+ self._request.queries = {}
+ self._request.add_query = self._request.queries.__setitem__
def http_method(self, method):
self._request.http_method = method
@@ -78,8 +85,16 @@ class _FakeRawResponse:
class _FakeMediaResponse:
- def __init__(self, content=b"image-bytes", success=True, code=0, msg="", headers=None):
- self.raw = _FakeRawResponse(content, headers=headers)
+ def __init__(
+ self,
+ content=b"image-bytes",
+ success=True,
+ code=0,
+ msg="",
+ headers=None,
+ status_code=200,
+ ):
+ self.raw = _FakeRawResponse(content, status_code=status_code, headers=headers)
self.code = code
self.msg = msg
self._success = success
@@ -234,6 +249,100 @@ def test_resolve_image_refs_falls_back_to_byte_magic_extension(monkeypatch):
assert images == {"images/img_token_webp.webp": webp_bytes}
+def test_resolve_image_refs_tries_distinct_permission_contexts(monkeypatch):
+ _install_fake_lark_modules(monkeypatch)
+ request_media = MagicMock(
+ side_effect=[
+ _FakeMediaResponse(success=False, code=400, status_code=400),
+ _FakeMediaResponse(b"\x89PNG\r\n"),
+ ]
+ )
+ accessor = FeishuAccessor()
+ accessor._config = SimpleNamespace(download_images=True)
+ accessor._client = SimpleNamespace(request=request_media)
+
+ updated, images = accessor._resolve_image_refs(
+ "![s](feishu://image/shared-token)",
+ media_download_extras={"shared-token": ["context-one", "context-two", None]},
+ )
+
+ assert updated == "![s](images/shared-token.png)"
+ assert images == {"images/shared-token.png": b"\x89PNG\r\n"}
+ requests = [call.args[0] for call in request_media.call_args_list]
+ assert [request.queries["extra"] for request in requests] == [
+ "context-one",
+ "context-two",
+ ]
+
+
+ ("feishu_access_token", "token_type"),
+ [(None, "tenant"), ("u-test", "user")],
+)
+def test_resolve_image_refs_falls_back_to_token_only(
+ monkeypatch,
+ feishu_access_token,
+ token_type,
+):
+ _install_fake_lark_modules(monkeypatch)
+ request_media = MagicMock(
+ side_effect=[
+ _FakeMediaResponse(success=False, code=400, status_code=400),
+ _FakeMediaResponse(b"\x89PNG\r\n"),
+ ]
+ )
+ accessor = FeishuAccessor()
+ accessor._config = SimpleNamespace(download_images=True)
+ accessor._client = SimpleNamespace(request=request_media)
+ accessor._user_token_client = SimpleNamespace(request=request_media)
+
+ updated, images = accessor._resolve_image_refs(
+ "![s](feishu://image/shared-token)",
+ feishu_access_token=feishu_access_token,
+ media_download_extras={"shared-token": ["permission-context"]},
+ )
+
+ assert updated == "![s](images/shared-token.png)"
+ assert images == {"images/shared-token.png": b"\x89PNG\r\n"}
+ requests = [call.args[0] for call in request_media.call_args_list]
+ assert [request.queries for request in requests] == [
+ {"extra": "permission-context"},
+ {},
+ ]
+ assert all(request.token_types == {token_type} for request in requests)
+ if feishu_access_token:
+ assert all(
+ call.args[1].user_access_token == feishu_access_token
+ for call in request_media.call_args_list
+ )
+ else:
+ assert all(len(call.args) == 1 for call in request_media.call_args_list)
+
+
+def test_resolve_image_refs_bounds_permission_context_attempts(monkeypatch):
+ _install_fake_lark_modules(monkeypatch)
+ request_media = MagicMock(
+ return_value=_FakeMediaResponse(success=False, code=400, status_code=400)
+ )
+ accessor = FeishuAccessor()
+ accessor._config = SimpleNamespace(download_images=True)
+ accessor._client = SimpleNamespace(request=request_media)
+ contexts = [f"context-{index}" for index in range(_MAX_MEDIA_DOWNLOAD_CONTEXTS + 2)]
+
+ updated, images = accessor._resolve_image_refs(
+ "![s](feishu://image/shared-token)",
+ media_download_extras={"shared-token": contexts},
+ )
+
+ assert updated == "![s](feishu://image/shared-token)"
+ assert images == {}
+ requests = [call.args[0] for call in request_media.call_args_list]
+ assert [request.queries.get("extra") for request in requests] == [
+ *contexts[:_MAX_MEDIA_DOWNLOAD_CONTEXTS],
+ None,
+ ]
+
+
def test_download_image_uses_tenant_token_without_user_token(monkeypatch):
_install_fake_lark_modules(monkeypatch)
request_media = MagicMock(return_value=_FakeMediaResponse(b"\x89PNG\r\n"))
@@ -294,6 +403,7 @@ def test_access_offloads_synchronous_download_to_thread(monkeypatch):
markdown_content="![s](feishu://image/img_token_123)",
title="Test Doc",
meta={},
+ media_download_extras={"img_token_123": ["permission-context"]},
)
monkeypatch.setattr(accessor, "_fetch_document", fake_fetch_document)
@@ -303,6 +413,7 @@ def test_access_offloads_synchronous_download_to_thread(monkeypatch):
def fake_resolve(markdown, **_):
ran_on["thread"] = threading.get_ident()
+ ran_on["extras"] = _["media_download_extras"]
return (
"![s](images/img_token_123.png)",
{"images/img_token_123.png": b"\x89PNG\r\n"},
@@ -317,6 +428,8 @@ def test_access_offloads_synchronous_download_to_thread(monkeypatch):
"_resolve_image_refs ran on the event-loop thread; "
"it must be offloaded via asyncio.to_thread"
)
+ assert ran_on["extras"] == {"img_token_123": ["permission-context"]}
+ assert "permission-context" not in str(resource.meta)
finally:
resource.cleanup()
@@ -395,7 +508,11 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):
)
assert wiki.title == "Wiki Base"
handlers["_parse_docx"].assert_called_once_with("doc_token", "u-test")
- handlers["_parse_sheets"].assert_called_once_with("sht_token", None)
+ handlers["_parse_sheets"].assert_called_once_with(
+ "sht_token",
+ None,
+ media_download_extras=sheets.media_download_extras,
+ )
assert handlers["_parse_bitable"].call_args_list[-1].args == ("wiki_app_token", None)
@@ -475,9 +592,9 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):
return_value=_SuccessResponse(
SimpleNamespace(
items=[
- SimpleNamespace(field_name="Topic"),
- SimpleNamespace(field_name="Cover"),
- SimpleNamespace(field_name="Brief"),
+ SimpleNamespace(field_name="Topic", field_id="fld-topic"),
+ SimpleNamespace(field_name="Cover", field_id="fld-cover"),
+ SimpleNamespace(field_name="Brief", field_id="fld-brief"),
],
has_more=False,
page_token=None,
@@ -489,6 +606,7 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):
SimpleNamespace(
items=[
SimpleNamespace(
+ record_id="rec-1",
fields={
"Topic": "Welcome",
"Cover": [
@@ -505,7 +623,7 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):
"type": "application/pdf",
}
],
- }
+ },
)
],
has_more=False,
@@ -530,7 +648,12 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):
),
)
- markdown, title = accessor._parse_sheets("sht_token", "u-test")
+ media_download_extras = {}
+ markdown, title = accessor._parse_sheets(
+ "sht_token",
+ "u-test",
+ media_download_extras=media_download_extras,
+ )
assert title == "Budget"
assert "| name | amount |" in markdown
@@ -542,12 +665,26 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):
assert "brief.pdf" in markdown
assert "feishu://image/brief-token" not in markdown
assert "Empty sheet" not in markdown
+ assert media_download_extras == {
+ "cover-token": [
+ json.dumps(
+ {
+ "bitablePerm": {
+ "tableId": "table-1",
+ "attachments": {"fld-cover": {"rec-1": ["cover-token"]}},
+ }
+ },
+ separators=(",", ":"),
+ )
+ ]
+ }
assert list_tables.call_count == 0
assert list_fields.call_args.args[0].table_id == "table-1"
resolved, images = accessor._resolve_image_refs(
markdown,
feishu_access_token="u-test",
+ media_download_extras=media_download_extras,
)
assert "![cover.png](images/cover-token.png)" in resolved
@@ -555,10 +692,45 @@ def test_parse_sheets_handles_grid_and_embedded_bitable(monkeypatch):
assert request.call_args_list[-1].args[0].uri == (
"/open-apis/drive/v1/medias/cover-token/download"
)
+ assert json.loads(request.call_args_list[-1].args[0].queries["extra"]) == {
+ "bitablePerm": {
+ "tableId": "table-1",
+ "attachments": {"fld-cover": {"rec-1": ["cover-token"]}},
+ }
+ }
assert all(call.args[0].token_types == {"user"} for call in request.call_args_list)
assert all(call.args[1].user_access_token == "u-test" for call in request.call_args_list)
+def test_bitable_media_context_falls_back_without_sdk_ids():
+ extras = {}
+
+ FeishuAccessor._collect_bitable_media_extras(
+ {"file_token": "cover-token", "name": "cover.png", "type": "image/png"},
+ table_id="table-1",
+ field_id=None,
+ record_id="rec-1",
+ media_download_extras=extras,
+ )
+
+ assert extras == {"cover-token": [None]}
+
+
+def test_bitable_media_context_collection_is_bounded():
+ extras = {}
+
+ for index in range(_MAX_MEDIA_DOWNLOAD_CONTEXTS + 2):
+ FeishuAccessor._collect_bitable_media_extras(
+ {"file_token": "cover-token", "name": "cover.png", "type": "image/png"},
+ table_id="table-1",
+ field_id="field-1",
+ record_id=f"record-{index}",
+ media_download_extras=extras,
+ )
+
+ assert len(extras["cover-token"]) == _MAX_MEDIA_DOWNLOAD_CONTEXTS
+
+
def test_parse_bitable_uses_user_token_and_formats_records(monkeypatch):
_install_fake_lark_modules(monkeypatch)
list_tables = MagicMock(