RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(data-source): add Feishu Wiki file connector (#19425)
Gonghan-Princessb50c13ff43e4
diff --git a/common/constants.py b/common/constants.pyindex febfd34071..46eededed2 100644--- a/common/constants.py+++ b/common/constants.py@@ -151,6 +151,7 @@ class FileSource(StrEnum):CONFLUENCE = "confluence"GMAIL = "gmail"GOOGLE_DRIVE = "google_drive"+ FEISHU_WIKI = "feishu_wiki"JIRA = "jira"SHAREPOINT = "sharepoint"SLACK = "slack"diff --git a/common/data_source/__init__.py b/common/data_source/__init__.pyindex fdb2871f2b..ec79f4d2a7 100644--- a/common/data_source/__init__.py+++ b/common/data_source/__init__.py@@ -46,6 +46,7 @@ from .exceptions import (InsufficientPermissionsError,UnexpectedValidationError,)+from .feishu_wiki_connector import FeishuWikiConnectorfrom .github.connector import GithubConnectorfrom .gitlab_connector import GitlabConnectorfrom .gmail_connector import GmailConnector@@ -82,6 +83,7 @@ CONNECTOR_BY_SOURCE: dict[str, type] = {FileSource.GMAIL: GmailConnector,FileSource.DROPBOX: DropboxConnector,FileSource.GOOGLE_DRIVE: GoogleDriveConnector,+ FileSource.FEISHU_WIKI: FeishuWikiConnector,FileSource.JIRA: JiraConnector,FileSource.SHAREPOINT: SharePointConnector,FileSource.SITEMAP: SitemapConnector,@@ -144,6 +146,7 @@ __all__ = ["Document","DocumentSource","DropboxConnector",+ "FeishuWikiConnector","GithubConnector","GitlabConnector","GmailConnector",diff --git a/common/data_source/config.py b/common/data_source/config.pyindex da3335760a..f6cba01557 100644--- a/common/data_source/config.py+++ b/common/data_source/config.py@@ -56,6 +56,7 @@ class DocumentSource(str, Enum):CONFLUENCE = "confluence"JIRA = "jira"GOOGLE_DRIVE = "google_drive"+ FEISHU_WIKI = "feishu_wiki"GMAIL = "gmail"DISCORD = "discord"WEBDAV = "webdav"diff --git a/common/data_source/feishu_wiki_connector.py b/common/data_source/feishu_wiki_connector.pynew file mode 100644index 0000000000..703adb4e3c--- /dev/null+++ b/common/data_source/feishu_wiki_connector.py@@ -0,0 +1,484 @@+"""Feishu Wiki data source connector.++The connector handles downloadable Wiki ``file`` nodes only. Native Feishu+documents require separate export APIs and are traversed as containers so that+downloadable files nested below them are still found.+"""++from __future__ import annotations++import hashlib+import json+import logging+import time+from collections.abc import Iterator+from datetime import UTC, datetime+from typing import Any+from urllib.parse import quote++import requests++from common.data_source.config import INDEX_BATCH_SIZE, DocumentSource+from common.data_source.exceptions import ConnectorMissingCredentialError, ConnectorValidationError+from common.data_source.interfaces import LoadConnector, PollConnector, SecondsSinceUnixEpoch+from common.data_source.models import Document, GenerateDocumentsOutput+from common.data_source.utils import get_file_ext++LOGGER = logging.getLogger(__name__)++FEISHU_OPEN_BASE_URL = "https://open.feishu.cn"+DEFAULT_REQUEST_TIMEOUT_SECONDS = 60+DEFAULT_MAX_FILE_SIZE_BYTES = 50 * 1024 * 1024+MAX_BATCH_SIZE = 10+DOWNLOAD_CHUNK_SIZE_BYTES = 1024 * 1024+MAX_DOWNLOAD_ERROR_BODY_BYTES = 8 * 1024+MAX_ERROR_DETAIL_LENGTH = 256+MAX_DOWNLOAD_RETRIES = 3+MAX_DOWNLOAD_RETRY_DELAY_SECONDS = 60+SUPPORTED_EXTENSIONS = {+ "csv",+ "doc",+ "docx",+ "eml",+ "gif",+ "html",+ "jpeg",+ "jpg",+ "json",+ "md",+ "mdx",+ "pdf",+ "png",+ "ppt",+ "pptx",+ "tif",+ "txt",+ "xls",+ "xlsx",+}+++def _normalized_terms(value: Any) -> set[str]:+ if value is None:+ return set()+ if isinstance(value, str):+ raw_values = value.split(",")+ elif isinstance(value, (list, tuple, set)):+ raw_values = value+ else:+ raw_values = [value]+ return {str(item).strip().lstrip(".").casefold() for item in raw_values if str(item).strip()}+++def _parse_timestamp(value: Any) -> datetime | None:+ if value in (None, ""):+ return None+ if isinstance(value, datetime):+ parsed = value+ else:+ text = str(value).strip()+ try:+ parsed = datetime.fromtimestamp(float(text), tz=UTC)+ except (TypeError, ValueError, OverflowError):+ try:+ parsed = datetime.fromisoformat(text)+ except (TypeError, ValueError):+ return None+ if parsed.tzinfo is None:+ return parsed.replace(tzinfo=UTC)+ return parsed.astimezone(UTC)+++class FeishuWikiConnector(LoadConnector, PollConnector):+ """Read downloadable attachment files from one Feishu Wiki subtree."""++ def __init__(+ self,+ *,+ space_id: str,+ root_node_token: str,+ include_extensions: Any = None,+ include_keywords: Any = None,+ exclude_keywords: Any = None,+ batch_size: int = INDEX_BATCH_SIZE,+ max_file_size_bytes: int = DEFAULT_MAX_FILE_SIZE_BYTES,+ session: requests.Session | None = None,+ ) -> None:+ self.space_id = space_id.strip()+ self.root_node_token = root_node_token.strip()+ self.include_extensions = _normalized_terms(include_extensions)+ self.include_keywords = _normalized_terms(include_keywords)+ self.exclude_keywords = _normalized_terms(exclude_keywords)+ self.batch_size = batch_size+ self.max_file_size_bytes = max_file_size_bytes+ self.session = session or requests.Session()+ self.app_id = ""+ self.app_secret = ""+ self._access_token = ""+ self._access_token_expires_at = 0.0++ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> FeishuWikiConnector:+ raw_batch_size = config.get("batch_size")+ if raw_batch_size in (None, ""):+ raw_batch_size = INDEX_BATCH_SIZE+ try:+ batch_size = int(raw_batch_size)+ except (TypeError, ValueError) as exc:+ raise ConnectorValidationError("Feishu Wiki batch_size must be an integer") from exc+ raw_max_file_size_bytes = config.get("max_file_size_bytes")+ if raw_max_file_size_bytes in (None, ""):+ raw_max_file_size_bytes = DEFAULT_MAX_FILE_SIZE_BYTES+ try:+ max_file_size_bytes = int(raw_max_file_size_bytes)+ except (TypeError, ValueError) as exc:+ raise ConnectorValidationError("Feishu Wiki max_file_size_bytes must be an integer") from exc++ connector = cls(+ space_id=str(config.get("space_id") or ""),+ root_node_token=str(config.get("root_node_token") or ""),+ include_extensions=config.get("include_extensions"),+ include_keywords=config.get("include_keywords"),+ exclude_keywords=config.get("exclude_keywords"),+ batch_size=batch_size,+ max_file_size_bytes=max_file_size_bytes,+ )+ connector.load_credentials(config.get("credentials") or {})+ connector._ensure_configured()+ return connector++ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:+ self.app_id = str(credentials.get("app_id") or "").strip()+ self.app_secret = str(credentials.get("app_secret") or "").strip()+ if not self.app_id or not self.app_secret:+ raise ConnectorMissingCredentialError("Feishu Wiki")+ self._access_token = ""+ self._access_token_expires_at = 0.0+ return None++ def _ensure_configured(self) -> None:+ if not self.space_id:+ raise ConnectorValidationError("Feishu Wiki space ID is required")+ if not self.root_node_token:+ raise ConnectorValidationError("Feishu Wiki root node token is required")+ if not 1 <= self.batch_size <= MAX_BATCH_SIZE:+ raise ConnectorValidationError(f"Feishu Wiki batch_size must be between 1 and {MAX_BATCH_SIZE}")+ if self.max_file_size_bytes <= 0:+ raise ConnectorValidationError("Feishu Wiki max_file_size_bytes must be a positive integer")+ unsupported = self.include_extensions - SUPPORTED_EXTENSIONS+ if unsupported:+ values = ", ".join(sorted(unsupported))+ raise ConnectorValidationError(f"Unsupported Feishu Wiki file extensions: {values}")++ def _get_access_token(self) -> str:+ now = time.monotonic()+ if self._access_token and now < self._access_token_expires_at:+ return self._access_token++ try:+ response = self.session.request(+ "POST",+ f"{FEISHU_OPEN_BASE_URL}/open-apis/auth/v3/tenant_access_token/internal",+ json={"app_id": self.app_id, "app_secret": self.app_secret},+ timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,+ )+ except requests.RequestException as exc:+ raise ConnectorValidationError("Unable to connect to Feishu while attempting to authenticate") from exc+ try:+ payload = self._decode_api_payload(response, "authenticate")+ finally:+ response.close()+ access_token = str(payload.get("tenant_access_token") or "")+ if not access_token:+ raise ConnectorValidationError("Feishu authentication response did not contain a tenant access token")+ try:+ expires_in = max(60, int(payload.get("expire") or 7200))+ except (TypeError, ValueError):+ expires_in = 7200+ self._access_token = access_token+ self._access_token_expires_at = now + max(30, expires_in - 60)+ return access_token++ @staticmethod+ def _decode_api_payload(response: requests.Response, operation: str) -> dict[str, Any]:+ payload: Any = None+ json_error: Exception | None = None+ try:+ payload = response.json()+ except (TypeError, ValueError) as exc:+ json_error = exc++ code = payload.get("code") if isinstance(payload, dict) else None+ message = str(payload.get("msg") or "unknown Feishu API error") if isinstance(payload, dict) else ""+ if response.status_code in {401, 403}:+ detail = f": {message}" if message else ""+ raise ConnectorValidationError(f"Feishu permission denied (HTTP {response.status_code}) while attempting to {operation}{detail}")+ if response.status_code >= 400:+ if code not in (0, "0", None):+ raise ConnectorValidationError(f"Feishu API error {code} (HTTP {response.status_code}) while attempting to {operation}: {message}")+ raise ConnectorValidationError(f"Feishu HTTP {response.status_code} while attempting to {operation}")+ if json_error is not None:+ raise ConnectorValidationError(f"Feishu returned invalid JSON while attempting to {operation}") from json_error+ if not isinstance(payload, dict):+ raise ConnectorValidationError(f"Feishu returned an invalid response while attempting to {operation}")+ code = payload.get("code", 0)+ if code not in (0, "0", None):+ message = str(payload.get("msg") or "unknown Feishu API error")+ raise ConnectorValidationError(f"Feishu API error {code} while attempting to {operation}: {message}")+ return payload++ def _request_json(+ self,+ method: str,+ path: str,+ *,+ params: dict[str, Any] | None = None,+ operation: str,+ ) -> dict[str, Any]:+ try:+ response = self.session.request(+ method,+ f"{FEISHU_OPEN_BASE_URL}{path}",+ headers={"Authorization": f"Bearer {self._get_access_token()}"},+ params=params,+ timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,+ )+ except requests.RequestException as exc:+ raise ConnectorValidationError(f"Unable to connect to Feishu while attempting to {operation}") from exc+ try:+ return self._decode_api_payload(response, operation)+ finally:+ response.close()++ def _iter_child_pages(self, parent_node_token: str) -> Iterator[list[dict[str, Any]]]:+ page_token = ""+ while True:+ params: dict[str, Any] = {"page_size": 50, "parent_node_token": parent_node_token}+ if page_token:+ params["page_token"] = page_token+ payload = self._request_json(+ "GET",+ f"/open-apis/wiki/v2/spaces/{quote(self.space_id, safe='')}/nodes",+ params=params,+ operation="list Wiki nodes",+ )+ data = payload.get("data") or {}+ if not isinstance(data, dict):+ raise ConnectorValidationError("Feishu Wiki node response contained invalid data")+ items = data.get("items") or []+ if not isinstance(items, list):+ raise ConnectorValidationError("Feishu Wiki node response contained invalid items")+ yield [item for item in items if isinstance(item, dict)]+ if not data.get("has_more"):+ break+ page_token = str(data.get("page_token") or "")+ if not page_token:+ raise ConnectorValidationError("Feishu Wiki pagination indicated more data without a page token")++ def _iter_nodes(self, parent_node_token: str) -> Iterator[dict[str, Any]]:+ for items in self._iter_child_pages(parent_node_token):+ for node in items:+ yield node+ node_token = str(node.get("node_token") or "")+ if node.get("has_child") and node_token:+ yield from self._iter_nodes(node_token)++ def _matches_filters(self, title: str) -> bool:+ normalized_title = title.casefold()+ extension = get_file_ext(title).lstrip(".").casefold()+ if extension not in SUPPORTED_EXTENSIONS:+ return False+ if self.include_extensions and extension not in self.include_extensions:+ return False+ if self.include_keywords and not any(keyword in normalized_title for keyword in self.include_keywords):+ return False+ return not any(keyword in normalized_title for keyword in self.exclude_keywords)++ @staticmethod+ def _is_within_window(+ updated_at: datetime | None,+ start: SecondsSinceUnixEpoch | None,+ end: SecondsSinceUnixEpoch | None,+ ) -> bool:+ if updated_at is None:+ return True+ timestamp = updated_at.timestamp()+ if start is not None and timestamp <= start:+ return False+ return end is None or timestamp <= end++ def _error_detail(self, value: Any) -> str:+ """Bound diagnostic fields without exposing credentials or multiline text."""+ if not isinstance(value, (str, int)):+ return ""+ text = str(value)+ for credential in (self.app_secret, self._access_token):+ if credential:+ text = text.replace(credential, "[redacted]")+ return " ".join(text.split())[:MAX_ERROR_DETAIL_LENGTH]++ def _download_error(self, response: requests.Response) -> ConnectorValidationError:+ """Extract only bounded JSON diagnostics from an unsuccessful download."""+ payload: Any = None+ body = bytearray()+ try:+ for chunk in response.iter_content(chunk_size=1024):+ body.extend(chunk[: MAX_DOWNLOAD_ERROR_BODY_BYTES - len(body)])+ if len(body) >= MAX_DOWNLOAD_ERROR_BODY_BYTES:+ break+ payload = json.loads(body)+ except (ValueError, RecursionError, OSError, requests.RequestException):+ payload = None++ code = self._error_detail(payload.get("code")) if isinstance(payload, dict) else ""+ message = self._error_detail(payload.get("msg")) if isinstance(payload, dict) else ""+ if code and code != "0":+ detail = f"Feishu API error {code} (HTTP {response.status_code}) while downloading a Wiki file"+ elif response.status_code in {401, 403}:+ detail = f"Feishu permission denied (HTTP {response.status_code}) while downloading a Wiki file"+ else:+ detail = f"Feishu HTTP {response.status_code} while downloading a Wiki file"+ if message:+ detail += f": {message}"+ error = payload.get("error") if isinstance(payload, dict) else None+ logid = self._error_detail(response.headers.get("x-tt-logid") or response.headers.get("x-request-id"))+ if not logid and isinstance(error, dict):+ logid = self._error_detail(error.get("logid"))+ if logid:+ detail += f" (logid={logid})"+ return ConnectorValidationError(detail)++ def _download_file(self, object_token: str) -> bytes:+ """Download a file, retrying throttling without leaking streamed responses."""+ for attempt in range(MAX_DOWNLOAD_RETRIES + 1):+ try:+ response = self.session.request(+ "GET",+ f"{FEISHU_OPEN_BASE_URL}/open-apis/drive/v1/files/{quote(object_token, safe='')}/download",+ headers={"Authorization": f"Bearer {self._get_access_token()}"},+ timeout=DEFAULT_REQUEST_TIMEOUT_SECONDS,+ stream=True,+ )+ except requests.RequestException as exc:+ raise ConnectorValidationError("Unable to connect to Feishu while downloading a Wiki file") from exc++ if response.status_code != 429 or attempt == MAX_DOWNLOAD_RETRIES:+ break+ # Feishu specifies a relative delay in seconds, not a Unix timestamp.+ try:+ delay = int(response.headers.get("x-ogw-ratelimit-reset", ""))+ except (TypeError, ValueError):+ delay = -1+ if delay < 0:+ delay = 2**attempt+ if delay > MAX_DOWNLOAD_RETRY_DELAY_SECONDS:+ # Fail rather than retry before the server's requested reset.+ break+ response.close()+ time.sleep(max(1, delay))++ try:+ if not 200 <= response.status_code < 300:+ raise self._download_error(response)++ declared_size = response.headers.get("Content-Length")+ if declared_size:+ try:+ declared_size_bytes = int(declared_size)+ except (TypeError, ValueError):+ declared_size_bytes = None+ if declared_size_bytes is not None and declared_size_bytes > self.max_file_size_bytes:+ raise ConnectorValidationError(f"Feishu Wiki file size exceeds the configured {self.max_file_size_bytes}-byte limit")++ chunks: list[bytes] = []+ downloaded_size = 0+ for chunk in response.iter_content(chunk_size=DOWNLOAD_CHUNK_SIZE_BYTES):+ if not chunk:+ continue+ downloaded_size += len(chunk)+ if downloaded_size > self.max_file_size_bytes:+ raise ConnectorValidationError(f"Feishu Wiki file size exceeds the configured {self.max_file_size_bytes}-byte limit")+ chunks.append(chunk)+ return b"".join(chunks)+ except ConnectorValidationError:+ raise+ except (OSError, requests.RequestException) as exc:+ raise ConnectorValidationError("Failed to download a Feishu Wiki file") from exc+ finally:+ response.close()++ def _node_to_document(self, node: dict[str, Any], *, fallback_updated_at: datetime) -> Document:+ node_token = str(node.get("node_token") or "")+ object_token = str(node.get("obj_token") or "")+ title = str(node.get("title") or node_token)+ if not node_token or not object_token:+ raise ConnectorValidationError("Feishu Wiki file node is missing a node token or object token")++ blob = self._download_file(object_token)+ content_sha256 = hashlib.sha256(blob).hexdigest()+ fingerprint = content_sha256[:32]+ updated_at = _parse_timestamp(node.get("obj_edit_time")) or fallback_updated_at+ metadata = {+ "source": DocumentSource.FEISHU_WIKI.value,+ "wiki_space_id": self.space_id,+ "wiki_node_token": node_token,+ "wiki_object_token": object_token,+ "wiki_object_type": str(node.get("obj_type") or "file"),+ "wiki_parent_node_token": str(node.get("parent_node_token") or self.root_node_token),+ "wiki_url": f"https://feishu.cn/wiki/{node_token}",+ "source_updated_at": updated_at.isoformat(),+ "content_sha256": content_sha256,+ }+ return Document(+ id=f"feishu_wiki:{self.space_id}:{node_token}",+ source=DocumentSource.FEISHU_WIKI,+ semantic_identifier=title,+ extension=get_file_ext(title),+ blob=blob,+ doc_updated_at=updated_at,+ size_bytes=len(blob),+ metadata=metadata,+ fingerprint=fingerprint,+ )++ def _yield_documents(+ self,+ *,+ start: SecondsSinceUnixEpoch | None,+ end: SecondsSinceUnixEpoch | None,+ ) -> GenerateDocumentsOutput:+ fallback_updated_at = datetime.fromtimestamp(end, tz=UTC) if end is not None else datetime.now(UTC)+ batch: list[Document] = []+ for node in self._iter_nodes(self.root_node_token):+ if node.get("obj_type") != "file":+ continue+ title = str(node.get("title") or "")+ if not self._matches_filters(title):+ continue+ updated_at = _parse_timestamp(node.get("obj_edit_time"))+ if not self._is_within_window(updated_at, start, end):+ continue+ try:+ batch.append(self._node_to_document(node, fallback_updated_at=fallback_updated_at))+ except ConnectorValidationError:+ raise+ except Exception as exc:+ node_token = str(node.get("node_token") or "<unknown>")+ LOGGER.exception("Failed to read Feishu Wiki node %s", node_token)+ raise ConnectorValidationError(f"Failed to read Feishu Wiki node {node_token}") from exc+ if len(batch) >= self.batch_size:+ yield batch+ batch = []+ if batch:+ yield batch++ def validate_connector_settings(self) -> None:+ self._ensure_configured()+ self._get_access_token()+ next(self._iter_child_pages(self.root_node_token), [])++ def load_from_state(self) -> GenerateDocumentsOutput:+ yield from self._yield_documents(start=None, end=None)++ def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> GenerateDocumentsOutput:+ yield from self._yield_documents(start=start, end=end)diff --git a/docs/guides/data_source/data_source_categories_and_selection.md b/docs/guides/data_source/data_source_categories_and_selection.mdindex 8949f8d8b9..1cb3f46f77 100644--- a/docs/guides/data_source/data_source_categories_and_selection.md+++ b/docs/guides/data_source/data_source_categories_and_selection.md@@ -12,7 +12,7 @@ sidebar_custom_props: {RAGFlow supports many data sources. It is recommended to first select a major category according to the content source, and then go to the corresponding data source configuration. The following categories help users quickly determine which connection method to choose.-**Documents and collaboration platforms**: Used to synchronize team documents, wikis, cloud drive files, and collaborative documents. This category is suitable for building enterprise knowledge bases or project repositories. It includes Confluence, Notion, Google Drive, OneDrive, SharePoint, Box, Dropbox, WebDAV, and SeaFile.+**Documents and collaboration platforms**: Used to synchronize team documents, wikis, cloud drive files, and collaborative documents. This category is suitable for building enterprise knowledge bases or project repositories. It includes Confluence, Notion, Feishu Wiki, Google Drive, OneDrive, SharePoint, Box, Dropbox, WebDAV, and SeaFile.**Object storage**: Used to synchronize large numbers of files in buckets or containers. This category is suitable for historical data archiving, batch file import, and centralized management of cloud files. It includes S3, Google Cloud Storage, Oracle Storage, R2, and Azure Blob Storage.diff --git a/docs/guides/data_source/data_source_configuration.md b/docs/guides/data_source/data_source_configuration.mdindex 5751793678..ead4e0a97b 100644--- a/docs/guides/data_source/data_source_configuration.md+++ b/docs/guides/data_source/data_source_configuration.md@@ -67,6 +67,35 @@ The Google Drive data source is used to synchronize files or folders in Google D+## Feishu Wiki++The Feishu Wiki data source is available with the Python backend. It recursively scans a configured Wiki subtree and imports matching downloadable `file` nodes. Native Feishu Docs, Sheets, Slides, and Bitable records are not imported directly, although the connector traverses non-file nodes to discover downloadable files below them.++The Feishu custom app needs read-only Wiki and Drive permissions. Add the app to the target Wiki with permission to list nodes and download files.++The connector supports this file subset: `csv`, `doc`, `docx`, `eml`, `gif`, `html`, `jpeg`, `jpg`, `json`, `md`, `mdx`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `txt`, `xls`, and `xlsx`.++Configure the following fields:++- **Name**: A descriptive name for the connection in RAGFlow.+- **Feishu app ID** and **Feishu app secret**: Credentials for the Feishu custom app. The secret is masked in the form.+- **Wiki space ID**: The ID of the target Wiki space.+- **Wiki root node token**: The node whose descendants are scanned recursively.+- **Allowed file extensions**: An optional allow-list from the supported subset above. Leave it empty to allow the complete subset.+- **Required filename keywords**: An optional list. A filename must contain at least one configured keyword.+- **Excluded filename keywords**: An optional list. A matching filename is excluded.+- **Maximum file size (bytes)**: The largest file body the connector will accept. The default is `52428800` bytes (50 MiB). Both the declared content length and the streamed byte count are enforced.+- **Batch size**: The number of files sent to the ingestion pipeline per batch. The default is `2`; valid values are `1` through `10`.+- **Refresh interval**: The interval between periodic scans.++Extension and filename filters are applied before a file body is downloaded. Every periodic sync scans node metadata in the subtree, but downloads only matching files whose edit time falls within the completed sync window. An empty completed scan advances that window, so the same time interval is not treated as pending again.++File downloads retry HTTP 429 responses up to three times. The connector honors the relative delay in seconds from Feishu's `x-ogw-ratelimit-reset` response header, with a minimum wait of one second. If the header is missing or invalid, retries wait one, two, then four seconds. A requested delay above 60 seconds stops the download instead of retrying before the server's reset. See [Feishu's rate-limit guide](https://open.feishu.cn/document/server-docs/api-call-guide/frequency-control).++Unsuccessful downloads raise a connector validation error with the HTTP status and, when available, Feishu's error code, message, and request log ID. Error-body parsing is limited to 8 KiB; diagnostic fields are length-limited and credentials are redacted. Successful file bodies, including JSON files, are imported unchanged. After retries are exhausted, the sync fails without advancing its successful window.++Normal periodic sync does not propagate source deletions: deleting a file in Feishu does not immediately delete the imported RAGFlow document. A manual rebuild follows the existing delete-then-import behavior, so previously imported files that are no longer present or no longer match the filters can be removed during the rebuild.+## OneDriveThe OneDrive data source is used to synchronize files in OneDrive or OneDrive for Business to a RAGFlow knowledge base. After configuration, personal or department cloud files can be queried in a unified way.diff --git a/rag/svr/feishu_wiki_sync.py b/rag/svr/feishu_wiki_sync.pynew file mode 100644index 0000000000..4970007096--- /dev/null+++ b/rag/svr/feishu_wiki_sync.py@@ -0,0 +1,26 @@+"""Small adapter between the Feishu Wiki connector and RAGFlow sync windows."""++from __future__ import annotations++from datetime import UTC, datetime+from typing import Any++from common.data_source import FeishuWikiConnector+++def _build_feishu_wiki_generator(+ conf: dict[str, Any],+ task: dict[str, Any],+ *,+ window_end: datetime | None = None,+):+ """Build a connector and select full or incremental document generation."""+ connector = FeishuWikiConnector.build_connector(conf)+ if task.get("reindex") == "1" or task.get("poll_range_start") is None:+ return connector, connector.load_from_state()++ effective_end = window_end or datetime.now(UTC)+ return connector, connector.poll_source(+ task["poll_range_start"].timestamp(),+ effective_end.timestamp(),+ )diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.pyindex 825a20ca85..e6efaed256 100644--- a/rag/svr/sync_data_source.py+++ b/rag/svr/sync_data_source.py@@ -87,6 +87,7 @@ from common.data_source.exceptions import ConnectorValidationErrorfrom common.log_utils import init_root_loggerfrom common.signal_utils import start_tracemalloc_and_snapshot, stop_tracemallocfrom common.versions import get_ragflow_version+from rag.svr.feishu_wiki_sync import _build_feishu_wiki_generatorfrom box_sdk_gen import BoxOAuth, OAuthConfig, AccessTokenMAX_CONCURRENT_TASKS = int(os.environ.get("MAX_CONCURRENT_TASKS", "5"))@@ -147,6 +148,7 @@ class SyncBase:"""SOURCE_NAME: str = None+ RAISE_ON_BATCH_ERROR = Falsedef __init__(self, conf: dict) -> None:self.conf = conf@@ -242,6 +244,7 @@ class SyncBase:added_docs = 0updated_docs = 0next_update = datetime(1970, 1, 1, tzinfo=timezone.utc)+ saw_documents = Falsesource_type = f"{self.SOURCE_NAME}/{task['connector_id']}"existing_doc_ids = {doc["id"]@@ -258,6 +261,7 @@ class SyncBase:if not document_batch:continue+ saw_documents = Truemax_update = max(doc.doc_updated_at for doc in document_batch)next_update = max(next_update, max_update)@@ -284,6 +288,8 @@ class SyncBase:err, dids = SyncLogsService.duplicate_and_parse(kb, docs, task["tenant_id"], f"{self.SOURCE_NAME}/{task['connector_id']}", task["auto_parse"])if err:had_parse_errors = True+ if self.RAISE_ON_BATCH_ERROR:+ raise RuntimeError(f"{self.SOURCE_NAME} failed to process {len(err)} document(s)")SyncLogsService.increase_docs(task["id"], max_update, len(docs), "\n".join(err), len(err))changed_doc_ids = set(dids)updated_in_batch = len(changed_doc_ids & existing_doc_ids)@@ -301,9 +307,14 @@ class SyncBase:else:logging.error(f"Error processing batch: {msg}")+ if self.RAISE_ON_BATCH_ERROR:+ raisefailed_docs += len(docs)continue+ if not saw_documents:+ next_update = self._get_empty_sync_cursor(task, next_update)+prefix = self._get_source_prefix()prefix = f"{prefix} " if prefix else ""next_update_info = self._format_window_boundary(next_update)@@ -319,6 +330,10 @@ class SyncBase:SyncLogsService.done(task["id"], task["connector_id"])task["poll_range_start"] = next_update+ def _get_empty_sync_cursor(self, task: dict, current_cursor: datetime) -> datetime:+ del task+ return current_cursor+async def _run_prune_task_logic(self, task: dict):if not self.conf.get("sync_deleted_files"):SyncLogsService.done(task["id"], task["connector_id"])@@ -1392,6 +1407,32 @@ class Slack(SyncBase):return document_generator+class FeishuWiki(SyncBase):+ """Synchronize downloadable files from a Feishu Wiki subtree."""++ SOURCE_NAME: str = FileSource.FEISHU_WIKI+ RAISE_ON_BATCH_ERROR = True++ async def _generate(self, task: dict):+ self._poll_window_end = datetime.now(UTC)+ self.connector, document_generator = _build_feishu_wiki_generator(+ self.conf,+ task,+ window_end=self._poll_window_end,+ )+ self.log_connection(+ "Feishu Wiki",+ f"space={self.conf.get('space_id', '<missing>')}",+ task,+ )+ return document_generator++ def _get_empty_sync_cursor(self, task: dict, current_cursor: datetime) -> datetime:+ if task.get("reindex") != "1" and task.get("poll_range_start") is not None:+ return self._poll_window_end+ return current_cursor++class Teams(SyncBase):SOURCE_NAME: str = FileSource.TEAMS@@ -2316,6 +2357,7 @@ func_factory = {FileSource.CONFLUENCE: Confluence,FileSource.GMAIL: Gmail,FileSource.GOOGLE_DRIVE: GoogleDrive,+ FileSource.FEISHU_WIKI: FeishuWiki,FileSource.JIRA: Jira,FileSource.SHAREPOINT: SharePoint,FileSource.ONEDRIVE: OneDrive,diff --git a/test/unit_test/data_source/test_feishu_wiki_connector.py b/test/unit_test/data_source/test_feishu_wiki_connector.pynew file mode 100644index 0000000000..f6e49f98b6--- /dev/null+++ b/test/unit_test/data_source/test_feishu_wiki_connector.py@@ -0,0 +1,683 @@+import hashlib+import importlib+import json+import sys+from datetime import UTC, datetime+from types import ModuleType, SimpleNamespace++import pytest+from requests.structures import CaseInsensitiveDict++import common+from common.constants import FileSource+from test.unit_test.feishu_wiki_test_support import install_unrelated_provider_stubs++_MISSING = object()+++def _data_source_modules() -> dict[str, ModuleType]:+ return {name: module for name, module in sys.modules.items() if name == "common.data_source" or name.startswith("common.data_source.")}+++def _load_real_data_source_stack() -> SimpleNamespace:+ saved_modules = _data_source_modules()+ saved_parent_attr = getattr(common, "data_source", _MISSING)+ for name in list(_data_source_modules()):+ sys.modules.pop(name, None)+ if hasattr(common, "data_source"):+ del common.data_source++ install_unrelated_provider_stubs()+ try:+ registry = importlib.import_module("common.data_source")+ config = importlib.import_module("common.data_source.config")+ exceptions = importlib.import_module("common.data_source.exceptions")+ interfaces = importlib.import_module("common.data_source.interfaces")+ models = importlib.import_module("common.data_source.models")+ utils = importlib.import_module("common.data_source.utils")+ return SimpleNamespace(+ registry=registry,+ DocumentSource=config.DocumentSource,+ ConnectorMissingCredentialError=exceptions.ConnectorMissingCredentialError,+ ConnectorValidationError=exceptions.ConnectorValidationError,+ LoadConnector=interfaces.LoadConnector,+ PollConnector=interfaces.PollConnector,+ Document=models.Document,+ get_file_ext=utils.get_file_ext,+ )+ finally:+ for name in list(_data_source_modules()):+ sys.modules.pop(name, None)+ sys.modules.update(saved_modules)+ if saved_parent_attr is _MISSING:+ if hasattr(common, "data_source"):+ del common.data_source+ else:+ common.data_source = saved_parent_attr+++_DATA_SOURCE_STATE_BEFORE = (_data_source_modules(), getattr(common, "data_source", _MISSING))+_real = _load_real_data_source_stack()+_DATA_SOURCE_STATE_AFTER = (_data_source_modules(), getattr(common, "data_source", _MISSING))++CONNECTOR_BY_SOURCE = _real.registry.CONNECTOR_BY_SOURCE+FeishuWikiConnector = _real.registry.FeishuWikiConnector+DocumentSource = _real.DocumentSource+ConnectorMissingCredentialError = _real.ConnectorMissingCredentialError+ConnectorValidationError = _real.ConnectorValidationError+LoadConnector = _real.LoadConnector+PollConnector = _real.PollConnector+Document = _real.Document+get_file_ext = _real.get_file_ext+++class FakeResponse:+ def __init__(self, *, payload=None, content=b"", chunks=None, headers=None, status_code=200):+ self._payload = payload+ self.content = content+ self.chunks = chunks+ self.headers = CaseInsensitiveDict(headers or {})+ self.status_code = status_code+ self.closed = 0+ self.iterated = 0++ def json(self):+ return self._payload++ def iter_content(self, chunk_size):+ self.iterated += 1+ del chunk_size+ if self.chunks is not None:+ yield from self.chunks+ elif self.content:+ yield self.content++ def close(self):+ self.closed += 1+++class FakeSession:+ def __init__(self, responses):+ self.responses = list(responses)+ self.requests = []+ self.downloaded_tokens = []++ def request(self, method, url, **kwargs):+ self.requests.append((method, url, kwargs))+ if "/drive/v1/files/" in url:+ self.downloaded_tokens.append(url.split("/files/", 1)[1].split("/", 1)[0])+ if not self.responses:+ raise AssertionError(f"unexpected request: {method} {url}")+ response = self.responses.pop(0)+ if isinstance(response, Exception):+ raise response+ return response+++def _success(payload):+ return FakeResponse(payload={"code": 0, "msg": "success", "data": payload})+++def _auth_success():+ return FakeResponse(+ payload={+ "code": 0,+ "msg": "success",+ "tenant_access_token": "token",+ "expire": 7200,+ }+ )+++def _build_connector(session, **overrides):+ config = {+ "space_id": "space-1",+ "root_node_token": "root-node",+ "batch_size": 2,+ "include_extensions": [],+ "include_keywords": [],+ "exclude_keywords": [],+ "credentials": {"app_id": "cli_test", "app_secret": "secret"},+ **overrides,+ }+ connector = FeishuWikiConnector.build_connector(config)+ connector.session = session+ return connector+++def test_real_registry_and_connector_contracts_are_wired():+ assert _DATA_SOURCE_STATE_AFTER[0] == _DATA_SOURCE_STATE_BEFORE[0]+ assert _DATA_SOURCE_STATE_AFTER[1] is _DATA_SOURCE_STATE_BEFORE[1]+ assert CONNECTOR_BY_SOURCE[FileSource.FEISHU_WIKI] is FeishuWikiConnector+ assert issubclass(FeishuWikiConnector, (LoadConnector, PollConnector))+ assert DocumentSource.FEISHU_WIKI.value == FileSource.FEISHU_WIKI.value+ assert get_file_ext("guide.PDF") == ".pdf"+ assert "fingerprint" in Document.model_fields++ saved_modules = _data_source_modules()+ saved_parent_attr = getattr(common, "data_source", _MISSING)+ sentinel = ModuleType("common.data_source")+ for name in list(saved_modules):+ sys.modules.pop(name, None)+ sys.modules["common.data_source"] = sentinel+ common.data_source = sentinel+ try:+ isolated = _load_real_data_source_stack()+ assert isolated.registry.CONNECTOR_BY_SOURCE[FileSource.FEISHU_WIKI] is isolated.registry.FeishuWikiConnector+ assert sys.modules["common.data_source"] is sentinel+ assert common.data_source is sentinel+ finally:+ for name in list(_data_source_modules()):+ sys.modules.pop(name, None)+ sys.modules.update(saved_modules)+ if saved_parent_attr is _MISSING:+ del common.data_source+ else:+ common.data_source = saved_parent_attr+++def test_build_connector_requires_app_credentials():+ with pytest.raises(ConnectorMissingCredentialError):+ FeishuWikiConnector.build_connector({"space_id": "space-1", "root_node_token": "root-node"})+++def test_screening_happens_before_download_and_nested_files_are_discovered():+ session = FakeSession(+ [+ _auth_success(),+ _success(+ {+ "items": [+ {+ "node_token": "folder-node",+ "obj_token": "folder-object",+ "obj_type": "docx",+ "title": "Folder",+ "has_child": True,+ "obj_edit_time": "1767225600",+ },+ {+ "node_token": "obsolete-node",+ "obj_token": "obsolete-token",+ "obj_type": "file",+ "title": "obsolete-file.pdf",+ "has_child": False,+ "obj_edit_time": "1767225600",+ },+ ],+ "has_more": False,+ }+ ),+ _success(+ {+ "items": [+ {+ "node_token": "current-node",+ "parent_node_token": "folder-node",+ "obj_token": "current-file-token",+ "obj_type": "file",+ "title": "project-overview.pdf",+ "has_child": False,+ "obj_edit_time": "1767225600",+ },+ {+ "node_token": "image-node",+ "parent_node_token": "folder-node",+ "obj_token": "image-token",+ "obj_type": "file",+ "title": "project-overview.png",+ "has_child": False,+ "obj_edit_time": "1767225600",+ },+ ],+ "has_more": False,+ }+ ),+ FakeResponse(content=b"current pdf"),+ ]+ )+ connector = _build_connector(+ session,+ include_extensions=["pdf"],+ exclude_keywords=["obsolete"],+ )++ documents = [document for batch in connector.load_from_state() for document in batch]++ assert [document.semantic_identifier for document in documents] == ["project-overview.pdf"]+ assert session.downloaded_tokens == ["current-file-token"]+++def test_pagination_incremental_window_metadata_and_fingerprint_are_preserved():+ session = FakeSession(+ [+ _auth_success(),+ _success(+ {+ "items": [+ {+ "node_token": "old-node",+ "obj_token": "old-token",+ "obj_type": "file",+ "title": "old.pdf",+ "has_child": False,+ "obj_edit_time": "1767225600",+ }+ ],+ "has_more": True,+ "page_token": "page-2",+ }+ ),+ _success(+ {+ "items": [+ {+ "node_token": "current-node",+ "parent_node_token": "root-node",+ "obj_token": "current-token",+ "obj_type": "file",+ "title": "project-plan.docx",+ "has_child": False,+ "obj_edit_time": "1767398400",+ }+ ],+ "has_more": False,+ }+ ),+ FakeResponse(content=b"current body"),+ ]+ )+ connector = _build_connector(session, include_keywords=["project"])++ documents = [document for batch in connector.poll_source(1767312000, 1767484800) for document in batch]++ assert len(documents) == 1+ document = documents[0]+ assert document.id == "feishu_wiki:space-1:current-node"+ assert document.doc_updated_at == datetime(2026, 1, 3, tzinfo=UTC)+ assert document.extension == ".docx"+ assert document.fingerprint == hashlib.sha256(b"current body").hexdigest()[:32]+ assert len(document.fingerprint) == 32+ assert document.metadata == {+ "source": "feishu_wiki",+ "wiki_space_id": "space-1",+ "wiki_node_token": "current-node",+ "wiki_object_token": "current-token",+ "wiki_object_type": "file",+ "wiki_parent_node_token": "root-node",+ "wiki_url": "https://feishu.cn/wiki/current-node",+ "source_updated_at": "2026-01-03T00:00:00+00:00",+ "content_sha256": hashlib.sha256(b"current body").hexdigest(),+ }+ assert session.requests[2][2]["params"]["page_token"] == "page-2"+++def test_validate_surfaces_feishu_api_errors_without_secrets():+ session = FakeSession(+ [+ _auth_success(),+ FakeResponse(payload={"code": 99991663, "msg": "permission denied"}),+ ]+ )+ connector = _build_connector(session)++ with pytest.raises(ConnectorValidationError, match="permission denied") as error:+ connector.validate_connector_settings()++ assert "secret" not in str(error.value)+ assert "token" not in str(error.value)+++def test_validate_surfaces_feishu_error_body_for_http_400_without_secrets():+ session = FakeSession(+ [+ _auth_success(),+ FakeResponse(+ status_code=400,+ payload={+ "code": 131006,+ "msg": "the parent node does not belong to the space",+ "data": {"app_secret": "must-not-leak"},+ },+ ),+ ]+ )+ connector = _build_connector(session)++ with pytest.raises(+ ConnectorValidationError,+ match=(+ r"Feishu API error 131006 \(HTTP 400\) while attempting to "+ r"list Wiki nodes: the parent node does not belong to the space"+ ),+ ) as error:+ connector.validate_connector_settings()++ assert "must-not-leak" not in str(error.value)+ assert "app_secret" not in str(error.value)+++def _prepare_download(connector):+ connector._access_token = "cached-access-token"+ connector._access_token_expires_at = float("inf")+++def test_download_rejects_declared_oversize_before_reading_body():+ response = FakeResponse(content=b"large", headers={"Content-Length": "5"})+ connector = _build_connector(FakeSession([response]), max_file_size_bytes=4)+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError, match=r"size.*limit|limit.*size"):+ connector._download_file("file-1")++ assert response.iterated == 0+ assert response.closed == 1+++def test_download_rejects_body_exceeding_limit_without_content_length():+ response = FakeResponse(chunks=[b"abc", b"de"])+ connector = _build_connector(FakeSession([response]), max_file_size_bytes=4)+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError, match=r"size.*limit|limit.*size"):+ connector._download_file("file-1")++ assert response.closed == 1+++def test_download_accepts_body_at_exact_limit_and_escapes_object_token():+ response = FakeResponse(chunks=[b"ab", b"cd"], headers={"Content-Length": "4"})+ session = FakeSession([response])+ connector = _build_connector(session, max_file_size_bytes=4)+ _prepare_download(connector)++ assert connector._download_file("folder/file ?") == b"abcd"+ assert session.requests[0][1].endswith("/folder%2Ffile%20%3F/download")+ assert session.requests[0][2]["stream"] is True+ assert response.closed == 1+++def test_download_closes_response_and_wraps_stream_read_failure():+ def failing_chunks():+ yield b"ab"+ raise OSError("socket reset")++ response = FakeResponse(chunks=failing_chunks())+ connector = _build_connector(FakeSession([response]))+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError, match="download") as error:+ connector._download_file("file-1")++ assert "socket reset" not in str(error.value)+ assert response.closed == 1+++def test_download_permission_failure_uses_connector_validation_contract():+ response = FakeResponse(status_code=403, payload={"code": 99991663, "msg": "forbidden"})+ connector = _build_connector(FakeSession([response]))+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError, match="permission"):+ connector._download_file("file-1")++ assert response.closed == 1++[email protected]("status_code", [300, 400, 403, 404, 500])+def test_download_surfaces_bounded_api_error_details(status_code):+ response = FakeResponse(+ status_code=status_code,+ content=json.dumps({"code": 99991672, "msg": "Access denied", "data": {"private": "must-not-leak"}}).encode(),+ headers={"X-Tt-Logid": "request-123"},+ )+ session = FakeSession([response])+ connector = _build_connector(session)+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError) as error:+ connector._download_file("file-1")++ detail = str(error.value)+ assert f"Feishu API error 99991672 (HTTP {status_code})" in detail+ assert "Access denied" in detail+ assert "logid=request-123" in detail+ assert "must-not-leak" not in detail+ assert len(session.requests) == 1+ assert response.closed == 1++[email protected]("body", [b"<html>gateway failed</html>", b"[]", b"\xff", b"{}"])+def test_download_keeps_status_and_logid_for_invalid_error_bodies(body):+ response = FakeResponse(status_code=502, content=body, headers={"x-tt-logid": "gateway-123"})+ connector = _build_connector(FakeSession([response]))+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError) as error:+ connector._download_file("file-1")++ assert "HTTP 502" in str(error.value)+ assert "logid=gateway-123" in str(error.value)+ assert "gateway failed" not in str(error.value)+ assert response.closed == 1+++def test_download_does_not_read_an_unbounded_error_body():+ def chunks():+ for _ in range(8):+ yield b"x" * 1024+ pytest.fail("error response was read beyond the 8 KiB limit")++ response = FakeResponse(status_code=503, chunks=chunks(), headers={"x-tt-logid": "large-error"})+ connector = _build_connector(FakeSession([response]))+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError) as error:+ connector._download_file("file-1")++ assert "HTTP 503" in str(error.value)+ assert "logid=large-error" in str(error.value)+ assert response.closed == 1+++def test_download_redacts_credentials_and_bounds_error_fields():+ response = FakeResponse(+ status_code=400,+ content=json.dumps({"code": 123, "msg": "secret cached-access-token\n" + "x" * 2000}).encode(),+ headers={"x-tt-logid": "secret\n" + "y" * 2000},+ )+ connector = _build_connector(FakeSession([response]))+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError) as error:+ connector._download_file("file-1")++ detail = str(error.value)+ assert "secret" not in detail+ assert "cached-access-token" not in detail+ assert "\n" not in detail+ assert len(detail) < 700+ assert response.closed == 1+++def test_download_preserves_successful_json_file_content():+ content = b'{"code":123,"msg":"this is an actual user file"}'+ response = FakeResponse(content=content, headers={"Content-Type": "application/json"})+ connector = _build_connector(FakeSession([response]))+ _prepare_download(connector)++ assert connector._download_file("file-1") == content+ assert response.closed == 1+++def test_download_retries_429_after_reset_and_closes_before_waiting(monkeypatch):+ throttled = FakeResponse(status_code=429, headers={"X-Ogw-Ratelimit-Reset": "2"})+ success = FakeResponse(content=b"downloaded file")+ session = FakeSession([throttled, success])+ connector = _build_connector(session)+ _prepare_download(connector)+ sleeps = []++ def sleep(delay):+ assert throttled.closed == 1+ sleeps.append(delay)++ monkeypatch.setattr("time.sleep", sleep)++ assert connector._download_file("file-1") == b"downloaded file"+ assert sleeps == [2]+ assert session.downloaded_tokens == ["file-1", "file-1"]+ assert session.requests[0] == session.requests[1]+ assert throttled.iterated == 0+ assert success.closed == 1++[email protected]("reset", [None, "invalid", "-10", "NaN", "Infinity"])+def test_download_uses_bounded_backoff_when_reset_is_missing_or_invalid(monkeypatch, reset):+ headers = {} if reset is None else {"x-ogw-ratelimit-reset": reset}+ throttled = [FakeResponse(status_code=429, headers=headers) for _ in range(3)]+ session = FakeSession([*throttled, FakeResponse(content=b"complete")])+ connector = _build_connector(session)+ _prepare_download(connector)+ sleeps = []+ monkeypatch.setattr("time.sleep", sleeps.append)++ assert connector._download_file("file-1") == b"complete"+ assert sleeps == [1, 2, 4]+ assert all(response.closed == 1 for response in throttled)+++def test_download_retry_exhaustion_preserves_final_error(monkeypatch):+ responses = [+ FakeResponse(+ status_code=429,+ content=b'{"code":99991400,"msg":"request limited"}',+ headers={"x-tt-logid": f"attempt-{attempt}"},+ )+ for attempt in range(4)+ ]+ session = FakeSession(responses)+ connector = _build_connector(session)+ _prepare_download(connector)+ sleeps = []+ monkeypatch.setattr("time.sleep", sleeps.append)++ with pytest.raises(ConnectorValidationError) as error:+ connector._download_file("file-1")++ assert "99991400" in str(error.value)+ assert "HTTP 429" in str(error.value)+ assert "logid=attempt-3" in str(error.value)+ assert sleeps == [1, 2, 4]+ assert len(session.requests) == 4+ assert all(response.closed == 1 for response in responses)+++def test_download_does_not_retry_before_an_excessive_server_reset(monkeypatch):+ response = FakeResponse(status_code=429, headers={"x-ogw-ratelimit-reset": "3600"})+ session = FakeSession([response])+ connector = _build_connector(session)+ _prepare_download(connector)+ sleeps = []+ monkeypatch.setattr("time.sleep", sleeps.append)++ with pytest.raises(ConnectorValidationError, match="HTTP 429"):+ connector._download_file("file-1")++ assert sleeps == []+ assert len(session.requests) == 1+ assert response.closed == 1++[email protected]("reset,expected_delay", [("0", 1), ("60", 60)])+def test_download_retry_handles_reset_boundaries(monkeypatch, reset, expected_delay):+ response = FakeResponse(status_code=429, headers={"x-ogw-ratelimit-reset": reset})+ connector = _build_connector(FakeSession([response, FakeResponse(content=b"complete")]))+ _prepare_download(connector)+ sleeps = []+ monkeypatch.setattr("time.sleep", sleeps.append)++ assert connector._download_file("file-1") == b"complete"+ assert sleeps == [expected_delay]+ assert response.closed == 1+++ "headers,payload,logid",+ [+ ({"X-Request-Id": "request-id"}, {}, "request-id"),+ ({}, {"error": {"logid": "body-log-id"}}, "body-log-id"),+ ({"X-Tt-Logid": "header-id"}, {"error": {"logid": "body-id"}}, "header-id"),+ ],+)+def test_download_preserves_available_request_identifiers(headers, payload, logid):+ response = FakeResponse(status_code=400, headers=headers, content=json.dumps(payload).encode())+ connector = _build_connector(FakeSession([response]))+ _prepare_download(connector)++ with pytest.raises(ConnectorValidationError) as error:+ connector._download_file("file-1")++ assert f"logid={logid}" in str(error.value)+ assert response.closed == 1+++def test_full_sync_imports_file_after_a_throttled_download(monkeypatch):+ session = FakeSession(+ [+ _auth_success(),+ _success({"items": [{"node_token": "node-1", "obj_token": "file-1", "obj_type": "file", "title": "guide.pdf"}], "has_more": False}),+ FakeResponse(status_code=429),+ FakeResponse(content=b"file after retry"),+ ]+ )+ connector = _build_connector(session)+ sleeps = []+ monkeypatch.setattr("time.sleep", sleeps.append)++ documents = [document for batch in connector.load_from_state() for document in batch]++ assert len(documents) == 1+ assert documents[0].blob == b"file after retry"+ assert documents[0].semantic_identifier == "guide.pdf"+ assert sleeps == [1]+++def test_validate_wraps_transport_failure_without_credentials():+ import requests++ connector = _build_connector(FakeSession([requests.ConnectionError("secret transport detail")]))++ with pytest.raises(ConnectorValidationError, match="connect|authenticate") as error:+ connector.validate_connector_settings()++ assert "secret" not in str(error.value)+++def test_http_permission_failure_is_a_connector_validation_error():+ response = FakeResponse(status_code=403, payload={"code": 99991663, "msg": "forbidden"})++ with pytest.raises(ConnectorValidationError, match="permission"):+ FeishuWikiConnector._decode_api_payload(response, "list Wiki nodes")++[email protected]("batch_size", [0, 11])+def test_batch_size_must_be_between_one_and_ten(batch_size):+ with pytest.raises(ConnectorValidationError, match="batch_size"):+ _build_connector(FakeSession([]), batch_size=batch_size)+++def test_default_limits_are_bounded():+ connector = _build_connector(FakeSession([]), batch_size=None)++ assert connector.batch_size == 2+ assert connector.max_file_size_bytes == 50 * 1024 * 1024+++def test_supported_extensions_match_the_document_and_image_upload_subset():+ connector = _build_connector(FakeSession([]))++ assert connector._matches_filters("guide.pdf")+ assert connector._matches_filters("diagram.tif")+ assert not connector._matches_filters("scan.tiff")+ assert not connector._matches_filters("archive.zip")diff --git a/test/unit_test/feishu_wiki_test_support.py b/test/unit_test/feishu_wiki_test_support.pynew file mode 100644index 0000000000..f79381b344--- /dev/null+++ b/test/unit_test/feishu_wiki_test_support.py@@ -0,0 +1,59 @@+"""Focused import support for Feishu Wiki tests without the global conftest."""++import sys+from types import ModuleType++_UNRELATED_PROVIDERS = {+ "airtable_connector": "AirtableConnector",+ "asana_connector": "AsanaConnector",+ "azure_blob_connector": "AzureBlobConnector",+ "bigquery_connector": "BigQueryConnector",+ "blob_connector": "BlobStorageConnector",+ "box_connector": "BoxConnector",+ "confluence_connector": "ConfluenceConnector",+ "dingtalk_ai_table_connector": "DingTalkAITableConnector",+ "discord_connector": "DiscordConnector",+ "dropbox_connector": "DropboxConnector",+ "gitlab_connector": "GitlabConnector",+ "gmail_connector": "GmailConnector",+ "imap_connector": "ImapConnector",+ "moodle_connector": "MoodleConnector",+ "notion_connector": "NotionConnector",+ "onedrive_connector": "OneDriveConnector",+ "outlook_connector": "OutlookConnector",+ "rdbms_connector": "RDBMSConnector",+ "rest_api_connector": "RestAPIConnector",+ "rss_connector": "RSSConnector",+ "salesforce_connector": "SalesforceConnector",+ "seafile_connector": "SeaFileConnector",+ "sharepoint_connector": "SharePointConnector",+ "sitemap_connector": "SitemapConnector",+ "slack_connector": "SlackConnector",+ "teams_connector": "TeamsConnector",+ "webdav_connector": "WebDAVConnector",+ "xquik_connector": "XquikConnector",+ "zendesk_connector": "ZendeskConnector",+ "azure_devops.connector": "AzureDevOpsConnector",+ "bitbucket.connector": "BitbucketConnector",+ "github.connector": "GithubConnector",+ "google_drive.connector": "GoogleDriveConnector",+ "jira.connector": "JiraConnector",+}+++def install_unrelated_provider_stubs(*, include_sitemap_runtime: bool = False) -> None:+ """Install import-only substitutes for providers outside the focused tests."""+ for relative_name, class_name in _UNRELATED_PROVIDERS.items():+ module_name = f"common.data_source.{relative_name}"+ module = ModuleType(module_name)+ setattr(module, class_name, type(class_name, (), {}))+ if relative_name == "sitemap_connector" and include_sitemap_runtime:+ module.iter_in_worker_thread = lambda value, **_kwargs: value+ module.validate_connector_in_thread = lambda *_args, **_kwargs: None+ sys.modules[module_name] = module++ if "." in relative_name:+ package_name = module_name.rsplit(".", 1)[0]+ package = ModuleType(package_name)+ package.__path__ = []+ sys.modules[package_name] = packagediff --git a/test/unit_test/rag/test_feishu_wiki_sync_adapter.py b/test/unit_test/rag/test_feishu_wiki_sync_adapter.pynew file mode 100644index 0000000000..41149fae52--- /dev/null+++ b/test/unit_test/rag/test_feishu_wiki_sync_adapter.py@@ -0,0 +1,107 @@+import importlib.util+import sys+from datetime import UTC, datetime+from pathlib import Path+from types import ModuleType+from typing import ClassVar+++class FakeConnector:+ def __init__(self):+ self.poll_calls = []+ self.full_calls = 0++ def load_from_state(self):+ self.full_calls += 1+ return iter((["full"],))++ def poll_source(self, start, end):+ self.poll_calls.append((start, end))+ return iter((["incremental"],))+++fake_connector = FakeConnector()+++class FakeConnectorFactory:+ configs: ClassVar[list] = []++ @classmethod+ def build_connector(cls, config):+ cls.configs.append(config)+ return fake_connector+++def _load_adapter_module():+ repo_root = Path(__file__).resolve().parents[3]+ adapter_path = repo_root / "rag" / "svr" / "feishu_wiki_sync.py"+ if not adapter_path.exists():+ return None++ saved_data_source = sys.modules.get("common.data_source")+ data_source_stub = ModuleType("common.data_source")+ data_source_stub.FeishuWikiConnector = FakeConnectorFactory+ sys.modules["common.data_source"] = data_source_stub+ try:+ spec = importlib.util.spec_from_file_location(+ "_feishu_wiki_sync_adapter_under_test",+ adapter_path,+ )+ module = importlib.util.module_from_spec(spec)+ assert spec.loader is not None+ spec.loader.exec_module(module)+ return module+ finally:+ if saved_data_source is None:+ sys.modules.pop("common.data_source", None)+ else:+ sys.modules["common.data_source"] = saved_data_source+++adapter_module = _load_adapter_module()+++def _build_generator():+ assert adapter_module is not None, "Feishu Wiki sync adapter is not implemented"+ return adapter_module._build_feishu_wiki_generator+++def setup_function():+ fake_connector.poll_calls.clear()+ fake_connector.full_calls = 0+ FakeConnectorFactory.configs.clear()+++def test_manual_rebuild_scans_from_the_root():+ build_generator = _build_generator()+ config = {"space_id": "space-1"}++ connector, batches = build_generator(+ config,+ {"reindex": "1", "poll_range_start": None},+ window_end=datetime(2026, 1, 2, tzinfo=UTC),+ )++ assert connector is fake_connector+ assert list(batches) == [["full"]]+ assert fake_connector.full_calls == 1+ assert fake_connector.poll_calls == []+ assert FakeConnectorFactory.configs == [config]+++def test_automatic_sync_uses_the_previous_successful_window():+ build_generator = _build_generator()++ connector, batches = build_generator(+ {"space_id": "space-1"},+ {+ "reindex": "0",+ "poll_range_start": datetime(2026, 1, 1, tzinfo=UTC),+ },+ window_end=datetime(2026, 1, 2, tzinfo=UTC),+ )++ assert connector is fake_connector+ assert list(batches) == [["incremental"]]+ assert fake_connector.full_calls == 0+ assert fake_connector.poll_calls == [(1767225600.0, 1767312000.0)]diff --git a/test/unit_test/rag/test_feishu_wiki_sync_failure.py b/test/unit_test/rag/test_feishu_wiki_sync_failure.pynew file mode 100644index 0000000000..c71ca6100c--- /dev/null+++ b/test/unit_test/rag/test_feishu_wiki_sync_failure.py@@ -0,0 +1,357 @@+import importlib+import json as stdlib_json+import sys+from datetime import UTC, datetime+from types import ModuleType, SimpleNamespace+from typing import ClassVar++import pytest+++class _DummyConnector:+ pass+++class _ConnectorService:+ pass+++class _DocumentService:+ @staticmethod+ def list_doc_headers_by_kb_and_source_type(*_args, **_kwargs):+ return []+++class _KnowledgebaseService:+ @staticmethod+ def get_by_id(*_args, **_kwargs):+ return True, object()+++class _SyncLogsService:+ starts: ClassVar[list] = []+ updates: ClassVar[list] = []+ schedules: ClassVar[list] = []+ done_calls: ClassVar[list] = []+ sink_error: ClassVar[Exception | None] = None+ parse_errors: ClassVar[list[str]] = []++ @classmethod+ def reset(cls):+ cls.starts.clear()+ cls.updates.clear()+ cls.schedules.clear()+ cls.done_calls.clear()+ cls.sink_error = None+ cls.parse_errors.clear()++ @classmethod+ def start(cls, *args):+ cls.starts.append(args)++ @classmethod+ def update_by_id(cls, *args):+ cls.updates.append(args)++ @classmethod+ def schedule(cls, *args, **kwargs):+ cls.schedules.append((args, kwargs))++ @classmethod+ def done(cls, *args):+ cls.done_calls.append(args)++ @classmethod+ def duplicate_and_parse(cls, *_args, **_kwargs):+ if cls.sink_error is not None:+ raise cls.sink_error+ return cls.parse_errors, ["stored-doc-id"]++ @staticmethod+ def increase_docs(*_args, **_kwargs):+ return None+++def _module(name, **attributes):+ module = ModuleType(name)+ for key, value in attributes.items():+ setattr(module, key, value)+ return module+++import common+from common.constants import ConnectorTaskType, FileSource, TaskStatus+from test.unit_test.feishu_wiki_test_support import install_unrelated_provider_stubs++rag_svr = importlib.import_module("rag.svr")+_MISSING = object()+++def _load_sync_stack() -> SimpleNamespace:+ stubs = {+ "flask": _module("flask", json=stdlib_json),+ "api": _module("api"),+ "api.db": _module("api.db"),+ "api.db.services": _module("api.db.services"),+ "api.db.services.connector_service": _module(+ "api.db.services.connector_service",+ ConnectorService=_ConnectorService,+ SyncLogsService=_SyncLogsService,+ resolve_connector_doc_id=lambda _kb, _connector, external_id, _existing: external_id,+ ),+ "api.db.services.document_service": _module(+ "api.db.services.document_service",+ DocumentService=_DocumentService,+ ),+ "api.db.services.knowledgebase_service": _module(+ "api.db.services.knowledgebase_service",+ KnowledgebaseService=_KnowledgebaseService,+ ),+ "common.config_utils": _module("common.config_utils", show_configs=lambda: None),+ "box_sdk_gen": _module(+ "box_sdk_gen",+ BoxOAuth=_DummyConnector,+ OAuthConfig=_DummyConnector,+ AccessToken=_DummyConnector,+ ),+ }+ exact_names = {*stubs, "rag.svr.feishu_wiki_sync", "rag.svr.sync_data_source"}+ saved_exact = {name: sys.modules[name] for name in exact_names if name in sys.modules}+ saved_data_source = {name: module for name, module in sys.modules.items() if name == "common.data_source" or name.startswith("common.data_source.")}+ saved_attrs = {+ (common, "config_utils"): getattr(common, "config_utils", _MISSING),+ (common, "data_source"): getattr(common, "data_source", _MISSING),+ (common, "settings"): getattr(common, "settings", _MISSING),+ (rag_svr, "feishu_wiki_sync"): getattr(rag_svr, "feishu_wiki_sync", _MISSING),+ (rag_svr, "sync_data_source"): getattr(rag_svr, "sync_data_source", _MISSING),+ }+ for name in list(sys.modules):+ if name in exact_names or name == "common.data_source" or name.startswith("common.data_source."):+ sys.modules.pop(name, None)+ for parent, attribute in saved_attrs:+ if hasattr(parent, attribute):+ delattr(parent, attribute)++ install_unrelated_provider_stubs(include_sitemap_runtime=True)+ common.settings = SimpleNamespace()+ sys.modules.update(stubs)+ try:+ registry = importlib.import_module("common.data_source")+ models = importlib.import_module("common.data_source.models")+ sync_owner = importlib.import_module("rag.svr.sync_data_source")+ return SimpleNamespace(registry=registry, Document=models.Document, sync_owner=sync_owner)+ finally:+ for name in list(sys.modules):+ if name in exact_names or name == "common.data_source" or name.startswith("common.data_source."):+ sys.modules.pop(name, None)+ sys.modules.update(saved_data_source)+ sys.modules.update(saved_exact)+ for (parent, attribute), value in saved_attrs.items():+ if value is _MISSING:+ if hasattr(parent, attribute):+ delattr(parent, attribute)+ else:+ setattr(parent, attribute, value)+++def _import_state() -> tuple[dict[str, ModuleType], tuple[object, ...]]:+ tracked_modules = {+ name: module+ for name, module in sys.modules.items()+ if name == "common.data_source" or name.startswith("common.data_source.") or name in {"rag.svr.feishu_wiki_sync", "rag.svr.sync_data_source"}+ }+ tracked_attrs = (+ getattr(common, "config_utils", _MISSING),+ getattr(common, "data_source", _MISSING),+ getattr(common, "settings", _MISSING),+ getattr(rag_svr, "feishu_wiki_sync", _MISSING),+ getattr(rag_svr, "sync_data_source", _MISSING),+ )+ return tracked_modules, tracked_attrs+++def _restore_import_state(state: tuple[dict[str, ModuleType], tuple[object, ...]]) -> None:+ modules, attrs = state+ for name in list(sys.modules):+ if name == "common.data_source" or name.startswith("common.data_source.") or name in {"rag.svr.feishu_wiki_sync", "rag.svr.sync_data_source"}:+ sys.modules.pop(name, None)+ sys.modules.update(modules)+ for (parent, attribute), value in zip(+ (+ (common, "config_utils"),+ (common, "data_source"),+ (common, "settings"),+ (rag_svr, "feishu_wiki_sync"),+ (rag_svr, "sync_data_source"),+ ),+ attrs,+ ):+ if value is _MISSING:+ if hasattr(parent, attribute):+ delattr(parent, attribute)+ else:+ setattr(parent, attribute, value)+++_IMPORT_STATE_BEFORE = _import_state()+_stack = _load_sync_stack()+_IMPORT_STATE_AFTER = _import_state()++CONNECTOR_BY_SOURCE = _stack.registry.CONNECTOR_BY_SOURCE+FeishuWikiConnector = _stack.registry.FeishuWikiConnector+Document = _stack.Document+sync_data_source = _stack.sync_owner+++class _FakeSync(sync_data_source.SyncBase):+ SOURCE_NAME = "other"++ def __init__(self, batches):+ super().__init__({})+ self.batches = batches++ async def _generate(self, task):+ del task+ return iter(self.batches)+++class _FakeFeishuSync(sync_data_source.FeishuWiki):+ def __init__(self, batches, *, window_end=None):+ super().__init__({})+ self.batches = batches+ self._poll_window_end = window_end++ async def _generate(self, task):+ del task+ return iter(self.batches)+++def _doc(updated_at):+ return Document(+ id="external-doc-id",+ source=FileSource.FEISHU_WIKI,+ semantic_identifier="guide.pdf",+ extension=".pdf",+ size_bytes=4,+ doc_updated_at=updated_at,+ blob=b"data",+ metadata=None,+ fingerprint="fingerprint",+ )+++def _task(*, poll_range_start=None):+ return {+ "id": "task-1",+ "connector_id": "connector-1",+ "kb_id": "kb-1",+ "tenant_id": "tenant-1",+ "poll_range_start": poll_range_start,+ "auto_parse": False,+ "reindex": "0",+ "timeout_secs": 10,+ "task_type": sync_data_source.ConnectorTaskType.SYNC,+ }+++def test_real_registries_and_models_reach_the_sync_owner():+ assert _IMPORT_STATE_AFTER[0] == _IMPORT_STATE_BEFORE[0]+ assert all(after is before for after, before in zip(_IMPORT_STATE_AFTER[1], _IMPORT_STATE_BEFORE[1]))+ assert CONNECTOR_BY_SOURCE[FileSource.FEISHU_WIKI] is FeishuWikiConnector+ assert sync_data_source.func_factory[FileSource.FEISHU_WIKI] is sync_data_source.FeishuWiki+ assert sync_data_source.FileSource is FileSource+ assert sync_data_source.ConnectorTaskType is ConnectorTaskType+ assert sync_data_source.TaskStatus is TaskStatus+ assert isinstance(_doc(datetime(2026, 1, 2, tzinfo=UTC)), Document)++ entry_state = _import_state()+ late_child = ModuleType("common.data_source.imported_after_collection")+ sys.modules[late_child.__name__] = late_child+ try:+ execution_state = _import_state()+ sentinel_data_source = ModuleType("common.data_source")+ sentinel_sync_owner = ModuleType("rag.svr.sync_data_source")+ sys.modules["common.data_source"] = sentinel_data_source+ sys.modules["rag.svr.sync_data_source"] = sentinel_sync_owner+ common.data_source = sentinel_data_source+ rag_svr.sync_data_source = sentinel_sync_owner+ try:+ isolated = _load_sync_stack()+ assert isolated.registry.CONNECTOR_BY_SOURCE[FileSource.FEISHU_WIKI] is isolated.registry.FeishuWikiConnector+ assert isolated.sync_owner.func_factory[FileSource.FEISHU_WIKI] is isolated.sync_owner.FeishuWiki+ assert sys.modules["common.data_source"] is sentinel_data_source+ assert sys.modules["rag.svr.sync_data_source"] is sentinel_sync_owner+ assert common.data_source is sentinel_data_source+ assert rag_svr.sync_data_source is sentinel_sync_owner+ finally:+ _restore_import_state(execution_state)++ assert sys.modules[late_child.__name__] is late_child+ finally:+ _restore_import_state(entry_state)++[email protected](autouse=True)+def _reset_sync_log_service():+ _SyncLogsService.reset()+++async def test_feishu_sink_failure_marks_task_failed_and_does_not_schedule():+ previous_cursor = datetime(2026, 1, 1, tzinfo=UTC)+ task = _task(poll_range_start=previous_cursor)+ _SyncLogsService.sink_error = RuntimeError("sink unavailable")++ await _FakeFeishuSync([[_doc(datetime(2026, 1, 2, tzinfo=UTC))]])(task)++ assert _SyncLogsService.done_calls == []+ assert _SyncLogsService.schedules == []+ assert _SyncLogsService.updates[-1][1]["status"] == sync_data_source.TaskStatus.FAIL+++async def test_feishu_reported_parse_error_marks_task_failed_and_does_not_schedule():+ task = _task(poll_range_start=datetime(2026, 1, 1, tzinfo=UTC))+ _SyncLogsService.parse_errors.append("parser rejected file")++ await _FakeFeishuSync([[_doc(datetime(2026, 1, 2, tzinfo=UTC))]])(task)++ assert _SyncLogsService.done_calls == []+ assert _SyncLogsService.schedules == []+ assert _SyncLogsService.updates[-1][1]["status"] == sync_data_source.TaskStatus.FAIL+++async def test_feishu_success_completes_and_schedules_at_document_cursor():+ updated_at = datetime(2026, 1, 2, tzinfo=UTC)+ task = _task(poll_range_start=datetime(2026, 1, 1, tzinfo=UTC))++ await _FakeFeishuSync([[_doc(updated_at)]])(task)++ assert _SyncLogsService.done_calls == [("task-1", "connector-1")]+ assert _SyncLogsService.updates == []+ assert _SyncLogsService.schedules[0][0][2] == updated_at+++async def test_default_source_preserves_skip_and_continue_behavior_on_sink_failure():+ task = _task(poll_range_start=datetime(2026, 1, 1, tzinfo=UTC))+ _SyncLogsService.sink_error = RuntimeError("sink unavailable")++ await _FakeSync([[_doc(datetime(2026, 1, 2, tzinfo=UTC))]])(task)++ assert _SyncLogsService.updates == []+ assert _SyncLogsService.done_calls == [("task-1", "connector-1")]+ assert len(_SyncLogsService.schedules) == 1+++async def test_empty_feishu_poll_schedules_the_completed_window_end():+ previous_cursor = datetime(2026, 1, 1, tzinfo=UTC)+ window_end = datetime(2026, 1, 2, tzinfo=UTC)+ task = _task(poll_range_start=previous_cursor)++ await _FakeFeishuSync([], window_end=window_end)(task)++ assert _SyncLogsService.done_calls == [("task-1", "connector-1")]+ assert _SyncLogsService.schedules[0][0][2] == window_enddiff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 7cc2f620a0..55ee1cb6bc 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1308,6 +1308,16 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-sdataSourceFieldClientSecret: 'Client Secret',dataSourceFieldFolderPathOptional: 'Folder Path (optional)',dataSourceFieldBatchSize: 'Batch Size',+ dataSourceFieldFeishuAppId: 'Feishu app ID',+ dataSourceFieldFeishuAppSecret: 'Feishu app secret',+ dataSourceFieldWikiSpaceId: 'Wiki space ID',+ dataSourceFieldRootNodeToken: 'Wiki root node token',+ dataSourceFieldIncludeExtensions: 'Allowed file extensions',+ dataSourceFieldIncludeKeywords: 'Required filename keywords',+ dataSourceFieldExcludeKeywords: 'Excluded filename keywords',+ dataSourceFieldMaxFileSizeBytes: 'Maximum file size (bytes)',+ dataSourceValidationFeishuBatchSize:+ 'Batch size must be between 1 and 10',dataSourceFieldMailFolder: 'Mail Folder',dataSourceFieldMailboxUserIds: 'Mailbox User IDs (optional)',dataSourceFieldInstanceUrl: 'Instance URL',@@ -1581,6 +1591,8 @@ Example: Virtual Hosted Style`,'Sync pages and databases from Notion for knowledge retrieval.',google_driveDescription:'Connect your Google Drive via OAuth and sync specific folders or drives.',+ feishu_wikiDescription:+ 'Sync downloadable file nodes from a Feishu Wiki subtree.',gmailDescription: 'Connect your Gmail via OAuth to sync emails.',webdavDescription: 'Connect to WebDAV servers to sync files.',webdavRemotePathTip:diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex 56604c1f49..e28f62edc9 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -1202,6 +1202,7 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R'连接你的 Oracle Cloud Object Storage 存储桶以导入和同步文件。',google_driveDescription:'通过 OAuth 连接 Google Drive,并同步指定的文件夹或云端硬盘。',+ feishu_wikiDescription: '同步飞书知识库指定范围内的可下载文件节点。',gmailDescription: '通过 OAuth 连接 Gmail,用于同步邮件。',google_driveTokenTip:'请上传由 OAuth helper 或 Google Cloud Console 导出的 OAuth token JSON。也支持上传 “installed” 或 “web” 类型的 client_secret JSON。若为首次同步,将自动弹出浏览器完成 OAuth 授权流程;如果该 JSON 已包含 refresh token,将会被自动复用。',@@ -1323,6 +1324,15 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 RdataSourceFieldClientSecret: '客户端密钥',dataSourceFieldFolderPathOptional: '文件夹路径(可选)',dataSourceFieldBatchSize: '批量大小',+ dataSourceFieldFeishuAppId: '飞书应用 App ID',+ dataSourceFieldFeishuAppSecret: '飞书应用 App Secret',+ dataSourceFieldWikiSpaceId: '知识库 Space ID',+ dataSourceFieldRootNodeToken: '知识库根节点 Token',+ dataSourceFieldIncludeExtensions: '允许的文件扩展名',+ dataSourceFieldIncludeKeywords: '文件名必须包含的关键词',+ dataSourceFieldExcludeKeywords: '文件名排除关键词',+ dataSourceFieldMaxFileSizeBytes: '最大文件大小(字节)',+ dataSourceValidationFeishuBatchSize: '批量大小必须介于 1 和 10 之间',dataSourceFieldMailFolder: '邮件文件夹',dataSourceFieldMailboxUserIds: '邮箱用户 ID(可选)',dataSourceFieldInstanceUrl: '实例 URL',diff --git a/web/src/pages/user-setting/data-source/backend-adapter.test.ts b/web/src/pages/user-setting/data-source/backend-adapter.test.tsnew file mode 100644index 0000000000..7121f87c00--- /dev/null+++ b/web/src/pages/user-setting/data-source/backend-adapter.test.ts@@ -0,0 +1,35 @@+(globalThis as any).CSS = {+ ...(globalThis as any).CSS,+ supports: () => false,+};++const loadAvailableDataSourceKeys = (language: 'go' | 'python') => {+ jest.resetModules();+ jest.doMock('@/utils/backend-runtime', () => ({+ getBackendLanguage: () => language,+ subscribeBackendLanguage: () => () => undefined,+ }));++ const { getAvailableDataSourceKeys } = require('./backend-adapter');+ return getAvailableDataSourceKeys();+};++describe('data source backend adapter', () => {+ afterEach(() => {+ jest.dontMock('@/utils/backend-runtime');+ });++ it('includes Feishu Wiki for the Python backend', () => {+ expect(loadAvailableDataSourceKeys('python')).toContain('feishu_wiki');+ });++ it('hides only Feishu Wiki for the Go backend', () => {+ const pythonKeys = loadAvailableDataSourceKeys('python');+ const goKeys = loadAvailableDataSourceKeys('go');++ expect(goKeys).not.toContain('feishu_wiki');+ expect(goKeys).toEqual(+ pythonKeys.filter((source: string) => source !== 'feishu_wiki'),+ );+ });+});diff --git a/web/src/pages/user-setting/data-source/backend-adapter.ts b/web/src/pages/user-setting/data-source/backend-adapter.tsnew file mode 100644index 0000000000..e018c3167a--- /dev/null+++ b/web/src/pages/user-setting/data-source/backend-adapter.ts@@ -0,0 +1,13 @@+import { pickByBackend } from '@/utils/backend-variant';+import { DataSourceKey } from './constant';++const PythonDataSourceKeys = Object.values(DataSourceKey);+const GoDataSourceKeys = PythonDataSourceKeys.filter(+ (source) => source !== DataSourceKey.FEISHU_WIKI,+);++export const getAvailableDataSourceKeys = (): DataSourceKey[] =>+ pickByBackend({+ go: GoDataSourceKeys,+ python: PythonDataSourceKeys,+ });diff --git a/web/src/pages/user-setting/data-source/constant/__tests__/feishu-wiki.test.tsx b/web/src/pages/user-setting/data-source/constant/__tests__/feishu-wiki.test.tsxnew file mode 100644index 0000000000..fbe7705772--- /dev/null+++ b/web/src/pages/user-setting/data-source/constant/__tests__/feishu-wiki.test.tsx@@ -0,0 +1,95 @@+(globalThis as any).CSS = {+ ...(globalThis as any).CSS,+ supports: () => false,+};++const { FormFieldType } = require('@/components/dynamic-form');+const {+ DataSourceFormDefaultValues,+ DataSourceKey,+ generateDataSourceInfo,+ getDataSourceFieldsWithExtras,+} = require('../index');++const translate = ((key: string) => key) as any;++describe('Feishu Wiki data source form', () => {+ it('registers the Python-backed file source and its generic description', () => {+ const key = (DataSourceKey as any).FEISHU_WIKI;++ expect(key).toBe('feishu_wiki');+ expect(generateDataSourceInfo(translate)[key]).toMatchObject({+ name: 'Feishu Wiki',+ description: 'setting.feishu_wikiDescription',+ });+ });++ it('exposes credentials, pre-download filters, and bounded resource controls', () => {+ const key = (DataSourceKey as any).FEISHU_WIKI;+ const fields = getDataSourceFieldsWithExtras(translate, key);+ const names = fields.map((field: { name: string }) => field.name);++ expect(names).toEqual(+ expect.arrayContaining([+ 'config.credentials.app_id',+ 'config.credentials.app_secret',+ 'config.space_id',+ 'config.root_node_token',+ 'config.include_extensions',+ 'config.include_keywords',+ 'config.exclude_keywords',+ 'config.max_file_size_bytes',+ 'config.batch_size',+ ]),+ );+ expect(+ fields.find(+ (field: { name: string; type: string }) =>+ field.name === 'config.credentials.app_secret',+ )?.type,+ ).toBe(FormFieldType.Password);+ expect(+ fields.find(+ (field: { name: string; type: string }) =>+ field.name === 'config.include_extensions',+ )?.type,+ ).toBe(FormFieldType.Tag);+ expect(+ fields.find(+ (field: {+ name: string;+ validation?: { min?: number; max?: number };+ }) => field.name === 'config.batch_size',+ )?.validation,+ ).toMatchObject({ min: 1, max: 10 });+ expect(+ fields.find(+ (field: { name: string; validation?: { min?: number } }) =>+ field.name === 'config.max_file_size_bytes',+ )?.validation,+ ).toMatchObject({ min: 1 });+ });++ it('defaults to empty filters, a 50 MiB file budget, and batch size two', () => {+ const key = (DataSourceKey as any).FEISHU_WIKI;+ const defaults = (DataSourceFormDefaultValues as any)[key];++ expect(defaults).toEqual({+ name: '',+ source: 'feishu_wiki',+ config: {+ space_id: '',+ root_node_token: '',+ include_extensions: [],+ include_keywords: [],+ exclude_keywords: [],+ max_file_size_bytes: 50 * 1024 * 1024,+ batch_size: 2,+ credentials: {+ app_id: '',+ app_secret: '',+ },+ },+ });+ });+});diff --git a/web/src/pages/user-setting/data-source/constant/feishu-wiki-constant.tsx b/web/src/pages/user-setting/data-source/constant/feishu-wiki-constant.tsxnew file mode 100644index 0000000000..e4919d62ea--- /dev/null+++ b/web/src/pages/user-setting/data-source/constant/feishu-wiki-constant.tsx@@ -0,0 +1,110 @@+/*+ * Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { FormFieldType } from '@/components/dynamic-form';+import { TFunction } from 'i18next';++export const feishuWikiConstant = (t: TFunction) => [+ {+ label: t('setting.dataSourceFieldFeishuAppId'),+ name: 'config.credentials.app_id',+ type: FormFieldType.Text,+ required: true,+ placeholder: 'cli_xxxxxxxxxxxxxxxx',+ },+ {+ label: t('setting.dataSourceFieldFeishuAppSecret'),+ name: 'config.credentials.app_secret',+ type: FormFieldType.Password,+ required: true,+ },+ {+ label: t('setting.dataSourceFieldWikiSpaceId'),+ name: 'config.space_id',+ type: FormFieldType.Text,+ required: true,+ placeholder: '1234567890123456789',+ },+ {+ label: t('setting.dataSourceFieldRootNodeToken'),+ name: 'config.root_node_token',+ type: FormFieldType.Text,+ required: true,+ placeholder: 'wikcnExampleRootNodeToken',+ },+ {+ label: t('setting.dataSourceFieldIncludeExtensions'),+ name: 'config.include_extensions',+ type: FormFieldType.Tag,+ required: false,+ placeholder: 'pdf, docx, xlsx, pptx, jpg, png',+ },+ {+ label: t('setting.dataSourceFieldIncludeKeywords'),+ name: 'config.include_keywords',+ type: FormFieldType.Tag,+ required: false,+ placeholder: 'project, handbook',+ },+ {+ label: t('setting.dataSourceFieldExcludeKeywords'),+ name: 'config.exclude_keywords',+ type: FormFieldType.Tag,+ required: false,+ placeholder: 'draft, archived',+ },+ {+ label: t('setting.dataSourceFieldMaxFileSizeBytes'),+ name: 'config.max_file_size_bytes',+ type: FormFieldType.Number,+ required: false,+ validation: {+ min: 1,+ message: t('setting.dataSourceValidationMinOne', {+ label: t('setting.dataSourceFieldMaxFileSizeBytes'),+ }),+ },+ },+ {+ label: t('setting.dataSourceFieldBatchSize'),+ name: 'config.batch_size',+ type: FormFieldType.Number,+ required: false,+ validation: {+ min: 1,+ max: 10,+ message: t('setting.dataSourceValidationFeishuBatchSize'),+ },+ },+];++export const feishuWikiDefaultValues = {+ name: '',+ source: 'feishu_wiki',+ config: {+ space_id: '',+ root_node_token: '',+ include_extensions: [],+ include_keywords: [],+ exclude_keywords: [],+ max_file_size_bytes: 50 * 1024 * 1024,+ batch_size: 2,+ credentials: {+ app_id: '',+ app_secret: '',+ },+ },+};diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsxindex b5c48d2cbb..eb33a360d3 100644--- a/web/src/pages/user-setting/data-source/constant/index.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.tsx@@ -18,7 +18,7 @@ import { FormFieldType } from '@/components/dynamic-form';import { IconFontFill } from '@/components/icon-font';import SvgIcon from '@/components/svg-icon';import { TFunction } from 'i18next';-import { Globe, Mail, Rss, Search } from 'lucide-react';+import { BookOpen, Globe, Mail, Rss, Search } from 'lucide-react';import { useEffect, useState } from 'react';import { useTranslation } from 'react-i18next';import BoxTokenField from '../component/box-token-field';@@ -28,6 +28,10 @@ import { IDataSourceInfoMap } from '../interface';import { azureDevOpsConstant } from './azure-devops-constant';import { bitbucketConstant } from './bitbucket-constant';import { confluenceConstant } from './confluence-constant';+import {+ feishuWikiConstant,+ feishuWikiDefaultValues,+} from './feishu-wiki-constant';import { jiraConstant } from './jira-constant';import { S3Constant } from './s3-constant';import { seafileConstant } from './seafile-constant';@@ -37,6 +41,7 @@ export enum DataSourceKey {CONFLUENCE = 'confluence',NOTION = 'notion',GOOGLE_DRIVE = 'google_drive',+ FEISHU_WIKI = 'feishu_wiki',GMAIL = 'gmail',GOOGLE_CLOUD_STORAGE = 'google_cloud_storage',OCI_STORAGE = 'oci_storage',@@ -263,6 +268,11 @@ export const generateDataSourceInfo = (t: TFunction) => {description: t(`setting.${DataSourceKey.GOOGLE_DRIVE}Description`),icon: <SvgIcon name={'data-source/google-drive'} width={38} />,},+ [DataSourceKey.FEISHU_WIKI]: {+ name: 'Feishu Wiki',+ description: t(`setting.${DataSourceKey.FEISHU_WIKI}Description`),+ icon: <BookOpen className="text-text-primary" size={22} />,+ },[DataSourceKey.GMAIL]: {name: 'Gmail',description: t(`setting.${DataSourceKey.GMAIL}Description`),@@ -488,6 +498,7 @@ export const getCommonExtraDefaultValues = () => ({});const generateDataSourceFormFields = (t: TFunction) => ({+ [DataSourceKey.FEISHU_WIKI]: feishuWikiConstant(t),[DataSourceKey.ONEDRIVE]: [{label: t('setting.dataSourceFieldTenantId'),@@ -2078,6 +2089,7 @@ const generateDataSourceFormFields = (t: TFunction) => ({});export const DataSourceFormDefaultValues = {+ [DataSourceKey.FEISHU_WIKI]: feishuWikiDefaultValues,[DataSourceKey.RSS]: {name: '',source: DataSourceKey.RSS,diff --git a/web/src/pages/user-setting/data-source/index.tsx b/web/src/pages/user-setting/data-source/index.tsxindex 9c14575758..7d719d4aaf 100644--- a/web/src/pages/user-setting/data-source/index.tsx+++ b/web/src/pages/user-setting/data-source/index.tsx@@ -20,8 +20,9 @@ import { Button } from '@/components/ui/button';import { Plus } from 'lucide-react';import { ProfileSettingWrapperCard } from '../components/user-setting-header';import AddDataSourceModal from './add-datasource-modal';+import { getAvailableDataSourceKeys } from './backend-adapter';import { AddedSourceCard } from './component/added-source-card';-import { DataSourceKey, useDataSourceInfo } from './constant';+import { useDataSourceInfo } from './constant';import { useAddDataSource, useListDataSource } from './hooks';import { IDataSorceInfo } from './interface';@@ -76,7 +77,7 @@ const AvailableSourceCard = ({const DataSource = () => {const { t } = useTranslation();const { dataSourceInfo } = useDataSourceInfo();- const dataSourceTemplates = Object.values(DataSourceKey).map((id) => {+ const dataSourceTemplates = getAvailableDataSourceKeys().map((id) => {return {id,name: dataSourceInfo[id].name,