RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: keep file API responsive during large connector ingest (#19632)
Öndery98ebe5f66f2c
diff --git a/api/apps/restful_apis/file_api.py b/api/apps/restful_apis/file_api.pyindex 3657a9be99..4545b6604d 100644--- a/api/apps/restful_apis/file_api.py+++ b/api/apps/restful_apis/file_api.py@@ -141,7 +141,7 @@ async def list_files(tenant_id: str = None):return get_error_argument_result(err)try:- success, result = file_api_service.list_files(tenant_id, args)+ success, result = await thread_pool_exec(file_api_service.list_files, tenant_id, args)if success:return get_result(data=result)else:diff --git a/api/apps/services/file_api_service.py b/api/apps/services/file_api_service.pyindex f3060741af..65762f400f 100644--- a/api/apps/services/file_api_service.py+++ b/api/apps/services/file_api_service.py@@ -41,10 +41,10 @@ async def upload_file(tenant_id: str, pf_id: str, file_objs: list)::return: (success, result_list) or (success, error_message)"""if not pf_id:- root_folder = FileService.get_root_folder(tenant_id)+ root_folder = await thread_pool_exec(FileService.get_root_folder, tenant_id)pf_id = root_folder["id"]- e, pf_folder = FileService.get_by_id(pf_id)+ e, pf_folder = await thread_pool_exec(FileService.get_by_id, pf_id)if not e:return False, "Can't find this folder!"@@ -114,6 +114,11 @@ async def create_folder(tenant_id: str, name: str, pf_id: str | None = None, filif "/" in name:return False, 'Folder name cannot contain "/"'+ return await thread_pool_exec(_create_folder_sync, tenant_id, name, pf_id, file_type)+++def _create_folder_sync(tenant_id: str, name: str, pf_id: str | None, file_type: str | None):+ """Insert a folder or virtual file; intended to run off the Quart event loop."""if not pf_id:root_folder = FileService.get_root_folder(tenant_id)pf_id = root_folder["id"]diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.pyindex a46b4fb028..a44570fabb 100644--- a/api/db/services/connector_service.py+++ b/api/db/services/connector_service.py@@ -542,11 +542,13 @@ class SyncLogsService(CommonService):cls.model.update(update_payload).where(cls.model.id == id).execute()@classmethod- def duplicate_and_parse(cls, kb, docs, tenant_id, src, auto_parse=True):+ def duplicate_and_parse(cls, kb, docs, tenant_id, src, auto_parse=True, should_cancel=None):from api.db.services.file_service import FileServiceif not docs:return None+ if FileService._is_sync_cancelled(should_cancel):+ return [], []class FileObj(BaseModel):id: str@@ -568,7 +570,7 @@ class SyncLogsService(CommonService):for d in docs]doc_ids = []- err, doc_blob_pairs = FileService.upload_document(kb, files, tenant_id, src)+ err, doc_blob_pairs = FileService.upload_document(kb, files, tenant_id, src, should_cancel=should_cancel)errs.extend(err)# Create a mapping from filename to metadata for later use@@ -579,16 +581,24 @@ class SyncLogsService(CommonService):metadata_map[filename] = d["metadata"]kb_table_num_map = {}- for doc, _ in doc_blob_pairs:- doc_ids.append(doc["id"])-- # Set metadata if available for this document- if doc["name"] in metadata_map:- DocMetadataService.update_document_metadata(doc["id"], metadata_map[doc["name"]])-- if not auto_parse or auto_parse == "0":- continue- DocumentService.run(tenant_id, doc, kb_table_num_map)+ wrote_meta = False+ try:+ for doc, _ in doc_blob_pairs:+ if FileService._is_sync_cancelled(should_cancel):+ break+ doc_ids.append(doc["id"])++ # Set metadata if available for this document+ if doc["name"] in metadata_map:+ DocMetadataService.update_document_metadata(doc["id"], metadata_map[doc["name"]], refresh_now=False)+ wrote_meta = True++ if not auto_parse or auto_parse == "0":+ continue+ DocumentService.run(tenant_id, doc, kb_table_num_map)+ finally:+ if wrote_meta:+ DocMetadataService.refresh_tenant_index(tenant_id)return errs, doc_idsdiff --git a/api/db/services/doc_metadata_service.py b/api/db/services/doc_metadata_service.pyindex 9e75b6d43a..093c9c86c9 100644--- a/api/db/services/doc_metadata_service.py+++ b/api/db/services/doc_metadata_service.py@@ -361,13 +361,16 @@ class DocMetadataService:@classmethod@DB.connection_context()- def insert_document_metadata(cls, doc_id: str, meta_fields: dict) -> bool:+ def insert_document_metadata(cls, doc_id: str, meta_fields: dict, *, refresh_now: bool = True) -> bool:"""Insert document metadata into ES/Infinity.Args:doc_id: Document IDmeta_fields: Metadata dictionary+ refresh_now: When True, wait for the index refresh so the row is+ searchable immediately. Connector ingest passes False and+ refreshes once per batch so agent retrieval is not blocked.Returns:True if successful, False otherwise@@ -419,8 +422,9 @@ class DocMetadataService:else:logging.debug(f"Metadata table already exists: {index_name}")- # Insert into ES/Infinity- result = settings.docStoreConn.insert([doc_meta], index_name, kb_id)+ # Insert into ES/Infinity. Connector ingest skips wait_for so agent+ # searches are not queued behind a refresh per uploaded row.+ result = settings.docStoreConn.insert([doc_meta], index_name, kb_id, refresh="wait_for" if refresh_now else False)if result:logging.error(f"Failed to insert metadata for document {doc_id}: {result}")@@ -428,18 +432,8 @@ class DocMetadataService:# Force refresh so metadata is immediately searchable.# Both Elasticsearch and OpenSearch backends expose refresh_idx;# Infinity does not need a manual refresh.- if not settings.DOC_ENGINE_INFINITY:- refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None)- if callable(refresh_idx):- if refresh_idx(index_name):- logging.debug(f"Refreshed metadata index: {index_name}")- else:- # A failed refresh can leave just-inserted metadata- # invisible to subsequent reads; surface it so operators- # can correlate stale-read complaints with the cause.- logging.warning(f"Failed to refresh metadata index {index_name} on backend {type(settings.docStoreConn).__name__}; metadata may not be immediately searchable")- else:- logging.debug(f"Backend {type(settings.docStoreConn).__name__} has no refresh_idx; skipping")+ if refresh_now:+ cls.refresh_tenant_index(tenant_id)logging.debug(f"Successfully inserted metadata for document {doc_id}")return True@@ -448,9 +442,24 @@ class DocMetadataService:logging.error(f"Error inserting metadata for document {doc_id}: {e}")return False+ @classmethod+ def refresh_tenant_index(cls, tenant_id: str) -> None:+ """Refresh the per-tenant metadata index when the backend needs it."""+ if settings.DOC_ENGINE_INFINITY:+ return+ index_name = cls._get_doc_meta_index_name(tenant_id)+ refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None)+ if not callable(refresh_idx):+ logging.debug(f"Backend {type(settings.docStoreConn).__name__} has no refresh_idx; skipping")+ return+ if refresh_idx(index_name):+ logging.debug(f"Refreshed metadata index: {index_name}")+ return+ logging.warning(f"Failed to refresh metadata index {index_name} on backend {type(settings.docStoreConn).__name__}; metadata may not be immediately searchable")+@classmethod@DB.connection_context()- def update_document_metadata(cls, doc_id: str, meta_fields: dict) -> bool:+ def update_document_metadata(cls, doc_id: str, meta_fields: dict, *, refresh_now: bool = True) -> bool:"""Update document metadata in ES/Infinity.@@ -460,6 +469,8 @@ class DocMetadataService:Args:doc_id: Document IDmeta_fields: Metadata dictionary+ refresh_now: When True, wait for the index refresh so the row is+ searchable immediately. Connector ingest passes False.Returns:True if successful, False otherwise@@ -511,7 +522,7 @@ class DocMetadataService:if result is False:logging.error(f"Failed to create metadata index {index_name}")return False- return cls.insert_document_metadata(doc_id, processed_meta)+ return cls.insert_document_metadata(doc_id, processed_meta, refresh_now=refresh_now)# Index exists - check if document existstry:@@ -536,18 +547,18 @@ class DocMetadataService:# replace still guarantees full overwrite semantics rather# than leaking through the "document not found" branch.cls.delete_document_metadata(doc_id, kb_id, tenant_id)- return cls.insert_document_metadata(doc_id, processed_meta)+ return cls.insert_document_metadata(doc_id, processed_meta, refresh_now=refresh_now)except Exception as e:logging.debug(f"Document {doc_id} not found in index, will insert: {e}")# Document doesn't exist - insert newlogging.debug(f"[update_document_metadata] Document {doc_id} not found, inserting new")- return cls.insert_document_metadata(doc_id, processed_meta)+ return cls.insert_document_metadata(doc_id, processed_meta, refresh_now=refresh_now)# For Infinity or as fallback: use delete+insertlogging.debug(f"[update_document_metadata] Using delete+insert method for doc_id: {doc_id}")cls.delete_document_metadata(doc_id, kb_id, tenant_id)- return cls.insert_document_metadata(doc_id, processed_meta)+ return cls.insert_document_metadata(doc_id, processed_meta, refresh_now=refresh_now)except Exception as e:logging.error(f"Error updating metadata for document {doc_id}: {e}")diff --git a/api/db/services/file_service.py b/api/db/services/file_service.pyindex 245b7852af..49005f6649 100644--- a/api/db/services/file_service.py+++ b/api/db/services/file_service.py@@ -575,9 +575,18 @@ class FileService(CommonService):DocumentService.delete_by_id(doc.id)return True+ @staticmethod+ def _is_sync_cancelled(should_cancel) -> bool:+ if should_cancel is None:+ return False+ is_set = getattr(should_cancel, "is_set", None)+ if callable(is_set):+ return bool(is_set())+ return bool(should_cancel())+@classmethod@DB.connection_context()- def upload_document(self, kb, file_objs, user_id, src="local", parent_path: str | None = None, parser_config_override: dict | None = None):+ def upload_document(self, kb, file_objs, user_id, src="local", parent_path: str | None = None, parser_config_override: dict | None = None, should_cancel=None):root_folder = self.get_root_folder(user_id)pf_id = root_folder["id"]self.init_knowledgebase_docs(pf_id, user_id)@@ -595,6 +604,8 @@ class FileService(CommonService):err, files = [], []for file in file_objs:+ if self._is_sync_cancelled(should_cancel):+ breakdoc_id = file.id if hasattr(file, "id") else get_uuid()e, doc = DocumentService.get_by_id(doc_id)if e and str(doc.kb_id) != str(kb.id):diff --git a/common/data_source/config.py b/common/data_source/config.pyindex f6cba01557..2198a4f360 100644--- a/common/data_source/config.py+++ b/common/data_source/config.py@@ -1,6 +1,7 @@"""Configuration constants and enum definitions"""import json+import mathimport osfrom datetime import datetime, timezonefrom enum import Enum@@ -117,12 +118,42 @@ _PAGE_EXPANSION_FIELDS = []+def _env_int(name: str, default: int, *, minimum: int = 1) -> int:+ """Parse a positive integer env var, returning ``default`` when unset or invalid."""+ raw = os.environ.get(name)+ if raw is None or str(raw).strip() == "":+ return default+ try:+ value = int(raw)+ except (TypeError, ValueError):+ return default+ return value if value >= minimum else default+++def _env_float(name: str, default: float, *, minimum: float = 0.0) -> float:+ """Parse a finite float env var, returning ``default`` when unset or invalid."""+ raw = os.environ.get(name)+ if raw is None or str(raw).strip() == "":+ return default+ try:+ value = float(raw)+ except (TypeError, ValueError):+ return default+ if not math.isfinite(value) or value < minimum:+ return default+ return value++# Configuration constantstry:BLOB_STORAGE_SIZE_THRESHOLD = int(os.environ.get("BLOB_STORAGE_SIZE_THRESHOLD", 20 * 1024 * 1024))except ValueError as error:raise ValueError("BLOB_STORAGE_SIZE_THRESHOLD must be an integer number of bytes") from error-INDEX_BATCH_SIZE = 2+# Connector ingest batch size (rows/files handed to duplicate_and_parse at once).+INDEX_BATCH_SIZE = _env_int("INDEX_BATCH_SIZE", 2)+# Yield the sync worker event loop (and MySQL) between nonempty write batches.+# 0 still schedules sleep(0) between batches, not after the last one.+SYNC_BATCH_PAUSE_SECONDS = _env_float("SYNC_BATCH_PAUSE_SECONDS", 0.0)SLACK_NUM_THREADS = 4ENABLE_EXPENSIVE_EXPERT_CALLS = Falsediff --git a/docker/.env b/docker/.envindex 5997e07846..147d1b6c13 100644--- a/docker/.env+++ b/docker/.env@@ -312,6 +312,13 @@ DOC_BULK_SIZE=${DOC_BULK_SIZE:-4}# Defaults to 16 if EMBEDDING_BATCH_SIZE is not set in the environment.EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-16}+# Connector ingest: documents (e.g. one MySQL row = one file) written per batch.+# Defaults to 2. Raise this for large RDBMS syncs so fewer write round-trips hit MySQL.+# INDEX_BATCH_SIZE=32+# Pause between connector write batches so the file/storage API can use MySQL.+# 0 still yields the sync worker event loop. A small value (e.g. 0.05) reduces lock wait.+# SYNC_BATCH_PAUSE_SECONDS=0.05+# Controls how rerank inputs that exceed the configured model token limit are handled.# Available options:# - `truncate` (default): truncate each document so its tokens plus the query tokens fit the limit.diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.pyindex 5cddf26de6..f8b0f9d80c 100644--- a/rag/svr/sync_data_source.py+++ b/rag/svr/sync_data_source.py@@ -44,7 +44,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseServicefrom common import settingsfrom common.constants import ConnectorTaskType, FileSource, TaskStatusfrom common.config_utils import show_configs-from common.data_source.config import INDEX_BATCH_SIZE+from common.data_source.config import INDEX_BATCH_SIZE, SYNC_BATCH_PAUSE_SECONDSfrom common.data_source import (BlobStorageConnector,RSSConnector,@@ -261,21 +261,25 @@ class SyncBase:next_update = datetime(1970, 1, 1, tzinfo=timezone.utc)saw_documents = Falsesource_type = f"{self.SOURCE_NAME}/{task['connector_id']}"- existing_doc_ids = {- doc["id"]- for doc in DocumentService.list_doc_headers_by_kb_and_source_type(- task["kb_id"],- source_type,- )- }+ existing_headers = await asyncio.to_thread(+ DocumentService.list_doc_headers_by_kb_and_source_type,+ task["kb_id"],+ source_type,+ )+ existing_doc_ids = {doc["id"] for doc in existing_headers}if task["poll_range_start"]:next_update = task["poll_range_start"]+ pause_before_next_batch = Falseasync for document_batch in _iterate_document_batches(document_batch_generator):if not document_batch:continue+ if pause_before_next_batch:+ await asyncio.sleep(SYNC_BATCH_PAUSE_SECONDS)+ pause_before_next_batch = True+saw_documents = Truemax_update = max(doc.doc_updated_at for doc in document_batch)next_update = max(next_update, max_update)@@ -298,14 +302,12 @@ class SyncBase:d["fingerprint"] = doc.fingerprintdocs.append(d)+ cancel_event = threading.Event()+ parent_task = asyncio.current_task()try:- e, kb = KnowledgebaseService.get_by_id(task["kb_id"])- err, dids = SyncLogsService.duplicate_and_parse(kb, docs, task["tenant_id"], f"{self.SOURCE_NAME}/{task['connector_id']}", task["auto_parse"])+ err, dids = await asyncio.to_thread(self._ingest_document_batch, task, docs, max_update, cancel_event, parent_task)if err:had_parse_errors = True- if self.RAISE_ON_BATCH_ERROR:- raise RuntimeError(f"{self.SOURCE_NAME} failed to process {len(err)} document(s)")- SyncLogsService.increase_docs(task["id"], max_update, len(docs), "\n".join(err), len(err))changed_doc_ids = set(dids)updated_in_batch = len(changed_doc_ids & existing_doc_ids)added_in_batch = len(changed_doc_ids) - updated_in_batch@@ -313,6 +315,9 @@ class SyncBase:updated_docs += updated_in_batchexisting_doc_ids.update(changed_doc_ids)+ except asyncio.CancelledError:+ cancel_event.set()+ raiseexcept Exception as batch_ex:msg = str(batch_ex)code = getattr(batch_ex, "args", [None])[0]@@ -325,7 +330,6 @@ class SyncBase:if self.RAISE_ON_BATCH_ERROR:raisefailed_docs += len(docs)- continueif not saw_documents:next_update = self._get_empty_sync_cursor(task, next_update)@@ -385,6 +389,40 @@ class SyncBase:async def _generate(self, task: dict):raise NotImplementedError+ def _ingest_document_batch(self, task: dict, docs: list, max_update: datetime, cancel_event: threading.Event | None = None, parent_task: asyncio.Task | None = None):+ """Write one connector batch (MinIO + document rows) off the event loop.++ Knowledgebase lookup and parse stay on the same worker thread so Peewee+ objects are not shared across threads. ``cancel_event`` is set when the+ awaiting coroutine catches ``CancelledError``. ``cancelled()`` stays false+ until that handler runs, so the worker also observes ``cancelling()`` to+ stop between files after ``wait_for`` times out. Uploads that already+ started cannot be force-stopped.+ """++ def should_cancel() -> bool:+ if cancel_event is not None and cancel_event.is_set():+ return True+ return parent_task is not None and (parent_task.cancelling() > 0 or parent_task.cancelled())++ if should_cancel():+ return [], []+ _e, kb = KnowledgebaseService.get_by_id(task["kb_id"])+ err, dids = SyncLogsService.duplicate_and_parse(+ kb,+ docs,+ task["tenant_id"],+ f"{self.SOURCE_NAME}/{task['connector_id']}",+ task["auto_parse"],+ should_cancel=should_cancel,+ )+ if should_cancel():+ return err, dids+ if err and self.RAISE_ON_BATCH_ERROR:+ raise RuntimeError(f"{self.SOURCE_NAME} failed to process {len(err)} document(s)")+ SyncLogsService.increase_docs(task["id"], max_update, len(docs), "\n".join(err), len(err))+ return err, dids+def _get_source_prefix(self):return ""@@ -2217,8 +2255,7 @@ class _RDBMSBase(_CursorPersistingSyncBase):raise ValueError(f"{self.DB_TYPE} connector is missing credentials.")self.connector.load_credentials(credentials)- self.connector.validate_connector_settings()- self.connector.prepare_sync_state(task["connector_id"], self.conf)+ await asyncio.to_thread(self._prepare_rdbms_connector, task)if task["reindex"] == "1" or not task["poll_range_start"]:document_generator = self.connector.load_from_state()@@ -2238,7 +2275,15 @@ class _RDBMSBase(_CursorPersistingSyncBase):_begin_info = f"from {poll_start}"self.log_connection(self.LOG_NAME, f"{self.conf.get('host')}:{self.conf.get('database')}", task)- return document_generator+ return iter_in_worker_thread(document_generator)++ def _prepare_rdbms_connector(self, task: dict) -> None:+ """Validate and snapshot cursor state off the event loop, then drop the setup connection."""+ self.connector.validate_connector_settings()+ self.connector.prepare_sync_state(task["connector_id"], self.conf)+ close_connection = getattr(self.connector, "_close_connection", None)+ if close_connection is not None:+ close_connection()class MySQL(_RDBMSBase):diff --git a/test/testcases/restful_api/test_file_routes_unit.py b/test/testcases/restful_api/test_file_routes_unit.pyindex 5988d04591..86e9542fc0 100644--- a/test/testcases/restful_api/test_file_routes_unit.py+++ b/test/testcases/restful_api/test_file_routes_unit.py@@ -268,6 +268,15 @@ def test_list_files_validation_error(monkeypatch):assert res["message"] == "bad args"+def test_list_files_success_offloads_to_thread_pool(monkeypatch):+ module = _load_file_api_module(monkeypatch)++ res = _run(module.list_files("tenant1"))+ assert res["code"] == 0+ assert res["data"] == {"files": [], "total": 0}++@pytest.mark.p2def test_move_uses_new_payload_shape(monkeypatch):module = _load_file_api_module(monkeypatch)@@ -901,6 +910,51 @@ def test_create_folder_rejects_slash_in_name(monkeypatch):assert message == 'Folder name cannot contain "/"'+def test_create_folder_success_offloads_sync_work(monkeypatch):+ module = _load_file_api_service(monkeypatch)++ ok, data = _run(module.create_folder("tenant1", "new-folder", "pf1", module.FileType.FOLDER.value))+ assert ok is True+ assert data["name"] == "new-folder"+ assert data["parent_id"] == "pf1"+ assert data["type"] == module.FileType.FOLDER.value+++def test_upload_file_uses_root_folder_when_parent_missing(monkeypatch):+ module = _load_file_api_service(monkeypatch)+ seen_ids = []++ monkeypatch.setattr(+ module.FileService,+ "get_by_id",+ lambda file_id: (True, SimpleNamespace(id=file_id, name=str(file_id))),+ )+ monkeypatch.setattr(module.FileService, "get_id_list_by_id", lambda *_args, **_kwargs: ["root"])+ monkeypatch.setattr(+ module.FileService,+ "create_folder",+ lambda _file, parent_id, _names, _len_id, *_args: SimpleNamespace(id=parent_id),+ )+ monkeypatch.setattr(+ module.settings,+ "STORAGE_IMPL",+ SimpleNamespace(+ obj_exist=lambda *_args, **_kwargs: False,+ put=lambda bucket, location, blob: seen_ids.append((bucket, location, blob)),+ rm=lambda *_args, **_kwargs: None,+ move=lambda *_args, **_kwargs: None,+ ),+ )++ ok, data = _run(module.upload_file("tenant1", "", [_DummyUploadFile("a.txt", b"hello")]))+ assert ok is True+ assert data[0]["name"] == "a.txt"+ assert data[0]["parent_id"] == "root"+ assert seen_ids == [("root", "a.txt", b"hello")]++@pytest.mark.p2def test_delete_files_checks_team_permission(monkeypatch):module = _load_file_api_service(monkeypatch)diff --git a/test/testcases/restful_api/test_files.py b/test/testcases/restful_api/test_files.pynew file mode 100644index 0000000000..566b14657a--- /dev/null+++ b/test/testcases/restful_api/test_files.py@@ -0,0 +1,64 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++import uuid++import pytest++from test.testcases.utils.file_utils import create_txt_file+++def _assert_ok(res, *, expect_list=False):+ assert res.status_code == 200, res.text+ payload = res.json()+ assert payload["code"] == 0, payload+ data = payload["data"]+ if expect_list:+ assert isinstance(data, dict), payload+ assert "files" in data and "total" in data, payload+ return data+++def test_files_list_create_folder_and_upload_to_root(rest_client, tmp_path):+ """Exercise GET/POST /files so connector offload paths run on the live server."""+ created_ids: list[str] = []+ suffix = uuid.uuid4().hex[:12]+ folder_name = f"rf_folder_{suffix}"+ upload_name = f"rf_upload_{suffix}.txt"++ try:+ list_data = _assert_ok(rest_client.get("/files"), expect_list=True)+ assert isinstance(list_data["files"], list), list_data++ folder = _assert_ok(rest_client.post("/files", json={"name": folder_name, "type": "folder"}))+ folder_id = folder.get("id")+ assert folder_id, folder+ created_ids.append(folder_id)+ assert folder.get("name") == folder_name++ fp = create_txt_file(tmp_path / upload_name)+ with fp.open("rb") as file_obj:+ upload_res = rest_client.post("/files", files=[("file", (fp.name, file_obj))])+ uploaded = _assert_ok(upload_res)+ assert isinstance(uploaded, list) and uploaded, uploaded+ file_id = uploaded[0].get("id")+ assert file_id, uploaded+ created_ids.append(file_id)+ assert uploaded[0].get("name") == upload_name+ finally:+ if created_ids:+ rest_client.delete("/files", json={"ids": created_ids})diff --git a/test/unit_test/api/db/services/test_connector_duplicate_and_parse.py b/test/unit_test/api/db/services/test_connector_duplicate_and_parse.pynew file mode 100644index 0000000000..da10d304dc--- /dev/null+++ b/test/unit_test/api/db/services/test_connector_duplicate_and_parse.py@@ -0,0 +1,101 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Deferred connector metadata refresh still runs if parse/queue fails."""++from types import SimpleNamespace++import pytest++from api.db.services import connector_service as connector_mod+from api.db.services.connector_service import SyncLogsService+from api.db.services.file_service import FileService+++TENANT_ID = "tenant-1"+FILENAME = "row-1.txt"+DOC = {"id": "doc-1", "name": FILENAME}+++def _docs():+ return [+ {+ "id": "row-1",+ "semantic_identifier": "row-1",+ "extension": ".txt",+ "blob": b"payload",+ "metadata": {"source": "mysql"},+ }+ ]+++def _stub_upload(monkeypatch, pairs):+ monkeypatch.setattr(FileService, "_is_sync_cancelled", staticmethod(lambda *_args, **_kwargs: False))+ monkeypatch.setattr(+ FileService,+ "upload_document",+ classmethod(lambda *_args, **_kwargs: ([], pairs)),+ )+++def test_duplicate_and_parse_refreshes_metadata_when_run_fails(monkeypatch):+ _stub_upload(monkeypatch, [(DOC, b"payload")])+ monkeypatch.setattr(connector_mod.DocMetadataService, "update_document_metadata", classmethod(lambda *_args, **_kwargs: True))+ refresh_calls = []+ monkeypatch.setattr(+ connector_mod.DocMetadataService,+ "refresh_tenant_index",+ classmethod(lambda cls, tenant_id: refresh_calls.append(tenant_id)),+ )++ def _fail_run(*_args, **_kwargs):+ raise RuntimeError("queue failed")++ monkeypatch.setattr(connector_mod.DocumentService, "run", classmethod(_fail_run))++ with pytest.raises(RuntimeError, match="queue failed"):+ SyncLogsService.duplicate_and_parse(SimpleNamespace(), _docs(), TENANT_ID, "mysql/1", auto_parse=True)++ assert refresh_calls == [TENANT_ID]+++def test_duplicate_and_parse_skips_refresh_when_no_metadata_was_written(monkeypatch):+ docs = [+ {+ "id": "row-1",+ "semantic_identifier": "row-1",+ "extension": ".txt",+ "blob": b"payload",+ }+ ]+ _stub_upload(monkeypatch, [(DOC, b"payload")])+ refresh_calls = []+ monkeypatch.setattr(+ connector_mod.DocMetadataService,+ "refresh_tenant_index",+ classmethod(lambda cls, tenant_id: refresh_calls.append(tenant_id)),+ )++ def _fail_run(*_args, **_kwargs):+ raise RuntimeError("queue failed")++ monkeypatch.setattr(connector_mod.DocumentService, "run", classmethod(_fail_run))++ with pytest.raises(RuntimeError, match="queue failed"):+ SyncLogsService.duplicate_and_parse(SimpleNamespace(), docs, TENANT_ID, "mysql/1", auto_parse=True)++ assert refresh_calls == []diff --git a/test/unit_test/api/db/services/test_doc_metadata_service.py b/test/unit_test/api/db/services/test_doc_metadata_service.pyindex 65c4b1d785..2e713baf81 100644--- a/test/unit_test/api/db/services/test_doc_metadata_service.py+++ b/test/unit_test/api/db/services/test_doc_metadata_service.py@@ -75,3 +75,33 @@ class TestDocMetadataServiceConnectorGetCalls:[],)assert result == {"author": "alice"}+++class TestDocMetadataServiceRefreshNow:+ def test_insert_skips_wait_for_when_refresh_now_is_false(self):+ mock_doc_store = MagicMock()+ mock_doc_store.index_exist.return_value = True+ mock_doc_store.insert.return_value = []+ mock_doc_store.refresh_idx = MagicMock(return_value=True)++ mock_doc = MagicMock()+ mock_doc.id = "doc_789"+ mock_doc.kb_id = "kb_123"+ mock_doc.knowledgebase.tenant_id = "tenant_456"++ with (+ patch("api.db.services.doc_metadata_service.settings") as mock_settings,+ patch("api.db.services.doc_metadata_service.Document") as mock_document_model,+ patch("api.db.db_models.DB.connect"),+ patch("api.db.db_models.DB.connection_context"),+ ):+ mock_settings.docStoreConn = mock_doc_store+ mock_settings.DOC_ENGINE_INFINITY = False+ mock_document_model.select.return_value.join.return_value.where.return_value.first.return_value = mock_doc++ ok = DocMetadataService.insert_document_metadata("doc_789", {"k": "v"}, refresh_now=False)++ assert ok is True+ mock_doc_store.insert.assert_called_once()+ assert mock_doc_store.insert.call_args.kwargs["refresh"] is False+ mock_doc_store.refresh_idx.assert_not_called()diff --git a/test/unit_test/api/db/services/test_file_service_upload_document.py b/test/unit_test/api/db/services/test_file_service_upload_document.pyindex 655f1f1476..e0410369e6 100644--- a/test/unit_test/api/db/services/test_file_service_upload_document.py+++ b/test/unit_test/api/db/services/test_file_service_upload_document.py@@ -247,6 +247,48 @@ def test_upload_document_reclaims_document_stranded_by_deleted_kb(monkeypatch):assert storage.written[("kb-target", "collision.txt")] == b"payload"+def test_upload_document_stops_remaining_files_when_cancelled(monkeypatch):+ kb = _target_kb()+ existing_doc = SimpleNamespace(+ id="doc-1",+ kb_id="kb-other",+ location="old-location.txt",+ content_hash="old-hash",+ to_dict=lambda: {"id": "doc-1"},+ )+ reads = []++ class _CountedFile(_DummyUploadFile):+ def read(self):+ reads.append(self.id)+ return super().read()++ _stub_folder_lookups(monkeypatch)+ monkeypatch.setattr(file_service_module.DocumentService, "get_by_id", lambda _doc_id: (True, existing_doc))+ monkeypatch.setattr(+ file_service_module.KnowledgebaseService,+ "get_or_none",+ classmethod(lambda cls, **_kwargs: SimpleNamespace(id="kb-other")),+ )++ cancel = types.SimpleNamespace(is_set=lambda: True)+ err, files = _unwrapped_upload_document()(+ FileService,+ kb,+ [+ _CountedFile(filename="one.txt", doc_id="doc-1", blob=b"one"),+ _CountedFile(filename="two.txt", doc_id="doc-2", blob=b"two"),+ ],+ "user-1",+ should_cancel=cancel,+ )++ assert err == []+ assert files == []+ assert reads == []++# ---------------------------------------------------------------------------# Helpers shared by TestValidateUrlForCrawl# ---------------------------------------------------------------------------diff --git a/test/unit_test/rag/test_sync_data_source.py b/test/unit_test/rag/test_sync_data_source.pyindex 021c2c7c2d..e2066d2db4 100644--- a/test/unit_test/rag/test_sync_data_source.py+++ b/test/unit_test/rag/test_sync_data_source.py@@ -13,10 +13,12 @@# See the License for the specific language governing permissions and# limitations under the License.#+import asyncioimport importlibimport importlib.utilimport osimport sys+import threadingimport typesimport warningsfrom datetime import datetime, timezone@@ -184,6 +186,256 @@ async def test_run_task_logic_skips_multiple_empty_sync_batches(monkeypatch):)._run_task_logic(_make_task())+async def test_run_task_logic_keeps_event_loop_responsive_during_parse(monkeypatch):+ _patch_common_dependencies(monkeypatch)+ started = threading.Event()+ release = threading.Event()++ def _slow_duplicate(*_args, **_kwargs):+ started.set()+ if not release.wait(timeout=5):+ pytest.fail("release was not signalled while parse was blocked")+ return [], ["doc-1"]++ monkeypatch.setattr(+ sync_data_source.KnowledgebaseService,+ "get_by_id",+ lambda *_args, **_kwargs: (True, object()),+ )+ monkeypatch.setattr(sync_data_source.SyncLogsService, "duplicate_and_parse", _slow_duplicate)+ monkeypatch.setattr(sync_data_source.SyncLogsService, "increase_docs", lambda *_args, **_kwargs: None)++ probe_done = False++ async def _probe():+ nonlocal probe_done+ flagged = await asyncio.to_thread(started.wait, 5)+ assert flagged, "ingest did not start"+ probe_done = True+ release.set()++ await asyncio.wait_for(+ asyncio.gather(+ _FakeSync(iter(([_make_fake_doc()],)))._run_task_logic(_make_task()),+ _probe(),+ ),+ timeout=10,+ )+ assert probe_done+++def _patch_successful_ingest(monkeypatch):+ monkeypatch.setattr(+ sync_data_source.KnowledgebaseService,+ "get_by_id",+ lambda *_args, **_kwargs: (True, object()),+ )+ monkeypatch.setattr(+ sync_data_source.SyncLogsService,+ "duplicate_and_parse",+ lambda *_args, **_kwargs: ([], ["doc-1"]),+ )+ monkeypatch.setattr(sync_data_source.SyncLogsService, "increase_docs", lambda *_args, **_kwargs: None)+++async def _collect_sync_sleeps(monkeypatch, generate_output):+ _patch_common_dependencies(monkeypatch)+ _patch_successful_ingest(monkeypatch)+ monkeypatch.setattr(sync_data_source, "SYNC_BATCH_PAUSE_SECONDS", 0.05)+ sleep_calls = []++ async def _track_sleep(delay, *_args, **_kwargs):+ sleep_calls.append(delay)++ monkeypatch.setattr(sync_data_source.asyncio, "sleep", _track_sleep)+ await _FakeSync(generate_output)._run_task_logic(_make_task())+ return sleep_calls+++async def test_run_task_logic_pauses_only_between_nonempty_batches(monkeypatch):+ sleep_calls = await _collect_sync_sleeps(+ monkeypatch,+ iter(([_make_fake_doc("doc-1")], [], [_make_fake_doc("doc-2")])),+ )+ assert sleep_calls == [0.05]+++async def test_run_task_logic_does_not_pause_after_last_batch(monkeypatch):+ sleep_calls = await _collect_sync_sleeps(monkeypatch, iter(([_make_fake_doc()],)))+ assert sleep_calls == []+++def test_ingest_document_batch_skips_writes_when_already_cancelled(monkeypatch):+ _patch_common_dependencies(monkeypatch)+ monkeypatch.setattr(+ sync_data_source.KnowledgebaseService,+ "get_by_id",+ lambda *_args, **_kwargs: pytest.fail("get_by_id should not run after cancel"),+ )+ monkeypatch.setattr(+ sync_data_source.SyncLogsService,+ "duplicate_and_parse",+ lambda *_args, **_kwargs: pytest.fail("duplicate_and_parse should not run after cancel"),+ )+ monkeypatch.setattr(+ sync_data_source.SyncLogsService,+ "increase_docs",+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not run after cancel"),+ )+ cancel_event = threading.Event()+ cancel_event.set()+ err, dids = _FakeSync(iter(()))._ingest_document_batch(+ _make_task(),+ [{"id": "doc-1"}],+ datetime(2026, 1, 1, tzinfo=timezone.utc),+ cancel_event,+ )+ assert err == []+ assert dids == []+++def test_ingest_document_batch_skips_progress_when_cancelled_during_parse(monkeypatch):+ _patch_common_dependencies(monkeypatch)+ cancel_event = threading.Event()+ monkeypatch.setattr(+ sync_data_source.KnowledgebaseService,+ "get_by_id",+ lambda *_args, **_kwargs: (True, object()),+ )++ def _duplicate_then_cancel(*_args, **_kwargs):+ cancel_event.set()+ return [], ["doc-1"]++ monkeypatch.setattr(sync_data_source.SyncLogsService, "duplicate_and_parse", _duplicate_then_cancel)+ monkeypatch.setattr(+ sync_data_source.SyncLogsService,+ "increase_docs",+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not advance after cancel"),+ )+ err, dids = _FakeSync(iter(()))._ingest_document_batch(+ _make_task(),+ [{"id": "doc-1"}],+ datetime(2026, 1, 1, tzinfo=timezone.utc),+ cancel_event,+ )+ assert err == []+ assert dids == ["doc-1"]+++class _PendingCancelTask:+ def cancelled(self):+ return False++ def cancelling(self):+ return 1+++class _DeferredCancelTask:+ def __init__(self):+ self._cancelling = 0++ def cancelled(self):+ return False++ def cancelling(self):+ return self._cancelling+++def test_ingest_document_batch_skips_writes_when_parent_task_is_cancelling(monkeypatch):+ _patch_common_dependencies(monkeypatch)+ monkeypatch.setattr(+ sync_data_source.KnowledgebaseService,+ "get_by_id",+ lambda *_args, **_kwargs: pytest.fail("get_by_id should not run after cancel"),+ )+ monkeypatch.setattr(+ sync_data_source.SyncLogsService,+ "duplicate_and_parse",+ lambda *_args, **_kwargs: pytest.fail("duplicate_and_parse should not run after cancel"),+ )+ monkeypatch.setattr(+ sync_data_source.SyncLogsService,+ "increase_docs",+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not run after cancel"),+ )+ err, dids = _FakeSync(iter(()))._ingest_document_batch(+ _make_task(),+ [{"id": "doc-1"}],+ datetime(2026, 1, 1, tzinfo=timezone.utc),+ threading.Event(),+ _PendingCancelTask(),+ )+ assert err == []+ assert dids == []+++def test_ingest_document_batch_skips_progress_when_parent_task_starts_cancelling(monkeypatch):+ _patch_common_dependencies(monkeypatch)+ parent_task = _DeferredCancelTask()+ monkeypatch.setattr(+ sync_data_source.KnowledgebaseService,+ "get_by_id",+ lambda *_args, **_kwargs: (True, object()),+ )++ def _duplicate_then_cancel(*_args, **_kwargs):+ parent_task._cancelling = 1+ return [], ["doc-1"]++ monkeypatch.setattr(sync_data_source.SyncLogsService, "duplicate_and_parse", _duplicate_then_cancel)+ monkeypatch.setattr(+ sync_data_source.SyncLogsService,+ "increase_docs",+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not advance after cancel"),+ )+ err, dids = _FakeSync(iter(()))._ingest_document_batch(+ _make_task(),+ [{"id": "doc-1"}],+ datetime(2026, 1, 1, tzinfo=timezone.utc),+ threading.Event(),+ parent_task,+ )+ assert err == []+ assert dids == ["doc-1"]+++def test_ingest_document_batch_forwards_cancel_callback(monkeypatch):+ _patch_common_dependencies(monkeypatch)+ captured = {}+ monkeypatch.setattr(+ sync_data_source.KnowledgebaseService,+ "get_by_id",+ lambda *_args, **_kwargs: (True, object()),+ )++ def _duplicate(*_args, **kwargs):+ captured["should_cancel"] = kwargs.get("should_cancel")+ return [], ["doc-1"]++ monkeypatch.setattr(sync_data_source.SyncLogsService, "duplicate_and_parse", _duplicate)+ monkeypatch.setattr(sync_data_source.SyncLogsService, "increase_docs", lambda *_args, **_kwargs: None)+ cancel_event = threading.Event()+ err, dids = _FakeSync(iter(()))._ingest_document_batch(+ _make_task(),+ [{"id": "doc-1"}],+ datetime(2026, 1, 1, tzinfo=timezone.utc),+ cancel_event,+ )+ assert err == []+ assert dids == ["doc-1"]+ assert captured["should_cancel"] is not None+ assert captured["should_cancel"]() is False+ cancel_event.set()+ assert captured["should_cancel"]() is True++@pytest.mark.asyncio@pytest.mark.p2async def test_run_prune_task_logic_cleans_up_for_empty_snapshot(monkeypatch):@@ -294,6 +546,10 @@ class _FakeRDBMSConnector:self.prepare_sync_state_called = Falseself.load_from_cursor_range_called = Falseself.persist_sync_state_called = False+ self.close_connection_called = False+ self.validate_thread_ident = None+ self.prepare_thread_ident = None+ self.close_thread_ident = Noneself._pending_sync_cursor_value = None_FakeRDBMSConnector.instance = self@@ -301,12 +557,17 @@ class _FakeRDBMSConnector:self.credentials = credentialsdef validate_connector_settings(self):- return None+ self.validate_thread_ident = threading.get_ident()def prepare_sync_state(self, connector_id, config):self.prepare_sync_state_called = True+ self.prepare_thread_ident = threading.get_ident()self.prepare_sync_state_args = (connector_id, config)+ def _close_connection(self):+ self.close_connection_called = True+ self.close_thread_ident = threading.get_ident()+def get_saved_sync_cursor_value(self):return None@@ -350,12 +611,18 @@ async def test_rdbms_generate_keeps_deleted_file_snapshot_without_timestamp_colu})+ main_ident = threading.get_ident()document_generator = await sync._generate(task)connector = _FakeRDBMSConnector.instanceassert connector is not Noneassert connector.load_from_state_called is Trueassert connector.load_from_cursor_range_called is False+ assert connector.prepare_sync_state_called is True+ assert connector.close_connection_called is True+ assert connector.validate_thread_ident != main_ident+ assert connector.prepare_thread_ident == connector.validate_thread_ident+ assert connector.close_thread_ident == connector.validate_thread_identfile_list = sync._collect_prune_snapshot(task)assert connector.retrieve_all_slim_docs_perm_sync_called is Trueassert file_list is not None@@ -933,6 +1200,32 @@ async def test_dropbox_generate_skips_snapshot_for_full_reindex(monkeypatch):assert connector.poll_source_called is False+def test_index_batch_size_env_helpers_use_defaults_for_invalid_values(monkeypatch):+ from common.data_source import config as data_source_config++ monkeypatch.delenv("RAGFLOW_TEST_INDEX_BATCH_SIZE", raising=False)+ assert data_source_config._env_int("RAGFLOW_TEST_INDEX_BATCH_SIZE", 2) == 2+ monkeypatch.setenv("RAGFLOW_TEST_INDEX_BATCH_SIZE", "32")+ assert data_source_config._env_int("RAGFLOW_TEST_INDEX_BATCH_SIZE", 2) == 32+ monkeypatch.setenv("RAGFLOW_TEST_INDEX_BATCH_SIZE", "0")+ assert data_source_config._env_int("RAGFLOW_TEST_INDEX_BATCH_SIZE", 2) == 2+ monkeypatch.setenv("RAGFLOW_TEST_INDEX_BATCH_SIZE", "nope")+ assert data_source_config._env_int("RAGFLOW_TEST_INDEX_BATCH_SIZE", 2) == 2++ monkeypatch.delenv("RAGFLOW_TEST_SYNC_PAUSE", raising=False)+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "0.05")+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.05+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "-1")+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "inf")+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "nan")+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "-inf")+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0++def test_redact_url_strips_credentials_query_and_fragment():_redact_url = sync_data_source._redact_urldiff --git a/web/src/pages/user-setting/data-source/constant/index.test.tsx b/web/src/pages/user-setting/data-source/constant/index.test.tsxindex dc0ff8ccf3..dd92b0f954 100644--- a/web/src/pages/user-setting/data-source/constant/index.test.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.test.tsx@@ -120,6 +120,24 @@ describe('Sitemap data source', () => {});});+describe.each([+ ['MySQL', DataSourceKey.MYSQL],+ ['PostgreSQL', DataSourceKey.POSTGRESQL],+])('%s data source', (_label, key) => {+ it('exposes a bounded batch size for large table syncs', () => {+ const fields = getDataSourceFieldsWithExtras(translate, key) as Array<{+ name: string;+ validation?: { min?: number };+ }>;+ const batchSize = fields.find((f) => f.name === 'config.batch_size');++ expect(batchSize?.validation?.min).toBe(1);+ expect(DataSourceFormDefaultValues[key].config).toMatchObject({+ batch_size: 2,+ });+ });+});+describe('Azure DevOps data source', () => {it('registers its catalog entry and defaults', () => {const info = generateDataSourceInfo(translate)[DataSourceKey.AZURE_DEVOPS];diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsxindex 5aa394cb64..307b321941 100644--- a/web/src/pages/user-setting/data-source/constant/index.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.tsx@@ -1568,6 +1568,19 @@ const generateDataSourceFormFields = (t: TFunction) => ({placeholder: '.txt',tooltip: t('setting.mysqlFileExtensionTip'),},+ {+ label: t('setting.dataSourceFieldBatchSize'),+ name: 'config.batch_size',+ type: FormFieldType.Number,+ required: false,+ placeholder: '2',+ validation: {+ min: 1,+ message: t('setting.dataSourceValidationMinOne', {+ label: t('setting.dataSourceFieldBatchSize'),+ }),+ },+ },],[DataSourceKey.POSTGRESQL]: [{@@ -1650,6 +1663,19 @@ const generateDataSourceFormFields = (t: TFunction) => ({placeholder: '.txt',tooltip: t('setting.postgresqlFileExtensionTip'),},+ {+ label: t('setting.dataSourceFieldBatchSize'),+ name: 'config.batch_size',+ type: FormFieldType.Number,+ required: false,+ placeholder: '2',+ validation: {+ min: 1,+ message: t('setting.dataSourceValidationMinOne', {+ label: t('setting.dataSourceFieldBatchSize'),+ }),+ },+ },],[DataSourceKey.BIGQUERY]: [{@@ -2508,6 +2534,7 @@ export const DataSourceFormDefaultValues = {metadata_columns: '',id_column: '',timestamp_column: '',+ batch_size: 2,credentials: {username: '',password: '',@@ -2526,6 +2553,7 @@ export const DataSourceFormDefaultValues = {metadata_columns: '',id_column: '',timestamp_column: '',+ batch_size: 2,credentials: {username: '',password: '',