Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: keep file API responsive during large connector ingest (#19632)

Öndery98ebe5f66f2c
diff --git a/api/apps/restful_apis/file_api.py b/api/apps/restful_apis/file_api.py
index 3657a9be99..4545b6604d 100644
--- a/api/apps/restful_apis/file_api.py
+++ b/api/apps/restful_apis/file_api.py
@@ -141,7 +141,7 @@ async def list_files(tenant_id: str = None):
return get_error_argument_result(err)
try:
- success, result = file_api_service.list_files(tenant_id, args)
+ success, result = await thread_pool_exec(file_api_service.list_files, tenant_id, args)
if success:
return get_result(data=result)
else:
diff --git a/api/apps/services/file_api_service.py b/api/apps/services/file_api_service.py
index f3060741af..65762f400f 100644
--- a/api/apps/services/file_api_service.py
+++ b/api/apps/services/file_api_service.py
@@ -41,10 +41,10 @@ async def upload_file(tenant_id: str, pf_id: str, file_objs: list):
:return: (success, result_list) or (success, error_message)
"""
if not pf_id:
- root_folder = FileService.get_root_folder(tenant_id)
+ root_folder = await thread_pool_exec(FileService.get_root_folder, tenant_id)
pf_id = root_folder["id"]
- e, pf_folder = FileService.get_by_id(pf_id)
+ e, pf_folder = await thread_pool_exec(FileService.get_by_id, pf_id)
if not e:
return False, "Can't find this folder!"
@@ -114,6 +114,11 @@ async def create_folder(tenant_id: str, name: str, pf_id: str | None = None, fil
if "/" in name:
return False, 'Folder name cannot contain "/"'
+ return await thread_pool_exec(_create_folder_sync, tenant_id, name, pf_id, file_type)
+
+
+def _create_folder_sync(tenant_id: str, name: str, pf_id: str | None, file_type: str | None):
+ """Insert a folder or virtual file; intended to run off the Quart event loop."""
if not pf_id:
root_folder = FileService.get_root_folder(tenant_id)
pf_id = root_folder["id"]
diff --git a/api/db/services/connector_service.py b/api/db/services/connector_service.py
index a46b4fb028..a44570fabb 100644
--- a/api/db/services/connector_service.py
+++ b/api/db/services/connector_service.py
@@ -542,11 +542,13 @@ class SyncLogsService(CommonService):
cls.model.update(update_payload).where(cls.model.id == id).execute()
@classmethod
- def duplicate_and_parse(cls, kb, docs, tenant_id, src, auto_parse=True):
+ def duplicate_and_parse(cls, kb, docs, tenant_id, src, auto_parse=True, should_cancel=None):
from api.db.services.file_service import FileService
if not docs:
return None
+ if FileService._is_sync_cancelled(should_cancel):
+ return [], []
class FileObj(BaseModel):
id: str
@@ -568,7 +570,7 @@ class SyncLogsService(CommonService):
for d in docs
]
doc_ids = []
- err, doc_blob_pairs = FileService.upload_document(kb, files, tenant_id, src)
+ err, doc_blob_pairs = FileService.upload_document(kb, files, tenant_id, src, should_cancel=should_cancel)
errs.extend(err)
# Create a mapping from filename to metadata for later use
@@ -579,16 +581,24 @@ class SyncLogsService(CommonService):
metadata_map[filename] = d["metadata"]
kb_table_num_map = {}
- for doc, _ in doc_blob_pairs:
- doc_ids.append(doc["id"])
-
- # Set metadata if available for this document
- if doc["name"] in metadata_map:
- DocMetadataService.update_document_metadata(doc["id"], metadata_map[doc["name"]])
-
- if not auto_parse or auto_parse == "0":
- continue
- DocumentService.run(tenant_id, doc, kb_table_num_map)
+ wrote_meta = False
+ try:
+ for doc, _ in doc_blob_pairs:
+ if FileService._is_sync_cancelled(should_cancel):
+ break
+ doc_ids.append(doc["id"])
+
+ # Set metadata if available for this document
+ if doc["name"] in metadata_map:
+ DocMetadataService.update_document_metadata(doc["id"], metadata_map[doc["name"]], refresh_now=False)
+ wrote_meta = True
+
+ if not auto_parse or auto_parse == "0":
+ continue
+ DocumentService.run(tenant_id, doc, kb_table_num_map)
+ finally:
+ if wrote_meta:
+ DocMetadataService.refresh_tenant_index(tenant_id)
return errs, doc_ids
diff --git a/api/db/services/doc_metadata_service.py b/api/db/services/doc_metadata_service.py
index 9e75b6d43a..093c9c86c9 100644
--- a/api/db/services/doc_metadata_service.py
+++ b/api/db/services/doc_metadata_service.py
@@ -361,13 +361,16 @@ class DocMetadataService:
@classmethod
@DB.connection_context()
- def insert_document_metadata(cls, doc_id: str, meta_fields: dict) -> bool:
+ def insert_document_metadata(cls, doc_id: str, meta_fields: dict, *, refresh_now: bool = True) -> bool:
"""
Insert document metadata into ES/Infinity.
Args:
doc_id: Document ID
meta_fields: Metadata dictionary
+ refresh_now: When True, wait for the index refresh so the row is
+ searchable immediately. Connector ingest passes False and
+ refreshes once per batch so agent retrieval is not blocked.
Returns:
True if successful, False otherwise
@@ -419,8 +422,9 @@ class DocMetadataService:
else:
logging.debug(f"Metadata table already exists: {index_name}")
- # Insert into ES/Infinity
- result = settings.docStoreConn.insert([doc_meta], index_name, kb_id)
+ # Insert into ES/Infinity. Connector ingest skips wait_for so agent
+ # searches are not queued behind a refresh per uploaded row.
+ result = settings.docStoreConn.insert([doc_meta], index_name, kb_id, refresh="wait_for" if refresh_now else False)
if result:
logging.error(f"Failed to insert metadata for document {doc_id}: {result}")
@@ -428,18 +432,8 @@ class DocMetadataService:
# Force refresh so metadata is immediately searchable.
# Both Elasticsearch and OpenSearch backends expose refresh_idx;
# Infinity does not need a manual refresh.
- if not settings.DOC_ENGINE_INFINITY:
- refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None)
- if callable(refresh_idx):
- if refresh_idx(index_name):
- logging.debug(f"Refreshed metadata index: {index_name}")
- else:
- # A failed refresh can leave just-inserted metadata
- # invisible to subsequent reads; surface it so operators
- # can correlate stale-read complaints with the cause.
- logging.warning(f"Failed to refresh metadata index {index_name} on backend {type(settings.docStoreConn).__name__}; metadata may not be immediately searchable")
- else:
- logging.debug(f"Backend {type(settings.docStoreConn).__name__} has no refresh_idx; skipping")
+ if refresh_now:
+ cls.refresh_tenant_index(tenant_id)
logging.debug(f"Successfully inserted metadata for document {doc_id}")
return True
@@ -448,9 +442,24 @@ class DocMetadataService:
logging.error(f"Error inserting metadata for document {doc_id}: {e}")
return False
+ @classmethod
+ def refresh_tenant_index(cls, tenant_id: str) -> None:
+ """Refresh the per-tenant metadata index when the backend needs it."""
+ if settings.DOC_ENGINE_INFINITY:
+ return
+ index_name = cls._get_doc_meta_index_name(tenant_id)
+ refresh_idx = getattr(settings.docStoreConn, "refresh_idx", None)
+ if not callable(refresh_idx):
+ logging.debug(f"Backend {type(settings.docStoreConn).__name__} has no refresh_idx; skipping")
+ return
+ if refresh_idx(index_name):
+ logging.debug(f"Refreshed metadata index: {index_name}")
+ return
+ logging.warning(f"Failed to refresh metadata index {index_name} on backend {type(settings.docStoreConn).__name__}; metadata may not be immediately searchable")
+
@classmethod
@DB.connection_context()
- def update_document_metadata(cls, doc_id: str, meta_fields: dict) -> bool:
+ def update_document_metadata(cls, doc_id: str, meta_fields: dict, *, refresh_now: bool = True) -> bool:
"""
Update document metadata in ES/Infinity.
@@ -460,6 +469,8 @@ class DocMetadataService:
Args:
doc_id: Document ID
meta_fields: Metadata dictionary
+ refresh_now: When True, wait for the index refresh so the row is
+ searchable immediately. Connector ingest passes False.
Returns:
True if successful, False otherwise
@@ -511,7 +522,7 @@ class DocMetadataService:
if result is False:
logging.error(f"Failed to create metadata index {index_name}")
return False
- return cls.insert_document_metadata(doc_id, processed_meta)
+ return cls.insert_document_metadata(doc_id, processed_meta, refresh_now=refresh_now)
# Index exists - check if document exists
try:
@@ -536,18 +547,18 @@ class DocMetadataService:
# replace still guarantees full overwrite semantics rather
# than leaking through the "document not found" branch.
cls.delete_document_metadata(doc_id, kb_id, tenant_id)
- return cls.insert_document_metadata(doc_id, processed_meta)
+ return cls.insert_document_metadata(doc_id, processed_meta, refresh_now=refresh_now)
except Exception as e:
logging.debug(f"Document {doc_id} not found in index, will insert: {e}")
# Document doesn't exist - insert new
logging.debug(f"[update_document_metadata] Document {doc_id} not found, inserting new")
- return cls.insert_document_metadata(doc_id, processed_meta)
+ return cls.insert_document_metadata(doc_id, processed_meta, refresh_now=refresh_now)
# For Infinity or as fallback: use delete+insert
logging.debug(f"[update_document_metadata] Using delete+insert method for doc_id: {doc_id}")
cls.delete_document_metadata(doc_id, kb_id, tenant_id)
- return cls.insert_document_metadata(doc_id, processed_meta)
+ return cls.insert_document_metadata(doc_id, processed_meta, refresh_now=refresh_now)
except Exception as e:
logging.error(f"Error updating metadata for document {doc_id}: {e}")
diff --git a/api/db/services/file_service.py b/api/db/services/file_service.py
index 245b7852af..49005f6649 100644
--- a/api/db/services/file_service.py
+++ b/api/db/services/file_service.py
@@ -575,9 +575,18 @@ class FileService(CommonService):
DocumentService.delete_by_id(doc.id)
return True
+ @staticmethod
+ def _is_sync_cancelled(should_cancel) -> bool:
+ if should_cancel is None:
+ return False
+ is_set = getattr(should_cancel, "is_set", None)
+ if callable(is_set):
+ return bool(is_set())
+ return bool(should_cancel())
+
@classmethod
@DB.connection_context()
- def upload_document(self, kb, file_objs, user_id, src="local", parent_path: str | None = None, parser_config_override: dict | None = None):
+ def upload_document(self, kb, file_objs, user_id, src="local", parent_path: str | None = None, parser_config_override: dict | None = None, should_cancel=None):
root_folder = self.get_root_folder(user_id)
pf_id = root_folder["id"]
self.init_knowledgebase_docs(pf_id, user_id)
@@ -595,6 +604,8 @@ class FileService(CommonService):
err, files = [], []
for file in file_objs:
+ if self._is_sync_cancelled(should_cancel):
+ break
doc_id = file.id if hasattr(file, "id") else get_uuid()
e, doc = DocumentService.get_by_id(doc_id)
if e and str(doc.kb_id) != str(kb.id):
diff --git a/common/data_source/config.py b/common/data_source/config.py
index f6cba01557..2198a4f360 100644
--- a/common/data_source/config.py
+++ b/common/data_source/config.py
@@ -1,6 +1,7 @@
"""Configuration constants and enum definitions"""
import json
+import math
import os
from datetime import datetime, timezone
from enum import Enum
@@ -117,12 +118,42 @@ _PAGE_EXPANSION_FIELDS = [
]
+def _env_int(name: str, default: int, *, minimum: int = 1) -> int:
+ """Parse a positive integer env var, returning ``default`` when unset or invalid."""
+ raw = os.environ.get(name)
+ if raw is None or str(raw).strip() == "":
+ return default
+ try:
+ value = int(raw)
+ except (TypeError, ValueError):
+ return default
+ return value if value >= minimum else default
+
+
+def _env_float(name: str, default: float, *, minimum: float = 0.0) -> float:
+ """Parse a finite float env var, returning ``default`` when unset or invalid."""
+ raw = os.environ.get(name)
+ if raw is None or str(raw).strip() == "":
+ return default
+ try:
+ value = float(raw)
+ except (TypeError, ValueError):
+ return default
+ if not math.isfinite(value) or value < minimum:
+ return default
+ return value
+
+
# Configuration constants
try:
BLOB_STORAGE_SIZE_THRESHOLD = int(os.environ.get("BLOB_STORAGE_SIZE_THRESHOLD", 20 * 1024 * 1024))
except ValueError as error:
raise ValueError("BLOB_STORAGE_SIZE_THRESHOLD must be an integer number of bytes") from error
-INDEX_BATCH_SIZE = 2
+# Connector ingest batch size (rows/files handed to duplicate_and_parse at once).
+INDEX_BATCH_SIZE = _env_int("INDEX_BATCH_SIZE", 2)
+# Yield the sync worker event loop (and MySQL) between nonempty write batches.
+# 0 still schedules sleep(0) between batches, not after the last one.
+SYNC_BATCH_PAUSE_SECONDS = _env_float("SYNC_BATCH_PAUSE_SECONDS", 0.0)
SLACK_NUM_THREADS = 4
ENABLE_EXPENSIVE_EXPERT_CALLS = False
diff --git a/docker/.env b/docker/.env
index 5997e07846..147d1b6c13 100644
--- a/docker/.env
+++ b/docker/.env
@@ -312,6 +312,13 @@ DOC_BULK_SIZE=${DOC_BULK_SIZE:-4}
# Defaults to 16 if EMBEDDING_BATCH_SIZE is not set in the environment.
EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-16}
+# Connector ingest: documents (e.g. one MySQL row = one file) written per batch.
+# Defaults to 2. Raise this for large RDBMS syncs so fewer write round-trips hit MySQL.
+# INDEX_BATCH_SIZE=32
+# Pause between connector write batches so the file/storage API can use MySQL.
+# 0 still yields the sync worker event loop. A small value (e.g. 0.05) reduces lock wait.
+# SYNC_BATCH_PAUSE_SECONDS=0.05
+
# Controls how rerank inputs that exceed the configured model token limit are handled.
# Available options:
# - `truncate` (default): truncate each document so its tokens plus the query tokens fit the limit.
diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.py
index 5cddf26de6..f8b0f9d80c 100644
--- a/rag/svr/sync_data_source.py
+++ b/rag/svr/sync_data_source.py
@@ -44,7 +44,7 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
from common import settings
from common.constants import ConnectorTaskType, FileSource, TaskStatus
from common.config_utils import show_configs
-from common.data_source.config import INDEX_BATCH_SIZE
+from common.data_source.config import INDEX_BATCH_SIZE, SYNC_BATCH_PAUSE_SECONDS
from common.data_source import (
BlobStorageConnector,
RSSConnector,
@@ -261,21 +261,25 @@ class SyncBase:
next_update = datetime(1970, 1, 1, tzinfo=timezone.utc)
saw_documents = False
source_type = f"{self.SOURCE_NAME}/{task['connector_id']}"
- existing_doc_ids = {
- doc["id"]
- for doc in DocumentService.list_doc_headers_by_kb_and_source_type(
- task["kb_id"],
- source_type,
- )
- }
+ existing_headers = await asyncio.to_thread(
+ DocumentService.list_doc_headers_by_kb_and_source_type,
+ task["kb_id"],
+ source_type,
+ )
+ existing_doc_ids = {doc["id"] for doc in existing_headers}
if task["poll_range_start"]:
next_update = task["poll_range_start"]
+ pause_before_next_batch = False
async for document_batch in _iterate_document_batches(document_batch_generator):
if not document_batch:
continue
+ if pause_before_next_batch:
+ await asyncio.sleep(SYNC_BATCH_PAUSE_SECONDS)
+ pause_before_next_batch = True
+
saw_documents = True
max_update = max(doc.doc_updated_at for doc in document_batch)
next_update = max(next_update, max_update)
@@ -298,14 +302,12 @@ class SyncBase:
d["fingerprint"] = doc.fingerprint
docs.append(d)
+ cancel_event = threading.Event()
+ parent_task = asyncio.current_task()
try:
- e, kb = KnowledgebaseService.get_by_id(task["kb_id"])
- err, dids = SyncLogsService.duplicate_and_parse(kb, docs, task["tenant_id"], f"{self.SOURCE_NAME}/{task['connector_id']}", task["auto_parse"])
+ err, dids = await asyncio.to_thread(self._ingest_document_batch, task, docs, max_update, cancel_event, parent_task)
if err:
had_parse_errors = True
- if self.RAISE_ON_BATCH_ERROR:
- raise RuntimeError(f"{self.SOURCE_NAME} failed to process {len(err)} document(s)")
- SyncLogsService.increase_docs(task["id"], max_update, len(docs), "\n".join(err), len(err))
changed_doc_ids = set(dids)
updated_in_batch = len(changed_doc_ids & existing_doc_ids)
added_in_batch = len(changed_doc_ids) - updated_in_batch
@@ -313,6 +315,9 @@ class SyncBase:
updated_docs += updated_in_batch
existing_doc_ids.update(changed_doc_ids)
+ except asyncio.CancelledError:
+ cancel_event.set()
+ raise
except Exception as batch_ex:
msg = str(batch_ex)
code = getattr(batch_ex, "args", [None])[0]
@@ -325,7 +330,6 @@ class SyncBase:
if self.RAISE_ON_BATCH_ERROR:
raise
failed_docs += len(docs)
- continue
if not saw_documents:
next_update = self._get_empty_sync_cursor(task, next_update)
@@ -385,6 +389,40 @@ class SyncBase:
async def _generate(self, task: dict):
raise NotImplementedError
+ def _ingest_document_batch(self, task: dict, docs: list, max_update: datetime, cancel_event: threading.Event | None = None, parent_task: asyncio.Task | None = None):
+ """Write one connector batch (MinIO + document rows) off the event loop.
+
+ Knowledgebase lookup and parse stay on the same worker thread so Peewee
+ objects are not shared across threads. ``cancel_event`` is set when the
+ awaiting coroutine catches ``CancelledError``. ``cancelled()`` stays false
+ until that handler runs, so the worker also observes ``cancelling()`` to
+ stop between files after ``wait_for`` times out. Uploads that already
+ started cannot be force-stopped.
+ """
+
+ def should_cancel() -> bool:
+ if cancel_event is not None and cancel_event.is_set():
+ return True
+ return parent_task is not None and (parent_task.cancelling() > 0 or parent_task.cancelled())
+
+ if should_cancel():
+ return [], []
+ _e, kb = KnowledgebaseService.get_by_id(task["kb_id"])
+ err, dids = SyncLogsService.duplicate_and_parse(
+ kb,
+ docs,
+ task["tenant_id"],
+ f"{self.SOURCE_NAME}/{task['connector_id']}",
+ task["auto_parse"],
+ should_cancel=should_cancel,
+ )
+ if should_cancel():
+ return err, dids
+ if err and self.RAISE_ON_BATCH_ERROR:
+ raise RuntimeError(f"{self.SOURCE_NAME} failed to process {len(err)} document(s)")
+ SyncLogsService.increase_docs(task["id"], max_update, len(docs), "\n".join(err), len(err))
+ return err, dids
+
def _get_source_prefix(self):
return ""
@@ -2217,8 +2255,7 @@ class _RDBMSBase(_CursorPersistingSyncBase):
raise ValueError(f"{self.DB_TYPE} connector is missing credentials.")
self.connector.load_credentials(credentials)
- self.connector.validate_connector_settings()
- self.connector.prepare_sync_state(task["connector_id"], self.conf)
+ await asyncio.to_thread(self._prepare_rdbms_connector, task)
if task["reindex"] == "1" or not task["poll_range_start"]:
document_generator = self.connector.load_from_state()
@@ -2238,7 +2275,15 @@ class _RDBMSBase(_CursorPersistingSyncBase):
_begin_info = f"from {poll_start}"
self.log_connection(self.LOG_NAME, f"{self.conf.get('host')}:{self.conf.get('database')}", task)
- return document_generator
+ return iter_in_worker_thread(document_generator)
+
+ def _prepare_rdbms_connector(self, task: dict) -> None:
+ """Validate and snapshot cursor state off the event loop, then drop the setup connection."""
+ self.connector.validate_connector_settings()
+ self.connector.prepare_sync_state(task["connector_id"], self.conf)
+ close_connection = getattr(self.connector, "_close_connection", None)
+ if close_connection is not None:
+ close_connection()
class MySQL(_RDBMSBase):
diff --git a/test/testcases/restful_api/test_file_routes_unit.py b/test/testcases/restful_api/test_file_routes_unit.py
index 5988d04591..86e9542fc0 100644
--- a/test/testcases/restful_api/test_file_routes_unit.py
+++ b/test/testcases/restful_api/test_file_routes_unit.py
@@ -268,6 +268,15 @@ def test_list_files_validation_error(monkeypatch):
assert res["message"] == "bad args"
+def test_list_files_success_offloads_to_thread_pool(monkeypatch):
+ module = _load_file_api_module(monkeypatch)
+
+ res = _run(module.list_files("tenant1"))
+ assert res["code"] == 0
+ assert res["data"] == {"files": [], "total": 0}
+
+
@pytest.mark.p2
def test_move_uses_new_payload_shape(monkeypatch):
module = _load_file_api_module(monkeypatch)
@@ -901,6 +910,51 @@ def test_create_folder_rejects_slash_in_name(monkeypatch):
assert message == 'Folder name cannot contain "/"'
+def test_create_folder_success_offloads_sync_work(monkeypatch):
+ module = _load_file_api_service(monkeypatch)
+
+ ok, data = _run(module.create_folder("tenant1", "new-folder", "pf1", module.FileType.FOLDER.value))
+ assert ok is True
+ assert data["name"] == "new-folder"
+ assert data["parent_id"] == "pf1"
+ assert data["type"] == module.FileType.FOLDER.value
+
+
+def test_upload_file_uses_root_folder_when_parent_missing(monkeypatch):
+ module = _load_file_api_service(monkeypatch)
+ seen_ids = []
+
+ monkeypatch.setattr(
+ module.FileService,
+ "get_by_id",
+ lambda file_id: (True, SimpleNamespace(id=file_id, name=str(file_id))),
+ )
+ monkeypatch.setattr(module.FileService, "get_id_list_by_id", lambda *_args, **_kwargs: ["root"])
+ monkeypatch.setattr(
+ module.FileService,
+ "create_folder",
+ lambda _file, parent_id, _names, _len_id, *_args: SimpleNamespace(id=parent_id),
+ )
+ monkeypatch.setattr(
+ module.settings,
+ "STORAGE_IMPL",
+ SimpleNamespace(
+ obj_exist=lambda *_args, **_kwargs: False,
+ put=lambda bucket, location, blob: seen_ids.append((bucket, location, blob)),
+ rm=lambda *_args, **_kwargs: None,
+ move=lambda *_args, **_kwargs: None,
+ ),
+ )
+
+ ok, data = _run(module.upload_file("tenant1", "", [_DummyUploadFile("a.txt", b"hello")]))
+ assert ok is True
+ assert data[0]["name"] == "a.txt"
+ assert data[0]["parent_id"] == "root"
+ assert seen_ids == [("root", "a.txt", b"hello")]
+
+
@pytest.mark.p2
def test_delete_files_checks_team_permission(monkeypatch):
module = _load_file_api_service(monkeypatch)
diff --git a/test/testcases/restful_api/test_files.py b/test/testcases/restful_api/test_files.py
new file mode 100644
index 0000000000..566b14657a
--- /dev/null
+++ b/test/testcases/restful_api/test_files.py
@@ -0,0 +1,64 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import uuid
+
+import pytest
+
+from test.testcases.utils.file_utils import create_txt_file
+
+
+def _assert_ok(res, *, expect_list=False):
+ assert res.status_code == 200, res.text
+ payload = res.json()
+ assert payload["code"] == 0, payload
+ data = payload["data"]
+ if expect_list:
+ assert isinstance(data, dict), payload
+ assert "files" in data and "total" in data, payload
+ return data
+
+
+def test_files_list_create_folder_and_upload_to_root(rest_client, tmp_path):
+ """Exercise GET/POST /files so connector offload paths run on the live server."""
+ created_ids: list[str] = []
+ suffix = uuid.uuid4().hex[:12]
+ folder_name = f"rf_folder_{suffix}"
+ upload_name = f"rf_upload_{suffix}.txt"
+
+ try:
+ list_data = _assert_ok(rest_client.get("/files"), expect_list=True)
+ assert isinstance(list_data["files"], list), list_data
+
+ folder = _assert_ok(rest_client.post("/files", json={"name": folder_name, "type": "folder"}))
+ folder_id = folder.get("id")
+ assert folder_id, folder
+ created_ids.append(folder_id)
+ assert folder.get("name") == folder_name
+
+ fp = create_txt_file(tmp_path / upload_name)
+ with fp.open("rb") as file_obj:
+ upload_res = rest_client.post("/files", files=[("file", (fp.name, file_obj))])
+ uploaded = _assert_ok(upload_res)
+ assert isinstance(uploaded, list) and uploaded, uploaded
+ file_id = uploaded[0].get("id")
+ assert file_id, uploaded
+ created_ids.append(file_id)
+ assert uploaded[0].get("name") == upload_name
+ finally:
+ if created_ids:
+ rest_client.delete("/files", json={"ids": created_ids})
diff --git a/test/unit_test/api/db/services/test_connector_duplicate_and_parse.py b/test/unit_test/api/db/services/test_connector_duplicate_and_parse.py
new file mode 100644
index 0000000000..da10d304dc
--- /dev/null
+++ b/test/unit_test/api/db/services/test_connector_duplicate_and_parse.py
@@ -0,0 +1,101 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Deferred connector metadata refresh still runs if parse/queue fails."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from api.db.services import connector_service as connector_mod
+from api.db.services.connector_service import SyncLogsService
+from api.db.services.file_service import FileService
+
+
+TENANT_ID = "tenant-1"
+FILENAME = "row-1.txt"
+DOC = {"id": "doc-1", "name": FILENAME}
+
+
+def _docs():
+ return [
+ {
+ "id": "row-1",
+ "semantic_identifier": "row-1",
+ "extension": ".txt",
+ "blob": b"payload",
+ "metadata": {"source": "mysql"},
+ }
+ ]
+
+
+def _stub_upload(monkeypatch, pairs):
+ monkeypatch.setattr(FileService, "_is_sync_cancelled", staticmethod(lambda *_args, **_kwargs: False))
+ monkeypatch.setattr(
+ FileService,
+ "upload_document",
+ classmethod(lambda *_args, **_kwargs: ([], pairs)),
+ )
+
+
+def test_duplicate_and_parse_refreshes_metadata_when_run_fails(monkeypatch):
+ _stub_upload(monkeypatch, [(DOC, b"payload")])
+ monkeypatch.setattr(connector_mod.DocMetadataService, "update_document_metadata", classmethod(lambda *_args, **_kwargs: True))
+ refresh_calls = []
+ monkeypatch.setattr(
+ connector_mod.DocMetadataService,
+ "refresh_tenant_index",
+ classmethod(lambda cls, tenant_id: refresh_calls.append(tenant_id)),
+ )
+
+ def _fail_run(*_args, **_kwargs):
+ raise RuntimeError("queue failed")
+
+ monkeypatch.setattr(connector_mod.DocumentService, "run", classmethod(_fail_run))
+
+ with pytest.raises(RuntimeError, match="queue failed"):
+ SyncLogsService.duplicate_and_parse(SimpleNamespace(), _docs(), TENANT_ID, "mysql/1", auto_parse=True)
+
+ assert refresh_calls == [TENANT_ID]
+
+
+def test_duplicate_and_parse_skips_refresh_when_no_metadata_was_written(monkeypatch):
+ docs = [
+ {
+ "id": "row-1",
+ "semantic_identifier": "row-1",
+ "extension": ".txt",
+ "blob": b"payload",
+ }
+ ]
+ _stub_upload(monkeypatch, [(DOC, b"payload")])
+ refresh_calls = []
+ monkeypatch.setattr(
+ connector_mod.DocMetadataService,
+ "refresh_tenant_index",
+ classmethod(lambda cls, tenant_id: refresh_calls.append(tenant_id)),
+ )
+
+ def _fail_run(*_args, **_kwargs):
+ raise RuntimeError("queue failed")
+
+ monkeypatch.setattr(connector_mod.DocumentService, "run", classmethod(_fail_run))
+
+ with pytest.raises(RuntimeError, match="queue failed"):
+ SyncLogsService.duplicate_and_parse(SimpleNamespace(), docs, TENANT_ID, "mysql/1", auto_parse=True)
+
+ assert refresh_calls == []
diff --git a/test/unit_test/api/db/services/test_doc_metadata_service.py b/test/unit_test/api/db/services/test_doc_metadata_service.py
index 65c4b1d785..2e713baf81 100644
--- a/test/unit_test/api/db/services/test_doc_metadata_service.py
+++ b/test/unit_test/api/db/services/test_doc_metadata_service.py
@@ -75,3 +75,33 @@ class TestDocMetadataServiceConnectorGetCalls:
[],
)
assert result == {"author": "alice"}
+
+
+class TestDocMetadataServiceRefreshNow:
+ def test_insert_skips_wait_for_when_refresh_now_is_false(self):
+ mock_doc_store = MagicMock()
+ mock_doc_store.index_exist.return_value = True
+ mock_doc_store.insert.return_value = []
+ mock_doc_store.refresh_idx = MagicMock(return_value=True)
+
+ mock_doc = MagicMock()
+ mock_doc.id = "doc_789"
+ mock_doc.kb_id = "kb_123"
+ mock_doc.knowledgebase.tenant_id = "tenant_456"
+
+ with (
+ patch("api.db.services.doc_metadata_service.settings") as mock_settings,
+ patch("api.db.services.doc_metadata_service.Document") as mock_document_model,
+ patch("api.db.db_models.DB.connect"),
+ patch("api.db.db_models.DB.connection_context"),
+ ):
+ mock_settings.docStoreConn = mock_doc_store
+ mock_settings.DOC_ENGINE_INFINITY = False
+ mock_document_model.select.return_value.join.return_value.where.return_value.first.return_value = mock_doc
+
+ ok = DocMetadataService.insert_document_metadata("doc_789", {"k": "v"}, refresh_now=False)
+
+ assert ok is True
+ mock_doc_store.insert.assert_called_once()
+ assert mock_doc_store.insert.call_args.kwargs["refresh"] is False
+ mock_doc_store.refresh_idx.assert_not_called()
diff --git a/test/unit_test/api/db/services/test_file_service_upload_document.py b/test/unit_test/api/db/services/test_file_service_upload_document.py
index 655f1f1476..e0410369e6 100644
--- a/test/unit_test/api/db/services/test_file_service_upload_document.py
+++ b/test/unit_test/api/db/services/test_file_service_upload_document.py
@@ -247,6 +247,48 @@ def test_upload_document_reclaims_document_stranded_by_deleted_kb(monkeypatch):
assert storage.written[("kb-target", "collision.txt")] == b"payload"
+def test_upload_document_stops_remaining_files_when_cancelled(monkeypatch):
+ kb = _target_kb()
+ existing_doc = SimpleNamespace(
+ id="doc-1",
+ kb_id="kb-other",
+ location="old-location.txt",
+ content_hash="old-hash",
+ to_dict=lambda: {"id": "doc-1"},
+ )
+ reads = []
+
+ class _CountedFile(_DummyUploadFile):
+ def read(self):
+ reads.append(self.id)
+ return super().read()
+
+ _stub_folder_lookups(monkeypatch)
+ monkeypatch.setattr(file_service_module.DocumentService, "get_by_id", lambda _doc_id: (True, existing_doc))
+ monkeypatch.setattr(
+ file_service_module.KnowledgebaseService,
+ "get_or_none",
+ classmethod(lambda cls, **_kwargs: SimpleNamespace(id="kb-other")),
+ )
+
+ cancel = types.SimpleNamespace(is_set=lambda: True)
+ err, files = _unwrapped_upload_document()(
+ FileService,
+ kb,
+ [
+ _CountedFile(filename="one.txt", doc_id="doc-1", blob=b"one"),
+ _CountedFile(filename="two.txt", doc_id="doc-2", blob=b"two"),
+ ],
+ "user-1",
+ should_cancel=cancel,
+ )
+
+ assert err == []
+ assert files == []
+ assert reads == []
+
+
# ---------------------------------------------------------------------------
# Helpers shared by TestValidateUrlForCrawl
# ---------------------------------------------------------------------------
diff --git a/test/unit_test/rag/test_sync_data_source.py b/test/unit_test/rag/test_sync_data_source.py
index 021c2c7c2d..e2066d2db4 100644
--- a/test/unit_test/rag/test_sync_data_source.py
+++ b/test/unit_test/rag/test_sync_data_source.py
@@ -13,10 +13,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import asyncio
import importlib
import importlib.util
import os
import sys
+import threading
import types
import warnings
from datetime import datetime, timezone
@@ -184,6 +186,256 @@ async def test_run_task_logic_skips_multiple_empty_sync_batches(monkeypatch):
)._run_task_logic(_make_task())
+async def test_run_task_logic_keeps_event_loop_responsive_during_parse(monkeypatch):
+ _patch_common_dependencies(monkeypatch)
+ started = threading.Event()
+ release = threading.Event()
+
+ def _slow_duplicate(*_args, **_kwargs):
+ started.set()
+ if not release.wait(timeout=5):
+ pytest.fail("release was not signalled while parse was blocked")
+ return [], ["doc-1"]
+
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: (True, object()),
+ )
+ monkeypatch.setattr(sync_data_source.SyncLogsService, "duplicate_and_parse", _slow_duplicate)
+ monkeypatch.setattr(sync_data_source.SyncLogsService, "increase_docs", lambda *_args, **_kwargs: None)
+
+ probe_done = False
+
+ async def _probe():
+ nonlocal probe_done
+ flagged = await asyncio.to_thread(started.wait, 5)
+ assert flagged, "ingest did not start"
+ probe_done = True
+ release.set()
+
+ await asyncio.wait_for(
+ asyncio.gather(
+ _FakeSync(iter(([_make_fake_doc()],)))._run_task_logic(_make_task()),
+ _probe(),
+ ),
+ timeout=10,
+ )
+ assert probe_done
+
+
+def _patch_successful_ingest(monkeypatch):
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: (True, object()),
+ )
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "duplicate_and_parse",
+ lambda *_args, **_kwargs: ([], ["doc-1"]),
+ )
+ monkeypatch.setattr(sync_data_source.SyncLogsService, "increase_docs", lambda *_args, **_kwargs: None)
+
+
+async def _collect_sync_sleeps(monkeypatch, generate_output):
+ _patch_common_dependencies(monkeypatch)
+ _patch_successful_ingest(monkeypatch)
+ monkeypatch.setattr(sync_data_source, "SYNC_BATCH_PAUSE_SECONDS", 0.05)
+ sleep_calls = []
+
+ async def _track_sleep(delay, *_args, **_kwargs):
+ sleep_calls.append(delay)
+
+ monkeypatch.setattr(sync_data_source.asyncio, "sleep", _track_sleep)
+ await _FakeSync(generate_output)._run_task_logic(_make_task())
+ return sleep_calls
+
+
+async def test_run_task_logic_pauses_only_between_nonempty_batches(monkeypatch):
+ sleep_calls = await _collect_sync_sleeps(
+ monkeypatch,
+ iter(([_make_fake_doc("doc-1")], [], [_make_fake_doc("doc-2")])),
+ )
+ assert sleep_calls == [0.05]
+
+
+async def test_run_task_logic_does_not_pause_after_last_batch(monkeypatch):
+ sleep_calls = await _collect_sync_sleeps(monkeypatch, iter(([_make_fake_doc()],)))
+ assert sleep_calls == []
+
+
+def test_ingest_document_batch_skips_writes_when_already_cancelled(monkeypatch):
+ _patch_common_dependencies(monkeypatch)
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: pytest.fail("get_by_id should not run after cancel"),
+ )
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "duplicate_and_parse",
+ lambda *_args, **_kwargs: pytest.fail("duplicate_and_parse should not run after cancel"),
+ )
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "increase_docs",
+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not run after cancel"),
+ )
+ cancel_event = threading.Event()
+ cancel_event.set()
+ err, dids = _FakeSync(iter(()))._ingest_document_batch(
+ _make_task(),
+ [{"id": "doc-1"}],
+ datetime(2026, 1, 1, tzinfo=timezone.utc),
+ cancel_event,
+ )
+ assert err == []
+ assert dids == []
+
+
+def test_ingest_document_batch_skips_progress_when_cancelled_during_parse(monkeypatch):
+ _patch_common_dependencies(monkeypatch)
+ cancel_event = threading.Event()
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: (True, object()),
+ )
+
+ def _duplicate_then_cancel(*_args, **_kwargs):
+ cancel_event.set()
+ return [], ["doc-1"]
+
+ monkeypatch.setattr(sync_data_source.SyncLogsService, "duplicate_and_parse", _duplicate_then_cancel)
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "increase_docs",
+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not advance after cancel"),
+ )
+ err, dids = _FakeSync(iter(()))._ingest_document_batch(
+ _make_task(),
+ [{"id": "doc-1"}],
+ datetime(2026, 1, 1, tzinfo=timezone.utc),
+ cancel_event,
+ )
+ assert err == []
+ assert dids == ["doc-1"]
+
+
+class _PendingCancelTask:
+ def cancelled(self):
+ return False
+
+ def cancelling(self):
+ return 1
+
+
+class _DeferredCancelTask:
+ def __init__(self):
+ self._cancelling = 0
+
+ def cancelled(self):
+ return False
+
+ def cancelling(self):
+ return self._cancelling
+
+
+def test_ingest_document_batch_skips_writes_when_parent_task_is_cancelling(monkeypatch):
+ _patch_common_dependencies(monkeypatch)
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: pytest.fail("get_by_id should not run after cancel"),
+ )
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "duplicate_and_parse",
+ lambda *_args, **_kwargs: pytest.fail("duplicate_and_parse should not run after cancel"),
+ )
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "increase_docs",
+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not run after cancel"),
+ )
+ err, dids = _FakeSync(iter(()))._ingest_document_batch(
+ _make_task(),
+ [{"id": "doc-1"}],
+ datetime(2026, 1, 1, tzinfo=timezone.utc),
+ threading.Event(),
+ _PendingCancelTask(),
+ )
+ assert err == []
+ assert dids == []
+
+
+def test_ingest_document_batch_skips_progress_when_parent_task_starts_cancelling(monkeypatch):
+ _patch_common_dependencies(monkeypatch)
+ parent_task = _DeferredCancelTask()
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: (True, object()),
+ )
+
+ def _duplicate_then_cancel(*_args, **_kwargs):
+ parent_task._cancelling = 1
+ return [], ["doc-1"]
+
+ monkeypatch.setattr(sync_data_source.SyncLogsService, "duplicate_and_parse", _duplicate_then_cancel)
+ monkeypatch.setattr(
+ sync_data_source.SyncLogsService,
+ "increase_docs",
+ lambda *_args, **_kwargs: pytest.fail("increase_docs should not advance after cancel"),
+ )
+ err, dids = _FakeSync(iter(()))._ingest_document_batch(
+ _make_task(),
+ [{"id": "doc-1"}],
+ datetime(2026, 1, 1, tzinfo=timezone.utc),
+ threading.Event(),
+ parent_task,
+ )
+ assert err == []
+ assert dids == ["doc-1"]
+
+
+def test_ingest_document_batch_forwards_cancel_callback(monkeypatch):
+ _patch_common_dependencies(monkeypatch)
+ captured = {}
+ monkeypatch.setattr(
+ sync_data_source.KnowledgebaseService,
+ "get_by_id",
+ lambda *_args, **_kwargs: (True, object()),
+ )
+
+ def _duplicate(*_args, **kwargs):
+ captured["should_cancel"] = kwargs.get("should_cancel")
+ return [], ["doc-1"]
+
+ monkeypatch.setattr(sync_data_source.SyncLogsService, "duplicate_and_parse", _duplicate)
+ monkeypatch.setattr(sync_data_source.SyncLogsService, "increase_docs", lambda *_args, **_kwargs: None)
+ cancel_event = threading.Event()
+ err, dids = _FakeSync(iter(()))._ingest_document_batch(
+ _make_task(),
+ [{"id": "doc-1"}],
+ datetime(2026, 1, 1, tzinfo=timezone.utc),
+ cancel_event,
+ )
+ assert err == []
+ assert dids == ["doc-1"]
+ assert captured["should_cancel"] is not None
+ assert captured["should_cancel"]() is False
+ cancel_event.set()
+ assert captured["should_cancel"]() is True
+
+
@pytest.mark.asyncio
@pytest.mark.p2
async def test_run_prune_task_logic_cleans_up_for_empty_snapshot(monkeypatch):
@@ -294,6 +546,10 @@ class _FakeRDBMSConnector:
self.prepare_sync_state_called = False
self.load_from_cursor_range_called = False
self.persist_sync_state_called = False
+ self.close_connection_called = False
+ self.validate_thread_ident = None
+ self.prepare_thread_ident = None
+ self.close_thread_ident = None
self._pending_sync_cursor_value = None
_FakeRDBMSConnector.instance = self
@@ -301,12 +557,17 @@ class _FakeRDBMSConnector:
self.credentials = credentials
def validate_connector_settings(self):
- return None
+ self.validate_thread_ident = threading.get_ident()
def prepare_sync_state(self, connector_id, config):
self.prepare_sync_state_called = True
+ self.prepare_thread_ident = threading.get_ident()
self.prepare_sync_state_args = (connector_id, config)
+ def _close_connection(self):
+ self.close_connection_called = True
+ self.close_thread_ident = threading.get_ident()
+
def get_saved_sync_cursor_value(self):
return None
@@ -350,12 +611,18 @@ async def test_rdbms_generate_keeps_deleted_file_snapshot_without_timestamp_colu
}
)
+ main_ident = threading.get_ident()
document_generator = await sync._generate(task)
connector = _FakeRDBMSConnector.instance
assert connector is not None
assert connector.load_from_state_called is True
assert connector.load_from_cursor_range_called is False
+ assert connector.prepare_sync_state_called is True
+ assert connector.close_connection_called is True
+ assert connector.validate_thread_ident != main_ident
+ assert connector.prepare_thread_ident == connector.validate_thread_ident
+ assert connector.close_thread_ident == connector.validate_thread_ident
file_list = sync._collect_prune_snapshot(task)
assert connector.retrieve_all_slim_docs_perm_sync_called is True
assert file_list is not None
@@ -933,6 +1200,32 @@ async def test_dropbox_generate_skips_snapshot_for_full_reindex(monkeypatch):
assert connector.poll_source_called is False
+def test_index_batch_size_env_helpers_use_defaults_for_invalid_values(monkeypatch):
+ from common.data_source import config as data_source_config
+
+ monkeypatch.delenv("RAGFLOW_TEST_INDEX_BATCH_SIZE", raising=False)
+ assert data_source_config._env_int("RAGFLOW_TEST_INDEX_BATCH_SIZE", 2) == 2
+ monkeypatch.setenv("RAGFLOW_TEST_INDEX_BATCH_SIZE", "32")
+ assert data_source_config._env_int("RAGFLOW_TEST_INDEX_BATCH_SIZE", 2) == 32
+ monkeypatch.setenv("RAGFLOW_TEST_INDEX_BATCH_SIZE", "0")
+ assert data_source_config._env_int("RAGFLOW_TEST_INDEX_BATCH_SIZE", 2) == 2
+ monkeypatch.setenv("RAGFLOW_TEST_INDEX_BATCH_SIZE", "nope")
+ assert data_source_config._env_int("RAGFLOW_TEST_INDEX_BATCH_SIZE", 2) == 2
+
+ monkeypatch.delenv("RAGFLOW_TEST_SYNC_PAUSE", raising=False)
+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0
+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "0.05")
+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.05
+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "-1")
+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0
+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "inf")
+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0
+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "nan")
+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0
+ monkeypatch.setenv("RAGFLOW_TEST_SYNC_PAUSE", "-inf")
+ assert data_source_config._env_float("RAGFLOW_TEST_SYNC_PAUSE", 0.0) == 0.0
+
+
def test_redact_url_strips_credentials_query_and_fragment():
_redact_url = sync_data_source._redact_url
diff --git a/web/src/pages/user-setting/data-source/constant/index.test.tsx b/web/src/pages/user-setting/data-source/constant/index.test.tsx
index dc0ff8ccf3..dd92b0f954 100644
--- a/web/src/pages/user-setting/data-source/constant/index.test.tsx
+++ b/web/src/pages/user-setting/data-source/constant/index.test.tsx
@@ -120,6 +120,24 @@ describe('Sitemap data source', () => {
});
});
+describe.each([
+ ['MySQL', DataSourceKey.MYSQL],
+ ['PostgreSQL', DataSourceKey.POSTGRESQL],
+])('%s data source', (_label, key) => {
+ it('exposes a bounded batch size for large table syncs', () => {
+ const fields = getDataSourceFieldsWithExtras(translate, key) as Array<{
+ name: string;
+ validation?: { min?: number };
+ }>;
+ const batchSize = fields.find((f) => f.name === 'config.batch_size');
+
+ expect(batchSize?.validation?.min).toBe(1);
+ expect(DataSourceFormDefaultValues[key].config).toMatchObject({
+ batch_size: 2,
+ });
+ });
+});
+
describe('Azure DevOps data source', () => {
it('registers its catalog entry and defaults', () => {
const info = generateDataSourceInfo(translate)[DataSourceKey.AZURE_DEVOPS];
diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsx
index 5aa394cb64..307b321941 100644
--- a/web/src/pages/user-setting/data-source/constant/index.tsx
+++ b/web/src/pages/user-setting/data-source/constant/index.tsx
@@ -1568,6 +1568,19 @@ const generateDataSourceFormFields = (t: TFunction) => ({
placeholder: '.txt',
tooltip: t('setting.mysqlFileExtensionTip'),
},
+ {
+ label: t('setting.dataSourceFieldBatchSize'),
+ name: 'config.batch_size',
+ type: FormFieldType.Number,
+ required: false,
+ placeholder: '2',
+ validation: {
+ min: 1,
+ message: t('setting.dataSourceValidationMinOne', {
+ label: t('setting.dataSourceFieldBatchSize'),
+ }),
+ },
+ },
],
[DataSourceKey.POSTGRESQL]: [
{
@@ -1650,6 +1663,19 @@ const generateDataSourceFormFields = (t: TFunction) => ({
placeholder: '.txt',
tooltip: t('setting.postgresqlFileExtensionTip'),
},
+ {
+ label: t('setting.dataSourceFieldBatchSize'),
+ name: 'config.batch_size',
+ type: FormFieldType.Number,
+ required: false,
+ placeholder: '2',
+ validation: {
+ min: 1,
+ message: t('setting.dataSourceValidationMinOne', {
+ label: t('setting.dataSourceFieldBatchSize'),
+ }),
+ },
+ },
],
[DataSourceKey.BIGQUERY]: [
{
@@ -2508,6 +2534,7 @@ export const DataSourceFormDefaultValues = {
metadata_columns: '',
id_column: '',
timestamp_column: '',
+ batch_size: 2,
credentials: {
username: '',
password: '',
@@ -2526,6 +2553,7 @@ export const DataSourceFormDefaultValues = {
metadata_columns: '',
id_column: '',
timestamp_column: '',
+ batch_size: 2,
credentials: {
username: '',
password: '',