OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: connector sync truncates the source listing at 100 files (#2379)
Rico Furtado15e3f38f439f
diff --git a/src/config/settings.py b/src/config/settings.pyindex e66dc1f62..a6026a00e 100644--- a/src/config/settings.py+++ b/src/config/settings.py@@ -317,6 +317,21 @@ def is_dev_azure_blob_enabled() -> bool:return raw in ("true", "1", "yes", "on")+def is_dev_aws_s3_enabled() -> bool:+ """Local dev: enable the AWS S3 connector without IBM_AUTH_ENABLED.++ Allows testing the S3 connector (e.g. against MinIO) in a local environment+ where IBM auth is not configured. Never enable in production. Requires+ ``OPENRAG_DEV_AWS_S3=true``.++ S3 was the only bucket connector with no dev bypass, so it could not be+ exercised end to end without standing up IBM auth — which is why its sync+ behaviour has historically had less real-world validation than COS's.+ """+ raw = os.getenv("OPENRAG_DEV_AWS_S3", "false").strip().lower()+ return raw in ("true", "1", "yes", "on")++def is_dev_ibm_cos_enabled() -> bool:"""Local dev: enable the IBM COS connector without IBM_AUTH_ENABLED.diff --git a/src/connectors/aws_s3/connector.py b/src/connectors/aws_s3/connector.pyindex 29980f93b..fdf1424d0 100644--- a/src/connectors/aws_s3/connector.py+++ b/src/connectors/aws_s3/connector.py@@ -6,7 +6,7 @@ from datetime import UTC, datetimefrom posixpath import basenamefrom typing import Any-from config.settings import IBM_AUTH_ENABLED+from config.settings import IBM_AUTH_ENABLED, is_dev_aws_s3_enabledfrom connectors.base import (CONTENT_ETAG_METADATA_KEY,BaseConnector,@@ -64,8 +64,10 @@ class S3Connector(BaseConnector):@classmethoddef is_available(cls, manager, user_id=None) -> bool:- # Gated by feature flag in OSS; SaaS / enterprise can flip it on.- return IBM_AUTH_ENABLED+ # Enterprise/SaaS gate is IBM_AUTH_ENABLED, like the other bucket+ # connectors (ibm_cos, azure_blob). OPENRAG_DEV_AWS_S3=true bypasses it+ # for local dev (e.g. against MinIO; never in production).+ return IBM_AUTH_ENABLED or is_dev_aws_s3_enabled()@classmethoddef register_routes(cls, app) -> None:diff --git a/src/connectors/service.py b/src/connectors/service.pyindex 11d0d2224..03ba8e987 100644--- a/src/connectors/service.py+++ b/src/connectors/service.py@@ -15,6 +15,18 @@ from .connection_manager import ConnectionManagerlogger = get_logger(__name__)+def _next_page_token(file_list: dict[str, Any]) -> str | None:+ """Read the continuation token out of a ``list_files`` result.++ Every connector in this repo returns ``next_page_token``; SharePoint is the+ one that populates it with a real value (a Graph ``$skiptoken``). This used+ to read ``nextPageToken`` only — a key no connector emits — so the token was+ always None and paging stopped after the first call. Both spellings are+ accepted now so a connector written against either convention still pages.+ """+ return file_list.get("next_page_token") or file_list.get("nextPageToken")++class ConnectorService:"""Service to manage document connectors and process files"""@@ -256,17 +268,24 @@ class ConnectorService:files_to_process: list[dict[str, Any]] = []page_token = None- # Calculate page size to minimize API calls- page_size = min(max_files or 100, 1000) if max_files else 100-- while True:- # List files from connector with limit- logger.debug("Calling list_files", page_size=page_size, page_token=page_token)- file_list = await connector.list_files(page_token, max_files=page_size)+ # A zero cap means "sync nothing" and has to short-circuit before the+ # first list_files call: every cap below is spelled `if max_files and …`,+ # so 0 would fall through as "no cap" and enumerate the whole source.+ # None (no cap) and positive caps take the loop as before.+ while max_files != 0:+ # Pass max_files straight through — None means "no cap". Asking for a+ # synthetic page size instead silently truncated every sync: the+ # connectors that paginate internally (all three bucket ones, and+ # Google Drive) honour the cap and then report next_page_token=None,+ # so there was no token to continue with and everything past the+ # first page was simply dropped.+ logger.debug("Calling list_files", max_files=max_files, page_token=page_token)+ file_list = await connector.list_files(page_token, max_files=max_files)logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))files = file_list["files"]+ page_token = _next_page_token(file_list)- if not files:+ if not files and not page_token:breakfor file_info in files:@@ -284,13 +303,9 @@ class ConnectorService:files_to_process.append(file_info)# Stop if we have enough files or no more pages- if (max_files and len(files_to_process) >= max_files) or not file_list.get(- "nextPageToken"- ):+ if (max_files and len(files_to_process) >= max_files) or not page_token:break- page_token = file_list.get("nextPageToken")-# Get user informationuser = self.session_manager.get_user(user_id) if self.session_manager else Noneowner_name = user.name if user else Nonediff --git a/tests/unit/connectors/test_aws_s3_connector.py b/tests/unit/connectors/test_aws_s3_connector.pynew file mode 100644index 000000000..5ef6620f2--- /dev/null+++ b/tests/unit/connectors/test_aws_s3_connector.py@@ -0,0 +1,176 @@+"""Unit tests for the AWS S3 connector.++Mirrors the IBM COS and Azure Blob coverage+(tests/unit/connectors/test_ibm_cos_connector.py,+tests/unit/connectors/test_azure_blob_connector.py) for the `is_available`+gating, and pins the change-detection signals S3 reports.++S3 is the bucket connector with the least real-world validation — until+OPENRAG_DEV_AWS_S3 it could not be enabled at all without IBM auth, so its sync+behaviour was only ever exercised through machinery shared with COS. These+assert the S3 path directly rather than inferring it from its siblings.+"""++import sys+from datetime import UTC, datetime+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))++from connectors.aws_s3 import connector as aws_s3_connector # noqa: E402+from connectors.aws_s3.connector import S3Connector # noqa: E402+from connectors.base import CONTENT_ETAG_METADATA_KEY # noqa: E402++MODIFIED = datetime(2026, 1, 1, tzinfo=UTC)+++# ---------------------------------------------------------------------------+# is_available gating+# ---------------------------------------------------------------------------+++def test_is_available_gated_on_ibm_auth_enabled(monkeypatch):+ monkeypatch.setattr(aws_s3_connector, "IBM_AUTH_ENABLED", True)+ monkeypatch.setattr(aws_s3_connector, "is_dev_aws_s3_enabled", lambda: False)+ assert S3Connector.is_available(MagicMock()) is True+ monkeypatch.setattr(aws_s3_connector, "IBM_AUTH_ENABLED", False)+ assert S3Connector.is_available(MagicMock()) is False+++def test_is_available_dev_flag_bypasses_ibm_auth(monkeypatch):+ """Without this, S3 could not be exercised locally at all (e.g. MinIO)."""+ monkeypatch.setattr(aws_s3_connector, "IBM_AUTH_ENABLED", False)+ monkeypatch.setattr(aws_s3_connector, "is_dev_aws_s3_enabled", lambda: True)+ assert S3Connector.is_available(MagicMock()) is True+++def test_is_dev_aws_s3_enabled_defaults_false_when_unset(monkeypatch):+ from config.settings import is_dev_aws_s3_enabled++ monkeypatch.delenv("OPENRAG_DEV_AWS_S3", raising=False)+ assert is_dev_aws_s3_enabled() is False+ monkeypatch.setenv("OPENRAG_DEV_AWS_S3", "true")+ assert is_dev_aws_s3_enabled() is True+++# ---------------------------------------------------------------------------+# Change-detection signals+# ---------------------------------------------------------------------------+++def _object(key: str, etag: str):+ return SimpleNamespace(key=key, size=10, last_modified=MODIFIED, e_tag=etag)+++def _connector_with(objects):+ connector = S3Connector({"bucket_names": ["b1"]})+ bucket = MagicMock()+ bucket.objects.all.return_value = objects+ resource = MagicMock()+ resource.Bucket.return_value = bucket+ connector._resource = resource+ return connector+++async def test_listing_reports_the_signals_sync_compares():+ """Sync needs both an id and an etag off the listing to spot an overwrite."""+ connector = _connector_with([_object("reports/q3.pdf", '"abc123"')])++ listed = (await connector.list_files())["files"][0]++ assert listed["id"] == "b1::reports/q3.pdf"+ assert listed["modified_time"] == MODIFIED.isoformat()+ # Normalized: S3 quotes its tags, the stored copy is unquoted.+ assert listed["etag"] == "abc123"+++async def test_listing_skips_directory_placeholders():+ connector = _connector_with(+ [_object("folder/", '"dir"'), _object("folder/real.pdf", '"abc123"')]+ )++ files = (await connector.list_files())["files"]++ assert [f["id"] for f in files] == ["b1::folder/real.pdf"]+++async def test_download_persists_the_etag_for_the_next_sync():+ connector = _connector_with([])+ connector._resource.Object.return_value.get.return_value = {+ "Body": SimpleNamespace(read=lambda: b"%PDF-1.4"),+ "ContentType": "application/pdf",+ "ContentLength": 8,+ "LastModified": MODIFIED,+ "ETag": '"abc123"',+ }+ connector._client = MagicMock()+ connector._client.get_object_acl.side_effect = RuntimeError("acls disabled")++ document = await connector.get_file_content("b1::reports/q3.pdf")++ assert document.metadata[CONTENT_ETAG_METADATA_KEY] == "abc123"+ assert document.modified_time == MODIFIED+++async def test_overwriting_an_object_classifies_as_changed():+ """The end-to-end question this connector kept getting asked.++ Drives the real classifier off the real listing, for each combination of+ stored signals a deployment can actually have.+ """+ from api.connectors import SyncedFileState, classify_remote_file_change++ fid = "b1::reports/q3.pdf"+ stored_ms = MODIFIED.timestamp() * 1000+ later = datetime(2026, 6, 1, tzinfo=UTC)++ overwritten = _connector_with([_object("reports/q3.pdf", '"v2"')])+ overwritten._resource.Bucket.return_value.objects.all.return_value = [+ SimpleNamespace(key="reports/q3.pdf", size=11, last_modified=later, e_tag='"v2"')+ ]+ remote = (await overwritten.list_files())["files"][0]++ for label, stored in (+ ("etag + timestamp", SyncedFileState(stored_ms, "v1")),+ ("timestamp only (ingested before etags)", SyncedFileState(stored_ms, None)),+ ("etag only", SyncedFileState(None, "v1")),+ ):+ assert (+ classify_remote_file_change(+ fid, remote["modified_time"], True, {fid: stored}, remote["etag"]+ )+ == "changed"+ ), label+++async def test_identical_bytes_are_not_re_ingested_when_the_timestamp_moves():+ """A copy or re-upload bumps LastModified without changing the content."""+ from api.connectors import SyncedFileState, classify_remote_file_change++ fid = "b1::reports/q3.pdf"+ later = datetime(2026, 6, 1, tzinfo=UTC)+ connector = _connector_with([])+ connector._resource.Bucket.return_value.objects.all.return_value = [+ SimpleNamespace(key="reports/q3.pdf", size=10, last_modified=later, e_tag='"v1"')+ ]+ remote = (await connector.list_files())["files"][0]++ stored = SyncedFileState(MODIFIED.timestamp() * 1000, "v1")+ assert (+ classify_remote_file_change(+ fid, remote["modified_time"], True, {fid: stored}, remote["etag"]+ )+ == "unchanged"+ )diff --git a/tests/unit/connectors/test_sync_connector_files_paging.py b/tests/unit/connectors/test_sync_connector_files_paging.pynew file mode 100644index 000000000..95f4f3d15--- /dev/null+++ b/tests/unit/connectors/test_sync_connector_files_paging.py@@ -0,0 +1,216 @@+"""sync_connector_files must enumerate the whole source, not just the first page.++Two bugs made it stop early on every connector:++ - it asked for a synthetic page size (100) whenever the caller set no+ max_files, and+ - it looked for the continuation token under ``nextPageToken``, a key no+ connector in this repo emits — they all return ``next_page_token``.++Together those silently capped a sync at 100 files: the connectors that+paginate internally (all three bucket ones, and Google Drive) honoured the cap+and then reported no token, so there was nothing to continue with. SharePoint,+which returns a real Graph ``$skiptoken``, had its token dropped on the floor.++The user-visible symptom is the same as an overwrite not propagating: a file+past the first page is never re-ingested, so its stale copy stays in Knowledge.+"""++import sys+from pathlib import Path+from unittest.mock import AsyncMock, MagicMock++import pytest++ROOT = Path(__file__).resolve().parent.parent.parent.parent+SRC = ROOT / "src"+if str(SRC) not in sys.path:+ sys.path.insert(0, str(SRC))+++def _file(i: int) -> dict:+ return {"id": f"b1::k{i}.pdf", "name": f"k{i}.pdf", "mimeType": "application/pdf"}+++def _internally_paginating_connector(total: int):+ """A bucket-style connector: caps to max_files itself, never returns a token.++ Mirrors S3/COS/Azure (and Google Drive), where the SDK walks the pages and+ the connector hands back one flat list.+ """+ connector = MagicMock()+ connector.CONNECTOR_TYPE = "aws_s3"+ connector.is_authenticated = True+ connector.cfg = None+ seen: list[int | None] = []++ async def list_files(page_token=None, max_files=None, **kwargs):+ seen.append(max_files)+ files = [_file(i) for i in range(total)]+ if isinstance(max_files, int) and max_files > 0:+ files = files[:max_files]+ return {"files": files, "next_page_token": None}++ connector.list_files = AsyncMock(side_effect=list_files)+ return connector, seen+++def _token_paginating_connector(pages: list[list[dict]]):+ """A SharePoint-style connector returning a real next_page_token per page."""+ connector = MagicMock()+ connector.CONNECTOR_TYPE = "sharepoint"+ connector.is_authenticated = True+ connector.cfg = None+ requested_tokens: list[str | None] = []++ async def list_files(page_token=None, max_files=None, **kwargs):+ requested_tokens.append(page_token)+ index = 0 if page_token is None else int(page_token)+ is_last = index == len(pages) - 1+ return {+ "files": pages[index],+ "next_page_token": None if is_last else str(index + 1),+ }++ connector.list_files = AsyncMock(side_effect=list_files)+ return connector, requested_tokens+++def _service(connector):+ from connectors.service import ConnectorService++ service = ConnectorService.__new__(ConnectorService)+ service.session_manager = MagicMock()+ service.session_manager.get_user = MagicMock(+ return_value=MagicMock(name="u", email="[email protected]")+ )+ service.models_service = MagicMock()+ service.langflow_service = None+ service.task_service = MagicMock()+ service.task_service.document_service = MagicMock()+ service.task_service.create_custom_task = AsyncMock(return_value="task-1")+ service.get_connector = AsyncMock(return_value=connector)+ service._get_effective_sync_jwt = AsyncMock(return_value="jwt")+ return service+++def _processed_ids(service) -> list[str]:+ """The file ids handed to the task processor."""+ return service.task_service.create_custom_task.await_args.args[1]+++async def test_uncapped_sync_takes_every_file_not_just_the_first_hundred():+ connector, requested_max_files = _internally_paginating_connector(total=150)+ service = _service(connector)++ await service.sync_connector_files("conn-1", "alice")++ assert len(_processed_ids(service)) == 150+ # No cap was requested, so the connector was free to return everything.+ assert requested_max_files == [None]+++async def test_explicit_max_files_still_caps():+ connector, requested_max_files = _internally_paginating_connector(total=150)+ service = _service(connector)++ await service.sync_connector_files("conn-1", "alice", max_files=20)++ assert len(_processed_ids(service)) == 20+ assert requested_max_files == [20]+++async def test_max_files_zero_lists_nothing():+ """A zero cap means "sync nothing", not "no cap".++ Every cap check is spelled `if max_files and ...`, so 0 reads as falsy there.+ Without the short-circuit it would list — and ingest — the entire source.+ """+ connector, requested_max_files = _internally_paginating_connector(total=150)+ service = _service(connector)++ await service.sync_connector_files("conn-1", "alice", max_files=0)++ assert _processed_ids(service) == []+ assert requested_max_files == []+ connector.list_files.assert_not_awaited()+++async def test_token_paginated_connector_is_followed_across_pages():+ """SharePoint's $skiptoken was read under the wrong key and dropped."""+ pages = [[_file(0), _file(1)], [_file(2), _file(3)], [_file(4)]]+ connector, requested_tokens = _token_paginating_connector(pages)+ service = _service(connector)++ await service.sync_connector_files("conn-1", "alice")++ assert len(_processed_ids(service)) == 5+ assert requested_tokens == [None, "1", "2"]+++async def test_token_paginated_connector_continues_after_empty_page():+ pages = [[], [_file(0), _file(1)]]+ connector, requested_tokens = _token_paginating_connector(pages)+ service = _service(connector)++ await service.sync_connector_files("conn-1", "alice")++ assert _processed_ids(service) == ["b1::k0.pdf", "b1::k1.pdf"]+ assert requested_tokens == [None, "1"]+++async def test_token_paginated_connector_stops_at_aggregate_max_files():+ pages = [[_file(i) for i in range(start, start + 15)] for start in (0, 15, 30)]+ connector, requested_tokens = _token_paginating_connector(pages)+ service = _service(connector)++ await service.sync_connector_files("conn-1", "alice", max_files=20)++ assert _processed_ids(service) == [f"b1::k{i}.pdf" for i in range(20)]+ assert requested_tokens == [None, "1"]+++async def test_camel_case_token_still_paginates():+ """Tolerated alias: a connector written against the old key still works."""+ connector = MagicMock()+ connector.CONNECTOR_TYPE = "sharepoint"+ connector.is_authenticated = True+ connector.cfg = None+ calls: list[str | None] = []++ async def list_files(page_token=None, max_files=None, **kwargs):+ calls.append(page_token)+ if page_token is None:+ return {"files": [_file(0)], "nextPageToken": "p2"}+ return {"files": [_file(1)], "nextPageToken": None}++ connector.list_files = AsyncMock(side_effect=list_files)+ service = _service(connector)++ await service.sync_connector_files("conn-1", "alice")++ assert len(_processed_ids(service)) == 2+ assert calls == [None, "p2"]+++async def test_filename_filter_matches_beyond_the_first_page():+ """The filter is why truncation is a correctness bug, not just a slow sync.++ _sync_existing_connector_files' fallback branch passes the indexed filenames+ here. A file past the cap was never listed, so it never matched the filter+ and never got re-ingested — its stale copy stayed in Knowledge.+ """+ connector, _ = _internally_paginating_connector(total=150)+ service = _service(connector)++ await service.sync_connector_files("conn-1", "alice", filename_filter={"k0.pdf", "k130.pdf"})++ assert sorted(_processed_ids(service)) == ["b1::k0.pdf", "b1::k130.pdf"]