Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: connector sync truncates the source listing at 100 files (#2379)

Rico Furtado15e3f38f439f
diff --git a/src/config/settings.py b/src/config/settings.py
index e66dc1f62..a6026a00e 100644
--- a/src/config/settings.py
+++ b/src/config/settings.py
@@ -317,6 +317,21 @@ def is_dev_azure_blob_enabled() -> bool:
return raw in ("true", "1", "yes", "on")
+def is_dev_aws_s3_enabled() -> bool:
+ """Local dev: enable the AWS S3 connector without IBM_AUTH_ENABLED.
+
+ Allows testing the S3 connector (e.g. against MinIO) in a local environment
+ where IBM auth is not configured. Never enable in production. Requires
+ ``OPENRAG_DEV_AWS_S3=true``.
+
+ S3 was the only bucket connector with no dev bypass, so it could not be
+ exercised end to end without standing up IBM auth — which is why its sync
+ behaviour has historically had less real-world validation than COS's.
+ """
+ raw = os.getenv("OPENRAG_DEV_AWS_S3", "false").strip().lower()
+ return raw in ("true", "1", "yes", "on")
+
+
def is_dev_ibm_cos_enabled() -> bool:
"""Local dev: enable the IBM COS connector without IBM_AUTH_ENABLED.
diff --git a/src/connectors/aws_s3/connector.py b/src/connectors/aws_s3/connector.py
index 29980f93b..fdf1424d0 100644
--- a/src/connectors/aws_s3/connector.py
+++ b/src/connectors/aws_s3/connector.py
@@ -6,7 +6,7 @@ from datetime import UTC, datetime
from posixpath import basename
from typing import Any
-from config.settings import IBM_AUTH_ENABLED
+from config.settings import IBM_AUTH_ENABLED, is_dev_aws_s3_enabled
from connectors.base import (
CONTENT_ETAG_METADATA_KEY,
BaseConnector,
@@ -64,8 +64,10 @@ class S3Connector(BaseConnector):
@classmethod
def is_available(cls, manager, user_id=None) -> bool:
- # Gated by feature flag in OSS; SaaS / enterprise can flip it on.
- return IBM_AUTH_ENABLED
+ # Enterprise/SaaS gate is IBM_AUTH_ENABLED, like the other bucket
+ # connectors (ibm_cos, azure_blob). OPENRAG_DEV_AWS_S3=true bypasses it
+ # for local dev (e.g. against MinIO; never in production).
+ return IBM_AUTH_ENABLED or is_dev_aws_s3_enabled()
@classmethod
def register_routes(cls, app) -> None:
diff --git a/src/connectors/service.py b/src/connectors/service.py
index 11d0d2224..03ba8e987 100644
--- a/src/connectors/service.py
+++ b/src/connectors/service.py
@@ -15,6 +15,18 @@ from .connection_manager import ConnectionManager
logger = get_logger(__name__)
+def _next_page_token(file_list: dict[str, Any]) -> str | None:
+ """Read the continuation token out of a ``list_files`` result.
+
+ Every connector in this repo returns ``next_page_token``; SharePoint is the
+ one that populates it with a real value (a Graph ``$skiptoken``). This used
+ to read ``nextPageToken`` only — a key no connector emits — so the token was
+ always None and paging stopped after the first call. Both spellings are
+ accepted now so a connector written against either convention still pages.
+ """
+ return file_list.get("next_page_token") or file_list.get("nextPageToken")
+
+
class ConnectorService:
"""Service to manage document connectors and process files"""
@@ -256,17 +268,24 @@ class ConnectorService:
files_to_process: list[dict[str, Any]] = []
page_token = None
- # Calculate page size to minimize API calls
- page_size = min(max_files or 100, 1000) if max_files else 100
-
- while True:
- # List files from connector with limit
- logger.debug("Calling list_files", page_size=page_size, page_token=page_token)
- file_list = await connector.list_files(page_token, max_files=page_size)
+ # A zero cap means "sync nothing" and has to short-circuit before the
+ # first list_files call: every cap below is spelled `if max_files and …`,
+ # so 0 would fall through as "no cap" and enumerate the whole source.
+ # None (no cap) and positive caps take the loop as before.
+ while max_files != 0:
+ # Pass max_files straight through — None means "no cap". Asking for a
+ # synthetic page size instead silently truncated every sync: the
+ # connectors that paginate internally (all three bucket ones, and
+ # Google Drive) honour the cap and then report next_page_token=None,
+ # so there was no token to continue with and everything past the
+ # first page was simply dropped.
+ logger.debug("Calling list_files", max_files=max_files, page_token=page_token)
+ file_list = await connector.list_files(page_token, max_files=max_files)
logger.debug("Got files from connector", file_count=len(file_list.get("files", [])))
files = file_list["files"]
+ page_token = _next_page_token(file_list)
- if not files:
+ if not files and not page_token:
break
for file_info in files:
@@ -284,13 +303,9 @@ class ConnectorService:
files_to_process.append(file_info)
# Stop if we have enough files or no more pages
- if (max_files and len(files_to_process) >= max_files) or not file_list.get(
- "nextPageToken"
- ):
+ if (max_files and len(files_to_process) >= max_files) or not page_token:
break
- page_token = file_list.get("nextPageToken")
-
# Get user information
user = self.session_manager.get_user(user_id) if self.session_manager else None
owner_name = user.name if user else None
diff --git a/tests/unit/connectors/test_aws_s3_connector.py b/tests/unit/connectors/test_aws_s3_connector.py
new file mode 100644
index 000000000..5ef6620f2
--- /dev/null
+++ b/tests/unit/connectors/test_aws_s3_connector.py
@@ -0,0 +1,176 @@
+"""Unit tests for the AWS S3 connector.
+
+Mirrors the IBM COS and Azure Blob coverage
+(tests/unit/connectors/test_ibm_cos_connector.py,
+tests/unit/connectors/test_azure_blob_connector.py) for the `is_available`
+gating, and pins the change-detection signals S3 reports.
+
+S3 is the bucket connector with the least real-world validation — until
+OPENRAG_DEV_AWS_S3 it could not be enabled at all without IBM auth, so its sync
+behaviour was only ever exercised through machinery shared with COS. These
+assert the S3 path directly rather than inferring it from its siblings.
+"""
+
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+from connectors.aws_s3 import connector as aws_s3_connector # noqa: E402
+from connectors.aws_s3.connector import S3Connector # noqa: E402
+from connectors.base import CONTENT_ETAG_METADATA_KEY # noqa: E402
+
+MODIFIED = datetime(2026, 1, 1, tzinfo=UTC)
+
+
+# ---------------------------------------------------------------------------
+# is_available gating
+# ---------------------------------------------------------------------------
+
+
+def test_is_available_gated_on_ibm_auth_enabled(monkeypatch):
+ monkeypatch.setattr(aws_s3_connector, "IBM_AUTH_ENABLED", True)
+ monkeypatch.setattr(aws_s3_connector, "is_dev_aws_s3_enabled", lambda: False)
+ assert S3Connector.is_available(MagicMock()) is True
+ monkeypatch.setattr(aws_s3_connector, "IBM_AUTH_ENABLED", False)
+ assert S3Connector.is_available(MagicMock()) is False
+
+
+def test_is_available_dev_flag_bypasses_ibm_auth(monkeypatch):
+ """Without this, S3 could not be exercised locally at all (e.g. MinIO)."""
+ monkeypatch.setattr(aws_s3_connector, "IBM_AUTH_ENABLED", False)
+ monkeypatch.setattr(aws_s3_connector, "is_dev_aws_s3_enabled", lambda: True)
+ assert S3Connector.is_available(MagicMock()) is True
+
+
+def test_is_dev_aws_s3_enabled_defaults_false_when_unset(monkeypatch):
+ from config.settings import is_dev_aws_s3_enabled
+
+ monkeypatch.delenv("OPENRAG_DEV_AWS_S3", raising=False)
+ assert is_dev_aws_s3_enabled() is False
+ monkeypatch.setenv("OPENRAG_DEV_AWS_S3", "true")
+ assert is_dev_aws_s3_enabled() is True
+
+
+# ---------------------------------------------------------------------------
+# Change-detection signals
+# ---------------------------------------------------------------------------
+
+
+def _object(key: str, etag: str):
+ return SimpleNamespace(key=key, size=10, last_modified=MODIFIED, e_tag=etag)
+
+
+def _connector_with(objects):
+ connector = S3Connector({"bucket_names": ["b1"]})
+ bucket = MagicMock()
+ bucket.objects.all.return_value = objects
+ resource = MagicMock()
+ resource.Bucket.return_value = bucket
+ connector._resource = resource
+ return connector
+
+
+async def test_listing_reports_the_signals_sync_compares():
+ """Sync needs both an id and an etag off the listing to spot an overwrite."""
+ connector = _connector_with([_object("reports/q3.pdf", '"abc123"')])
+
+ listed = (await connector.list_files())["files"][0]
+
+ assert listed["id"] == "b1::reports/q3.pdf"
+ assert listed["modified_time"] == MODIFIED.isoformat()
+ # Normalized: S3 quotes its tags, the stored copy is unquoted.
+ assert listed["etag"] == "abc123"
+
+
+async def test_listing_skips_directory_placeholders():
+ connector = _connector_with(
+ [_object("folder/", '"dir"'), _object("folder/real.pdf", '"abc123"')]
+ )
+
+ files = (await connector.list_files())["files"]
+
+ assert [f["id"] for f in files] == ["b1::folder/real.pdf"]
+
+
+async def test_download_persists_the_etag_for_the_next_sync():
+ connector = _connector_with([])
+ connector._resource.Object.return_value.get.return_value = {
+ "Body": SimpleNamespace(read=lambda: b"%PDF-1.4"),
+ "ContentType": "application/pdf",
+ "ContentLength": 8,
+ "LastModified": MODIFIED,
+ "ETag": '"abc123"',
+ }
+ connector._client = MagicMock()
+ connector._client.get_object_acl.side_effect = RuntimeError("acls disabled")
+
+ document = await connector.get_file_content("b1::reports/q3.pdf")
+
+ assert document.metadata[CONTENT_ETAG_METADATA_KEY] == "abc123"
+ assert document.modified_time == MODIFIED
+
+
+async def test_overwriting_an_object_classifies_as_changed():
+ """The end-to-end question this connector kept getting asked.
+
+ Drives the real classifier off the real listing, for each combination of
+ stored signals a deployment can actually have.
+ """
+ from api.connectors import SyncedFileState, classify_remote_file_change
+
+ fid = "b1::reports/q3.pdf"
+ stored_ms = MODIFIED.timestamp() * 1000
+ later = datetime(2026, 6, 1, tzinfo=UTC)
+
+ overwritten = _connector_with([_object("reports/q3.pdf", '"v2"')])
+ overwritten._resource.Bucket.return_value.objects.all.return_value = [
+ SimpleNamespace(key="reports/q3.pdf", size=11, last_modified=later, e_tag='"v2"')
+ ]
+ remote = (await overwritten.list_files())["files"][0]
+
+ for label, stored in (
+ ("etag + timestamp", SyncedFileState(stored_ms, "v1")),
+ ("timestamp only (ingested before etags)", SyncedFileState(stored_ms, None)),
+ ("etag only", SyncedFileState(None, "v1")),
+ ):
+ assert (
+ classify_remote_file_change(
+ fid, remote["modified_time"], True, {fid: stored}, remote["etag"]
+ )
+ == "changed"
+ ), label
+
+
+async def test_identical_bytes_are_not_re_ingested_when_the_timestamp_moves():
+ """A copy or re-upload bumps LastModified without changing the content."""
+ from api.connectors import SyncedFileState, classify_remote_file_change
+
+ fid = "b1::reports/q3.pdf"
+ later = datetime(2026, 6, 1, tzinfo=UTC)
+ connector = _connector_with([])
+ connector._resource.Bucket.return_value.objects.all.return_value = [
+ SimpleNamespace(key="reports/q3.pdf", size=10, last_modified=later, e_tag='"v1"')
+ ]
+ remote = (await connector.list_files())["files"][0]
+
+ stored = SyncedFileState(MODIFIED.timestamp() * 1000, "v1")
+ assert (
+ classify_remote_file_change(
+ fid, remote["modified_time"], True, {fid: stored}, remote["etag"]
+ )
+ == "unchanged"
+ )
diff --git a/tests/unit/connectors/test_sync_connector_files_paging.py b/tests/unit/connectors/test_sync_connector_files_paging.py
new file mode 100644
index 000000000..95f4f3d15
--- /dev/null
+++ b/tests/unit/connectors/test_sync_connector_files_paging.py
@@ -0,0 +1,216 @@
+"""sync_connector_files must enumerate the whole source, not just the first page.
+
+Two bugs made it stop early on every connector:
+
+ - it asked for a synthetic page size (100) whenever the caller set no
+ max_files, and
+ - it looked for the continuation token under ``nextPageToken``, a key no
+ connector in this repo emits — they all return ``next_page_token``.
+
+Together those silently capped a sync at 100 files: the connectors that
+paginate internally (all three bucket ones, and Google Drive) honoured the cap
+and then reported no token, so there was nothing to continue with. SharePoint,
+which returns a real Graph ``$skiptoken``, had its token dropped on the floor.
+
+The user-visible symptom is the same as an overwrite not propagating: a file
+past the first page is never re-ingested, so its stale copy stays in Knowledge.
+"""
+
+import sys
+from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+ROOT = Path(__file__).resolve().parent.parent.parent.parent
+SRC = ROOT / "src"
+if str(SRC) not in sys.path:
+ sys.path.insert(0, str(SRC))
+
+
+def _file(i: int) -> dict:
+ return {"id": f"b1::k{i}.pdf", "name": f"k{i}.pdf", "mimeType": "application/pdf"}
+
+
+def _internally_paginating_connector(total: int):
+ """A bucket-style connector: caps to max_files itself, never returns a token.
+
+ Mirrors S3/COS/Azure (and Google Drive), where the SDK walks the pages and
+ the connector hands back one flat list.
+ """
+ connector = MagicMock()
+ connector.CONNECTOR_TYPE = "aws_s3"
+ connector.is_authenticated = True
+ connector.cfg = None
+ seen: list[int | None] = []
+
+ async def list_files(page_token=None, max_files=None, **kwargs):
+ seen.append(max_files)
+ files = [_file(i) for i in range(total)]
+ if isinstance(max_files, int) and max_files > 0:
+ files = files[:max_files]
+ return {"files": files, "next_page_token": None}
+
+ connector.list_files = AsyncMock(side_effect=list_files)
+ return connector, seen
+
+
+def _token_paginating_connector(pages: list[list[dict]]):
+ """A SharePoint-style connector returning a real next_page_token per page."""
+ connector = MagicMock()
+ connector.CONNECTOR_TYPE = "sharepoint"
+ connector.is_authenticated = True
+ connector.cfg = None
+ requested_tokens: list[str | None] = []
+
+ async def list_files(page_token=None, max_files=None, **kwargs):
+ requested_tokens.append(page_token)
+ index = 0 if page_token is None else int(page_token)
+ is_last = index == len(pages) - 1
+ return {
+ "files": pages[index],
+ "next_page_token": None if is_last else str(index + 1),
+ }
+
+ connector.list_files = AsyncMock(side_effect=list_files)
+ return connector, requested_tokens
+
+
+def _service(connector):
+ from connectors.service import ConnectorService
+
+ service = ConnectorService.__new__(ConnectorService)
+ service.session_manager = MagicMock()
+ service.session_manager.get_user = MagicMock(
+ return_value=MagicMock(name="u", email="[email protected]")
+ )
+ service.models_service = MagicMock()
+ service.langflow_service = None
+ service.task_service = MagicMock()
+ service.task_service.document_service = MagicMock()
+ service.task_service.create_custom_task = AsyncMock(return_value="task-1")
+ service.get_connector = AsyncMock(return_value=connector)
+ service._get_effective_sync_jwt = AsyncMock(return_value="jwt")
+ return service
+
+
+def _processed_ids(service) -> list[str]:
+ """The file ids handed to the task processor."""
+ return service.task_service.create_custom_task.await_args.args[1]
+
+
+async def test_uncapped_sync_takes_every_file_not_just_the_first_hundred():
+ connector, requested_max_files = _internally_paginating_connector(total=150)
+ service = _service(connector)
+
+ await service.sync_connector_files("conn-1", "alice")
+
+ assert len(_processed_ids(service)) == 150
+ # No cap was requested, so the connector was free to return everything.
+ assert requested_max_files == [None]
+
+
+async def test_explicit_max_files_still_caps():
+ connector, requested_max_files = _internally_paginating_connector(total=150)
+ service = _service(connector)
+
+ await service.sync_connector_files("conn-1", "alice", max_files=20)
+
+ assert len(_processed_ids(service)) == 20
+ assert requested_max_files == [20]
+
+
+async def test_max_files_zero_lists_nothing():
+ """A zero cap means "sync nothing", not "no cap".
+
+ Every cap check is spelled `if max_files and ...`, so 0 reads as falsy there.
+ Without the short-circuit it would list — and ingest — the entire source.
+ """
+ connector, requested_max_files = _internally_paginating_connector(total=150)
+ service = _service(connector)
+
+ await service.sync_connector_files("conn-1", "alice", max_files=0)
+
+ assert _processed_ids(service) == []
+ assert requested_max_files == []
+ connector.list_files.assert_not_awaited()
+
+
+async def test_token_paginated_connector_is_followed_across_pages():
+ """SharePoint's $skiptoken was read under the wrong key and dropped."""
+ pages = [[_file(0), _file(1)], [_file(2), _file(3)], [_file(4)]]
+ connector, requested_tokens = _token_paginating_connector(pages)
+ service = _service(connector)
+
+ await service.sync_connector_files("conn-1", "alice")
+
+ assert len(_processed_ids(service)) == 5
+ assert requested_tokens == [None, "1", "2"]
+
+
+async def test_token_paginated_connector_continues_after_empty_page():
+ pages = [[], [_file(0), _file(1)]]
+ connector, requested_tokens = _token_paginating_connector(pages)
+ service = _service(connector)
+
+ await service.sync_connector_files("conn-1", "alice")
+
+ assert _processed_ids(service) == ["b1::k0.pdf", "b1::k1.pdf"]
+ assert requested_tokens == [None, "1"]
+
+
+async def test_token_paginated_connector_stops_at_aggregate_max_files():
+ pages = [[_file(i) for i in range(start, start + 15)] for start in (0, 15, 30)]
+ connector, requested_tokens = _token_paginating_connector(pages)
+ service = _service(connector)
+
+ await service.sync_connector_files("conn-1", "alice", max_files=20)
+
+ assert _processed_ids(service) == [f"b1::k{i}.pdf" for i in range(20)]
+ assert requested_tokens == [None, "1"]
+
+
+async def test_camel_case_token_still_paginates():
+ """Tolerated alias: a connector written against the old key still works."""
+ connector = MagicMock()
+ connector.CONNECTOR_TYPE = "sharepoint"
+ connector.is_authenticated = True
+ connector.cfg = None
+ calls: list[str | None] = []
+
+ async def list_files(page_token=None, max_files=None, **kwargs):
+ calls.append(page_token)
+ if page_token is None:
+ return {"files": [_file(0)], "nextPageToken": "p2"}
+ return {"files": [_file(1)], "nextPageToken": None}
+
+ connector.list_files = AsyncMock(side_effect=list_files)
+ service = _service(connector)
+
+ await service.sync_connector_files("conn-1", "alice")
+
+ assert len(_processed_ids(service)) == 2
+ assert calls == [None, "p2"]
+
+
+async def test_filename_filter_matches_beyond_the_first_page():
+ """The filter is why truncation is a correctness bug, not just a slow sync.
+
+ _sync_existing_connector_files' fallback branch passes the indexed filenames
+ here. A file past the cap was never listed, so it never matched the filter
+ and never got re-ingested — its stale copy stayed in Knowledge.
+ """
+ connector, _ = _internally_paginating_connector(total=150)
+ service = _service(connector)
+
+ await service.sync_connector_files("conn-1", "alice", filename_filter={"k0.pdf", "k130.pdf"})
+
+ assert sorted(_processed_ids(service)) == ["b1::k0.pdf", "b1::k130.pdf"]