Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(common): propagate WebDAV listing errors from _list_files_recursive to avoid prune data loss (#18694)

Harsh Kashyap24abfcd43d55
diff --git a/common/data_source/webdav_connector.py b/common/data_source/webdav_connector.py
index c205a6b3aa..5a8750d684 100644
--- a/common/data_source/webdav_connector.py
+++ b/common/data_source/webdav_connector.py
@@ -170,71 +170,67 @@ class WebDAVConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):
files = []
- try:
- logging.debug(f"Listing directory: {path}")
- for item in self.client.ls(path, detail=True):
- item_path = item["name"]
+ logging.debug(f"Listing directory: {path}")
+ for item in self.client.ls(path, detail=True):
+ item_path = item["name"]
- if item_path == path or item_path == path + "/":
- continue
+ if item_path == path or item_path == path + "/":
+ continue
- logging.debug(f"Found item: {item_path}, type: {item.get('type')}")
-
- if item.get("type") == "directory":
- try:
- files.extend(
- self._list_files_recursive(
- item_path,
- start,
- end,
- filter_by_mtime=filter_by_mtime,
- )
+ logging.debug(f"Found item: {item_path}, type: {item.get('type')}")
+
+ if item.get("type") == "directory":
+ try:
+ files.extend(
+ self._list_files_recursive(
+ item_path,
+ start,
+ end,
+ filter_by_mtime=filter_by_mtime,
)
- except Exception as e:
- logging.error(f"Error recursing into directory {item_path}: {e}")
+ )
+ except Exception as e:
+ logging.error(f"Error recursing into directory {item_path}: {e}")
+ continue
+ else:
+ try:
+ file_name = os.path.basename(item_path)
+ if not self._is_supported_file(file_name):
+ logging.debug(f"Skipping file {item_path} due to unsupported extension.")
continue
- else:
- try:
- file_name = os.path.basename(item_path)
- if not self._is_supported_file(file_name):
- logging.debug(f"Skipping file {item_path} due to unsupported extension.")
- continue
-
- modified_time = item.get("modified")
- if modified_time:
- if isinstance(modified_time, datetime):
- modified = modified_time
- if modified.tzinfo is None:
- modified = modified.replace(tzinfo=timezone.utc)
- elif isinstance(modified_time, str):
+
+ modified_time = item.get("modified")
+ if modified_time:
+ if isinstance(modified_time, datetime):
+ modified = modified_time
+ if modified.tzinfo is None:
+ modified = modified.replace(tzinfo=timezone.utc)
+ elif isinstance(modified_time, str):
+ try:
+ modified = datetime.strptime(modified_time, "%a, %d %b %Y %H:%M:%S %Z")
+ modified = modified.replace(tzinfo=timezone.utc)
+ except (ValueError, TypeError):
try:
- modified = datetime.strptime(modified_time, "%a, %d %b %Y %H:%M:%S %Z")
- modified = modified.replace(tzinfo=timezone.utc)
+ modified = datetime.fromisoformat(modified_time.replace("Z", "+00:00"))
except (ValueError, TypeError):
- try:
- modified = datetime.fromisoformat(modified_time.replace("Z", "+00:00"))
- except (ValueError, TypeError):
- logging.warning(f"Could not parse modified time for {item_path}: {modified_time}")
- modified = datetime.now(timezone.utc)
- else:
- modified = datetime.now(timezone.utc)
+ logging.warning(f"Could not parse modified time for {item_path}: {modified_time}")
+ modified = datetime.now(timezone.utc)
else:
modified = datetime.now(timezone.utc)
+ else:
+ modified = datetime.now(timezone.utc)
- logging.debug(f"File {item_path}: modified={modified}, start={start}, end={end}, include={start < modified <= end}")
- if filter_by_mtime:
- if start < modified <= end:
- files.append((item_path, item))
- else:
- logging.debug(f"File {item_path} filtered out by time range")
- else:
+ logging.debug(f"File {item_path}: modified={modified}, start={start}, end={end}, include={start < modified <= end}")
+ if filter_by_mtime:
+ if start < modified <= end:
files.append((item_path, item))
- except Exception as e:
- logging.error(f"Error processing file {item_path}: {e}")
- continue
-
- except Exception as e:
- logging.error(f"Error listing directory {path}: {e}")
+ else:
+ logging.debug(f"File {item_path} filtered out by time range")
+ else:
+ files.append((item_path, item))
+ except Exception as e:
+ logging.error(f"Error processing file {item_path}: {e}")
+ continue
return files
diff --git a/test/unit_test/data_source/test_webdav_connector_unit.py b/test/unit_test/data_source/test_webdav_connector_unit.py
index fe6dd9c4b9..786f1820e2 100644
--- a/test/unit_test/data_source/test_webdav_connector_unit.py
+++ b/test/unit_test/data_source/test_webdav_connector_unit.py
@@ -165,6 +165,27 @@ def _stub_files(files):
return _list_files_recursive
+def _stub_list_raises(exc):
+ """Stub ``_list_files_recursive`` so the very first call raises.
+
+ Used by the ``#16636`` regression tests: pre-fix the top-level
+ ``except Exception`` swallowed the error and returned an empty
+ list, which then caused the prune path to delete every indexed
+ document because the snapshot looked "all gone". Post-fix the
+ exception propagates to the caller, where the prune snapshot
+ collector converts it to ``None`` and aborts the prune.
+ """
+ call_count = {"n": 0}
+
+ def _list_files_recursive(*args, **kwargs):
+ call_count["n"] += 1
+ if call_count["n"] == 1:
+ raise exc
+ return []
+
+ return _list_files_recursive
+
+
@pytest.mark.p2
def test_get_size_bytes_accepts_integer_strings():
assert WebDAVConnector._get_size_bytes({"size": 128}) == 128
@@ -299,3 +320,84 @@ def test_retrieve_all_slim_docs_skips_missing_size_metadata(caplog):
"webdav:https://webdav.example:/small.txt",
]
assert ("missing.txt: size metadata missing from WebDAV server response, skipping to avoid processing potentially large files.") in caplog.text
+
+
+class _RaisingClient:
+ """Test client whose ``ls`` raises the given exception.
+
+ Used by the ``#16636`` regression tests to simulate a transient
+ WebDAV listing error (network blip / TLS error / momentary
+ server restart). The connector must propagate the error rather
+ than swallow it and return an empty file list.
+ """
+
+ def __init__(self, exc):
+ self._exc = exc
+
+ def ls(self, path, detail=True):
+ raise self._exc
+
+
+def test_list_files_recursive_propagates_transient_listing_error():
+ """Regression for #16636: a transient ``self.client.ls`` failure
+ must propagate to the caller (so the prune snapshot collector
+ can return ``None`` and abort the prune), not be swallowed
+ and returned as an empty file list (which would cause the prune
+ to delete every indexed document because the snapshot looks
+ "all gone")."""
+ connector = WebDAVConnector("https://webdav.example", batch_size=10)
+ connector.client = _RaisingClient(ConnectionError("server restart"))
+
+ with pytest.raises(ConnectionError, match="server restart"):
+ connector._list_files_recursive(
+ "/",
+ datetime(1970, 1, 1, tzinfo=timezone.utc),
+ datetime.now(timezone.utc),
+ filter_by_mtime=False,
+ )
+
+
+def test_retrieve_all_slim_docs_perm_sync_propagates_transient_listing_error():
+ """Regression for #16636: ``retrieve_all_slim_docs_perm_sync`` is
+ the prune-snapshot entry point. A transient listing error must
+ surface as a raised exception, not a silently-empty generator.
+
+ Pre-fix, ``_list_files_recursive`` returned ``[]`` on the same
+ input and the caller saw a successful empty snapshot.
+ """
+ connector = WebDAVConnector("https://webdav.example", batch_size=10)
+ connector.client = _RaisingClient(ConnectionError("server restart"))
+
+ with pytest.raises(ConnectionError, match="server restart"):
+ # Force the inner _list_files_recursive call (rather than the
+ # stub) so the exception path is exercised end-to-end.
+ for _ in connector._yield_webdav_documents(
+ datetime(1970, 1, 1, tzinfo=timezone.utc),
+ datetime.now(timezone.utc),
+ ):
+ pass
+
+
+def test_list_files_recursive_returns_empty_when_directory_truly_empty():
+ """Regression guard: an empty WebDAV directory (no exception,
+ just zero items) must still return ``[]`` — the empty list is
+ the correct signal for a directory with no files, not for a
+ failed listing. The fix should not change this case."""
+ connector = WebDAVConnector("https://webdav.example", batch_size=10)
+ connector.client = object() # _list_files_recursive has its own for-loop; not used here
+
+ def _empty(*args, **kwargs):
+ return []
+
+ connector._list_files_recursive = _empty
+
+ result = connector._list_files_recursive(
+ "/",
+ datetime(1970, 1, 1, tzinfo=timezone.utc),
+ datetime.now(timezone.utc),
+ filter_by_mtime=False,
+ )
+ assert result == []