OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: Remove indexed chunks when connector file deleted (#1677)
Edwin Josef320c2db953d
diff --git a/src/models/processors.py b/src/models/processors.pyindex 3c84ffe7e..78ac66a72 100644--- a/src/models/processors.py+++ b/src/models/processors.py@@ -533,14 +533,42 @@ class ConnectorFileProcessor(TaskProcessor):except (FileNotFoundError, ValueError) as e:msg = str(e).lower()if "not found" in msg or "404" in msg:+ # File gone at source — remove indexed chunks by document_id+ # (= connector file_id) so it stops appearing in search/chat.+ # Filename rename (e.g. .txt → .md) is irrelevant here.+ deleted_chunks = 0+ try:+ from api.documents import delete_chunks_by_document_ids++ opensearch_client = (+ self.document_service.session_manager.get_user_opensearch_client(+ self.user_id, self.jwt_token+ )+ )+ deleted_chunks = await delete_chunks_by_document_ids(+ [file_id], opensearch_client, get_index_name()+ )+ except Exception as cleanup_err:+ logger.error(+ "Failed to clean up chunks for deleted source file",+ file_id=file_id,+ connection_id=self.connection_id,+ error=str(cleanup_err),+ )+logger.warning(- "File no longer exists at source — skipping",+ "File no longer exists at source — removed from index",file_id=file_id,connection_id=self.connection_id,+ deleted_chunks=deleted_chunks,error=str(e),)file_task.status = TaskStatus.SKIPPED- file_task.result = {"status": "skipped", "reason": "deleted_at_source"}+ file_task.result = {+ "status": "skipped",+ "reason": "deleted_at_source",+ "deleted_chunks": deleted_chunks,+ }file_task.updated_at = time.time()upload_task.successful_files += 1return@@ -681,7 +709,49 @@ class LangflowConnectorFileProcessor(TaskProcessor):raise ValueError(f"Connection '{self.connection_id}' not found")# Get file content from connector- document = await connector.get_file_content(file_id)+ try:+ document = await connector.get_file_content(file_id)+ except (FileNotFoundError, ValueError) as e:+ msg = str(e).lower()+ if "not found" in msg or "404" in msg:+ # File gone at source — remove indexed chunks by document_id+ # (= connector file_id) so it stops appearing in search/chat.+ # Filename rename (e.g. .txt → .md) is irrelevant here.+ deleted_chunks = 0+ try:+ from api.documents import delete_chunks_by_document_ids++ opensearch_client = self.langflow_connector_service.session_manager.get_user_opensearch_client(+ self.user_id, self.jwt_token+ )+ deleted_chunks = await delete_chunks_by_document_ids(+ [file_id], opensearch_client, get_index_name()+ )+ except Exception as cleanup_err:+ logger.error(+ "Failed to clean up chunks for deleted source file",+ file_id=file_id,+ connection_id=self.connection_id,+ error=str(cleanup_err),+ )++ logger.warning(+ "File no longer exists at source — removed from index",+ file_id=file_id,+ connection_id=self.connection_id,+ deleted_chunks=deleted_chunks,+ error=str(e),+ )+ file_task.status = TaskStatus.SKIPPED+ file_task.result = {+ "status": "skipped",+ "reason": "deleted_at_source",+ "deleted_chunks": deleted_chunks,+ }+ file_task.updated_at = time.time()+ upload_task.successful_files += 1+ return+ raise# Update filename in task once we have it from the connectorfile_task.filename = clean_connector_filename(document.filename, document.mimetype)diff --git a/src/models/tasks.py b/src/models/tasks.pyindex 747579f62..49eff03a5 100644--- a/src/models/tasks.py+++ b/src/models/tasks.py@@ -1,8 +1,14 @@+from __future__ import annotations++import asyncioimport itertoolsimport timefrom dataclasses import dataclass, fieldfrom enum import Enum-from typing import ClassVar+from typing import TYPE_CHECKING, ClassVar++if TYPE_CHECKING:+ from models.processors import TaskProcessorclass TaskStatus(Enum):@@ -72,6 +78,8 @@ class UploadTask:status: TaskStatus = TaskStatus.PENDINGcreated_at: float = field(default_factory=time.time)updated_at: float = field(default_factory=time.time)+ processor: TaskProcessor | None = field(default=None, repr=False)+ background_task: asyncio.Task[None] | None = field(default=None, repr=False)_sequence_number: int = field(init=False, repr=False)def __post_init__(self):diff --git a/src/services/task_service.py b/src/services/task_service.pyindex 4c7176990..25d197dac 100644--- a/src/services/task_service.py+++ b/src/services/task_service.py@@ -1,16 +1,17 @@-import tracebackimport asyncioimport osimport randomimport time+import tracebackimport uuid-from typing import Any, Coroutine, TypeVar+from collections.abc import Coroutine+from typing import Any, TypeVarfrom models.tasks import FileTask, TaskStatus, UploadTaskfrom session_manager import AnonymousUserfrom utils.gpu_detection import get_worker_countfrom utils.logging_config import get_logger-from utils.telemetry import TelemetryClient, Category, MessageId+from utils.telemetry import Category, MessageId, TelemetryClientT = TypeVar("T")@@ -112,7 +113,7 @@ class TaskService:timeout: int = timeout_seconds or self.ingestion_timeouttry:return await asyncio.wait_for(coro, timeout=timeout)- except asyncio.TimeoutError:+ except TimeoutError:raise IngestionTimeoutError(f"File processing timed out after {timeout} seconds.") from None@@ -431,8 +432,16 @@ class TaskService:finally:file_task.updated_at = time.time()# Only increment processed_files if the file reached a terminal state- # This prevents counter inconsistency on cancellation- if file_task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]:+ # This prevents counter inconsistency on cancellation.+ # SKIPPED is terminal too — e.g. source-deleted files in+ # connector sync — and must count, or the upload task+ # never reaches processed_files >= total_files and stays+ # open forever.+ if file_task.status in [+ TaskStatus.COMPLETED,+ TaskStatus.FAILED,+ TaskStatus.SKIPPED,+ ]:async with self._get_task_lock(task_id):upload_task.processed_files += 1upload_task.updated_at = time.time()@@ -824,7 +833,7 @@ class TaskService:if self.background_tasks:results = await asyncio.gather(*self.background_tasks, return_exceptions=True)# Log any unexpected errors (not CancelledError)- for i, result in enumerate(results):+ for result in results:if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):logger.warning("Background task raised exception during shutdown", error=str(result)diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.pyindex 4d167dc42..9cdfe1685 100644--- a/tests/unit/test_connector_processor_filename_dedupe.py+++ b/tests/unit/test_connector_processor_filename_dedupe.py@@ -128,6 +128,47 @@ async def test_connector_processor_deletes_then_ingests_when_replace_true():assert mock_process.await_args.kwargs["original_filename"] == "My Report.pdf"+async def test_connector_processor_deletes_chunks_when_source_returns_404():+ """When the source connector reports the file is gone (404), the processor+ must remove the already-indexed chunks for that document_id. Regression+ test for SharePoint sync leaving orphan chunks after a source-side delete.+ """+ processor = _build_connector_processor(replace_duplicates=False)++ opensearch_client = AsyncMock()+ # collect_visible_document_ids issues a scroll search; return one chunk _id+ opensearch_client.search = AsyncMock(+ return_value={"hits": {"hits": [{"_id": "chunk-1"}]}, "_scroll_id": None}+ )+ # delete_document_ids issues individual deletes per _id+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})+ processor.document_service.session_manager.get_user_opensearch_client.return_value = (+ opensearch_client+ )++ connector = MagicMock()+ connector.get_file_content = AsyncMock(side_effect=FileNotFoundError("404 Not Found"))+ processor.connector_service.get_connector = AsyncMock(return_value=connector)+ connection = MagicMock()+ connection.connector_type = "sharepoint"+ processor.connector_service.connection_manager = MagicMock()+ processor.connector_service.connection_manager.get_connection = AsyncMock(+ return_value=connection+ )++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.status == TaskStatus.SKIPPED+ assert (file_task.result or {}).get("reason") == "deleted_at_source"+ assert (file_task.result or {}).get("deleted_chunks") == 1+ assert upload_task.successful_files == 1+ opensearch_client.delete.assert_awaited_once()++@pytest.mark.asyncioasync def test_connector_processor_proceeds_when_filename_absent():processor = _build_connector_processor(replace_duplicates=False)@@ -232,6 +273,51 @@ async def test_langflow_connector_processor_overwrites_when_replace_true():processor.langflow_connector_service.process_connector_document.assert_awaited_once()+async def test_langflow_connector_processor_deletes_chunks_when_source_returns_404():+ """When the source connector reports the file is gone (404), the Langflow+ processor must remove the already-indexed chunks instead of surfacing+ 'File not found: <id>' as a task error. Regression test for SharePoint+ webhook-triggered sync of a deleted source file.+ """+ processor = _build_langflow_processor(replace_duplicates=False)++ opensearch_client = AsyncMock()+ opensearch_client.search = AsyncMock(+ return_value={"hits": {"hits": [{"_id": "chunk-1"}]}, "_scroll_id": None}+ )+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})+ processor.langflow_connector_service.session_manager.get_user_opensearch_client.return_value = (+ opensearch_client+ )++ connector = MagicMock()+ connector.get_file_content = AsyncMock(+ side_effect=ValueError("File not found: 01BYMO7NCRKVAJFSPPABBKQXS4PPDHBVUY")+ )+ processor.langflow_connector_service.get_connector = AsyncMock(return_value=connector)+ connection = MagicMock()+ connection.connector_type = "sharepoint"+ processor.langflow_connector_service.connection_manager = MagicMock()+ processor.langflow_connector_service.connection_manager.get_connection = AsyncMock(+ return_value=connection+ )++ file_task = _make_file_task()+ upload_task = _make_upload_task()++ await processor.process_item(upload_task, "file-id-1", file_task)++ assert file_task.status == TaskStatus.SKIPPED+ assert (file_task.result or {}).get("reason") == "deleted_at_source"+ assert (file_task.result or {}).get("deleted_chunks") == 1+ assert file_task.error is None+ assert upload_task.successful_files == 1+ assert upload_task.failed_files == 0+ processor.langflow_connector_service.process_connector_document.assert_not_called()+ opensearch_client.delete.assert_awaited_once()++@pytest.mark.asyncioasync def test_langflow_connector_processor_hash_unchanged_path_preserved():"""When the filename is new but the byte hash already exists, the filediff --git a/tests/unit/test_task_service.py b/tests/unit/test_task_service.pyindex c5e590603..3b346c48a 100644--- a/tests/unit/test_task_service.py+++ b/tests/unit/test_task_service.py@@ -2,12 +2,15 @@Basic tests for TaskService improvementsTests timeout handling, cancellation, and cleanup functionality"""+import asyncio-import pytestimport time-from unittest.mock import Mock, AsyncMock, patch-from services.task_service import TaskService, IngestionTimeoutError-from models.tasks import TaskStatus, UploadTask, FileTask+from unittest.mock import Mock++import pytest++from models.tasks import FileTask, TaskStatus, UploadTask+from services.task_service import IngestionTimeoutError, TaskService@pytest.fixture@@ -16,13 +19,14 @@ def task_service():mock_doc_service = Mock()return TaskService(document_service=mock_doc_service,- ingestion_timeout=2 # Short timeout for testing+ ingestion_timeout=2, # Short timeout for testing)@pytest.mark.asyncioasync def test_process_with_timeout_success(task_service):"""Test that _process_with_timeout completes successfully within timeout"""+async def quick_task():await asyncio.sleep(0.01)return "success"@@ -34,6 +38,7 @@ async def test_process_with_timeout_success(task_service):@pytest.mark.asyncioasync def test_process_with_timeout_exceeds(task_service):"""Test that _process_with_timeout raises IngestionTimeoutError when timeout is exceeded"""+async def slow_task():await asyncio.sleep(10)return "should not reach here"@@ -52,7 +57,7 @@ async def test_cleanup_old_tasks(task_service):task_id="old_task",total_files=1,file_tasks={"file1": FileTask(file_path="file1")},- status=TaskStatus.COMPLETED+ status=TaskStatus.COMPLETED,)old_task.updated_at = time.time() - 7200 # 2 hours ago@@ -60,7 +65,7 @@ async def test_cleanup_old_tasks(task_service):task_id="recent_task",total_files=1,file_tasks={"file2": FileTask(file_path="file2")},- status=TaskStatus.COMPLETED+ status=TaskStatus.COMPLETED,)recent_task.updated_at = time.time() - 600 # 10 minutes ago@@ -68,7 +73,7 @@ async def test_cleanup_old_tasks(task_service):task_id="running_task",total_files=1,file_tasks={"file3": FileTask(file_path="file3")},- status=TaskStatus.RUNNING+ status=TaskStatus.RUNNING,)running_task.updated_at = time.time() - 7200 # 2 hours ago but still running@@ -76,7 +81,7 @@ async def test_cleanup_old_tasks(task_service):task_service.task_store["user1"] = {"old_task": old_task,"recent_task": recent_task,- "running_task": running_task+ "running_task": running_task,}# Pre-create locks for the taskstask_service._task_locks["old_task"] = asyncio.Lock()@@ -98,6 +103,7 @@ async def test_cleanup_old_tasks(task_service):@pytest.mark.asyncioasync def test_shutdown_cancels_background_tasks(task_service):"""Test that shutdown properly cancels background tasks"""+async def background_work():try:await asyncio.sleep(10)@@ -120,28 +126,37 @@ async def test_shutdown_cancels_background_tasks(task_service):def test_counter_consistency_logic(task_service):- """Test that processed_files counter logic only counts terminal states"""+ """Test that processed_files counter logic only counts terminal states.++ SKIPPED must count as terminal — connector sync marks source-deleted+ files SKIPPED, and excluding them caused the upload task to hang+ forever (processed_files never reaching total_files).+ """# This tests the logic pattern used in the finally blocktask = UploadTask(task_id="test_task",- total_files=3,+ total_files=4,file_tasks={"file1": FileTask(file_path="file1", status=TaskStatus.COMPLETED),"file2": FileTask(file_path="file2", status=TaskStatus.FAILED),"file3": FileTask(file_path="file3", status=TaskStatus.RUNNING),- }+ "file4": FileTask(file_path="file4", status=TaskStatus.SKIPPED),+ },)# Simulate the counter logic from the finally blockprocessed_count = 0for file_task in task.file_tasks.values():- if file_task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]:+ if file_task.status in [+ TaskStatus.COMPLETED,+ TaskStatus.FAILED,+ TaskStatus.SKIPPED,+ ]:processed_count += 1- # Should only count completed and failed, not running- assert processed_count == 2-+ # Should count completed, failed, and skipped — but not running+ assert processed_count == 3@pytest.mark.asyncio@@ -152,38 +167,37 @@ async def test_concurrent_counter_updates(task_service):task_id="concurrent_task",total_files=10,file_tasks={- f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING)- for i in range(10)+ f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING) for i in range(10)},- status=TaskStatus.RUNNING+ status=TaskStatus.RUNNING,)-+task_service.task_store["user1"] = {"concurrent_task": task}-+# Simulate concurrent updates to countersasync def update_counter(file_id: str):"""Simulate processing a file and updating counters"""file_task = task.file_tasks[file_id]-+# Simulate some processing timeawait asyncio.sleep(0.01)-+# Update file statusfile_task.status = TaskStatus.COMPLETED-+# Update counters with lock (as done in the actual code)async with task_service._get_task_lock("concurrent_task"):task.processed_files += 1task.successful_files += 1-+# Run all updates concurrentlyawait asyncio.gather(*[update_counter(f"file{i}") for i in range(10)])-+# Verify all counters are correct (no race conditions)assert task.processed_files == 10assert task.successful_files == 10assert task.failed_files == 0-+# Verify all file tasks are completedfor file_task in task.file_tasks.values():assert file_task.status == TaskStatus.COMPLETED@@ -196,19 +210,18 @@ async def test_concurrent_mixed_counter_updates(task_service):task_id="mixed_task",total_files=20,file_tasks={- f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING)- for i in range(20)+ f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING) for i in range(20)},- status=TaskStatus.RUNNING+ status=TaskStatus.RUNNING,)-+task_service.task_store["user1"] = {"mixed_task": task}-+async def update_counter(file_id: str, should_fail: bool):"""Simulate processing with success or failure"""file_task = task.file_tasks[file_id]await asyncio.sleep(0.01)-+# Update file status and counters atomicallyasync with task_service._get_task_lock("mixed_task"):if should_fail:@@ -218,18 +231,14 @@ async def test_concurrent_mixed_counter_updates(task_service):file_task.status = TaskStatus.COMPLETEDtask.successful_files += 1task.processed_files += 1-+# Create mix of successful and failed tasks- tasks = [- update_counter(f"file{i}", should_fail=(i % 3 == 0))- for i in range(20)- ]-+ tasks = [update_counter(f"file{i}", should_fail=(i % 3 == 0)) for i in range(20)]+await asyncio.gather(*tasks)-+# Verify counters: 7 failures (0,3,6,9,12,15,18), 13 successesassert task.processed_files == 20assert task.failed_files == 7assert task.successful_files == 13assert task.processed_files == task.successful_files + task.failed_files-