Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python

fix: Remove indexed chunks when connector file deleted (#1677)

Edwin Josef320c2db953d
diff --git a/src/models/processors.py b/src/models/processors.py
index 3c84ffe7e..78ac66a72 100644
--- a/src/models/processors.py
+++ b/src/models/processors.py
@@ -533,14 +533,42 @@ class ConnectorFileProcessor(TaskProcessor):
except (FileNotFoundError, ValueError) as e:
msg = str(e).lower()
if "not found" in msg or "404" in msg:
+ # File gone at source — remove indexed chunks by document_id
+ # (= connector file_id) so it stops appearing in search/chat.
+ # Filename rename (e.g. .txt → .md) is irrelevant here.
+ deleted_chunks = 0
+ try:
+ from api.documents import delete_chunks_by_document_ids
+
+ opensearch_client = (
+ self.document_service.session_manager.get_user_opensearch_client(
+ self.user_id, self.jwt_token
+ )
+ )
+ deleted_chunks = await delete_chunks_by_document_ids(
+ [file_id], opensearch_client, get_index_name()
+ )
+ except Exception as cleanup_err:
+ logger.error(
+ "Failed to clean up chunks for deleted source file",
+ file_id=file_id,
+ connection_id=self.connection_id,
+ error=str(cleanup_err),
+ )
+
logger.warning(
- "File no longer exists at source — skipping",
+ "File no longer exists at source — removed from index",
file_id=file_id,
connection_id=self.connection_id,
+ deleted_chunks=deleted_chunks,
error=str(e),
)
file_task.status = TaskStatus.SKIPPED
- file_task.result = {"status": "skipped", "reason": "deleted_at_source"}
+ file_task.result = {
+ "status": "skipped",
+ "reason": "deleted_at_source",
+ "deleted_chunks": deleted_chunks,
+ }
file_task.updated_at = time.time()
upload_task.successful_files += 1
return
@@ -681,7 +709,49 @@ class LangflowConnectorFileProcessor(TaskProcessor):
raise ValueError(f"Connection '{self.connection_id}' not found")
# Get file content from connector
- document = await connector.get_file_content(file_id)
+ try:
+ document = await connector.get_file_content(file_id)
+ except (FileNotFoundError, ValueError) as e:
+ msg = str(e).lower()
+ if "not found" in msg or "404" in msg:
+ # File gone at source — remove indexed chunks by document_id
+ # (= connector file_id) so it stops appearing in search/chat.
+ # Filename rename (e.g. .txt → .md) is irrelevant here.
+ deleted_chunks = 0
+ try:
+ from api.documents import delete_chunks_by_document_ids
+
+ opensearch_client = self.langflow_connector_service.session_manager.get_user_opensearch_client(
+ self.user_id, self.jwt_token
+ )
+ deleted_chunks = await delete_chunks_by_document_ids(
+ [file_id], opensearch_client, get_index_name()
+ )
+ except Exception as cleanup_err:
+ logger.error(
+ "Failed to clean up chunks for deleted source file",
+ file_id=file_id,
+ connection_id=self.connection_id,
+ error=str(cleanup_err),
+ )
+
+ logger.warning(
+ "File no longer exists at source — removed from index",
+ file_id=file_id,
+ connection_id=self.connection_id,
+ deleted_chunks=deleted_chunks,
+ error=str(e),
+ )
+ file_task.status = TaskStatus.SKIPPED
+ file_task.result = {
+ "status": "skipped",
+ "reason": "deleted_at_source",
+ "deleted_chunks": deleted_chunks,
+ }
+ file_task.updated_at = time.time()
+ upload_task.successful_files += 1
+ return
+ raise
# Update filename in task once we have it from the connector
file_task.filename = clean_connector_filename(document.filename, document.mimetype)
diff --git a/src/models/tasks.py b/src/models/tasks.py
index 747579f62..49eff03a5 100644
--- a/src/models/tasks.py
+++ b/src/models/tasks.py
@@ -1,8 +1,14 @@
+from __future__ import annotations
+
+import asyncio
import itertools
import time
from dataclasses import dataclass, field
from enum import Enum
-from typing import ClassVar
+from typing import TYPE_CHECKING, ClassVar
+
+if TYPE_CHECKING:
+ from models.processors import TaskProcessor
class TaskStatus(Enum):
@@ -72,6 +78,8 @@ class UploadTask:
status: TaskStatus = TaskStatus.PENDING
created_at: float = field(default_factory=time.time)
updated_at: float = field(default_factory=time.time)
+ processor: TaskProcessor | None = field(default=None, repr=False)
+ background_task: asyncio.Task[None] | None = field(default=None, repr=False)
_sequence_number: int = field(init=False, repr=False)
def __post_init__(self):
diff --git a/src/services/task_service.py b/src/services/task_service.py
index 4c7176990..25d197dac 100644
--- a/src/services/task_service.py
+++ b/src/services/task_service.py
@@ -1,16 +1,17 @@
-import traceback
import asyncio
import os
import random
import time
+import traceback
import uuid
-from typing import Any, Coroutine, TypeVar
+from collections.abc import Coroutine
+from typing import Any, TypeVar
from models.tasks import FileTask, TaskStatus, UploadTask
from session_manager import AnonymousUser
from utils.gpu_detection import get_worker_count
from utils.logging_config import get_logger
-from utils.telemetry import TelemetryClient, Category, MessageId
+from utils.telemetry import Category, MessageId, TelemetryClient
T = TypeVar("T")
@@ -112,7 +113,7 @@ class TaskService:
timeout: int = timeout_seconds or self.ingestion_timeout
try:
return await asyncio.wait_for(coro, timeout=timeout)
- except asyncio.TimeoutError:
+ except TimeoutError:
raise IngestionTimeoutError(
f"File processing timed out after {timeout} seconds."
) from None
@@ -431,8 +432,16 @@ class TaskService:
finally:
file_task.updated_at = time.time()
# Only increment processed_files if the file reached a terminal state
- # This prevents counter inconsistency on cancellation
- if file_task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
+ # This prevents counter inconsistency on cancellation.
+ # SKIPPED is terminal too — e.g. source-deleted files in
+ # connector sync — and must count, or the upload task
+ # never reaches processed_files >= total_files and stays
+ # open forever.
+ if file_task.status in [
+ TaskStatus.COMPLETED,
+ TaskStatus.FAILED,
+ TaskStatus.SKIPPED,
+ ]:
async with self._get_task_lock(task_id):
upload_task.processed_files += 1
upload_task.updated_at = time.time()
@@ -824,7 +833,7 @@ class TaskService:
if self.background_tasks:
results = await asyncio.gather(*self.background_tasks, return_exceptions=True)
# Log any unexpected errors (not CancelledError)
- for i, result in enumerate(results):
+ for result in results:
if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
logger.warning(
"Background task raised exception during shutdown", error=str(result)
diff --git a/tests/unit/test_connector_processor_filename_dedupe.py b/tests/unit/test_connector_processor_filename_dedupe.py
index 4d167dc42..9cdfe1685 100644
--- a/tests/unit/test_connector_processor_filename_dedupe.py
+++ b/tests/unit/test_connector_processor_filename_dedupe.py
@@ -128,6 +128,47 @@ async def test_connector_processor_deletes_then_ingests_when_replace_true():
assert mock_process.await_args.kwargs["original_filename"] == "My Report.pdf"
+async def test_connector_processor_deletes_chunks_when_source_returns_404():
+ """When the source connector reports the file is gone (404), the processor
+ must remove the already-indexed chunks for that document_id. Regression
+ test for SharePoint sync leaving orphan chunks after a source-side delete.
+ """
+ processor = _build_connector_processor(replace_duplicates=False)
+
+ opensearch_client = AsyncMock()
+ # collect_visible_document_ids issues a scroll search; return one chunk _id
+ opensearch_client.search = AsyncMock(
+ return_value={"hits": {"hits": [{"_id": "chunk-1"}]}, "_scroll_id": None}
+ )
+ # delete_document_ids issues individual deletes per _id
+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
+ processor.document_service.session_manager.get_user_opensearch_client.return_value = (
+ opensearch_client
+ )
+
+ connector = MagicMock()
+ connector.get_file_content = AsyncMock(side_effect=FileNotFoundError("404 Not Found"))
+ processor.connector_service.get_connector = AsyncMock(return_value=connector)
+ connection = MagicMock()
+ connection.connector_type = "sharepoint"
+ processor.connector_service.connection_manager = MagicMock()
+ processor.connector_service.connection_manager.get_connection = AsyncMock(
+ return_value=connection
+ )
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.status == TaskStatus.SKIPPED
+ assert (file_task.result or {}).get("reason") == "deleted_at_source"
+ assert (file_task.result or {}).get("deleted_chunks") == 1
+ assert upload_task.successful_files == 1
+ opensearch_client.delete.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_connector_processor_proceeds_when_filename_absent():
processor = _build_connector_processor(replace_duplicates=False)
@@ -232,6 +273,51 @@ async def test_langflow_connector_processor_overwrites_when_replace_true():
processor.langflow_connector_service.process_connector_document.assert_awaited_once()
+async def test_langflow_connector_processor_deletes_chunks_when_source_returns_404():
+ """When the source connector reports the file is gone (404), the Langflow
+ processor must remove the already-indexed chunks instead of surfacing
+ 'File not found: <id>' as a task error. Regression test for SharePoint
+ webhook-triggered sync of a deleted source file.
+ """
+ processor = _build_langflow_processor(replace_duplicates=False)
+
+ opensearch_client = AsyncMock()
+ opensearch_client.search = AsyncMock(
+ return_value={"hits": {"hits": [{"_id": "chunk-1"}]}, "_scroll_id": None}
+ )
+ opensearch_client.delete = AsyncMock(return_value={"result": "deleted"})
+ processor.langflow_connector_service.session_manager.get_user_opensearch_client.return_value = (
+ opensearch_client
+ )
+
+ connector = MagicMock()
+ connector.get_file_content = AsyncMock(
+ side_effect=ValueError("File not found: 01BYMO7NCRKVAJFSPPABBKQXS4PPDHBVUY")
+ )
+ processor.langflow_connector_service.get_connector = AsyncMock(return_value=connector)
+ connection = MagicMock()
+ connection.connector_type = "sharepoint"
+ processor.langflow_connector_service.connection_manager = MagicMock()
+ processor.langflow_connector_service.connection_manager.get_connection = AsyncMock(
+ return_value=connection
+ )
+
+ file_task = _make_file_task()
+ upload_task = _make_upload_task()
+
+ await processor.process_item(upload_task, "file-id-1", file_task)
+
+ assert file_task.status == TaskStatus.SKIPPED
+ assert (file_task.result or {}).get("reason") == "deleted_at_source"
+ assert (file_task.result or {}).get("deleted_chunks") == 1
+ assert file_task.error is None
+ assert upload_task.successful_files == 1
+ assert upload_task.failed_files == 0
+ processor.langflow_connector_service.process_connector_document.assert_not_called()
+ opensearch_client.delete.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_langflow_connector_processor_hash_unchanged_path_preserved():
"""When the filename is new but the byte hash already exists, the file
diff --git a/tests/unit/test_task_service.py b/tests/unit/test_task_service.py
index c5e590603..3b346c48a 100644
--- a/tests/unit/test_task_service.py
+++ b/tests/unit/test_task_service.py
@@ -2,12 +2,15 @@
Basic tests for TaskService improvements
Tests timeout handling, cancellation, and cleanup functionality
"""
+
import asyncio
-import pytest
import time
-from unittest.mock import Mock, AsyncMock, patch
-from services.task_service import TaskService, IngestionTimeoutError
-from models.tasks import TaskStatus, UploadTask, FileTask
+from unittest.mock import Mock
+
+import pytest
+
+from models.tasks import FileTask, TaskStatus, UploadTask
+from services.task_service import IngestionTimeoutError, TaskService
@pytest.fixture
@@ -16,13 +19,14 @@ def task_service():
mock_doc_service = Mock()
return TaskService(
document_service=mock_doc_service,
- ingestion_timeout=2 # Short timeout for testing
+ ingestion_timeout=2, # Short timeout for testing
)
@pytest.mark.asyncio
async def test_process_with_timeout_success(task_service):
"""Test that _process_with_timeout completes successfully within timeout"""
+
async def quick_task():
await asyncio.sleep(0.01)
return "success"
@@ -34,6 +38,7 @@ async def test_process_with_timeout_success(task_service):
@pytest.mark.asyncio
async def test_process_with_timeout_exceeds(task_service):
"""Test that _process_with_timeout raises IngestionTimeoutError when timeout is exceeded"""
+
async def slow_task():
await asyncio.sleep(10)
return "should not reach here"
@@ -52,7 +57,7 @@ async def test_cleanup_old_tasks(task_service):
task_id="old_task",
total_files=1,
file_tasks={"file1": FileTask(file_path="file1")},
- status=TaskStatus.COMPLETED
+ status=TaskStatus.COMPLETED,
)
old_task.updated_at = time.time() - 7200 # 2 hours ago
@@ -60,7 +65,7 @@ async def test_cleanup_old_tasks(task_service):
task_id="recent_task",
total_files=1,
file_tasks={"file2": FileTask(file_path="file2")},
- status=TaskStatus.COMPLETED
+ status=TaskStatus.COMPLETED,
)
recent_task.updated_at = time.time() - 600 # 10 minutes ago
@@ -68,7 +73,7 @@ async def test_cleanup_old_tasks(task_service):
task_id="running_task",
total_files=1,
file_tasks={"file3": FileTask(file_path="file3")},
- status=TaskStatus.RUNNING
+ status=TaskStatus.RUNNING,
)
running_task.updated_at = time.time() - 7200 # 2 hours ago but still running
@@ -76,7 +81,7 @@ async def test_cleanup_old_tasks(task_service):
task_service.task_store["user1"] = {
"old_task": old_task,
"recent_task": recent_task,
- "running_task": running_task
+ "running_task": running_task,
}
# Pre-create locks for the tasks
task_service._task_locks["old_task"] = asyncio.Lock()
@@ -98,6 +103,7 @@ async def test_cleanup_old_tasks(task_service):
@pytest.mark.asyncio
async def test_shutdown_cancels_background_tasks(task_service):
"""Test that shutdown properly cancels background tasks"""
+
async def background_work():
try:
await asyncio.sleep(10)
@@ -120,28 +126,37 @@ async def test_shutdown_cancels_background_tasks(task_service):
def test_counter_consistency_logic(task_service):
- """Test that processed_files counter logic only counts terminal states"""
+ """Test that processed_files counter logic only counts terminal states.
+
+ SKIPPED must count as terminal — connector sync marks source-deleted
+ files SKIPPED, and excluding them caused the upload task to hang
+ forever (processed_files never reaching total_files).
+ """
# This tests the logic pattern used in the finally block
task = UploadTask(
task_id="test_task",
- total_files=3,
+ total_files=4,
file_tasks={
"file1": FileTask(file_path="file1", status=TaskStatus.COMPLETED),
"file2": FileTask(file_path="file2", status=TaskStatus.FAILED),
"file3": FileTask(file_path="file3", status=TaskStatus.RUNNING),
- }
+ "file4": FileTask(file_path="file4", status=TaskStatus.SKIPPED),
+ },
)
# Simulate the counter logic from the finally block
processed_count = 0
for file_task in task.file_tasks.values():
- if file_task.status in [TaskStatus.COMPLETED, TaskStatus.FAILED]:
+ if file_task.status in [
+ TaskStatus.COMPLETED,
+ TaskStatus.FAILED,
+ TaskStatus.SKIPPED,
+ ]:
processed_count += 1
- # Should only count completed and failed, not running
- assert processed_count == 2
-
+ # Should count completed, failed, and skipped — but not running
+ assert processed_count == 3
@pytest.mark.asyncio
@@ -152,38 +167,37 @@ async def test_concurrent_counter_updates(task_service):
task_id="concurrent_task",
total_files=10,
file_tasks={
- f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING)
- for i in range(10)
+ f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING) for i in range(10)
},
- status=TaskStatus.RUNNING
+ status=TaskStatus.RUNNING,
)
-
+
task_service.task_store["user1"] = {"concurrent_task": task}
-
+
# Simulate concurrent updates to counters
async def update_counter(file_id: str):
"""Simulate processing a file and updating counters"""
file_task = task.file_tasks[file_id]
-
+
# Simulate some processing time
await asyncio.sleep(0.01)
-
+
# Update file status
file_task.status = TaskStatus.COMPLETED
-
+
# Update counters with lock (as done in the actual code)
async with task_service._get_task_lock("concurrent_task"):
task.processed_files += 1
task.successful_files += 1
-
+
# Run all updates concurrently
await asyncio.gather(*[update_counter(f"file{i}") for i in range(10)])
-
+
# Verify all counters are correct (no race conditions)
assert task.processed_files == 10
assert task.successful_files == 10
assert task.failed_files == 0
-
+
# Verify all file tasks are completed
for file_task in task.file_tasks.values():
assert file_task.status == TaskStatus.COMPLETED
@@ -196,19 +210,18 @@ async def test_concurrent_mixed_counter_updates(task_service):
task_id="mixed_task",
total_files=20,
file_tasks={
- f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING)
- for i in range(20)
+ f"file{i}": FileTask(file_path=f"file{i}", status=TaskStatus.PENDING) for i in range(20)
},
- status=TaskStatus.RUNNING
+ status=TaskStatus.RUNNING,
)
-
+
task_service.task_store["user1"] = {"mixed_task": task}
-
+
async def update_counter(file_id: str, should_fail: bool):
"""Simulate processing with success or failure"""
file_task = task.file_tasks[file_id]
await asyncio.sleep(0.01)
-
+
# Update file status and counters atomically
async with task_service._get_task_lock("mixed_task"):
if should_fail:
@@ -218,18 +231,14 @@ async def test_concurrent_mixed_counter_updates(task_service):
file_task.status = TaskStatus.COMPLETED
task.successful_files += 1
task.processed_files += 1
-
+
# Create mix of successful and failed tasks
- tasks = [
- update_counter(f"file{i}", should_fail=(i % 3 == 0))
- for i in range(20)
- ]
-
+ tasks = [update_counter(f"file{i}", should_fail=(i % 3 == 0)) for i in range(20)]
+
await asyncio.gather(*tasks)
-
+
# Verify counters: 7 failures (0,3,6,9,12,15,18), 13 successes
assert task.processed_files == 20
assert task.failed_files == 7
assert task.successful_files == 13
assert task.processed_files == task.successful_files + task.failed_files
-