Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix: dataset hitcount deadlock (#41979)
FFXNbb2945a1a3a5
diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.pyindex a9a3cbb16b9..2caa266c37c 100644--- a/api/core/rag/retrieval/dataset_retrieval.py+++ b/api/core/rag/retrieval/dataset_retrieval.py@@ -11,7 +11,9 @@ from typing import Any, Union, castfrom flask import Flask, current_appfrom opentelemetry.trace import get_current_spanfrom sqlalchemy import and_, func, literal, or_, select, update+from sqlalchemy.exc import DBAPIErrorfrom sqlalchemy.orm import Session, sessionmaker+from tenacity import before_sleep_log, retry, retry_if_exception, stop_after_attempt, wait_exponentialfrom core.app.app_config.entities import (DatasetEntity,@@ -107,6 +109,18 @@ default_retrieval_model: DefaultRetrievalModelDict = {logger = logging.getLogger(__name__)+_POSTGRES_DEADLOCK_SQLSTATE = "40P01"+_HIT_COUNT_UPDATE_MAX_ATTEMPTS = 3+++def _is_postgres_deadlock_error(exc: BaseException) -> bool:+ if not isinstance(exc, DBAPIError) or exc.orig is None:+ return False+ orig = cast(Any, exc.orig)+ return (hasattr(orig, "sqlstate") and orig.sqlstate == _POSTGRES_DEADLOCK_SQLSTATE) or (+ hasattr(orig, "pgcode") and orig.pgcode == _POSTGRES_DEADLOCK_SQLSTATE+ )+class DatasetRetrieval:def __init__(self, application_generate_entity: EasyUIBasedAppGenerateEntity | None = None):@@ -915,6 +929,13 @@ class DatasetRetrieval:retrieval_resource_list.append(document)return retrieval_resource_list+ @retry(+ retry=retry_if_exception(_is_postgres_deadlock_error),+ stop=stop_after_attempt(_HIT_COUNT_UPDATE_MAX_ATTEMPTS),+ wait=wait_exponential(multiplier=0.05, min=0.05, max=0.1),+ before_sleep=before_sleep_log(logger, logging.WARNING),+ reraise=True,+ )def _on_retrieval_end(self,flask_app: Flask,@@ -1016,6 +1037,14 @@ class DatasetRetrieval:# Batch update hit_count for all segmentsif segment_ids_to_update:+ # PostgreSQL does not guarantee that an IN predicate is visited in parameter order.+ # Lock every target row explicitly and consistently before the multi-row update.+ session.scalars(+ select(DocumentSegment.id)+ .where(DocumentSegment.id.in_(segment_ids_to_update))+ .order_by(DocumentSegment.id)+ .with_for_update()+ ).all()session.execute(update(DocumentSegment).where(DocumentSegment.id.in_(segment_ids_to_update))diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.pyindex 890cc0068ed..461258033ee 100644--- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py+++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py@@ -9,10 +9,13 @@ from uuid import uuid4import pytestfrom flask import Flask, current_app-from sqlalchemy import select+from sqlalchemy import event, select+from sqlalchemy.dialects import postgresqlfrom sqlalchemy.engine import Engine+from sqlalchemy.exc import DBAPIErrorfrom sqlalchemy.orm import Session+import core.rag.retrieval.dataset_retrieval as dataset_retrieval_modulefrom core.app.app_config.entities import (DatasetEntity,DatasetRetrieveConfigEntity,@@ -5477,15 +5480,133 @@ class TestInternalHooksCoverage:),]+ executed_statements: list[object] = []++ def capture_statement(*args: object) -> None:+ executed_statements.append(args[1])++ event.listen(self.orm_engine, "before_execute", capture_statement)+ try:+ with (+ patch("core.rag.retrieval.dataset_retrieval.db", SimpleNamespace(engine=self.orm_engine)),+ patch.object(retrieval, "_send_trace_task") as mock_trace,+ ):+ retrieval._on_retrieval_end(flask_app=app, documents=docs, message_id="m1", timer={"cost": 1})+ finally:+ event.remove(self.orm_engine, "before_execute", capture_statement)++ self.orm_session.expire_all()+ assert [segment.hit_count for segment in segments] == [1, 1, 1, 1]+ locking_statements = [+ statement+ for statement in executed_statements+ if "ORDER BY document_segments.id FOR UPDATE" in str(statement.compile(dialect=postgresql.dialect())) # type: ignore[union-attr]+ ]+ assert len(locking_statements) == 1+ mock_trace.assert_called_once()++ def test_on_retrieval_end_retries_postgres_deadlock(self, retrieval: DatasetRetrieval) -> None:+ app = Flask(__name__)+ deadlock = DBAPIError(+ "UPDATE document_segments",+ {},+ SimpleNamespace(sqlstate="40P01"),+ connection_invalidated=False,+ )+ doc = _doc(provider="dify")+ retry_session = MagicMock()+with (patch("core.rag.retrieval.dataset_retrieval.db", SimpleNamespace(engine=self.orm_engine)),+ patch("core.rag.retrieval.dataset_retrieval.sessionmaker") as mock_sessionmaker,patch.object(retrieval, "_send_trace_task") as mock_trace,):- retrieval._on_retrieval_end(flask_app=app, documents=docs, message_id="m1", timer={"cost": 1})+ mock_sessionmaker.return_value.begin.side_effect = [deadlock, nullcontext(retry_session)]+ retrieval._on_retrieval_end(flask_app=app, documents=[doc], message_id="m1", timer={"cost": 1})- self.orm_session.expire_all()- assert [segment.hit_count for segment in segments] == [1, 1, 1, 1]- mock_trace.assert_called_once()+ assert mock_sessionmaker.return_value.begin.call_count == 2+ mock_trace.assert_called_once_with("m1", [doc], {"cost": 1})++ @pytest.mark.parametrize(+ ("error", "expected"),+ [+ (RuntimeError("not a database error"), False),+ (DBAPIError("SELECT 1", {}, None, connection_invalidated=False), False),+ (+ DBAPIError(+ "UPDATE document_segments",+ {},+ SimpleNamespace(pgcode="40P01"),+ connection_invalidated=False,+ ),+ True,+ ),+ (+ DBAPIError(+ "UPDATE document_segments",+ {},+ SimpleNamespace(sqlstate="40001", pgcode="40P01"),+ connection_invalidated=False,+ ),+ True,+ ),+ (+ DBAPIError(+ "UPDATE document_segments",+ {},+ SimpleNamespace(sqlstate="40001", pgcode="40001"),+ connection_invalidated=False,+ ),+ False,+ ),+ ],+ )+ def test_is_postgres_deadlock_error(self, error: BaseException, expected: bool) -> None:+ assert dataset_retrieval_module._is_postgres_deadlock_error(error) is expected++ def test_on_retrieval_end_reraises_non_deadlock(self, retrieval: DatasetRetrieval) -> None:+ app = Flask(__name__)+ error = DBAPIError(+ "UPDATE document_segments",+ {},+ SimpleNamespace(sqlstate="40001"),+ connection_invalidated=False,+ )+ doc = _doc(provider="dify")++ with (+ patch("core.rag.retrieval.dataset_retrieval.db", SimpleNamespace(engine=self.orm_engine)),+ patch("core.rag.retrieval.dataset_retrieval.sessionmaker") as mock_sessionmaker,+ patch.object(retrieval, "_send_trace_task") as mock_trace,+ ):+ mock_sessionmaker.return_value.begin.side_effect = error+ with pytest.raises(DBAPIError):+ retrieval._on_retrieval_end(flask_app=app, documents=[doc])++ assert mock_sessionmaker.return_value.begin.call_count == 1+ mock_trace.assert_not_called()++ def test_on_retrieval_end_reraises_after_deadlock_retries_are_exhausted(self, retrieval: DatasetRetrieval) -> None:+ app = Flask(__name__)+ deadlock = DBAPIError(+ "UPDATE document_segments",+ {},+ SimpleNamespace(sqlstate="40P01"),+ connection_invalidated=False,+ )+ doc = _doc(provider="dify")++ with (+ patch("core.rag.retrieval.dataset_retrieval.db", SimpleNamespace(engine=self.orm_engine)),+ patch("core.rag.retrieval.dataset_retrieval.sessionmaker") as mock_sessionmaker,+ patch.object(retrieval, "_send_trace_task") as mock_trace,+ ):+ mock_sessionmaker.return_value.begin.side_effect = deadlock+ with pytest.raises(DBAPIError):+ retrieval._on_retrieval_end(flask_app=app, documents=[doc])++ assert mock_sessionmaker.return_value.begin.call_count == 3+ mock_trace.assert_not_called()def test_retriever_variants(self, retrieval: DatasetRetrieval) -> None:flask_app = SimpleNamespace(app_context=lambda: nullcontext())