Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: dataset hitcount deadlock (#41979)

FFXNbb2945a1a3a5
diff --git a/api/core/rag/retrieval/dataset_retrieval.py b/api/core/rag/retrieval/dataset_retrieval.py
index a9a3cbb16b9..2caa266c37c 100644
--- a/api/core/rag/retrieval/dataset_retrieval.py
+++ b/api/core/rag/retrieval/dataset_retrieval.py
@@ -11,7 +11,9 @@ from typing import Any, Union, cast
from flask import Flask, current_app
from opentelemetry.trace import get_current_span
from sqlalchemy import and_, func, literal, or_, select, update
+from sqlalchemy.exc import DBAPIError
from sqlalchemy.orm import Session, sessionmaker
+from tenacity import before_sleep_log, retry, retry_if_exception, stop_after_attempt, wait_exponential
from core.app.app_config.entities import (
DatasetEntity,
@@ -107,6 +109,18 @@ default_retrieval_model: DefaultRetrievalModelDict = {
logger = logging.getLogger(__name__)
+_POSTGRES_DEADLOCK_SQLSTATE = "40P01"
+_HIT_COUNT_UPDATE_MAX_ATTEMPTS = 3
+
+
+def _is_postgres_deadlock_error(exc: BaseException) -> bool:
+ if not isinstance(exc, DBAPIError) or exc.orig is None:
+ return False
+ orig = cast(Any, exc.orig)
+ return (hasattr(orig, "sqlstate") and orig.sqlstate == _POSTGRES_DEADLOCK_SQLSTATE) or (
+ hasattr(orig, "pgcode") and orig.pgcode == _POSTGRES_DEADLOCK_SQLSTATE
+ )
+
class DatasetRetrieval:
def __init__(self, application_generate_entity: EasyUIBasedAppGenerateEntity | None = None):
@@ -915,6 +929,13 @@ class DatasetRetrieval:
retrieval_resource_list.append(document)
return retrieval_resource_list
+ @retry(
+ retry=retry_if_exception(_is_postgres_deadlock_error),
+ stop=stop_after_attempt(_HIT_COUNT_UPDATE_MAX_ATTEMPTS),
+ wait=wait_exponential(multiplier=0.05, min=0.05, max=0.1),
+ before_sleep=before_sleep_log(logger, logging.WARNING),
+ reraise=True,
+ )
def _on_retrieval_end(
self,
flask_app: Flask,
@@ -1016,6 +1037,14 @@ class DatasetRetrieval:
# Batch update hit_count for all segments
if segment_ids_to_update:
+ # PostgreSQL does not guarantee that an IN predicate is visited in parameter order.
+ # Lock every target row explicitly and consistently before the multi-row update.
+ session.scalars(
+ select(DocumentSegment.id)
+ .where(DocumentSegment.id.in_(segment_ids_to_update))
+ .order_by(DocumentSegment.id)
+ .with_for_update()
+ ).all()
session.execute(
update(DocumentSegment)
.where(DocumentSegment.id.in_(segment_ids_to_update))
diff --git a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
index 890cc0068ed..461258033ee 100644
--- a/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
+++ b/api/tests/unit_tests/core/rag/retrieval/test_dataset_retrieval.py
@@ -9,10 +9,13 @@ from uuid import uuid4
import pytest
from flask import Flask, current_app
-from sqlalchemy import select
+from sqlalchemy import event, select
+from sqlalchemy.dialects import postgresql
from sqlalchemy.engine import Engine
+from sqlalchemy.exc import DBAPIError
from sqlalchemy.orm import Session
+import core.rag.retrieval.dataset_retrieval as dataset_retrieval_module
from core.app.app_config.entities import (
DatasetEntity,
DatasetRetrieveConfigEntity,
@@ -5477,15 +5480,133 @@ class TestInternalHooksCoverage:
),
]
+ executed_statements: list[object] = []
+
+ def capture_statement(*args: object) -> None:
+ executed_statements.append(args[1])
+
+ event.listen(self.orm_engine, "before_execute", capture_statement)
+ try:
+ with (
+ patch("core.rag.retrieval.dataset_retrieval.db", SimpleNamespace(engine=self.orm_engine)),
+ patch.object(retrieval, "_send_trace_task") as mock_trace,
+ ):
+ retrieval._on_retrieval_end(flask_app=app, documents=docs, message_id="m1", timer={"cost": 1})
+ finally:
+ event.remove(self.orm_engine, "before_execute", capture_statement)
+
+ self.orm_session.expire_all()
+ assert [segment.hit_count for segment in segments] == [1, 1, 1, 1]
+ locking_statements = [
+ statement
+ for statement in executed_statements
+ if "ORDER BY document_segments.id FOR UPDATE" in str(statement.compile(dialect=postgresql.dialect())) # type: ignore[union-attr]
+ ]
+ assert len(locking_statements) == 1
+ mock_trace.assert_called_once()
+
+ def test_on_retrieval_end_retries_postgres_deadlock(self, retrieval: DatasetRetrieval) -> None:
+ app = Flask(__name__)
+ deadlock = DBAPIError(
+ "UPDATE document_segments",
+ {},
+ SimpleNamespace(sqlstate="40P01"),
+ connection_invalidated=False,
+ )
+ doc = _doc(provider="dify")
+ retry_session = MagicMock()
+
with (
patch("core.rag.retrieval.dataset_retrieval.db", SimpleNamespace(engine=self.orm_engine)),
+ patch("core.rag.retrieval.dataset_retrieval.sessionmaker") as mock_sessionmaker,
patch.object(retrieval, "_send_trace_task") as mock_trace,
):
- retrieval._on_retrieval_end(flask_app=app, documents=docs, message_id="m1", timer={"cost": 1})
+ mock_sessionmaker.return_value.begin.side_effect = [deadlock, nullcontext(retry_session)]
+ retrieval._on_retrieval_end(flask_app=app, documents=[doc], message_id="m1", timer={"cost": 1})
- self.orm_session.expire_all()
- assert [segment.hit_count for segment in segments] == [1, 1, 1, 1]
- mock_trace.assert_called_once()
+ assert mock_sessionmaker.return_value.begin.call_count == 2
+ mock_trace.assert_called_once_with("m1", [doc], {"cost": 1})
+
+ @pytest.mark.parametrize(
+ ("error", "expected"),
+ [
+ (RuntimeError("not a database error"), False),
+ (DBAPIError("SELECT 1", {}, None, connection_invalidated=False), False),
+ (
+ DBAPIError(
+ "UPDATE document_segments",
+ {},
+ SimpleNamespace(pgcode="40P01"),
+ connection_invalidated=False,
+ ),
+ True,
+ ),
+ (
+ DBAPIError(
+ "UPDATE document_segments",
+ {},
+ SimpleNamespace(sqlstate="40001", pgcode="40P01"),
+ connection_invalidated=False,
+ ),
+ True,
+ ),
+ (
+ DBAPIError(
+ "UPDATE document_segments",
+ {},
+ SimpleNamespace(sqlstate="40001", pgcode="40001"),
+ connection_invalidated=False,
+ ),
+ False,
+ ),
+ ],
+ )
+ def test_is_postgres_deadlock_error(self, error: BaseException, expected: bool) -> None:
+ assert dataset_retrieval_module._is_postgres_deadlock_error(error) is expected
+
+ def test_on_retrieval_end_reraises_non_deadlock(self, retrieval: DatasetRetrieval) -> None:
+ app = Flask(__name__)
+ error = DBAPIError(
+ "UPDATE document_segments",
+ {},
+ SimpleNamespace(sqlstate="40001"),
+ connection_invalidated=False,
+ )
+ doc = _doc(provider="dify")
+
+ with (
+ patch("core.rag.retrieval.dataset_retrieval.db", SimpleNamespace(engine=self.orm_engine)),
+ patch("core.rag.retrieval.dataset_retrieval.sessionmaker") as mock_sessionmaker,
+ patch.object(retrieval, "_send_trace_task") as mock_trace,
+ ):
+ mock_sessionmaker.return_value.begin.side_effect = error
+ with pytest.raises(DBAPIError):
+ retrieval._on_retrieval_end(flask_app=app, documents=[doc])
+
+ assert mock_sessionmaker.return_value.begin.call_count == 1
+ mock_trace.assert_not_called()
+
+ def test_on_retrieval_end_reraises_after_deadlock_retries_are_exhausted(self, retrieval: DatasetRetrieval) -> None:
+ app = Flask(__name__)
+ deadlock = DBAPIError(
+ "UPDATE document_segments",
+ {},
+ SimpleNamespace(sqlstate="40P01"),
+ connection_invalidated=False,
+ )
+ doc = _doc(provider="dify")
+
+ with (
+ patch("core.rag.retrieval.dataset_retrieval.db", SimpleNamespace(engine=self.orm_engine)),
+ patch("core.rag.retrieval.dataset_retrieval.sessionmaker") as mock_sessionmaker,
+ patch.object(retrieval, "_send_trace_task") as mock_trace,
+ ):
+ mock_sessionmaker.return_value.begin.side_effect = deadlock
+ with pytest.raises(DBAPIError):
+ retrieval._on_retrieval_end(flask_app=app, documents=[doc])
+
+ assert mock_sessionmaker.return_value.begin.call_count == 3
+ mock_trace.assert_not_called()
def test_retriever_variants(self, retrieval: DatasetRetrieval) -> None:
flask_app = SimpleNamespace(app_context=lambda: nullcontext())