Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(knowledge-fs): serialize TiDB vector initialization (#44845)

Jyongc0fa7b26c46d
diff --git a/api/controllers/inner_api/knowledge_fs/vectors.py b/api/controllers/inner_api/knowledge_fs/vectors.py
index 65d46902374..7acadf5466f 100644
--- a/api/controllers/inner_api/knowledge_fs/vectors.py
+++ b/api/controllers/inner_api/knowledge_fs/vectors.py
@@ -12,6 +12,7 @@ from fields.base import ResponseModel
from libs.exception import BaseHTTPException
from libs.helper import dump_response
from services.knowledge_fs.vector_store import (
+ VectorCollectionPendingError,
VectorPoint,
VectorRequest,
admit_vector_request,
@@ -63,7 +64,7 @@ class KnowledgeFSVectorApi(Resource):
payload.scope.tenant_id, allow_create=payload.operation == "upsert"
) as client:
result = execute_vector_request(client, payload, check_admission=False)
- except TidbBindingPendingError:
+ except (TidbBindingPendingError, VectorCollectionPendingError):
raise ServiceUnavailable("KnowledgeFS vector backend is being provisioned", retry_after=5) from None
except VectorSpaceAdmissionUnavailableError as error:
raise KnowledgeFSVectorQuotaUnavailableError(str(error)) from None
diff --git a/api/services/knowledge_fs/vector_store.py b/api/services/knowledge_fs/vector_store.py
index 15c6be1f59d..e382cd67ab6 100644
--- a/api/services/knowledge_fs/vector_store.py
+++ b/api/services/knowledge_fs/vector_store.py
@@ -8,7 +8,7 @@ import hashlib
import json
import math
from concurrent.futures import ThreadPoolExecutor
-from contextlib import contextmanager
+from contextlib import ExitStack, contextmanager
from typing import TYPE_CHECKING, Annotated, Any, Literal, TypedDict
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, FiniteFloat, model_validator
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
from collections.abc import Generator
from qdrant_client import QdrantClient
- from qdrant_client.http.models import Record
+ from qdrant_client.http.models import CollectionInfo, Record
from services.knowledge_fs.vector_store_elasticsearch import ElasticsearchVectorStore
from services.knowledge_fs.vector_store_weaviate import WeaviateVectorStore
@@ -105,6 +105,10 @@ class VectorStoreUnavailableError(Exception):
"""Safe error; must not include native exceptions containing backend secrets."""
+class VectorCollectionPendingError(VectorStoreUnavailableError):
+ """Another writer owns collection initialization; retry the same admitted batch."""
+
+
class VectorMatch(TypedDict):
id: str
score: float
@@ -236,57 +240,100 @@ def execute_vector_request(
}
-def _execute_qdrant_collection(client: "QdrantClient", payload: VectorRequest, collection: str) -> VectorResult | None:
- """Return None only for a missing collection; all uncertain I/O must fail."""
-
- from qdrant_client.http import models
+def _read_qdrant_collection(client: "QdrantClient", collection: str) -> "CollectionInfo | None":
from qdrant_client.http.exceptions import UnexpectedResponse
- from configs import dify_config
-
- scope = payload.scope
- point_ids: list[int | str] = list(payload.ids)
- tidb = dify_config.VECTOR_STORE == "tidb_on_qdrant"
try:
- info = client.get_collection(collection)
+ return client.get_collection(collection)
except UnexpectedResponse as error:
if error.status_code != 404:
raise
- if payload.operation != "upsert":
- return None
- try:
- client.create_collection(
- collection_name=collection,
- vectors_config=models.VectorParams(size=scope.dimension, distance=models.Distance.COSINE),
- )
- except Exception:
- # Concurrent first writers can race. Accept only a compatible existing
- # collection; never recreate or clear another writer's collection.
- info = client.get_collection(collection)
- else:
- info = client.get_collection(collection)
- vectors = info.config.params.vectors
- if (
- not isinstance(vectors, models.VectorParams)
- or vectors.size != scope.dimension
- or vectors.distance != models.Distance.COSINE
- ):
- raise VectorStoreUnavailableError("KnowledgeFS vector collection has incompatible dimensions or distance")
+ return None
- id_index = info.payload_schema.get(TIDB_POINT_ID_FIELD)
- if (
- tidb
- and payload.operation in {"upsert", "search"}
- and id_index is not None
- and id_index.data_type != models.PayloadSchemaType.KEYWORD
- ):
- raise VectorStoreUnavailableError("KnowledgeFS vector ID index has incompatible schema")
- if payload.operation == "upsert":
- if tidb and TIDB_POINT_ID_FIELD not in info.payload_schema and info.points_count == 0:
+def _prepare_qdrant_collection(
+ client: "QdrantClient", payload: VectorRequest, collection: str
+) -> "CollectionInfo | None":
+ """Serialize TiDB schema initialization across API processes, never normal vector I/O."""
+ from qdrant_client.http import models
+ from redis.exceptions import LockError
+
+ from configs import dify_config
+ from extensions.ext_redis import redis_client
+
+ tidb = dify_config.VECTOR_STORE == "tidb_on_qdrant"
+ upsert = payload.operation == "upsert"
+ info = _read_qdrant_collection(client, collection)
+ if info is None and not upsert:
+ return None
+ with ExitStack() as initialization:
+ if (
+ tidb
+ and upsert
+ and (info is None or (info.points_count == 0 and TIDB_POINT_ID_FIELD not in info.payload_schema))
+ ):
+ # TiDB's concurrent DDL can return 409/500 before the winner's schema
+ # is visible. One re-read after an error is not sufficient. Contenders
+ # return explicit pending so KnowledgeFS can use its bounded retry.
+ try:
+ initialization.enter_context(
+ redis_client.lock(
+ f"knowledge_fs:vector_init:{payload.scope.tenant_id}:{collection}",
+ # At most five SDK calls, each bounded to 60 seconds.
+ timeout=360,
+ blocking_timeout=0,
+ )
+ )
+ except LockError:
+ raise VectorCollectionPendingError("KnowledgeFS vector collection is being initialized") from None
+ info = _read_qdrant_collection(client, collection)
+ if info is None:
+ try:
+ client.create_collection(
+ collection_name=collection,
+ vectors_config=models.VectorParams(size=payload.scope.dimension, distance=models.Distance.COSINE),
+ )
+ except Exception:
+ # Accept a concurrent creator only after verifying its schema.
+ info = client.get_collection(collection)
+ else:
+ info = client.get_collection(collection)
+ vectors = info.config.params.vectors
+ if (
+ not isinstance(vectors, models.VectorParams)
+ or vectors.size != payload.scope.dimension
+ or vectors.distance != models.Distance.COSINE
+ ):
+ raise VectorStoreUnavailableError("KnowledgeFS vector collection has incompatible dimensions or distance")
+ id_index = info.payload_schema.get(TIDB_POINT_ID_FIELD)
+ if (
+ tidb
+ and payload.operation in {"upsert", "search"}
+ and id_index is not None
+ and id_index.data_type != models.PayloadSchemaType.KEYWORD
+ ):
+ raise VectorStoreUnavailableError("KnowledgeFS vector ID index has incompatible schema")
+ if tidb and upsert and TIDB_POINT_ID_FIELD not in info.payload_schema and info.points_count == 0:
# An index on an empty collection proves every subsequent point is
# written with our ID payload. Do not mark old, unbackfilled data ready.
_ensure_qdrant_keyword_index(client, collection, TIDB_POINT_ID_FIELD)
+ return info
+
+
+def _execute_qdrant_collection(client: "QdrantClient", payload: VectorRequest, collection: str) -> VectorResult | None:
+ """Return None only for a missing collection; all uncertain I/O must fail."""
+ from qdrant_client.http import models
+ from qdrant_client.http.exceptions import UnexpectedResponse
+
+ from configs import dify_config
+
+ info = _prepare_qdrant_collection(client, payload, collection)
+ if info is None:
+ return None
+ point_ids: list[int | str] = list(payload.ids)
+ tidb = dify_config.VECTOR_STORE == "tidb_on_qdrant"
+ id_index = info.payload_schema.get(TIDB_POINT_ID_FIELD)
+ if payload.operation == "upsert":
client.upsert(
collection_name=collection,
wait=True,
diff --git a/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_vectors.py b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_vectors.py
index 14eea1c6ba6..e4eb7ebf338 100644
--- a/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_vectors.py
+++ b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_vectors.py
@@ -197,9 +197,13 @@ def test_pending_cluster_returns_explicit_retry_after_without_secrets(app: Flask
assert "private" not in str(error.value)
-def test_http_pending_response_preserves_retry_after_header(config_overrides: Callable[..., None]) -> None:
[email protected]("pending_kind", ["binding", "collection"])
+def test_http_pending_response_preserves_retry_after_header(
+ config_overrides: Callable[..., None], pending_kind: str
+) -> None:
from controllers.inner_api import inner_api_ns
from libs.external_api import ExternalApi
+ from services.knowledge_fs.vector_store import VectorCollectionPendingError
from services.tidb_binding_service import TidbBindingPendingError
config_overrides(PLUGIN_DAEMON_KEY="enabled", INNER_API_KEY_FOR_PLUGIN="trusted-key")
@@ -209,7 +213,9 @@ def test_http_pending_response_preserves_retry_after_header(config_overrides: Ca
api.add_resource(KnowledgeFSVectorApi, "/vectors")
with patch(
"controllers.inner_api.knowledge_fs.vectors.configured_vector_client",
- side_effect=TidbBindingPendingError("private endpoint"),
+ side_effect=(TidbBindingPendingError if pending_kind == "binding" else VectorCollectionPendingError)(
+ "private endpoint"
+ ),
):
response = http_app.test_client().post("/vectors", json=PAYLOAD, headers={"X-Inner-Api-Key": "trusted-key"})
assert response.status_code == 503
diff --git a/api/tests/unit_tests/services/knowledge_fs/test_vector_store.py b/api/tests/unit_tests/services/knowledge_fs/test_vector_store.py
index f001b3ab710..13923d5ff0a 100644
--- a/api/tests/unit_tests/services/knowledge_fs/test_vector_store.py
+++ b/api/tests/unit_tests/services/knowledge_fs/test_vector_store.py
@@ -1,6 +1,7 @@
from __future__ import annotations
-from collections.abc import Callable, Iterator, Mapping
+from collections.abc import Callable, Generator, Iterator, Mapping
+from contextlib import contextmanager
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, cast
from unittest.mock import patch
@@ -1507,6 +1508,102 @@ class TestKnowledgeFsVectorStoreCases:
TIDB_POINT_ID_FIELD: _KNOWLEDGE_FS_VECTOR_STORE_A,
}
+ @staticmethod
+ @pytest.mark.parametrize("collection_missing", [False, True])
+ def test_tidb_first_writer_rechecks_schema_after_acquiring_initialization_lock(
+ tidb_client: TidbHarness, collection_missing: bool
+ ) -> None:
+ client, info = tidb_client
+ scope = VectorScope.model_validate(_KNOWLEDGE_FS_VECTOR_STORE_SCOPE)
+ if collection_missing:
+ client.collections.clear()
+ ready = info.model_copy(deep=True)
+ ready.payload_schema[TIDB_POINT_ID_FIELD] = models.PayloadIndexInfo(data_type="keyword", points=0)
+
+ @contextmanager
+ def initialize_elsewhere(*_args: object, **_kwargs: object) -> Generator[None]:
+ client.collections[scope.collection_name] = ready
+ yield
+
+ payload = _knowledge_fs_vector_store_request("upsert", points=[{**POINT, "vector": [1, 0]}])
+ with (
+ patch("extensions.ext_redis.redis_client.lock", side_effect=initialize_elsewhere),
+ patch.object(client, "create_collection", side_effect=RuntimeError("collection creation raced")),
+ patch.object(client, "create_payload_index", side_effect=RuntimeError("index creation raced")),
+ ):
+ client.execute(payload)
+ assert len(client.upserted_points) == 1
+
+ @staticmethod
+ @pytest.mark.parametrize("collection_missing", [False, True])
+ def test_tidb_busy_initialization_does_not_race_schema_or_write_points(
+ tidb_client: TidbHarness, collection_missing: bool
+ ) -> None:
+ from redis.exceptions import LockError
+
+ client, _ = tidb_client
+ if collection_missing:
+ client.collections.clear()
+ payload = _knowledge_fs_vector_store_request("upsert", points=[{**POINT, "vector": [1, 0]}])
+ with patch("extensions.ext_redis.redis_client.lock") as lock:
+ lock.return_value.__enter__.side_effect = LockError("busy")
+ with pytest.raises(VectorStoreUnavailableError, match="initializ"):
+ client.execute(payload)
+ assert client.create_collection_calls == []
+ assert client.create_payload_index_calls == []
+ assert client.upsert_calls == []
+
+ @staticmethod
+ def test_tidb_ready_collection_does_not_serialize_vector_writes(tidb_client: TidbHarness) -> None:
+ client, info = tidb_client
+ info.payload_schema[TIDB_POINT_ID_FIELD] = models.PayloadIndexInfo(data_type="keyword", points=1)
+ payload = _knowledge_fs_vector_store_request("upsert", points=[{**POINT, "vector": [1, 0]}])
+ with patch("extensions.ext_redis.redis_client.lock") as lock:
+ client.execute(payload)
+ lock.assert_not_called()
+ assert len(client.upserted_points) == 1
+
+ @staticmethod
+ def test_tidb_initialization_lock_covers_schema_but_releases_before_point_io(tidb_client: TidbHarness) -> None:
+ client, info = tidb_client
+ scope = VectorScope.model_validate(_KNOWLEDGE_FS_VECTOR_STORE_SCOPE)
+ client.collections.clear()
+ held = False
+
+ @contextmanager
+ def initialization_lock(*_args: object, **_kwargs: object) -> Generator[None]:
+ nonlocal held
+ held = True
+ try:
+ yield
+ finally:
+ held = False
+
+ def create_collection(**_kwargs: object) -> None:
+ assert held
+ client.collections[scope.collection_name] = info
+
+ def create_index(**_kwargs: object) -> None:
+ assert held
+ info.payload_schema[TIDB_POINT_ID_FIELD] = models.PayloadIndexInfo(data_type="keyword", points=0)
+
+ def write_points(**_kwargs: object) -> None:
+ assert not held
+
+ payload = _knowledge_fs_vector_store_request("upsert", points=[{**POINT, "vector": [1, 0]}])
+ with (
+ patch("extensions.ext_redis.redis_client.lock", side_effect=initialization_lock) as lock,
+ patch.object(client, "create_collection", side_effect=create_collection),
+ patch.object(client, "create_payload_index", side_effect=create_index),
+ patch.object(client, "upsert", side_effect=write_points) as upsert,
+ ):
+ client.execute(payload)
+ assert scope.tenant_id in lock.call_args.args[0]
+ assert scope.collection_name in lock.call_args.args[0]
+ assert lock.call_args.kwargs["blocking_timeout"] == 0
+ upsert.assert_called_once()
+ assert not held
+
@staticmethod
@pytest.mark.parametrize("compatible", [False, True])
def test_tidb_payload_index_creation_race_requires_compatible_index(
diff --git a/web/features/new-rag/settings/__tests__/form.spec.tsx b/web/features/new-rag/settings/__tests__/form.spec.tsx
index d6e82d0327c..f2d53671338 100644
--- a/web/features/new-rag/settings/__tests__/form.spec.tsx
+++ b/web/features/new-rag/settings/__tests__/form.spec.tsx
@@ -1572,6 +1572,35 @@ describe('KnowledgeSettingsPage workflows', () => {
)
})
+ it.each(['unavailable', 'binding_missing'] as const)(
+ 'does not show model setup warnings for an active configuration with a %s publication',
+ async (code) => {
+ await renderForm({
+ settings: {
+ ...settings,
+ capabilities: {
+ ...settings.capabilities,
+ deep: false,
+ query: false,
+ research: false,
+ },
+ issues: [{ code, field: 'publication', retryable: code === 'unavailable' }],
+ },
+ })
+
+ expect(screen.queryByRole('status')).not.toBeInTheDocument()
+ expect(
+ screen.getByRole('button', { name: 'knowledgeSettings.settings.embeddingModelLabel' }),
+ ).toHaveTextContent(settings.embedding.model)
+ expect(
+ screen.getByRole('button', {
+ name: 'knowledgeSettings.settings.systemReasoningModelLabel',
+ }),
+ ).toHaveTextContent(settings.retrieval.reasoning_model.model)
+ expectNoWrites()
+ },
+ )
+
it('keeps active-profile controls enabled when a replacement candidate fails validation', async () => {
await renderForm({
settings: {
diff --git a/web/features/new-rag/settings/retrieval-settings.tsx b/web/features/new-rag/settings/retrieval-settings.tsx
index 0e66980c6f0..f4a279a8249 100644
--- a/web/features/new-rag/settings/retrieval-settings.tsx
+++ b/web/features/new-rag/settings/retrieval-settings.tsx
@@ -105,7 +105,7 @@ export function RetrievalSettingsSection() {
return (
<>
{settings.configuration_state !== 'pending-validation' &&
- (settings.configuration_state !== 'active' || settings.issues.length > 0) && (
+ settings.configuration_state !== 'active' && (
<KnowledgeModelReadinessNotice
className="mb-3"
description={