Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(knowledge-fs): serialize TiDB vector initialization (#44845)
Jyongc0fa7b26c46d
diff --git a/api/controllers/inner_api/knowledge_fs/vectors.py b/api/controllers/inner_api/knowledge_fs/vectors.pyindex 65d46902374..7acadf5466f 100644--- a/api/controllers/inner_api/knowledge_fs/vectors.py+++ b/api/controllers/inner_api/knowledge_fs/vectors.py@@ -12,6 +12,7 @@ from fields.base import ResponseModelfrom libs.exception import BaseHTTPExceptionfrom libs.helper import dump_responsefrom services.knowledge_fs.vector_store import (+ VectorCollectionPendingError,VectorPoint,VectorRequest,admit_vector_request,@@ -63,7 +64,7 @@ class KnowledgeFSVectorApi(Resource):payload.scope.tenant_id, allow_create=payload.operation == "upsert") as client:result = execute_vector_request(client, payload, check_admission=False)- except TidbBindingPendingError:+ except (TidbBindingPendingError, VectorCollectionPendingError):raise ServiceUnavailable("KnowledgeFS vector backend is being provisioned", retry_after=5) from Noneexcept VectorSpaceAdmissionUnavailableError as error:raise KnowledgeFSVectorQuotaUnavailableError(str(error)) from Nonediff --git a/api/services/knowledge_fs/vector_store.py b/api/services/knowledge_fs/vector_store.pyindex 15c6be1f59d..e382cd67ab6 100644--- a/api/services/knowledge_fs/vector_store.py+++ b/api/services/knowledge_fs/vector_store.py@@ -8,7 +8,7 @@ import hashlibimport jsonimport mathfrom concurrent.futures import ThreadPoolExecutor-from contextlib import contextmanager+from contextlib import ExitStack, contextmanagerfrom typing import TYPE_CHECKING, Annotated, Any, Literal, TypedDictfrom pydantic import AfterValidator, BaseModel, ConfigDict, Field, FiniteFloat, model_validator@@ -17,7 +17,7 @@ if TYPE_CHECKING:from collections.abc import Generatorfrom qdrant_client import QdrantClient- from qdrant_client.http.models import Record+ from qdrant_client.http.models import CollectionInfo, Recordfrom services.knowledge_fs.vector_store_elasticsearch import ElasticsearchVectorStorefrom services.knowledge_fs.vector_store_weaviate import WeaviateVectorStore@@ -105,6 +105,10 @@ class VectorStoreUnavailableError(Exception):"""Safe error; must not include native exceptions containing backend secrets."""+class VectorCollectionPendingError(VectorStoreUnavailableError):+ """Another writer owns collection initialization; retry the same admitted batch."""++class VectorMatch(TypedDict):id: strscore: float@@ -236,57 +240,100 @@ def execute_vector_request(}-def _execute_qdrant_collection(client: "QdrantClient", payload: VectorRequest, collection: str) -> VectorResult | None:- """Return None only for a missing collection; all uncertain I/O must fail."""-- from qdrant_client.http import models+def _read_qdrant_collection(client: "QdrantClient", collection: str) -> "CollectionInfo | None":from qdrant_client.http.exceptions import UnexpectedResponse- from configs import dify_config-- scope = payload.scope- point_ids: list[int | str] = list(payload.ids)- tidb = dify_config.VECTOR_STORE == "tidb_on_qdrant"try:- info = client.get_collection(collection)+ return client.get_collection(collection)except UnexpectedResponse as error:if error.status_code != 404:raise- if payload.operation != "upsert":- return None- try:- client.create_collection(- collection_name=collection,- vectors_config=models.VectorParams(size=scope.dimension, distance=models.Distance.COSINE),- )- except Exception:- # Concurrent first writers can race. Accept only a compatible existing- # collection; never recreate or clear another writer's collection.- info = client.get_collection(collection)- else:- info = client.get_collection(collection)- vectors = info.config.params.vectors- if (- not isinstance(vectors, models.VectorParams)- or vectors.size != scope.dimension- or vectors.distance != models.Distance.COSINE- ):- raise VectorStoreUnavailableError("KnowledgeFS vector collection has incompatible dimensions or distance")+ return None- id_index = info.payload_schema.get(TIDB_POINT_ID_FIELD)- if (- tidb- and payload.operation in {"upsert", "search"}- and id_index is not None- and id_index.data_type != models.PayloadSchemaType.KEYWORD- ):- raise VectorStoreUnavailableError("KnowledgeFS vector ID index has incompatible schema")- if payload.operation == "upsert":- if tidb and TIDB_POINT_ID_FIELD not in info.payload_schema and info.points_count == 0:+def _prepare_qdrant_collection(+ client: "QdrantClient", payload: VectorRequest, collection: str+) -> "CollectionInfo | None":+ """Serialize TiDB schema initialization across API processes, never normal vector I/O."""+ from qdrant_client.http import models+ from redis.exceptions import LockError++ from configs import dify_config+ from extensions.ext_redis import redis_client++ tidb = dify_config.VECTOR_STORE == "tidb_on_qdrant"+ upsert = payload.operation == "upsert"+ info = _read_qdrant_collection(client, collection)+ if info is None and not upsert:+ return None+ with ExitStack() as initialization:+ if (+ tidb+ and upsert+ and (info is None or (info.points_count == 0 and TIDB_POINT_ID_FIELD not in info.payload_schema))+ ):+ # TiDB's concurrent DDL can return 409/500 before the winner's schema+ # is visible. One re-read after an error is not sufficient. Contenders+ # return explicit pending so KnowledgeFS can use its bounded retry.+ try:+ initialization.enter_context(+ redis_client.lock(+ f"knowledge_fs:vector_init:{payload.scope.tenant_id}:{collection}",+ # At most five SDK calls, each bounded to 60 seconds.+ timeout=360,+ blocking_timeout=0,+ )+ )+ except LockError:+ raise VectorCollectionPendingError("KnowledgeFS vector collection is being initialized") from None+ info = _read_qdrant_collection(client, collection)+ if info is None:+ try:+ client.create_collection(+ collection_name=collection,+ vectors_config=models.VectorParams(size=payload.scope.dimension, distance=models.Distance.COSINE),+ )+ except Exception:+ # Accept a concurrent creator only after verifying its schema.+ info = client.get_collection(collection)+ else:+ info = client.get_collection(collection)+ vectors = info.config.params.vectors+ if (+ not isinstance(vectors, models.VectorParams)+ or vectors.size != payload.scope.dimension+ or vectors.distance != models.Distance.COSINE+ ):+ raise VectorStoreUnavailableError("KnowledgeFS vector collection has incompatible dimensions or distance")+ id_index = info.payload_schema.get(TIDB_POINT_ID_FIELD)+ if (+ tidb+ and payload.operation in {"upsert", "search"}+ and id_index is not None+ and id_index.data_type != models.PayloadSchemaType.KEYWORD+ ):+ raise VectorStoreUnavailableError("KnowledgeFS vector ID index has incompatible schema")+ if tidb and upsert and TIDB_POINT_ID_FIELD not in info.payload_schema and info.points_count == 0:# An index on an empty collection proves every subsequent point is# written with our ID payload. Do not mark old, unbackfilled data ready._ensure_qdrant_keyword_index(client, collection, TIDB_POINT_ID_FIELD)+ return info+++def _execute_qdrant_collection(client: "QdrantClient", payload: VectorRequest, collection: str) -> VectorResult | None:+ """Return None only for a missing collection; all uncertain I/O must fail."""+ from qdrant_client.http import models+ from qdrant_client.http.exceptions import UnexpectedResponse++ from configs import dify_config++ info = _prepare_qdrant_collection(client, payload, collection)+ if info is None:+ return None+ point_ids: list[int | str] = list(payload.ids)+ tidb = dify_config.VECTOR_STORE == "tidb_on_qdrant"+ id_index = info.payload_schema.get(TIDB_POINT_ID_FIELD)+ if payload.operation == "upsert":client.upsert(collection_name=collection,wait=True,diff --git a/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_vectors.py b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_vectors.pyindex 14eea1c6ba6..e4eb7ebf338 100644--- a/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_vectors.py+++ b/api/tests/unit_tests/controllers/inner_api/knowledge_fs/test_vectors.py@@ -197,9 +197,13 @@ def test_pending_cluster_returns_explicit_retry_after_without_secrets(app: Flaskassert "private" not in str(error.value)-def test_http_pending_response_preserves_retry_after_header(config_overrides: Callable[..., None]) -> None:[email protected]("pending_kind", ["binding", "collection"])+def test_http_pending_response_preserves_retry_after_header(+ config_overrides: Callable[..., None], pending_kind: str+) -> None:from controllers.inner_api import inner_api_nsfrom libs.external_api import ExternalApi+ from services.knowledge_fs.vector_store import VectorCollectionPendingErrorfrom services.tidb_binding_service import TidbBindingPendingErrorconfig_overrides(PLUGIN_DAEMON_KEY="enabled", INNER_API_KEY_FOR_PLUGIN="trusted-key")@@ -209,7 +213,9 @@ def test_http_pending_response_preserves_retry_after_header(config_overrides: Caapi.add_resource(KnowledgeFSVectorApi, "/vectors")with patch("controllers.inner_api.knowledge_fs.vectors.configured_vector_client",- side_effect=TidbBindingPendingError("private endpoint"),+ side_effect=(TidbBindingPendingError if pending_kind == "binding" else VectorCollectionPendingError)(+ "private endpoint"+ ),):response = http_app.test_client().post("/vectors", json=PAYLOAD, headers={"X-Inner-Api-Key": "trusted-key"})assert response.status_code == 503diff --git a/api/tests/unit_tests/services/knowledge_fs/test_vector_store.py b/api/tests/unit_tests/services/knowledge_fs/test_vector_store.pyindex f001b3ab710..13923d5ff0a 100644--- a/api/tests/unit_tests/services/knowledge_fs/test_vector_store.py+++ b/api/tests/unit_tests/services/knowledge_fs/test_vector_store.py@@ -1,6 +1,7 @@from __future__ import annotations-from collections.abc import Callable, Iterator, Mapping+from collections.abc import Callable, Generator, Iterator, Mapping+from contextlib import contextmanagerfrom dataclasses import dataclass, fieldfrom typing import TYPE_CHECKING, castfrom unittest.mock import patch@@ -1507,6 +1508,102 @@ class TestKnowledgeFsVectorStoreCases:TIDB_POINT_ID_FIELD: _KNOWLEDGE_FS_VECTOR_STORE_A,}+ @staticmethod+ @pytest.mark.parametrize("collection_missing", [False, True])+ def test_tidb_first_writer_rechecks_schema_after_acquiring_initialization_lock(+ tidb_client: TidbHarness, collection_missing: bool+ ) -> None:+ client, info = tidb_client+ scope = VectorScope.model_validate(_KNOWLEDGE_FS_VECTOR_STORE_SCOPE)+ if collection_missing:+ client.collections.clear()+ ready = info.model_copy(deep=True)+ ready.payload_schema[TIDB_POINT_ID_FIELD] = models.PayloadIndexInfo(data_type="keyword", points=0)++ @contextmanager+ def initialize_elsewhere(*_args: object, **_kwargs: object) -> Generator[None]:+ client.collections[scope.collection_name] = ready+ yield++ payload = _knowledge_fs_vector_store_request("upsert", points=[{**POINT, "vector": [1, 0]}])+ with (+ patch("extensions.ext_redis.redis_client.lock", side_effect=initialize_elsewhere),+ patch.object(client, "create_collection", side_effect=RuntimeError("collection creation raced")),+ patch.object(client, "create_payload_index", side_effect=RuntimeError("index creation raced")),+ ):+ client.execute(payload)+ assert len(client.upserted_points) == 1++ @staticmethod+ @pytest.mark.parametrize("collection_missing", [False, True])+ def test_tidb_busy_initialization_does_not_race_schema_or_write_points(+ tidb_client: TidbHarness, collection_missing: bool+ ) -> None:+ from redis.exceptions import LockError++ client, _ = tidb_client+ if collection_missing:+ client.collections.clear()+ payload = _knowledge_fs_vector_store_request("upsert", points=[{**POINT, "vector": [1, 0]}])+ with patch("extensions.ext_redis.redis_client.lock") as lock:+ lock.return_value.__enter__.side_effect = LockError("busy")+ with pytest.raises(VectorStoreUnavailableError, match="initializ"):+ client.execute(payload)+ assert client.create_collection_calls == []+ assert client.create_payload_index_calls == []+ assert client.upsert_calls == []++ @staticmethod+ def test_tidb_ready_collection_does_not_serialize_vector_writes(tidb_client: TidbHarness) -> None:+ client, info = tidb_client+ info.payload_schema[TIDB_POINT_ID_FIELD] = models.PayloadIndexInfo(data_type="keyword", points=1)+ payload = _knowledge_fs_vector_store_request("upsert", points=[{**POINT, "vector": [1, 0]}])+ with patch("extensions.ext_redis.redis_client.lock") as lock:+ client.execute(payload)+ lock.assert_not_called()+ assert len(client.upserted_points) == 1++ @staticmethod+ def test_tidb_initialization_lock_covers_schema_but_releases_before_point_io(tidb_client: TidbHarness) -> None:+ client, info = tidb_client+ scope = VectorScope.model_validate(_KNOWLEDGE_FS_VECTOR_STORE_SCOPE)+ client.collections.clear()+ held = False++ @contextmanager+ def initialization_lock(*_args: object, **_kwargs: object) -> Generator[None]:+ nonlocal held+ held = True+ try:+ yield+ finally:+ held = False++ def create_collection(**_kwargs: object) -> None:+ assert held+ client.collections[scope.collection_name] = info++ def create_index(**_kwargs: object) -> None:+ assert held+ info.payload_schema[TIDB_POINT_ID_FIELD] = models.PayloadIndexInfo(data_type="keyword", points=0)++ def write_points(**_kwargs: object) -> None:+ assert not held++ payload = _knowledge_fs_vector_store_request("upsert", points=[{**POINT, "vector": [1, 0]}])+ with (+ patch("extensions.ext_redis.redis_client.lock", side_effect=initialization_lock) as lock,+ patch.object(client, "create_collection", side_effect=create_collection),+ patch.object(client, "create_payload_index", side_effect=create_index),+ patch.object(client, "upsert", side_effect=write_points) as upsert,+ ):+ client.execute(payload)+ assert scope.tenant_id in lock.call_args.args[0]+ assert scope.collection_name in lock.call_args.args[0]+ assert lock.call_args.kwargs["blocking_timeout"] == 0+ upsert.assert_called_once()+ assert not held+@staticmethod@pytest.mark.parametrize("compatible", [False, True])def test_tidb_payload_index_creation_race_requires_compatible_index(diff --git a/web/features/new-rag/settings/__tests__/form.spec.tsx b/web/features/new-rag/settings/__tests__/form.spec.tsxindex d6e82d0327c..f2d53671338 100644--- a/web/features/new-rag/settings/__tests__/form.spec.tsx+++ b/web/features/new-rag/settings/__tests__/form.spec.tsx@@ -1572,6 +1572,35 @@ describe('KnowledgeSettingsPage workflows', () => {)})+ it.each(['unavailable', 'binding_missing'] as const)(+ 'does not show model setup warnings for an active configuration with a %s publication',+ async (code) => {+ await renderForm({+ settings: {+ ...settings,+ capabilities: {+ ...settings.capabilities,+ deep: false,+ query: false,+ research: false,+ },+ issues: [{ code, field: 'publication', retryable: code === 'unavailable' }],+ },+ })++ expect(screen.queryByRole('status')).not.toBeInTheDocument()+ expect(+ screen.getByRole('button', { name: 'knowledgeSettings.settings.embeddingModelLabel' }),+ ).toHaveTextContent(settings.embedding.model)+ expect(+ screen.getByRole('button', {+ name: 'knowledgeSettings.settings.systemReasoningModelLabel',+ }),+ ).toHaveTextContent(settings.retrieval.reasoning_model.model)+ expectNoWrites()+ },+ )+it('keeps active-profile controls enabled when a replacement candidate fails validation', async () => {await renderForm({settings: {diff --git a/web/features/new-rag/settings/retrieval-settings.tsx b/web/features/new-rag/settings/retrieval-settings.tsxindex 0e66980c6f0..f4a279a8249 100644--- a/web/features/new-rag/settings/retrieval-settings.tsx+++ b/web/features/new-rag/settings/retrieval-settings.tsx@@ -105,7 +105,7 @@ export function RetrievalSettingsSection() {return (<>{settings.configuration_state !== 'pending-validation' &&- (settings.configuration_state !== 'active' || settings.issues.length > 0) && (+ settings.configuration_state !== 'active' && (<KnowledgeModelReadinessNoticeclassName="mb-3"description={