Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): keep embedding vectors aligned when one in a batch is dropped (#45569)

lianbie686b51f0d7f5
diff --git a/api/core/rag/embedding/cached_embedding.py b/api/core/rag/embedding/cached_embedding.py
index 91d228b92c6..95b0b2a7827 100644
--- a/api/core/rag/embedding/cached_embedding.py
+++ b/api/core/rag/embedding/cached_embedding.py
@@ -70,7 +70,6 @@ class CacheEmbedding(Embeddings):
if embedding_queue_indices:
embedding_queue_texts = [texts[i] for i in embedding_queue_indices]
- embedding_queue_embeddings = []
try:
model_type_instance = cast(TextEmbeddingModel, self._model_instance.model_type_instance)
model_schema = model_type_instance.get_model_schema(
@@ -88,7 +87,12 @@ class CacheEmbedding(Embeddings):
texts=batch_texts, input_type=EmbeddingInputType.DOCUMENT
)
- for vector in embedding_result.embeddings:
+ # Each vector belongs to a specific text. A vector that cannot be used
+ # is dropped where it was produced, so collecting the usable vectors in a
+ # separate list and zipping that list with the indices afterwards would
+ # pair every later text with its neighbour's vector.
+ batch_indices = embedding_queue_indices[i : i + len(batch_texts)]
+ for queue_index, vector in zip(batch_indices, embedding_result.embeddings, strict=False):
try:
# FIXME: type ignore for numpy here
normalized_embedding = (vector / np.linalg.norm(vector)).tolist() # type: ignore
@@ -97,16 +101,18 @@ class CacheEmbedding(Embeddings):
# for issue #11827 float values are not json compliant
logger.warning("Normalized embedding is nan: %s", normalized_embedding)
continue
- embedding_queue_embeddings.append(normalized_embedding)
+ text_embeddings[queue_index] = normalized_embedding
except IntegrityError:
db.session.rollback()
except Exception:
logger.exception("Failed transform embedding")
cache_embeddings = []
try:
- for i, n_embedding in zip(embedding_queue_indices, embedding_queue_embeddings, strict=False):
- text_embeddings[i] = n_embedding
- hash = helper.generate_text_hash(texts[i])
+ for queue_index in embedding_queue_indices:
+ n_embedding = text_embeddings[queue_index]
+ if n_embedding is None:
+ continue
+ hash = helper.generate_text_hash(texts[queue_index])
if hash not in cache_embeddings:
embedding_cache = Embedding(
model_name=self._model_instance.model_name,
@@ -152,7 +158,6 @@ class CacheEmbedding(Embeddings):
if embedding_queue_indices:
embedding_queue_multimodel_documents = [multimodel_documents[i] for i in embedding_queue_indices]
- embedding_queue_embeddings = []
try:
model_type_instance = cast(TextEmbeddingModel, self._model_instance.model_type_instance)
model_schema = model_type_instance.get_model_schema(
@@ -171,7 +176,11 @@ class CacheEmbedding(Embeddings):
input_type=EmbeddingInputType.DOCUMENT,
)
- for vector in embedding_result.embeddings:
+ # Each vector belongs to a specific file. A vector that cannot be used is
+ # dropped where it was produced; collecting the usable vectors separately
+ # and zipping them with the indices afterwards would shift every later file.
+ batch_indices = embedding_queue_indices[i : i + len(batch_multimodel_documents)]
+ for queue_index, vector in zip(batch_indices, embedding_result.embeddings, strict=False):
try:
# FIXME: type ignore for numpy here
normalized_embedding = (vector / np.linalg.norm(vector)).tolist() # type: ignore
@@ -180,16 +189,18 @@ class CacheEmbedding(Embeddings):
# for issue #11827 float values are not json compliant
logger.warning("Normalized embedding is nan: %s", normalized_embedding)
continue
- embedding_queue_embeddings.append(normalized_embedding)
+ multimodel_embeddings[queue_index] = normalized_embedding
except IntegrityError:
db.session.rollback()
except Exception:
logger.exception("Failed transform embedding")
cache_embeddings = []
try:
- for i, n_embedding in zip(embedding_queue_indices, embedding_queue_embeddings, strict=False):
- multimodel_embeddings[i] = n_embedding
- file_id = multimodel_documents[i]["file_id"]
+ for queue_index in embedding_queue_indices:
+ n_embedding = multimodel_embeddings[queue_index]
+ if n_embedding is None:
+ continue
+ file_id = multimodel_documents[queue_index]["file_id"]
if file_id not in cache_embeddings:
embedding_cache = Embedding(
model_name=self._model_instance.model_name,
diff --git a/api/tests/unit_tests/core/rag/embedding/test_cached_embedding.py b/api/tests/unit_tests/core/rag/embedding/test_cached_embedding.py
index 8748bccf54d..9084a421798 100644
--- a/api/tests/unit_tests/core/rag/embedding/test_cached_embedding.py
+++ b/api/tests/unit_tests/core/rag/embedding/test_cached_embedding.py
@@ -281,6 +281,52 @@ class TestCachedEmbeddingCases:
assert any(record.levelno == logging.WARNING for record in caplog.records)
+ def test_multimodal_nan_vector_does_not_shift_the_files_after_it(
+ self,
+ mock_model_instance: Mock,
+ embedding_session: Session,
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """A dropped multimodal vector must not be handed to the next file."""
+ cache_embedding = CacheEmbedding(mock_model_instance)
+ documents = [{"file_id": "alpha"}, {"file_id": "beta"}, {"file_id": "gamma"}]
+
+ alpha_vector = np.zeros(4)
+ alpha_vector[0] = 1.0
+ gamma_vector = np.zeros(4)
+ gamma_vector[2] = 1.0
+ nan_vector = np.full(4, np.nan)
+
+ usage = EmbeddingUsage(
+ tokens=30,
+ total_tokens=30,
+ unit_price=Decimal("0.0001"),
+ price_unit=Decimal(1000),
+ total_price=Decimal("0.000003"),
+ currency="USD",
+ latency=0.5,
+ )
+ mock_model_instance.invoke_multimodal_embedding.return_value = EmbeddingResult(
+ model="vision-embedding-model",
+ embeddings=[alpha_vector, nan_vector, gamma_vector],
+ usage=usage,
+ )
+
+ with caplog.at_level(logging.WARNING, logger="core.rag.embedding.cached_embedding"):
+ result = cache_embedding.embed_multimodal_documents(documents)
+
+ assert result[0] == pytest.approx(alpha_vector.tolist())
+ assert result[1] is None
+ assert result[2] == pytest.approx(gamma_vector.tolist())
+
+ stored: dict[str, list[float]] = {
+ row.hash: row.get_embedding() for row in embedding_session.scalars(select(Embedding)).all()
+ }
+ assert stored == {
+ "alpha": pytest.approx(alpha_vector.tolist()),
+ "gamma": pytest.approx(gamma_vector.tolist()),
+ }
+
def test_embed_multimodal_documents_large_batch(
self, mock_model_instance: Mock, embedding_session: Session
) -> None:
@@ -795,6 +841,70 @@ class TestEmbeddingServiceCases:
assert "Normalized embedding is nan" in caplog.text
assert sqlite_embedding_session.scalar(select(Embedding)) is None
+ def test_nan_vector_does_not_shift_the_vectors_after_it(
+ self,
+ sqlite_embedding_session: Session,
+ model_instance: Mock,
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """A dropped vector must not be handed to the next text in the batch."""
+ alpha_vector = np.array([1.0, 0.0, 0.0, 0.0])
+ gamma_vector = np.array([0.0, 0.0, 1.0, 0.0])
+ nan_vector = np.array([np.nan, np.nan, np.nan, np.nan])
+ model_instance.invoke_text_embedding.return_value = _result(alpha_vector, nan_vector, gamma_vector)
+
+ with caplog.at_level("WARNING", logger="core.rag.embedding.cached_embedding"):
+ result = CacheEmbedding(model_instance).embed_documents(["alpha", "beta", "gamma"])
+
+ assert result[0] == pytest.approx(alpha_vector.tolist())
+ assert result[1] is None
+ assert result[2] == pytest.approx(gamma_vector.tolist())
+
+ stored: dict[str, list[float]] = {
+ row.hash: row.get_embedding() for row in sqlite_embedding_session.scalars(select(Embedding)).all()
+ }
+ assert stored == {
+ helper.generate_text_hash("alpha"): pytest.approx(alpha_vector.tolist()),
+ helper.generate_text_hash("gamma"): pytest.approx(gamma_vector.tolist()),
+ }
+
+ def test_nan_vector_does_not_shift_later_provider_batches(
+ self,
+ sqlite_embedding_session: Session,
+ model_instance: Mock,
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """A vector dropped in one provider batch must not shift the next batch.
+
+ MAX_CHUNKS is 10 in the fixture, so 12 texts span two provider calls.
+ """
+ documents = [f"doc{i:02d}" for i in range(12)]
+ first_batch = [np.eye(4)[i % 4] for i in range(10)]
+ second_batch = [np.eye(4)[(i + 1) % 4] for i in range(2)]
+ # drop a vector in the middle of the first batch
+ first_batch[5] = np.full(4, np.nan)
+
+ model_instance.invoke_text_embedding.side_effect = [
+ _result(*first_batch),
+ _result(*second_batch),
+ ]
+
+ with caplog.at_level("WARNING", logger="core.rag.embedding.cached_embedding"):
+ result = CacheEmbedding(model_instance).embed_documents(documents)
+
+ assert result[5] is None
+ for index in (0, 4, 6, 9):
+ assert result[index] == pytest.approx(first_batch[index].tolist()), f"text doc{index:02d}"
+ for offset, index in enumerate((10, 11)):
+ assert result[index] == pytest.approx(second_batch[offset].tolist()), f"text {documents[index]}"
+
+ stored: dict[str, list[float]] = {
+ row.hash: row.get_embedding() for row in sqlite_embedding_session.scalars(select(Embedding)).all()
+ }
+ assert len(stored) == 11
+ assert helper.generate_text_hash(documents[5]) not in stored
+ assert stored[helper.generate_text_hash(documents[11])] == pytest.approx(second_batch[1].tolist())
+
def test_duplicate_text_is_cached_once(self, sqlite_embedding_session: Session, model_instance: Mock) -> None:
model_instance.invoke_text_embedding.return_value = _result(_vector(), _vector())