Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): report batch count as the progress denominator in Vector (#43005)

huoex1222-hub1366cadbb4c9
diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py
index 5fbc55ffd91..0806dfb2745 100644
--- a/api/core/rag/datasource/vdb/vector_factory.py
+++ b/api/core/rag/datasource/vdb/vector_factory.py
@@ -180,7 +180,7 @@ class Vector:
start = time.time()
logger.info("start embedding %s texts %s", len(texts), start)
batch_size = 1000
- total_batches = len(texts) + batch_size - 1
+ total_batches = (len(texts) + batch_size - 1) // batch_size
for i in range(0, len(texts), batch_size):
batch = texts[i : i + batch_size]
batch_start = time.time()
@@ -197,7 +197,7 @@ class Vector:
start = time.time()
logger.info("start embedding %s files %s", len(file_documents), start)
batch_size = 1000
- total_batches = len(file_documents) + batch_size - 1
+ total_batches = (len(file_documents) + batch_size - 1) // batch_size
for i in range(0, len(file_documents), batch_size):
batch = file_documents[i : i + batch_size]
batch_start = time.time()
diff --git a/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py b/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py
index afa3b56d8fe..5ed354d15ac 100644
--- a/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py
+++ b/api/tests/unit_tests/core/rag/datasource/vdb/test_vector_factory.py
@@ -1,5 +1,6 @@
import base64
import json
+import logging
import sys
import types
from datetime import UTC, datetime
@@ -358,6 +359,28 @@ def test_create_batches_texts_and_skips_empty_input(vector_factory_module):
vector._vector_processor.create.assert_not_called()
+def test_create_logs_batch_count_as_progress_denominator(vector_factory_module, caplog):
+ """Progress logs must report the batch count, not a text count with the raw remainder."""
+ vector = vector_factory_module.Vector.__new__(vector_factory_module.Vector)
+ vector._embeddings = MagicMock()
+ vector._embeddings.embed_documents.side_effect = [
+ [[0.1] for _ in range(1000)],
+ [[0.2] for _ in range(500)],
+ ]
+ vector._vector_processor = MagicMock()
+
+ docs = [Document(page_content=f"doc-{i}", metadata={"doc_id": f"id-{i}"}) for i in range(1500)]
+
+ with caplog.at_level(logging.INFO):
+ vector.create(texts=docs)
+
+ progress = [record.getMessage() for record in caplog.records if "Processing batch" in record.getMessage()]
+ assert progress == [
+ "Processing batch 1/2 (1000 texts)",
+ "Processing batch 2/2 (500 texts)",
+ ]
+
+
def test_create_skips_empty_text_documents_before_embedding(vector_factory_module):
vector = vector_factory_module.Vector.__new__(vector_factory_module.Vector)
vector._embeddings = MagicMock()