Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: validate declared blob size before allocating buffer in chunk merger (#42021)

Harsh Kashyap75f203cdb998
diff --git a/api/core/plugin/utils/chunk_merger.py b/api/core/plugin/utils/chunk_merger.py
index 941d2082055..66a8e564178 100644
--- a/api/core/plugin/utils/chunk_merger.py
+++ b/api/core/plugin/utils/chunk_merger.py
@@ -55,6 +55,13 @@ def merge_blob_chunks[T: ToolInvokeMessage | AgentInvokeMessage](
# Initialize buffer for this file if it doesn't exist
if chunk_id not in files:
+ # Validate the declared size before allocating the buffer.
+ # total_length comes from the plugin and the whole buffer is
+ # pre-allocated, so an oversized claim would exhaust memory
+ # (or raise an unhandled MemoryError/OverflowError) before the
+ # size limit below is ever checked.
+ if total_length > max_file_size:
+ raise ValueError(f"File is too large which reached the limit of {max_file_size / 1024 / 1024}MB")
files[chunk_id] = FileChunk(total_length)
# Check if file is too large (before appending)
diff --git a/api/tests/unit_tests/core/plugin/utils/test_chunk_merger.py b/api/tests/unit_tests/core/plugin/utils/test_chunk_merger.py
index 00a42077860..d53515239c4 100644
--- a/api/tests/unit_tests/core/plugin/utils/test_chunk_merger.py
+++ b/api/tests/unit_tests/core/plugin/utils/test_chunk_merger.py
@@ -546,3 +546,31 @@ class TestConverter:
assert converted["empty_list"] == []
assert converted["mixed_list"] == [file_one, "raw"]
assert converted["none_value"] is None
+
+ def test_merge_blob_chunks_rejects_declared_size_above_limit(self):
+ """A declared total_length above the limit is rejected before allocation."""
+
+ def mock_generator() -> Generator[ToolInvokeMessage, None, None]:
+ yield ToolInvokeMessage(
+ type=ToolInvokeMessage.MessageType.BLOB_CHUNK,
+ message=ToolInvokeMessage.BlobChunkMessage(
+ id="file1", sequence=0, total_length=1025, blob=b"x", end=False
+ ),
+ )
+
+ with pytest.raises(ValueError, match="File is too large"):
+ list(merge_blob_chunks(mock_generator(), max_file_size=1024))
+
+ def test_merge_blob_chunks_rejects_absurd_declared_size(self):
+ """An absurd declared total_length raises ValueError, not MemoryError/OverflowError."""
+
+ def mock_generator() -> Generator[ToolInvokeMessage, None, None]:
+ yield ToolInvokeMessage(
+ type=ToolInvokeMessage.MessageType.BLOB_CHUNK,
+ message=ToolInvokeMessage.BlobChunkMessage(
+ id="file1", sequence=0, total_length=2**62, blob=b"x", end=False
+ ),
+ )
+
+ with pytest.raises(ValueError, match="File is too large"):
+ list(merge_blob_chunks(mock_generator()))