Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): reject negative segmentation overlap before indexing (#45455)

ChengFa312c2c28a3cd
diff --git a/api/core/rag/entities/processing_entities.py b/api/core/rag/entities/processing_entities.py
index 3a568062f40..898f3d5b25c 100644
--- a/api/core/rag/entities/processing_entities.py
+++ b/api/core/rag/entities/processing_entities.py
@@ -27,7 +27,7 @@ class Segmentation(BaseModel):
validation_alias=AliasChoices("separator", "delimiter"),
)
max_tokens: int = Field(description="Maximum token count per chunk.")
- chunk_overlap: int = Field(default=0, description="Token overlap between chunks.")
+ chunk_overlap: int = Field(default=0, ge=0, description="Token overlap between chunks.")
class Rule(BaseModel):
diff --git a/api/tests/unit_tests/core/rag/entities/test_processing_entities.py b/api/tests/unit_tests/core/rag/entities/test_processing_entities.py
new file mode 100644
index 00000000000..6dba26acfe1
--- /dev/null
+++ b/api/tests/unit_tests/core/rag/entities/test_processing_entities.py
@@ -0,0 +1,27 @@
+import pytest
+from pydantic import ValidationError
+
+from core.rag.entities.processing_entities import Rule
+
+
[email protected]("field", ["segmentation", "subchunk_segmentation"])
+def test_processing_rule_rejects_negative_chunk_overlap(field: str) -> None:
+ with pytest.raises(ValidationError) as exc_info:
+ Rule.model_validate({field: {"max_tokens": 50, "chunk_overlap": -1}})
+
+ assert exc_info.value.errors()[0]["loc"] == (field, "chunk_overlap")
+
+
[email protected]("overlap", [0, 10])
+def test_processing_rule_accepts_nonnegative_chunk_overlap(overlap: int) -> None:
+ rule = Rule.model_validate(
+ {
+ "segmentation": {"max_tokens": 50, "chunk_overlap": overlap},
+ "subchunk_segmentation": {"max_tokens": 25, "chunk_overlap": overlap},
+ }
+ )
+
+ assert rule.segmentation is not None
+ assert rule.segmentation.chunk_overlap == overlap
+ assert rule.subchunk_segmentation is not None
+ assert rule.subchunk_segmentation.chunk_overlap == overlap
diff --git a/packages/contracts/generated/api/console/datasets/zod.gen.ts b/packages/contracts/generated/api/console/datasets/zod.gen.ts
index 636b949a454..abbd597bf66 100644
--- a/packages/contracts/generated/api/console/datasets/zod.gen.ts
+++ b/packages/contracts/generated/api/console/datasets/zod.gen.ts
@@ -989,7 +989,7 @@ export const zRerankingModel = z.object({
* Segmentation
*/
export const zSegmentation = z.object({
- chunk_overlap: z.int().optional().default(0),
+ chunk_overlap: z.int().gte(0).optional().default(0),
max_tokens: z.int(),
separator: z.string().optional().default('\n'),
})
diff --git a/packages/contracts/generated/api/service/zod.gen.ts b/packages/contracts/generated/api/service/zod.gen.ts
index ead033638a9..eb644d357e3 100644
--- a/packages/contracts/generated/api/service/zod.gen.ts
+++ b/packages/contracts/generated/api/service/zod.gen.ts
@@ -4802,7 +4802,7 @@ export const zSegmentUpdatePayload = z.object({
* Segmentation
*/
export const zSegmentation = z.object({
- chunk_overlap: z.int().optional().default(0),
+ chunk_overlap: z.int().gte(0).optional().default(0),
max_tokens: z.int(),
separator: z.string().optional().default('\n'),
})