Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: normalize legacy parser configuration (#17761)

buua4363e1191414486
diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.py
index cb1a575de7..195fcd72d1 100644
--- a/api/apps/restful_apis/document_api.py
+++ b/api/apps/restful_apis/document_api.py
@@ -79,6 +79,34 @@ from common.ssrf_guard import assert_url_is_safe
from rag.nlp import search
+def _normalize_legacy_raptor_config(req: dict) -> None:
+ """Drop RAPTOR fields removed from the current parser-config schema."""
+ parser_config = req.get("parser_config")
+ if not isinstance(parser_config, dict):
+ return
+ raptor = parser_config.get("raptor")
+ if not isinstance(raptor, dict):
+ return
+
+ normalized_fields = []
+ legacy_ext = raptor.pop("ext", None)
+ if legacy_ext is not None:
+ normalized_fields.append("ext")
+ if isinstance(legacy_ext, dict) and "clustering_threshold" in legacy_ext and "clustering_threshold" not in raptor:
+ raptor["clustering_threshold"] = legacy_ext["clustering_threshold"]
+ normalized_fields.append("ext.clustering_threshold")
+ for field in ("threshold", "clustering_method", "tree_builder"):
+ if field in raptor:
+ raptor.pop(field)
+ normalized_fields.append(field)
+ max_token = raptor.get("max_token")
+ if isinstance(max_token, (int, float)) and not isinstance(max_token, bool) and max_token < 512:
+ raptor["max_token"] = 512
+ normalized_fields.append("max_token")
+ if normalized_fields:
+ logging.debug("Document RAPTOR config normalized legacy fields: %s", sorted(normalized_fields))
+
+
def _normalize_parser_config_compilation_template_group_ids(parser_config) -> bool:
from rag.svr.task_executor_refactor.chunk_post_processor import (
_parser_config_compilation_template_group_ids,
@@ -212,6 +240,7 @@ async def update_document(tenant_id, dataset_id, document_id):
type: object
"""
req = await get_request_json()
+ _normalize_legacy_raptor_config(req)
# An explicit null name is a type error, not an unset field.
if "name" in req and req["name"] is None:
diff --git a/api/db/services/llm_service.py b/api/db/services/llm_service.py
index 148cba7373..9537e260da 100644
--- a/api/db/services/llm_service.py
+++ b/api/db/services/llm_service.py
@@ -27,7 +27,7 @@ from langfuse import propagate_attributes
from api.db.db_models import LLM
from api.db.services.common_service import CommonService
from api.db.services.tenant_llm_service import LLM4Tenant
-from common.token_utils import num_tokens_from_string, record_run_token_usage, langfuse_run_attrs
+from common.token_utils import langfuse_run_attrs, num_tokens_from_string, record_run_token_usage, truncate
class LLMService(CommonService):
@@ -144,7 +144,14 @@ class LLMBundle(LLM4Tenant):
token_size = num_tokens_from_string(text)
if token_size > self.max_length * 0.95:
target_len = int(self.max_length * 0.95)
- safe_texts.append(text[:target_len])
+ logging.debug(
+ "LLMBundle.encode truncating input: index=%d model=%s original_tokens=%d target_tokens=%d",
+ idx,
+ self.model_config["llm_name"],
+ token_size,
+ target_len,
+ )
+ safe_texts.append(truncate(text, target_len))
else:
safe_texts.append(text)
diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py
index cc3e7cdcc9..4a54756e73 100644
--- a/api/utils/validation_utils.py
+++ b/api/utils/validation_utils.py
@@ -371,6 +371,35 @@ class RaptorConfig(Base):
scope: Annotated[Literal["file", "dataset"], Field(default="file")]
auto_disable_for_structured_data: Annotated[bool, Field(default=True)]
+ @model_validator(mode="before")
+ @classmethod
+ def normalize_legacy_fields(cls, value: Any) -> Any:
+ """Accept old RAPTOR fields but do not retain them in the config."""
+ if not isinstance(value, dict):
+ return value
+
+ normalized = dict(value)
+ changed_fields = []
+ legacy_ext = normalized.pop("ext", None)
+ if legacy_ext is not None:
+ changed_fields.append("ext")
+ if isinstance(legacy_ext, dict) and normalized.get("clustering_threshold") is None:
+ if "clustering_threshold" in legacy_ext:
+ normalized["clustering_threshold"] = legacy_ext["clustering_threshold"]
+ changed_fields.append("ext.clustering_threshold")
+
+ for field in ("threshold", "clustering_method", "tree_builder"):
+ if field in normalized:
+ normalized.pop(field)
+ changed_fields.append(field)
+ max_token = normalized.get("max_token")
+ if isinstance(max_token, (int, float)) and not isinstance(max_token, bool) and max_token < 512:
+ normalized["max_token"] = 512
+ changed_fields.append("max_token")
+ if changed_fields:
+ logging.debug("RaptorConfig normalized legacy fields: %s", sorted(changed_fields))
+ return normalized
+
class GraphragConfig(Base):
"""Dataset parser configuration for GraphRAG generation."""
diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.py
index 560b324d6a..a1c137dfe8 100644
--- a/rag/llm/embedding_model.py
+++ b/rag/llm/embedding_model.py
@@ -973,7 +973,41 @@ class SILICONFLOWEmbed(Base):
def _clean_batch(self, batch):
if self.model_name in ["BAAI/bge-large-zh-v1.5", "BAAI/bge-large-en-v1.5"]:
# limit 512, 340 is almost safe
- return [" " if not text.strip() else truncate(text, 256) for text in batch]
+ limit = 256
+ cleaned = []
+ for index, text in enumerate(batch):
+ if not text.strip():
+ cleaned.append(" ")
+ continue
+ original_tokens = num_tokens_from_string(text)
+ if original_tokens > limit:
+ logger.debug(
+ "Embedding input truncated: model=%s input_index=%d original_tokens=%d target_tokens=%d",
+ self.model_name,
+ index,
+ original_tokens,
+ limit,
+ )
+ cleaned.append(truncate(text, limit))
+ return cleaned
+ if self.model_name in ["BAAI/bge-m3", "Pro/BAAI/bge-m3"]:
+ limit = 4096
+ cleaned = []
+ for index, text in enumerate(batch):
+ if not text.strip():
+ cleaned.append(" ")
+ continue
+ original_tokens = num_tokens_from_string(text)
+ if original_tokens > limit:
+ logger.debug(
+ "Embedding input truncated: model=%s input_index=%d original_tokens=%d target_tokens=%d",
+ self.model_name,
+ index,
+ original_tokens,
+ limit,
+ )
+ cleaned.append(truncate(text, limit))
+ return cleaned
return [" " if not text.strip() else text for text in batch]
def _call(self, batch):
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index f4f8d22145..b1f8d70cad 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -1020,9 +1020,7 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase
({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
({"raptor": {"prompt": ""}}, "String should have at least 1 character"),
({"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),
({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
- ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
@@ -1745,9 +1743,7 @@ def test_dataset_create_parser_config_invalid_contract(rest_client, clear_datase
("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"),
("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),
("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
- ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
diff --git a/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py b/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py
index ed2c4319cf..68787962d2 100644
--- a/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py
+++ b/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py
@@ -563,9 +563,7 @@ class TestDatasetCreate:
("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"),
("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"),
("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
- ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
@@ -621,9 +619,7 @@ class TestDatasetCreate:
"raptor_type_invalid",
"raptor_prompt_empty",
"raptor_prompt_space",
- "raptor_max_token_min_limit",
"raptor_max_token_max_limit",
- "raptor_max_token_float_not_allowed",
"raptor_max_token_type_invalid",
"raptor_clustering_threshold_min_limit",
"raptor_clustering_threshold_max_limit",
diff --git a/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py b/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py
index d216b35793..65fe4b2441 100644
--- a/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py
+++ b/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py
@@ -693,9 +693,7 @@ class TestDatasetUpdate:
({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
({"raptor": {"prompt": ""}}, "String should have at least 1 character"),
({"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"),
({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
- ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
@@ -749,9 +747,7 @@ class TestDatasetUpdate:
"raptor_type_invalid",
"raptor_prompt_empty",
"raptor_prompt_space",
- "raptor_max_token_min_limit",
"raptor_max_token_max_limit",
- "raptor_max_token_float_not_allowed",
"raptor_max_token_type_invalid",
"raptor_clustering_threshold_min_limit",
"raptor_clustering_threshold_max_limit",
diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
index ab626f0b48..8a60697e14 100644
--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
@@ -512,9 +512,7 @@ class TestDatasetCreate:
("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"),
("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),
("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
- ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
@@ -568,9 +566,7 @@ class TestDatasetCreate:
"raptor_type_invalid",
"raptor_prompt_empty",
"raptor_prompt_space",
- "raptor_max_token_min_limit",
"raptor_max_token_max_limit",
- "raptor_max_token_float_not_allowed",
"raptor_max_token_type_invalid",
"raptor_clustering_threshold_min_limit",
"raptor_clustering_threshold_max_limit",
diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
index daba719c6c..4047ec9b26 100644
--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
@@ -581,9 +581,7 @@ class TestDatasetUpdate:
({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
({"raptor": {"prompt": ""}}, "String should have at least 1 character"),
({"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),
({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
- ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
@@ -637,9 +635,7 @@ class TestDatasetUpdate:
"raptor_type_invalid",
"raptor_prompt_empty",
"raptor_prompt_space",
- "raptor_max_token_min_limit",
"raptor_max_token_max_limit",
- "raptor_max_token_float_not_allowed",
"raptor_max_token_type_invalid",
"raptor_clustering_threshold_min_limit",
"raptor_clustering_threshold_max_limit",