RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: normalize legacy parser configuration (#17761)
buua4363e1191414486
diff --git a/api/apps/restful_apis/document_api.py b/api/apps/restful_apis/document_api.pyindex cb1a575de7..195fcd72d1 100644--- a/api/apps/restful_apis/document_api.py+++ b/api/apps/restful_apis/document_api.py@@ -79,6 +79,34 @@ from common.ssrf_guard import assert_url_is_safefrom rag.nlp import search+def _normalize_legacy_raptor_config(req: dict) -> None:+ """Drop RAPTOR fields removed from the current parser-config schema."""+ parser_config = req.get("parser_config")+ if not isinstance(parser_config, dict):+ return+ raptor = parser_config.get("raptor")+ if not isinstance(raptor, dict):+ return++ normalized_fields = []+ legacy_ext = raptor.pop("ext", None)+ if legacy_ext is not None:+ normalized_fields.append("ext")+ if isinstance(legacy_ext, dict) and "clustering_threshold" in legacy_ext and "clustering_threshold" not in raptor:+ raptor["clustering_threshold"] = legacy_ext["clustering_threshold"]+ normalized_fields.append("ext.clustering_threshold")+ for field in ("threshold", "clustering_method", "tree_builder"):+ if field in raptor:+ raptor.pop(field)+ normalized_fields.append(field)+ max_token = raptor.get("max_token")+ if isinstance(max_token, (int, float)) and not isinstance(max_token, bool) and max_token < 512:+ raptor["max_token"] = 512+ normalized_fields.append("max_token")+ if normalized_fields:+ logging.debug("Document RAPTOR config normalized legacy fields: %s", sorted(normalized_fields))++def _normalize_parser_config_compilation_template_group_ids(parser_config) -> bool:from rag.svr.task_executor_refactor.chunk_post_processor import (_parser_config_compilation_template_group_ids,@@ -212,6 +240,7 @@ async def update_document(tenant_id, dataset_id, document_id):type: object"""req = await get_request_json()+ _normalize_legacy_raptor_config(req)# An explicit null name is a type error, not an unset field.if "name" in req and req["name"] is None:diff --git a/api/db/services/llm_service.py b/api/db/services/llm_service.pyindex 148cba7373..9537e260da 100644--- a/api/db/services/llm_service.py+++ b/api/db/services/llm_service.py@@ -27,7 +27,7 @@ from langfuse import propagate_attributesfrom api.db.db_models import LLMfrom api.db.services.common_service import CommonServicefrom api.db.services.tenant_llm_service import LLM4Tenant-from common.token_utils import num_tokens_from_string, record_run_token_usage, langfuse_run_attrs+from common.token_utils import langfuse_run_attrs, num_tokens_from_string, record_run_token_usage, truncateclass LLMService(CommonService):@@ -144,7 +144,14 @@ class LLMBundle(LLM4Tenant):token_size = num_tokens_from_string(text)if token_size > self.max_length * 0.95:target_len = int(self.max_length * 0.95)- safe_texts.append(text[:target_len])+ logging.debug(+ "LLMBundle.encode truncating input: index=%d model=%s original_tokens=%d target_tokens=%d",+ idx,+ self.model_config["llm_name"],+ token_size,+ target_len,+ )+ safe_texts.append(truncate(text, target_len))else:safe_texts.append(text)diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.pyindex cc3e7cdcc9..4a54756e73 100644--- a/api/utils/validation_utils.py+++ b/api/utils/validation_utils.py@@ -371,6 +371,35 @@ class RaptorConfig(Base):scope: Annotated[Literal["file", "dataset"], Field(default="file")]auto_disable_for_structured_data: Annotated[bool, Field(default=True)]+ @model_validator(mode="before")+ @classmethod+ def normalize_legacy_fields(cls, value: Any) -> Any:+ """Accept old RAPTOR fields but do not retain them in the config."""+ if not isinstance(value, dict):+ return value++ normalized = dict(value)+ changed_fields = []+ legacy_ext = normalized.pop("ext", None)+ if legacy_ext is not None:+ changed_fields.append("ext")+ if isinstance(legacy_ext, dict) and normalized.get("clustering_threshold") is None:+ if "clustering_threshold" in legacy_ext:+ normalized["clustering_threshold"] = legacy_ext["clustering_threshold"]+ changed_fields.append("ext.clustering_threshold")++ for field in ("threshold", "clustering_method", "tree_builder"):+ if field in normalized:+ normalized.pop(field)+ changed_fields.append(field)+ max_token = normalized.get("max_token")+ if isinstance(max_token, (int, float)) and not isinstance(max_token, bool) and max_token < 512:+ normalized["max_token"] = 512+ changed_fields.append("max_token")+ if changed_fields:+ logging.debug("RaptorConfig normalized legacy fields: %s", sorted(changed_fields))+ return normalized+class GraphragConfig(Base):"""Dataset parser configuration for GraphRAG generation."""diff --git a/rag/llm/embedding_model.py b/rag/llm/embedding_model.pyindex 560b324d6a..a1c137dfe8 100644--- a/rag/llm/embedding_model.py+++ b/rag/llm/embedding_model.py@@ -973,7 +973,41 @@ class SILICONFLOWEmbed(Base):def _clean_batch(self, batch):if self.model_name in ["BAAI/bge-large-zh-v1.5", "BAAI/bge-large-en-v1.5"]:# limit 512, 340 is almost safe- return [" " if not text.strip() else truncate(text, 256) for text in batch]+ limit = 256+ cleaned = []+ for index, text in enumerate(batch):+ if not text.strip():+ cleaned.append(" ")+ continue+ original_tokens = num_tokens_from_string(text)+ if original_tokens > limit:+ logger.debug(+ "Embedding input truncated: model=%s input_index=%d original_tokens=%d target_tokens=%d",+ self.model_name,+ index,+ original_tokens,+ limit,+ )+ cleaned.append(truncate(text, limit))+ return cleaned+ if self.model_name in ["BAAI/bge-m3", "Pro/BAAI/bge-m3"]:+ limit = 4096+ cleaned = []+ for index, text in enumerate(batch):+ if not text.strip():+ cleaned.append(" ")+ continue+ original_tokens = num_tokens_from_string(text)+ if original_tokens > limit:+ logger.debug(+ "Embedding input truncated: model=%s input_index=%d original_tokens=%d target_tokens=%d",+ self.model_name,+ index,+ original_tokens,+ limit,+ )+ cleaned.append(truncate(text, limit))+ return cleanedreturn [" " if not text.strip() else text for text in batch]def _call(self, batch):diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.pyindex f4f8d22145..b1f8d70cad 100644--- a/test/testcases/restful_api/test_datasets.py+++ b/test/testcases/restful_api/test_datasets.py@@ -1020,9 +1020,7 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),({"raptor": {"prompt": ""}}, "String should have at least 1 character"),({"raptor": {"prompt": " "}}, "String should have at least 1 character"),- ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),- ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),@@ -1745,9 +1743,7 @@ def test_dataset_create_parser_config_invalid_contract(rest_client, clear_datase("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"),("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"),- ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),- ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),diff --git a/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py b/test/testcases/test_http_api/test_dataset_management/test_create_dataset.pyindex ed2c4319cf..68787962d2 100644--- a/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py+++ b/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py@@ -563,9 +563,7 @@ class TestDatasetCreate:("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"),("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"),- ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"),("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),- ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),@@ -621,9 +619,7 @@ class TestDatasetCreate:"raptor_type_invalid","raptor_prompt_empty","raptor_prompt_space",- "raptor_max_token_min_limit","raptor_max_token_max_limit",- "raptor_max_token_float_not_allowed","raptor_max_token_type_invalid","raptor_clustering_threshold_min_limit","raptor_clustering_threshold_max_limit",diff --git a/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py b/test/testcases/test_http_api/test_dataset_management/test_update_dataset.pyindex d216b35793..65fe4b2441 100644--- a/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py+++ b/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py@@ -693,9 +693,7 @@ class TestDatasetUpdate:({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),({"raptor": {"prompt": ""}}, "String should have at least 1 character"),({"raptor": {"prompt": " "}}, "String should have at least 1 character"),- ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"),({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),- ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),@@ -749,9 +747,7 @@ class TestDatasetUpdate:"raptor_type_invalid","raptor_prompt_empty","raptor_prompt_space",- "raptor_max_token_min_limit","raptor_max_token_max_limit",- "raptor_max_token_float_not_allowed","raptor_max_token_type_invalid","raptor_clustering_threshold_min_limit","raptor_clustering_threshold_max_limit",diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.pyindex ab626f0b48..8a60697e14 100644--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py@@ -512,9 +512,7 @@ class TestDatasetCreate:("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"),("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"),- ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),- ("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),@@ -568,9 +566,7 @@ class TestDatasetCreate:"raptor_type_invalid","raptor_prompt_empty","raptor_prompt_space",- "raptor_max_token_min_limit","raptor_max_token_max_limit",- "raptor_max_token_float_not_allowed","raptor_max_token_type_invalid","raptor_clustering_threshold_min_limit","raptor_clustering_threshold_max_limit",diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.pyindex daba719c6c..4047ec9b26 100644--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py@@ -581,9 +581,7 @@ class TestDatasetUpdate:({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),({"raptor": {"prompt": ""}}, "String should have at least 1 character"),({"raptor": {"prompt": " "}}, "String should have at least 1 character"),- ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),- ({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),@@ -637,9 +635,7 @@ class TestDatasetUpdate:"raptor_type_invalid","raptor_prompt_empty","raptor_prompt_space",- "raptor_max_token_min_limit","raptor_max_token_max_limit",- "raptor_max_token_float_not_allowed","raptor_max_token_type_invalid","raptor_clustering_threshold_min_limit","raptor_clustering_threshold_max_limit",