RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat: add LLM-guided semantic rechunking for knowledge compilation (#17546)
buua43644e13d1cb6ec
diff --git a/api/db/init_data/compilation_templates/knowledge_graph.yaml b/api/db/init_data/compilation_templates/knowledge_graph.yamlindex 27a8535dd1..c7d4386af8 100644--- a/api/db/init_data/compilation_templates/knowledge_graph.yaml+++ b/api/db/init_data/compilation_templates/knowledge_graph.yaml@@ -2,6 +2,12 @@ kind: knowledge_graphdisplay_name: Knowledge graphconfig:kind: knowledge_graph+ rechunk_rules: |+ - Group only adjacent source chunks into coherent semantic units for this knowledge-compilation task.+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.+ - Split when the topic, section, or semantic focus changes; do not merge unrelated content.+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.entity:description: >-You are a robust graph entity extractor for knowledge graphs.diff --git a/api/db/init_data/compilation_templates/mind_map.yaml b/api/db/init_data/compilation_templates/mind_map.yamlindex 383be48e6e..136d79b44e 100644--- a/api/db/init_data/compilation_templates/mind_map.yaml+++ b/api/db/init_data/compilation_templates/mind_map.yaml@@ -2,6 +2,12 @@ kind: mind_mapdisplay_name: Mind map - Radial concept hierarchyconfig:kind: mind_map+ rechunk_rules: |+ - Group only adjacent source chunks into coherent semantic units for this knowledge-compilation task.+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.+ - Keep each main idea with its supporting content, and split when the topic or branch changes.+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.entity:description: >-You are a robust mind-map extractor. Extract the central concept,diff --git a/api/db/init_data/compilation_templates/page_index.yaml b/api/db/init_data/compilation_templates/page_index.yamlindex f17519daf1..18213b5f00 100644--- a/api/db/init_data/compilation_templates/page_index.yaml+++ b/api/db/init_data/compilation_templates/page_index.yaml@@ -2,6 +2,14 @@ kind: page_indexdisplay_name: PageIndex — Hierarchical table of contentsconfig:kind: page_index+ rechunk_rules: |+ - Group only adjacent source chunks into coherent semantic units for this page-index task.+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.+ - Keep each heading with the content it introduces, and split when the topic, section, or semantic focus changes.+ - Preserve document heading hierarchy: keep a parent heading with its introductory content and nearby child-section context.+ - Do not combine separate sections merely because their subjects are related; keep each section boundary recoverable.+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal rows or items.+ - Do not merge unrelated content. Do not invent, rewrite, or return source text; return only chunk grouping metadata.entity:description: >-You are a source-grounded document index extractor. First extract everydiff --git a/api/db/init_data/compilation_templates/session_essence.yaml b/api/db/init_data/compilation_templates/session_essence.yamlindex dfe7bb6888..62ddef4e4a 100644--- a/api/db/init_data/compilation_templates/session_essence.yaml+++ b/api/db/init_data/compilation_templates/session_essence.yaml@@ -2,6 +2,12 @@ kind: session_essencedisplay_name: Session Essence — Cross-source entity synopses from conversationsconfig:kind: knowledge_graph+ rechunk_rules: |+ - Group only adjacent source chunks that discuss the same conversation topic or entity.+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.+ - Split when the speaker, topic, intent, or semantic focus changes.+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.entity:description: >-You are a robust entity and fact extractor for conversational data.diff --git a/api/db/init_data/compilation_templates/session_graph.yaml b/api/db/init_data/compilation_templates/session_graph.yamlindex 3bbdb1efb5..d9dd23daf4 100644--- a/api/db/init_data/compilation_templates/session_graph.yaml+++ b/api/db/init_data/compilation_templates/session_graph.yaml@@ -2,6 +2,12 @@ kind: session_graphdisplay_name: Session Graph — Knowledge graph from conversationsconfig:kind: knowledge_graph+ rechunk_rules: |+ - Group only adjacent source chunks that discuss the same conversation topic or entity.+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.+ - Split when the speaker, topic, intent, or semantic focus changes.+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.entity:description: >-You are a robust entity and fact extractor for conversational data.diff --git a/api/db/init_data/compilation_templates/timeline.yaml b/api/db/init_data/compilation_templates/timeline.yamlindex 00bee1485d..728603a315 100644--- a/api/db/init_data/compilation_templates/timeline.yaml+++ b/api/db/init_data/compilation_templates/timeline.yaml@@ -2,6 +2,12 @@ kind: timelinedisplay_name: List (Timeline) — Chronological events / Graphconfig:kind: timeline+ rechunk_rules: |+ - Group only adjacent source chunks that belong to the same event or continuous time period.+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.+ - Split when the event, time period, or semantic focus changes.+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.entity:description: >-You are a robust events-timeline extractor.diff --git a/rag/advanced_rag/knowlege_compile/_common.py b/rag/advanced_rag/knowlege_compile/_common.pyindex f426b61071..77393d6427 100644--- a/rag/advanced_rag/knowlege_compile/_common.py+++ b/rag/advanced_rag/knowlege_compile/_common.py@@ -57,6 +57,7 @@ def knowledge_compile_gen_conf(chat_mdl, gen_conf: Optional[dict] = None) -> dicmodel_name = str(model_config.get("llm_name") or getattr(chat_mdl, "llm_name", "")).lower()if "deepseek-v4" in model_name:+ conf["max_completion_tokens"] = 32768extra_body = conf.get("extra_body")extra_body = dict(extra_body) if isinstance(extra_body, dict) else {}extra_body["thinking"] = {"type": "disabled"}diff --git a/rag/advanced_rag/knowlege_compile/runner.py b/rag/advanced_rag/knowlege_compile/runner.pyindex 61d37cf6de..3f855b5d5c 100644--- a/rag/advanced_rag/knowlege_compile/runner.py+++ b/rag/advanced_rag/knowlege_compile/runner.py@@ -65,6 +65,9 @@ DOC_STRUCTURE_COMPILE_BATCH_CHUNKS = 4# before the LLM call.STRUCTURE_CONTEXT_FRACTION = 0.5STRUCTURE_DEFAULT_CONTEXT = 100_000+KNOWLEDGE_GRAPH_CONTEXT_FRACTION = 0.1+KNOWLEDGE_GRAPH_MIN_BATCH_TOKENS = 2048+KNOWLEDGE_GRAPH_MAX_BATCH_TOKENS = 4096# Bound the number of batch/template extraction calls in flight. Results are# committed in submission order so accumulator updates and merge flushes stay@@ -310,7 +313,7 @@ async def run_structure_compile_over_batches(# compile_kwd(s) each template actually wrote, harvested from flush results# so a dataset-scope template can rebuild its dataset graph once at the end.compile_kwds_by_tid: dict[str, set[str]] = {tid: set() for tid, _ in active_templates}- agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0} for tid, _ in active_templates}+ agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0, "rechunked_chunks": []} for tid, _ in active_templates}chunks_by_id: dict[str, str] = {}flush_sequence = 0flush_tasks: set[asyncio.Task[None]] = set()@@ -414,6 +417,10 @@ async def run_structure_compile_over_batches(async def _commit_result(batch_no: int, batch_len: int, template_id: str, docs: list[dict]) -> None:if docs:accumulators[template_id].extend(docs)+ rechunked_chunks = getattr(docs, "rechunked_chunks", None)+ if rechunked_chunks:+ known_ids = {chunk.get("id") for chunk in agg_infos[template_id]["rechunked_chunks"]}+ agg_infos[template_id]["rechunked_chunks"].extend(chunk for chunk in rechunked_chunks if chunk.get("id") not in known_ids)if len(accumulators[template_id]) >= DOC_STRUCTURE_MERGE_MAX_DOCS:progress_cb(msg=f" merge flush ({len(accumulators[template_id])} docs) for batch {batch_no} ({batch_len} chunks) for template ({template_ids_by_id[template_id]}/{total})")await _flush(template_id)@@ -428,6 +435,11 @@ async def run_structure_compile_over_batches(def _dynamic_batch_budget(template_id: str) -> int:max_length = getattr(chat_mdl_by_tid[template_id], "max_length", None) or STRUCTURE_DEFAULT_CONTEXT+ if template_kinds.get(template_id) == "knowledge_graph":+ return min(+ max(int(max_length * KNOWLEDGE_GRAPH_CONTEXT_FRACTION), KNOWLEDGE_GRAPH_MIN_BATCH_TOKENS),+ KNOWLEDGE_GRAPH_MAX_BATCH_TOKENS,+ )return max(int(max_length * STRUCTURE_CONTEXT_FRACTION), 1024)async def _commit_ready() -> None:@@ -489,9 +501,6 @@ async def run_structure_compile_over_batches(for template_id, parser_cfg in active_templates:if cancel_check():raise TaskCanceledException("Task was cancelled during document knowledge compilation")- if template_kinds[template_id] == "knowledge_graph":- await _submit_one(incoming_batch, template_id, parser_cfg)- continuebuffer = dynamic_buffers[template_id]budget = _dynamic_batch_budget(template_id)buffer_tokens = dynamic_buffer_tokens[template_id]@@ -622,8 +631,26 @@ async def run_structure_compile_over_batches(raise TaskCanceledException("Task was cancelled during document knowledge compilation")agg = agg_infos[template_id]if record:- record(f"document_structure_compile:{template_id}", agg)- progress_cb(msg=f"Document knowledge compilation done ({idx + 1}/{total}): {agg}")+ recorded_agg = {key: value for key, value in agg.items() if key != "rechunked_chunks"}+ recorded_agg["rechunked_chunk_count"] = len(agg.get("rechunked_chunks") or [])+ record(f"document_structure_compile:{template_id}", recorded_agg)+ rechunked_chunks = agg.get("rechunked_chunks") or []+ if rechunked_chunks:+ progress_cb(+ msg=(+ f"Rechunk: {len(chunks_by_id)} -> {len(rechunked_chunks)} chunks; "+ f"inserted={agg.get('inserted', 0)}, updated={agg.get('updated', 0)}, "+ f"duplicates_dropped={agg.get('duplicates_dropped', 0)}"+ )+ )+ else:+ progress_cb(+ msg=(+ f"Document knowledge compilation done ({idx + 1}/{total}): "+ f"inserted={agg.get('inserted', 0)}, updated={agg.get('updated', 0)}, "+ f"duplicates_dropped={agg.get('duplicates_dropped', 0)}"+ )+ )# ── Synthesis phase ──────────────────────────────────────────────# If the template has synthesis.enabled, run wiki PLAN+REFINEdiff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.pyindex bc45452bf7..66385c1aee 100644--- a/rag/advanced_rag/knowlege_compile/structure.py+++ b/rag/advanced_rag/knowlege_compile/structure.py@@ -18,6 +18,8 @@ import asyncioimport heapqimport jsonimport logging+import re+import uuidfrom typing import Awaitable, Callable, Tupleimport xxhash@@ -68,6 +70,14 @@ _STRUCT_MERGE_LOCK_TIMEOUT_S = 60_STRUCT_MERGE_LOCK_BLOCKING_TIMEOUT_S = 5+class _RechunkedDocs(list):+ """Compiled structure rows plus the formal chunks created by rechunking."""++ def __init__(self, docs=None, rechunked_chunks=None):+ super().__init__(docs or [])+ self.rechunked_chunks = rechunked_chunks or []++def _struct_merge_lock_key(kb_id: str, compilation_template_id: str | None) -> str:"""Per-(kb, template) lock so different templates can merge in parallel."""return f"struct_merge:{kb_id}:{compilation_template_id or ''}"@@ -280,7 +290,7 @@ def _struct_render_type_fields(fields: list, language: str, *, kind: str) -> Tupreturn "\n".join(lines), skeleton-def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tuple[str, str]:+def _struct_hypergraph_prompts(parser_config: dict, language: str = "en", rechunk: bool = False) -> Tuple[str, str]:autotype = _struct_infer_type(parser_config)guideline = _struct_get(parser_config, "guideline", default={}) or {}output = _struct_get(parser_config, "output", default={}) or {}@@ -292,6 +302,7 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tuprules_r = _struct_localize(_struct_get(guideline, "rules_for_relations"), language)rules_t = _struct_localize(_struct_get(guideline, "rules_for_time"), language)global_rules = _struct_localize(_struct_get(parser_config, "global_rules"), language)+ rechunk_rules = _struct_localize(_struct_get(parser_config, "rechunk_rules"), language)observation_time = _struct_get(options, "observation_time") or datetime.date.today().isoformat()if rules_t and "{observation_time}" in rules_t:@@ -318,12 +329,26 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tupif ent_desc:node_parts.append(f"## Entity Description:\n{ent_desc}")node_parts.append(f"## Entity Fields:\n{ent_fields_text}")- node_parts.append(- "## Response Format:\n"- "Reply with a single JSON object of the form: "- f'{{"items": [{ent_skel}, ...]}}.\n'- f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."- )+ if rechunk:+ node_parts.append(+ "## Semantic Chunking Rules:\n"+ f"{rechunk_rules}\n\n"+ "## Response Format:\n"+ "First group the source chunks according to the rules above. "+ "Do not return chunk text. Return temporary chunk ids (c1, c2, ...) and the source chunk ids included in each group. "+ 'Use compact inclusive ranges for consecutive source ids, for example ["t1-t3", "t8"]. '+ "Then extract entities and use only those temporary chunk ids in each entity's source_chunk_ids. "+ "Reply with a single JSON object of the form: "+ f'{{"chunks": [{{"id": "c1", "source_chunk_ids": ["source-id", ...]}}], "items": [{ent_skel}, ...]}}.\n'+ f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."+ )+ else:+ node_parts.append(+ "## Response Format:\n"+ "Reply with a single JSON object of the form: "+ f'{{"items": [{ent_skel}, ...]}}.\n'+ f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."+ )node_prompt = "\n\n".join(node_parts)if not relations_cfg:@@ -383,8 +408,38 @@ def _struct_unwrap_items(res) -> list:return []-async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, language: str) -> Tuple[list[dict], list[dict]]:- node_prompt, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language)+def _struct_expand_source_chunk_ids(raw_ids, source_texts: dict[str, str]) -> list[str]:+ """Expand compact positional source IDs such as t1-t3."""+ if isinstance(raw_ids, str):+ raw_ids = [raw_ids]+ if not isinstance(raw_ids, list):+ return []++ expanded: list[str] = []+ available = set(source_texts)+ for raw_id in raw_ids:+ value = str(raw_id).strip()+ match = re.fullmatch(r"t(\d+)\s*-\s*t(\d+)", value, flags=re.IGNORECASE)+ if match:+ start, end = (int(match.group(1)), int(match.group(2)))+ step = 1 if start <= end else -1+ values = [f"t{index}" for index in range(start, end + step, step)]+ else:+ values = [value]+ for chunk_id in values:+ if chunk_id in available and chunk_id not in expanded:+ expanded.append(chunk_id)+ return expanded+++async def _struct_extract_hypergraph(+ text: str,+ parser_config: dict,+ chat_mdl,+ language: str,+ rechunk: bool = False,+) -> Tuple[list[dict], list[dict], dict[str, str], list[dict]]:+ node_prompt, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk)user_prompt = ("## Source Text:\n"@@ -395,6 +450,56 @@ async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, l)node_res = await gen_json(node_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))nodes = _struct_unwrap_items(node_res)+ chunk_id_map: dict[str, str] = {}+ rechunked_chunks: list[dict] = []+ relation_text = text++ if rechunk:+ source_texts = {match.group(1).strip(): match.group(2).strip() for match in re.finditer(r"\[CHUNK_ID:\s*([^\]]+)\]\n(.*?)\n\[END_CHUNK\]", text, flags=re.DOTALL)}+ groups = node_res.get("chunks") if isinstance(node_res, dict) else None+ valid_groups = []+ claimed_sources: set[str] = set()+ if isinstance(groups, list):+ for group in groups:+ if not isinstance(group, dict):+ continue+ sources = [source for source in _struct_expand_source_chunk_ids(group.get("source_chunk_ids"), source_texts) if source not in claimed_sources]+ if not sources:+ continue+ claimed_sources.update(sources)+ temp_id = str(group.get("id") or f"c{len(valid_groups) + 1}").strip()+ if temp_id in {item[0] for item in valid_groups}:+ temp_id = f"c{len(valid_groups) + 1}"+ valid_groups.append((temp_id, sources))++ for source_id in source_texts:+ if source_id not in claimed_sources:+ valid_groups.append((f"c{len(valid_groups) + 1}", [source_id]))++ relation_segments = []+ temp_to_uuid: dict[str, str] = {}+ for temp_id, source_ids in valid_groups:+ new_id = uuid.uuid4().hex+ temp_to_uuid[temp_id] = new_id+ for source_id in source_ids:+ chunk_id_map[source_id] = new_id+ grouped_text = "\n\n".join(source_texts[source_id] for source_id in source_ids)+ relation_segments.append(f"[CHUNK_ID: {new_id}]\n{grouped_text}\n[END_CHUNK]")+ rechunked_chunks.append(+ {+ "id": new_id,+ "text": grouped_text,+ "source_chunk_ids": source_ids,+ }+ )+ relation_text = "\n\n".join(relation_segments) or text++ for node in nodes:+ raw_ids = node.get("source_chunk_ids")+ if isinstance(raw_ids, str):+ raw_ids = [raw_ids]+ if isinstance(raw_ids, list):+ node["source_chunk_ids"] = [temp_to_uuid.get(str(item).strip(), str(item).strip()) for item in raw_ids if temp_to_uuid.get(str(item).strip())]id_field = _struct_entity_id_field(parser_config)known_keys = []@@ -408,13 +513,31 @@ async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, lknown_str = "- " + "\n- ".join(known_keys) if known_keys else "(none)"if not edge_prompt_template:- return nodes, []+ return nodes, [], chunk_id_map, rechunked_chunksedge_prompt = edge_prompt_template.replace("{known_nodes}", known_str)- edge_res = await gen_json(edge_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))+ edge_user_prompt = (+ user_prompt+ if not rechunk+ else (+ "## Source Text:\n"+ "Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK]. "+ "For every relation, return source_chunk_ids containing only the IDs of chunks that support that relation.\n"+ f"{relation_text}\n\n## Output (JSON only):"+ )+ )+ edge_res = await gen_json(edge_prompt, edge_user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))edges = _struct_unwrap_items(edge_res)- return nodes, edges+ if rechunk:+ for edge in edges:+ raw_ids = edge.get("source_chunk_ids")+ if isinstance(raw_ids, str):+ raw_ids = [raw_ids]+ if isinstance(raw_ids, list):+ edge["source_chunk_ids"] = [item for item in raw_ids if isinstance(item, str) and item in set(chunk_id_map.values())]++ return nodes, edges, chunk_id_map, rechunked_chunksdef _struct_payload_chunk_ids(payload: dict, batch_ids: list) -> list:@@ -677,7 +800,7 @@ async def _struct_process_batch(semaphore,compilation_template_id: str | None = None,compilation_template_kind: str | None = None,-) -> list[dict]:+) -> _RechunkedDocs:"""Process one packed batch end-to-end (extract → embed → ES docs).``packed`` is the per-batch shape produced by@@ -690,50 +813,57 @@ async def _struct_process_batch(embedding work to bound peak concurrency."""if not packed:- return []+ return _RechunkedDocs()batch_ids: list = [e["chunk_id"] for e in packed if e.get("chunk_id")]batch_segments: list[str] = [f"[CHUNK_ID: {e['chunk_id']}]\n{e['text']}\n[END_CHUNK]" for e in packed if e.get("chunk_id") and isinstance(e.get("text"), str)]combined_text = "\n\n".join(batch_segments)-src_field, target_field = _struct_relation_member_fields(parser_config)+ rechunk = bool(parser_config.get("rechunk"))- async def _run() -> list[dict]:+ async def _run() -> _RechunkedDocs:# For hypergraph, entity extraction MUST complete before edge extraction# within the same batch, because the edge prompt's {known_nodes}# placeholder is filled from this batch's extracted nodes — see# _struct_extract_hypergraph. Parallelism across batches is fine; the# two stages within one batch are strictly sequential.try:- items, relations = await _struct_extract_hypergraph(combined_text, parser_config, chat_mdl, language)+ items, relations, chunk_id_map, formal_chunks = await _struct_extract_hypergraph(+ combined_text,+ parser_config,+ chat_mdl,+ language,+ rechunk=rechunk,+ )except Exception as e:logging.exception(f"compile_structure_from_text: extraction failed for batch {batch_idx}: {e}")- return []+ return _RechunkedDocs()payloads = items + relationskinds = ["entity"] * len(items) + ["relation"] * len(relations)+ payload_chunk_ids = list(dict.fromkeys(chunk_id_map.values())) if chunk_id_map else batch_idsif not payloads:if callback:callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: 0 items")- return []+ return _RechunkedDocs(rechunked_chunks=formal_chunks)embed_inputs = [_struct_payload_description(p) for p in payloads]try:embeddings = await _struct_embed(embd_mdl, embed_inputs)except Exception as e:logging.exception(f"compile_structure_from_text: embedding failed for batch {batch_idx}: {e}")- return []+ return _RechunkedDocs(rechunked_chunks=formal_chunks)if len(embeddings) != len(payloads):logging.error(f"compile_structure_from_text: embedding count mismatch ({len(embeddings)} vs {len(payloads)}) for batch {batch_idx}")- return []+ return _RechunkedDocs(rechunked_chunks=formal_chunks)docs = [_struct_to_doc_storage_doc(payload,autotype,doc_id,- _struct_payload_chunk_ids(payload, batch_ids),+ _struct_payload_chunk_ids(payload, payload_chunk_ids),vec,kind,src_field=src_field,@@ -747,7 +877,7 @@ async def _struct_process_batch(if callback:callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: {len(payloads)} items")- return docs+ return _RechunkedDocs(docs, formal_chunks)if semaphore is not None:async with semaphore:@@ -810,7 +940,8 @@ async def compile_structure_from_text(logging.error(f"compile_structure_from_text: unsupported type '{autotype}'")return []- node_prompt, edge_prompt = _struct_hypergraph_prompts(parser_config, language)+ rechunk = bool(parser_config.get("rechunk"))+ node_prompt, edge_prompt = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk)prompt_overhead = max(num_tokens_from_string(node_prompt), num_tokens_from_string(edge_prompt))# ``kind`` for the row stamp follows the template's ``kind`` field if@@ -847,12 +978,15 @@ async def compile_structure_from_text(compilation_template_kind=template_kind,)- def _flatten(per_batch: list) -> list[dict]:+ def _flatten(per_batch: list) -> _RechunkedDocs:out: list[dict] = []+ formal_chunks: list[dict] = []for br in per_batch or []:- if br:- out.extend(br)- return out+ if br is None:+ continue+ out.extend(br)+ formal_chunks.extend(getattr(br, "rechunked_chunks", []))+ return _RechunkedDocs(out, formal_chunks)return await _run_chunked_pipeline(packed_batches,diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.pyindex b0dec7a1d4..eb1bbfb078 100644--- a/rag/flow/compiler/compiler.py+++ b/rag/flow/compiler/compiler.py@@ -14,6 +14,7 @@# limitations under the License.import loggingimport random+import refrom copy import deepcopyfrom types import SimpleNamespace@@ -26,6 +27,8 @@ from api.db.services.knowledgebase_service import KnowledgebaseServicefrom api.db.services.llm_service import LLMBundlefrom api.db.services.task_service import has_canceledfrom common.constants import LLMType+from common.token_utils import num_tokens_from_string+from rag.nlp import naive_mergefrom rag.advanced_rag.knowlege_compile.runner import (DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,load_active_templates,@@ -58,6 +61,288 @@ class CompilerParam(ProcessParamBase, LLMParam):class Compiler(ProcessBase, LLM):component_name = "Compiler"+ _PARSER_TEXT_CHUNK_TOKEN_SIZE = 1024+ _PARSER_CANDIDATE_TOKEN_SIZE = 128+ _PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE = 256+ _PARSER_MIN_SPLIT_CHARACTERS = 24++ @classmethod+ def _split_slumber_level(cls, text: str, delimiters: list[str]) -> list[str]:+ """Split at boundaries while keeping delimiters with the previous part."""+ if not text:+ return []+ pattern = "|".join(re.escape(delimiter) for delimiter in sorted(delimiters, key=len, reverse=True))+ if not pattern:+ return [text]++ parts = []+ start = 0+ for match in re.finditer(pattern, text, flags=re.DOTALL):+ end = match.end()+ if end - start >= cls._PARSER_MIN_SPLIT_CHARACTERS:+ parts.append(text[start:end])+ start = end+ if start < len(text):+ parts.append(text[start:])++ # Keep very short delimiter fragments with their neighbour, matching+ # Chonkie's min_chars behavior instead of producing tiny candidates.+ merged = []+ for part in parts:+ if merged and len(part.strip()) < cls._PARSER_MIN_SPLIT_CHARACTERS:+ merged[-1] += part+ else:+ merged.append(part)+ return [part for part in merged if part.strip()]++ @classmethod+ def _slumber_candidates(cls, text: str, level: int = 0, target_token_size: int | None = None) -> list[str]:+ """Apply paragraph→sentence splitting with token fallback."""+ target_token_size = target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE+ levels = [+ ["\n\n", "\r\n", "\n", "\r"],+ [". ", "! ", "? ", "。", "!", "?", ";", ";"],+ ]+ if not text.strip():+ return []+ if level >= len(levels):+ return [part for part in naive_merge(text, target_token_size, "", 0) if part.strip()]++ splits = cls._split_slumber_level(text, levels[level])+ if len(splits) <= 1 and level < len(levels) - 1:+ return cls._slumber_candidates(text, level + 1, target_token_size)++ candidates = []+ for split in splits:+ if num_tokens_from_string(split) > target_token_size:+ candidates.extend(cls._slumber_candidates(split, level + 1, target_token_size))+ else:+ candidates.append(split)+ return candidates++ @staticmethod+ def _is_atomic_json_record(record: dict) -> bool:+ """Keep structured or position-bound Parser records intact."""+ doc_type = str(record.get("doc_type_kwd") or "").strip().lower()+ layout_type = str(record.get("layout_type") or "").strip().lower()+ text = record.get("text") or record.get("content_with_weight")+ has_markdown_table = False+ has_html_table = False+ has_fenced_code = False+ if isinstance(text, str):+ lines = text.splitlines()+ has_markdown_table = any(index + 1 < len(lines) and re.match(r"^\s*\|", lines[index]) and Compiler._is_markdown_table_separator(lines[index + 1]) for index in range(len(lines)))+ has_html_table = bool(re.search(r"<table(?:\s|>)", text, flags=re.IGNORECASE))+ has_fenced_code = bool(re.search(r"^\s*(`{3,}|~{3,})", text, flags=re.MULTILINE))+ return bool(+ doc_type in {"table", "image"}+ or record.get("img_id")+ or record.get("image")+ or layout_type in {"figure", "table"}+ or record.get("bbox")+ or has_markdown_table+ or has_html_table+ or has_fenced_code+ )++ @staticmethod+ def _formalize_rechunked_chunks(chunks: list[dict], rechunked_chunks: list[dict], doc_id: str) -> list[dict]:+ """Replace parser chunks with the semantic groups produced by rechunking."""+ originals = {str(chunk.get("id")): chunk for chunk in chunks if chunk.get("id")}+ result = []+ for grouped in rechunked_chunks:+ source_ids = [str(value) for value in grouped.get("source_chunk_ids") or []]+ source = next((originals[source_id] for source_id in source_ids if source_id in originals), None)+ if source is None:+ logging.warning(+ "Compiler: skip rechunked group %s because none of its source chunks exist",+ grouped.get("id"),+ )+ continue+ item = deepcopy(source)+ item["id"] = str(grouped.get("id"))+ item["doc_id"] = doc_id+ item["text"] = grouped.get("text") or ""+ for key in list(item):+ if key.startswith("q_") and key.endswith("_vec"):+ del item[key]+ for key in ("content_with_weight", "content_ltks", "content_sm_ltks", "position_int", "page_num_int", "top_int"):+ item.pop(key, None)+ result.append(item)+ return result++ @staticmethod+ def _is_markdown_table_separator(line: str) -> bool:+ cells = line.strip().strip("|").split("|")+ return len(cells) >= 1 and all(re.fullmatch(r"\s*:?-{3,}:?\s*", cell) for cell in cells)++ @classmethod+ def _split_markdown_blocks(cls, text: str) -> list[tuple[str, str | None]]:+ """Separate Markdown tables and fenced code blocks as atomic blocks."""+ lines = text.splitlines(keepends=True)+ blocks: list[tuple[str, str | None]] = []+ ordinary: list[str] = []+ index = 0++ def flush_ordinary() -> None:+ if ordinary:+ blocks.append(("".join(ordinary), False))+ ordinary.clear()++ while index < len(lines):+ fence_match = re.match(r"^\s*(`{3,}|~{3,})", lines[index])+ if fence_match:+ flush_ordinary()+ fence = fence_match.group(1)+ fence_char = fence[0]+ fence_length = len(fence)+ code = [lines[index]]+ index += 1+ while index < len(lines):+ code.append(lines[index])+ closing = re.match(+ rf"^\s*{re.escape(fence_char)}{{{fence_length},}}\s*$",+ lines[index].rstrip("\r\n"),+ )+ index += 1+ if closing:+ break+ blocks.append(("".join(code), "fenced_code"))+ continue++ if index + 1 < len(lines) and re.match(r"^\s*\|", lines[index]) and cls._is_markdown_table_separator(lines[index + 1]):+ flush_ordinary()+ table = [lines[index], lines[index + 1]]+ index += 2+ while index < len(lines) and re.match(r"^\s*\|", lines[index]):+ table.append(lines[index])+ index += 1+ blocks.append(("".join(table), "table"))+ continue+ ordinary.append(lines[index])+ index += 1++ flush_ordinary()+ return blocks++ @classmethod+ def _merge_small_text_chunks(cls, chunks: list[dict], target_token_size: int) -> list[dict]:+ """Merge adjacent ordinary text chunks up to the coarse target."""+ if target_token_size < cls._PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE:+ return chunks++ merged = []+ pending = None+ for chunk in chunks:+ text = chunk.get("text")+ if not isinstance(text, str) or not text.strip() or cls._is_atomic_json_record(chunk):+ if pending is not None:+ merged.append(pending)+ pending = None+ merged.append(chunk)+ continue++ if pending is None:+ pending = chunk+ continue++ combined = f"{pending['text']}\n\n{text}"+ if num_tokens_from_string(pending["text"]) < cls._PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE and num_tokens_from_string(combined) <= target_token_size:+ pending["text"] = combined+ else:+ merged.append(pending)+ pending = chunk++ if pending is not None:+ merged.append(pending)+ return merged++ @classmethod+ def _normalize_upstream_chunks(+ cls,+ upstream: dict,+ split_json_text: bool = False,+ target_token_size: int | None = None,+ ) -> list[dict]:+ """Normalize direct Parser output into Compiler chunks.++ Parser JSON is a list of chunk-like records. JSON records are split by+ the requested token target while retaining their metadata; upstream+ ``chunks`` preserve atomic table/image records unless rechunking is+ explicitly requested. Textual Parser outputs use the same split.+ """+ output_format = upstream.get("output_format")+ if output_format == "chunks":+ chunks = upstream.get("chunks") or []+ if not isinstance(chunks, list):+ return []+ if not split_json_text:+ normalized = deepcopy(chunks)+ for chunk in normalized:+ if not isinstance(chunk, dict):+ continue+ text = chunk.get("text")+ if not isinstance(text, str):+ text = chunk.get("content_with_weight")+ chunk["text"] = text if isinstance(text, str) else ""+ return normalized+ records = chunks+ elif output_format == "json":+ records = upstream.get("json") or upstream.get("json_result") or []+ if not isinstance(records, list):+ return []+ else:+ records = None++ if records is not None:+ chunks = []+ for record in records:+ if not isinstance(record, dict):+ continue+ chunk = deepcopy(record)+ text = chunk.get("text")+ if not isinstance(text, str):+ text = chunk.get("content_with_weight")+ if isinstance(text, str) and text.strip():+ if Compiler._is_atomic_json_record(chunk) or not split_json_text:+ chunk["text"] = text+ chunks.append(chunk)+ else:+ for part in Compiler._slumber_candidates(text, target_token_size=target_token_size):+ split_chunk = deepcopy(chunk)+ split_chunk["text"] = part+ chunks.append(split_chunk)+ else:+ # Keep metadata-only records (for example, an image+ # placeholder) instead of silently dropping them.+ chunk["text"] = text if isinstance(text, str) else ""+ chunks.append(chunk)+ if split_json_text:+ return cls._merge_small_text_chunks(chunks, target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE)+ return chunks++ if output_format in {"markdown", "text", "html"}:+ text = upstream.get(output_format)+ if not isinstance(text, str):+ text = upstream.get(f"{output_format}_result")+ if not isinstance(text, str) or not text.strip():+ return []+ if not split_json_text:+ return [{"text": text}]+ if output_format == "markdown":+ chunks = []+ for block, block_type in cls._split_markdown_blocks(text):+ if block_type == "table":+ chunks.append({"text": block, "doc_type_kwd": "table"})+ elif block_type == "fenced_code":+ chunks.append({"text": block})+ else:+ chunks.extend({"text": part} for part in cls._slumber_candidates(block, target_token_size=target_token_size) if part.strip())+ else:+ chunks = [{"text": part} for part in cls._slumber_candidates(text, target_token_size=target_token_size) if part.strip()]+ return cls._merge_small_text_chunks(chunks, target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE)++ return []def _compile_progress(self, prog=None, msg=""):"""Adapt the knowledge-compile ``callback`` protocol to the flow@@ -210,31 +495,35 @@ class Compiler(ProcessBase, LLM):# kwargs. Do not call LLM.get_input_elements() here: it resolves the# inherited prompt variables through Canvas.globals, while Pipeline# is a Graph and has no globals.- chunks = deepcopy(kwargs.get("chunks") or [])- if not chunks:- for val in kwargs.values():- if isinstance(val, list):- chunks = deepcopy(val)+ output_format = kwargs.get("output_format")+ if output_format == "chunks":+ raw_chunks = kwargs.get("chunks") or []+ if not isinstance(raw_chunks, list) or not raw_chunks:+ self.set_output("chunks", [])+ return+ # Normalize chunks only after the active templates are known, so+ # rechunk mode does not normalize the same input twice.+ chunks = None+ else:+ chunks = self._normalize_upstream_chunks(kwargs)tenant_id = self._canvas.get_tenant_id()doc_id = self._canvas._doc_idkb_id = getattr(self._canvas, "_kb_id", None) or DocumentService.get_knowledgebase_id(doc_id)language = self._compile_language(kwargs)- if not chunks:+ if chunks is not None and not chunks:self.set_output("chunks", chunks)return- for ck in chunks:- ck["doc_id"] = doc_id- ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()-# Resolve the configured template groups to concrete, active# (non-artifact) structure-compilation templates.template_ids = resolve_template_ids_from_groups(self._param.compilation_template_group_ids, tenant_id)active_templates = load_active_templates(template_ids, tenant_id)if not active_templates:self.callback(0, "No active compilation templates resolved from the configured groups.")+ if chunks is None:+ chunks = self._normalize_upstream_chunks(kwargs)self.set_output("chunks", chunks)return@@ -279,10 +568,52 @@ class Compiler(ProcessBase, LLM):filtered_templates.append((template_id, parser_cfg))if not filtered_templates:+ if chunks is None:+ chunks = self._normalize_upstream_chunks(kwargs)self.set_output("chunks", chunks)returnactive_templates = filtered_templates+ def _template_requests_rechunk(cfg: dict) -> bool:+ if not isinstance(cfg, dict):+ return False+ # ``raptor.rechunk`` belongs to the Tree path, which is explicitly+ # excluded from the semantic rechunk staging below. It must not+ # shrink non-Tree parser chunks unless a structure template has+ # requested the actual regrouping pass.+ return bool(cfg.get("rechunk"))++ should_rechunk = any(_template_requests_rechunk(cfg) for _, cfg in active_templates)+ target_token_size = self._PARSER_CANDIDATE_TOKEN_SIZE if should_rechunk else self._PARSER_TEXT_CHUNK_TOKEN_SIZE+ if output_format == "chunks":+ chunks = self._normalize_upstream_chunks(+ kwargs,+ split_json_text=should_rechunk,+ target_token_size=target_token_size,+ )+ elif output_format in {"markdown", "text", "html"}:+ chunks = self._normalize_upstream_chunks(+ kwargs,+ split_json_text=should_rechunk,+ target_token_size=target_token_size,+ )+ elif output_format == "json":+ chunks = self._normalize_upstream_chunks(+ kwargs,+ split_json_text=should_rechunk,+ target_token_size=target_token_size,+ )+ if not chunks:+ self.set_output("chunks", chunks)+ return+ if not chunks:+ self.set_output("chunks", chunks)+ return++ for ck in chunks:+ ck["doc_id"] = doc_id+ ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()+if self._canvas._kb_id:e, kb = KnowledgebaseService.get_by_id(self._canvas._kb_id)if kb.tenant_embd_id:@@ -315,6 +646,17 @@ class Compiler(ProcessBase, LLM):)if non_tree_templates:+ first_rechunk_index = next(+ (index for index, (_, cfg) in enumerate(non_tree_templates) if _template_requests_rechunk(cfg)),+ None,+ )+ if first_rechunk_index is not None:+ # Keep the stable hash IDs for Tree. Non-tree rechunking uses+ # compact positional IDs to keep the LLM grouping response+ # small (e.g. t1-t3 instead of three long hash IDs).+ for index, chunk in enumerate(chunks, start=1):+ chunk["id"] = f"t{index}"+task_id = getattr(self._canvas, "task_id", None)def _cancelled() -> bool:@@ -324,17 +666,48 @@ class Compiler(ProcessBase, LLM):for i in range(0, len(chunks), DOC_STRUCTURE_COMPILE_BATCH_CHUNKS):yield chunks[i : i + DOC_STRUCTURE_COMPILE_BATCH_CHUNKS]- await run_structure_compile_over_batches(- active_templates=non_tree_templates,- chat_mdl_by_tid=chat_mdl_by_tid,- embedding_model=embedding_model,- tenant_id=tenant_id,- kb_id=kb_id,- doc_id=doc_id,- language=language,- chunk_batches=_chunk_batches(),- progress_cb=self._compile_progress,- cancel_check=_cancelled,- )+ async def _run_templates(template_specs):+ return await run_structure_compile_over_batches(+ active_templates=template_specs,+ chat_mdl_by_tid=chat_mdl_by_tid,+ embedding_model=embedding_model,+ tenant_id=tenant_id,+ kb_id=kb_id,+ doc_id=doc_id,+ language=language,+ chunk_batches=_chunk_batches(),+ progress_cb=self._compile_progress,+ cancel_check=_cancelled,+ )++ if first_rechunk_index is None:+ await _run_templates(non_tree_templates)+ else:+ if first_rechunk_index:+ await _run_templates(non_tree_templates[:first_rechunk_index])+ first_template = [non_tree_templates[first_rechunk_index]]+ compile_info = await _run_templates(first_template)+ first_template_id = first_template[0][0]+ formal_chunks = (compile_info.get(first_template_id) or {}).get("rechunked_chunks")+ if formal_chunks:+ formalized_chunks = self._formalize_rechunked_chunks(chunks, formal_chunks, doc_id)+ if formalized_chunks:+ chunks = formalized_chunks+ else:+ logging.warning(+ "Compiler: rechunking returned no valid formal chunks for doc %s; keeping original chunks",+ doc_id,+ )++ # The first rechunk template defines the semantic boundaries+ # for all later templates. They consume those formal chunks+ # and must not perform another independent rechunking pass.+ later_templates = []+ for template_id, parser_cfg in non_tree_templates[first_rechunk_index + 1 :]:+ later_config = dict(parser_cfg or {})+ later_config["rechunk"] = False+ later_templates.append((template_id, later_config))+ if later_templates:+ await _run_templates(later_templates)self.set_output("chunks", chunks)diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.pyindex d4015e1673..0ab017bd2f 100644--- a/rag/llm/chat_model.py+++ b/rag/llm/chat_model.py@@ -1691,8 +1691,30 @@ class LiteLLMBase(ABC):gen_conf=gen_conf,)- gen_conf.pop("max_tokens", None)+ deepseek_max_tokens = None+ if self.provider == SupportedLiteLLMProvider.DeepSeek:+ # DeepSeek's API uses the legacy OpenAI-compatible max_tokens field.+ # LiteLLM accepts max_completion_tokens generically, but does not+ # translate it for DeepSeek and the provider then falls back to 8192.+ # Knowledge compilation supplies max_completion_tokens explicitly+ # through its model-specific generation configuration. Otherwise,+ # preserve the legacy max_tokens value used by existing callers.+ raw_max_completion_tokens = gen_conf.pop("max_completion_tokens", None)+ raw_max_tokens = gen_conf.pop("max_tokens", None)+ raw_limit = raw_max_completion_tokens if raw_max_completion_tokens is not None else raw_max_tokens+ if raw_limit is not None and not isinstance(raw_limit, bool):+ try:+ candidate = int(raw_limit)+ except (TypeError, ValueError):+ candidate = 0+ if candidate > 0:+ deepseek_max_tokens = candidate+ else:+ gen_conf.pop("max_tokens", None)+gen_conf = {k: v for k, v in gen_conf.items() if k in LITELLM_ALLOWED_GEN_CONF_KEYS}+ if deepseek_max_tokens is not None:+ gen_conf["max_tokens"] = deepseek_max_tokensreturn gen_confdef _need_reasoning_content_back(self) -> bool:diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.pyindex 3e6cb4b300..484ff20920 100644--- a/rag/svr/task_executor_refactor/chunk_post_processor.py+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py@@ -378,8 +378,6 @@ from rag.advanced_rag.knowlege_compile.runner import ( # noqa: E402load_active_templates,run_structure_compile_over_batches,)--# ----- parser_config helpers -----------------------------------------diff --git a/web/src/interfaces/database/compilation-template.ts b/web/src/interfaces/database/compilation-template.tsindex 564611c3e6..b29876e69e 100644--- a/web/src/interfaces/database/compilation-template.ts+++ b/web/src/interfaces/database/compilation-template.ts@@ -24,6 +24,8 @@ export interface ICompilationTemplateConfig {relation?: ICompilationTemplateSection;raptor?: ICompilationTemplateRaptorConfig;global_rules?: string;+ rechunk?: boolean;+ rechunk_rules?: string;[section: string]:| ICompilationTemplateSection| ICompilationTemplateRaptorConfigdiff --git a/web/src/interfaces/request/compilation-template.ts b/web/src/interfaces/request/compilation-template.tsindex 81058f246a..6ab69cb280 100644--- a/web/src/interfaces/request/compilation-template.ts+++ b/web/src/interfaces/request/compilation-template.ts@@ -24,6 +24,8 @@ export interface ICompilationTemplateConfigRequest {relation?: ICompilationTemplateSectionRequest;raptor?: ICompilationTemplateRaptorConfigRequest;global_rules?: string;+ rechunk?: boolean;+ rechunk_rules?: string;[section: string]:| ICompilationTemplateSectionRequest| ICompilationTemplateRaptorConfigRequestdiff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 7ad38f99cc..25be19c67e 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1900,6 +1900,12 @@ Example: Virtual Hosted Style`,rechunkByTreeLeaves: 'Re-chunk by tree leaves',rechunkByTreeLeavesTip:"Merge each leaf cluster's source chunks into a single replacement chunk. Originals are kept but marked unavailable for retrieval. Only one tree template per group may enable this.",+ rechunkInput: 'Re-chunk parser output',+ rechunkInputTip:+ 'Let the LLM determine chunk boundaries based on the knowledge compilation task.',+ rechunkRules: 'Rechunking rules',+ rechunkRulesPlaceholder:+ 'Describe how the LLM should group source chunks for this compilation task.',jsonPreview: 'JSON preview',processFlow: 'Process flow',processFlowComingSoon: 'Process flow preview coming soon',diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex 9b0f22c1a5..68f3632ee9 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -1590,6 +1590,11 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系rechunkByTreeLeaves: '按树叶重新分块',rechunkByTreeLeavesTip:'将每个叶簇的源数据块合并为单个替换数据块。原始数据块保留但标记为不可检索。每个分组最多只能有一个树模板启用此功能。',+ rechunkInput: '重新切分 Parser 输出',+ rechunkInputTip: '根据知识编译任务,由 LLM 决定 chunk 边界。',+ rechunkRules: 'Rechunk 规则',+ rechunkRulesPlaceholder:+ '描述此知识编译任务下,LLM 应如何合并和划分源 chunk。',jsonPreview: 'JSON 预览',processFlow: '流程视图',processFlowComingSoon: '流程视图预览即将到来',diff --git a/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx b/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsxindex def1f126ac..65a826160f 100644--- a/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx+++ b/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx@@ -1,6 +1,7 @@import { ModelTreeSelectFormField } from '@/components/model-tree-select';import { SelectWithSearch } from '@/components/originui/select-with-search';import { RAGFlowFormItem } from '@/components/ragflow-form';+import { SwitchFormField } from '@/components/switch-fom-field';import { Button } from '@/components/ui/button';import { Input } from '@/components/ui/input';import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';@@ -65,6 +66,10 @@ export function TemplateConfiguration({control: form.control,name: `templates.${selectedTemplateIndex}.kind`,});+ const rechunk = useWatch({+ control: form.control,+ name: `templates.${selectedTemplateIndex}.config.rechunk`,+ });const availableKindOptions = useAvailableKindOptions(form,@@ -198,39 +203,62 @@ export function TemplateConfiguration({{kind === CompilationTemplateKind.Tree ? (<TreeTemplateFields index={selectedTemplateIndex} />) : (- sectionNames.length > 0 &&- activeSectionTab && (- <Tabs- value={activeSectionTab}- onValueChange={setActiveSectionTab}- className="w-full"- >- <TabsList className="w-full justify-start">+ <>+ {kind !== CompilationTemplateKind.Artifacts && (+ <>+ <SwitchFormField+ name={`templates.${selectedTemplateIndex}.config.rechunk`}+ label={t('setting.rechunkInput')}+ tooltip={t('setting.rechunkInputTip')}+ vertical={false}+ />+ {rechunk && (+ <RAGFlowFormItem+ name={`templates.${selectedTemplateIndex}.config.rechunk_rules`}+ label={t('setting.rechunkRules')}+ >+ <Textarea+ placeholder={t('setting.rechunkRulesPlaceholder')}+ rows={6}+ resize="vertical"+ />+ </RAGFlowFormItem>+ )}+ </>+ )}+ {sectionNames.length > 0 && activeSectionTab && (+ <Tabs+ value={activeSectionTab}+ onValueChange={setActiveSectionTab}+ className="w-full"+ >+ <TabsList className="w-full justify-start">+ {sectionNames.map((sectionName) => (+ <TabsTrigger+ key={sectionName}+ value={sectionName}+ className="flex-1"+ >+ {t(+ SectionTitleKeyMap[sectionName] ??+ startCase(sectionName),+ )}+ </TabsTrigger>+ ))}+ </TabsList>+{sectionNames.map((sectionName) => (- <TabsTrigger+ <TabsContentkey={sectionName}value={sectionName}- className="flex-1"+ className="mt-4">- {t(- SectionTitleKeyMap[sectionName] ??- startCase(sectionName),- )}- </TabsTrigger>+ {renderSectionTabs(sectionName)}+ </TabsContent>))}- </TabsList>-- {sectionNames.map((sectionName) => (- <TabsContent- key={sectionName}- value={sectionName}- className="mt-4"- >- {renderSectionTabs(sectionName)}- </TabsContent>- ))}- </Tabs>- )+ </Tabs>+ )}+ </>)}</div></div>diff --git a/web/src/pages/user-setting/compilation-templates/create-next/utils.ts b/web/src/pages/user-setting/compilation-templates/create-next/utils.tsindex 834430cbd8..d9cd24640e 100644--- a/web/src/pages/user-setting/compilation-templates/create-next/utils.ts+++ b/web/src/pages/user-setting/compilation-templates/create-next/utils.ts@@ -61,6 +61,8 @@ export const DefaultTemplateValues: TemplateSchemaType = {instruction: '',page_example: '',use_blueprint: false,+ rechunk: false,+ rechunk_rules: '',},};@@ -81,6 +83,8 @@ export const isConfigMetaKey = (key: string) =>'page_example','synthesis','use_blueprint',+ 'rechunk',+ 'rechunk_rules',].includes(key);export const createEmptyField = (keys: string[]) =>@@ -132,6 +136,15 @@ export const buildConfigFromBuiltin = (: {}),use_blueprint:kind === CompilationTemplateKind.Artifacts && example.length > 0,+ ...(kind !== CompilationTemplateKind.Tree+ ? {+ rechunk: builtinTemplate.config?.rechunk === true,+ rechunk_rules:+ typeof builtinTemplate.config?.rechunk_rules === 'string'+ ? builtinTemplate.config.rechunk_rules+ : '',+ }+ : {}),};if (kind === CompilationTemplateKind.Artifacts && example.length > 0) {@@ -183,6 +196,15 @@ export const transformDetailToForm = (: {}),use_blueprint:detail.kind === CompilationTemplateKind.Artifacts && example.length > 0,+ ...(detail.kind !== CompilationTemplateKind.Tree+ ? {+ rechunk: config.rechunk === true,+ rechunk_rules:+ typeof config.rechunk_rules === 'string'+ ? config.rechunk_rules+ : '',+ }+ : {}),};if (detail.kind === CompilationTemplateKind.Artifacts && example.length > 0) {@@ -258,7 +280,8 @@ export const transformTemplateToPayload = (template: TemplateSchemaType) => {return;}if (isConfigMetaKey(key)) {- if (typeof value === 'string') config[key] = value;+ if (typeof value === 'string' || typeof value === 'boolean')+ config[key] = value;} else {config[key] = value as ICompilationTemplateConfigRequest[string];}diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx b/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsxindex adacf90b1f..89a41ec0e8 100644--- a/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx+++ b/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx@@ -1,6 +1,7 @@import { ModelTreeSelectFormField } from '@/components/model-tree-select';import { SelectWithSearch } from '@/components/originui/select-with-search';import { RAGFlowFormItem } from '@/components/ragflow-form';+import { SwitchFormField } from '@/components/switch-fom-field';import { Input } from '@/components/ui/input';import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';import { Textarea } from '@/components/ui/textarea';@@ -58,6 +59,10 @@ export function TemplateConfiguration({control: form.control,name: `templates.${selectedTemplateIndex}.kind`,});+ const rechunk = useWatch({+ control: form.control,+ name: `templates.${selectedTemplateIndex}.config.rechunk`,+ });const availableKindOptions = useAvailableKindOptions(form,@@ -191,39 +196,62 @@ export function TemplateConfiguration({{kind === CompilationTemplateKind.Tree ? (<TreeTemplateFields index={selectedTemplateIndex} />) : (- sectionNames.length > 0 &&- activeSectionTab && (- <Tabs- value={activeSectionTab}- onValueChange={setActiveSectionTab}- className="w-full"- >- <TabsList className="w-full justify-start">+ <>+ {kind !== CompilationTemplateKind.Artifacts && (+ <>+ <SwitchFormField+ name={`templates.${selectedTemplateIndex}.config.rechunk`}+ label={t('setting.rechunkInput')}+ tooltip={t('setting.rechunkInputTip')}+ vertical={false}+ />+ {rechunk && (+ <RAGFlowFormItem+ name={`templates.${selectedTemplateIndex}.config.rechunk_rules`}+ label={t('setting.rechunkRules')}+ >+ <Textarea+ placeholder={t('setting.rechunkRulesPlaceholder')}+ rows={6}+ resize="vertical"+ />+ </RAGFlowFormItem>+ )}+ </>+ )}+ {sectionNames.length > 0 && activeSectionTab && (+ <Tabs+ value={activeSectionTab}+ onValueChange={setActiveSectionTab}+ className="w-full"+ >+ <TabsList className="w-full justify-start">+ {sectionNames.map((sectionName) => (+ <TabsTrigger+ key={sectionName}+ value={sectionName}+ className="flex-1"+ >+ {t(+ SectionTitleKeyMap[sectionName] ??+ startCase(sectionName),+ )}+ </TabsTrigger>+ ))}+ </TabsList>+{sectionNames.map((sectionName) => (- <TabsTrigger+ <TabsContentkey={sectionName}value={sectionName}- className="flex-1"+ className="mt-4">- {t(- SectionTitleKeyMap[sectionName] ??- startCase(sectionName),- )}- </TabsTrigger>+ {renderSectionTabs(sectionName)}+ </TabsContent>))}- </TabsList>-- {sectionNames.map((sectionName) => (- <TabsContent- key={sectionName}- value={sectionName}- className="mt-4"- >- {renderSectionTabs(sectionName)}- </TabsContent>- ))}- </Tabs>- )+ </Tabs>+ )}+ </>)}{children}diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts b/web/src/pages/user-setting/compilation-templates/edit-next/utils.tsindex 13b23d9a76..f21a6d275e 100644--- a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts+++ b/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts@@ -57,6 +57,8 @@ export const DefaultTemplateValues: TemplateSchemaType = {instruction: '',page_example: '',use_blueprint: false,+ rechunk: false,+ rechunk_rules: '',},};@@ -77,6 +79,8 @@ export const isConfigMetaKey = (key: string) =>'page_example','synthesis','use_blueprint',+ 'rechunk',+ 'rechunk_rules',].includes(key);export const createEmptyField = (keys: string[]) =>@@ -128,6 +132,15 @@ export const buildConfigFromBuiltin = (: {}),use_blueprint:kind === CompilationTemplateKind.Artifacts && example.length > 0,+ ...(kind !== CompilationTemplateKind.Tree+ ? {+ rechunk: builtinTemplate.config?.rechunk === true,+ rechunk_rules:+ typeof builtinTemplate.config?.rechunk_rules === 'string'+ ? builtinTemplate.config.rechunk_rules+ : '',+ }+ : {}),};if (kind === CompilationTemplateKind.Artifacts && example.length > 0) {@@ -179,6 +192,15 @@ export const transformDetailToForm = (: {}),use_blueprint:detail.kind === CompilationTemplateKind.Artifacts && example.length > 0,+ ...(detail.kind !== CompilationTemplateKind.Tree+ ? {+ rechunk: config.rechunk === true,+ rechunk_rules:+ typeof config.rechunk_rules === 'string'+ ? config.rechunk_rules+ : '',+ }+ : {}),};if (detail.kind === CompilationTemplateKind.Artifacts && example.length > 0) {@@ -254,7 +276,8 @@ export const transformTemplateToPayload = (template: TemplateSchemaType) => {return;}if (isConfigMetaKey(key)) {- if (typeof value === 'string') config[key] = value;+ if (typeof value === 'string' || typeof value === 'boolean')+ config[key] = value;} else {config[key] = value as ICompilationTemplateConfigRequest[string];}