Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat: add LLM-guided semantic rechunking for knowledge compilation (#17546)

buua43644e13d1cb6ec
diff --git a/api/db/init_data/compilation_templates/knowledge_graph.yaml b/api/db/init_data/compilation_templates/knowledge_graph.yaml
index 27a8535dd1..c7d4386af8 100644
--- a/api/db/init_data/compilation_templates/knowledge_graph.yaml
+++ b/api/db/init_data/compilation_templates/knowledge_graph.yaml
@@ -2,6 +2,12 @@ kind: knowledge_graph
display_name: Knowledge graph
config:
kind: knowledge_graph
+ rechunk_rules: |
+ - Group only adjacent source chunks into coherent semantic units for this knowledge-compilation task.
+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.
+ - Split when the topic, section, or semantic focus changes; do not merge unrelated content.
+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.
+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.
entity:
description: >-
You are a robust graph entity extractor for knowledge graphs.
diff --git a/api/db/init_data/compilation_templates/mind_map.yaml b/api/db/init_data/compilation_templates/mind_map.yaml
index 383be48e6e..136d79b44e 100644
--- a/api/db/init_data/compilation_templates/mind_map.yaml
+++ b/api/db/init_data/compilation_templates/mind_map.yaml
@@ -2,6 +2,12 @@ kind: mind_map
display_name: Mind map - Radial concept hierarchy
config:
kind: mind_map
+ rechunk_rules: |
+ - Group only adjacent source chunks into coherent semantic units for this knowledge-compilation task.
+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.
+ - Keep each main idea with its supporting content, and split when the topic or branch changes.
+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.
+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.
entity:
description: >-
You are a robust mind-map extractor. Extract the central concept,
diff --git a/api/db/init_data/compilation_templates/page_index.yaml b/api/db/init_data/compilation_templates/page_index.yaml
index f17519daf1..18213b5f00 100644
--- a/api/db/init_data/compilation_templates/page_index.yaml
+++ b/api/db/init_data/compilation_templates/page_index.yaml
@@ -2,6 +2,14 @@ kind: page_index
display_name: PageIndex — Hierarchical table of contents
config:
kind: page_index
+ rechunk_rules: |
+ - Group only adjacent source chunks into coherent semantic units for this page-index task.
+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.
+ - Keep each heading with the content it introduces, and split when the topic, section, or semantic focus changes.
+ - Preserve document heading hierarchy: keep a parent heading with its introductory content and nearby child-section context.
+ - Do not combine separate sections merely because their subjects are related; keep each section boundary recoverable.
+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal rows or items.
+ - Do not merge unrelated content. Do not invent, rewrite, or return source text; return only chunk grouping metadata.
entity:
description: >-
You are a source-grounded document index extractor. First extract every
diff --git a/api/db/init_data/compilation_templates/session_essence.yaml b/api/db/init_data/compilation_templates/session_essence.yaml
index dfe7bb6888..62ddef4e4a 100644
--- a/api/db/init_data/compilation_templates/session_essence.yaml
+++ b/api/db/init_data/compilation_templates/session_essence.yaml
@@ -2,6 +2,12 @@ kind: session_essence
display_name: Session Essence — Cross-source entity synopses from conversations
config:
kind: knowledge_graph
+ rechunk_rules: |
+ - Group only adjacent source chunks that discuss the same conversation topic or entity.
+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.
+ - Split when the speaker, topic, intent, or semantic focus changes.
+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.
+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.
entity:
description: >-
You are a robust entity and fact extractor for conversational data.
diff --git a/api/db/init_data/compilation_templates/session_graph.yaml b/api/db/init_data/compilation_templates/session_graph.yaml
index 3bbdb1efb5..d9dd23daf4 100644
--- a/api/db/init_data/compilation_templates/session_graph.yaml
+++ b/api/db/init_data/compilation_templates/session_graph.yaml
@@ -2,6 +2,12 @@ kind: session_graph
display_name: Session Graph — Knowledge graph from conversations
config:
kind: knowledge_graph
+ rechunk_rules: |
+ - Group only adjacent source chunks that discuss the same conversation topic or entity.
+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.
+ - Split when the speaker, topic, intent, or semantic focus changes.
+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.
+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.
entity:
description: >-
You are a robust entity and fact extractor for conversational data.
diff --git a/api/db/init_data/compilation_templates/timeline.yaml b/api/db/init_data/compilation_templates/timeline.yaml
index 00bee1485d..728603a315 100644
--- a/api/db/init_data/compilation_templates/timeline.yaml
+++ b/api/db/init_data/compilation_templates/timeline.yaml
@@ -2,6 +2,12 @@ kind: timeline
display_name: List (Timeline) — Chronological events / Graph
config:
kind: timeline
+ rechunk_rules: |
+ - Group only adjacent source chunks that belong to the same event or continuous time period.
+ - Preserve source order: use every source chunk exactly once; do not omit, overlap, duplicate, or reorder chunks.
+ - Split when the event, time period, or semantic focus changes.
+ - Keep self-contained tables, lists, figures, and code blocks intact; do not split their internal structure.
+ - Do not invent, rewrite, or return source text; return only chunk grouping metadata.
entity:
description: >-
You are a robust events-timeline extractor.
diff --git a/rag/advanced_rag/knowlege_compile/_common.py b/rag/advanced_rag/knowlege_compile/_common.py
index f426b61071..77393d6427 100644
--- a/rag/advanced_rag/knowlege_compile/_common.py
+++ b/rag/advanced_rag/knowlege_compile/_common.py
@@ -57,6 +57,7 @@ def knowledge_compile_gen_conf(chat_mdl, gen_conf: Optional[dict] = None) -> dic
model_name = str(model_config.get("llm_name") or getattr(chat_mdl, "llm_name", "")).lower()
if "deepseek-v4" in model_name:
+ conf["max_completion_tokens"] = 32768
extra_body = conf.get("extra_body")
extra_body = dict(extra_body) if isinstance(extra_body, dict) else {}
extra_body["thinking"] = {"type": "disabled"}
diff --git a/rag/advanced_rag/knowlege_compile/runner.py b/rag/advanced_rag/knowlege_compile/runner.py
index 61d37cf6de..3f855b5d5c 100644
--- a/rag/advanced_rag/knowlege_compile/runner.py
+++ b/rag/advanced_rag/knowlege_compile/runner.py
@@ -65,6 +65,9 @@ DOC_STRUCTURE_COMPILE_BATCH_CHUNKS = 4
# before the LLM call.
STRUCTURE_CONTEXT_FRACTION = 0.5
STRUCTURE_DEFAULT_CONTEXT = 100_000
+KNOWLEDGE_GRAPH_CONTEXT_FRACTION = 0.1
+KNOWLEDGE_GRAPH_MIN_BATCH_TOKENS = 2048
+KNOWLEDGE_GRAPH_MAX_BATCH_TOKENS = 4096
# Bound the number of batch/template extraction calls in flight. Results are
# committed in submission order so accumulator updates and merge flushes stay
@@ -310,7 +313,7 @@ async def run_structure_compile_over_batches(
# compile_kwd(s) each template actually wrote, harvested from flush results
# so a dataset-scope template can rebuild its dataset graph once at the end.
compile_kwds_by_tid: dict[str, set[str]] = {tid: set() for tid, _ in active_templates}
- agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0} for tid, _ in active_templates}
+ agg_infos: dict[str, dict] = {tid: {"inserted": 0, "updated": 0, "duplicates_dropped": 0, "rechunked_chunks": []} for tid, _ in active_templates}
chunks_by_id: dict[str, str] = {}
flush_sequence = 0
flush_tasks: set[asyncio.Task[None]] = set()
@@ -414,6 +417,10 @@ async def run_structure_compile_over_batches(
async def _commit_result(batch_no: int, batch_len: int, template_id: str, docs: list[dict]) -> None:
if docs:
accumulators[template_id].extend(docs)
+ rechunked_chunks = getattr(docs, "rechunked_chunks", None)
+ if rechunked_chunks:
+ known_ids = {chunk.get("id") for chunk in agg_infos[template_id]["rechunked_chunks"]}
+ agg_infos[template_id]["rechunked_chunks"].extend(chunk for chunk in rechunked_chunks if chunk.get("id") not in known_ids)
if len(accumulators[template_id]) >= DOC_STRUCTURE_MERGE_MAX_DOCS:
progress_cb(msg=f" merge flush ({len(accumulators[template_id])} docs) for batch {batch_no} ({batch_len} chunks) for template ({template_ids_by_id[template_id]}/{total})")
await _flush(template_id)
@@ -428,6 +435,11 @@ async def run_structure_compile_over_batches(
def _dynamic_batch_budget(template_id: str) -> int:
max_length = getattr(chat_mdl_by_tid[template_id], "max_length", None) or STRUCTURE_DEFAULT_CONTEXT
+ if template_kinds.get(template_id) == "knowledge_graph":
+ return min(
+ max(int(max_length * KNOWLEDGE_GRAPH_CONTEXT_FRACTION), KNOWLEDGE_GRAPH_MIN_BATCH_TOKENS),
+ KNOWLEDGE_GRAPH_MAX_BATCH_TOKENS,
+ )
return max(int(max_length * STRUCTURE_CONTEXT_FRACTION), 1024)
async def _commit_ready() -> None:
@@ -489,9 +501,6 @@ async def run_structure_compile_over_batches(
for template_id, parser_cfg in active_templates:
if cancel_check():
raise TaskCanceledException("Task was cancelled during document knowledge compilation")
- if template_kinds[template_id] == "knowledge_graph":
- await _submit_one(incoming_batch, template_id, parser_cfg)
- continue
buffer = dynamic_buffers[template_id]
budget = _dynamic_batch_budget(template_id)
buffer_tokens = dynamic_buffer_tokens[template_id]
@@ -622,8 +631,26 @@ async def run_structure_compile_over_batches(
raise TaskCanceledException("Task was cancelled during document knowledge compilation")
agg = agg_infos[template_id]
if record:
- record(f"document_structure_compile:{template_id}", agg)
- progress_cb(msg=f"Document knowledge compilation done ({idx + 1}/{total}): {agg}")
+ recorded_agg = {key: value for key, value in agg.items() if key != "rechunked_chunks"}
+ recorded_agg["rechunked_chunk_count"] = len(agg.get("rechunked_chunks") or [])
+ record(f"document_structure_compile:{template_id}", recorded_agg)
+ rechunked_chunks = agg.get("rechunked_chunks") or []
+ if rechunked_chunks:
+ progress_cb(
+ msg=(
+ f"Rechunk: {len(chunks_by_id)} -> {len(rechunked_chunks)} chunks; "
+ f"inserted={agg.get('inserted', 0)}, updated={agg.get('updated', 0)}, "
+ f"duplicates_dropped={agg.get('duplicates_dropped', 0)}"
+ )
+ )
+ else:
+ progress_cb(
+ msg=(
+ f"Document knowledge compilation done ({idx + 1}/{total}): "
+ f"inserted={agg.get('inserted', 0)}, updated={agg.get('updated', 0)}, "
+ f"duplicates_dropped={agg.get('duplicates_dropped', 0)}"
+ )
+ )
# ── Synthesis phase ──────────────────────────────────────────────
# If the template has synthesis.enabled, run wiki PLAN+REFINE
diff --git a/rag/advanced_rag/knowlege_compile/structure.py b/rag/advanced_rag/knowlege_compile/structure.py
index bc45452bf7..66385c1aee 100644
--- a/rag/advanced_rag/knowlege_compile/structure.py
+++ b/rag/advanced_rag/knowlege_compile/structure.py
@@ -18,6 +18,8 @@ import asyncio
import heapq
import json
import logging
+import re
+import uuid
from typing import Awaitable, Callable, Tuple
import xxhash
@@ -68,6 +70,14 @@ _STRUCT_MERGE_LOCK_TIMEOUT_S = 60
_STRUCT_MERGE_LOCK_BLOCKING_TIMEOUT_S = 5
+class _RechunkedDocs(list):
+ """Compiled structure rows plus the formal chunks created by rechunking."""
+
+ def __init__(self, docs=None, rechunked_chunks=None):
+ super().__init__(docs or [])
+ self.rechunked_chunks = rechunked_chunks or []
+
+
def _struct_merge_lock_key(kb_id: str, compilation_template_id: str | None) -> str:
"""Per-(kb, template) lock so different templates can merge in parallel."""
return f"struct_merge:{kb_id}:{compilation_template_id or ''}"
@@ -280,7 +290,7 @@ def _struct_render_type_fields(fields: list, language: str, *, kind: str) -> Tup
return "\n".join(lines), skeleton
-def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tuple[str, str]:
+def _struct_hypergraph_prompts(parser_config: dict, language: str = "en", rechunk: bool = False) -> Tuple[str, str]:
autotype = _struct_infer_type(parser_config)
guideline = _struct_get(parser_config, "guideline", default={}) or {}
output = _struct_get(parser_config, "output", default={}) or {}
@@ -292,6 +302,7 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tup
rules_r = _struct_localize(_struct_get(guideline, "rules_for_relations"), language)
rules_t = _struct_localize(_struct_get(guideline, "rules_for_time"), language)
global_rules = _struct_localize(_struct_get(parser_config, "global_rules"), language)
+ rechunk_rules = _struct_localize(_struct_get(parser_config, "rechunk_rules"), language)
observation_time = _struct_get(options, "observation_time") or datetime.date.today().isoformat()
if rules_t and "{observation_time}" in rules_t:
@@ -318,12 +329,26 @@ def _struct_hypergraph_prompts(parser_config: dict, language: str = "en") -> Tup
if ent_desc:
node_parts.append(f"## Entity Description:\n{ent_desc}")
node_parts.append(f"## Entity Fields:\n{ent_fields_text}")
- node_parts.append(
- "## Response Format:\n"
- "Reply with a single JSON object of the form: "
- f'{{"items": [{ent_skel}, ...]}}.\n'
- f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."
- )
+ if rechunk:
+ node_parts.append(
+ "## Semantic Chunking Rules:\n"
+ f"{rechunk_rules}\n\n"
+ "## Response Format:\n"
+ "First group the source chunks according to the rules above. "
+ "Do not return chunk text. Return temporary chunk ids (c1, c2, ...) and the source chunk ids included in each group. "
+ 'Use compact inclusive ranges for consecutive source ids, for example ["t1-t3", "t8"]. '
+ "Then extract entities and use only those temporary chunk ids in each entity's source_chunk_ids. "
+ "Reply with a single JSON object of the form: "
+ f'{{"chunks": [{{"id": "c1", "source_chunk_ids": ["source-id", ...]}}], "items": [{ent_skel}, ...]}}.\n'
+ f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."
+ )
+ else:
+ node_parts.append(
+ "## Response Format:\n"
+ "Reply with a single JSON object of the form: "
+ f'{{"items": [{ent_skel}, ...]}}.\n'
+ f'Auto-type: "{_struct_infer_type(parser_config)}". ' + ("Items must be unique. " if autotype == "set" else "") + "Return JSON only, no commentary."
+ )
node_prompt = "\n\n".join(node_parts)
if not relations_cfg:
@@ -383,8 +408,38 @@ def _struct_unwrap_items(res) -> list:
return []
-async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, language: str) -> Tuple[list[dict], list[dict]]:
- node_prompt, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language)
+def _struct_expand_source_chunk_ids(raw_ids, source_texts: dict[str, str]) -> list[str]:
+ """Expand compact positional source IDs such as t1-t3."""
+ if isinstance(raw_ids, str):
+ raw_ids = [raw_ids]
+ if not isinstance(raw_ids, list):
+ return []
+
+ expanded: list[str] = []
+ available = set(source_texts)
+ for raw_id in raw_ids:
+ value = str(raw_id).strip()
+ match = re.fullmatch(r"t(\d+)\s*-\s*t(\d+)", value, flags=re.IGNORECASE)
+ if match:
+ start, end = (int(match.group(1)), int(match.group(2)))
+ step = 1 if start <= end else -1
+ values = [f"t{index}" for index in range(start, end + step, step)]
+ else:
+ values = [value]
+ for chunk_id in values:
+ if chunk_id in available and chunk_id not in expanded:
+ expanded.append(chunk_id)
+ return expanded
+
+
+async def _struct_extract_hypergraph(
+ text: str,
+ parser_config: dict,
+ chat_mdl,
+ language: str,
+ rechunk: bool = False,
+) -> Tuple[list[dict], list[dict], dict[str, str], list[dict]]:
+ node_prompt, edge_prompt_template = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk)
user_prompt = (
"## Source Text:\n"
@@ -395,6 +450,56 @@ async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, l
)
node_res = await gen_json(node_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))
nodes = _struct_unwrap_items(node_res)
+ chunk_id_map: dict[str, str] = {}
+ rechunked_chunks: list[dict] = []
+ relation_text = text
+
+ if rechunk:
+ source_texts = {match.group(1).strip(): match.group(2).strip() for match in re.finditer(r"\[CHUNK_ID:\s*([^\]]+)\]\n(.*?)\n\[END_CHUNK\]", text, flags=re.DOTALL)}
+ groups = node_res.get("chunks") if isinstance(node_res, dict) else None
+ valid_groups = []
+ claimed_sources: set[str] = set()
+ if isinstance(groups, list):
+ for group in groups:
+ if not isinstance(group, dict):
+ continue
+ sources = [source for source in _struct_expand_source_chunk_ids(group.get("source_chunk_ids"), source_texts) if source not in claimed_sources]
+ if not sources:
+ continue
+ claimed_sources.update(sources)
+ temp_id = str(group.get("id") or f"c{len(valid_groups) + 1}").strip()
+ if temp_id in {item[0] for item in valid_groups}:
+ temp_id = f"c{len(valid_groups) + 1}"
+ valid_groups.append((temp_id, sources))
+
+ for source_id in source_texts:
+ if source_id not in claimed_sources:
+ valid_groups.append((f"c{len(valid_groups) + 1}", [source_id]))
+
+ relation_segments = []
+ temp_to_uuid: dict[str, str] = {}
+ for temp_id, source_ids in valid_groups:
+ new_id = uuid.uuid4().hex
+ temp_to_uuid[temp_id] = new_id
+ for source_id in source_ids:
+ chunk_id_map[source_id] = new_id
+ grouped_text = "\n\n".join(source_texts[source_id] for source_id in source_ids)
+ relation_segments.append(f"[CHUNK_ID: {new_id}]\n{grouped_text}\n[END_CHUNK]")
+ rechunked_chunks.append(
+ {
+ "id": new_id,
+ "text": grouped_text,
+ "source_chunk_ids": source_ids,
+ }
+ )
+ relation_text = "\n\n".join(relation_segments) or text
+
+ for node in nodes:
+ raw_ids = node.get("source_chunk_ids")
+ if isinstance(raw_ids, str):
+ raw_ids = [raw_ids]
+ if isinstance(raw_ids, list):
+ node["source_chunk_ids"] = [temp_to_uuid.get(str(item).strip(), str(item).strip()) for item in raw_ids if temp_to_uuid.get(str(item).strip())]
id_field = _struct_entity_id_field(parser_config)
known_keys = []
@@ -408,13 +513,31 @@ async def _struct_extract_hypergraph(text: str, parser_config: dict, chat_mdl, l
known_str = "- " + "\n- ".join(known_keys) if known_keys else "(none)"
if not edge_prompt_template:
- return nodes, []
+ return nodes, [], chunk_id_map, rechunked_chunks
edge_prompt = edge_prompt_template.replace("{known_nodes}", known_str)
- edge_res = await gen_json(edge_prompt, user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))
+ edge_user_prompt = (
+ user_prompt
+ if not rechunk
+ else (
+ "## Source Text:\n"
+ "Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK]. "
+ "For every relation, return source_chunk_ids containing only the IDs of chunks that support that relation.\n"
+ f"{relation_text}\n\n## Output (JSON only):"
+ )
+ )
+ edge_res = await gen_json(edge_prompt, edge_user_prompt, chat_mdl, gen_conf=_knowledge_compile_gen_conf(chat_mdl, {"temperature": 0.1}))
edges = _struct_unwrap_items(edge_res)
- return nodes, edges
+ if rechunk:
+ for edge in edges:
+ raw_ids = edge.get("source_chunk_ids")
+ if isinstance(raw_ids, str):
+ raw_ids = [raw_ids]
+ if isinstance(raw_ids, list):
+ edge["source_chunk_ids"] = [item for item in raw_ids if isinstance(item, str) and item in set(chunk_id_map.values())]
+
+ return nodes, edges, chunk_id_map, rechunked_chunks
def _struct_payload_chunk_ids(payload: dict, batch_ids: list) -> list:
@@ -677,7 +800,7 @@ async def _struct_process_batch(
semaphore,
compilation_template_id: str | None = None,
compilation_template_kind: str | None = None,
-) -> list[dict]:
+) -> _RechunkedDocs:
"""Process one packed batch end-to-end (extract → embed → ES docs).
``packed`` is the per-batch shape produced by
@@ -690,50 +813,57 @@ async def _struct_process_batch(
embedding work to bound peak concurrency.
"""
if not packed:
- return []
+ return _RechunkedDocs()
batch_ids: list = [e["chunk_id"] for e in packed if e.get("chunk_id")]
batch_segments: list[str] = [f"[CHUNK_ID: {e['chunk_id']}]\n{e['text']}\n[END_CHUNK]" for e in packed if e.get("chunk_id") and isinstance(e.get("text"), str)]
combined_text = "\n\n".join(batch_segments)
-
src_field, target_field = _struct_relation_member_fields(parser_config)
+ rechunk = bool(parser_config.get("rechunk"))
- async def _run() -> list[dict]:
+ async def _run() -> _RechunkedDocs:
# For hypergraph, entity extraction MUST complete before edge extraction
# within the same batch, because the edge prompt's {known_nodes}
# placeholder is filled from this batch's extracted nodes — see
# _struct_extract_hypergraph. Parallelism across batches is fine; the
# two stages within one batch are strictly sequential.
try:
- items, relations = await _struct_extract_hypergraph(combined_text, parser_config, chat_mdl, language)
+ items, relations, chunk_id_map, formal_chunks = await _struct_extract_hypergraph(
+ combined_text,
+ parser_config,
+ chat_mdl,
+ language,
+ rechunk=rechunk,
+ )
except Exception as e:
logging.exception(f"compile_structure_from_text: extraction failed for batch {batch_idx}: {e}")
- return []
+ return _RechunkedDocs()
payloads = items + relations
kinds = ["entity"] * len(items) + ["relation"] * len(relations)
+ payload_chunk_ids = list(dict.fromkeys(chunk_id_map.values())) if chunk_id_map else batch_ids
if not payloads:
if callback:
callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: 0 items")
- return []
+ return _RechunkedDocs(rechunked_chunks=formal_chunks)
embed_inputs = [_struct_payload_description(p) for p in payloads]
try:
embeddings = await _struct_embed(embd_mdl, embed_inputs)
except Exception as e:
logging.exception(f"compile_structure_from_text: embedding failed for batch {batch_idx}: {e}")
- return []
+ return _RechunkedDocs(rechunked_chunks=formal_chunks)
if len(embeddings) != len(payloads):
logging.error(f"compile_structure_from_text: embedding count mismatch ({len(embeddings)} vs {len(payloads)}) for batch {batch_idx}")
- return []
+ return _RechunkedDocs(rechunked_chunks=formal_chunks)
docs = [
_struct_to_doc_storage_doc(
payload,
autotype,
doc_id,
- _struct_payload_chunk_ids(payload, batch_ids),
+ _struct_payload_chunk_ids(payload, payload_chunk_ids),
vec,
kind,
src_field=src_field,
@@ -747,7 +877,7 @@ async def _struct_process_batch(
if callback:
callback((batch_idx + 1) / total, f"{batch_idx + 1}/{total} batches: {len(payloads)} items")
- return docs
+ return _RechunkedDocs(docs, formal_chunks)
if semaphore is not None:
async with semaphore:
@@ -810,7 +940,8 @@ async def compile_structure_from_text(
logging.error(f"compile_structure_from_text: unsupported type '{autotype}'")
return []
- node_prompt, edge_prompt = _struct_hypergraph_prompts(parser_config, language)
+ rechunk = bool(parser_config.get("rechunk"))
+ node_prompt, edge_prompt = _struct_hypergraph_prompts(parser_config, language, rechunk=rechunk)
prompt_overhead = max(num_tokens_from_string(node_prompt), num_tokens_from_string(edge_prompt))
# ``kind`` for the row stamp follows the template's ``kind`` field if
@@ -847,12 +978,15 @@ async def compile_structure_from_text(
compilation_template_kind=template_kind,
)
- def _flatten(per_batch: list) -> list[dict]:
+ def _flatten(per_batch: list) -> _RechunkedDocs:
out: list[dict] = []
+ formal_chunks: list[dict] = []
for br in per_batch or []:
- if br:
- out.extend(br)
- return out
+ if br is None:
+ continue
+ out.extend(br)
+ formal_chunks.extend(getattr(br, "rechunked_chunks", []))
+ return _RechunkedDocs(out, formal_chunks)
return await _run_chunked_pipeline(
packed_batches,
diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.py
index b0dec7a1d4..eb1bbfb078 100644
--- a/rag/flow/compiler/compiler.py
+++ b/rag/flow/compiler/compiler.py
@@ -14,6 +14,7 @@
# limitations under the License.
import logging
import random
+import re
from copy import deepcopy
from types import SimpleNamespace
@@ -26,6 +27,8 @@ from api.db.services.knowledgebase_service import KnowledgebaseService
from api.db.services.llm_service import LLMBundle
from api.db.services.task_service import has_canceled
from common.constants import LLMType
+from common.token_utils import num_tokens_from_string
+from rag.nlp import naive_merge
from rag.advanced_rag.knowlege_compile.runner import (
DOC_STRUCTURE_COMPILE_BATCH_CHUNKS,
load_active_templates,
@@ -58,6 +61,288 @@ class CompilerParam(ProcessParamBase, LLMParam):
class Compiler(ProcessBase, LLM):
component_name = "Compiler"
+ _PARSER_TEXT_CHUNK_TOKEN_SIZE = 1024
+ _PARSER_CANDIDATE_TOKEN_SIZE = 128
+ _PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE = 256
+ _PARSER_MIN_SPLIT_CHARACTERS = 24
+
+ @classmethod
+ def _split_slumber_level(cls, text: str, delimiters: list[str]) -> list[str]:
+ """Split at boundaries while keeping delimiters with the previous part."""
+ if not text:
+ return []
+ pattern = "|".join(re.escape(delimiter) for delimiter in sorted(delimiters, key=len, reverse=True))
+ if not pattern:
+ return [text]
+
+ parts = []
+ start = 0
+ for match in re.finditer(pattern, text, flags=re.DOTALL):
+ end = match.end()
+ if end - start >= cls._PARSER_MIN_SPLIT_CHARACTERS:
+ parts.append(text[start:end])
+ start = end
+ if start < len(text):
+ parts.append(text[start:])
+
+ # Keep very short delimiter fragments with their neighbour, matching
+ # Chonkie's min_chars behavior instead of producing tiny candidates.
+ merged = []
+ for part in parts:
+ if merged and len(part.strip()) < cls._PARSER_MIN_SPLIT_CHARACTERS:
+ merged[-1] += part
+ else:
+ merged.append(part)
+ return [part for part in merged if part.strip()]
+
+ @classmethod
+ def _slumber_candidates(cls, text: str, level: int = 0, target_token_size: int | None = None) -> list[str]:
+ """Apply paragraph→sentence splitting with token fallback."""
+ target_token_size = target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE
+ levels = [
+ ["\n\n", "\r\n", "\n", "\r"],
+ [". ", "! ", "? ", "。", "!", "?", ";", ";"],
+ ]
+ if not text.strip():
+ return []
+ if level >= len(levels):
+ return [part for part in naive_merge(text, target_token_size, "", 0) if part.strip()]
+
+ splits = cls._split_slumber_level(text, levels[level])
+ if len(splits) <= 1 and level < len(levels) - 1:
+ return cls._slumber_candidates(text, level + 1, target_token_size)
+
+ candidates = []
+ for split in splits:
+ if num_tokens_from_string(split) > target_token_size:
+ candidates.extend(cls._slumber_candidates(split, level + 1, target_token_size))
+ else:
+ candidates.append(split)
+ return candidates
+
+ @staticmethod
+ def _is_atomic_json_record(record: dict) -> bool:
+ """Keep structured or position-bound Parser records intact."""
+ doc_type = str(record.get("doc_type_kwd") or "").strip().lower()
+ layout_type = str(record.get("layout_type") or "").strip().lower()
+ text = record.get("text") or record.get("content_with_weight")
+ has_markdown_table = False
+ has_html_table = False
+ has_fenced_code = False
+ if isinstance(text, str):
+ lines = text.splitlines()
+ has_markdown_table = any(index + 1 < len(lines) and re.match(r"^\s*\|", lines[index]) and Compiler._is_markdown_table_separator(lines[index + 1]) for index in range(len(lines)))
+ has_html_table = bool(re.search(r"<table(?:\s|>)", text, flags=re.IGNORECASE))
+ has_fenced_code = bool(re.search(r"^\s*(`{3,}|~{3,})", text, flags=re.MULTILINE))
+ return bool(
+ doc_type in {"table", "image"}
+ or record.get("img_id")
+ or record.get("image")
+ or layout_type in {"figure", "table"}
+ or record.get("bbox")
+ or has_markdown_table
+ or has_html_table
+ or has_fenced_code
+ )
+
+ @staticmethod
+ def _formalize_rechunked_chunks(chunks: list[dict], rechunked_chunks: list[dict], doc_id: str) -> list[dict]:
+ """Replace parser chunks with the semantic groups produced by rechunking."""
+ originals = {str(chunk.get("id")): chunk for chunk in chunks if chunk.get("id")}
+ result = []
+ for grouped in rechunked_chunks:
+ source_ids = [str(value) for value in grouped.get("source_chunk_ids") or []]
+ source = next((originals[source_id] for source_id in source_ids if source_id in originals), None)
+ if source is None:
+ logging.warning(
+ "Compiler: skip rechunked group %s because none of its source chunks exist",
+ grouped.get("id"),
+ )
+ continue
+ item = deepcopy(source)
+ item["id"] = str(grouped.get("id"))
+ item["doc_id"] = doc_id
+ item["text"] = grouped.get("text") or ""
+ for key in list(item):
+ if key.startswith("q_") and key.endswith("_vec"):
+ del item[key]
+ for key in ("content_with_weight", "content_ltks", "content_sm_ltks", "position_int", "page_num_int", "top_int"):
+ item.pop(key, None)
+ result.append(item)
+ return result
+
+ @staticmethod
+ def _is_markdown_table_separator(line: str) -> bool:
+ cells = line.strip().strip("|").split("|")
+ return len(cells) >= 1 and all(re.fullmatch(r"\s*:?-{3,}:?\s*", cell) for cell in cells)
+
+ @classmethod
+ def _split_markdown_blocks(cls, text: str) -> list[tuple[str, str | None]]:
+ """Separate Markdown tables and fenced code blocks as atomic blocks."""
+ lines = text.splitlines(keepends=True)
+ blocks: list[tuple[str, str | None]] = []
+ ordinary: list[str] = []
+ index = 0
+
+ def flush_ordinary() -> None:
+ if ordinary:
+ blocks.append(("".join(ordinary), False))
+ ordinary.clear()
+
+ while index < len(lines):
+ fence_match = re.match(r"^\s*(`{3,}|~{3,})", lines[index])
+ if fence_match:
+ flush_ordinary()
+ fence = fence_match.group(1)
+ fence_char = fence[0]
+ fence_length = len(fence)
+ code = [lines[index]]
+ index += 1
+ while index < len(lines):
+ code.append(lines[index])
+ closing = re.match(
+ rf"^\s*{re.escape(fence_char)}{{{fence_length},}}\s*$",
+ lines[index].rstrip("\r\n"),
+ )
+ index += 1
+ if closing:
+ break
+ blocks.append(("".join(code), "fenced_code"))
+ continue
+
+ if index + 1 < len(lines) and re.match(r"^\s*\|", lines[index]) and cls._is_markdown_table_separator(lines[index + 1]):
+ flush_ordinary()
+ table = [lines[index], lines[index + 1]]
+ index += 2
+ while index < len(lines) and re.match(r"^\s*\|", lines[index]):
+ table.append(lines[index])
+ index += 1
+ blocks.append(("".join(table), "table"))
+ continue
+ ordinary.append(lines[index])
+ index += 1
+
+ flush_ordinary()
+ return blocks
+
+ @classmethod
+ def _merge_small_text_chunks(cls, chunks: list[dict], target_token_size: int) -> list[dict]:
+ """Merge adjacent ordinary text chunks up to the coarse target."""
+ if target_token_size < cls._PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE:
+ return chunks
+
+ merged = []
+ pending = None
+ for chunk in chunks:
+ text = chunk.get("text")
+ if not isinstance(text, str) or not text.strip() or cls._is_atomic_json_record(chunk):
+ if pending is not None:
+ merged.append(pending)
+ pending = None
+ merged.append(chunk)
+ continue
+
+ if pending is None:
+ pending = chunk
+ continue
+
+ combined = f"{pending['text']}\n\n{text}"
+ if num_tokens_from_string(pending["text"]) < cls._PARSER_MIN_COARSE_CHUNK_TOKEN_SIZE and num_tokens_from_string(combined) <= target_token_size:
+ pending["text"] = combined
+ else:
+ merged.append(pending)
+ pending = chunk
+
+ if pending is not None:
+ merged.append(pending)
+ return merged
+
+ @classmethod
+ def _normalize_upstream_chunks(
+ cls,
+ upstream: dict,
+ split_json_text: bool = False,
+ target_token_size: int | None = None,
+ ) -> list[dict]:
+ """Normalize direct Parser output into Compiler chunks.
+
+ Parser JSON is a list of chunk-like records. JSON records are split by
+ the requested token target while retaining their metadata; upstream
+ ``chunks`` preserve atomic table/image records unless rechunking is
+ explicitly requested. Textual Parser outputs use the same split.
+ """
+ output_format = upstream.get("output_format")
+ if output_format == "chunks":
+ chunks = upstream.get("chunks") or []
+ if not isinstance(chunks, list):
+ return []
+ if not split_json_text:
+ normalized = deepcopy(chunks)
+ for chunk in normalized:
+ if not isinstance(chunk, dict):
+ continue
+ text = chunk.get("text")
+ if not isinstance(text, str):
+ text = chunk.get("content_with_weight")
+ chunk["text"] = text if isinstance(text, str) else ""
+ return normalized
+ records = chunks
+ elif output_format == "json":
+ records = upstream.get("json") or upstream.get("json_result") or []
+ if not isinstance(records, list):
+ return []
+ else:
+ records = None
+
+ if records is not None:
+ chunks = []
+ for record in records:
+ if not isinstance(record, dict):
+ continue
+ chunk = deepcopy(record)
+ text = chunk.get("text")
+ if not isinstance(text, str):
+ text = chunk.get("content_with_weight")
+ if isinstance(text, str) and text.strip():
+ if Compiler._is_atomic_json_record(chunk) or not split_json_text:
+ chunk["text"] = text
+ chunks.append(chunk)
+ else:
+ for part in Compiler._slumber_candidates(text, target_token_size=target_token_size):
+ split_chunk = deepcopy(chunk)
+ split_chunk["text"] = part
+ chunks.append(split_chunk)
+ else:
+ # Keep metadata-only records (for example, an image
+ # placeholder) instead of silently dropping them.
+ chunk["text"] = text if isinstance(text, str) else ""
+ chunks.append(chunk)
+ if split_json_text:
+ return cls._merge_small_text_chunks(chunks, target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE)
+ return chunks
+
+ if output_format in {"markdown", "text", "html"}:
+ text = upstream.get(output_format)
+ if not isinstance(text, str):
+ text = upstream.get(f"{output_format}_result")
+ if not isinstance(text, str) or not text.strip():
+ return []
+ if not split_json_text:
+ return [{"text": text}]
+ if output_format == "markdown":
+ chunks = []
+ for block, block_type in cls._split_markdown_blocks(text):
+ if block_type == "table":
+ chunks.append({"text": block, "doc_type_kwd": "table"})
+ elif block_type == "fenced_code":
+ chunks.append({"text": block})
+ else:
+ chunks.extend({"text": part} for part in cls._slumber_candidates(block, target_token_size=target_token_size) if part.strip())
+ else:
+ chunks = [{"text": part} for part in cls._slumber_candidates(text, target_token_size=target_token_size) if part.strip()]
+ return cls._merge_small_text_chunks(chunks, target_token_size or cls._PARSER_TEXT_CHUNK_TOKEN_SIZE)
+
+ return []
def _compile_progress(self, prog=None, msg=""):
"""Adapt the knowledge-compile ``callback`` protocol to the flow
@@ -210,31 +495,35 @@ class Compiler(ProcessBase, LLM):
# kwargs. Do not call LLM.get_input_elements() here: it resolves the
# inherited prompt variables through Canvas.globals, while Pipeline
# is a Graph and has no globals.
- chunks = deepcopy(kwargs.get("chunks") or [])
- if not chunks:
- for val in kwargs.values():
- if isinstance(val, list):
- chunks = deepcopy(val)
+ output_format = kwargs.get("output_format")
+ if output_format == "chunks":
+ raw_chunks = kwargs.get("chunks") or []
+ if not isinstance(raw_chunks, list) or not raw_chunks:
+ self.set_output("chunks", [])
+ return
+ # Normalize chunks only after the active templates are known, so
+ # rechunk mode does not normalize the same input twice.
+ chunks = None
+ else:
+ chunks = self._normalize_upstream_chunks(kwargs)
tenant_id = self._canvas.get_tenant_id()
doc_id = self._canvas._doc_id
kb_id = getattr(self._canvas, "_kb_id", None) or DocumentService.get_knowledgebase_id(doc_id)
language = self._compile_language(kwargs)
- if not chunks:
+ if chunks is not None and not chunks:
self.set_output("chunks", chunks)
return
- for ck in chunks:
- ck["doc_id"] = doc_id
- ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()
-
# Resolve the configured template groups to concrete, active
# (non-artifact) structure-compilation templates.
template_ids = resolve_template_ids_from_groups(self._param.compilation_template_group_ids, tenant_id)
active_templates = load_active_templates(template_ids, tenant_id)
if not active_templates:
self.callback(0, "No active compilation templates resolved from the configured groups.")
+ if chunks is None:
+ chunks = self._normalize_upstream_chunks(kwargs)
self.set_output("chunks", chunks)
return
@@ -279,10 +568,52 @@ class Compiler(ProcessBase, LLM):
filtered_templates.append((template_id, parser_cfg))
if not filtered_templates:
+ if chunks is None:
+ chunks = self._normalize_upstream_chunks(kwargs)
self.set_output("chunks", chunks)
return
active_templates = filtered_templates
+ def _template_requests_rechunk(cfg: dict) -> bool:
+ if not isinstance(cfg, dict):
+ return False
+ # ``raptor.rechunk`` belongs to the Tree path, which is explicitly
+ # excluded from the semantic rechunk staging below. It must not
+ # shrink non-Tree parser chunks unless a structure template has
+ # requested the actual regrouping pass.
+ return bool(cfg.get("rechunk"))
+
+ should_rechunk = any(_template_requests_rechunk(cfg) for _, cfg in active_templates)
+ target_token_size = self._PARSER_CANDIDATE_TOKEN_SIZE if should_rechunk else self._PARSER_TEXT_CHUNK_TOKEN_SIZE
+ if output_format == "chunks":
+ chunks = self._normalize_upstream_chunks(
+ kwargs,
+ split_json_text=should_rechunk,
+ target_token_size=target_token_size,
+ )
+ elif output_format in {"markdown", "text", "html"}:
+ chunks = self._normalize_upstream_chunks(
+ kwargs,
+ split_json_text=should_rechunk,
+ target_token_size=target_token_size,
+ )
+ elif output_format == "json":
+ chunks = self._normalize_upstream_chunks(
+ kwargs,
+ split_json_text=should_rechunk,
+ target_token_size=target_token_size,
+ )
+ if not chunks:
+ self.set_output("chunks", chunks)
+ return
+ if not chunks:
+ self.set_output("chunks", chunks)
+ return
+
+ for ck in chunks:
+ ck["doc_id"] = doc_id
+ ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()
+
if self._canvas._kb_id:
e, kb = KnowledgebaseService.get_by_id(self._canvas._kb_id)
if kb.tenant_embd_id:
@@ -315,6 +646,17 @@ class Compiler(ProcessBase, LLM):
)
if non_tree_templates:
+ first_rechunk_index = next(
+ (index for index, (_, cfg) in enumerate(non_tree_templates) if _template_requests_rechunk(cfg)),
+ None,
+ )
+ if first_rechunk_index is not None:
+ # Keep the stable hash IDs for Tree. Non-tree rechunking uses
+ # compact positional IDs to keep the LLM grouping response
+ # small (e.g. t1-t3 instead of three long hash IDs).
+ for index, chunk in enumerate(chunks, start=1):
+ chunk["id"] = f"t{index}"
+
task_id = getattr(self._canvas, "task_id", None)
def _cancelled() -> bool:
@@ -324,17 +666,48 @@ class Compiler(ProcessBase, LLM):
for i in range(0, len(chunks), DOC_STRUCTURE_COMPILE_BATCH_CHUNKS):
yield chunks[i : i + DOC_STRUCTURE_COMPILE_BATCH_CHUNKS]
- await run_structure_compile_over_batches(
- active_templates=non_tree_templates,
- chat_mdl_by_tid=chat_mdl_by_tid,
- embedding_model=embedding_model,
- tenant_id=tenant_id,
- kb_id=kb_id,
- doc_id=doc_id,
- language=language,
- chunk_batches=_chunk_batches(),
- progress_cb=self._compile_progress,
- cancel_check=_cancelled,
- )
+ async def _run_templates(template_specs):
+ return await run_structure_compile_over_batches(
+ active_templates=template_specs,
+ chat_mdl_by_tid=chat_mdl_by_tid,
+ embedding_model=embedding_model,
+ tenant_id=tenant_id,
+ kb_id=kb_id,
+ doc_id=doc_id,
+ language=language,
+ chunk_batches=_chunk_batches(),
+ progress_cb=self._compile_progress,
+ cancel_check=_cancelled,
+ )
+
+ if first_rechunk_index is None:
+ await _run_templates(non_tree_templates)
+ else:
+ if first_rechunk_index:
+ await _run_templates(non_tree_templates[:first_rechunk_index])
+ first_template = [non_tree_templates[first_rechunk_index]]
+ compile_info = await _run_templates(first_template)
+ first_template_id = first_template[0][0]
+ formal_chunks = (compile_info.get(first_template_id) or {}).get("rechunked_chunks")
+ if formal_chunks:
+ formalized_chunks = self._formalize_rechunked_chunks(chunks, formal_chunks, doc_id)
+ if formalized_chunks:
+ chunks = formalized_chunks
+ else:
+ logging.warning(
+ "Compiler: rechunking returned no valid formal chunks for doc %s; keeping original chunks",
+ doc_id,
+ )
+
+ # The first rechunk template defines the semantic boundaries
+ # for all later templates. They consume those formal chunks
+ # and must not perform another independent rechunking pass.
+ later_templates = []
+ for template_id, parser_cfg in non_tree_templates[first_rechunk_index + 1 :]:
+ later_config = dict(parser_cfg or {})
+ later_config["rechunk"] = False
+ later_templates.append((template_id, later_config))
+ if later_templates:
+ await _run_templates(later_templates)
self.set_output("chunks", chunks)
diff --git a/rag/llm/chat_model.py b/rag/llm/chat_model.py
index d4015e1673..0ab017bd2f 100644
--- a/rag/llm/chat_model.py
+++ b/rag/llm/chat_model.py
@@ -1691,8 +1691,30 @@ class LiteLLMBase(ABC):
gen_conf=gen_conf,
)
- gen_conf.pop("max_tokens", None)
+ deepseek_max_tokens = None
+ if self.provider == SupportedLiteLLMProvider.DeepSeek:
+ # DeepSeek's API uses the legacy OpenAI-compatible max_tokens field.
+ # LiteLLM accepts max_completion_tokens generically, but does not
+ # translate it for DeepSeek and the provider then falls back to 8192.
+ # Knowledge compilation supplies max_completion_tokens explicitly
+ # through its model-specific generation configuration. Otherwise,
+ # preserve the legacy max_tokens value used by existing callers.
+ raw_max_completion_tokens = gen_conf.pop("max_completion_tokens", None)
+ raw_max_tokens = gen_conf.pop("max_tokens", None)
+ raw_limit = raw_max_completion_tokens if raw_max_completion_tokens is not None else raw_max_tokens
+ if raw_limit is not None and not isinstance(raw_limit, bool):
+ try:
+ candidate = int(raw_limit)
+ except (TypeError, ValueError):
+ candidate = 0
+ if candidate > 0:
+ deepseek_max_tokens = candidate
+ else:
+ gen_conf.pop("max_tokens", None)
+
gen_conf = {k: v for k, v in gen_conf.items() if k in LITELLM_ALLOWED_GEN_CONF_KEYS}
+ if deepseek_max_tokens is not None:
+ gen_conf["max_tokens"] = deepseek_max_tokens
return gen_conf
def _need_reasoning_content_back(self) -> bool:
diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py
index 3e6cb4b300..484ff20920 100644
--- a/rag/svr/task_executor_refactor/chunk_post_processor.py
+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py
@@ -378,8 +378,6 @@ from rag.advanced_rag.knowlege_compile.runner import ( # noqa: E402
load_active_templates,
run_structure_compile_over_batches,
)
-
-
# ----- parser_config helpers -----------------------------------------
diff --git a/web/src/interfaces/database/compilation-template.ts b/web/src/interfaces/database/compilation-template.ts
index 564611c3e6..b29876e69e 100644
--- a/web/src/interfaces/database/compilation-template.ts
+++ b/web/src/interfaces/database/compilation-template.ts
@@ -24,6 +24,8 @@ export interface ICompilationTemplateConfig {
relation?: ICompilationTemplateSection;
raptor?: ICompilationTemplateRaptorConfig;
global_rules?: string;
+ rechunk?: boolean;
+ rechunk_rules?: string;
[section: string]:
| ICompilationTemplateSection
| ICompilationTemplateRaptorConfig
diff --git a/web/src/interfaces/request/compilation-template.ts b/web/src/interfaces/request/compilation-template.ts
index 81058f246a..6ab69cb280 100644
--- a/web/src/interfaces/request/compilation-template.ts
+++ b/web/src/interfaces/request/compilation-template.ts
@@ -24,6 +24,8 @@ export interface ICompilationTemplateConfigRequest {
relation?: ICompilationTemplateSectionRequest;
raptor?: ICompilationTemplateRaptorConfigRequest;
global_rules?: string;
+ rechunk?: boolean;
+ rechunk_rules?: string;
[section: string]:
| ICompilationTemplateSectionRequest
| ICompilationTemplateRaptorConfigRequest
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 7ad38f99cc..25be19c67e 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -1900,6 +1900,12 @@ Example: Virtual Hosted Style`,
rechunkByTreeLeaves: 'Re-chunk by tree leaves',
rechunkByTreeLeavesTip:
"Merge each leaf cluster's source chunks into a single replacement chunk. Originals are kept but marked unavailable for retrieval. Only one tree template per group may enable this.",
+ rechunkInput: 'Re-chunk parser output',
+ rechunkInputTip:
+ 'Let the LLM determine chunk boundaries based on the knowledge compilation task.',
+ rechunkRules: 'Rechunking rules',
+ rechunkRulesPlaceholder:
+ 'Describe how the LLM should group source chunks for this compilation task.',
jsonPreview: 'JSON preview',
processFlow: 'Process flow',
processFlowComingSoon: 'Process flow preview coming soon',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 9b0f22c1a5..68f3632ee9 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -1590,6 +1590,11 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系
rechunkByTreeLeaves: '按树叶重新分块',
rechunkByTreeLeavesTip:
'将每个叶簇的源数据块合并为单个替换数据块。原始数据块保留但标记为不可检索。每个分组最多只能有一个树模板启用此功能。',
+ rechunkInput: '重新切分 Parser 输出',
+ rechunkInputTip: '根据知识编译任务,由 LLM 决定 chunk 边界。',
+ rechunkRules: 'Rechunk 规则',
+ rechunkRulesPlaceholder:
+ '描述此知识编译任务下,LLM 应如何合并和划分源 chunk。',
jsonPreview: 'JSON 预览',
processFlow: '流程视图',
processFlowComingSoon: '流程视图预览即将到来',
diff --git a/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx b/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx
index def1f126ac..65a826160f 100644
--- a/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx
+++ b/web/src/pages/user-setting/compilation-templates/create-next/components/template-configuration.tsx
@@ -1,6 +1,7 @@
import { ModelTreeSelectFormField } from '@/components/model-tree-select';
import { SelectWithSearch } from '@/components/originui/select-with-search';
import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { SwitchFormField } from '@/components/switch-fom-field';
import { Button } from '@/components/ui/button';
import { Input } from '@/components/ui/input';
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
@@ -65,6 +66,10 @@ export function TemplateConfiguration({
control: form.control,
name: `templates.${selectedTemplateIndex}.kind`,
});
+ const rechunk = useWatch({
+ control: form.control,
+ name: `templates.${selectedTemplateIndex}.config.rechunk`,
+ });
const availableKindOptions = useAvailableKindOptions(
form,
@@ -198,39 +203,62 @@ export function TemplateConfiguration({
{kind === CompilationTemplateKind.Tree ? (
<TreeTemplateFields index={selectedTemplateIndex} />
) : (
- sectionNames.length > 0 &&
- activeSectionTab && (
- <Tabs
- value={activeSectionTab}
- onValueChange={setActiveSectionTab}
- className="w-full"
- >
- <TabsList className="w-full justify-start">
+ <>
+ {kind !== CompilationTemplateKind.Artifacts && (
+ <>
+ <SwitchFormField
+ name={`templates.${selectedTemplateIndex}.config.rechunk`}
+ label={t('setting.rechunkInput')}
+ tooltip={t('setting.rechunkInputTip')}
+ vertical={false}
+ />
+ {rechunk && (
+ <RAGFlowFormItem
+ name={`templates.${selectedTemplateIndex}.config.rechunk_rules`}
+ label={t('setting.rechunkRules')}
+ >
+ <Textarea
+ placeholder={t('setting.rechunkRulesPlaceholder')}
+ rows={6}
+ resize="vertical"
+ />
+ </RAGFlowFormItem>
+ )}
+ </>
+ )}
+ {sectionNames.length > 0 && activeSectionTab && (
+ <Tabs
+ value={activeSectionTab}
+ onValueChange={setActiveSectionTab}
+ className="w-full"
+ >
+ <TabsList className="w-full justify-start">
+ {sectionNames.map((sectionName) => (
+ <TabsTrigger
+ key={sectionName}
+ value={sectionName}
+ className="flex-1"
+ >
+ {t(
+ SectionTitleKeyMap[sectionName] ??
+ startCase(sectionName),
+ )}
+ </TabsTrigger>
+ ))}
+ </TabsList>
+
{sectionNames.map((sectionName) => (
- <TabsTrigger
+ <TabsContent
key={sectionName}
value={sectionName}
- className="flex-1"
+ className="mt-4"
>
- {t(
- SectionTitleKeyMap[sectionName] ??
- startCase(sectionName),
- )}
- </TabsTrigger>
+ {renderSectionTabs(sectionName)}
+ </TabsContent>
))}
- </TabsList>
-
- {sectionNames.map((sectionName) => (
- <TabsContent
- key={sectionName}
- value={sectionName}
- className="mt-4"
- >
- {renderSectionTabs(sectionName)}
- </TabsContent>
- ))}
- </Tabs>
- )
+ </Tabs>
+ )}
+ </>
)}
</div>
</div>
diff --git a/web/src/pages/user-setting/compilation-templates/create-next/utils.ts b/web/src/pages/user-setting/compilation-templates/create-next/utils.ts
index 834430cbd8..d9cd24640e 100644
--- a/web/src/pages/user-setting/compilation-templates/create-next/utils.ts
+++ b/web/src/pages/user-setting/compilation-templates/create-next/utils.ts
@@ -61,6 +61,8 @@ export const DefaultTemplateValues: TemplateSchemaType = {
instruction: '',
page_example: '',
use_blueprint: false,
+ rechunk: false,
+ rechunk_rules: '',
},
};
@@ -81,6 +83,8 @@ export const isConfigMetaKey = (key: string) =>
'page_example',
'synthesis',
'use_blueprint',
+ 'rechunk',
+ 'rechunk_rules',
].includes(key);
export const createEmptyField = (keys: string[]) =>
@@ -132,6 +136,15 @@ export const buildConfigFromBuiltin = (
: {}),
use_blueprint:
kind === CompilationTemplateKind.Artifacts && example.length > 0,
+ ...(kind !== CompilationTemplateKind.Tree
+ ? {
+ rechunk: builtinTemplate.config?.rechunk === true,
+ rechunk_rules:
+ typeof builtinTemplate.config?.rechunk_rules === 'string'
+ ? builtinTemplate.config.rechunk_rules
+ : '',
+ }
+ : {}),
};
if (kind === CompilationTemplateKind.Artifacts && example.length > 0) {
@@ -183,6 +196,15 @@ export const transformDetailToForm = (
: {}),
use_blueprint:
detail.kind === CompilationTemplateKind.Artifacts && example.length > 0,
+ ...(detail.kind !== CompilationTemplateKind.Tree
+ ? {
+ rechunk: config.rechunk === true,
+ rechunk_rules:
+ typeof config.rechunk_rules === 'string'
+ ? config.rechunk_rules
+ : '',
+ }
+ : {}),
};
if (detail.kind === CompilationTemplateKind.Artifacts && example.length > 0) {
@@ -258,7 +280,8 @@ export const transformTemplateToPayload = (template: TemplateSchemaType) => {
return;
}
if (isConfigMetaKey(key)) {
- if (typeof value === 'string') config[key] = value;
+ if (typeof value === 'string' || typeof value === 'boolean')
+ config[key] = value;
} else {
config[key] = value as ICompilationTemplateConfigRequest[string];
}
diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx b/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx
index adacf90b1f..89a41ec0e8 100644
--- a/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx
+++ b/web/src/pages/user-setting/compilation-templates/edit-next/components/template-configuration.tsx
@@ -1,6 +1,7 @@
import { ModelTreeSelectFormField } from '@/components/model-tree-select';
import { SelectWithSearch } from '@/components/originui/select-with-search';
import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { SwitchFormField } from '@/components/switch-fom-field';
import { Input } from '@/components/ui/input';
import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs';
import { Textarea } from '@/components/ui/textarea';
@@ -58,6 +59,10 @@ export function TemplateConfiguration({
control: form.control,
name: `templates.${selectedTemplateIndex}.kind`,
});
+ const rechunk = useWatch({
+ control: form.control,
+ name: `templates.${selectedTemplateIndex}.config.rechunk`,
+ });
const availableKindOptions = useAvailableKindOptions(
form,
@@ -191,39 +196,62 @@ export function TemplateConfiguration({
{kind === CompilationTemplateKind.Tree ? (
<TreeTemplateFields index={selectedTemplateIndex} />
) : (
- sectionNames.length > 0 &&
- activeSectionTab && (
- <Tabs
- value={activeSectionTab}
- onValueChange={setActiveSectionTab}
- className="w-full"
- >
- <TabsList className="w-full justify-start">
+ <>
+ {kind !== CompilationTemplateKind.Artifacts && (
+ <>
+ <SwitchFormField
+ name={`templates.${selectedTemplateIndex}.config.rechunk`}
+ label={t('setting.rechunkInput')}
+ tooltip={t('setting.rechunkInputTip')}
+ vertical={false}
+ />
+ {rechunk && (
+ <RAGFlowFormItem
+ name={`templates.${selectedTemplateIndex}.config.rechunk_rules`}
+ label={t('setting.rechunkRules')}
+ >
+ <Textarea
+ placeholder={t('setting.rechunkRulesPlaceholder')}
+ rows={6}
+ resize="vertical"
+ />
+ </RAGFlowFormItem>
+ )}
+ </>
+ )}
+ {sectionNames.length > 0 && activeSectionTab && (
+ <Tabs
+ value={activeSectionTab}
+ onValueChange={setActiveSectionTab}
+ className="w-full"
+ >
+ <TabsList className="w-full justify-start">
+ {sectionNames.map((sectionName) => (
+ <TabsTrigger
+ key={sectionName}
+ value={sectionName}
+ className="flex-1"
+ >
+ {t(
+ SectionTitleKeyMap[sectionName] ??
+ startCase(sectionName),
+ )}
+ </TabsTrigger>
+ ))}
+ </TabsList>
+
{sectionNames.map((sectionName) => (
- <TabsTrigger
+ <TabsContent
key={sectionName}
value={sectionName}
- className="flex-1"
+ className="mt-4"
>
- {t(
- SectionTitleKeyMap[sectionName] ??
- startCase(sectionName),
- )}
- </TabsTrigger>
+ {renderSectionTabs(sectionName)}
+ </TabsContent>
))}
- </TabsList>
-
- {sectionNames.map((sectionName) => (
- <TabsContent
- key={sectionName}
- value={sectionName}
- className="mt-4"
- >
- {renderSectionTabs(sectionName)}
- </TabsContent>
- ))}
- </Tabs>
- )
+ </Tabs>
+ )}
+ </>
)}
{children}
diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts b/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts
index 13b23d9a76..f21a6d275e 100644
--- a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts
+++ b/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts
@@ -57,6 +57,8 @@ export const DefaultTemplateValues: TemplateSchemaType = {
instruction: '',
page_example: '',
use_blueprint: false,
+ rechunk: false,
+ rechunk_rules: '',
},
};
@@ -77,6 +79,8 @@ export const isConfigMetaKey = (key: string) =>
'page_example',
'synthesis',
'use_blueprint',
+ 'rechunk',
+ 'rechunk_rules',
].includes(key);
export const createEmptyField = (keys: string[]) =>
@@ -128,6 +132,15 @@ export const buildConfigFromBuiltin = (
: {}),
use_blueprint:
kind === CompilationTemplateKind.Artifacts && example.length > 0,
+ ...(kind !== CompilationTemplateKind.Tree
+ ? {
+ rechunk: builtinTemplate.config?.rechunk === true,
+ rechunk_rules:
+ typeof builtinTemplate.config?.rechunk_rules === 'string'
+ ? builtinTemplate.config.rechunk_rules
+ : '',
+ }
+ : {}),
};
if (kind === CompilationTemplateKind.Artifacts && example.length > 0) {
@@ -179,6 +192,15 @@ export const transformDetailToForm = (
: {}),
use_blueprint:
detail.kind === CompilationTemplateKind.Artifacts && example.length > 0,
+ ...(detail.kind !== CompilationTemplateKind.Tree
+ ? {
+ rechunk: config.rechunk === true,
+ rechunk_rules:
+ typeof config.rechunk_rules === 'string'
+ ? config.rechunk_rules
+ : '',
+ }
+ : {}),
};
if (detail.kind === CompilationTemplateKind.Artifacts && example.length > 0) {
@@ -254,7 +276,8 @@ export const transformTemplateToPayload = (template: TemplateSchemaType) => {
return;
}
if (isConfigMetaKey(key)) {
- if (typeof value === 'string') config[key] = value;
+ if (typeof value === 'string' || typeof value === 'boolean')
+ config[key] = value;
} else {
config[key] = value as ICompilationTemplateConfigRequest[string];
}