RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: handle title and truncate document preview by sentence (#18407)
Lynn2368c546898b
diff --git a/rag/svr/task_executor_refactor/dataset_skill_generator.py b/rag/svr/task_executor_refactor/dataset_skill_generator.pyindex bbb160d2a0..e592dbe87e 100644--- a/rag/svr/task_executor_refactor/dataset_skill_generator.py+++ b/rag/svr/task_executor_refactor/dataset_skill_generator.py@@ -122,9 +122,10 @@ class SkillNode:def skill_safe_name(text: str, max_len: int = 50) -> str:"""Lowercase, hyphen-only, max-len-clamped slug. MirrorsCorpus2Skill ``_safe_name`` for cross-system stability of folder- names."""+ names. Keeps CJK (Chinese) characters so Chinese labels survive+ the slugification."""name = (text or "").lower().strip()- name = re.sub(r"[^a-z0-9\s-]", "", name)+ name = re.sub(r"[^a-z0-9\s\u4e00-\u9fff-]", "", name)name = re.sub(r"\s+", "-", name)name = name.strip("-")[:max_len]return name@@ -136,8 +137,8 @@ async def label_skill_node_one(semaphore: asyncio.Semaphore,) -> str:"""Generate a single fs-safe label: 2–5 word lowercase- hyphenated label, max_tokens=20, sanitized to [a-z0-9-], capped- at 50 chars, falls back to "cluster" on any failure.+ hyphenated label, max_tokens=20, sanitized to [a-z0-9-\u4e00-\u9fff],+ capped at 50 chars, falls back to "cluster" on any failure."""async with semaphore:try:@@ -156,7 +157,7 @@ async def label_skill_node_one({"max_tokens": 20, "temperature": 0.0},)raw = (cnt or "").strip().lower()- label = re.sub(r"[^a-z0-9-]", "-", raw)+ label = re.sub(r"[^a-z0-9\u4e00-\u9fff-]", "-", raw)label = re.sub(r"-+", "-", label).strip("-")[:50]return label or "cluster"except Exception:@@ -220,22 +221,57 @@ async def doc_summary_for_skill()-def build_skill_md(node: "SkillNode") -> str:+_SENTENCE_ENDINGS = "。!?!?;;"+++def _strip_md_headings(text: str) -> str:+ """Convert ATX heading markers (``###`` at line start) into bold text,+ keeping the heading text and the line structure. e.g. ``### 客户服务``+ becomes ``**客户服务**``. Line breaks survive, so a multi-section+ summary keeps its readability inside the YAML ``description`` field."""+ if not text:+ return text+ return re.sub(r"^\s*#{1,6}\s+(.*?)\s*$", r"**\g<1>**", text, flags=re.MULTILINE)+++def _truncate_at_sentence_end(text: str, limit: int = 300) -> str:+ """Clip ``text`` at the last sentence-ending punctuation (or line+ break) within ``limit``, so a clipped summary never ends mid-sentence.+ Falls back to a hard cut at ``limit`` when no good boundary occurs in+ the first half of the budget."""+ if len(text) <= limit:+ return text+ head = text[:limit]+ cut = max(head.rfind(p) for p in _SENTENCE_ENDINGS + "\n")+ if cut >= limit // 2:+ return head[: cut + 1]+ return head+++def _escape_md_text(text: str) -> str:+ """Escape markdown metacharacters so a document name cannot break+ the emphasis / inline-code / link syntax it is embedded in."""+ return re.sub(r"([\\`*_\[\]])", r"\\\1", text)+++def build_skill_md(node: "SkillNode", doc_names: Optional[Dict] = None) -> str:"""SKILL.md (depth 0) / INDEX.md (deeper) text. MirrorsCorpus2Skill's ``_format_skill_md`` (skill_builder.py:193): YAMLfrontmatter (name / description / level / num_documents), then``## Overview`` with the full summary, then ``## Contents`` —- sub-groups for branches, ``- `doc_id`: <first 120 chars>`` for- leaves.+ sub-groups for branches, ``- **doc_name** (`doc_id`): <first 120+ chars>`` for leaves. ``doc_names`` maps doc_id → display name; a+ missing entry falls back to the bare ``doc_id``."""depth = node.levelname = node.folder_name or node.label or f"cluster-{depth}"- desc = (node.summary or "")[:300].replace("\n", " ").strip()+ desc = _truncate_at_sentence_end(_strip_md_headings(node.summary or ""))+ desc_block = "\n".join(f" {ln}" if ln.strip() else "" for ln in desc.split("\n"))lines: list[str] = ["---",f"name: {name}",- "description: >",- f" {desc}",+ "description: |",+ desc_block,f"level: {depth}",f"num_documents: {len(node.doc_ids)}","---",@@ -252,7 +288,7 @@ def build_skill_md(node: "SkillNode") -> str:lines.append("")for child in node.children:child_name = child.folder_name or child.label or "cluster"- summary_snip = (child.summary or "")[:200].replace("\n", " ").strip()+ summary_snip = _truncate_at_sentence_end((child.summary or "").replace("\n", " ").strip(), limit=200)lines.append(f"- **{child_name}/** ({len(child.doc_ids)} docs): {summary_snip}")lines.append("")else:@@ -260,13 +296,17 @@ def build_skill_md(node: "SkillNode") -> str:lines.append("")for doc_id in node.doc_ids:preview = node.doc_texts.get(doc_id, "")- first_line = (preview.split("\n", 1)[0] if preview else "").strip()[:120]- lines.append(f"- `{doc_id}`: {first_line}")+ first_line = _truncate_at_sentence_end((preview.split("\n", 1)[0] if preview else "").strip(), limit=120)+ doc_name = ((doc_names or {}).get(doc_id) or "").replace("\n", " ").strip()+ if doc_name:+ lines.append(f"- **{_escape_md_text(doc_name)}** (`{doc_id}`): {first_line}")+ else:+ lines.append(f"- `{doc_id}`: {first_line}")lines.append("")return "\n".join(lines)-def skill_node_es_row(ctx: TaskContext, node: "SkillNode", parent_kwd: str = "") -> Dict:+def skill_node_es_row(ctx: TaskContext, node: "SkillNode", parent_kwd: str = "", doc_names: Optional[Dict] = None) -> Dict:"""Build the ES row for one tree node. Stable id from(kb_id, folder_name) so re-runs upsert cleanly."""kb_id_str = str(ctx.kb_id)@@ -283,38 +323,38 @@ def skill_node_es_row(ctx: TaskContext, node: "SkillNode", parent_kwd: str = "")"depth_int": int(node.level),"children_kwd": [c.folder_name for c in node.children],"source_doc_ids": list(node.doc_ids),- "md_with_weight": build_skill_md(node),+ "md_with_weight": build_skill_md(node, doc_names),"available_int": 1,}-def _collect_skill_rows(ctx: TaskContext, node: "SkillNode", parent_kwd: str, out: list[dict]) -> None:+def _collect_skill_rows(ctx: TaskContext, node: "SkillNode", parent_kwd: str, out: list[dict], doc_names: Optional[Dict] = None) -> None:"""Depth-first traversal: build one ``skill`` row per node, passing``parent_kwd`` down so every non-root node records its parent."""- out.append(skill_node_es_row(ctx, node, parent_kwd))+ out.append(skill_node_es_row(ctx, node, parent_kwd, doc_names))for child in node.children:- _collect_skill_rows(ctx, child, node.folder_name, out)+ _collect_skill_rows(ctx, child, node.folder_name, out, doc_names)-def skill_tree_md_snippet(node: "SkillNode") -> str:+def skill_tree_md_snippet(node: "SkillNode", doc_names: Optional[Dict] = None) -> str:"""Return only the frontmatter/preamble before the Overview body.The one-shot tree browser needs enough metadata to render the skilldirectory without loading every full node body up front."""- md = build_skill_md(node)+ md = build_skill_md(node, doc_names)return md.split("\n## Overview", 1)[0].strip()-def skill_tree_node(node: "SkillNode") -> Dict:+def skill_tree_node(node: "SkillNode", doc_names: Optional[Dict] = None) -> Dict:return {"skill_kwd": node.folder_name,- "md_with_weight": skill_tree_md_snippet(node),- "children_kwd": [skill_tree_node(child) for child in node.children],+ "md_with_weight": skill_tree_md_snippet(node, doc_names),+ "children_kwd": [skill_tree_node(child, doc_names) for child in node.children],}-def skill_all_es_row(ctx: TaskContext, roots: list["SkillNode"]) -> Dict:+def skill_all_es_row(ctx: TaskContext, roots: list["SkillNode"], doc_names: Optional[Dict] = None) -> Dict:"""Build the aggregate tree row loaded by the Skills sidebar."""kb_id_str = str(ctx.kb_id)row_id = xxhash.xxh64(@@ -326,7 +366,7 @@ def skill_all_es_row(ctx: TaskContext, roots: list["SkillNode"]) -> Dict:"doc_id": kb_id_str,"compile_kwd": "skill_all","skill_with_weight": json.dumps(- [skill_tree_node(root) for root in roots],+ [skill_tree_node(root, doc_names) for root in roots],ensure_ascii=False,indent=2,),@@ -395,6 +435,7 @@ async def run_corpus2skill(suffix=[],)eligible_docs = [d for d in (all_docs or []) if d.get("id")]+ doc_names = {d["id"]: (d.get("name") or "") for d in eligible_docs}if not eligible_docs:progress(1.0, "skill: no documents in KB")return@@ -577,8 +618,8 @@ async def run_corpus2skill(rows: list[dict] = []for r in roots:- _collect_skill_rows(ctx, r, "", rows)- rows.append(skill_all_es_row(ctx, roots))+ _collect_skill_rows(ctx, r, "", rows, doc_names)+ rows.append(skill_all_es_row(ctx, roots, doc_names))if not rows:progress(1.0, "skill: nothing to persist")return