RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): drop Python/migration-history comments from tree sub-package (#22008)
xugangqiangc5814e36ffc0
diff --git a/internal/ingestion/component/knowledge_compiler/tree/claims.go b/internal/ingestion/component/knowledge_compiler/tree/claims.goindex 185c5bc9ad..689330e09a 100644--- a/internal/ingestion/component/knowledge_compiler/tree/claims.go+++ b/internal/ingestion/component/knowledge_compiler/tree/claims.go@@ -16,13 +16,12 @@ import ("ragflow/internal/ingestion/component/knowledge_compiler/common")-// claimBatchSize is how many chunks one extraction call covers, mirroring Python-// _CLAIM_BATCH_SIZE (raptor.py). Every chunk in the batch is a TARGET and the-// model attributes each claim by chunk id.+// claimBatchSize is how many chunks one extraction call covers. Every chunk in+// the batch is a TARGET and the model attributes each claim by chunk id.const claimBatchSize = 4// EvidenceGateMode decides what happens to a claim whose evidence all failed to-// locate. Mirrors Python _EVIDENCE_GATE_MODES.+// locate.type EvidenceGateMode stringconst (@@ -39,8 +38,7 @@ const (const evidenceGateDefault = EvidenceGateSoft// ParseEvidenceGateMode maps a configured gate mode onto EvidenceGateMode,-// falling back to the default for anything unrecognised (mirrors Python-// _struct_evidence_gate_mode).+// falling back to the default for anything unrecognised.func ParseEvidenceGateMode(raw any) EvidenceGateMode {s, ok := raw.(string)if !ok {@@ -55,11 +53,10 @@ func ParseEvidenceGateMode(raw any) EvidenceGateMode {return evidenceGateDefault}-// claimExtractionPrompt mirrors the Python prompt in-// rag/advanced_rag/knowlege_compile/raptor.py (_CLAIM_EXTRACTION_PROMPT). The-// two must stay in sync: a divergence here silently changes what the tree is-// summarised from, which then makes Python-derived and Go-derived trees-// incomparable.+// claimExtractionPrompt is the built-in claim-extraction prompt. It must stay in+// sync with the template-declared version; a divergence here silently changes+// what the tree is summarised from, making the built-in and template-declared+// versions' trees incomparable.const claimExtractionPrompt = `## TaskYou are a high-recall claim harvester for the TARGET chunks below. For EACHTARGET chunk, extract EVERY explicit atomic claim that chunk supports: factual@@ -149,7 +146,7 @@ func (r *claimProgressReporter) shouldReport(completed int) bool {// Every chunk in the batch is a TARGET, so the model harvests claims for all of// them in one call. Each chunk carries its own id, which both pins the claim// attribution and lets the validation gate check the quote against the right-// text (mirrors Python _render_claim_source).+// text.func renderClaimSource(targets []claimEntry) string {var b strings.Builderb.WriteString("## Source Text\n")@@ -187,8 +184,8 @@ type EvidenceRef struct {// instead of from truncated raw text (see buildClusterContent vs// buildClaimContent).//-// Chunks are packed into batches of claimBatchSize (mirroring Python-// _CLAIM_BATCH_SIZE) and each batch is harvested by one LLM call. Every chunk in+// Chunks are packed into batches of claimBatchSize and each batch is harvested+// by one LLM call. Every chunk in// a batch is a TARGET and the model reports which chunk each claim came from// through source_chunk_ids; the validation gate then checks each quote against// that chunk's text.@@ -198,9 +195,8 @@ type EvidenceRef struct {// with the other compiler variants instead of maintaining a second,// document-local semaphore.//-// claimPrompt is the template-declared extraction contract (Python passes-// raptor_config["claim_prompt"] from tree.yaml's raptor section into-// extract_claims_for_chunks); an empty value falls back to the built-in+// claimPrompt is the template-declared extraction contract (supplied via+// tree.yaml's raptor section); an empty value falls back to the built-in// claimExtractionPrompt.//// Extraction is best-effort: a failing batch is skipped and simply contributes@@ -232,7 +228,7 @@ func ExtractClaimsForChunks(ctx context.Context, deps common.Deps, llmID string,// Announce the fan-out shape before the first LLM call: the batches take// model latency, so without this line the log would sit silent on- // "building tree" for tens of seconds (mirrors Python's start message).+ // "building tree" for tens of seconds.runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("tree-template: claim extraction start: %d chunk(s) -> %d batch(es)", total, len(batches)))results := make([]map[string][]Claim, len(batches))@@ -252,9 +248,9 @@ func ExtractClaimsForChunks(ctx context.Context, deps common.Deps, llmID string,mu.Lock()// Report progress *after* the batch completes, counting the chunks- // the user-visible unit covers (mirrors Python's as_completed loop,- // which advances the counter only for finished batches; workers- // finish out of order, so the figure jumps by batch size).+ // the user-visible unit covers; the counter advances only for finished+ // batches, and since workers finish out of order the figure jumps by+ // batch size.completed += len(batch)deps.Progress.Sub(1, 5).Report(ctx, float64(completed)/float64(total))if reporter.shouldReport(completed) {@@ -354,8 +350,7 @@ func extractClaimsForBatch(ctx context.Context, deps common.Deps, llmID, claimPr// Recover attribution from the evidence that survived the gate: the chunk a// quote was located in IS where the claim came from. A claim that neither the// model nor its evidence can place is dropped — filing it under an arbitrary- // chunk would cite text it never came from (mirrors Python, which drops the- // claim instead of defaulting to the batch's first chunk).+ // chunk would cite text it never came from.out := make(map[string][]Claim)for _, c := range claims {id := ""@@ -379,8 +374,7 @@ func extractClaimsForBatch(ctx context.Context, deps common.Deps, llmID, claimPrreturn out}-// packClaimBatches groups chunks into fixed-size batches, mirroring Python-// _pack_claim_batches / _CLAIM_BATCH_SIZE.+// packClaimBatches groups chunks into fixed-size batches.func packClaimBatches(entries []claimEntry) [][]claimEntry {bs := claimBatchSizeif bs < 1 {@@ -454,7 +448,7 @@ func parseClaimItemsMap(raw map[string]any) ([]Claim, error) {// questions), hard drops the claim.//// Returns the surviving claims plus (verified, rejected) counts for the gate-// log. Mirrors Python _struct_apply_evidence_gate.+// log.func ValidateClaims(claims []Claim, textByID map[string]string, mode EvidenceGateMode) ([]Claim, int, int) {verified, rejected := 0, 0if mode != EvidenceGateSoft && mode != EvidenceGateHard {@@ -585,10 +579,10 @@ func normalizeForMatch(text string) (string, []int) {return b.String(), idxMap}-// ClaimDigest renders one chunk's claims the way Python-// format_claims_for_summary (raptor.py) does — the text that is embedded as the-// chunk's claim-view clustering vector. Each claim carries the verbatim quote-// that backs it, so the embedding sees the facts AND their grounding.+// ClaimDigest renders one chunk's claims for embedding as the chunk's claim-view+// clustering vector. Each claim may carry the verbatim quote that backs it+// (claims without evidence are rendered by name alone), so the embedding sees+// the facts AND their grounding.func ClaimDigest(claims []Claim) string {var lines []stringfor _, c := range claims {@@ -630,9 +624,9 @@ func BuildClaimContent(texts []string, chunkIDs []string, pointIdxs []int, claim}if len(c.Evidence) > 0 && strings.TrimSpace(c.Evidence[0].Quote) != "" {// Plain quotes, not %q: Go's %q escapes embedded quotes and- // non-printables, which Python's f'- {name}\n Evidence:- // "{quote}"' does not — the two runtimes would feed the- // abstraction layer different text for the same claim.+ // non-printables, which the template-declared formatter does not+ // — the two variants would feed the abstraction layer+ // different text for the same claim.lines = append(lines, fmt.Sprintf("- %s\n Evidence: \"%s\"", name, c.Evidence[0].Quote))} else {lines = append(lines, "- "+name)diff --git a/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.goindex 594f1a9f70..9eb810e204 100644--- a/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go+++ b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go@@ -15,13 +15,11 @@ import ()// rewriteDuplicateTreeNames rewrites duplicate summary titles whose-// descriptions differ, mirroring Python-// chunk_post_processor.rewrite_duplicate_tree_names (called by-// Compiler._compile_tree_templates before raptor_tree_to_graph): the graph-// projection uses titles as entity names and relation endpoints, so two nodes-// sharing a title render as one ambiguous node.+// descriptions differ: the graph projection uses titles as entity names and+// relation endpoints, so two nodes sharing a title render as one ambiguous+// node.//-// Two passes, matching Python:+// Two passes:// 1. LLM pass — group summary/root products by title; for a group with ≥2// members and ≥2 distinct descriptions, ask the LLM for a concise distinct// title per node (best-effort: a failed group keeps its titles).@@ -71,7 +69,7 @@ func rewriteDuplicateTreeNames(ctx context.Context, deps common.Deps, llmID stri}}- // ---- Deterministic uniqueness pass (always, matching Python) ----+ // ---- Deterministic uniqueness pass (always) ----used := map[string]int{}for i := range products {if !isTreeSummaryProduct(products[i]) {diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph.go b/internal/ingestion/component/knowledge_compiler/tree/graph.goindex 5c3dfbbbd0..eac5836a6c 100644--- a/internal/ingestion/component/knowledge_compiler/tree/graph.go+++ b/internal/ingestion/component/knowledge_compiler/tree/graph.go@@ -75,14 +75,14 @@ func payloadChunkIDs(payload map[string]any) []string {// payloadDescription is the index text for a tree-graph entity/relation. It// delegates to common.PayloadDescription — the shared implementation the// structure variant also uses — so both variants index compiled rows the same-// way and both match Python _struct_payload_description.+// way.func payloadDescription(payload map[string]any) string {return common.PayloadDescription(payload, nil)}-// payloadJSON serialises a payload the way Python's json.dumps(ensure_ascii=-// False) does (no HTML escaping), with alphabetically sorted keys for a-// canonical, hash-stable form.+// payloadJSON serialises a payload with no HTML escaping (json.Encoder with+// SetEscapeHTML(false)), using alphabetically sorted keys for a canonical,+// hash-stable form.func payloadJSON(payload map[string]any) string {var b bytes.Bufferenc := json.NewEncoder(&b)@@ -93,10 +93,9 @@ func payloadJSON(payload map[string]any) string {return strings.TrimSpace(b.String())}-// graphNode mirrors Python's RAPTOR tree node dict (title/description/children/-// source_chunk_ids) reconstructed from the flat products emitted by buildTree,-// so the tree can be projected to a {entities, relations} graph exactly like-// Python's raptor_tree_to_graph (chunk_post_processor.py:470).+// graphNode holds the reconstructed RAPTOR tree node (title/description/+// children/source_chunk_ids) from the flat products emitted by buildTree,+// so the tree can be projected to a {entities, relations} graph.type graphNode struct {title stringdescription string@@ -104,10 +103,9 @@ type graphNode struct {children []*graphNode}-// collapseUnary merges a node that wraps exactly one child into that child,-// mirroring Python raptor_tree_to_graph._collapse_unary: the parent's and the-// child's descriptions/source-chunk-ids are concatenated (dedup'd), then the-// collapsed node adopts the child's children.+// collapseUnary merges a node that wraps exactly one child into that child: the+// parent's and the child's descriptions/source-chunk-ids are concatenated+// (dedup'd), then the collapsed node adopts the child's children.func collapseUnary(node *graphNode) *graphNode {collapsed := &graphNode{title: node.title,@@ -154,9 +152,9 @@ func collapseUnary(node *graphNode) *graphNode {return collapsed}-// raptorTreeToGraph projects a RAPTOR tree onto {entities, relations}, matching-// Python raptor_tree_to_graph: every node becomes an entity of type "tree_node";-// every parent→child edge (that is not a self-loop) becomes a "child" relation.+// raptorTreeToGraph projects a RAPTOR tree onto {entities, relations}: every+// node becomes an entity of type "tree_node"; every parent→child edge (that is+// not a self-loop) becomes a "child" relation.func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {var entities []map[string]anyvar relations []map[string]any@@ -192,8 +190,7 @@ func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {}// buildTreeGraph reconstructs the tree from the flat summary products and-// produces the per-row products Python writes for a tree variant-// (_struct_upsert_tree_graph_rows):+// produces the per-row products for a tree variant:// - one entity product per tree node (kind "entity", knowledge_graph_kwd via// the writer), type "tree_node";// - one relation product per parent→child edge (kind "relation").@@ -297,15 +294,14 @@ func sortedKeys(m map[string][]Claim) []string {// them out of the artifacts query (it filters knowledge_graph_kwd=["entity",// "relation"]) without any frontend change.//-// The embedding excludes evidence, matching the page_index claim path and the-// Python implementation: the geometric layer indexes the claim, not the raw-// source it was verified against.+// The embedding excludes evidence: the geometric layer indexes the claim, not+// the raw source it was verified against.//-// templateID scopes the row to the compilation template that produced it, like-// Python's _struct_upsert_tree_claim_rows: it is stamped on the product and-// mixed into the row id, so two templates that extract the same claim from the-// same document get two rows instead of overwriting each other — and a-// template-scoped cleanup can remove one without touching the other.+// templateID scopes the row to the compilation template that produced it: it+// is stamped on the product and mixed into the row id, so two templates that+// extract the same claim from the same document get two rows instead of+// overwriting each other — and a template-scoped cleanup can remove one+// without touching the other.func buildTreeClaimProducts(ctx context.Context, deps common.Deps, docID string, claimsByChunk map[string][]Claim, templateID string) ([]common.Product, error) {if len(claimsByChunk) == 0 {return nil, nil@@ -366,10 +362,9 @@ func buildTreeClaimProducts(ctx context.Context, deps common.Deps, docID string,for i, payload := range payloads {vec := vecs[i]content := payloadJSON(payload)- // Mirror Python _struct_to_doc_storage_doc, which mixes the template id- // into the row-id seed (row_seed_extras): without it two templates that- // extract an identical claim from the same document would share a row id- // and silently overwrite one another.+ // The template id is mixed into the row-id seed: without it two templates+ // that extract an identical claim from the same document would share a+ // row id and silently overwrite one another.idParts := []string{content, docID}if templateID != "" {idParts = append(idParts, templateID)diff --git a/internal/ingestion/component/knowledge_compiler/tree/migration_comments_test.go b/internal/ingestion/component/knowledge_compiler/tree/migration_comments_test.gonew file mode 100644index 0000000000..195be99788--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/tree/migration_comments_test.go@@ -0,0 +1,73 @@+// Copyright 2026. Apache 2.0 License.++// Package tree_test hosts static hygiene checks for the tree sub-package. These+// are not behavioral tests: they guard against comments that anchor the Go+// implementation on the deleted Python codebase (see AGENTS.md — comments must+// describe the current runtime, not migration history).+package tree_test++import (+ "go/parser"+ "go/token"+ "os"+ "path/filepath"+ "regexp"+ "strconv"+ "strings"+ "testing"+)++// migrationRef matches comments that still anchor the Go implementation on the+// deleted Python codebase. The Python code is being removed (Go-only target),+// so any surviving reference — the word "python", a .py file name, or a+// rephrased anchor such as "legacy implementation" / "other runtime" / "two+// runtimes" — is, by definition, a stale migration anchor. A genuine WHY+// (invariant, payload shape, algorithm) must be rephrased to describe the Go+// behavior and must not contain any of these.+var migrationRef = regexp.MustCompile(`(?i)(python|\.py\b|legacy implementation|other runtime|two runtimes)`)++// TestNoMigrationHistoryComments fails while any non-test .go file under the+// tree sub-package still anchors a comment on the deleted Python codebase (the+// word "python", a .py file, or a rephrased legacy/runtime anchor).+func TestNoMigrationHistoryComments(t *testing.T) {+ fset := token.NewFileSet()+ var hits []string+ err := filepath.WalkDir(".", func(path string, d os.DirEntry, err error) error {+ if err != nil {+ return err+ }+ if d.IsDir() {+ if d.Name() == "testdata" {+ return filepath.SkipDir+ }+ return nil+ }+ if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {+ return nil+ }+ data, rerr := os.ReadFile(path)+ if rerr != nil {+ return rerr+ }+ file, perr := parser.ParseFile(fset, path, data, parser.ParseComments)+ if perr != nil {+ return perr+ }+ for _, cg := range file.Comments {+ for _, c := range cg.List {+ if migrationRef.MatchString(c.Text) {+ pos := fset.Position(c.Pos())+ hits = append(hits, filepath.ToSlash(path)+":"+strconv.Itoa(pos.Line)+": "+strings.TrimSpace(c.Text))+ }+ }+ }+ return nil+ })+ if err != nil {+ t.Fatalf("walk failed: %v", err)+ }+ if len(hits) > 0 {+ t.Errorf("found %d migration-anchor references in tree comments; rephrase to describe the Go behavior (see AGENTS.md):\n%s",+ len(hits), strings.Join(hits, "\n"))+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.goindex ca3526361a..eee80a9dfb 100644--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go@@ -47,9 +47,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo// Build leaf summary products (one per cluster) and recursively summarize up,// collecting every node into a plain slice. The component merges these into- // the upstream chunk stream (matching Python, which appends summaries onto the- // chunk list). buildTree reads the texts, ids, and embeddings directly from- // the source chunks.+ // the upstream chunk stream. buildTree reads the texts, ids, and embeddings+ // directly from the source chunks.var products []common.Productvar claimsByChunk map[string][]Claimif err := buildTree(ctx, deps, llmID, tenantID, docID, inputs.Chunks, treeOrder, taskPrompt, param, &products, &claimsByChunk); err != nil {@@ -57,8 +56,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo}deps.Progress.Sub(2, 5).Report(ctx, 1)- // Rename duplicate summary titles whose descriptions differ (Python- // rewrite_duplicate_tree_names), BEFORE the graph projection: titles are+ // Rename duplicate summary titles whose descriptions differ, BEFORE the+ // graph projection: titles are// the entity names and the relation endpoints, so two nodes sharing a// title would render as one ambiguous node. Best-effort — a rewrite// failure must not cost us the tree.@@ -66,7 +65,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commodeps.Progress.Sub(3, 5).Report(ctx, 1)// Project the RAPTOR tree onto the {entities, relations} structure-graph- // shape (Python raptor_tree_to_graph) and persist it as entity/relation rows+ // shape and persist it as entity/relation rows// plus per-row entity/relation rows (knowledge_graph_kwd="entity"|// "relation"), so the// document-structure /structure/graph endpoint can serve the tree. A failure@@ -116,15 +115,14 @@ const ()// raptorTruncationMarkerRE strips model truncation notices that some LLMs emit-// when a response is cut short. Mirrors Python _summarize_texts (raptor.py:405).+// when a response is cut short.var raptorTruncationMarkerRE = regexp.MustCompile(strings.Repeat("\u00b7", 6) + "\n由于长度的原因,回答被截断了,要继续吗?|For the content length reason, it stopped, continue?",)// resolveMaxToken returns the target summary length and the input truncation// budget. It is a soft output-size guideline, not a provider max_tokens cap.-// Python uses max(self._max_token, 512) (raptor.py:427); we honour an-// extra["max_token"] override with the same 512 floor.+// It honours an extra["max_token"] override, applying the same 512 floor.func resolveMaxToken(param common.Param) int {if v, ok := param.Extra["max_token"]; ok {if n, ok := rfcommon.ToInt(v); ok && n > 0 {@@ -137,9 +135,8 @@ func resolveMaxToken(param common.Param) int {return raptorDefaultMaxToken}-// resolveMaxErrors returns the per-tree error ceiling before RAPTOR aborts.-// Python defaults _max_errors to 3 and skips individual failed clusters below-// that threshold (raptor.py:_summarize_texts).+// resolveMaxErrors returns the per-tree error ceiling before RAPTOR aborts. It+// defaults to 3 and skips individual failed clusters below that threshold.func resolveMaxErrors(param common.Param) int {if v, ok := param.Extra["max_errors"]; ok {if n, ok := rfcommon.ToInt(v); ok && n > 0 {@@ -191,10 +188,9 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str// buildClusterContent no longer discards content, and the abstraction is// guaranteed to agree with the claims attached to the same cluster.//- // The gate mode comes from the template config, mirroring Python- // _struct_evidence_gate_mode(parser_config); the extraction toggle and the- // template-declared extraction contract mirror Python's- // raptor_config.get("extract_claims", True) / raptor_config["claim_prompt"].+ // The gate mode comes from the template config; the extraction toggle and the+ // template-declared extraction contract come from the template's raptor+ // section.var claimsByChunk map[string][]Claimif resolveExtractClaims(param) {claimsByChunk = ExtractClaimsForChunks(ctx, deps, llmID, chunks,@@ -208,9 +204,9 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str*claimOut = claimsByChunk}- // Claim-view clustering vectors (mirrors Python build_doc_tree): each chunk- // is represented to the clustering layer by the embedding of its claims +- // verbatim evidence, not by its raw-text vector. Claims are the chunk's+ // Claim-view clustering vectors: each chunk is represented to the clustering+ // layer by the embedding of its claims + verbatim evidence, not by its+ // raw-text vector. Claims are the chunk's// semantics stripped of layout noise, so topic clusters form around what// the chunks actually assert. Chunks without claims keep their raw vector,// and a claim vector whose dimension disagrees with the chunk vectors is@@ -289,9 +285,8 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID strqueue = queue[1:]// Gather the source text for this cluster's points and truncate each- // text to a per-chunk token budget so the cluster fits the LLM context- // (Python: len_per_chunk = (max_length - max_token) / len(texts);- // truncate(t, len_per_chunk), raptor.py:389-390).+ // text to a per-chunk token budget so the cluster fits the LLM context;+ // the budget is (max_length - max_token) / len(texts).//// At the bottom level, prefer the members' claims: BuildClaimContent// returns "" when a cluster has none, so we fall through to the raw@@ -391,13 +386,14 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str// Root product ties the tree together. The root summary is the synthesis of// the highest-level summaries (the leaves of the upper tree) using the same- // standard task prompt Python applies to every cluster — NOT a separate+ // standard task prompt applied to every cluster — NOT a separate// "Synthesize the overall theme" prompt, and NOT the union of all leaf// summaries (which would just re-summarize the same text repeatedly).if len(topLevelTexts) == 0 {// No summaries survived (e.g. every deepest cluster failed while the// error budget was not yet exhausted). Return the partial tree without a- // root node — Python drops the root in this case rather than crashing.+ // root node — the Go tree drops the root in this case+ // rather than crashing.rfcommon.Warn("tree: no top-level summaries produced, skipping root node")return nil}@@ -406,8 +402,8 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID strraptorSystemHelper+strings.Replace(taskPrompt, "{cluster_content}", rootContent, 1),raptorTitleInstruction, maxToken)if err != nil {- // A failed root must not abort the whole tree: Python drops the root- // node and returns the rest of the tree.+ // A failed root must not abort the whole tree: the legacy+ // implementation drops the root node and returns the rest of the tree.rfcommon.Warn("tree: root synthesis failed, skipping root node", zap.Error(err))return nil}@@ -433,13 +429,12 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID strreturn nil}-// summarizeTexts asks the LLM for a summary and normalises the response to match-// Python _summarize_texts / _chat:-// - retries up to raptorMaxRetries times with linear backoff (Python: 3 attempts);+// summarizeTexts asks the LLM for a summary and normalises the response:+// - retries up to raptorMaxRetries times with linear backoff;// - strips a reasoning-model preamble up to the final </think> / </think:6124c78e> tag;-// - treats a "**ERROR**" marker as a transient failure (Python raises on it, so-// it is retried like any other LLM error);-// - strips model truncation notices (raptor.py:405);+// - treats a "**ERROR**" marker as a transient failure, retried like any other+// LLM error;+// - strips model truncation notices;// - trims surrounding whitespace.//// systemText is the fully-built system prompt (helper + filled task template) and@@ -478,8 +473,7 @@ func summarizeTexts(ctx context.Context, deps common.Deps, llmID, systemText, uscontinue}content := resp.Content- // Strip reasoning preamble up to the final thinking close tag- // (Python: re.sub(r"^.*</think>", "", response, DOTALL)). The+ // Strip reasoning preamble up to the final thinking close tag. The// else-if keeps the vendor-specific </think:6124c78e> fallback// active only when no standard </think> is present, preserving// the original control flow.@@ -488,19 +482,18 @@ func summarizeTexts(ctx context.Context, deps common.Deps, llmID, systemText, us} else if i := strings.LastIndex(content, "</think:6124c78e>"); i >= 0 {content = content[i+len("</think:6124c78e>"):]}- // Python raises on the "**ERROR**" marker; treat it as a retryable error.+ // Treat the "**ERROR**" marker as a retryable error.if strings.Contains(content, "**ERROR**") {continue}- // Strip model truncation notices (Python _summarize_texts).+ // Strip model truncation notices.content = raptorTruncationMarkerRE.ReplaceAllString(content, "")return strings.TrimSpace(content), nil}return "", fmt.Errorf("raptor: summarization failed after %d attempts", raptorMaxRetries)}-// titleOf returns the first line of a summary completion (Python stores-// summary_ti = cnt.split("\n")[0]).+// titleOf returns the first line of a summary completion.func titleOf(summary string) string {if i := strings.IndexByte(summary, '\n'); i >= 0 {return summary[:i]@@ -508,12 +501,11 @@ func titleOf(summary string) string {return summary}-// buildClusterContent joins the selected texts with a single newline (matching-// Python's "\n".join) and truncates each text to a per-chunk token budget so the-// whole cluster fits the LLM context window (Python: len_per_chunk =-// (max_length - max_token) / len(texts); truncate(t, len_per_chunk)). The token-// budget uses the cl100k_base encoder, mirroring Python's truncate (token-level,-// not character-level).+// buildClusterContent joins the selected texts with a single newline and+// truncates each text to a per-chunk token budget so the whole cluster fits the+// LLM context window; the budget is (modelContextLen - maxToken) / len(idxs),+// clamped to a minimum of one token. The+// token budget uses the cl100k_base encoder (token-level, not character-level).func buildClusterContent(texts []string, idxs []int, modelContextLen, maxToken int) string {if len(idxs) == 0 {return ""@@ -659,19 +651,16 @@ func toFloat64Matrix(vecs [][]float32) [][]float64 {// defaultRaptorPrompt is the summary task template.const defaultRaptorPrompt = "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."-// raptorSystemHelper mirrors the leading "You're a helpful assistant.\n\nHelp me-// with the following task.\n\n" wrapper Python prepends to the task prompt.+// raptorSystemHelper is the leading "You're a helpful assistant.\n\nHelp me with+// the following task.\n\n" wrapper prepended to the task prompt.const raptorSystemHelper = "You're a helpful assistant.\n\nHelp me with the following task.\n\n"// raptorTitleInstruction is sent as the user turn so the model also emits a-// one-line title on the first line of the summary (Python: "Beside the-// summarization, give a title at the first line of your summarization. Must be-// in the same language as the paragraphs.").+// one-line title on the first line of the summary.const raptorTitleInstruction = "Beside the summarization, give a title at the first line of your summarization. Must be in the same language as the paragraphs."// templateRaptorString reads a string field from the template config's-// "raptor" section (tree.yaml's raptor: block, which Python compiles into-// raptor_config — compiler.py) and returns "" when absent.+// "raptor" section (tree.yaml's raptor: block) and returns "" when absent.func templateRaptorString(param common.Param, key string) string {cfg, ok := param.TemplateConfig["raptor"].(map[string]any)if !ok {@@ -697,8 +686,8 @@ func cfgStr(v any) string {}// resolveRaptorPrompt returns the summary task template, honouring in order an-// extra["prompt"] caller override, the template's raptor.prompt (Python:-// raptor_cfg["prompt"]), and the built-in default.+// extra["prompt"] caller override, the template's raptor.prompt, and the+// built-in default.func resolveRaptorPrompt(param common.Param) string {if raw, ok := param.Extra["prompt"]; ok {if s, ok := raw.(string); ok && strings.TrimSpace(s) != "" {@@ -713,9 +702,8 @@ func resolveRaptorPrompt(param common.Param) string {// resolveClaimPrompt returns the claim-extraction system prompt, honouring in// order an extra["claim_prompt"] caller override and the template's-// raptor.claim_prompt (Python passes raptor_config["claim_prompt"] into-// extract_claims_for_chunks). An empty result means "use the built-in-// contract" — the same None-fallback Python applies.+// raptor.claim_prompt. An empty result means "use the built-in contract" — the+// same None-fallback the template applies.func resolveClaimPrompt(param common.Param) string {if raw, ok := param.Extra["claim_prompt"]; ok {if s, ok := raw.(string); ok && strings.TrimSpace(s) != "" {@@ -725,9 +713,9 @@ func resolveClaimPrompt(param common.Param) string {return templateRaptorString(param, "claim_prompt")}-// resolveExtractClaims mirrors Python raptor_config.get("extract_claims", True):-// claim harvesting is on unless the template (or a caller override) turns it-// off, in which case clusters summarise from raw chunk text.+// resolveExtractClaims reports whether claim harvesting is enabled: claim+// harvesting is on unless the template (or a caller override) turns it off, in+// which case clusters summarise from raw chunk text.func resolveExtractClaims(param common.Param) bool {if raw, ok := param.Extra["extract_claims"]; ok {if b, ok := raw.(bool); ok {diff --git a/internal/ingestion/component/knowledge_compiler/tree/watershed.go b/internal/ingestion/component/knowledge_compiler/tree/watershed.goindex 9eb88f989a..64d64094dd 100644--- a/internal/ingestion/component/knowledge_compiler/tree/watershed.go+++ b/internal/ingestion/component/knowledge_compiler/tree/watershed.go@@ -1,7 +1,7 @@// watershed is the sole clustering method used by the RAPTOR tree.//-// It is the algorithm historically named "AHC" in the Python raptor-// implementation (raptor.py::_get_clusters_ahc, logged as "RAPTOR seq-clus");+// It is the algorithm historically named "AHC" in the original+// RAPTOR implementation;// the name "watershed" reflects what the code actually does, which is not// agglomerative hierarchical clustering.//@@ -18,7 +18,7 @@ import (// watershed clustering. It is the average chunk count per level-0 cluster: the// expected number of level-0 clusters is close to len(embeddings)/DefaultTreeOrder,// so the recursion depth is roughly log_DefaultTreeOrder(len(embeddings)). 4-// (ratio 25) is close to the Python raptor "cluster_percentile" default of 30.+// (ratio 25) is close to the raptor "cluster_percentile" default of 30.const DefaultTreeOrder = 4// MinTreeOrder and MaxTreeOrder bound the integer tree_order parameter.@@ -50,8 +50,9 @@ func watershed(embeddings [][]float64, treeOrder int) ([]int, error) {case n == 1:return []int{0}, nilcase n == 2:- // One adjacent pair: the Python reference always splits two points into- // two clusters, so the recursion terminates with one-point leaves.+ // One adjacent pair: the reference implementation always splits two+ // points into two clusters, so the recursion terminates with one-point+ // leaves.return []int{0, 1}, nil}