Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): drop Python/migration-history comments from tree sub-package (#22008)

xugangqiangc5814e36ffc0
diff --git a/internal/ingestion/component/knowledge_compiler/tree/claims.go b/internal/ingestion/component/knowledge_compiler/tree/claims.go
index 185c5bc9ad..689330e09a 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/claims.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/claims.go
@@ -16,13 +16,12 @@ import (
"ragflow/internal/ingestion/component/knowledge_compiler/common"
)
-// claimBatchSize is how many chunks one extraction call covers, mirroring Python
-// _CLAIM_BATCH_SIZE (raptor.py). Every chunk in the batch is a TARGET and the
-// model attributes each claim by chunk id.
+// claimBatchSize is how many chunks one extraction call covers. Every chunk in
+// the batch is a TARGET and the model attributes each claim by chunk id.
const claimBatchSize = 4
// EvidenceGateMode decides what happens to a claim whose evidence all failed to
-// locate. Mirrors Python _EVIDENCE_GATE_MODES.
+// locate.
type EvidenceGateMode string
const (
@@ -39,8 +38,7 @@ const (
const evidenceGateDefault = EvidenceGateSoft
// ParseEvidenceGateMode maps a configured gate mode onto EvidenceGateMode,
-// falling back to the default for anything unrecognised (mirrors Python
-// _struct_evidence_gate_mode).
+// falling back to the default for anything unrecognised.
func ParseEvidenceGateMode(raw any) EvidenceGateMode {
s, ok := raw.(string)
if !ok {
@@ -55,11 +53,10 @@ func ParseEvidenceGateMode(raw any) EvidenceGateMode {
return evidenceGateDefault
}
-// claimExtractionPrompt mirrors the Python prompt in
-// rag/advanced_rag/knowlege_compile/raptor.py (_CLAIM_EXTRACTION_PROMPT). The
-// two must stay in sync: a divergence here silently changes what the tree is
-// summarised from, which then makes Python-derived and Go-derived trees
-// incomparable.
+// claimExtractionPrompt is the built-in claim-extraction prompt. It must stay in
+// sync with the template-declared version; a divergence here silently changes
+// what the tree is summarised from, making the built-in and template-declared
+// versions' trees incomparable.
const claimExtractionPrompt = `## Task
You are a high-recall claim harvester for the TARGET chunks below. For EACH
TARGET chunk, extract EVERY explicit atomic claim that chunk supports: factual
@@ -149,7 +146,7 @@ func (r *claimProgressReporter) shouldReport(completed int) bool {
// Every chunk in the batch is a TARGET, so the model harvests claims for all of
// them in one call. Each chunk carries its own id, which both pins the claim
// attribution and lets the validation gate check the quote against the right
-// text (mirrors Python _render_claim_source).
+// text.
func renderClaimSource(targets []claimEntry) string {
var b strings.Builder
b.WriteString("## Source Text\n")
@@ -187,8 +184,8 @@ type EvidenceRef struct {
// instead of from truncated raw text (see buildClusterContent vs
// buildClaimContent).
//
-// Chunks are packed into batches of claimBatchSize (mirroring Python
-// _CLAIM_BATCH_SIZE) and each batch is harvested by one LLM call. Every chunk in
+// Chunks are packed into batches of claimBatchSize and each batch is harvested
+// by one LLM call. Every chunk in
// a batch is a TARGET and the model reports which chunk each claim came from
// through source_chunk_ids; the validation gate then checks each quote against
// that chunk's text.
@@ -198,9 +195,8 @@ type EvidenceRef struct {
// with the other compiler variants instead of maintaining a second,
// document-local semaphore.
//
-// claimPrompt is the template-declared extraction contract (Python passes
-// raptor_config["claim_prompt"] from tree.yaml's raptor section into
-// extract_claims_for_chunks); an empty value falls back to the built-in
+// claimPrompt is the template-declared extraction contract (supplied via
+// tree.yaml's raptor section); an empty value falls back to the built-in
// claimExtractionPrompt.
//
// Extraction is best-effort: a failing batch is skipped and simply contributes
@@ -232,7 +228,7 @@ func ExtractClaimsForChunks(ctx context.Context, deps common.Deps, llmID string,
// Announce the fan-out shape before the first LLM call: the batches take
// model latency, so without this line the log would sit silent on
- // "building tree" for tens of seconds (mirrors Python's start message).
+ // "building tree" for tens of seconds.
runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("tree-template: claim extraction start: %d chunk(s) -> %d batch(es)", total, len(batches)))
results := make([]map[string][]Claim, len(batches))
@@ -252,9 +248,9 @@ func ExtractClaimsForChunks(ctx context.Context, deps common.Deps, llmID string,
mu.Lock()
// Report progress *after* the batch completes, counting the chunks
- // the user-visible unit covers (mirrors Python's as_completed loop,
- // which advances the counter only for finished batches; workers
- // finish out of order, so the figure jumps by batch size).
+ // the user-visible unit covers; the counter advances only for finished
+ // batches, and since workers finish out of order the figure jumps by
+ // batch size.
completed += len(batch)
deps.Progress.Sub(1, 5).Report(ctx, float64(completed)/float64(total))
if reporter.shouldReport(completed) {
@@ -354,8 +350,7 @@ func extractClaimsForBatch(ctx context.Context, deps common.Deps, llmID, claimPr
// Recover attribution from the evidence that survived the gate: the chunk a
// quote was located in IS where the claim came from. A claim that neither the
// model nor its evidence can place is dropped — filing it under an arbitrary
- // chunk would cite text it never came from (mirrors Python, which drops the
- // claim instead of defaulting to the batch's first chunk).
+ // chunk would cite text it never came from.
out := make(map[string][]Claim)
for _, c := range claims {
id := ""
@@ -379,8 +374,7 @@ func extractClaimsForBatch(ctx context.Context, deps common.Deps, llmID, claimPr
return out
}
-// packClaimBatches groups chunks into fixed-size batches, mirroring Python
-// _pack_claim_batches / _CLAIM_BATCH_SIZE.
+// packClaimBatches groups chunks into fixed-size batches.
func packClaimBatches(entries []claimEntry) [][]claimEntry {
bs := claimBatchSize
if bs < 1 {
@@ -454,7 +448,7 @@ func parseClaimItemsMap(raw map[string]any) ([]Claim, error) {
// questions), hard drops the claim.
//
// Returns the surviving claims plus (verified, rejected) counts for the gate
-// log. Mirrors Python _struct_apply_evidence_gate.
+// log.
func ValidateClaims(claims []Claim, textByID map[string]string, mode EvidenceGateMode) ([]Claim, int, int) {
verified, rejected := 0, 0
if mode != EvidenceGateSoft && mode != EvidenceGateHard {
@@ -585,10 +579,10 @@ func normalizeForMatch(text string) (string, []int) {
return b.String(), idxMap
}
-// ClaimDigest renders one chunk's claims the way Python
-// format_claims_for_summary (raptor.py) does — the text that is embedded as the
-// chunk's claim-view clustering vector. Each claim carries the verbatim quote
-// that backs it, so the embedding sees the facts AND their grounding.
+// ClaimDigest renders one chunk's claims for embedding as the chunk's claim-view
+// clustering vector. Each claim may carry the verbatim quote that backs it
+// (claims without evidence are rendered by name alone), so the embedding sees
+// the facts AND their grounding.
func ClaimDigest(claims []Claim) string {
var lines []string
for _, c := range claims {
@@ -630,9 +624,9 @@ func BuildClaimContent(texts []string, chunkIDs []string, pointIdxs []int, claim
}
if len(c.Evidence) > 0 && strings.TrimSpace(c.Evidence[0].Quote) != "" {
// Plain quotes, not %q: Go's %q escapes embedded quotes and
- // non-printables, which Python's f'- {name}\n Evidence:
- // "{quote}"' does not — the two runtimes would feed the
- // abstraction layer different text for the same claim.
+ // non-printables, which the template-declared formatter does not
+ // — the two variants would feed the abstraction layer
+ // different text for the same claim.
lines = append(lines, fmt.Sprintf("- %s\n Evidence: \"%s\"", name, c.Evidence[0].Quote))
} else {
lines = append(lines, "- "+name)
diff --git a/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go
index 594f1a9f70..9eb810e204 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go
@@ -15,13 +15,11 @@ import (
)
// rewriteDuplicateTreeNames rewrites duplicate summary titles whose
-// descriptions differ, mirroring Python
-// chunk_post_processor.rewrite_duplicate_tree_names (called by
-// Compiler._compile_tree_templates before raptor_tree_to_graph): the graph
-// projection uses titles as entity names and relation endpoints, so two nodes
-// sharing a title render as one ambiguous node.
+// descriptions differ: the graph projection uses titles as entity names and
+// relation endpoints, so two nodes sharing a title render as one ambiguous
+// node.
//
-// Two passes, matching Python:
+// Two passes:
// 1. LLM pass — group summary/root products by title; for a group with ≥2
// members and ≥2 distinct descriptions, ask the LLM for a concise distinct
// title per node (best-effort: a failed group keeps its titles).
@@ -71,7 +69,7 @@ func rewriteDuplicateTreeNames(ctx context.Context, deps common.Deps, llmID stri
}
}
- // ---- Deterministic uniqueness pass (always, matching Python) ----
+ // ---- Deterministic uniqueness pass (always) ----
used := map[string]int{}
for i := range products {
if !isTreeSummaryProduct(products[i]) {
diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph.go b/internal/ingestion/component/knowledge_compiler/tree/graph.go
index 5c3dfbbbd0..eac5836a6c 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/graph.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/graph.go
@@ -75,14 +75,14 @@ func payloadChunkIDs(payload map[string]any) []string {
// payloadDescription is the index text for a tree-graph entity/relation. It
// delegates to common.PayloadDescription — the shared implementation the
// structure variant also uses — so both variants index compiled rows the same
-// way and both match Python _struct_payload_description.
+// way.
func payloadDescription(payload map[string]any) string {
return common.PayloadDescription(payload, nil)
}
-// payloadJSON serialises a payload the way Python's json.dumps(ensure_ascii=
-// False) does (no HTML escaping), with alphabetically sorted keys for a
-// canonical, hash-stable form.
+// payloadJSON serialises a payload with no HTML escaping (json.Encoder with
+// SetEscapeHTML(false)), using alphabetically sorted keys for a canonical,
+// hash-stable form.
func payloadJSON(payload map[string]any) string {
var b bytes.Buffer
enc := json.NewEncoder(&b)
@@ -93,10 +93,9 @@ func payloadJSON(payload map[string]any) string {
return strings.TrimSpace(b.String())
}
-// graphNode mirrors Python's RAPTOR tree node dict (title/description/children/
-// source_chunk_ids) reconstructed from the flat products emitted by buildTree,
-// so the tree can be projected to a {entities, relations} graph exactly like
-// Python's raptor_tree_to_graph (chunk_post_processor.py:470).
+// graphNode holds the reconstructed RAPTOR tree node (title/description/
+// children/source_chunk_ids) from the flat products emitted by buildTree,
+// so the tree can be projected to a {entities, relations} graph.
type graphNode struct {
title string
description string
@@ -104,10 +103,9 @@ type graphNode struct {
children []*graphNode
}
-// collapseUnary merges a node that wraps exactly one child into that child,
-// mirroring Python raptor_tree_to_graph._collapse_unary: the parent's and the
-// child's descriptions/source-chunk-ids are concatenated (dedup'd), then the
-// collapsed node adopts the child's children.
+// collapseUnary merges a node that wraps exactly one child into that child: the
+// parent's and the child's descriptions/source-chunk-ids are concatenated
+// (dedup'd), then the collapsed node adopts the child's children.
func collapseUnary(node *graphNode) *graphNode {
collapsed := &graphNode{
title: node.title,
@@ -154,9 +152,9 @@ func collapseUnary(node *graphNode) *graphNode {
return collapsed
}
-// raptorTreeToGraph projects a RAPTOR tree onto {entities, relations}, matching
-// Python raptor_tree_to_graph: every node becomes an entity of type "tree_node";
-// every parent→child edge (that is not a self-loop) becomes a "child" relation.
+// raptorTreeToGraph projects a RAPTOR tree onto {entities, relations}: every
+// node becomes an entity of type "tree_node"; every parent→child edge (that is
+// not a self-loop) becomes a "child" relation.
func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {
var entities []map[string]any
var relations []map[string]any
@@ -192,8 +190,7 @@ func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {
}
// buildTreeGraph reconstructs the tree from the flat summary products and
-// produces the per-row products Python writes for a tree variant
-// (_struct_upsert_tree_graph_rows):
+// produces the per-row products for a tree variant:
// - one entity product per tree node (kind "entity", knowledge_graph_kwd via
// the writer), type "tree_node";
// - one relation product per parent→child edge (kind "relation").
@@ -297,15 +294,14 @@ func sortedKeys(m map[string][]Claim) []string {
// them out of the artifacts query (it filters knowledge_graph_kwd=["entity",
// "relation"]) without any frontend change.
//
-// The embedding excludes evidence, matching the page_index claim path and the
-// Python implementation: the geometric layer indexes the claim, not the raw
-// source it was verified against.
+// The embedding excludes evidence: the geometric layer indexes the claim, not
+// the raw source it was verified against.
//
-// templateID scopes the row to the compilation template that produced it, like
-// Python's _struct_upsert_tree_claim_rows: it is stamped on the product and
-// mixed into the row id, so two templates that extract the same claim from the
-// same document get two rows instead of overwriting each other — and a
-// template-scoped cleanup can remove one without touching the other.
+// templateID scopes the row to the compilation template that produced it: it
+// is stamped on the product and mixed into the row id, so two templates that
+// extract the same claim from the same document get two rows instead of
+// overwriting each other — and a template-scoped cleanup can remove one
+// without touching the other.
func buildTreeClaimProducts(ctx context.Context, deps common.Deps, docID string, claimsByChunk map[string][]Claim, templateID string) ([]common.Product, error) {
if len(claimsByChunk) == 0 {
return nil, nil
@@ -366,10 +362,9 @@ func buildTreeClaimProducts(ctx context.Context, deps common.Deps, docID string,
for i, payload := range payloads {
vec := vecs[i]
content := payloadJSON(payload)
- // Mirror Python _struct_to_doc_storage_doc, which mixes the template id
- // into the row-id seed (row_seed_extras): without it two templates that
- // extract an identical claim from the same document would share a row id
- // and silently overwrite one another.
+ // The template id is mixed into the row-id seed: without it two templates
+ // that extract an identical claim from the same document would share a
+ // row id and silently overwrite one another.
idParts := []string{content, docID}
if templateID != "" {
idParts = append(idParts, templateID)
diff --git a/internal/ingestion/component/knowledge_compiler/tree/migration_comments_test.go b/internal/ingestion/component/knowledge_compiler/tree/migration_comments_test.go
new file mode 100644
index 0000000000..195be99788
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/tree/migration_comments_test.go
@@ -0,0 +1,73 @@
+// Copyright 2026. Apache 2.0 License.
+
+// Package tree_test hosts static hygiene checks for the tree sub-package. These
+// are not behavioral tests: they guard against comments that anchor the Go
+// implementation on the deleted Python codebase (see AGENTS.md — comments must
+// describe the current runtime, not migration history).
+package tree_test
+
+import (
+ "go/parser"
+ "go/token"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+// migrationRef matches comments that still anchor the Go implementation on the
+// deleted Python codebase. The Python code is being removed (Go-only target),
+// so any surviving reference — the word "python", a .py file name, or a
+// rephrased anchor such as "legacy implementation" / "other runtime" / "two
+// runtimes" — is, by definition, a stale migration anchor. A genuine WHY
+// (invariant, payload shape, algorithm) must be rephrased to describe the Go
+// behavior and must not contain any of these.
+var migrationRef = regexp.MustCompile(`(?i)(python|\.py\b|legacy implementation|other runtime|two runtimes)`)
+
+// TestNoMigrationHistoryComments fails while any non-test .go file under the
+// tree sub-package still anchors a comment on the deleted Python codebase (the
+// word "python", a .py file, or a rephrased legacy/runtime anchor).
+func TestNoMigrationHistoryComments(t *testing.T) {
+ fset := token.NewFileSet()
+ var hits []string
+ err := filepath.WalkDir(".", func(path string, d os.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ if d.Name() == "testdata" {
+ return filepath.SkipDir
+ }
+ return nil
+ }
+ if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
+ return nil
+ }
+ data, rerr := os.ReadFile(path)
+ if rerr != nil {
+ return rerr
+ }
+ file, perr := parser.ParseFile(fset, path, data, parser.ParseComments)
+ if perr != nil {
+ return perr
+ }
+ for _, cg := range file.Comments {
+ for _, c := range cg.List {
+ if migrationRef.MatchString(c.Text) {
+ pos := fset.Position(c.Pos())
+ hits = append(hits, filepath.ToSlash(path)+":"+strconv.Itoa(pos.Line)+": "+strings.TrimSpace(c.Text))
+ }
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("walk failed: %v", err)
+ }
+ if len(hits) > 0 {
+ t.Errorf("found %d migration-anchor references in tree comments; rephrase to describe the Go behavior (see AGENTS.md):\n%s",
+ len(hits), strings.Join(hits, "\n"))
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
index ca3526361a..eee80a9dfb 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
@@ -47,9 +47,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
// Build leaf summary products (one per cluster) and recursively summarize up,
// collecting every node into a plain slice. The component merges these into
- // the upstream chunk stream (matching Python, which appends summaries onto the
- // chunk list). buildTree reads the texts, ids, and embeddings directly from
- // the source chunks.
+ // the upstream chunk stream. buildTree reads the texts, ids, and embeddings
+ // directly from the source chunks.
var products []common.Product
var claimsByChunk map[string][]Claim
if err := buildTree(ctx, deps, llmID, tenantID, docID, inputs.Chunks, treeOrder, taskPrompt, param, &products, &claimsByChunk); err != nil {
@@ -57,8 +56,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
}
deps.Progress.Sub(2, 5).Report(ctx, 1)
- // Rename duplicate summary titles whose descriptions differ (Python
- // rewrite_duplicate_tree_names), BEFORE the graph projection: titles are
+ // Rename duplicate summary titles whose descriptions differ, BEFORE the
+ // graph projection: titles are
// the entity names and the relation endpoints, so two nodes sharing a
// title would render as one ambiguous node. Best-effort — a rewrite
// failure must not cost us the tree.
@@ -66,7 +65,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
deps.Progress.Sub(3, 5).Report(ctx, 1)
// Project the RAPTOR tree onto the {entities, relations} structure-graph
- // shape (Python raptor_tree_to_graph) and persist it as entity/relation rows
+ // shape and persist it as entity/relation rows
// plus per-row entity/relation rows (knowledge_graph_kwd="entity"|
// "relation"), so the
// document-structure /structure/graph endpoint can serve the tree. A failure
@@ -116,15 +115,14 @@ const (
)
// raptorTruncationMarkerRE strips model truncation notices that some LLMs emit
-// when a response is cut short. Mirrors Python _summarize_texts (raptor.py:405).
+// when a response is cut short.
var raptorTruncationMarkerRE = regexp.MustCompile(
strings.Repeat("\u00b7", 6) + "\n由于长度的原因,回答被截断了,要继续吗?|For the content length reason, it stopped, continue?",
)
// resolveMaxToken returns the target summary length and the input truncation
// budget. It is a soft output-size guideline, not a provider max_tokens cap.
-// Python uses max(self._max_token, 512) (raptor.py:427); we honour an
-// extra["max_token"] override with the same 512 floor.
+// It honours an extra["max_token"] override, applying the same 512 floor.
func resolveMaxToken(param common.Param) int {
if v, ok := param.Extra["max_token"]; ok {
if n, ok := rfcommon.ToInt(v); ok && n > 0 {
@@ -137,9 +135,8 @@ func resolveMaxToken(param common.Param) int {
return raptorDefaultMaxToken
}
-// resolveMaxErrors returns the per-tree error ceiling before RAPTOR aborts.
-// Python defaults _max_errors to 3 and skips individual failed clusters below
-// that threshold (raptor.py:_summarize_texts).
+// resolveMaxErrors returns the per-tree error ceiling before RAPTOR aborts. It
+// defaults to 3 and skips individual failed clusters below that threshold.
func resolveMaxErrors(param common.Param) int {
if v, ok := param.Extra["max_errors"]; ok {
if n, ok := rfcommon.ToInt(v); ok && n > 0 {
@@ -191,10 +188,9 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
// buildClusterContent no longer discards content, and the abstraction is
// guaranteed to agree with the claims attached to the same cluster.
//
- // The gate mode comes from the template config, mirroring Python
- // _struct_evidence_gate_mode(parser_config); the extraction toggle and the
- // template-declared extraction contract mirror Python's
- // raptor_config.get("extract_claims", True) / raptor_config["claim_prompt"].
+ // The gate mode comes from the template config; the extraction toggle and the
+ // template-declared extraction contract come from the template's raptor
+ // section.
var claimsByChunk map[string][]Claim
if resolveExtractClaims(param) {
claimsByChunk = ExtractClaimsForChunks(ctx, deps, llmID, chunks,
@@ -208,9 +204,9 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
*claimOut = claimsByChunk
}
- // Claim-view clustering vectors (mirrors Python build_doc_tree): each chunk
- // is represented to the clustering layer by the embedding of its claims +
- // verbatim evidence, not by its raw-text vector. Claims are the chunk's
+ // Claim-view clustering vectors: each chunk is represented to the clustering
+ // layer by the embedding of its claims + verbatim evidence, not by its
+ // raw-text vector. Claims are the chunk's
// semantics stripped of layout noise, so topic clusters form around what
// the chunks actually assert. Chunks without claims keep their raw vector,
// and a claim vector whose dimension disagrees with the chunk vectors is
@@ -289,9 +285,8 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
queue = queue[1:]
// Gather the source text for this cluster's points and truncate each
- // text to a per-chunk token budget so the cluster fits the LLM context
- // (Python: len_per_chunk = (max_length - max_token) / len(texts);
- // truncate(t, len_per_chunk), raptor.py:389-390).
+ // text to a per-chunk token budget so the cluster fits the LLM context;
+ // the budget is (max_length - max_token) / len(texts).
//
// At the bottom level, prefer the members' claims: BuildClaimContent
// returns "" when a cluster has none, so we fall through to the raw
@@ -391,13 +386,14 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
// Root product ties the tree together. The root summary is the synthesis of
// the highest-level summaries (the leaves of the upper tree) using the same
- // standard task prompt Python applies to every cluster — NOT a separate
+ // standard task prompt applied to every cluster — NOT a separate
// "Synthesize the overall theme" prompt, and NOT the union of all leaf
// summaries (which would just re-summarize the same text repeatedly).
if len(topLevelTexts) == 0 {
// No summaries survived (e.g. every deepest cluster failed while the
// error budget was not yet exhausted). Return the partial tree without a
- // root node — Python drops the root in this case rather than crashing.
+ // root node — the Go tree drops the root in this case
+ // rather than crashing.
rfcommon.Warn("tree: no top-level summaries produced, skipping root node")
return nil
}
@@ -406,8 +402,8 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
raptorSystemHelper+strings.Replace(taskPrompt, "{cluster_content}", rootContent, 1),
raptorTitleInstruction, maxToken)
if err != nil {
- // A failed root must not abort the whole tree: Python drops the root
- // node and returns the rest of the tree.
+ // A failed root must not abort the whole tree: the legacy
+ // implementation drops the root node and returns the rest of the tree.
rfcommon.Warn("tree: root synthesis failed, skipping root node", zap.Error(err))
return nil
}
@@ -433,13 +429,12 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
return nil
}
-// summarizeTexts asks the LLM for a summary and normalises the response to match
-// Python _summarize_texts / _chat:
-// - retries up to raptorMaxRetries times with linear backoff (Python: 3 attempts);
+// summarizeTexts asks the LLM for a summary and normalises the response:
+// - retries up to raptorMaxRetries times with linear backoff;
// - strips a reasoning-model preamble up to the final </think> / </think:6124c78e> tag;
-// - treats a "**ERROR**" marker as a transient failure (Python raises on it, so
-// it is retried like any other LLM error);
-// - strips model truncation notices (raptor.py:405);
+// - treats a "**ERROR**" marker as a transient failure, retried like any other
+// LLM error;
+// - strips model truncation notices;
// - trims surrounding whitespace.
//
// systemText is the fully-built system prompt (helper + filled task template) and
@@ -478,8 +473,7 @@ func summarizeTexts(ctx context.Context, deps common.Deps, llmID, systemText, us
continue
}
content := resp.Content
- // Strip reasoning preamble up to the final thinking close tag
- // (Python: re.sub(r"^.*</think>", "", response, DOTALL)). The
+ // Strip reasoning preamble up to the final thinking close tag. The
// else-if keeps the vendor-specific </think:6124c78e> fallback
// active only when no standard </think> is present, preserving
// the original control flow.
@@ -488,19 +482,18 @@ func summarizeTexts(ctx context.Context, deps common.Deps, llmID, systemText, us
} else if i := strings.LastIndex(content, "</think:6124c78e>"); i >= 0 {
content = content[i+len("</think:6124c78e>"):]
}
- // Python raises on the "**ERROR**" marker; treat it as a retryable error.
+ // Treat the "**ERROR**" marker as a retryable error.
if strings.Contains(content, "**ERROR**") {
continue
}
- // Strip model truncation notices (Python _summarize_texts).
+ // Strip model truncation notices.
content = raptorTruncationMarkerRE.ReplaceAllString(content, "")
return strings.TrimSpace(content), nil
}
return "", fmt.Errorf("raptor: summarization failed after %d attempts", raptorMaxRetries)
}
-// titleOf returns the first line of a summary completion (Python stores
-// summary_ti = cnt.split("\n")[0]).
+// titleOf returns the first line of a summary completion.
func titleOf(summary string) string {
if i := strings.IndexByte(summary, '\n'); i >= 0 {
return summary[:i]
@@ -508,12 +501,11 @@ func titleOf(summary string) string {
return summary
}
-// buildClusterContent joins the selected texts with a single newline (matching
-// Python's "\n".join) and truncates each text to a per-chunk token budget so the
-// whole cluster fits the LLM context window (Python: len_per_chunk =
-// (max_length - max_token) / len(texts); truncate(t, len_per_chunk)). The token
-// budget uses the cl100k_base encoder, mirroring Python's truncate (token-level,
-// not character-level).
+// buildClusterContent joins the selected texts with a single newline and
+// truncates each text to a per-chunk token budget so the whole cluster fits the
+// LLM context window; the budget is (modelContextLen - maxToken) / len(idxs),
+// clamped to a minimum of one token. The
+// token budget uses the cl100k_base encoder (token-level, not character-level).
func buildClusterContent(texts []string, idxs []int, modelContextLen, maxToken int) string {
if len(idxs) == 0 {
return ""
@@ -659,19 +651,16 @@ func toFloat64Matrix(vecs [][]float32) [][]float64 {
// defaultRaptorPrompt is the summary task template.
const defaultRaptorPrompt = "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."
-// raptorSystemHelper mirrors the leading "You're a helpful assistant.\n\nHelp me
-// with the following task.\n\n" wrapper Python prepends to the task prompt.
+// raptorSystemHelper is the leading "You're a helpful assistant.\n\nHelp me with
+// the following task.\n\n" wrapper prepended to the task prompt.
const raptorSystemHelper = "You're a helpful assistant.\n\nHelp me with the following task.\n\n"
// raptorTitleInstruction is sent as the user turn so the model also emits a
-// one-line title on the first line of the summary (Python: "Beside the
-// summarization, give a title at the first line of your summarization. Must be
-// in the same language as the paragraphs.").
+// one-line title on the first line of the summary.
const raptorTitleInstruction = "Beside the summarization, give a title at the first line of your summarization. Must be in the same language as the paragraphs."
// templateRaptorString reads a string field from the template config's
-// "raptor" section (tree.yaml's raptor: block, which Python compiles into
-// raptor_config — compiler.py) and returns "" when absent.
+// "raptor" section (tree.yaml's raptor: block) and returns "" when absent.
func templateRaptorString(param common.Param, key string) string {
cfg, ok := param.TemplateConfig["raptor"].(map[string]any)
if !ok {
@@ -697,8 +686,8 @@ func cfgStr(v any) string {
}
// resolveRaptorPrompt returns the summary task template, honouring in order an
-// extra["prompt"] caller override, the template's raptor.prompt (Python:
-// raptor_cfg["prompt"]), and the built-in default.
+// extra["prompt"] caller override, the template's raptor.prompt, and the
+// built-in default.
func resolveRaptorPrompt(param common.Param) string {
if raw, ok := param.Extra["prompt"]; ok {
if s, ok := raw.(string); ok && strings.TrimSpace(s) != "" {
@@ -713,9 +702,8 @@ func resolveRaptorPrompt(param common.Param) string {
// resolveClaimPrompt returns the claim-extraction system prompt, honouring in
// order an extra["claim_prompt"] caller override and the template's
-// raptor.claim_prompt (Python passes raptor_config["claim_prompt"] into
-// extract_claims_for_chunks). An empty result means "use the built-in
-// contract" — the same None-fallback Python applies.
+// raptor.claim_prompt. An empty result means "use the built-in contract" — the
+// same None-fallback the template applies.
func resolveClaimPrompt(param common.Param) string {
if raw, ok := param.Extra["claim_prompt"]; ok {
if s, ok := raw.(string); ok && strings.TrimSpace(s) != "" {
@@ -725,9 +713,9 @@ func resolveClaimPrompt(param common.Param) string {
return templateRaptorString(param, "claim_prompt")
}
-// resolveExtractClaims mirrors Python raptor_config.get("extract_claims", True):
-// claim harvesting is on unless the template (or a caller override) turns it
-// off, in which case clusters summarise from raw chunk text.
+// resolveExtractClaims reports whether claim harvesting is enabled: claim
+// harvesting is on unless the template (or a caller override) turns it off, in
+// which case clusters summarise from raw chunk text.
func resolveExtractClaims(param common.Param) bool {
if raw, ok := param.Extra["extract_claims"]; ok {
if b, ok := raw.(bool); ok {
diff --git a/internal/ingestion/component/knowledge_compiler/tree/watershed.go b/internal/ingestion/component/knowledge_compiler/tree/watershed.go
index 9eb88f989a..64d64094dd 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/watershed.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/watershed.go
@@ -1,7 +1,7 @@
// watershed is the sole clustering method used by the RAPTOR tree.
//
-// It is the algorithm historically named "AHC" in the Python raptor
-// implementation (raptor.py::_get_clusters_ahc, logged as "RAPTOR seq-clus");
+// It is the algorithm historically named "AHC" in the original
+// RAPTOR implementation;
// the name "watershed" reflects what the code actually does, which is not
// agglomerative hierarchical clustering.
//
@@ -18,7 +18,7 @@ import (
// watershed clustering. It is the average chunk count per level-0 cluster: the
// expected number of level-0 clusters is close to len(embeddings)/DefaultTreeOrder,
// so the recursion depth is roughly log_DefaultTreeOrder(len(embeddings)). 4
-// (ratio 25) is close to the Python raptor "cluster_percentile" default of 30.
+// (ratio 25) is close to the raptor "cluster_percentile" default of 30.
const DefaultTreeOrder = 4
// MinTreeOrder and MaxTreeOrder bound the integer tree_order parameter.
@@ -50,8 +50,9 @@ func watershed(embeddings [][]float64, treeOrder int) ([]int, error) {
case n == 1:
return []int{0}, nil
case n == 2:
- // One adjacent pair: the Python reference always splits two points into
- // two clusters, so the recursion terminates with one-point leaves.
+ // One adjacent pair: the reference implementation always splits two
+ // points into two clusters, so the recursion terminates with one-point
+ // leaves.
return []int{0, 1}, nil
}