Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: align mindmap relations with Python (#19579)

buua4360a00a732905d
diff --git a/go.mod b/go.mod
index 8044548db1..f57c9f487c 100644
--- a/go.mod
+++ b/go.mod
@@ -11,7 +11,6 @@ require (
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/JohannesKaufmann/html-to-markdown v1.6.0
github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0
- github.com/RealAlexandreAI/json-repair v0.0.17
github.com/alibabacloud-go/agentrun-20250910/v5 v5.8.4
github.com/alibabacloud-go/darabonba-openapi/v2 v2.2.1
github.com/alicebob/miniredis/v2 v2.38.0
diff --git a/go.sum b/go.sum
index 4c7d7a2cac..0826ca25f9 100644
--- a/go.sum
+++ b/go.sum
@@ -62,8 +62,6 @@ github.com/LuxorLabs/tenki-sdk-go/sandbox v0.7.0/go.mod h1:rBypesed6hSrj+OgpJwyM
github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE=
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
-github.com/RealAlexandreAI/json-repair v0.0.17 h1:nXppoV05wAPo/+/HjpFkvvAgTb98s3bPcAALldG7j/g=
-github.com/RealAlexandreAI/json-repair v0.0.17/go.mod h1:GKJi5borR78O8c7HCVbgqjhoiVibZ6hJldxbc6dGrAI=
github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o=
github.com/alibabacloud-go/agentrun-20250910/v5 v5.8.4 h1:hiAsm9pz6aICOPLI1FC54vga10xwd/XxfNj06ow5jVM=
github.com/alibabacloud-go/agentrun-20250910/v5 v5.8.4/go.mod h1:CYOwrIjr05fOHvdhid7MWtP/IT5ILLcE0eqiolzQT/Q=
diff --git a/internal/ingestion/component/knowledge_compiler/common/common_test.go b/internal/ingestion/component/knowledge_compiler/common/common_test.go
index f9679f4b46..e9b24023d3 100644
--- a/internal/ingestion/component/knowledge_compiler/common/common_test.go
+++ b/internal/ingestion/component/knowledge_compiler/common/common_test.go
@@ -2,6 +2,7 @@ package common
import (
"context"
+ "encoding/json"
"errors"
"strings"
"testing"
@@ -26,6 +27,20 @@ func TestGenJSONDisablesInvokerRetry(t *testing.T) {
}
}
+func TestRepairJSONTextRepairsMalformedResponse(t *testing.T) {
+ got, err := RepairJSONText("Here is the result: {id: 'root', children: [{id: 'child',},]}")
+ if err != nil {
+ t.Fatalf("RepairJSONText: %v", err)
+ }
+ var parsed map[string]any
+ if err := json.Unmarshal([]byte(got), &parsed); err != nil {
+ t.Fatalf("repaired JSON is invalid: %v", err)
+ }
+ if parsed["id"] != "root" {
+ t.Errorf("repaired id = %v, want root", parsed["id"])
+ }
+}
+
type failingChat struct{ err error }
func (c failingChat) Chat(context.Context, ChatRequest) (*ChatResponse, error) {
diff --git a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
index e2917aafac..0c5b55ddf6 100644
--- a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
+++ b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
@@ -9,6 +9,7 @@ import (
"strings"
"time"
+ "github.com/kaptinlin/jsonrepair"
"ragflow/internal/agent/runtime"
appcommon "ragflow/internal/common"
@@ -69,6 +70,11 @@ func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ..
// with a fresh LLM call.
lastErr = err
} else {
+ if repaired, repairErr := RepairJSONText(resp.Content); repairErr == nil {
+ if m, unmarshalErr := tryUnmarshalJSONErr(repaired); unmarshalErr == nil {
+ return m, nil
+ }
+ }
candidates := jsonCandidates(resp.Content)
for _, candidate := range candidates {
if m, ok := tryUnmarshalJSON(candidate); ok {
@@ -111,6 +117,46 @@ func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ..
return nil, lastErr
}
+// RepairJSONText extracts and repairs a JSON object or array from an LLM response.
+// It accepts plain JSON, fenced JSON, JSON surrounded by prose, and common
+// malformed forms such as trailing commas or unquoted keys. The returned text
+// is guaranteed to be a valid JSON object or array.
+func RepairJSONText(s string) (string, error) {
+ s = appcommon.StripThinkTrailing(s)
+ var lastErr error
+ for _, candidate := range jsonCandidates(s) {
+ candidate = strings.TrimSpace(candidate)
+ if candidate == "" {
+ lastErr = fmt.Errorf("empty JSON candidate")
+ continue
+ }
+ if isJSONObjectOrArray(candidate) {
+ return candidate, nil
+ }
+ repaired, err := jsonrepair.Repair(candidate)
+ if err != nil {
+ lastErr = err
+ continue
+ }
+ if isJSONObjectOrArray(repaired) {
+ return repaired, nil
+ }
+ lastErr = fmt.Errorf("repaired candidate is not a JSON object or array")
+ }
+ if lastErr == nil {
+ lastErr = fmt.Errorf("no JSON candidate")
+ }
+ return "", lastErr
+}
+
+func isJSONObjectOrArray(s string) bool {
+ s = strings.TrimSpace(s)
+ if len(s) < 2 || (s[0] != '{' && s[0] != '[') {
+ return false
+ }
+ return json.Valid([]byte(s))
+}
+
func reportLLMFailure(ctx context.Context, attempt, maxRetries int, delay time.Duration, err error) {
message := fmt.Sprintf("[ERROR] LLM call failed (attempt %d/%d): %s", attempt+1, maxRetries+1, CompactError(err))
if delay > 0 {
@@ -152,6 +198,11 @@ func jsonCandidates(s string) []string {
cands = append(cands, s[i:j+1])
}
}
+ if i := strings.Index(s, "["); i >= 0 {
+ if j := strings.LastIndex(s, "]"); j > i {
+ cands = append(cands, s[i:j+1])
+ }
+ }
return cands
}
diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
index 61460d84b3..30c5b5a876 100644
--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
@@ -12,6 +12,7 @@ package mindmap
import (
"context"
+ "encoding/json"
"fmt"
"strings"
@@ -19,6 +20,8 @@ import (
"ragflow/internal/utility"
)
+const mindmapJSONRetryMax = 3
+
// batchSubmitter fans out the batch extraction jobs on the process-wide
// knowledge-compilation pool. It is injected by the knowledge_compiler wiring
// (component.go) so every stage shares one vCPU-sized concurrency bound; when
@@ -62,25 +65,35 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
return common.Outputs{}, fmt.Errorf("mindmap: chat model required")
}
- sections := chunkTexts(inputs.Chunks)
-
// One LLM task per token-budget batch (mirrors __call__'s task fan-out).
- batches := packSections(sections, deps.Tokenizer)
- results := make([]utility.OMap, len(batches))
+ batches := packMindmapBatches(inputs.Chunks, deps.Tokenizer)
+ results := make([]mindmapBatchResult, len(batches))
jobs := make([]func() error, 0, len(batches))
- for i, text := range batches {
- i, text := i, text
+ for i, batch := range batches {
+ i, batch := i, batch
jobs = append(jobs, func() error {
- resp, err := deps.Chat.Chat(ctx, common.ChatRequest{
- LLMID: llmID,
- SystemPrompt: renderPrompt(text),
- UserPrompt: userMessage,
- })
- if err != nil {
- return err
+ var lastContent string
+ for attempt := 0; attempt < mindmapJSONRetryMax; attempt++ {
+ resp, err := deps.Chat.Chat(ctx, common.ChatRequest{
+ LLMID: llmID,
+ SystemPrompt: renderPrompt(batch.text),
+ UserPrompt: userMessage,
+ JSONMode: true,
+ DisableThinking: true,
+ })
+ if err != nil {
+ return err
+ }
+ lastContent = resp.Content
+ // Distinct slice index per batch → no cross-goroutine contention.
+ if tree, ok := parseJSONTree(resp.Content, batch.ids); ok {
+ results[i].tree = tree
+ return nil
+ }
}
- // Distinct slice index per batch → no cross-goroutine contention.
- results[i] = utility.Todict(utility.Dictify(utility.StripFences(resp.Content)))
+ // Keep the old Markdown protocol as a compatibility fallback after
+ // exhausting JSON retries.
+ results[i].outline = utility.Todict(utility.Dictify(utility.StripFences(lastContent)))
return nil
})
}
@@ -93,16 +106,31 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
// Merge batch dicts in batch order (mirrors reduce(self._merge, res)) and
// shape the final tree. Python returns a bare root when nothing parsed.
- var merged utility.OMap
- if len(results) > 0 {
- merged = results[0]
- for _, r := range results[1:] {
- merged = utility.MergeDicts(merged, r)
+ var root *utility.Node
+ allJSON := len(results) > 0
+ for _, result := range results {
+ if result.tree == nil {
+ allJSON = false
+ break
+ }
+ root = mergeMindmapTrees(root, result.tree)
+ }
+ if !allJSON {
+ var merged utility.OMap
+ for _, result := range results {
+ if len(result.outline) == 0 {
+ continue
+ }
+ if len(merged) == 0 {
+ merged = result.outline
+ } else {
+ merged = utility.MergeDicts(merged, result.outline)
+ }
}
+ root = utility.ShapeTree(merged)
}
- root := utility.ShapeTree(merged)
- products := treeToProducts(tenantID, docID, root)
+ products := treeToProducts(tenantID, docID, root, mindmapSourceChunkIDs(inputs.Chunks))
// Batched embedding of each node's content for downstream vector search.
if len(products) > 0 && deps.Embed != nil {
@@ -140,30 +168,38 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
// - each node (including the root) → an entity product (kind="entity",
// name = node id, type = "mindmap").
// - each parent→child edge → a relation product (kind="relation",
-// from = parent id, to = child id, type = "related" — Python's default).
+// from = parent id, to = child id, type = "related", matching Python's
+// mindmap structure-graph projection).
//
// The entity/relation discriminator is carried in Meta["kind"] so the consumer's
// mergeStructureDataset buckets entities by (name,type) and relations by
// (from,type,to), matching graph/timeline.
-func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product {
+func treeToProducts(tenantID, docID string, root *utility.Node, fallbackSourceChunkIDs []string) []common.Product {
var out []common.Product
if root == nil || root.ID == "" {
return out
}
seen := map[string]bool{}
// Entity: root node.
+ rootSourceChunkIDs := nodeSourceChunkIDs(root, fallbackSourceChunkIDs)
+ rootPayload := map[string]any{"name": root.ID, "type": "mindmap"}
+ rootMeta := map[string]any{
+ "kind": "entity",
+ "name": root.ID,
+ "entity_type": "mindmap",
+ "compile_kwd": "mindmap",
+ }
+ if len(rootSourceChunkIDs) > 0 {
+ rootPayload["source_chunk_ids"] = rootSourceChunkIDs
+ rootMeta["source_chunk_ids"] = rootSourceChunkIDs
+ }
out = append(out, common.Product{
ID: common.StableRowID(tenantID, docID, string(common.VariantMindmap), "entity", root.ID),
DocID: docID,
TenantID: tenantID,
Variant: common.VariantMindmap,
- Content: root.ID,
- Meta: map[string]any{
- "kind": "entity",
- "name": root.ID,
- "entity_type": "mindmap",
- "compile_kwd": "mindmap",
- },
+ Content: payloadJSON(rootPayload),
+ Meta: rootMeta,
})
seen[root.ID] = true
@@ -183,18 +219,25 @@ func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product
// the same node twice).
if !seen[child.ID] {
seen[child.ID] = true
+ childSourceChunkIDs := nodeSourceChunkIDs(child, fallbackSourceChunkIDs)
+ childPayload := map[string]any{"name": child.ID, "type": "mindmap"}
+ childMeta := map[string]any{
+ "kind": "entity",
+ "name": child.ID,
+ "entity_type": "mindmap",
+ "compile_kwd": "mindmap",
+ }
+ if len(childSourceChunkIDs) > 0 {
+ childPayload["source_chunk_ids"] = childSourceChunkIDs
+ childMeta["source_chunk_ids"] = childSourceChunkIDs
+ }
out = append(out, common.Product{
ID: common.StableRowID(tenantID, docID, string(common.VariantMindmap), "entity", child.ID),
DocID: docID,
TenantID: tenantID,
Variant: common.VariantMindmap,
- Content: child.ID,
- Meta: map[string]any{
- "kind": "entity",
- "name": child.ID,
- "entity_type": "mindmap",
- "compile_kwd": "mindmap",
- },
+ Content: payloadJSON(childPayload),
+ Meta: childMeta,
})
}
// Relation: parent → child edge (type = "related", Python default).
@@ -203,7 +246,11 @@ func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product
DocID: docID,
TenantID: tenantID,
Variant: common.VariantMindmap,
- Content: p.parent + " related " + child.ID,
+ Content: payloadJSON(map[string]any{
+ "source": p.parent,
+ "target": child.ID,
+ "type": "related",
+ }),
Meta: map[string]any{
"kind": "relation",
"from": p.parent,
@@ -218,6 +265,157 @@ func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product
return out
}
+func nodeSourceChunkIDs(node *utility.Node, fallback []string) []string {
+ if node != nil && len(node.SourceChunkIDs) > 0 {
+ return append([]string(nil), node.SourceChunkIDs...)
+ }
+ return append([]string(nil), fallback...)
+}
+
+func mindmapSourceChunkIDs(chunks []common.Chunk) []string {
+ seen := make(map[string]struct{}, len(chunks))
+ ids := make([]string, 0, len(chunks))
+ for i, chunk := range chunks {
+ if strings.TrimSpace(firstNonEmpty(chunk.Text, chunk.Content)) == "" {
+ continue
+ }
+ id := strings.TrimSpace(chunk.ID)
+ if id == "" {
+ id = fmt.Sprintf("chunk-%d", i+1)
+ }
+ if _, ok := seen[id]; !ok {
+ seen[id] = struct{}{}
+ ids = append(ids, id)
+ }
+ }
+ return ids
+}
+
+type mindmapBatchResult struct {
+ tree *utility.Node
+ outline utility.OMap
+}
+
+type jsonMindmapNode struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ SourceChunkIDs []string `json:"source_chunk_ids"`
+ Children []jsonMindmapNode `json:"children"`
+}
+
+func parseJSONTree(content string, batchIDs []string) (*utility.Node, bool) {
+ content, err := common.RepairJSONText(content)
+ if err != nil {
+ return nil, false
+ }
+ var raw jsonMindmapNode
+ if err := json.Unmarshal([]byte(content), &raw); err != nil {
+ return nil, false
+ }
+ if strings.TrimSpace(raw.ID) == "" {
+ raw.ID = raw.Name
+ }
+ if strings.TrimSpace(raw.ID) == "" {
+ return nil, false
+ }
+ return convertJSONMindmapNode(raw, batchIDs), true
+}
+
+func convertJSONMindmapNode(raw jsonMindmapNode, batchIDs []string) *utility.Node {
+ if strings.TrimSpace(raw.ID) == "" {
+ raw.ID = raw.Name
+ }
+ node := &utility.Node{
+ ID: strings.TrimSpace(raw.ID),
+ SourceChunkIDs: filterMindmapChunkIDs(raw.SourceChunkIDs, batchIDs),
+ }
+ for _, child := range raw.Children {
+ converted := convertJSONMindmapNode(child, batchIDs)
+ if converted.ID != "" {
+ node.Children = append(node.Children, converted)
+ }
+ }
+ return node
+}
+
+func filterMindmapChunkIDs(ids, batchIDs []string) []string {
+ allowed := make(map[string]struct{}, len(batchIDs))
+ for _, id := range batchIDs {
+ allowed[id] = struct{}{}
+ }
+ seen := make(map[string]struct{}, len(ids))
+ selected := make([]string, 0, len(ids))
+ for _, id := range ids {
+ id = strings.TrimSpace(id)
+ if id == "" {
+ continue
+ }
+ if _, ok := allowed[id]; !ok {
+ continue
+ }
+ if _, ok := seen[id]; !ok {
+ seen[id] = struct{}{}
+ selected = append(selected, id)
+ }
+ }
+ if len(selected) == 0 {
+ return append([]string(nil), batchIDs...)
+ }
+ return selected
+}
+
+func mergeMindmapTrees(left, right *utility.Node) *utility.Node {
+ if left == nil {
+ return right
+ }
+ if right == nil {
+ return left
+ }
+ if left.ID != right.ID {
+ return &utility.Node{ID: "root", Children: []*utility.Node{left, right}}
+ }
+ left.SourceChunkIDs = mergeMindmapIDs(left.SourceChunkIDs, right.SourceChunkIDs)
+ byID := make(map[string]*utility.Node, len(left.Children))
+ for _, child := range left.Children {
+ byID[child.ID] = child
+ }
+ for _, child := range right.Children {
+ if existing := byID[child.ID]; existing != nil {
+ mergeMindmapTrees(existing, child)
+ } else {
+ left.Children = append(left.Children, child)
+ byID[child.ID] = child
+ }
+ }
+ return left
+}
+
+func mergeMindmapIDs(left, right []string) []string {
+ seen := make(map[string]struct{}, len(left)+len(right))
+ merged := make([]string, 0, len(left)+len(right))
+ for _, ids := range [][]string{left, right} {
+ for _, id := range ids {
+ if id != "" {
+ if _, ok := seen[id]; !ok {
+ seen[id] = struct{}{}
+ merged = append(merged, id)
+ }
+ }
+ }
+ }
+ return merged
+}
+
+// payloadJSON serializes the graph payload stored in content_with_weight.
+// Structure-graph projection expects entity and relation content to be JSON.
+func payloadJSON(payload map[string]any) string {
+ data, err := json.Marshal(payload)
+ if err != nil {
+ return "{}"
+ }
+ return string(data)
+}
+
func chunkTexts(chunks []common.Chunk) []string {
var out []string
for _, c := range chunks {
diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
index a354ee2a25..5027a49eeb 100644
--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
@@ -1,9 +1,13 @@
package mindmap
import (
+ "context"
+ "encoding/json"
+ "fmt"
"strings"
"testing"
+ "ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/utility"
)
@@ -15,6 +19,9 @@ func TestRenderPrompt(t *testing.T) {
if strings.Contains(got, "{input_text}") {
t.Fatalf("placeholder survived: %q", got)
}
+ if !strings.Contains(got, `"source_chunk_ids"`) {
+ t.Fatalf("source chunk provenance requirement missing: %q", got)
+ }
// The verbatim prompt body must be present.
if !strings.Contains(got, "Generate a title for user's 'TEXT'。") {
t.Fatalf("prompt body drifted")
@@ -46,7 +53,7 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {
{ID: "A", Children: []*utility.Node{{ID: "A1"}, {ID: "A2"}}},
{ID: "B"},
}}
- products := treeToProducts("t1", "d1", root)
+ products := treeToProducts("t1", "d1", root, []string{"c1", "c2"})
// 5 entities (root, A, A1, A2, B) + 4 relations (root→A, A→A1, A→A2, root→B).
if len(products) != 9 {
t.Fatalf("products = %d, want 9 (5 entities + 4 relations)", len(products))
@@ -58,6 +65,13 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {
switch kind {
case "entity":
entCount++
+ var payload map[string]any
+ if err := json.Unmarshal([]byte(p.Content), &payload); err != nil {
+ t.Fatalf("entity %q content is not JSON: %v", p.Meta["name"], err)
+ }
+ if payload["name"] != p.Meta["name"] || payload["type"] != "mindmap" {
+ t.Errorf("entity payload = %v, want name=%v and type=mindmap", payload, p.Meta["name"])
+ }
if p.Meta["entity_type"] != "mindmap" {
t.Errorf("entity %v type = %v, want mindmap", p.Meta["name"], p.Meta["entity_type"])
}
@@ -68,6 +82,13 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {
relCount++
from, _ := p.Meta["from"].(string)
to, _ := p.Meta["to"].(string)
+ var payload map[string]any
+ if err := json.Unmarshal([]byte(p.Content), &payload); err != nil {
+ t.Fatalf("relation %q content is not JSON: %v", p.Meta["name"], err)
+ }
+ if payload["source"] != from || payload["target"] != to || payload["type"] != "related" {
+ t.Errorf("relation payload = %v, want source=%v target=%v type=related", payload, from, to)
+ }
fromTo[from+"->"+to] = true
if p.Meta["relation_type"] != "related" {
t.Errorf("relation %v->%v type = %v, want related", from, to, p.Meta["relation_type"])
@@ -87,10 +108,64 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {
}
func TestTreeToProducts_EmptyAndNil(t *testing.T) {
- if got := treeToProducts("t1", "d1", nil); len(got) != 0 {
+ if got := treeToProducts("t1", "d1", nil, nil); len(got) != 0 {
t.Errorf("nil root produced %d products", len(got))
}
- if got := treeToProducts("t1", "d1", &utility.Node{ID: ""}); len(got) != 0 {
+ if got := treeToProducts("t1", "d1", &utility.Node{ID: ""}, nil); len(got) != 0 {
t.Errorf("empty root id produced %d products", len(got))
}
}
+
+func TestParseJSONTree_SourceChunkIDs(t *testing.T) {
+ content := `{"id":"root","source_chunk_ids":["c1","outside"],"children":[{"id":"child","source_chunk_ids":["c2"],"children":[]}]}`
+ root, ok := parseJSONTree(content, []string{"c1", "c2"})
+ if !ok || root == nil {
+ t.Fatal("JSON mindmap response was not parsed")
+ }
+ if len(root.SourceChunkIDs) != 1 || root.SourceChunkIDs[0] != "c1" {
+ t.Fatalf("root source chunk ids = %v, want [c1]", root.SourceChunkIDs)
+ }
+ if len(root.Children) != 1 || len(root.Children[0].SourceChunkIDs) != 1 || root.Children[0].SourceChunkIDs[0] != "c2" {
+ t.Fatalf("child source chunk ids = %+v, want [c2]", root.Children)
+ }
+
+ products := treeToProducts("t1", "d1", root, nil)
+ if ids, ok := products[0].Meta["source_chunk_ids"].([]string); !ok || len(ids) != 1 || ids[0] != "c1" {
+ t.Fatalf("entity meta source chunk ids = %#v, want [c1]", products[0].Meta["source_chunk_ids"])
+ }
+}
+
+type retryChat struct {
+ calls int
+}
+
+func (c *retryChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {
+ c.calls++
+ if !req.JSONMode || !req.DisableThinking {
+ return nil, fmt.Errorf("mindmap request must enable JSON mode and disable thinking")
+ }
+ if c.calls == 1 {
+ return &common.ChatResponse{Content: "not json"}, nil
+ }
+ return &common.ChatResponse{Content: `{"id":"root","children":[{"id":"child","children":[]}]}`}, nil
+}
+
+func TestRunRetriesInvalidJSON(t *testing.T) {
+ chat := &retryChat{}
+ outputs, err := Run(context.Background(), common.Deps{
+ Chat: chat,
+ TenantID: "t1",
+ }, common.Param{}, common.Inputs{
+ Chunks: []common.Chunk{{ID: "c1", Text: "source text"}},
+ DocID: "d1",
+ })
+ if err != nil {
+ t.Fatalf("Run returned error: %v", err)
+ }
+ if chat.calls != 2 {
+ t.Fatalf("chat calls = %d, want 2 after one invalid JSON response", chat.calls)
+ }
+ if len(outputs.Products) != 3 {
+ t.Fatalf("products = %d, want root, child, and relation", len(outputs.Products))
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/prompt.go b/internal/ingestion/component/knowledge_compiler/mindmap/prompt.go
index d3ac5e1c5b..81b6385215 100644
--- a/internal/ingestion/component/knowledge_compiler/mindmap/prompt.go
+++ b/internal/ingestion/component/knowledge_compiler/mindmap/prompt.go
@@ -1,14 +1,16 @@
package mindmap
import (
+ "fmt"
"strings"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
)
-// mindMapExtractionPrompt mirrors MIND_MAP_EXTRACTION_PROMPT verbatim
-// (rag/graphrag/general/mind_map_prompt.py). It is the SYSTEM message; the
-// user turn is the fixed string "Output:" (see userMessage).
+// mindMapExtractionPrompt is based on MIND_MAP_EXTRACTION_PROMPT
+// (rag/graphrag/general/mind_map_prompt.py), extended with the JSON tree and
+// source-chunk provenance contract. It is the SYSTEM message; the user turn is
+// the fixed string "Output:" (see userMessage).
const mindMapExtractionPrompt = `
- Role: You're a talent text processor to summarize a piece of text into a mind map.
@@ -22,7 +24,10 @@ const mindMapExtractionPrompt = `
- Generate at least 4 levels.
- Always try to maximize the number of sub-sections.
- In language of 'Text'
- - MUST IN FORMAT OF MARKDOWN
+ - Return JSON only, with this shape:
+ {"id":"node title","source_chunk_ids":["chunk id",...],"children":[{"id":"child title","source_chunk_ids":["chunk id",...],"children":[]}]}
+ - Every node MUST include source_chunk_ids containing only the IDs of source chunks that support that node.
+ - Each source chunk is enclosed by [CHUNK_ID: ...] and [END_CHUNK].
-TEXT-
{input_text}
@@ -55,6 +60,50 @@ func renderPrompt(text string) string {
return strings.ReplaceAll(mindMapExtractionPrompt, "{input_text}", text)
}
+type mindmapBatch struct {
+ text string
+ ids []string
+}
+
+func packMindmapBatches(chunks []common.Chunk, tok common.Tokenizer) []mindmapBatch {
+ var batches []mindmapBatch
+ var sections []string
+ var ids []string
+ cnt := 0
+ flush := func() {
+ if len(sections) == 0 {
+ return
+ }
+ batches = append(batches, mindmapBatch{
+ text: strings.Join(sections, "\n\n"),
+ ids: append([]string(nil), ids...),
+ })
+ sections = nil
+ ids = nil
+ cnt = 0
+ }
+ for i, chunk := range chunks {
+ text := common.FirstNonEmpty(chunk.Text, chunk.Content)
+ if strings.TrimSpace(text) == "" {
+ continue
+ }
+ id := strings.TrimSpace(chunk.ID)
+ if id == "" {
+ id = "chunk-" + fmt.Sprint(i+1)
+ }
+ section := "[CHUNK_ID: " + id + "]\n" + text + "\n[END_CHUNK]"
+ sectionTokens := numTokens(tok, section)
+ if cnt+sectionTokens >= batchBudget() && len(sections) > 0 {
+ flush()
+ }
+ sections = append(sections, section)
+ ids = append(ids, id)
+ cnt += sectionTokens
+ }
+ flush()
+ return batches
+}
+
// packSections mirrors MindMapExtractor.__call__'s batching: sections are
// concatenated WITHOUT a separator into batches whose token count stays
// under the budget; a single section is never split (an oversized section
diff --git a/internal/ingestion/component/knowledge_compiler/tree/claims.go b/internal/ingestion/component/knowledge_compiler/tree/claims.go
index 7c66ea45e4..91082adee3 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/claims.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/claims.go
@@ -441,19 +441,10 @@ func isRetryableClaimErr(err error) bool {
// parseClaimItems tolerates the model wrapping JSON in prose or fences.
func parseClaimItems(raw string) []Claim {
- s := strings.TrimSpace(raw)
- if s == "" {
+ s, err := common.RepairJSONText(raw)
+ if err != nil {
return nil
}
- if i := strings.Index(s, "{"); i > 0 {
- s = s[i:]
- }
- if j := strings.LastIndex(s, "}"); j >= 0 && j+1 < len(s) {
- s = s[:j+1]
- }
- s = strings.TrimPrefix(s, "```json")
- s = strings.TrimPrefix(s, "```")
- s = strings.TrimSuffix(s, "```")
var out struct {
Items []Claim `json:"items"`
diff --git a/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go
index fe361fc5c1..c89f4cc434 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/duplicate_titles.go
@@ -177,19 +177,12 @@ func rewriteTitleGroup(ctx context.Context, deps common.Deps, llmID, title strin
// unmarshalRenames parses the LLM rename array, tolerating code fences.
func unmarshalRenames(raw string, out *[]renameOut) error {
- s := strings.TrimSpace(raw)
- s = strings.TrimPrefix(s, "```json")
- s = strings.TrimPrefix(s, "```")
- s = strings.TrimSuffix(s, "```")
- if err := json.Unmarshal([]byte(s), out); err == nil {
- return nil
- }
- // Fallback: the model may have wrapped the array in extra prose — grab the
- // first '[' … last ']' span.
- start := strings.IndexByte(s, '[')
- end := strings.LastIndexByte(s, ']')
- if start < 0 || end <= start {
- return fmt.Errorf("no JSON array in rename response")
- }
- return json.Unmarshal([]byte(s[start:end+1]), out)
+ s, err := common.RepairJSONText(raw)
+ if err != nil {
+ return fmt.Errorf("invalid rename response: %w", err)
+ }
+ if err := json.Unmarshal([]byte(s), out); err != nil {
+ return fmt.Errorf("rename response is not a JSON array: %w", err)
+ }
+ return nil
}
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 68312b11f3..31df1251d5 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -1689,7 +1689,28 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {
// dataset row's description stays a plain-text string (mirror Python
// _struct_merge_graph_entities, which folds the entity description, not the
// whole payload). Fall back to Content only when it is not a JSON object.
- if desc := structureProductDescription(p.Content); desc != "" {
+ desc := structureProductDescription(p.Content)
+ if desc == "" {
+ // Match Python's dataset merger fallback: mindmap entities only carry
+ // name/type, while their dataset row still needs searchable text.
+ // Relations likewise use the endpoint names and relation type when the
+ // source payload has no description.
+ from := metaString(p.Meta, "from")
+ to := metaString(p.Meta, "to")
+ if kind := metaString(p.Meta, "kind"); kind == "relation" || (from != "" && to != "") {
+ relType := metaString(p.Meta, "relation_type")
+ if relType == "" {
+ relType = metaString(p.Meta, "type")
+ }
+ if relType == "" {
+ relType = "related"
+ }
+ desc = strings.TrimSpace(strings.Join([]string{from, relType, to}, " "))
+ } else {
+ desc = strings.TrimSpace(metaString(p.Meta, "name"))
+ }
+ }
+ if desc != "" {
if b.Description != "" {
b.Description += "\n"
}
diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.go
index 4e10a1c922..bd1dd2a5f5 100644
--- a/internal/ingestion/knowledge_compile/structure_merge_test.go
+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go
@@ -181,6 +181,39 @@ func TestMergeStructureDataset_DescriptionIsPlainText(t *testing.T) {
}
}
+func TestMergeStructureDataset_DescriptionFallbacks(t *testing.T) {
+ c := &Consumer{writer: &fakeWriter{}}
+ products := []kccommon.Product{
+ {Variant: kccommon.VariantMindmap, DocID: "d1",
+ Content: `{"name":"Root","type":"mindmap"}`,
+ Meta: map[string]any{"kind": "entity", "name": "Root", "entity_type": "mindmap"}},
+ {Variant: kccommon.VariantMindmap, DocID: "d1",
+ Content: `{"from":"Root","to":"Child","type":"related"}`,
+ Meta: map[string]any{"kind": "relation", "from": "Root", "to": "Child", "relation_type": "related"}},
+ }
+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {
+ t.Fatalf("mergeStructureDataset: %v", err)
+ }
+ fw := c.writer.(*fakeWriter)
+ if len(fw.buckets) != 2 {
+ t.Fatalf("want entity and relation buckets, got %d", len(fw.buckets))
+ }
+ for _, b := range fw.buckets {
+ switch b.Type {
+ case "mindmap":
+ if b.Description != "Root" {
+ t.Errorf("entity fallback description = %q, want %q", b.Description, "Root")
+ }
+ case "relation":
+ if b.Description != "Root related Child" {
+ t.Errorf("relation fallback description = %q, want %q", b.Description, "Root related Child")
+ }
+ default:
+ t.Errorf("unexpected bucket: %+v", b)
+ }
+ }
+}
+
// TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind covers the
// option-A alignment: the dataset row's compile_kwd is the doc row's inferred
// compile type (autotype, e.g. "hypergraph"/"mindmap"), NOT the template kind.
diff --git a/internal/rag/advanced_rag/harness/action_session.go b/internal/rag/advanced_rag/harness/action_session.go
index 333eab5afa..dea5623337 100644
--- a/internal/rag/advanced_rag/harness/action_session.go
+++ b/internal/rag/advanced_rag/harness/action_session.go
@@ -30,9 +30,9 @@ import (
"time"
"unicode/utf8"
- "github.com/RealAlexandreAI/json-repair"
"github.com/cloudwego/eino/compose"
"github.com/cloudwego/eino/schema"
+ "github.com/kaptinlin/jsonrepair"
"gorm.io/gorm"
"ragflow/internal/agent/chat"
@@ -791,10 +791,9 @@ func stripRawControlChars(s string) string {
// the single-object path cannot — unquoted object keys ({a: 1}), missing
// separators between adjacent values ({"a":1} {"b":2} or "x" "y"), bare string
// values ({name: foo}), and any combination the model emits across the full
-// output. It delegates to github.com/RealAlexandreAI/json-repair, the Go port of
-// the same json_repair library Python uses, so behavior matches exactly. It
-// returns nil when nothing parses, exactly as json_repair would yield a parse
-// error.
+// output. It delegates to github.com/kaptinlin/jsonrepair, which repairs
+// malformed JSON returned by LLMs. It returns nil when nothing parses, exactly
+// as json_repair would yield a parse error.
func repairJSONWhole(text string) any {
cleaned := reThinkWrap.ReplaceAllString(text, "")
cleaned = reFencedJSON.ReplaceAllString(cleaned, "$1")
@@ -808,10 +807,10 @@ func repairJSONWhole(text string) any {
// value; quoting keys first sidesteps that and matches what Python's
// json_repair does internally. Already-valid JSON is left untouched.
normalized := quoteUnquotedKeys(cleaned)
- repaired, err := jsonrepair.RepairJSON(normalized)
+ repaired, err := jsonrepair.Repair(normalized)
if err != nil {
// Fall back to the un-normalized input in case the heuristic mis-fired.
- repaired, err = jsonrepair.RepairJSON(cleaned)
+ repaired, err = jsonrepair.Repair(cleaned)
if err != nil {
return nil
}
diff --git a/internal/utility/markdown_to_json.go b/internal/utility/markdown_to_json.go
index 73cccdfe06..439a0bf2c7 100644
--- a/internal/utility/markdown_to_json.go
+++ b/internal/utility/markdown_to_json.go
@@ -88,10 +88,13 @@ type kv struct {
}
type OMap []kv
-// Node is one shaped mind-map node (Python's {"id", "children"} shape).
+// Node is one shaped mind-map node. SourceChunkIDs is populated by the Go
+// mindmap compiler when the model returns provenance in its JSON response;
+// legacy Markdown shaping leaves it empty.
type Node struct {
- ID string
- Children []*Node
+ ID string
+ SourceChunkIDs []string
+ Children []*Node
}
type blockType int
diff --git a/web/src/components/structure-graph/adapters.tsx b/web/src/components/structure-graph/adapters.tsx
index c9bb97c2ee..1f194942a2 100644
--- a/web/src/components/structure-graph/adapters.tsx
+++ b/web/src/components/structure-graph/adapters.tsx
@@ -371,7 +371,10 @@ export function adaptMindMapToIndentedTree(
const roots = buildUniqueTreeDataItems(
template.entities,
template.relations,
- ['has_branch', 'has_sub_branch'],
+ // Python's mindmap structure-graph projection stores parent-child edges
+ // as generic "related" relations. Keep the branch spellings for older
+ // data, but accept the canonical Python form as well.
+ ['related', 'has_branch', 'has_sub_branch'],
);
const g6Roots = roots.map(treeDataItemToG6TreeData);