Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: knowledge compilation skip writting extra field (#21278)

Wang Qid6ad95429819
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 471a45bf4f..c5da9b7d53 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -6,7 +6,6 @@ package knowledge_compiler
import (
"context"
- "encoding/json"
"fmt"
"maps"
"slices"
@@ -448,17 +447,6 @@ func productsToChunkDocs(products []common.Product, language string) ([]schema.C
if err := doc.SetExtraValue("compile_kwd", compileKWD); err != nil {
return nil, err
}
- if p.Variant == common.VariantStructure {
- if inferred := metaString(p.Meta, "compile_kwd"); inferred != "" {
- extra, err := json.Marshal(map[string]string{"compile_type": inferred})
- if err != nil {
- return nil, err
- }
- if err := doc.SetExtraValue("extra", string(extra)); err != nil {
- return nil, err
- }
- }
- }
// scope_kwd marks doc/dataset-level rows (B8/O1=B). A doc-level compiled
// product is always a per-document (scope="doc") input to the dataset-level
// merge; the consumer rewrites dataset-level rows with scope_kwd="dataset".
diff --git a/internal/ingestion/component/knowledge_compiler/storage_test.go b/internal/ingestion/component/knowledge_compiler/storage_test.go
index a957f0dedc..cf18aa2da6 100644
--- a/internal/ingestion/component/knowledge_compiler/storage_test.go
+++ b/internal/ingestion/component/knowledge_compiler/storage_test.go
@@ -1,7 +1,6 @@
package knowledge_compiler
import (
- "encoding/json"
"testing"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
@@ -38,12 +37,8 @@ func TestProductsToChunkDocsCanonicalKinds(t *testing.T) {
}
}
if test.variant == common.VariantStructure {
- var extra map[string]string
- if err := json.Unmarshal([]byte(row["extra"].(string)), &extra); err != nil {
- t.Fatal(err)
- }
- if extra["compile_type"] != "list" {
- t.Fatalf("compile type = %v", extra)
+ if _, exists := row["extra"]; exists {
+ t.Fatalf("compiler must not write extra: %#v", row["extra"])
}
}
if test.variant == common.VariantWiki && (row["type_kwd"] != "wiki_entity" || row["entity_type_kwd"] != "concept") {
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 2a2ee30522..4408edd64b 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -1672,7 +1672,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
k := rkey{template: template, from: normalizedStructureEntityName(from), typ: structureRelationType(relType), to: normalizedStructureEntityName(to), ckwd: ckwd}
b := relByKey[k]
if b == nil {
- b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, CompileType: metaString(p.Meta, "compile_type"), RelationType: relType}
+ b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, RelationType: relType}
relByKey[k] = b
}
appendBucket(b, p)
@@ -1690,7 +1690,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
k := ekey{template: template, name: normalizedStructureEntityName(name), ckwd: ckwd}
b := entByKey[k]
if b == nil {
- b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, CompileType: metaString(p.Meta, "compile_type")}
+ b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind}
entByKey[k] = b
} else {
b.Type = preferredStructureEntityType(b.Type, typ)
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index 4e8c683135..20f523c69f 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -110,7 +110,7 @@ type engineReader struct {
// unknown column); payload = content_with_weight, kind = compile_kwd.
var compiledSelectFields = []string{
"id", "doc_id", "compile_kwd",
- "available_int", "extra",
+ "available_int",
"content_with_weight",
"source_chunk_ids", "source_doc_ids",
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
@@ -368,20 +368,8 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
payload = nil
}
}
- if raw := asString(c["extra"]); raw != "" {
- var extra map[string]any
- if err := json.Unmarshal([]byte(raw), &extra); err != nil {
- return kccommon.Product{}, false
- }
- if inferred, ok := extra["compile_type"].(string); ok {
- meta["compile_type"] = inferred
- }
- }
if v := asString(c["compile_kwd"]); v != "" {
meta["compile_kwd"] = v
- if _, ok := meta["compile_type"]; !ok && types.CanonicalCompilationKind(v) != v {
- meta["compile_type"] = v
- }
}
if v, ok := c["name_kwd"].(string); ok && v != "" {
name := v
diff --git a/internal/ingestion/knowledge_compile/storage.md b/internal/ingestion/knowledge_compile/storage.md
index 8d8f82c1af..10ff043754 100644
--- a/internal/ingestion/knowledge_compile/storage.md
+++ b/internal/ingestion/knowledge_compile/storage.md
@@ -15,8 +15,8 @@ session kinds remain `session_graph` and `session_essence`.
Wiki page categories such as `concept`, `entity`, and `topic` are stored in
`entity_type_kwd`. Inferred structure extraction types such as `list`, `set`,
-and `hypergraph` are retained as `compile_type` inside the JSON string in `extra`.
-Neither changes the compilation kind.
+and `hypergraph` guide extraction but are not persisted. Compilation products do
+not write `extra`.
New writes omit `compilation_template_kind_kwd` and `page_type_kwd`.
Their engine columns remain available for reading existing data.
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 1f441a13c5..135000d768 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -95,7 +95,6 @@ type Writer interface {
// folded descriptions and the union of source docs/chunks. It is written as a
// scope_kwd="dataset" row. Relations carry FromEntity/ToEntity instead of Name.
type StructureBucket struct {
- CompileType string
Name string
Type string
Description string // folded entity descriptions
@@ -570,13 +569,6 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
"create_time": now.Format("2006-01-02 15:04:05"),
"create_timestamp_flt": float64(now.Unix()),
}
- if b.CompileType != "" {
- extra, err := json.Marshal(map[string]string{"compile_type": b.CompileType})
- if err != nil {
- return err
- }
- row["extra"] = string(extra)
- }
if b.TemplateID != "" {
row["compilation_template_ids"] = []string{b.TemplateID}
}
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 12e7be8a74..8c8080f19d 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -262,6 +262,9 @@ func TestWriteMergedStructureKeepsEntityKeywordLowercase(t *testing.T) {
t.Errorf("%s = %q, want searchable entity payload text", field, text)
}
}
+ if _, exists := row["extra"]; exists {
+ t.Fatalf("compiler must not write extra: %#v", row["extra"])
+ }
return
}
t.Fatal("inserted rows contain no entity")