RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: knowledge compilation skip writting extra field (#21278)
Wang Qid6ad95429819
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 471a45bf4f..c5da9b7d53 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -6,7 +6,6 @@ package knowledge_compilerimport ("context"- "encoding/json""fmt""maps""slices"@@ -448,17 +447,6 @@ func productsToChunkDocs(products []common.Product, language string) ([]schema.Cif err := doc.SetExtraValue("compile_kwd", compileKWD); err != nil {return nil, err}- if p.Variant == common.VariantStructure {- if inferred := metaString(p.Meta, "compile_kwd"); inferred != "" {- extra, err := json.Marshal(map[string]string{"compile_type": inferred})- if err != nil {- return nil, err- }- if err := doc.SetExtraValue("extra", string(extra)); err != nil {- return nil, err- }- }- }// scope_kwd marks doc/dataset-level rows (B8/O1=B). A doc-level compiled// product is always a per-document (scope="doc") input to the dataset-level// merge; the consumer rewrites dataset-level rows with scope_kwd="dataset".diff --git a/internal/ingestion/component/knowledge_compiler/storage_test.go b/internal/ingestion/component/knowledge_compiler/storage_test.goindex a957f0dedc..cf18aa2da6 100644--- a/internal/ingestion/component/knowledge_compiler/storage_test.go+++ b/internal/ingestion/component/knowledge_compiler/storage_test.go@@ -1,7 +1,6 @@package knowledge_compilerimport (- "encoding/json""testing""ragflow/internal/ingestion/component/knowledge_compiler/common"@@ -38,12 +37,8 @@ func TestProductsToChunkDocsCanonicalKinds(t *testing.T) {}}if test.variant == common.VariantStructure {- var extra map[string]string- if err := json.Unmarshal([]byte(row["extra"].(string)), &extra); err != nil {- t.Fatal(err)- }- if extra["compile_type"] != "list" {- t.Fatalf("compile type = %v", extra)+ if _, exists := row["extra"]; exists {+ t.Fatalf("compiler must not write extra: %#v", row["extra"])}}if test.variant == common.VariantWiki && (row["type_kwd"] != "wiki_entity" || row["entity_type_kwd"] != "concept") {diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 2a2ee30522..4408edd64b 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -1672,7 +1672,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,k := rkey{template: template, from: normalizedStructureEntityName(from), typ: structureRelationType(relType), to: normalizedStructureEntityName(to), ckwd: ckwd}b := relByKey[k]if b == nil {- b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, CompileType: metaString(p.Meta, "compile_type"), RelationType: relType}+ b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, RelationType: relType}relByKey[k] = b}appendBucket(b, p)@@ -1690,7 +1690,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,k := ekey{template: template, name: normalizedStructureEntityName(name), ckwd: ckwd}b := entByKey[k]if b == nil {- b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, CompileType: metaString(p.Meta, "compile_type")}+ b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind}entByKey[k] = b} else {b.Type = preferredStructureEntityType(b.Type, typ)diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 4e8c683135..20f523c69f 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -110,7 +110,7 @@ type engineReader struct {// unknown column); payload = content_with_weight, kind = compile_kwd.var compiledSelectFields = []string{"id", "doc_id", "compile_kwd",- "available_int", "extra",+ "available_int","content_with_weight","source_chunk_ids", "source_doc_ids","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",@@ -368,20 +368,8 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommopayload = nil}}- if raw := asString(c["extra"]); raw != "" {- var extra map[string]any- if err := json.Unmarshal([]byte(raw), &extra); err != nil {- return kccommon.Product{}, false- }- if inferred, ok := extra["compile_type"].(string); ok {- meta["compile_type"] = inferred- }- }if v := asString(c["compile_kwd"]); v != "" {meta["compile_kwd"] = v- if _, ok := meta["compile_type"]; !ok && types.CanonicalCompilationKind(v) != v {- meta["compile_type"] = v- }}if v, ok := c["name_kwd"].(string); ok && v != "" {name := vdiff --git a/internal/ingestion/knowledge_compile/storage.md b/internal/ingestion/knowledge_compile/storage.mdindex 8d8f82c1af..10ff043754 100644--- a/internal/ingestion/knowledge_compile/storage.md+++ b/internal/ingestion/knowledge_compile/storage.md@@ -15,8 +15,8 @@ session kinds remain `session_graph` and `session_essence`.Wiki page categories such as `concept`, `entity`, and `topic` are stored in`entity_type_kwd`. Inferred structure extraction types such as `list`, `set`,-and `hypergraph` are retained as `compile_type` inside the JSON string in `extra`.-Neither changes the compilation kind.+and `hypergraph` guide extraction but are not persisted. Compilation products do+not write `extra`.New writes omit `compilation_template_kind_kwd` and `page_type_kwd`.Their engine columns remain available for reading existing data.diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 1f441a13c5..135000d768 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -95,7 +95,6 @@ type Writer interface {// folded descriptions and the union of source docs/chunks. It is written as a// scope_kwd="dataset" row. Relations carry FromEntity/ToEntity instead of Name.type StructureBucket struct {- CompileType stringName stringType stringDescription string // folded entity descriptions@@ -570,13 +569,6 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin"create_time": now.Format("2006-01-02 15:04:05"),"create_timestamp_flt": float64(now.Unix()),}- if b.CompileType != "" {- extra, err := json.Marshal(map[string]string{"compile_type": b.CompileType})- if err != nil {- return err- }- row["extra"] = string(extra)- }if b.TemplateID != "" {row["compilation_template_ids"] = []string{b.TemplateID}}diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex 12e7be8a74..8c8080f19d 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -262,6 +262,9 @@ func TestWriteMergedStructureKeepsEntityKeywordLowercase(t *testing.T) {t.Errorf("%s = %q, want searchable entity payload text", field, text)}}+ if _, exists := row["extra"]; exists {+ t.Fatalf("compiler must not write extra: %#v", row["extra"])+ }return}t.Fatal("inserted rows contain no entity")