RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): consolidate duplicated meta-reader helpers into kccommon (#21983)
xugangqiang106c7d5cbf39
diff --git a/internal/common/format.go b/internal/common/format.goindex 39c0d1399f..8f82d2cb76 100644--- a/internal/common/format.go+++ b/internal/common/format.go@@ -191,3 +191,16 @@ func IsValidString(v interface{}) bool {func ChunkID(docID, text string) string {return fmt.Sprintf("%016x", xxhash.Sum64String(text+docID))}++// FirstNonEmpty returns the first value that is non-empty after trimming,+// trimmed. Trimming the returned value means callers receive a normalized result+// and need not TrimSpace again at every use site. Used as a generic+// settings/value precedence helper across the codebase.+func FirstNonEmpty(vals ...string) string {+ for _, v := range vals {+ if strings.TrimSpace(v) != "" {+ return strings.TrimSpace(v)+ }+ }+ return ""+}diff --git a/internal/common/format_test.go b/internal/common/format_test.goindex 936c896b25..ef1ee6622c 100644--- a/internal/common/format_test.go+++ b/internal/common/format_test.go@@ -97,3 +97,26 @@ func TestBaseModelName(t *testing.T) {})}}++func TestFirstNonEmpty(t *testing.T) {+ tests := []struct {+ name string+ in []string+ want string+ }{+ {"plain value unchanged", []string{"a", "b"}, "a"},+ {"returns first when later match", []string{"x", "y", "z"}, "x"},+ {"trims chosen value", []string{" picked ", "b"}, "picked"},+ {"chosen whitespace later clean", []string{" picked ", "clean"}, "picked"},+ {"skips whitespace-only candidates", []string{" ", " ", "real"}, "real"},+ {"all empty returns empty", []string{"", " ", " "}, ""},+ {"nil returns empty", nil, ""},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := FirstNonEmpty(tt.in...); got != tt.want {+ t.Fatalf("FirstNonEmpty(%v) = %q, want %q", tt.in, got, tt.want)+ }+ })+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/common/migration_helpers_test.go b/internal/ingestion/component/knowledge_compiler/common/migration_helpers_test.gonew file mode 100644index 0000000000..1fad8a3d20--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/common/migration_helpers_test.go@@ -0,0 +1,124 @@+package common++import (+ "reflect"+ "testing"+)++// TestFirstNonEmptyReturnsTrimmed locks the contract that FirstNonEmpty returns+// the *trimmed* first non-empty value. The wiki pipeline historically relied on+// this (its local firstNonEmpty trimmed and returned the trimmed value); the+// consolidated helper must preserve that so the four local copies can be deleted+// without a behavior change.+func TestFirstNonEmptyReturnsTrimmed(t *testing.T) {+ cases := []struct {+ name string+ in []string+ want string+ }{+ {"plain value unchanged", []string{"a", "b"}, "a"},+ {"returns first when later match", []string{"x", "y", "z"}, "x"},+ {"trims leading/trailing space of the chosen value", []string{" picked ", "b"}, "picked"},+ // Adversarial: the chosen candidate carries whitespace and a later+ // candidate does not. A wrong (untrimmed-return) implementation would+ // return " picked " here.+ {"chosen whitespace, later clean", []string{" picked ", "clean"}, "picked"},+ {"skips whitespace-only candidates", []string{" ", " ", "real"}, "real"},+ {"all empty returns empty", []string{"", " ", " "}, ""},+ {"nil returns empty", nil, ""},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ if got := FirstNonEmpty(c.in...); got != c.want {+ t.Fatalf("FirstNonEmpty(%v) = %q, want %q", c.in, got, c.want)+ }+ })+ }+}++// TestMetaString locks the canonical MetaString behavior: returns the string+// value as-is, and "" for a missing or non-string key.+func TestMetaString(t *testing.T) {+ m := map[string]any{"k": "v", "n": 123}+ if got := MetaString(m, "k"); got != "v" {+ t.Fatalf("MetaString(k) = %q, want v", got)+ }+ if got := MetaString(m, "n"); got != "" {+ t.Fatalf("MetaString(non-string) = %q, want empty", got)+ }+ if got := MetaString(m, "missing"); got != "" {+ t.Fatalf("MetaString(missing) = %q, want empty", got)+ }+}++// TestMetaInt locks the canonical int64 MetaInt: tolerant of int64, int,+// float64, and numeric string boxings; missing/non-numeric yields (0, false).+func TestMetaInt(t *testing.T) {+ m := map[string]any{+ "i64": int64(7),+ "i": int(8),+ "f": float64(9.0),+ "s": "42",+ "bad": "abc",+ "non": true,+ }+ if v, ok := MetaInt(m, "i64"); v != 7 || !ok {+ t.Fatalf("MetaInt(i64) = %d,%v want 7,true", v, ok)+ }+ if v, ok := MetaInt(m, "i"); v != 8 || !ok {+ t.Fatalf("MetaInt(i) = %d,%v want 8,true", v, ok)+ }+ if v, ok := MetaInt(m, "f"); v != 9 || !ok {+ t.Fatalf("MetaInt(f) = %d,%v want 9,true", v, ok)+ }+ // Adversarial: numeric string boxed value must still parse.+ if v, ok := MetaInt(m, "s"); v != 42 || !ok {+ t.Fatalf("MetaInt(s) = %d,%v want 42,true", v, ok)+ }+ if v, ok := MetaInt(m, "bad"); v != 0 || ok {+ t.Fatalf("MetaInt(bad) = %d,%v want 0,false", v, ok)+ }+ if v, ok := MetaInt(m, "non"); v != 0 || ok {+ t.Fatalf("MetaInt(non-numeric) = %d,%v want 0,false", v, ok)+ }+ if v, ok := MetaInt(m, "missing"); v != 0 || ok {+ t.Fatalf("MetaInt(missing) = %d,%v want 0,false", v, ok)+ }+}++// TestMetaStringSlice locks the canonical filter-empty metaStringSlice: reads+// []string and []any, keeps non-empty strings, drops empties (the []any branch+// previously filtered; the source data is produced empty-free by the dedup+// helpers so this is behavior-preserving with both legacy readers).+func TestMetaStringSlice(t *testing.T) {+ m := map[string]any{+ "strs": []string{"a", "", "b"},+ "strsEmpty": []string{"", "", ""},+ "anys": []any{"a", "", 5, "b"},+ "anyson": []any{1, 2},+ "missing": "not-a-slice",+ }+ // Both boxings must drop empties: a unified []string/[]any reader must not let+ // the source type change the result. The []string branch previously kept+ // empties (inconsistent with []any) — this guards the fix.+ if got := MetaStringSlice(m, "strs"); !reflect.DeepEqual(got, []string{"a", "b"}) {+ t.Fatalf("MetaStringSlice(strs) = %v, want [a b]", got)+ }+ // A []string that is all-empty yields an empty (non-nil) slice, mirroring the+ // []any branch — never nil for a present typed []string key.+ if got := MetaStringSlice(m, "strsEmpty"); len(got) != 0 {+ t.Fatalf("MetaStringSlice(strsEmpty) = %v, want empty slice", got)+ }+ // []any: drops non-strings and empties.+ if got := MetaStringSlice(m, "anys"); !reflect.DeepEqual(got, []string{"a", "b"}) {+ t.Fatalf("MetaStringSlice(anys) = %v, want [a b]", got)+ }+ // All entries dropped: the []any branch returns a non-nil empty slice+ // (matches the legacy reader); a non-slice/missing key returns nil.+ if got := MetaStringSlice(m, "anyson"); len(got) != 0 {+ t.Fatalf("MetaStringSlice(anyson) = %v, want empty slice", got)+ }+ if got := MetaStringSlice(m, "missing"); got != nil {+ t.Fatalf("MetaStringSlice(missing) = %v, want nil", got)+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/common/relations.go b/internal/ingestion/component/knowledge_compiler/common/relations.goindex f0e81e8ad1..717e9bb90e 100644--- a/internal/ingestion/component/knowledge_compiler/common/relations.go+++ b/internal/ingestion/component/knowledge_compiler/common/relations.go@@ -9,10 +9,10 @@ import "strings"func FilterOrphanRelations(products []Product) []Product {entities := make(map[string]struct{}, len(products))for _, product := range products {- if product.Meta == nil || strings.TrimSpace(metaString(product.Meta, "kind")) != "entity" {+ if product.Meta == nil || strings.TrimSpace(MetaString(product.Meta, "kind")) != "entity" {continue}- name := normalizeRelationEndpoint(metaString(product.Meta, "name"))+ name := normalizeRelationEndpoint(MetaString(product.Meta, "name"))if name != "" {entities[relationScope(product)+"\x00"+name] = struct{}{}}@@ -20,9 +20,9 @@ func FilterOrphanRelations(products []Product) []Product {filtered := make([]Product, 0, len(products))for _, product := range products {- from := normalizeRelationEndpoint(metaString(product.Meta, "from"))- to := normalizeRelationEndpoint(metaString(product.Meta, "to"))- isRelation := strings.TrimSpace(metaString(product.Meta, "kind")) == "relation" || (from != "" || to != "")+ from := normalizeRelationEndpoint(MetaString(product.Meta, "from"))+ to := normalizeRelationEndpoint(MetaString(product.Meta, "to"))+ isRelation := strings.TrimSpace(MetaString(product.Meta, "kind")) == "relation" || (from != "" || to != "")if isRelation && (from == "" || to == "" || !hasRelationEntity(entities, product, from) || !hasRelationEntity(entities, product, to)) {continue}@@ -37,7 +37,7 @@ func hasRelationEntity(entities map[string]struct{}, product Product, name strin}func relationScope(product Product) string {- compileKind := metaString(product.Meta, "compile_kwd")+ compileKind := MetaString(product.Meta, "compile_kwd")if compileKind == "" {compileKind = string(product.Variant)}@@ -52,7 +52,7 @@ func normalizeRelationEndpoint(value string) string {return strings.ToLower(strings.Join(strings.Fields(value), " "))}-func metaString(meta map[string]any, key string) string {+func MetaString(meta map[string]any, key string) string {value, _ := meta[key].(string)return value}diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.goindex 34c2136bb6..68a99e2988 100644--- a/internal/ingestion/component/knowledge_compiler/common/types.go+++ b/internal/ingestion/component/knowledge_compiler/common/types.go@@ -12,6 +12,8 @@ import ("errors""fmt""strings"++ appcommon "ragflow/internal/common")// Variant identifies which knowledge-compile strategy to run.@@ -279,14 +281,64 @@ func VariantToTaskType(variant Variant) string {}}-// FirstNonEmpty returns the first string that is non-empty after trimming.+// FirstNonEmpty returns the first string that is non-empty after trimming,+// trimmed. It delegates to the generic helper in internal/common so the+// knowledge_compiler package shares one implementation with the rest of the+// codebase (the generic component layer no longer needs to reach into this+// package for a meta-agnostic string helper). The trimmed return value preserves+// the historical contract of the wiki pipeline's local helper.func FirstNonEmpty(vals ...string) string {- for _, v := range vals {- if strings.TrimSpace(v) != "" {- return v+ return appcommon.FirstNonEmpty(vals...)+}++// MetaInt extracts an integer from a Product.Meta value that may be boxed as+// int64, int, float64 (JSON number), or even a numeric string — the+// engine/JSON round-trip does not guarantee a single numeric type. It returns+// (0, false) when the key is absent or not a parseable number. The int64 form+// is the canonical one so callers can store it directly without a narrowing+// conversion.+func MetaInt(meta map[string]any, key string) (int64, bool) {+ switch v := meta[key].(type) {+ case int64:+ return v, true+ case int:+ return int64(v), true+ case float64:+ return int64(v), true+ case string:+ var n int64+ if _, err := fmt.Sscanf(v, "%d", &n); err == nil {+ return n, true+ }+ }+ return 0, false+}++// MetaStringSlice reads a []string (or []any) Product.Meta key, keeping+// non-empty strings and dropping empties. Both the []string and []any branches+// filter empties so the result is independent of how the value was boxed (the+// engine/JSON round-trip does not guarantee a single type). Returns nil when no+// strings are present; a present but empty key yields a non-nil empty slice.+func MetaStringSlice(meta map[string]any, key string) []string {+ switch v := meta[key].(type) {+ case []string:+ out := make([]string, 0, len(v))+ for _, e := range v {+ if e != "" {+ out = append(out, e)+ }}+ return out+ case []any:+ out := make([]string, 0, len(v))+ for _, e := range v {+ if s, ok := e.(string); ok && s != "" {+ out = append(out, s)+ }+ }+ return out}- return ""+ return nil}// VectorFromChunkMap extracts a pre-computed embedding from a chunk map. Thediff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 57a451df21..e576f838a6 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -248,13 +248,13 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, invar pageCount, sectionCount, entityCount, relationCount intfor _, p := range out.Products {switch {- case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "page":+ case p.Variant == common.VariantWiki && common.MetaString(p.Meta, "kind") == "page":pageCount++- case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "section":+ case p.Variant == common.VariantWiki && common.MetaString(p.Meta, "kind") == "section":sectionCount++- case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "entity":+ case p.Variant == common.VariantWiki && common.MetaString(p.Meta, "kind") == "entity":entityCount++- case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "relation":+ case p.Variant == common.VariantWiki && common.MetaString(p.Meta, "kind") == "relation":relationCount++}}@@ -452,12 +452,12 @@ func productsToChunkDocs(products []common.Product, language string) ([]schema.C}}// Provenance columns shared by all compiled rows.- if ids := metaStringSlice(p.Meta, "source_chunk_ids"); len(ids) > 0 {+ if ids := common.MetaStringSlice(p.Meta, "source_chunk_ids"); len(ids) > 0 {if err := doc.SetExtraValue("source_chunk_ids", ids); err != nil {return nil, err}}- if ids := metaStringSlice(p.Meta, "source_doc_ids"); len(ids) > 0 {+ if ids := common.MetaStringSlice(p.Meta, "source_doc_ids"); len(ids) > 0 {if err := doc.SetExtraValue("source_doc_ids", ids); err != nil {return nil, err}@@ -465,7 +465,7 @@ func productsToChunkDocs(products []common.Product, language string) ([]schema.C// compilation_template_ids: the resolved template ids that produced// this row (Python stamps one per row from the active template; the// document-structure endpoint groups rows by this column).- if ids := metaStringSlice(p.Meta, "compilation_template_ids"); len(ids) > 0 {+ if ids := common.MetaStringSlice(p.Meta, "compilation_template_ids"); len(ids) > 0 {if err := doc.SetExtraValue("compilation_template_ids", ids); err != nil {return nil, err}@@ -483,7 +483,7 @@ func productsToChunkDocs(products []common.Product, language string) ([]schema.C// conf/infinity_mapping.json lines 45–77, driven by Product.Meta keys that// each variant's build site populates. Unknown/absent keys are skipped.func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.Tokenizer) error {- kind := metaString(p.Meta, "kind")+ kind := common.MetaString(p.Meta, "kind")switch p.Variant {case common.VariantStructure:@@ -502,8 +502,8 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T// slug_kwd stores the full "<page_type>/<slug>" form, so retrieval// filters (GetWikiPage) can reconstruct it directly. page_type is also// stored separately for topic grouping.- if pageType := metaString(p.Meta, "page_type"); pageType != "" {- if slug := metaString(p.Meta, "slug"); slug != "" {+ if pageType := common.MetaString(p.Meta, "page_type"); pageType != "" {+ if slug := common.MetaString(p.Meta, "slug"); slug != "" {// Normalize to the full "<page_type>/<slug>" form (Python writer// contract). Idempotent: a slug that already carries the prefix// (some producers emit pageType/slug directly) is left as-is.@@ -516,39 +516,39 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T}}}- if v := metaString(p.Meta, "title"); v != "" {+ if v := common.MetaString(p.Meta, "title"); v != "" {if err := doc.SetExtraValue("title_kwd", v); err != nil {return err}setTitleTokens(doc, v, tok)}- if v := metaString(p.Meta, "page_type"); v != "" {+ if v := common.MetaString(p.Meta, "page_type"); v != "" {if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {return err}}- if v := metaString(p.Meta, "topic"); v != "" {+ if v := common.MetaString(p.Meta, "topic"); v != "" {if err := doc.SetExtraValue("topic_kwd", v); err != nil {return err}}- if v := metaString(p.Meta, "summary"); v != "" {+ if v := common.MetaString(p.Meta, "summary"); v != "" {if err := doc.SetExtraValue("summary_with_weight", v); err != nil {return err}}// Section depth goes to depth_int; there is no section_level_int column.- if v, ok := metaInt(p.Meta, "section_level"); ok {+ if v, ok := common.MetaInt(p.Meta, "section_level"); ok {if err := doc.SetExtraValue("depth_int", v); err != nil {return err}}- if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {+ if v := common.MetaStringSlice(p.Meta, "entity_names"); len(v) > 0 {if err := doc.SetExtraValue("entity_names_kwd", v); err != nil {return err}}- if v := metaStringSlice(p.Meta, "outlinks"); len(v) > 0 {+ if v := common.MetaStringSlice(p.Meta, "outlinks"); len(v) > 0 {if err := doc.SetExtraValue("outlinks_kwd", v); err != nil {return err}@@ -556,7 +556,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.Treturn err}}- if v := metaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {+ if v := common.MetaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {if err := doc.SetExtraValue("related_kb_pages_kwd", v); err != nil {return err}@@ -580,17 +580,17 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T// them out of the artifacts query (it filters// type_kwd=["entity","relation"]) without a frontend// change. Python mirrors this in _struct_upsert_tree_claim_rows.- if v := metaString(p.Meta, "name"); v != "" {+ if v := common.MetaString(p.Meta, "name"); v != "" {if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {return err}}- if v := metaString(p.Meta, "entity_type"); v != "" {+ if v := common.MetaString(p.Meta, "entity_type"); v != "" {if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {return err}}- if v, ok := metaInt(p.Meta, "mention_count"); ok {+ if v, ok := common.MetaInt(p.Meta, "mention_count"); ok {if err := doc.SetExtraValue("mention_count_int", v); err != nil {return err}@@ -604,7 +604,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.Treturn err}}- if v, ok := metaInt(p.Meta, "level"); ok {+ if v, ok := common.MetaInt(p.Meta, "level"); ok {if err := doc.SetExtraValue("raptor_layer_int", v); err != nil {return err}@@ -612,7 +612,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.Treturn err}}- if v := metaStringSlice(p.Meta, "children"); len(v) > 0 {+ if v := common.MetaStringSlice(p.Meta, "children"); len(v) > 0 {if err := doc.SetExtraValue("children_kwd", v); err != nil {return err}@@ -649,30 +649,30 @@ func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind str}}if kind == "relation" {- if v := metaString(p.Meta, "from"); v != "" {+ if v := common.MetaString(p.Meta, "from"); v != "" {if err := doc.SetExtraValue("from_entity_kwd", v); err != nil {return err}}- if v := metaString(p.Meta, "to"); v != "" {+ if v := common.MetaString(p.Meta, "to"); v != "" {if err := doc.SetExtraValue("to_entity_kwd", v); err != nil {return err}}}if kind == "entity" {- if v := metaString(p.Meta, "name"); v != "" {+ if v := common.MetaString(p.Meta, "name"); v != "" {if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {return err}}- if v := metaString(p.Meta, "entity_type"); v != "" {+ if v := common.MetaString(p.Meta, "entity_type"); v != "" {if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {return err}}}- if v, ok := metaInt(p.Meta, "mention_count"); ok {+ if v, ok := common.MetaInt(p.Meta, "mention_count"); ok {if err := doc.SetExtraValue("mention_count_int", v); err != nil {return err}@@ -681,10 +681,6 @@ func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind str}// metaString reads a string-valued Product.Meta key.-func metaString(m map[string]any, key string) string {- v, _ := m[key].(string)- return v-}// setTitleTokens populates ChunkDoc.title_tks. Python's page row has no// title_sm_tks, and the Infinity writer folds title_kwd / title_sm_tks into@@ -698,34 +694,6 @@ func setTitleTokens(doc *schema.ChunkDoc, title string, tok tokenizer.Tokenizer)}}-// metaInt reads an int-valued Product.Meta key (tolerant of float64 from JSON).-func metaInt(m map[string]any, key string) (int, bool) {- switch v := m[key].(type) {- case int:- return v, true- case float64:- return int(v), true- }- return 0, false-}--// metaStringSlice reads a []string Product.Meta key (tolerant of []any).-func metaStringSlice(m map[string]any, key string) []string {- switch v := m[key].(type) {- case []string:- return v- case []any:- out := make([]string, 0, len(v))- for _, e := range v {- if s, ok := e.(string); ok && s != "" {- out = append(out, s)- }- }- return out- }- return nil-}-// mergeChunks returns the canonical chunker-shaped output: the upstream input// chunks followed by the freshly compiled chunk docs, all under the "chunks"// key with output_format "chunks". This makes KnowledgeCompiler's output schemadiff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.goindex f9ef23bbed..d9bd97ec30 100644--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go@@ -55,11 +55,11 @@ func runBatches(ctx context.Context, jobs []func() error) error {// Run executes the mindmap variant.func Run(ctx context.Context, deps common.Deps, param common.Param, inputs common.Inputs) (common.Outputs, error) {- docID := firstNonEmpty(inputs.DocID, deps.DatasetID)+ docID := common.FirstNonEmpty(inputs.DocID, deps.DatasetID)if docID == "" {docID = "unknown"}- llmID := firstNonEmpty(param.LLMID, inputs.LLMID)+ llmID := common.FirstNonEmpty(param.LLMID, inputs.LLMID)tenantID := deps.TenantIDif deps.Chat == nil {@@ -268,7 +268,7 @@ func mindmapSourceChunkIDs(chunks []common.Chunk) []string {seen := make(map[string]struct{}, len(chunks))ids := make([]string, 0, len(chunks))for i, chunk := range chunks {- if strings.TrimSpace(firstNonEmpty(chunk.Text, chunk.Content)) == "" {+ if strings.TrimSpace(common.FirstNonEmpty(chunk.Text, chunk.Content)) == "" {continue}id := strings.TrimSpace(chunk.ID)@@ -425,7 +425,7 @@ func payloadJSON(payload map[string]any) string {func chunkTexts(chunks []common.Chunk) []string {var out []stringfor _, c := range chunks {- t := firstNonEmpty(c.Text, c.Content)+ t := common.FirstNonEmpty(c.Text, c.Content)if strings.TrimSpace(t) == "" {continue}@@ -433,12 +433,3 @@ func chunkTexts(chunks []common.Chunk) []string {}return out}--func firstNonEmpty(vals ...string) string {- for _, v := range vals {- if v != "" {- return v- }- }- return ""-}diff --git a/internal/ingestion/component/knowledge_compiler/structure/chain.go b/internal/ingestion/component/knowledge_compiler/structure/chain.goindex c8d19babb7..5b4fed8aa1 100644--- a/internal/ingestion/component/knowledge_compiler/structure/chain.go+++ b/internal/ingestion/component/knowledge_compiler/structure/chain.go@@ -226,7 +226,7 @@ func chainGatherChunkText(badRows []common.Product, chunksByID map[string]stringseen := map[string]bool{}var out [][2]stringfor _, row := range badRows {- for _, cid := range metaStrings(row.Meta, "source_chunk_ids") {+ for _, cid := range common.MetaStringSlice(row.Meta, "source_chunk_ids") {if cid == "" || seen[cid] {continue}diff --git a/internal/ingestion/component/knowledge_compiler/structure/graph.go b/internal/ingestion/component/knowledge_compiler/structure/graph.goindex 460ef77da7..4e45cae5f4 100644--- a/internal/ingestion/component/knowledge_compiler/structure/graph.go+++ b/internal/ingestion/component/knowledge_compiler/structure/graph.go@@ -147,7 +147,7 @@ func RebuildStructureGraph(products []common.Product, compileType string) map[strelations = append(relations, rel)}case "entity":- if ent := graphEntity(payload, metaStrings(p.Meta, "source_chunk_ids")); ent != nil {+ if ent := graphEntity(payload, common.MetaStringSlice(p.Meta, "source_chunk_ids")); ent != nil {entities = append(entities, ent)}}diff --git a/internal/ingestion/component/knowledge_compiler/structure/merge.go b/internal/ingestion/component/knowledge_compiler/structure/merge.goindex 19eea60e82..e3a71f238c 100644--- a/internal/ingestion/component/knowledge_compiler/structure/merge.go+++ b/internal/ingestion/component/knowledge_compiler/structure/merge.go@@ -286,7 +286,7 @@ func (d *LLMMergeDecider) BuildReplacement(ctx context.Context, existing, incomi}merged = applyMergeInvariants(existing, merged)- chunkIDs := unionOrdered(metaStrings(existing.Meta, "source_chunk_ids"), metaStrings(incoming.Meta, "source_chunk_ids"))+ chunkIDs := unionOrdered(common.MetaStringSlice(existing.Meta, "source_chunk_ids"), common.MetaStringSlice(incoming.Meta, "source_chunk_ids"))texts := []string{payloadDescription(merged)}vecs, err := d.Embed.Encode(ctx, texts)if err != nil {@@ -662,23 +662,6 @@ func entityNameValue(p common.Product) string {return strings.TrimSpace(name)}-// metaStrings reads a []string meta value (tolerating []any).-func metaStrings(meta map[string]any, key string) []string {- switch v := meta[key].(type) {- case []string:- return v- case []any:- out := make([]string, 0, len(v))- for _, e := range v {- if s, ok := e.(string); ok {- out = append(out, s)- }- }- return out- }- return nil-}-// unionOrdered mirrors _common.union_ordered: concatenated, deduped,// first-seen order preserved.func unionOrdered(lists ...[]string) []string {@@ -747,7 +730,7 @@ func MergeIntoStore(ctx context.Context, store *common.MemStore, decider MergeDestats.survivorID = droppedExisting.ID// Fold the incoming source_chunk_ids into the surviving entry.if dropped && droppedExisting.ID != "" {- if ids := metaStrings(row.Meta, "source_chunk_ids"); len(ids) > 0 {+ if ids := common.MetaStringSlice(row.Meta, "source_chunk_ids"); len(ids) > 0 {store.MergeSourceChunkIDs(droppedExisting.ID, ids)}}@@ -830,7 +813,7 @@ func (g *GroupedDeduper) Add(ctx context.Context, row common.Product) error {}})if err == nil && action == common.KeepDrop {- store.MergeSourceChunkIDs(candidateID, metaStrings(row.Meta, "source_chunk_ids"))+ store.MergeSourceChunkIDs(candidateID, common.MetaStringSlice(row.Meta, "source_chunk_ids"))}} else {var stats MergeStatsdiff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.goindex 8d92f82127..4d5f679e54 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go@@ -491,7 +491,7 @@ func TestStructureRunGraphKind(t *testing.T) {if betaProduct == nil {t.Fatal("no Beta entity product")}- ids := metaStrings(betaProduct.Meta, "source_chunk_ids")+ ids := common.MetaStringSlice(betaProduct.Meta, "source_chunk_ids")if len(ids) != 2 {t.Fatalf("Beta provenance = %v, want {c1,c2} unioned through the merge", ids)}@@ -636,7 +636,7 @@ func TestLLMMergeDeciderContracts(t *testing.T) {if got := entityName(parsePayload(merged.Content)); got != "Alpha" {t.Errorf("merged canonical name = %q, want Alpha", got)}- if ids := metaStrings(merged.Meta, "source_chunk_ids"); len(ids) != 2 {+ if ids := common.MetaStringSlice(merged.Meta, "source_chunk_ids"); len(ids) != 2 {t.Errorf("merged provenance = %v, want {c1,c2}", ids)}if aliases := d.Aliases(); aliases["Al"] != "Alpha" {@@ -857,7 +857,7 @@ func TestGroupedDeduperMergesSameNameBeforeVectorSimilarity(t *testing.T) {if payload["type"] != "component" {t.Fatalf("specific type should replace other: %v", payload)}- if ids := metaStrings(got[0].Meta, "source_chunk_ids"); len(ids) != 2 {+ if ids := common.MetaStringSlice(got[0].Meta, "source_chunk_ids"); len(ids) != 2 {t.Fatalf("source chunks were not merged: %v", ids)}if chat.mergeCalls != 0 {@@ -889,7 +889,7 @@ func (similarityMergeDecider) Decide(_ context.Context, existing, incoming commofor key, value := range existing.Meta {replacement.Meta[key] = value}- replacement.Meta["source_chunk_ids"] = unionOrdered(metaStrings(existing.Meta, "source_chunk_ids"), metaStrings(incoming.Meta, "source_chunk_ids"))+ replacement.Meta["source_chunk_ids"] = unionOrdered(common.MetaStringSlice(existing.Meta, "source_chunk_ids"), common.MetaStringSlice(incoming.Meta, "source_chunk_ids"))return DecisionMerge, replacement, nil}@@ -909,7 +909,7 @@ func TestGroupedDeduperTracksNameAfterSemanticMerge(t *testing.T) {if len(got) != 1 {t.Fatalf("semantic alias must remain addressable by its own name: %+v", got)}- if ids := metaStrings(got[0].Meta, "source_chunk_ids"); len(ids) != 3 {+ if ids := common.MetaStringSlice(got[0].Meta, "source_chunk_ids"); len(ids) != 3 {t.Fatalf("semantic and exact-name merges lost provenance: %v", ids)}}diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph.go b/internal/ingestion/component/knowledge_compiler/tree/graph.goindex 9aafe2c3f2..5c3dfbbbd0 100644--- a/internal/ingestion/component/knowledge_compiler/tree/graph.go+++ b/internal/ingestion/component/knowledge_compiler/tree/graph.go@@ -169,7 +169,7 @@ func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {ent := map[string]any{"name": title,"type": "tree_node",- "description": firstNonEmpty(node.description, title),+ "description": common.FirstNonEmpty(node.description, title),"mention_count": 1,}if len(node.sourceChunkIDs) > 0 {diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.goindex ad25e506ed..ca3526361a 100644--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go@@ -35,11 +35,11 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoif deps.Chat == nil {return common.Outputs{}, fmt.Errorf("tree: chat model required")}- docID := firstNonEmpty(inputs.DocID, deps.DatasetID)+ docID := common.FirstNonEmpty(inputs.DocID, deps.DatasetID)if docID == "" {docID = "unknown"}- llmID := firstNonEmpty(param.LLMID, inputs.LLMID)+ llmID := common.FirstNonEmpty(param.LLMID, inputs.LLMID)tenantID := deps.TenantIDtreeOrder := resolveTreeOrder(param)@@ -544,7 +544,7 @@ func allIndices(n int) []int {// chunkTextOf returns the text of a chunk, preferring Text over Content.func chunkTextOf(c common.Chunk) string {- return firstNonEmpty(c.Text, c.Content)+ return common.FirstNonEmpty(c.Text, c.Content)}// chunkHasText reports whether a chunk carries non-blank text. Empty chunks@@ -644,15 +644,6 @@ func collectIDs(chunkIDs []string, pointIdxs []int) []string {return out}-func firstNonEmpty(vals ...string) string {- for _, v := range vals {- if v != "" {- return v- }- }- return ""-}-func toFloat64Matrix(vecs [][]float32) [][]float64 {out := make([][]float64, len(vecs))for i, v := range vecs {diff --git a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go b/internal/ingestion/component/knowledge_compiler/wiki/prompt.goindex a42f7f77c1..a5ba92156d 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go@@ -257,7 +257,7 @@ func formatWikiChunkBatch(docID string, batch []common.Chunk) (string, []string)id = fmt.Sprintf("chunk-%d", i+1)}ids = append(ids, id)- text := firstNonEmpty(c.Text, c.Content)+ text := common.FirstNonEmpty(c.Text, c.Content)if strings.TrimSpace(text) == "" {continue}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex 64cc98ab15..f9bd3e393f 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -216,7 +216,7 @@ const (// Run executes the wiki variant.func Run(ctx context.Context, deps common.Deps, param common.Param, inputs common.Inputs) (common.Outputs, error) {docID := inputs.DocID- llmID := firstNonEmpty(inputs.LLMID, param.LLMID)+ llmID := common.FirstNonEmpty(inputs.LLMID, param.LLMID)if docID == "" {docID = "unknown"}@@ -1010,16 +1010,16 @@ func (p *wikiPipeline) runRefinePage(existingSection := ""existingRaw := ""if existing != nil {- existingRaw = firstNonEmpty(existing.ContentMDRaw, existing.ContentMD)+ existingRaw = common.FirstNonEmpty(existing.ContentMDRaw, existing.ContentMD)if strings.TrimSpace(existingRaw) != "" {existingSection = "## Existing page content (UPDATE — integrate new evidence into this)\n\n" + existingRaw + "\n"}}user := renderWikiTemplate(wikiRefineWriterUserTemplate, map[string]string{- "action": firstNonEmpty(planItem.Action, "CREATE"),+ "action": common.FirstNonEmpty(planItem.Action, "CREATE"),"slug": planItem.Slug,- "title": firstNonEmpty(planItem.Title, planItem.Slug),- "page_type": firstNonEmpty(planItem.PageType, "concept"),+ "title": common.FirstNonEmpty(planItem.Title, planItem.Slug),+ "page_type": common.FirstNonEmpty(planItem.PageType, "concept"),"all_plan_slugs": strings.Join(available, "\n"),"related_kb_pages": formatWikiRelatedKB(planItem.RelatedKB, pageTitles),"existing_section": existingSection,@@ -1038,9 +1038,9 @@ func (p *wikiPipeline) runRefinePage(if resp == nil {return wikiPageResult{}, fmt.Errorf("knowledge_compiler: wiki refine returned no response")}- contentRaw := strings.TrimSpace(firstNonEmpty(resp.Content))+ contentRaw := strings.TrimSpace(common.FirstNonEmpty(resp.Content))if contentRaw == "" {- contentRaw = "# " + firstNonEmpty(planItem.Title, planItem.Slug) + "\n\n(Page generation produced no content.)"+ contentRaw = "# " + common.FirstNonEmpty(planItem.Title, planItem.Slug) + "\n\n(Page generation produced no content.)"}if strings.TrimSpace(existingRaw) != "" {contentRaw, err = p.mergeWikiPageContent(existingRaw, contentRaw, planItem.Slug)@@ -1048,7 +1048,7 @@ func (p *wikiPipeline) runRefinePage(return wikiPageResult{}, err}}- contentRendered, outlinks := transformWikiLinks(contentRaw, firstNonEmpty(p.datasetID, p.docID), pageTitles, slugToPageType)+ contentRendered, outlinks := transformWikiLinks(contentRaw, common.FirstNonEmpty(p.datasetID, p.docID), pageTitles, slugToPageType)// wiki_incremental port (O2): deterministically guarantee a "See also"// cross-link section from RelatedKB. transformWikiLinks/resolveSlug drop// links whose target cannot be resolved to a known slug, so an LLM-authored@@ -1058,13 +1058,13 @@ func (p *wikiPipeline) runRefinePage(// full-slug is linked once, only if not already present in Outlinks.contentRendered, outlinks = appendWikiSeeAlso(contentRendered, outlinks, planItem.RelatedKB, p.datasetID, pageTitles, slugToPageType)sourceDocIDs := collectWikiSourceDocIDs(p.inputs.Chunks, sourceChunkIDs, p.docID)- topic := firstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)+ topic := common.FirstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)return wikiPageResult{Slug: planItem.Slug,- Title: firstNonEmpty(planItem.Title, planItem.Slug),- PageType: firstNonEmpty(planItem.PageType, "concept"),+ Title: common.FirstNonEmpty(planItem.Title, planItem.Slug),+ PageType: common.FirstNonEmpty(planItem.PageType, "concept"),Topic: topic,- Action: firstNonEmpty(planItem.Action, "CREATE"),+ Action: common.FirstNonEmpty(planItem.Action, "CREATE"),EntityNames: appcommon.DeduplicateTrimmed(planItem.EntityNames),RelatedKBPages: appcommon.DeduplicateTrimmed(planItem.RelatedKB),ContentRaw: contentRaw,@@ -1259,13 +1259,13 @@ func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {continue}page.Action = "UPDATE"- page.Slug = firstNonEmpty(existing.Slug, page.Slug)- page.Title = firstNonEmpty(existing.Title, page.Title)- page.PageType = firstNonEmpty(existing.PageType, page.PageType)+ page.Slug = common.FirstNonEmpty(existing.Slug, page.Slug)+ page.Title = common.FirstNonEmpty(existing.Title, page.Title)+ page.PageType = common.FirstNonEmpty(existing.PageType, page.PageType)if existing.RoutedTopic != "" {page.Topic = common.NormalizeWikiTopicPath(existing.RoutedTopic)} else {- page.Topic = common.NormalizeWikiTopicPath(firstNonEmpty(existing.Topic, page.Topic))+ page.Topic = common.NormalizeWikiTopicPath(common.FirstNonEmpty(existing.Topic, page.Topic))}page.RelatedKB = mergeStrings(page.RelatedKB, existing.RelatedKBPages)if len(existing.RoutedEntityNames) > 0 {@@ -1549,7 +1549,7 @@ func (p *wikiPipeline) mergeWikiPageContent(existingMD, newMD, slug string) (strif resp == nil {return fallback, nil}- merged := strings.TrimSpace(firstNonEmpty(resp.Content))+ merged := strings.TrimSpace(common.FirstNonEmpty(resp.Content))if merged == "" {return fallback, nil}@@ -1570,7 +1570,7 @@ func conservativeMergeWikiMarkdown(existingMD, newMD string) string {if strings.Contains(newMD, existingMD) {return strings.TrimSpace(newMD)}- title := firstNonEmpty(wikiMarkdownTitle(existingMD), wikiMarkdownTitle(newMD))+ title := common.FirstNonEmpty(wikiMarkdownTitle(existingMD), wikiMarkdownTitle(newMD))blocks := make([]string, 0, 8)seen := map[string]bool{}appendBlocks := func(body string) {@@ -2117,7 +2117,7 @@ func buildWikiFallbackPages(reduced wikiExtract) []wikiPlanPage {if len(out) == 0 {for _, t := range reduced.Topics {title := common.WikiTopicLeaf(t.Path)- lead := firstNonEmpty(t.Description, title)+ lead := common.FirstNonEmpty(t.Description, title)appendPage(wikiPlanPage{Action: "CREATE",Slug: "topic/" + slugify(title),@@ -2489,7 +2489,7 @@ func buildSourceContext(chunks []common.Chunk, sourceChunkIDs []string) string {if len(sourceChunkIDs) == 0 {var b strings.Builderfor _, ch := range chunks {- text := firstNonEmpty(ch.Text, ch.Content)+ text := common.FirstNonEmpty(ch.Text, ch.Content)if strings.TrimSpace(text) == "" {continue}@@ -2504,7 +2504,7 @@ func buildSourceContext(chunks []common.Chunk, sourceChunkIDs []string) string {if id == "" {id = fmt.Sprintf("chunk-%d", i+1)}- lookup[id] = firstNonEmpty(ch.Text, ch.Content)+ lookup[id] = common.FirstNonEmpty(ch.Text, ch.Content)}var b strings.Builderfor _, id := range sourceChunkIDs {@@ -2598,7 +2598,7 @@ func assembleWikiPageEvidence(planItem wikiPlanPage, claims []wikiClaim, entityBevidence = append(evidence, wikiEvidenceItem{Statement: claim.Statement,Subject: claim.Subject,- Confidence: firstNonEmpty(claim.Confidence, "explicit"),+ Confidence: common.FirstNonEmpty(claim.Confidence, "explicit"),ChunkIDs: appcommon.DeduplicateTrimmed(claim.SourceChunkIDs),})}@@ -2679,7 +2679,7 @@ func formatWikiEvidenceBlocks(evidence []wikiEvidenceItem) string {if ev.Synthetic {continue}- confidence := strings.ToUpper(firstNonEmpty(ev.Confidence, "explicit"))+ confidence := strings.ToUpper(common.FirstNonEmpty(ev.Confidence, "explicit"))lines = append(lines, fmt.Sprintf("%d. [%s] %s\n %s", len(lines)+1, confidence, ev.Subject, ev.Statement))}if len(lines) == 0 {@@ -3064,15 +3064,6 @@ func prefixWithDash(items []string) []string {return out}-func firstNonEmpty(vals ...string) string {- for _, v := range vals {- if s := strings.TrimSpace(v); s != "" {- return s- }- }- return ""-}-func mergeStrings(a, b []string) []string {return appcommon.DeduplicateTrimmed(append(append([]string{}, a...), b...))}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.goindex aeeadff05b..8e13bf0a80 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go@@ -70,7 +70,7 @@ func (p *wikiPipeline) runVersionedMap() error {if strings.TrimSpace(chunk.ID) == "" {continue}- contentHash := wikiMapHash(firstNonEmpty(chunk.Text, chunk.Content))+ contentHash := wikiMapHash(common.FirstNonEmpty(chunk.Text, chunk.Content))key := wikiMapHash(strings.Join([]string{p.tenantID,p.datasetID,diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex 7bb9b79e2a..7fc30d88d2 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -368,7 +368,7 @@ func describeImage(// (the key the Python parser's image branch used) stays readable for canvases// saved before the move. Do NOT read setup["prompt"] here — that key is for// the video family, not image.- if v := firstNonEmpty(vision.systemPrompt, getStringOr(setup, "system_prompt", "")); v != "" {+ if v := common.FirstNonEmpty(vision.systemPrompt, getStringOr(setup, "system_prompt", "")); v != "" {prompt = v}messages := []modelModule.Message{{diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex a378345f6d..413ea4552a 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -621,7 +621,7 @@ func dispatchMinerUPDF(// That default used to arrive implicitly through defaultSetups() - which// also fed captions, so a knowledge base that never chose a language got// Chinese captions (issue #20727). Keeping it here confines it to the engine.- lang := firstNonEmpty(getStringOr(setup, "mineru_lang", ""), getStringOr(setup, "lang", ""))+ lang := common.FirstNonEmpty(getStringOr(setup, "mineru_lang", ""), getStringOr(setup, "lang", ""))if lang == "" {lang = minerUDefaultLanguage}diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex 1c0bb2b162..bf3c5ba428 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -1183,18 +1183,6 @@ func getStringOr(m map[string]any, key, def string) string {return def}-// firstNonEmpty returns the first non-empty value, or "" when all are empty.-// Used to walk a settings precedence chain (global choice → legacy per-family-// choice).-func firstNonEmpty(values ...string) string {- for _, v := range values {- if v != "" {- return v- }- }- return ""-}-// cloneInputs returns a shallow copy of m with room for one extra key.// Used to inject the Globals-resolved `name` into the decode input without// mutating the caller's input snapshot.diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 4f4797caa6..9e6e3d4a98 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -555,7 +555,7 @@ func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Productif cand.Variant != kccommon.VariantWiki {continue // tree/structure products never enter the wiki merge}- if metaString(cand.Meta, "kind") != "page" {+ if kccommon.MetaString(cand.Meta, "kind") != "page" {continue}filtered = append(filtered, cand)@@ -1256,7 +1256,7 @@ func (c *Consumer) mergeEntityModeCandidates(ctx context.Context, tenant, kb strvar entityCandidates []kccommon.Productvar topic []kccommon.Productfor _, candidate := range candidates {- pageType := strings.ToLower(strings.TrimSpace(metaString(candidate.Meta, "page_type")))+ pageType := strings.ToLower(strings.TrimSpace(kccommon.MetaString(candidate.Meta, "page_type")))if pageType != "entity" && pageType != "concept" {topic = append(topic, candidate)continue@@ -1307,7 +1307,7 @@ func (c *Consumer) mergeWikiProductsBySlug(ctx context.Context, tenant, kb strinexists = truecontinue}- if strings.EqualFold(strings.TrimSpace(metaString(current.Meta, "page_type")), "topic") {+ if strings.EqualFold(strings.TrimSpace(kccommon.MetaString(current.Meta, "page_type")), "topic") {current = mergeTopicPage(current, candidate)} else {current = wikiEntityMerge(current, candidate)@@ -1358,11 +1358,11 @@ func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommonjobs = append(jobs, func() error {page := pages[idx]prompt := fmt.Sprintf("PAGE TYPE: %s\nTITLE: %s\nTOPIC: %s\nENTITIES: %s\nSUMMARY: %s\n\nEVIDENCE MARKDOWN:\n%s",- metaString(page.Meta, "page_type"),- metaString(page.Meta, "title"),- metaString(page.Meta, "topic"),- strings.Join(metaStringSlice(page.Meta, "entity_names"), ", "),- metaString(page.Meta, "summary"),+ kccommon.MetaString(page.Meta, "page_type"),+ kccommon.MetaString(page.Meta, "title"),+ kccommon.MetaString(page.Meta, "topic"),+ strings.Join(kccommon.MetaStringSlice(page.Meta, "entity_names"), ", "),+ kccommon.MetaString(page.Meta, "summary"),page.Content)response, err := deps.Chat.Chat(ctx, kccommon.ChatRequest{LLMID: defaultLLMID,@@ -1371,14 +1371,14 @@ func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommonTemperature: floatPtr(0.1),})if err != nil {- return fmt.Errorf("rewrite Wiki page %q: %w", metaString(page.Meta, "slug"), err)+ return fmt.Errorf("rewrite Wiki page %q: %w", kccommon.MetaString(page.Meta, "slug"), err)}if response == nil {- return fmt.Errorf("rewrite Wiki page %q returned no response", metaString(page.Meta, "slug"))+ return fmt.Errorf("rewrite Wiki page %q returned no response", kccommon.MetaString(page.Meta, "slug"))}content := strings.TrimSpace(response.Content)if content == "" {- return fmt.Errorf("rewrite Wiki page %q returned empty content", metaString(page.Meta, "slug"))+ return fmt.Errorf("rewrite Wiki page %q returned empty content", kccommon.MetaString(page.Meta, "slug"))}pages[idx].Content = contentreturn nil@@ -1390,11 +1390,11 @@ func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommonfunc floatPtr(value float64) *float64 { return &value }func wikiPageMergeKey(product kccommon.Product) string {- slug := strings.TrimSpace(metaString(product.Meta, "slug"))+ slug := strings.TrimSpace(kccommon.MetaString(product.Meta, "slug"))if slug == "" {return ""}- pageType := strings.ToLower(strings.TrimSpace(metaString(product.Meta, "page_type")))+ pageType := strings.ToLower(strings.TrimSpace(kccommon.MetaString(product.Meta, "page_type")))return pageType + "\x00" + strings.ToLower(strings.Join(strings.Fields(slug), " "))}@@ -1643,14 +1643,14 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,}// Dataset rows retain the producing compilation kind. Extraction shape// is separate metadata and does not determine the stored kind.- ckwd := enginetypes.CanonicalCompilationKind(kccommon.FirstNonEmpty(p.Kind, metaString(p.Meta, "compile_kwd")))+ ckwd := enginetypes.CanonicalCompilationKind(kccommon.FirstNonEmpty(p.Kind, kccommon.MetaString(p.Meta, "compile_kwd")))if ckwd == "" {ckwd = compileKwdForVariant(p.Variant)}- kind := metaString(p.Meta, "kind")+ kind := kccommon.MetaString(p.Meta, "kind")template := structureTemplateIdentity(p.TemplateID, p.Kind)- from := metaString(p.Meta, "from")- to := metaString(p.Meta, "to")+ from := kccommon.MetaString(p.Meta, "from")+ to := kccommon.MetaString(p.Meta, "to")if kind == "relation" || (from != "" && to != "") {from = strings.TrimSpace(from)to = strings.TrimSpace(to)@@ -1660,9 +1660,9 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,if from == "" || to == "" {continue}- relType := metaString(p.Meta, "relation_type")+ relType := kccommon.MetaString(p.Meta, "relation_type")if relType == "" {- relType = metaString(p.Meta, "type")+ relType = kccommon.MetaString(p.Meta, "type")}if relType == "" {relType = "related"@@ -1677,10 +1677,10 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,continue}// entity bucket- name := metaString(p.Meta, "name")- typ := metaString(p.Meta, "entity_type")+ name := kccommon.MetaString(p.Meta, "name")+ typ := kccommon.MetaString(p.Meta, "entity_type")if typ == "" {- typ = metaString(p.Meta, "type")+ typ = kccommon.MetaString(p.Meta, "type")}if name == "" {continue@@ -1773,19 +1773,19 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {// name/type, while their dataset row still needs searchable text.// Relations likewise use the endpoint names and relation type when the// source payload has no description.- from := metaString(p.Meta, "from")- to := metaString(p.Meta, "to")- if kind := metaString(p.Meta, "kind"); kind == "relation" || (from != "" && to != "") {- relType := metaString(p.Meta, "relation_type")+ from := kccommon.MetaString(p.Meta, "from")+ to := kccommon.MetaString(p.Meta, "to")+ if kind := kccommon.MetaString(p.Meta, "kind"); kind == "relation" || (from != "" && to != "") {+ relType := kccommon.MetaString(p.Meta, "relation_type")if relType == "" {- relType = metaString(p.Meta, "type")+ relType = kccommon.MetaString(p.Meta, "type")}if relType == "" {relType = "related"}desc = strings.TrimSpace(strings.Join([]string{from, relType, to}, " "))} else {- desc = strings.TrimSpace(metaString(p.Meta, "name"))+ desc = strings.TrimSpace(kccommon.MetaString(p.Meta, "name"))}}if desc != "" {@@ -1795,13 +1795,13 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {b.Description += desc}b.SourceDocIDs = common.AppendUnique(b.SourceDocIDs, []string{p.DocID})- b.SourceChunkIDs = common.AppendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))+ b.SourceChunkIDs = common.AppendUnique(b.SourceChunkIDs, kccommon.MetaStringSlice(p.Meta, "source_chunk_ids"))// mention_count_int: sum the per-entity mention counts (Python// _struct_merge_graph_entities sums mention_count across the merged entities).// Each entity product carries mention_count ≥ 1 (structure compile.go L231);// mindmap entities carry none, so they contribute a default of 0 and are// simply not stamped (mention_count_int is omitted when 0).- if mc, ok := metaInt(p.Meta, "mention_count"); ok && mc > 0 {+ if mc, ok := kccommon.MetaInt(p.Meta, "mention_count"); ok && mc > 0 {b.MentionCount += int(mc)}if len(p.Vector) > 0 {diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 20f523c69f..18a762b31a 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -265,7 +265,7 @@ func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb stringbreak}for _, row := range res.Chunks {- if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && metaString(product.Meta, "kind") == "page" {+ if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && kccommon.MetaString(product.Meta, "kind") == "page" {pages = append(pages, product)}}@@ -315,7 +315,7 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,break}for _, row := range res.Chunks {- if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && metaString(product.Meta, "kind") == "page" {+ if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && kccommon.MetaString(product.Meta, "kind") == "page" {pages = append(pages, product)}}@@ -395,12 +395,12 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommometa["to"] = vmeta["kind"] = "relation"}- if metaString(meta, "kind") == "relation" {+ if kccommon.MetaString(meta, "kind") == "relation" {if v, ok := payload["type"].(string); ok && v != "" {meta["relation_type"] = v}}- if v, ok := metaInt(c, "mention_count_int"); ok {+ if v, ok := kccommon.MetaInt(c, "mention_count_int"); ok {meta["mention_count"] = v}if v, ok := c["slug_kwd"].(string); ok && v != "" {@@ -422,20 +422,20 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommoif v, ok := c["summary_with_weight"].(string); ok && v != "" {meta["summary"] = v}- if v := metaStringSlice(c, "entity_names_kwd"); len(v) > 0 {+ if v := kccommon.MetaStringSlice(c, "entity_names_kwd"); len(v) > 0 {meta["entity_names"] = v}- if v := metaStringSlice(c, "related_kb_pages_kwd"); len(v) > 0 {+ if v := kccommon.MetaStringSlice(c, "related_kb_pages_kwd"); len(v) > 0 {meta["related_kb_pages"] = v}- if v := metaStringSlice(c, "outlinks_kwd"); len(v) > 0 {+ if v := kccommon.MetaStringSlice(c, "outlinks_kwd"); len(v) > 0 {meta["outlinks"] = v}// Section depth comes from depth_int. Only the wiki variant consumes// Meta["section_level"] (applyVariantColumns); tree rows also stamp depth_int,// but they keep their depth in raptor_layer_int.if expect == kccommon.VariantWiki {- if v, ok := metaInt(c, "depth_int"); ok {+ if v, ok := kccommon.MetaInt(c, "depth_int"); ok {meta["section_level"] = v}}@@ -478,10 +478,10 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommoif v, ok := c["create_time"].(string); ok && v != "" {meta["created_at"] = v}- if v := metaStringSlice(c, "source_chunk_ids"); len(v) > 0 {+ if v := kccommon.MetaStringSlice(c, "source_chunk_ids"); len(v) > 0 {meta["source_chunk_ids"] = v}- if v := metaStringSlice(c, "source_doc_ids"); len(v) > 0 {+ if v := kccommon.MetaStringSlice(c, "source_doc_ids"); len(v) > 0 {meta["source_doc_ids"] = v}@@ -494,7 +494,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo// more than one the first is used (multi-template rows are a config error// surfaced elsewhere).templateID := ""- if ids := metaStringSlice(c, "compilation_template_ids"); len(ids) > 0 {+ if ids := kccommon.MetaStringSlice(c, "compilation_template_ids"); len(ids) > 0 {templateID = ids[0]}return kccommon.Product{diff --git a/internal/ingestion/knowledge_compile/topic_merge.go b/internal/ingestion/knowledge_compile/topic_merge.goindex 1638a2be3d..0ef90dceab 100644--- a/internal/ingestion/knowledge_compile/topic_merge.go+++ b/internal/ingestion/knowledge_compile/topic_merge.go@@ -38,26 +38,26 @@ func (x *llmDeduper) RouteTopic(ctx context.Context, incoming, existing kccommon}func isTopicPage(product kccommon.Product) bool {- return strings.EqualFold(strings.TrimSpace(metaString(product.Meta, "page_type")), "topic")+ return strings.EqualFold(strings.TrimSpace(kccommon.MetaString(product.Meta, "page_type")), "topic")}func productTopic(product kccommon.Product) string {- return kccommon.NormalizeWikiTopicPath(metaString(product.Meta, "topic"))+ return kccommon.NormalizeWikiTopicPath(kccommon.MetaString(product.Meta, "topic"))}func productTitle(product kccommon.Product) string {- return strings.TrimSpace(metaString(product.Meta, "title"))+ return strings.TrimSpace(kccommon.MetaString(product.Meta, "title"))}func productSummary(product kccommon.Product) string {- if summary := strings.TrimSpace(metaString(product.Meta, "summary")); summary != "" {+ if summary := strings.TrimSpace(kccommon.MetaString(product.Meta, "summary")); summary != "" {return summary}return strings.TrimSpace(product.Content)}func productEntities(product kccommon.Product) []string {- return metaStringSlice(product.Meta, "entity_names")+ return kccommon.MetaStringSlice(product.Meta, "entity_names")}func firstTopicString(values ...string) string {@@ -152,11 +152,11 @@ func mergeTopicProducts(tenant, kb string, products []kccommon.Product) ([]kccomfunc mergeTopicPage(existing, incoming kccommon.Product) kccommon.Product {merged := wikiEntityMerge(existing, incoming)merged.Meta = copyMeta(merged.Meta)- merged.Meta["slug"] = metaString(existing.Meta, "slug")+ merged.Meta["slug"] = kccommon.MetaString(existing.Meta, "slug")merged.Meta["page_type"] = "topic"- topic := kccommon.NormalizeWikiTopicPath(firstTopicString(metaString(incoming.Meta, "topic"), metaString(existing.Meta, "topic")))+ topic := kccommon.NormalizeWikiTopicPath(firstTopicString(kccommon.MetaString(incoming.Meta, "topic"), kccommon.MetaString(existing.Meta, "topic")))merged.Meta["topic"] = topic- merged.Meta["title"] = firstTopicString(metaString(existing.Meta, "title"), metaString(incoming.Meta, "title"), topic)- merged.Meta["entity_names"] = unionStrs(metaStringSlice(existing.Meta, "entity_names"), metaStringSlice(incoming.Meta, "entity_names"))+ merged.Meta["title"] = firstTopicString(kccommon.MetaString(existing.Meta, "title"), kccommon.MetaString(incoming.Meta, "title"), topic)+ merged.Meta["entity_names"] = unionStrs(kccommon.MetaStringSlice(existing.Meta, "entity_names"), kccommon.MetaStringSlice(incoming.Meta, "entity_names"))return merged}diff --git a/internal/ingestion/knowledge_compile/topic_merge_test.go b/internal/ingestion/knowledge_compile/topic_merge_test.goindex a886f5a1dc..1e2264a71c 100644--- a/internal/ingestion/knowledge_compile/topic_merge_test.go+++ b/internal/ingestion/knowledge_compile/topic_merge_test.go@@ -36,13 +36,13 @@ func TestMergeTopicProductsPreservesEvidenceAndStableTopicIdentity(t *testing.T)if !strings.Contains(merged[0].Content, "first") || !strings.Contains(merged[0].Content, "second") {t.Fatalf("merged content lost evidence: %q", merged[0].Content)}- if got := metaString(merged[0].Meta, "slug"); got != "topic/history-a" {+ if got := kccommon.MetaString(merged[0].Meta, "slug"); got != "topic/history-a" {t.Fatalf("topic slug = %q, want existing slug", got)}- if got := metaString(merged[0].Meta, "title"); got != "History" {+ if got := kccommon.MetaString(merged[0].Meta, "title"); got != "History" {t.Fatalf("topic title = %q, want existing title", got)}- if got := metaStringSlice(merged[0].Meta, "entity_names"); len(got) != 2 {+ if got := kccommon.MetaStringSlice(merged[0].Meta, "entity_names"); len(got) != 2 {t.Fatalf("entity names = %v, want both entities", got)}}diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.goindex a65104ea48..2f29a53e49 100644--- a/internal/ingestion/knowledge_compile/wiki_contribution.go+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go@@ -199,7 +199,7 @@ func (s *memoryWikiContributionStore) ClearDataset(_ context.Context, _, datasetfunc buildWikiDocumentContribution(documentID string, products []kccommon.Product) wikiDocumentContribution {pagesByKey := make(map[string]wikiContributionPage)for _, product := range products {- if product.Variant != kccommon.VariantWiki || metaString(product.Meta, "kind") != "page" {+ if product.Variant != kccommon.VariantWiki || kccommon.MetaString(product.Meta, "kind") != "page" {continue}key := wikiPageMergeKey(product)@@ -208,8 +208,8 @@ func buildWikiDocumentContribution(documentID string, products []kccommon.Produc}pagesByKey[key] = wikiContributionPage{Key: key,- Slug: strings.TrimSpace(metaString(product.Meta, "slug")),- PageType: strings.ToLower(strings.TrimSpace(metaString(product.Meta, "page_type"))),+ Slug: strings.TrimSpace(kccommon.MetaString(product.Meta, "slug")),+ PageType: strings.ToLower(strings.TrimSpace(kccommon.MetaString(product.Meta, "page_type"))),Signature: wikiContributionSignature(product),}}@@ -270,13 +270,13 @@ func wikiContributionSignature(product kccommon.Product) string {SourceChunkIDs []string}{Content: strings.TrimSpace(product.Content),- Title: strings.TrimSpace(metaString(product.Meta, "title")),- Topic: kccommon.NormalizeWikiTopicPath(metaString(product.Meta, "topic")),- Summary: strings.TrimSpace(metaString(product.Meta, "summary")),- EntityNames: common.DeduplicateSorted(metaStringSlice(product.Meta, "entity_names")),- RelatedKBPages: common.DeduplicateSorted(metaStringSlice(product.Meta, "related_kb_pages")),- Outlinks: common.DeduplicateSorted(metaStringSlice(product.Meta, "outlinks")),- SourceChunkIDs: common.DeduplicateSorted(metaStringSlice(product.Meta, "source_chunk_ids")),+ Title: strings.TrimSpace(kccommon.MetaString(product.Meta, "title")),+ Topic: kccommon.NormalizeWikiTopicPath(kccommon.MetaString(product.Meta, "topic")),+ Summary: strings.TrimSpace(kccommon.MetaString(product.Meta, "summary")),+ EntityNames: common.DeduplicateSorted(kccommon.MetaStringSlice(product.Meta, "entity_names")),+ RelatedKBPages: common.DeduplicateSorted(kccommon.MetaStringSlice(product.Meta, "related_kb_pages")),+ Outlinks: common.DeduplicateSorted(kccommon.MetaStringSlice(product.Meta, "outlinks")),+ SourceChunkIDs: common.DeduplicateSorted(kccommon.MetaStringSlice(product.Meta, "source_chunk_ids")),}payload, _ := json.Marshal(normalized)sum := sha256.Sum256(payload)@@ -352,7 +352,7 @@ func (c *Consumer) prepareWikiContributionDiff(ctx context.Context, tenantID, da}legacyProducts := make(map[string][]kccommon.Product, len(missingPrevious))for _, page := range mergedPages {- for _, docID := range metaStringSlice(page.Meta, "source_doc_ids") {+ for _, docID := range kccommon.MetaStringSlice(page.Meta, "source_doc_ids") {if _, wanted := missingPrevious[docID]; wanted {legacyProducts[docID] = append(legacyProducts[docID], page)}diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.goindex b5a76d220d..0bdfc9d48d 100644--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go@@ -21,6 +21,8 @@ import ("errors""strings""testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")// page builds a wikiPageProjection with the full "<page_type>/<slug>" identity.@@ -69,7 +71,7 @@ func TestProjectWikiGraphRowsWritesSearchContent(t *testing.T) {var payload struct {Description string `json:"description"`}- if err := json.Unmarshal([]byte(metaString(entity, "content_with_weight")), &payload); err != nil {+ if err := json.Unmarshal([]byte(kccommon.MetaString(entity, "content_with_weight")), &payload); err != nil {t.Fatalf("decode graph entity: %v", err)}if payload.Description != p.Summary {diff --git a/internal/ingestion/knowledge_compile/wiki_merge.go b/internal/ingestion/knowledge_compile/wiki_merge.goindex d6ab34ff48..8e0115df2d 100644--- a/internal/ingestion/knowledge_compile/wiki_merge.go+++ b/internal/ingestion/knowledge_compile/wiki_merge.go@@ -66,7 +66,7 @@ func selectMergedWikiTopicPath(products []kccommon.Product) string {byKey[key] = current}current.products++- docIDs := metaStringSliceAny(product.Meta, "source_doc_ids")+ docIDs := kccommon.MetaStringSlice(product.Meta, "source_doc_ids")if len(docIDs) == 0 && product.DocID != "" {docIDs = []string{product.DocID}}@@ -75,7 +75,7 @@ func selectMergedWikiTopicPath(products []kccommon.Product) string {current.docs[docID] = struct{}{}}}- for _, chunkID := range metaStringSliceAny(product.Meta, "source_chunk_ids") {+ for _, chunkID := range kccommon.MetaStringSlice(product.Meta, "source_chunk_ids") {if chunkID = strings.TrimSpace(chunkID); chunkID != "" {current.chunks[chunkID] = struct{}{}}@@ -180,8 +180,8 @@ func unionWikiProvenance(a, b map[string]any) map[string]any {out[key] = v}}- out["source_doc_ids"] = unionStrs(metaStringSliceAny(a, "source_doc_ids"), metaStringSliceAny(b, "source_doc_ids"))- out["source_chunk_ids"] = unionStrs(metaStringSliceAny(a, "source_chunk_ids"), metaStringSliceAny(b, "source_chunk_ids"))+ out["source_doc_ids"] = unionStrs(kccommon.MetaStringSlice(a, "source_doc_ids"), kccommon.MetaStringSlice(b, "source_doc_ids"))+ out["source_chunk_ids"] = unionStrs(kccommon.MetaStringSlice(a, "source_chunk_ids"), kccommon.MetaStringSlice(b, "source_chunk_ids"))if v, ok := b["entity_names"]; ok {out["entity_names"] = v}@@ -213,24 +213,6 @@ func unionStrs(a, b []string) []string {return out}-// metaStringSliceAny reads a []string from a meta map that may box the value as-// []string or []any (engine/JSON round-trip does not guarantee a single type).-func metaStringSliceAny(m map[string]any, key string) []string {- switch v := m[key].(type) {- case []string:- return v- case []any:- out := make([]string, 0, len(v))- for _, e := range v {- if s, ok := e.(string); ok {- out = append(out, s)- }- }- return out- }- return nil-}-// isWikiGroup reports whether a merge group targets the wiki variant and needs// the page-specific Markdown merge instead of the generic JSON decider.func isWikiGroup(g MergeGroup) bool {diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex f093e317f0..cd6f8d161a 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -292,9 +292,9 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, producif len(chunks) > 0 {common.Info("knowledge_compile: WriteMerged chunk sample",zap.String("kb_id", kb),- zap.String("compile_kwd", metaString(chunks[0], "compile_kwd")),+ zap.String("compile_kwd", kccommon.MetaString(chunks[0], "compile_kwd")),zap.String("variant", string(batch[0].Variant)),- zap.String("id", metaString(chunks[0], "id")),+ zap.String("id", kccommon.MetaString(chunks[0], "id")),)}_, err := eng.InsertChunks(ctx, chunks, baseName, kb, "")@@ -432,12 +432,12 @@ func (w engineWriter) recordWikiPageCommits(ctx context.Context, kb string, prod}func wikiPageCommitTargetForProduct(product kccommon.Product) (wikiPageCommitTarget, bool) {- if product.Variant != kccommon.VariantWiki || strings.ToLower(strings.TrimSpace(metaString(product.Meta, "kind"))) != "page" {+ if product.Variant != kccommon.VariantWiki || strings.ToLower(strings.TrimSpace(kccommon.MetaString(product.Meta, "kind"))) != "page" {return wikiPageCommitTarget{}, false}return wikiPageCommitTargetForRow(- metaString(product.Meta, "page_type"),- metaString(product.Meta, "slug"),+ kccommon.MetaString(product.Meta, "page_type"),+ kccommon.MetaString(product.Meta, "slug"),)}@@ -965,8 +965,8 @@ func f32ToF64Slice(v []float32) []float64 {// (input_hash_kwd), and now stamps both wall-clock audit fields. We deliberately// do not write the full PLAN JSON into the engine.func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommon.Product) map[string]interface{} {- srcDocIDs := metaStringSlice(p.Meta, "source_doc_ids")- srcChunkIDs := metaStringSlice(p.Meta, "source_chunk_ids")+ srcDocIDs := kccommon.MetaStringSlice(p.Meta, "source_doc_ids")+ srcChunkIDs := kccommon.MetaStringSlice(p.Meta, "source_chunk_ids")m := map[string]interface{}{"id": datasetLevelID(tenant, kb, p),"doc_id": kb,@@ -987,7 +987,7 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm"create_timestamp_flt": float64(now.Unix()),}if p.Variant == kccommon.VariantWiki {- m["type_kwd"] = "wiki_" + kccommon.FirstNonEmpty(metaString(p.Meta, "kind"), "page")+ m["type_kwd"] = "wiki_" + kccommon.FirstNonEmpty(kccommon.MetaString(p.Meta, "kind"), "page")}// wiki_incremental port: preserve the original creation timestamp across a// page merge. If the incoming merged product already carries@@ -1005,8 +1005,8 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm// slug_kwd follows the Python writer contract (api/db/db_models.py): it is// stored as the full "<page_type>/<slug>" form so GetWikiPage's filter// (page_type + "/" + slug) matches directly.- pageType := metaString(p.Meta, "page_type")- if slug := metaString(p.Meta, "slug"); slug != "" {+ pageType := kccommon.MetaString(p.Meta, "page_type")+ if slug := kccommon.MetaString(p.Meta, "slug"); slug != "" {// Normalize to the full "<page_type>/<slug>" form (Python writer// contract). Idempotent: slugs that already carry the prefix are kept.fullSlug := slug@@ -1015,31 +1015,31 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm}m["slug_kwd"] = fullSlug}- if v := metaString(p.Meta, "title"); v != "" {+ if v := kccommon.MetaString(p.Meta, "title"); v != "" {m["title_kwd"] = v}if pageType != "" {m["entity_type_kwd"] = pageType}- if v := kccommon.NormalizeWikiTopicPath(metaString(p.Meta, "topic")); v != "" {+ if v := kccommon.NormalizeWikiTopicPath(kccommon.MetaString(p.Meta, "topic")); v != "" {m["topic_kwd"] = v}- if v := metaString(p.Meta, "plan_group"); v != "" {+ if v := kccommon.MetaString(p.Meta, "plan_group"); v != "" {m["plan_group_kwd"] = v}- if v := metaString(p.Meta, "generation"); v != "" {+ if v := kccommon.MetaString(p.Meta, "generation"); v != "" {m["generation_kwd"] = v}- if v := metaString(p.Meta, "summary"); v != "" {+ if v := kccommon.MetaString(p.Meta, "summary"); v != "" {m["summary_with_weight"] = v}- if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {+ if v := kccommon.MetaStringSlice(p.Meta, "entity_names"); len(v) > 0 {m["entity_names_kwd"] = v}- if v := metaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {+ if v := kccommon.MetaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {m["related_kb_pages_kwd"] = v}- if v := metaStringSlice(p.Meta, "outlinks"); len(v) > 0 {+ if v := kccommon.MetaStringSlice(p.Meta, "outlinks"); len(v) > 0 {m["outlinks_kwd"] = v}// Persist the merged product's embedding under the dimension-suffixed column@@ -1123,7 +1123,7 @@ func (w engineWriter) StripMergedSources(ctx context.Context, tenant, kb string,if id == "" {continue}- src := metaStringSlice(c, "source_doc_ids")+ src := kccommon.MetaStringSlice(c, "source_doc_ids")kept := make([]string, 0, len(src))changed := falsefor _, d := range src {@@ -1204,7 +1204,7 @@ func mergedInputHash(products []kccommon.Product) string {seen := make(map[string]struct{}, 16)docs := make([]string, 0, 16)for _, p := range products {- for _, d := range metaStringSlice(p.Meta, "source_doc_ids") {+ for _, d := range kccommon.MetaStringSlice(p.Meta, "source_doc_ids") {if _, ok := seen[d]; ok {continue}@@ -1216,57 +1216,6 @@ func mergedInputHash(products []kccommon.Product) string {return hashStr(strings.Join(docs, "\n"))}-// metaString extracts a string from a map value, tolerating a missing or-// non-string entry.-func metaString(m map[string]any, key string) string {- if v, ok := m[key].(string); ok {- return v- }- return ""-}--// firstStringSlice extracts a []string from an engine row value, tolerating both-// []string and []any forms; it returns nil when the value is not a string slice.-func firstStringSlice(v any) []string {- switch s := v.(type) {- case []string:- return s- case []any:- out := make([]string, 0, len(s))- for _, x := range s {- if str, ok := x.(string); ok {- out = append(out, str)- }- }- return out- }- return nil-}--func metaStringSlice(m map[string]any, key string) []string {- return firstStringSlice(m[key])-}--// metaInt extracts an integer from a map value that may be boxed as float64-// (JSON number), int64, string, or a typed int — the engine/JSON round-trip does-// not guarantee a single numeric type.-func metaInt(m map[string]any, key string) (int64, bool) {- switch v := m[key].(type) {- case int64:- return v, true- case int:- return int64(v), true- case float64:- return int64(v), true- case string:- var n int64- if _, err := fmt.Sscanf(v, "%d", &n); err == nil {- return n, true- }- }- return 0, false-}-// metaFloat extracts a float64 from a map value that may be boxed as float64,// int64, int, or string — the engine/JSON round-trip does not guarantee a// single numeric type. Used to recover create_timestamp_flt so the reader can@@ -1288,6 +1237,24 @@ func metaFloat(m map[string]any, key string) (float64, bool) {return 0, false}+// firstStringSlice extracts a []string from an engine row value, tolerating both+// []string and []any forms; it returns nil when the value is not a string slice.+func firstStringSlice(v any) []string {+ switch s := v.(type) {+ case []string:+ return s+ case []any:+ out := make([]string, 0, len(s))+ for _, x := range s {+ if str, ok := x.(string); ok {+ out = append(out, str)+ }+ }+ return out+ }+ return nil+}+// KwdToVariant maps a stored compilation kind to its execution variant.func KwdToVariant(kwd string) (kccommon.Variant, error) {switch kwd {@@ -1477,7 +1444,7 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kbreak}for _, row := range result.Chunks {- slug := metaString(row, "slug_kwd")+ slug := kccommon.MetaString(row, "slug_kwd")if slug == "" {continue}@@ -1485,13 +1452,13 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kif page.Slug == "" {page.Slug = slugpage.PageType = types.WikiPageCategory(row)- page.Title = metaString(row, "title_kwd")- page.Summary = metaString(row, "summary_with_weight")+ page.Title = kccommon.MetaString(row, "title_kwd")+ page.Summary = kccommon.MetaString(row, "summary_with_weight")}- page.Aliases = unionStrs(page.Aliases, metaStringSlice(row, "entity_names_kwd"))- page.Outlinks = unionStrs(page.Outlinks, metaStringSlice(row, "outlinks_kwd"))- page.SourceDocIDs = unionStrs(page.SourceDocIDs, metaStringSlice(row, "source_doc_ids"))- page.SourceChunkIDs = unionStrs(page.SourceChunkIDs, metaStringSlice(row, "source_chunk_ids"))+ page.Aliases = unionStrs(page.Aliases, kccommon.MetaStringSlice(row, "entity_names_kwd"))+ page.Outlinks = unionStrs(page.Outlinks, kccommon.MetaStringSlice(row, "outlinks_kwd"))+ page.SourceDocIDs = unionStrs(page.SourceDocIDs, kccommon.MetaStringSlice(row, "source_doc_ids"))+ page.SourceChunkIDs = unionStrs(page.SourceChunkIDs, kccommon.MetaStringSlice(row, "source_chunk_ids"))bySlug[slug] = page}if len(result.Chunks) < batchSize {@@ -1673,21 +1640,21 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string// Filter map is being ignored and wiki_entity rows leak into the page// projection (which then has nothing under compile_kwd=wiki_page).for _, c := range res.Chunks {- kwdSeen[metaString(c, "compile_kwd")]+++ kwdSeen[kccommon.MetaString(c, "compile_kwd")]++}if len(res.Chunks) == 0 {break}for _, c := range res.Chunks {p := wikiPageProjection{- Slug: metaString(c, "slug_kwd"),+ Slug: kccommon.MetaString(c, "slug_kwd"),PageType: types.WikiPageCategory(c),- Title: metaString(c, "title_kwd"),- Aliases: metaStringSlice(c, "entity_names_kwd"),- Summary: metaString(c, "summary_with_weight"),- Outlinks: metaStringSlice(c, "outlinks_kwd"),- SourceDocIDs: metaStringSlice(c, "source_doc_ids"),- SourceChunkIDs: metaStringSlice(c, "source_chunk_ids"),+ Title: kccommon.MetaString(c, "title_kwd"),+ Aliases: kccommon.MetaStringSlice(c, "entity_names_kwd"),+ Summary: kccommon.MetaString(c, "summary_with_weight"),+ Outlinks: kccommon.MetaStringSlice(c, "outlinks_kwd"),+ SourceDocIDs: kccommon.MetaStringSlice(c, "source_doc_ids"),+ SourceChunkIDs: kccommon.MetaStringSlice(c, "source_chunk_ids"),}out = append(out, p)}