Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): consolidate duplicated meta-reader helpers into kccommon (#21983)

xugangqiang106c7d5cbf39
diff --git a/internal/common/format.go b/internal/common/format.go
index 39c0d1399f..8f82d2cb76 100644
--- a/internal/common/format.go
+++ b/internal/common/format.go
@@ -191,3 +191,16 @@ func IsValidString(v interface{}) bool {
func ChunkID(docID, text string) string {
return fmt.Sprintf("%016x", xxhash.Sum64String(text+docID))
}
+
+// FirstNonEmpty returns the first value that is non-empty after trimming,
+// trimmed. Trimming the returned value means callers receive a normalized result
+// and need not TrimSpace again at every use site. Used as a generic
+// settings/value precedence helper across the codebase.
+func FirstNonEmpty(vals ...string) string {
+ for _, v := range vals {
+ if strings.TrimSpace(v) != "" {
+ return strings.TrimSpace(v)
+ }
+ }
+ return ""
+}
diff --git a/internal/common/format_test.go b/internal/common/format_test.go
index 936c896b25..ef1ee6622c 100644
--- a/internal/common/format_test.go
+++ b/internal/common/format_test.go
@@ -97,3 +97,26 @@ func TestBaseModelName(t *testing.T) {
})
}
}
+
+func TestFirstNonEmpty(t *testing.T) {
+ tests := []struct {
+ name string
+ in []string
+ want string
+ }{
+ {"plain value unchanged", []string{"a", "b"}, "a"},
+ {"returns first when later match", []string{"x", "y", "z"}, "x"},
+ {"trims chosen value", []string{" picked ", "b"}, "picked"},
+ {"chosen whitespace later clean", []string{" picked ", "clean"}, "picked"},
+ {"skips whitespace-only candidates", []string{" ", " ", "real"}, "real"},
+ {"all empty returns empty", []string{"", " ", " "}, ""},
+ {"nil returns empty", nil, ""},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := FirstNonEmpty(tt.in...); got != tt.want {
+ t.Fatalf("FirstNonEmpty(%v) = %q, want %q", tt.in, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/common/migration_helpers_test.go b/internal/ingestion/component/knowledge_compiler/common/migration_helpers_test.go
new file mode 100644
index 0000000000..1fad8a3d20
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/common/migration_helpers_test.go
@@ -0,0 +1,124 @@
+package common
+
+import (
+ "reflect"
+ "testing"
+)
+
+// TestFirstNonEmptyReturnsTrimmed locks the contract that FirstNonEmpty returns
+// the *trimmed* first non-empty value. The wiki pipeline historically relied on
+// this (its local firstNonEmpty trimmed and returned the trimmed value); the
+// consolidated helper must preserve that so the four local copies can be deleted
+// without a behavior change.
+func TestFirstNonEmptyReturnsTrimmed(t *testing.T) {
+ cases := []struct {
+ name string
+ in []string
+ want string
+ }{
+ {"plain value unchanged", []string{"a", "b"}, "a"},
+ {"returns first when later match", []string{"x", "y", "z"}, "x"},
+ {"trims leading/trailing space of the chosen value", []string{" picked ", "b"}, "picked"},
+ // Adversarial: the chosen candidate carries whitespace and a later
+ // candidate does not. A wrong (untrimmed-return) implementation would
+ // return " picked " here.
+ {"chosen whitespace, later clean", []string{" picked ", "clean"}, "picked"},
+ {"skips whitespace-only candidates", []string{" ", " ", "real"}, "real"},
+ {"all empty returns empty", []string{"", " ", " "}, ""},
+ {"nil returns empty", nil, ""},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ if got := FirstNonEmpty(c.in...); got != c.want {
+ t.Fatalf("FirstNonEmpty(%v) = %q, want %q", c.in, got, c.want)
+ }
+ })
+ }
+}
+
+// TestMetaString locks the canonical MetaString behavior: returns the string
+// value as-is, and "" for a missing or non-string key.
+func TestMetaString(t *testing.T) {
+ m := map[string]any{"k": "v", "n": 123}
+ if got := MetaString(m, "k"); got != "v" {
+ t.Fatalf("MetaString(k) = %q, want v", got)
+ }
+ if got := MetaString(m, "n"); got != "" {
+ t.Fatalf("MetaString(non-string) = %q, want empty", got)
+ }
+ if got := MetaString(m, "missing"); got != "" {
+ t.Fatalf("MetaString(missing) = %q, want empty", got)
+ }
+}
+
+// TestMetaInt locks the canonical int64 MetaInt: tolerant of int64, int,
+// float64, and numeric string boxings; missing/non-numeric yields (0, false).
+func TestMetaInt(t *testing.T) {
+ m := map[string]any{
+ "i64": int64(7),
+ "i": int(8),
+ "f": float64(9.0),
+ "s": "42",
+ "bad": "abc",
+ "non": true,
+ }
+ if v, ok := MetaInt(m, "i64"); v != 7 || !ok {
+ t.Fatalf("MetaInt(i64) = %d,%v want 7,true", v, ok)
+ }
+ if v, ok := MetaInt(m, "i"); v != 8 || !ok {
+ t.Fatalf("MetaInt(i) = %d,%v want 8,true", v, ok)
+ }
+ if v, ok := MetaInt(m, "f"); v != 9 || !ok {
+ t.Fatalf("MetaInt(f) = %d,%v want 9,true", v, ok)
+ }
+ // Adversarial: numeric string boxed value must still parse.
+ if v, ok := MetaInt(m, "s"); v != 42 || !ok {
+ t.Fatalf("MetaInt(s) = %d,%v want 42,true", v, ok)
+ }
+ if v, ok := MetaInt(m, "bad"); v != 0 || ok {
+ t.Fatalf("MetaInt(bad) = %d,%v want 0,false", v, ok)
+ }
+ if v, ok := MetaInt(m, "non"); v != 0 || ok {
+ t.Fatalf("MetaInt(non-numeric) = %d,%v want 0,false", v, ok)
+ }
+ if v, ok := MetaInt(m, "missing"); v != 0 || ok {
+ t.Fatalf("MetaInt(missing) = %d,%v want 0,false", v, ok)
+ }
+}
+
+// TestMetaStringSlice locks the canonical filter-empty metaStringSlice: reads
+// []string and []any, keeps non-empty strings, drops empties (the []any branch
+// previously filtered; the source data is produced empty-free by the dedup
+// helpers so this is behavior-preserving with both legacy readers).
+func TestMetaStringSlice(t *testing.T) {
+ m := map[string]any{
+ "strs": []string{"a", "", "b"},
+ "strsEmpty": []string{"", "", ""},
+ "anys": []any{"a", "", 5, "b"},
+ "anyson": []any{1, 2},
+ "missing": "not-a-slice",
+ }
+ // Both boxings must drop empties: a unified []string/[]any reader must not let
+ // the source type change the result. The []string branch previously kept
+ // empties (inconsistent with []any) — this guards the fix.
+ if got := MetaStringSlice(m, "strs"); !reflect.DeepEqual(got, []string{"a", "b"}) {
+ t.Fatalf("MetaStringSlice(strs) = %v, want [a b]", got)
+ }
+ // A []string that is all-empty yields an empty (non-nil) slice, mirroring the
+ // []any branch — never nil for a present typed []string key.
+ if got := MetaStringSlice(m, "strsEmpty"); len(got) != 0 {
+ t.Fatalf("MetaStringSlice(strsEmpty) = %v, want empty slice", got)
+ }
+ // []any: drops non-strings and empties.
+ if got := MetaStringSlice(m, "anys"); !reflect.DeepEqual(got, []string{"a", "b"}) {
+ t.Fatalf("MetaStringSlice(anys) = %v, want [a b]", got)
+ }
+ // All entries dropped: the []any branch returns a non-nil empty slice
+ // (matches the legacy reader); a non-slice/missing key returns nil.
+ if got := MetaStringSlice(m, "anyson"); len(got) != 0 {
+ t.Fatalf("MetaStringSlice(anyson) = %v, want empty slice", got)
+ }
+ if got := MetaStringSlice(m, "missing"); got != nil {
+ t.Fatalf("MetaStringSlice(missing) = %v, want nil", got)
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/common/relations.go b/internal/ingestion/component/knowledge_compiler/common/relations.go
index f0e81e8ad1..717e9bb90e 100644
--- a/internal/ingestion/component/knowledge_compiler/common/relations.go
+++ b/internal/ingestion/component/knowledge_compiler/common/relations.go
@@ -9,10 +9,10 @@ import "strings"
func FilterOrphanRelations(products []Product) []Product {
entities := make(map[string]struct{}, len(products))
for _, product := range products {
- if product.Meta == nil || strings.TrimSpace(metaString(product.Meta, "kind")) != "entity" {
+ if product.Meta == nil || strings.TrimSpace(MetaString(product.Meta, "kind")) != "entity" {
continue
}
- name := normalizeRelationEndpoint(metaString(product.Meta, "name"))
+ name := normalizeRelationEndpoint(MetaString(product.Meta, "name"))
if name != "" {
entities[relationScope(product)+"\x00"+name] = struct{}{}
}
@@ -20,9 +20,9 @@ func FilterOrphanRelations(products []Product) []Product {
filtered := make([]Product, 0, len(products))
for _, product := range products {
- from := normalizeRelationEndpoint(metaString(product.Meta, "from"))
- to := normalizeRelationEndpoint(metaString(product.Meta, "to"))
- isRelation := strings.TrimSpace(metaString(product.Meta, "kind")) == "relation" || (from != "" || to != "")
+ from := normalizeRelationEndpoint(MetaString(product.Meta, "from"))
+ to := normalizeRelationEndpoint(MetaString(product.Meta, "to"))
+ isRelation := strings.TrimSpace(MetaString(product.Meta, "kind")) == "relation" || (from != "" || to != "")
if isRelation && (from == "" || to == "" || !hasRelationEntity(entities, product, from) || !hasRelationEntity(entities, product, to)) {
continue
}
@@ -37,7 +37,7 @@ func hasRelationEntity(entities map[string]struct{}, product Product, name strin
}
func relationScope(product Product) string {
- compileKind := metaString(product.Meta, "compile_kwd")
+ compileKind := MetaString(product.Meta, "compile_kwd")
if compileKind == "" {
compileKind = string(product.Variant)
}
@@ -52,7 +52,7 @@ func normalizeRelationEndpoint(value string) string {
return strings.ToLower(strings.Join(strings.Fields(value), " "))
}
-func metaString(meta map[string]any, key string) string {
+func MetaString(meta map[string]any, key string) string {
value, _ := meta[key].(string)
return value
}
diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.go
index 34c2136bb6..68a99e2988 100644
--- a/internal/ingestion/component/knowledge_compiler/common/types.go
+++ b/internal/ingestion/component/knowledge_compiler/common/types.go
@@ -12,6 +12,8 @@ import (
"errors"
"fmt"
"strings"
+
+ appcommon "ragflow/internal/common"
)
// Variant identifies which knowledge-compile strategy to run.
@@ -279,14 +281,64 @@ func VariantToTaskType(variant Variant) string {
}
}
-// FirstNonEmpty returns the first string that is non-empty after trimming.
+// FirstNonEmpty returns the first string that is non-empty after trimming,
+// trimmed. It delegates to the generic helper in internal/common so the
+// knowledge_compiler package shares one implementation with the rest of the
+// codebase (the generic component layer no longer needs to reach into this
+// package for a meta-agnostic string helper). The trimmed return value preserves
+// the historical contract of the wiki pipeline's local helper.
func FirstNonEmpty(vals ...string) string {
- for _, v := range vals {
- if strings.TrimSpace(v) != "" {
- return v
+ return appcommon.FirstNonEmpty(vals...)
+}
+
+// MetaInt extracts an integer from a Product.Meta value that may be boxed as
+// int64, int, float64 (JSON number), or even a numeric string — the
+// engine/JSON round-trip does not guarantee a single numeric type. It returns
+// (0, false) when the key is absent or not a parseable number. The int64 form
+// is the canonical one so callers can store it directly without a narrowing
+// conversion.
+func MetaInt(meta map[string]any, key string) (int64, bool) {
+ switch v := meta[key].(type) {
+ case int64:
+ return v, true
+ case int:
+ return int64(v), true
+ case float64:
+ return int64(v), true
+ case string:
+ var n int64
+ if _, err := fmt.Sscanf(v, "%d", &n); err == nil {
+ return n, true
+ }
+ }
+ return 0, false
+}
+
+// MetaStringSlice reads a []string (or []any) Product.Meta key, keeping
+// non-empty strings and dropping empties. Both the []string and []any branches
+// filter empties so the result is independent of how the value was boxed (the
+// engine/JSON round-trip does not guarantee a single type). Returns nil when no
+// strings are present; a present but empty key yields a non-nil empty slice.
+func MetaStringSlice(meta map[string]any, key string) []string {
+ switch v := meta[key].(type) {
+ case []string:
+ out := make([]string, 0, len(v))
+ for _, e := range v {
+ if e != "" {
+ out = append(out, e)
+ }
}
+ return out
+ case []any:
+ out := make([]string, 0, len(v))
+ for _, e := range v {
+ if s, ok := e.(string); ok && s != "" {
+ out = append(out, s)
+ }
+ }
+ return out
}
- return ""
+ return nil
}
// VectorFromChunkMap extracts a pre-computed embedding from a chunk map. The
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 57a451df21..e576f838a6 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -248,13 +248,13 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
var pageCount, sectionCount, entityCount, relationCount int
for _, p := range out.Products {
switch {
- case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "page":
+ case p.Variant == common.VariantWiki && common.MetaString(p.Meta, "kind") == "page":
pageCount++
- case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "section":
+ case p.Variant == common.VariantWiki && common.MetaString(p.Meta, "kind") == "section":
sectionCount++
- case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "entity":
+ case p.Variant == common.VariantWiki && common.MetaString(p.Meta, "kind") == "entity":
entityCount++
- case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "relation":
+ case p.Variant == common.VariantWiki && common.MetaString(p.Meta, "kind") == "relation":
relationCount++
}
}
@@ -452,12 +452,12 @@ func productsToChunkDocs(products []common.Product, language string) ([]schema.C
}
}
// Provenance columns shared by all compiled rows.
- if ids := metaStringSlice(p.Meta, "source_chunk_ids"); len(ids) > 0 {
+ if ids := common.MetaStringSlice(p.Meta, "source_chunk_ids"); len(ids) > 0 {
if err := doc.SetExtraValue("source_chunk_ids", ids); err != nil {
return nil, err
}
}
- if ids := metaStringSlice(p.Meta, "source_doc_ids"); len(ids) > 0 {
+ if ids := common.MetaStringSlice(p.Meta, "source_doc_ids"); len(ids) > 0 {
if err := doc.SetExtraValue("source_doc_ids", ids); err != nil {
return nil, err
}
@@ -465,7 +465,7 @@ func productsToChunkDocs(products []common.Product, language string) ([]schema.C
// compilation_template_ids: the resolved template ids that produced
// this row (Python stamps one per row from the active template; the
// document-structure endpoint groups rows by this column).
- if ids := metaStringSlice(p.Meta, "compilation_template_ids"); len(ids) > 0 {
+ if ids := common.MetaStringSlice(p.Meta, "compilation_template_ids"); len(ids) > 0 {
if err := doc.SetExtraValue("compilation_template_ids", ids); err != nil {
return nil, err
}
@@ -483,7 +483,7 @@ func productsToChunkDocs(products []common.Product, language string) ([]schema.C
// conf/infinity_mapping.json lines 45–77, driven by Product.Meta keys that
// each variant's build site populates. Unknown/absent keys are skipped.
func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.Tokenizer) error {
- kind := metaString(p.Meta, "kind")
+ kind := common.MetaString(p.Meta, "kind")
switch p.Variant {
case common.VariantStructure:
@@ -502,8 +502,8 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T
// slug_kwd stores the full "<page_type>/<slug>" form, so retrieval
// filters (GetWikiPage) can reconstruct it directly. page_type is also
// stored separately for topic grouping.
- if pageType := metaString(p.Meta, "page_type"); pageType != "" {
- if slug := metaString(p.Meta, "slug"); slug != "" {
+ if pageType := common.MetaString(p.Meta, "page_type"); pageType != "" {
+ if slug := common.MetaString(p.Meta, "slug"); slug != "" {
// Normalize to the full "<page_type>/<slug>" form (Python writer
// contract). Idempotent: a slug that already carries the prefix
// (some producers emit pageType/slug directly) is left as-is.
@@ -516,39 +516,39 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T
}
}
}
- if v := metaString(p.Meta, "title"); v != "" {
+ if v := common.MetaString(p.Meta, "title"); v != "" {
if err := doc.SetExtraValue("title_kwd", v); err != nil {
return err
}
setTitleTokens(doc, v, tok)
}
- if v := metaString(p.Meta, "page_type"); v != "" {
+ if v := common.MetaString(p.Meta, "page_type"); v != "" {
if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {
return err
}
}
- if v := metaString(p.Meta, "topic"); v != "" {
+ if v := common.MetaString(p.Meta, "topic"); v != "" {
if err := doc.SetExtraValue("topic_kwd", v); err != nil {
return err
}
}
- if v := metaString(p.Meta, "summary"); v != "" {
+ if v := common.MetaString(p.Meta, "summary"); v != "" {
if err := doc.SetExtraValue("summary_with_weight", v); err != nil {
return err
}
}
// Section depth goes to depth_int; there is no section_level_int column.
- if v, ok := metaInt(p.Meta, "section_level"); ok {
+ if v, ok := common.MetaInt(p.Meta, "section_level"); ok {
if err := doc.SetExtraValue("depth_int", v); err != nil {
return err
}
}
- if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {
+ if v := common.MetaStringSlice(p.Meta, "entity_names"); len(v) > 0 {
if err := doc.SetExtraValue("entity_names_kwd", v); err != nil {
return err
}
}
- if v := metaStringSlice(p.Meta, "outlinks"); len(v) > 0 {
+ if v := common.MetaStringSlice(p.Meta, "outlinks"); len(v) > 0 {
if err := doc.SetExtraValue("outlinks_kwd", v); err != nil {
return err
}
@@ -556,7 +556,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T
return err
}
}
- if v := metaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {
+ if v := common.MetaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {
if err := doc.SetExtraValue("related_kb_pages_kwd", v); err != nil {
return err
}
@@ -580,17 +580,17 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T
// them out of the artifacts query (it filters
// type_kwd=["entity","relation"]) without a frontend
// change. Python mirrors this in _struct_upsert_tree_claim_rows.
- if v := metaString(p.Meta, "name"); v != "" {
+ if v := common.MetaString(p.Meta, "name"); v != "" {
if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {
return err
}
}
- if v := metaString(p.Meta, "entity_type"); v != "" {
+ if v := common.MetaString(p.Meta, "entity_type"); v != "" {
if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {
return err
}
}
- if v, ok := metaInt(p.Meta, "mention_count"); ok {
+ if v, ok := common.MetaInt(p.Meta, "mention_count"); ok {
if err := doc.SetExtraValue("mention_count_int", v); err != nil {
return err
}
@@ -604,7 +604,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T
return err
}
}
- if v, ok := metaInt(p.Meta, "level"); ok {
+ if v, ok := common.MetaInt(p.Meta, "level"); ok {
if err := doc.SetExtraValue("raptor_layer_int", v); err != nil {
return err
}
@@ -612,7 +612,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product, tok tokenizer.T
return err
}
}
- if v := metaStringSlice(p.Meta, "children"); len(v) > 0 {
+ if v := common.MetaStringSlice(p.Meta, "children"); len(v) > 0 {
if err := doc.SetExtraValue("children_kwd", v); err != nil {
return err
}
@@ -649,30 +649,30 @@ func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind str
}
}
if kind == "relation" {
- if v := metaString(p.Meta, "from"); v != "" {
+ if v := common.MetaString(p.Meta, "from"); v != "" {
if err := doc.SetExtraValue("from_entity_kwd", v); err != nil {
return err
}
}
- if v := metaString(p.Meta, "to"); v != "" {
+ if v := common.MetaString(p.Meta, "to"); v != "" {
if err := doc.SetExtraValue("to_entity_kwd", v); err != nil {
return err
}
}
}
if kind == "entity" {
- if v := metaString(p.Meta, "name"); v != "" {
+ if v := common.MetaString(p.Meta, "name"); v != "" {
if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {
return err
}
}
- if v := metaString(p.Meta, "entity_type"); v != "" {
+ if v := common.MetaString(p.Meta, "entity_type"); v != "" {
if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {
return err
}
}
}
- if v, ok := metaInt(p.Meta, "mention_count"); ok {
+ if v, ok := common.MetaInt(p.Meta, "mention_count"); ok {
if err := doc.SetExtraValue("mention_count_int", v); err != nil {
return err
}
@@ -681,10 +681,6 @@ func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind str
}
// metaString reads a string-valued Product.Meta key.
-func metaString(m map[string]any, key string) string {
- v, _ := m[key].(string)
- return v
-}
// setTitleTokens populates ChunkDoc.title_tks. Python's page row has no
// title_sm_tks, and the Infinity writer folds title_kwd / title_sm_tks into
@@ -698,34 +694,6 @@ func setTitleTokens(doc *schema.ChunkDoc, title string, tok tokenizer.Tokenizer)
}
}
-// metaInt reads an int-valued Product.Meta key (tolerant of float64 from JSON).
-func metaInt(m map[string]any, key string) (int, bool) {
- switch v := m[key].(type) {
- case int:
- return v, true
- case float64:
- return int(v), true
- }
- return 0, false
-}
-
-// metaStringSlice reads a []string Product.Meta key (tolerant of []any).
-func metaStringSlice(m map[string]any, key string) []string {
- switch v := m[key].(type) {
- case []string:
- return v
- case []any:
- out := make([]string, 0, len(v))
- for _, e := range v {
- if s, ok := e.(string); ok && s != "" {
- out = append(out, s)
- }
- }
- return out
- }
- return nil
-}
-
// mergeChunks returns the canonical chunker-shaped output: the upstream input
// chunks followed by the freshly compiled chunk docs, all under the "chunks"
// key with output_format "chunks". This makes KnowledgeCompiler's output schema
diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
index f9ef23bbed..d9bd97ec30 100644
--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
@@ -55,11 +55,11 @@ func runBatches(ctx context.Context, jobs []func() error) error {
// Run executes the mindmap variant.
func Run(ctx context.Context, deps common.Deps, param common.Param, inputs common.Inputs) (common.Outputs, error) {
- docID := firstNonEmpty(inputs.DocID, deps.DatasetID)
+ docID := common.FirstNonEmpty(inputs.DocID, deps.DatasetID)
if docID == "" {
docID = "unknown"
}
- llmID := firstNonEmpty(param.LLMID, inputs.LLMID)
+ llmID := common.FirstNonEmpty(param.LLMID, inputs.LLMID)
tenantID := deps.TenantID
if deps.Chat == nil {
@@ -268,7 +268,7 @@ func mindmapSourceChunkIDs(chunks []common.Chunk) []string {
seen := make(map[string]struct{}, len(chunks))
ids := make([]string, 0, len(chunks))
for i, chunk := range chunks {
- if strings.TrimSpace(firstNonEmpty(chunk.Text, chunk.Content)) == "" {
+ if strings.TrimSpace(common.FirstNonEmpty(chunk.Text, chunk.Content)) == "" {
continue
}
id := strings.TrimSpace(chunk.ID)
@@ -425,7 +425,7 @@ func payloadJSON(payload map[string]any) string {
func chunkTexts(chunks []common.Chunk) []string {
var out []string
for _, c := range chunks {
- t := firstNonEmpty(c.Text, c.Content)
+ t := common.FirstNonEmpty(c.Text, c.Content)
if strings.TrimSpace(t) == "" {
continue
}
@@ -433,12 +433,3 @@ func chunkTexts(chunks []common.Chunk) []string {
}
return out
}
-
-func firstNonEmpty(vals ...string) string {
- for _, v := range vals {
- if v != "" {
- return v
- }
- }
- return ""
-}
diff --git a/internal/ingestion/component/knowledge_compiler/structure/chain.go b/internal/ingestion/component/knowledge_compiler/structure/chain.go
index c8d19babb7..5b4fed8aa1 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/chain.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/chain.go
@@ -226,7 +226,7 @@ func chainGatherChunkText(badRows []common.Product, chunksByID map[string]string
seen := map[string]bool{}
var out [][2]string
for _, row := range badRows {
- for _, cid := range metaStrings(row.Meta, "source_chunk_ids") {
+ for _, cid := range common.MetaStringSlice(row.Meta, "source_chunk_ids") {
if cid == "" || seen[cid] {
continue
}
diff --git a/internal/ingestion/component/knowledge_compiler/structure/graph.go b/internal/ingestion/component/knowledge_compiler/structure/graph.go
index 460ef77da7..4e45cae5f4 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/graph.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/graph.go
@@ -147,7 +147,7 @@ func RebuildStructureGraph(products []common.Product, compileType string) map[st
relations = append(relations, rel)
}
case "entity":
- if ent := graphEntity(payload, metaStrings(p.Meta, "source_chunk_ids")); ent != nil {
+ if ent := graphEntity(payload, common.MetaStringSlice(p.Meta, "source_chunk_ids")); ent != nil {
entities = append(entities, ent)
}
}
diff --git a/internal/ingestion/component/knowledge_compiler/structure/merge.go b/internal/ingestion/component/knowledge_compiler/structure/merge.go
index 19eea60e82..e3a71f238c 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/merge.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/merge.go
@@ -286,7 +286,7 @@ func (d *LLMMergeDecider) BuildReplacement(ctx context.Context, existing, incomi
}
merged = applyMergeInvariants(existing, merged)
- chunkIDs := unionOrdered(metaStrings(existing.Meta, "source_chunk_ids"), metaStrings(incoming.Meta, "source_chunk_ids"))
+ chunkIDs := unionOrdered(common.MetaStringSlice(existing.Meta, "source_chunk_ids"), common.MetaStringSlice(incoming.Meta, "source_chunk_ids"))
texts := []string{payloadDescription(merged)}
vecs, err := d.Embed.Encode(ctx, texts)
if err != nil {
@@ -662,23 +662,6 @@ func entityNameValue(p common.Product) string {
return strings.TrimSpace(name)
}
-// metaStrings reads a []string meta value (tolerating []any).
-func metaStrings(meta map[string]any, key string) []string {
- switch v := meta[key].(type) {
- case []string:
- return v
- case []any:
- out := make([]string, 0, len(v))
- for _, e := range v {
- if s, ok := e.(string); ok {
- out = append(out, s)
- }
- }
- return out
- }
- return nil
-}
-
// unionOrdered mirrors _common.union_ordered: concatenated, deduped,
// first-seen order preserved.
func unionOrdered(lists ...[]string) []string {
@@ -747,7 +730,7 @@ func MergeIntoStore(ctx context.Context, store *common.MemStore, decider MergeDe
stats.survivorID = droppedExisting.ID
// Fold the incoming source_chunk_ids into the surviving entry.
if dropped && droppedExisting.ID != "" {
- if ids := metaStrings(row.Meta, "source_chunk_ids"); len(ids) > 0 {
+ if ids := common.MetaStringSlice(row.Meta, "source_chunk_ids"); len(ids) > 0 {
store.MergeSourceChunkIDs(droppedExisting.ID, ids)
}
}
@@ -830,7 +813,7 @@ func (g *GroupedDeduper) Add(ctx context.Context, row common.Product) error {
}
})
if err == nil && action == common.KeepDrop {
- store.MergeSourceChunkIDs(candidateID, metaStrings(row.Meta, "source_chunk_ids"))
+ store.MergeSourceChunkIDs(candidateID, common.MetaStringSlice(row.Meta, "source_chunk_ids"))
}
} else {
var stats MergeStats
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
index 8d92f82127..4d5f679e54 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
@@ -491,7 +491,7 @@ func TestStructureRunGraphKind(t *testing.T) {
if betaProduct == nil {
t.Fatal("no Beta entity product")
}
- ids := metaStrings(betaProduct.Meta, "source_chunk_ids")
+ ids := common.MetaStringSlice(betaProduct.Meta, "source_chunk_ids")
if len(ids) != 2 {
t.Fatalf("Beta provenance = %v, want {c1,c2} unioned through the merge", ids)
}
@@ -636,7 +636,7 @@ func TestLLMMergeDeciderContracts(t *testing.T) {
if got := entityName(parsePayload(merged.Content)); got != "Alpha" {
t.Errorf("merged canonical name = %q, want Alpha", got)
}
- if ids := metaStrings(merged.Meta, "source_chunk_ids"); len(ids) != 2 {
+ if ids := common.MetaStringSlice(merged.Meta, "source_chunk_ids"); len(ids) != 2 {
t.Errorf("merged provenance = %v, want {c1,c2}", ids)
}
if aliases := d.Aliases(); aliases["Al"] != "Alpha" {
@@ -857,7 +857,7 @@ func TestGroupedDeduperMergesSameNameBeforeVectorSimilarity(t *testing.T) {
if payload["type"] != "component" {
t.Fatalf("specific type should replace other: %v", payload)
}
- if ids := metaStrings(got[0].Meta, "source_chunk_ids"); len(ids) != 2 {
+ if ids := common.MetaStringSlice(got[0].Meta, "source_chunk_ids"); len(ids) != 2 {
t.Fatalf("source chunks were not merged: %v", ids)
}
if chat.mergeCalls != 0 {
@@ -889,7 +889,7 @@ func (similarityMergeDecider) Decide(_ context.Context, existing, incoming commo
for key, value := range existing.Meta {
replacement.Meta[key] = value
}
- replacement.Meta["source_chunk_ids"] = unionOrdered(metaStrings(existing.Meta, "source_chunk_ids"), metaStrings(incoming.Meta, "source_chunk_ids"))
+ replacement.Meta["source_chunk_ids"] = unionOrdered(common.MetaStringSlice(existing.Meta, "source_chunk_ids"), common.MetaStringSlice(incoming.Meta, "source_chunk_ids"))
return DecisionMerge, replacement, nil
}
@@ -909,7 +909,7 @@ func TestGroupedDeduperTracksNameAfterSemanticMerge(t *testing.T) {
if len(got) != 1 {
t.Fatalf("semantic alias must remain addressable by its own name: %+v", got)
}
- if ids := metaStrings(got[0].Meta, "source_chunk_ids"); len(ids) != 3 {
+ if ids := common.MetaStringSlice(got[0].Meta, "source_chunk_ids"); len(ids) != 3 {
t.Fatalf("semantic and exact-name merges lost provenance: %v", ids)
}
}
diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph.go b/internal/ingestion/component/knowledge_compiler/tree/graph.go
index 9aafe2c3f2..5c3dfbbbd0 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/graph.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/graph.go
@@ -169,7 +169,7 @@ func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {
ent := map[string]any{
"name": title,
"type": "tree_node",
- "description": firstNonEmpty(node.description, title),
+ "description": common.FirstNonEmpty(node.description, title),
"mention_count": 1,
}
if len(node.sourceChunkIDs) > 0 {
diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
index ad25e506ed..ca3526361a 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
@@ -35,11 +35,11 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
if deps.Chat == nil {
return common.Outputs{}, fmt.Errorf("tree: chat model required")
}
- docID := firstNonEmpty(inputs.DocID, deps.DatasetID)
+ docID := common.FirstNonEmpty(inputs.DocID, deps.DatasetID)
if docID == "" {
docID = "unknown"
}
- llmID := firstNonEmpty(param.LLMID, inputs.LLMID)
+ llmID := common.FirstNonEmpty(param.LLMID, inputs.LLMID)
tenantID := deps.TenantID
treeOrder := resolveTreeOrder(param)
@@ -544,7 +544,7 @@ func allIndices(n int) []int {
// chunkTextOf returns the text of a chunk, preferring Text over Content.
func chunkTextOf(c common.Chunk) string {
- return firstNonEmpty(c.Text, c.Content)
+ return common.FirstNonEmpty(c.Text, c.Content)
}
// chunkHasText reports whether a chunk carries non-blank text. Empty chunks
@@ -644,15 +644,6 @@ func collectIDs(chunkIDs []string, pointIdxs []int) []string {
return out
}
-func firstNonEmpty(vals ...string) string {
- for _, v := range vals {
- if v != "" {
- return v
- }
- }
- return ""
-}
-
func toFloat64Matrix(vecs [][]float32) [][]float64 {
out := make([][]float64, len(vecs))
for i, v := range vecs {
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
index a42f7f77c1..a5ba92156d 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
@@ -257,7 +257,7 @@ func formatWikiChunkBatch(docID string, batch []common.Chunk) (string, []string)
id = fmt.Sprintf("chunk-%d", i+1)
}
ids = append(ids, id)
- text := firstNonEmpty(c.Text, c.Content)
+ text := common.FirstNonEmpty(c.Text, c.Content)
if strings.TrimSpace(text) == "" {
continue
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
index 64cc98ab15..f9bd3e393f 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
@@ -216,7 +216,7 @@ const (
// Run executes the wiki variant.
func Run(ctx context.Context, deps common.Deps, param common.Param, inputs common.Inputs) (common.Outputs, error) {
docID := inputs.DocID
- llmID := firstNonEmpty(inputs.LLMID, param.LLMID)
+ llmID := common.FirstNonEmpty(inputs.LLMID, param.LLMID)
if docID == "" {
docID = "unknown"
}
@@ -1010,16 +1010,16 @@ func (p *wikiPipeline) runRefinePage(
existingSection := ""
existingRaw := ""
if existing != nil {
- existingRaw = firstNonEmpty(existing.ContentMDRaw, existing.ContentMD)
+ existingRaw = common.FirstNonEmpty(existing.ContentMDRaw, existing.ContentMD)
if strings.TrimSpace(existingRaw) != "" {
existingSection = "## Existing page content (UPDATE — integrate new evidence into this)\n\n" + existingRaw + "\n"
}
}
user := renderWikiTemplate(wikiRefineWriterUserTemplate, map[string]string{
- "action": firstNonEmpty(planItem.Action, "CREATE"),
+ "action": common.FirstNonEmpty(planItem.Action, "CREATE"),
"slug": planItem.Slug,
- "title": firstNonEmpty(planItem.Title, planItem.Slug),
- "page_type": firstNonEmpty(planItem.PageType, "concept"),
+ "title": common.FirstNonEmpty(planItem.Title, planItem.Slug),
+ "page_type": common.FirstNonEmpty(planItem.PageType, "concept"),
"all_plan_slugs": strings.Join(available, "\n"),
"related_kb_pages": formatWikiRelatedKB(planItem.RelatedKB, pageTitles),
"existing_section": existingSection,
@@ -1038,9 +1038,9 @@ func (p *wikiPipeline) runRefinePage(
if resp == nil {
return wikiPageResult{}, fmt.Errorf("knowledge_compiler: wiki refine returned no response")
}
- contentRaw := strings.TrimSpace(firstNonEmpty(resp.Content))
+ contentRaw := strings.TrimSpace(common.FirstNonEmpty(resp.Content))
if contentRaw == "" {
- contentRaw = "# " + firstNonEmpty(planItem.Title, planItem.Slug) + "\n\n(Page generation produced no content.)"
+ contentRaw = "# " + common.FirstNonEmpty(planItem.Title, planItem.Slug) + "\n\n(Page generation produced no content.)"
}
if strings.TrimSpace(existingRaw) != "" {
contentRaw, err = p.mergeWikiPageContent(existingRaw, contentRaw, planItem.Slug)
@@ -1048,7 +1048,7 @@ func (p *wikiPipeline) runRefinePage(
return wikiPageResult{}, err
}
}
- contentRendered, outlinks := transformWikiLinks(contentRaw, firstNonEmpty(p.datasetID, p.docID), pageTitles, slugToPageType)
+ contentRendered, outlinks := transformWikiLinks(contentRaw, common.FirstNonEmpty(p.datasetID, p.docID), pageTitles, slugToPageType)
// wiki_incremental port (O2): deterministically guarantee a "See also"
// cross-link section from RelatedKB. transformWikiLinks/resolveSlug drop
// links whose target cannot be resolved to a known slug, so an LLM-authored
@@ -1058,13 +1058,13 @@ func (p *wikiPipeline) runRefinePage(
// full-slug is linked once, only if not already present in Outlinks.
contentRendered, outlinks = appendWikiSeeAlso(contentRendered, outlinks, planItem.RelatedKB, p.datasetID, pageTitles, slugToPageType)
sourceDocIDs := collectWikiSourceDocIDs(p.inputs.Chunks, sourceChunkIDs, p.docID)
- topic := firstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)
+ topic := common.FirstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)
return wikiPageResult{
Slug: planItem.Slug,
- Title: firstNonEmpty(planItem.Title, planItem.Slug),
- PageType: firstNonEmpty(planItem.PageType, "concept"),
+ Title: common.FirstNonEmpty(planItem.Title, planItem.Slug),
+ PageType: common.FirstNonEmpty(planItem.PageType, "concept"),
Topic: topic,
- Action: firstNonEmpty(planItem.Action, "CREATE"),
+ Action: common.FirstNonEmpty(planItem.Action, "CREATE"),
EntityNames: appcommon.DeduplicateTrimmed(planItem.EntityNames),
RelatedKBPages: appcommon.DeduplicateTrimmed(planItem.RelatedKB),
ContentRaw: contentRaw,
@@ -1259,13 +1259,13 @@ func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {
continue
}
page.Action = "UPDATE"
- page.Slug = firstNonEmpty(existing.Slug, page.Slug)
- page.Title = firstNonEmpty(existing.Title, page.Title)
- page.PageType = firstNonEmpty(existing.PageType, page.PageType)
+ page.Slug = common.FirstNonEmpty(existing.Slug, page.Slug)
+ page.Title = common.FirstNonEmpty(existing.Title, page.Title)
+ page.PageType = common.FirstNonEmpty(existing.PageType, page.PageType)
if existing.RoutedTopic != "" {
page.Topic = common.NormalizeWikiTopicPath(existing.RoutedTopic)
} else {
- page.Topic = common.NormalizeWikiTopicPath(firstNonEmpty(existing.Topic, page.Topic))
+ page.Topic = common.NormalizeWikiTopicPath(common.FirstNonEmpty(existing.Topic, page.Topic))
}
page.RelatedKB = mergeStrings(page.RelatedKB, existing.RelatedKBPages)
if len(existing.RoutedEntityNames) > 0 {
@@ -1549,7 +1549,7 @@ func (p *wikiPipeline) mergeWikiPageContent(existingMD, newMD, slug string) (str
if resp == nil {
return fallback, nil
}
- merged := strings.TrimSpace(firstNonEmpty(resp.Content))
+ merged := strings.TrimSpace(common.FirstNonEmpty(resp.Content))
if merged == "" {
return fallback, nil
}
@@ -1570,7 +1570,7 @@ func conservativeMergeWikiMarkdown(existingMD, newMD string) string {
if strings.Contains(newMD, existingMD) {
return strings.TrimSpace(newMD)
}
- title := firstNonEmpty(wikiMarkdownTitle(existingMD), wikiMarkdownTitle(newMD))
+ title := common.FirstNonEmpty(wikiMarkdownTitle(existingMD), wikiMarkdownTitle(newMD))
blocks := make([]string, 0, 8)
seen := map[string]bool{}
appendBlocks := func(body string) {
@@ -2117,7 +2117,7 @@ func buildWikiFallbackPages(reduced wikiExtract) []wikiPlanPage {
if len(out) == 0 {
for _, t := range reduced.Topics {
title := common.WikiTopicLeaf(t.Path)
- lead := firstNonEmpty(t.Description, title)
+ lead := common.FirstNonEmpty(t.Description, title)
appendPage(wikiPlanPage{
Action: "CREATE",
Slug: "topic/" + slugify(title),
@@ -2489,7 +2489,7 @@ func buildSourceContext(chunks []common.Chunk, sourceChunkIDs []string) string {
if len(sourceChunkIDs) == 0 {
var b strings.Builder
for _, ch := range chunks {
- text := firstNonEmpty(ch.Text, ch.Content)
+ text := common.FirstNonEmpty(ch.Text, ch.Content)
if strings.TrimSpace(text) == "" {
continue
}
@@ -2504,7 +2504,7 @@ func buildSourceContext(chunks []common.Chunk, sourceChunkIDs []string) string {
if id == "" {
id = fmt.Sprintf("chunk-%d", i+1)
}
- lookup[id] = firstNonEmpty(ch.Text, ch.Content)
+ lookup[id] = common.FirstNonEmpty(ch.Text, ch.Content)
}
var b strings.Builder
for _, id := range sourceChunkIDs {
@@ -2598,7 +2598,7 @@ func assembleWikiPageEvidence(planItem wikiPlanPage, claims []wikiClaim, entityB
evidence = append(evidence, wikiEvidenceItem{
Statement: claim.Statement,
Subject: claim.Subject,
- Confidence: firstNonEmpty(claim.Confidence, "explicit"),
+ Confidence: common.FirstNonEmpty(claim.Confidence, "explicit"),
ChunkIDs: appcommon.DeduplicateTrimmed(claim.SourceChunkIDs),
})
}
@@ -2679,7 +2679,7 @@ func formatWikiEvidenceBlocks(evidence []wikiEvidenceItem) string {
if ev.Synthetic {
continue
}
- confidence := strings.ToUpper(firstNonEmpty(ev.Confidence, "explicit"))
+ confidence := strings.ToUpper(common.FirstNonEmpty(ev.Confidence, "explicit"))
lines = append(lines, fmt.Sprintf("%d. [%s] %s\n %s", len(lines)+1, confidence, ev.Subject, ev.Statement))
}
if len(lines) == 0 {
@@ -3064,15 +3064,6 @@ func prefixWithDash(items []string) []string {
return out
}
-func firstNonEmpty(vals ...string) string {
- for _, v := range vals {
- if s := strings.TrimSpace(v); s != "" {
- return s
- }
- }
- return ""
-}
-
func mergeStrings(a, b []string) []string {
return appcommon.DeduplicateTrimmed(append(append([]string{}, a...), b...))
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
index aeeadff05b..8e13bf0a80 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go
@@ -70,7 +70,7 @@ func (p *wikiPipeline) runVersionedMap() error {
if strings.TrimSpace(chunk.ID) == "" {
continue
}
- contentHash := wikiMapHash(firstNonEmpty(chunk.Text, chunk.Content))
+ contentHash := wikiMapHash(common.FirstNonEmpty(chunk.Text, chunk.Content))
key := wikiMapHash(strings.Join([]string{
p.tenantID,
p.datasetID,
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index 7bb9b79e2a..7fc30d88d2 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -368,7 +368,7 @@ func describeImage(
// (the key the Python parser's image branch used) stays readable for canvases
// saved before the move. Do NOT read setup["prompt"] here — that key is for
// the video family, not image.
- if v := firstNonEmpty(vision.systemPrompt, getStringOr(setup, "system_prompt", "")); v != "" {
+ if v := common.FirstNonEmpty(vision.systemPrompt, getStringOr(setup, "system_prompt", "")); v != "" {
prompt = v
}
messages := []modelModule.Message{{
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index a378345f6d..413ea4552a 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -621,7 +621,7 @@ func dispatchMinerUPDF(
// That default used to arrive implicitly through defaultSetups() - which
// also fed captions, so a knowledge base that never chose a language got
// Chinese captions (issue #20727). Keeping it here confines it to the engine.
- lang := firstNonEmpty(getStringOr(setup, "mineru_lang", ""), getStringOr(setup, "lang", ""))
+ lang := common.FirstNonEmpty(getStringOr(setup, "mineru_lang", ""), getStringOr(setup, "lang", ""))
if lang == "" {
lang = minerUDefaultLanguage
}
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index 1c0bb2b162..bf3c5ba428 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -1183,18 +1183,6 @@ func getStringOr(m map[string]any, key, def string) string {
return def
}
-// firstNonEmpty returns the first non-empty value, or "" when all are empty.
-// Used to walk a settings precedence chain (global choice → legacy per-family
-// choice).
-func firstNonEmpty(values ...string) string {
- for _, v := range values {
- if v != "" {
- return v
- }
- }
- return ""
-}
-
// cloneInputs returns a shallow copy of m with room for one extra key.
// Used to inject the Globals-resolved `name` into the decode input without
// mutating the caller's input snapshot.
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 4f4797caa6..9e6e3d4a98 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -555,7 +555,7 @@ func filterWikiPageCandidates(candidates []kccommon.Product) []kccommon.Product
if cand.Variant != kccommon.VariantWiki {
continue // tree/structure products never enter the wiki merge
}
- if metaString(cand.Meta, "kind") != "page" {
+ if kccommon.MetaString(cand.Meta, "kind") != "page" {
continue
}
filtered = append(filtered, cand)
@@ -1256,7 +1256,7 @@ func (c *Consumer) mergeEntityModeCandidates(ctx context.Context, tenant, kb str
var entityCandidates []kccommon.Product
var topic []kccommon.Product
for _, candidate := range candidates {
- pageType := strings.ToLower(strings.TrimSpace(metaString(candidate.Meta, "page_type")))
+ pageType := strings.ToLower(strings.TrimSpace(kccommon.MetaString(candidate.Meta, "page_type")))
if pageType != "entity" && pageType != "concept" {
topic = append(topic, candidate)
continue
@@ -1307,7 +1307,7 @@ func (c *Consumer) mergeWikiProductsBySlug(ctx context.Context, tenant, kb strin
exists = true
continue
}
- if strings.EqualFold(strings.TrimSpace(metaString(current.Meta, "page_type")), "topic") {
+ if strings.EqualFold(strings.TrimSpace(kccommon.MetaString(current.Meta, "page_type")), "topic") {
current = mergeTopicPage(current, candidate)
} else {
current = wikiEntityMerge(current, candidate)
@@ -1358,11 +1358,11 @@ func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommon
jobs = append(jobs, func() error {
page := pages[idx]
prompt := fmt.Sprintf("PAGE TYPE: %s\nTITLE: %s\nTOPIC: %s\nENTITIES: %s\nSUMMARY: %s\n\nEVIDENCE MARKDOWN:\n%s",
- metaString(page.Meta, "page_type"),
- metaString(page.Meta, "title"),
- metaString(page.Meta, "topic"),
- strings.Join(metaStringSlice(page.Meta, "entity_names"), ", "),
- metaString(page.Meta, "summary"),
+ kccommon.MetaString(page.Meta, "page_type"),
+ kccommon.MetaString(page.Meta, "title"),
+ kccommon.MetaString(page.Meta, "topic"),
+ strings.Join(kccommon.MetaStringSlice(page.Meta, "entity_names"), ", "),
+ kccommon.MetaString(page.Meta, "summary"),
page.Content)
response, err := deps.Chat.Chat(ctx, kccommon.ChatRequest{
LLMID: defaultLLMID,
@@ -1371,14 +1371,14 @@ func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommon
Temperature: floatPtr(0.1),
})
if err != nil {
- return fmt.Errorf("rewrite Wiki page %q: %w", metaString(page.Meta, "slug"), err)
+ return fmt.Errorf("rewrite Wiki page %q: %w", kccommon.MetaString(page.Meta, "slug"), err)
}
if response == nil {
- return fmt.Errorf("rewrite Wiki page %q returned no response", metaString(page.Meta, "slug"))
+ return fmt.Errorf("rewrite Wiki page %q returned no response", kccommon.MetaString(page.Meta, "slug"))
}
content := strings.TrimSpace(response.Content)
if content == "" {
- return fmt.Errorf("rewrite Wiki page %q returned empty content", metaString(page.Meta, "slug"))
+ return fmt.Errorf("rewrite Wiki page %q returned empty content", kccommon.MetaString(page.Meta, "slug"))
}
pages[idx].Content = content
return nil
@@ -1390,11 +1390,11 @@ func rewriteMergedWikiPages(ctx context.Context, tenant string, pages []kccommon
func floatPtr(value float64) *float64 { return &value }
func wikiPageMergeKey(product kccommon.Product) string {
- slug := strings.TrimSpace(metaString(product.Meta, "slug"))
+ slug := strings.TrimSpace(kccommon.MetaString(product.Meta, "slug"))
if slug == "" {
return ""
}
- pageType := strings.ToLower(strings.TrimSpace(metaString(product.Meta, "page_type")))
+ pageType := strings.ToLower(strings.TrimSpace(kccommon.MetaString(product.Meta, "page_type")))
return pageType + "\x00" + strings.ToLower(strings.Join(strings.Fields(slug), " "))
}
@@ -1643,14 +1643,14 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
}
// Dataset rows retain the producing compilation kind. Extraction shape
// is separate metadata and does not determine the stored kind.
- ckwd := enginetypes.CanonicalCompilationKind(kccommon.FirstNonEmpty(p.Kind, metaString(p.Meta, "compile_kwd")))
+ ckwd := enginetypes.CanonicalCompilationKind(kccommon.FirstNonEmpty(p.Kind, kccommon.MetaString(p.Meta, "compile_kwd")))
if ckwd == "" {
ckwd = compileKwdForVariant(p.Variant)
}
- kind := metaString(p.Meta, "kind")
+ kind := kccommon.MetaString(p.Meta, "kind")
template := structureTemplateIdentity(p.TemplateID, p.Kind)
- from := metaString(p.Meta, "from")
- to := metaString(p.Meta, "to")
+ from := kccommon.MetaString(p.Meta, "from")
+ to := kccommon.MetaString(p.Meta, "to")
if kind == "relation" || (from != "" && to != "") {
from = strings.TrimSpace(from)
to = strings.TrimSpace(to)
@@ -1660,9 +1660,9 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
if from == "" || to == "" {
continue
}
- relType := metaString(p.Meta, "relation_type")
+ relType := kccommon.MetaString(p.Meta, "relation_type")
if relType == "" {
- relType = metaString(p.Meta, "type")
+ relType = kccommon.MetaString(p.Meta, "type")
}
if relType == "" {
relType = "related"
@@ -1677,10 +1677,10 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
continue
}
// entity bucket
- name := metaString(p.Meta, "name")
- typ := metaString(p.Meta, "entity_type")
+ name := kccommon.MetaString(p.Meta, "name")
+ typ := kccommon.MetaString(p.Meta, "entity_type")
if typ == "" {
- typ = metaString(p.Meta, "type")
+ typ = kccommon.MetaString(p.Meta, "type")
}
if name == "" {
continue
@@ -1773,19 +1773,19 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {
// name/type, while their dataset row still needs searchable text.
// Relations likewise use the endpoint names and relation type when the
// source payload has no description.
- from := metaString(p.Meta, "from")
- to := metaString(p.Meta, "to")
- if kind := metaString(p.Meta, "kind"); kind == "relation" || (from != "" && to != "") {
- relType := metaString(p.Meta, "relation_type")
+ from := kccommon.MetaString(p.Meta, "from")
+ to := kccommon.MetaString(p.Meta, "to")
+ if kind := kccommon.MetaString(p.Meta, "kind"); kind == "relation" || (from != "" && to != "") {
+ relType := kccommon.MetaString(p.Meta, "relation_type")
if relType == "" {
- relType = metaString(p.Meta, "type")
+ relType = kccommon.MetaString(p.Meta, "type")
}
if relType == "" {
relType = "related"
}
desc = strings.TrimSpace(strings.Join([]string{from, relType, to}, " "))
} else {
- desc = strings.TrimSpace(metaString(p.Meta, "name"))
+ desc = strings.TrimSpace(kccommon.MetaString(p.Meta, "name"))
}
}
if desc != "" {
@@ -1795,13 +1795,13 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {
b.Description += desc
}
b.SourceDocIDs = common.AppendUnique(b.SourceDocIDs, []string{p.DocID})
- b.SourceChunkIDs = common.AppendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))
+ b.SourceChunkIDs = common.AppendUnique(b.SourceChunkIDs, kccommon.MetaStringSlice(p.Meta, "source_chunk_ids"))
// mention_count_int: sum the per-entity mention counts (Python
// _struct_merge_graph_entities sums mention_count across the merged entities).
// Each entity product carries mention_count ≥ 1 (structure compile.go L231);
// mindmap entities carry none, so they contribute a default of 0 and are
// simply not stamped (mention_count_int is omitted when 0).
- if mc, ok := metaInt(p.Meta, "mention_count"); ok && mc > 0 {
+ if mc, ok := kccommon.MetaInt(p.Meta, "mention_count"); ok && mc > 0 {
b.MentionCount += int(mc)
}
if len(p.Vector) > 0 {
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index 20f523c69f..18a762b31a 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -265,7 +265,7 @@ func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb string
break
}
for _, row := range res.Chunks {
- if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && metaString(product.Meta, "kind") == "page" {
+ if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && kccommon.MetaString(product.Meta, "kind") == "page" {
pages = append(pages, product)
}
}
@@ -315,7 +315,7 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,
break
}
for _, row := range res.Chunks {
- if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && metaString(product.Meta, "kind") == "page" {
+ if product, ok := productFromChunkMap(row, tenant, kccommon.VariantWiki); ok && kccommon.MetaString(product.Meta, "kind") == "page" {
pages = append(pages, product)
}
}
@@ -395,12 +395,12 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
meta["to"] = v
meta["kind"] = "relation"
}
- if metaString(meta, "kind") == "relation" {
+ if kccommon.MetaString(meta, "kind") == "relation" {
if v, ok := payload["type"].(string); ok && v != "" {
meta["relation_type"] = v
}
}
- if v, ok := metaInt(c, "mention_count_int"); ok {
+ if v, ok := kccommon.MetaInt(c, "mention_count_int"); ok {
meta["mention_count"] = v
}
if v, ok := c["slug_kwd"].(string); ok && v != "" {
@@ -422,20 +422,20 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
if v, ok := c["summary_with_weight"].(string); ok && v != "" {
meta["summary"] = v
}
- if v := metaStringSlice(c, "entity_names_kwd"); len(v) > 0 {
+ if v := kccommon.MetaStringSlice(c, "entity_names_kwd"); len(v) > 0 {
meta["entity_names"] = v
}
- if v := metaStringSlice(c, "related_kb_pages_kwd"); len(v) > 0 {
+ if v := kccommon.MetaStringSlice(c, "related_kb_pages_kwd"); len(v) > 0 {
meta["related_kb_pages"] = v
}
- if v := metaStringSlice(c, "outlinks_kwd"); len(v) > 0 {
+ if v := kccommon.MetaStringSlice(c, "outlinks_kwd"); len(v) > 0 {
meta["outlinks"] = v
}
// Section depth comes from depth_int. Only the wiki variant consumes
// Meta["section_level"] (applyVariantColumns); tree rows also stamp depth_int,
// but they keep their depth in raptor_layer_int.
if expect == kccommon.VariantWiki {
- if v, ok := metaInt(c, "depth_int"); ok {
+ if v, ok := kccommon.MetaInt(c, "depth_int"); ok {
meta["section_level"] = v
}
}
@@ -478,10 +478,10 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
if v, ok := c["create_time"].(string); ok && v != "" {
meta["created_at"] = v
}
- if v := metaStringSlice(c, "source_chunk_ids"); len(v) > 0 {
+ if v := kccommon.MetaStringSlice(c, "source_chunk_ids"); len(v) > 0 {
meta["source_chunk_ids"] = v
}
- if v := metaStringSlice(c, "source_doc_ids"); len(v) > 0 {
+ if v := kccommon.MetaStringSlice(c, "source_doc_ids"); len(v) > 0 {
meta["source_doc_ids"] = v
}
@@ -494,7 +494,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
// more than one the first is used (multi-template rows are a config error
// surfaced elsewhere).
templateID := ""
- if ids := metaStringSlice(c, "compilation_template_ids"); len(ids) > 0 {
+ if ids := kccommon.MetaStringSlice(c, "compilation_template_ids"); len(ids) > 0 {
templateID = ids[0]
}
return kccommon.Product{
diff --git a/internal/ingestion/knowledge_compile/topic_merge.go b/internal/ingestion/knowledge_compile/topic_merge.go
index 1638a2be3d..0ef90dceab 100644
--- a/internal/ingestion/knowledge_compile/topic_merge.go
+++ b/internal/ingestion/knowledge_compile/topic_merge.go
@@ -38,26 +38,26 @@ func (x *llmDeduper) RouteTopic(ctx context.Context, incoming, existing kccommon
}
func isTopicPage(product kccommon.Product) bool {
- return strings.EqualFold(strings.TrimSpace(metaString(product.Meta, "page_type")), "topic")
+ return strings.EqualFold(strings.TrimSpace(kccommon.MetaString(product.Meta, "page_type")), "topic")
}
func productTopic(product kccommon.Product) string {
- return kccommon.NormalizeWikiTopicPath(metaString(product.Meta, "topic"))
+ return kccommon.NormalizeWikiTopicPath(kccommon.MetaString(product.Meta, "topic"))
}
func productTitle(product kccommon.Product) string {
- return strings.TrimSpace(metaString(product.Meta, "title"))
+ return strings.TrimSpace(kccommon.MetaString(product.Meta, "title"))
}
func productSummary(product kccommon.Product) string {
- if summary := strings.TrimSpace(metaString(product.Meta, "summary")); summary != "" {
+ if summary := strings.TrimSpace(kccommon.MetaString(product.Meta, "summary")); summary != "" {
return summary
}
return strings.TrimSpace(product.Content)
}
func productEntities(product kccommon.Product) []string {
- return metaStringSlice(product.Meta, "entity_names")
+ return kccommon.MetaStringSlice(product.Meta, "entity_names")
}
func firstTopicString(values ...string) string {
@@ -152,11 +152,11 @@ func mergeTopicProducts(tenant, kb string, products []kccommon.Product) ([]kccom
func mergeTopicPage(existing, incoming kccommon.Product) kccommon.Product {
merged := wikiEntityMerge(existing, incoming)
merged.Meta = copyMeta(merged.Meta)
- merged.Meta["slug"] = metaString(existing.Meta, "slug")
+ merged.Meta["slug"] = kccommon.MetaString(existing.Meta, "slug")
merged.Meta["page_type"] = "topic"
- topic := kccommon.NormalizeWikiTopicPath(firstTopicString(metaString(incoming.Meta, "topic"), metaString(existing.Meta, "topic")))
+ topic := kccommon.NormalizeWikiTopicPath(firstTopicString(kccommon.MetaString(incoming.Meta, "topic"), kccommon.MetaString(existing.Meta, "topic")))
merged.Meta["topic"] = topic
- merged.Meta["title"] = firstTopicString(metaString(existing.Meta, "title"), metaString(incoming.Meta, "title"), topic)
- merged.Meta["entity_names"] = unionStrs(metaStringSlice(existing.Meta, "entity_names"), metaStringSlice(incoming.Meta, "entity_names"))
+ merged.Meta["title"] = firstTopicString(kccommon.MetaString(existing.Meta, "title"), kccommon.MetaString(incoming.Meta, "title"), topic)
+ merged.Meta["entity_names"] = unionStrs(kccommon.MetaStringSlice(existing.Meta, "entity_names"), kccommon.MetaStringSlice(incoming.Meta, "entity_names"))
return merged
}
diff --git a/internal/ingestion/knowledge_compile/topic_merge_test.go b/internal/ingestion/knowledge_compile/topic_merge_test.go
index a886f5a1dc..1e2264a71c 100644
--- a/internal/ingestion/knowledge_compile/topic_merge_test.go
+++ b/internal/ingestion/knowledge_compile/topic_merge_test.go
@@ -36,13 +36,13 @@ func TestMergeTopicProductsPreservesEvidenceAndStableTopicIdentity(t *testing.T)
if !strings.Contains(merged[0].Content, "first") || !strings.Contains(merged[0].Content, "second") {
t.Fatalf("merged content lost evidence: %q", merged[0].Content)
}
- if got := metaString(merged[0].Meta, "slug"); got != "topic/history-a" {
+ if got := kccommon.MetaString(merged[0].Meta, "slug"); got != "topic/history-a" {
t.Fatalf("topic slug = %q, want existing slug", got)
}
- if got := metaString(merged[0].Meta, "title"); got != "History" {
+ if got := kccommon.MetaString(merged[0].Meta, "title"); got != "History" {
t.Fatalf("topic title = %q, want existing title", got)
}
- if got := metaStringSlice(merged[0].Meta, "entity_names"); len(got) != 2 {
+ if got := kccommon.MetaStringSlice(merged[0].Meta, "entity_names"); len(got) != 2 {
t.Fatalf("entity names = %v, want both entities", got)
}
}
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.go
index a65104ea48..2f29a53e49 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go
@@ -199,7 +199,7 @@ func (s *memoryWikiContributionStore) ClearDataset(_ context.Context, _, dataset
func buildWikiDocumentContribution(documentID string, products []kccommon.Product) wikiDocumentContribution {
pagesByKey := make(map[string]wikiContributionPage)
for _, product := range products {
- if product.Variant != kccommon.VariantWiki || metaString(product.Meta, "kind") != "page" {
+ if product.Variant != kccommon.VariantWiki || kccommon.MetaString(product.Meta, "kind") != "page" {
continue
}
key := wikiPageMergeKey(product)
@@ -208,8 +208,8 @@ func buildWikiDocumentContribution(documentID string, products []kccommon.Produc
}
pagesByKey[key] = wikiContributionPage{
Key: key,
- Slug: strings.TrimSpace(metaString(product.Meta, "slug")),
- PageType: strings.ToLower(strings.TrimSpace(metaString(product.Meta, "page_type"))),
+ Slug: strings.TrimSpace(kccommon.MetaString(product.Meta, "slug")),
+ PageType: strings.ToLower(strings.TrimSpace(kccommon.MetaString(product.Meta, "page_type"))),
Signature: wikiContributionSignature(product),
}
}
@@ -270,13 +270,13 @@ func wikiContributionSignature(product kccommon.Product) string {
SourceChunkIDs []string
}{
Content: strings.TrimSpace(product.Content),
- Title: strings.TrimSpace(metaString(product.Meta, "title")),
- Topic: kccommon.NormalizeWikiTopicPath(metaString(product.Meta, "topic")),
- Summary: strings.TrimSpace(metaString(product.Meta, "summary")),
- EntityNames: common.DeduplicateSorted(metaStringSlice(product.Meta, "entity_names")),
- RelatedKBPages: common.DeduplicateSorted(metaStringSlice(product.Meta, "related_kb_pages")),
- Outlinks: common.DeduplicateSorted(metaStringSlice(product.Meta, "outlinks")),
- SourceChunkIDs: common.DeduplicateSorted(metaStringSlice(product.Meta, "source_chunk_ids")),
+ Title: strings.TrimSpace(kccommon.MetaString(product.Meta, "title")),
+ Topic: kccommon.NormalizeWikiTopicPath(kccommon.MetaString(product.Meta, "topic")),
+ Summary: strings.TrimSpace(kccommon.MetaString(product.Meta, "summary")),
+ EntityNames: common.DeduplicateSorted(kccommon.MetaStringSlice(product.Meta, "entity_names")),
+ RelatedKBPages: common.DeduplicateSorted(kccommon.MetaStringSlice(product.Meta, "related_kb_pages")),
+ Outlinks: common.DeduplicateSorted(kccommon.MetaStringSlice(product.Meta, "outlinks")),
+ SourceChunkIDs: common.DeduplicateSorted(kccommon.MetaStringSlice(product.Meta, "source_chunk_ids")),
}
payload, _ := json.Marshal(normalized)
sum := sha256.Sum256(payload)
@@ -352,7 +352,7 @@ func (c *Consumer) prepareWikiContributionDiff(ctx context.Context, tenantID, da
}
legacyProducts := make(map[string][]kccommon.Product, len(missingPrevious))
for _, page := range mergedPages {
- for _, docID := range metaStringSlice(page.Meta, "source_doc_ids") {
+ for _, docID := range kccommon.MetaStringSlice(page.Meta, "source_doc_ids") {
if _, wanted := missingPrevious[docID]; wanted {
legacyProducts[docID] = append(legacyProducts[docID], page)
}
diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.go
index b5a76d220d..0bdfc9d48d 100644
--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go
@@ -21,6 +21,8 @@ import (
"errors"
"strings"
"testing"
+
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
// page builds a wikiPageProjection with the full "<page_type>/<slug>" identity.
@@ -69,7 +71,7 @@ func TestProjectWikiGraphRowsWritesSearchContent(t *testing.T) {
var payload struct {
Description string `json:"description"`
}
- if err := json.Unmarshal([]byte(metaString(entity, "content_with_weight")), &payload); err != nil {
+ if err := json.Unmarshal([]byte(kccommon.MetaString(entity, "content_with_weight")), &payload); err != nil {
t.Fatalf("decode graph entity: %v", err)
}
if payload.Description != p.Summary {
diff --git a/internal/ingestion/knowledge_compile/wiki_merge.go b/internal/ingestion/knowledge_compile/wiki_merge.go
index d6ab34ff48..8e0115df2d 100644
--- a/internal/ingestion/knowledge_compile/wiki_merge.go
+++ b/internal/ingestion/knowledge_compile/wiki_merge.go
@@ -66,7 +66,7 @@ func selectMergedWikiTopicPath(products []kccommon.Product) string {
byKey[key] = current
}
current.products++
- docIDs := metaStringSliceAny(product.Meta, "source_doc_ids")
+ docIDs := kccommon.MetaStringSlice(product.Meta, "source_doc_ids")
if len(docIDs) == 0 && product.DocID != "" {
docIDs = []string{product.DocID}
}
@@ -75,7 +75,7 @@ func selectMergedWikiTopicPath(products []kccommon.Product) string {
current.docs[docID] = struct{}{}
}
}
- for _, chunkID := range metaStringSliceAny(product.Meta, "source_chunk_ids") {
+ for _, chunkID := range kccommon.MetaStringSlice(product.Meta, "source_chunk_ids") {
if chunkID = strings.TrimSpace(chunkID); chunkID != "" {
current.chunks[chunkID] = struct{}{}
}
@@ -180,8 +180,8 @@ func unionWikiProvenance(a, b map[string]any) map[string]any {
out[key] = v
}
}
- out["source_doc_ids"] = unionStrs(metaStringSliceAny(a, "source_doc_ids"), metaStringSliceAny(b, "source_doc_ids"))
- out["source_chunk_ids"] = unionStrs(metaStringSliceAny(a, "source_chunk_ids"), metaStringSliceAny(b, "source_chunk_ids"))
+ out["source_doc_ids"] = unionStrs(kccommon.MetaStringSlice(a, "source_doc_ids"), kccommon.MetaStringSlice(b, "source_doc_ids"))
+ out["source_chunk_ids"] = unionStrs(kccommon.MetaStringSlice(a, "source_chunk_ids"), kccommon.MetaStringSlice(b, "source_chunk_ids"))
if v, ok := b["entity_names"]; ok {
out["entity_names"] = v
}
@@ -213,24 +213,6 @@ func unionStrs(a, b []string) []string {
return out
}
-// metaStringSliceAny reads a []string from a meta map that may box the value as
-// []string or []any (engine/JSON round-trip does not guarantee a single type).
-func metaStringSliceAny(m map[string]any, key string) []string {
- switch v := m[key].(type) {
- case []string:
- return v
- case []any:
- out := make([]string, 0, len(v))
- for _, e := range v {
- if s, ok := e.(string); ok {
- out = append(out, s)
- }
- }
- return out
- }
- return nil
-}
-
// isWikiGroup reports whether a merge group targets the wiki variant and needs
// the page-specific Markdown merge instead of the generic JSON decider.
func isWikiGroup(g MergeGroup) bool {
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index f093e317f0..cd6f8d161a 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -292,9 +292,9 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
if len(chunks) > 0 {
common.Info("knowledge_compile: WriteMerged chunk sample",
zap.String("kb_id", kb),
- zap.String("compile_kwd", metaString(chunks[0], "compile_kwd")),
+ zap.String("compile_kwd", kccommon.MetaString(chunks[0], "compile_kwd")),
zap.String("variant", string(batch[0].Variant)),
- zap.String("id", metaString(chunks[0], "id")),
+ zap.String("id", kccommon.MetaString(chunks[0], "id")),
)
}
_, err := eng.InsertChunks(ctx, chunks, baseName, kb, "")
@@ -432,12 +432,12 @@ func (w engineWriter) recordWikiPageCommits(ctx context.Context, kb string, prod
}
func wikiPageCommitTargetForProduct(product kccommon.Product) (wikiPageCommitTarget, bool) {
- if product.Variant != kccommon.VariantWiki || strings.ToLower(strings.TrimSpace(metaString(product.Meta, "kind"))) != "page" {
+ if product.Variant != kccommon.VariantWiki || strings.ToLower(strings.TrimSpace(kccommon.MetaString(product.Meta, "kind"))) != "page" {
return wikiPageCommitTarget{}, false
}
return wikiPageCommitTargetForRow(
- metaString(product.Meta, "page_type"),
- metaString(product.Meta, "slug"),
+ kccommon.MetaString(product.Meta, "page_type"),
+ kccommon.MetaString(product.Meta, "slug"),
)
}
@@ -965,8 +965,8 @@ func f32ToF64Slice(v []float32) []float64 {
// (input_hash_kwd), and now stamps both wall-clock audit fields. We deliberately
// do not write the full PLAN JSON into the engine.
func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommon.Product) map[string]interface{} {
- srcDocIDs := metaStringSlice(p.Meta, "source_doc_ids")
- srcChunkIDs := metaStringSlice(p.Meta, "source_chunk_ids")
+ srcDocIDs := kccommon.MetaStringSlice(p.Meta, "source_doc_ids")
+ srcChunkIDs := kccommon.MetaStringSlice(p.Meta, "source_chunk_ids")
m := map[string]interface{}{
"id": datasetLevelID(tenant, kb, p),
"doc_id": kb,
@@ -987,7 +987,7 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
"create_timestamp_flt": float64(now.Unix()),
}
if p.Variant == kccommon.VariantWiki {
- m["type_kwd"] = "wiki_" + kccommon.FirstNonEmpty(metaString(p.Meta, "kind"), "page")
+ m["type_kwd"] = "wiki_" + kccommon.FirstNonEmpty(kccommon.MetaString(p.Meta, "kind"), "page")
}
// wiki_incremental port: preserve the original creation timestamp across a
// page merge. If the incoming merged product already carries
@@ -1005,8 +1005,8 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
// slug_kwd follows the Python writer contract (api/db/db_models.py): it is
// stored as the full "<page_type>/<slug>" form so GetWikiPage's filter
// (page_type + "/" + slug) matches directly.
- pageType := metaString(p.Meta, "page_type")
- if slug := metaString(p.Meta, "slug"); slug != "" {
+ pageType := kccommon.MetaString(p.Meta, "page_type")
+ if slug := kccommon.MetaString(p.Meta, "slug"); slug != "" {
// Normalize to the full "<page_type>/<slug>" form (Python writer
// contract). Idempotent: slugs that already carry the prefix are kept.
fullSlug := slug
@@ -1015,31 +1015,31 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
}
m["slug_kwd"] = fullSlug
}
- if v := metaString(p.Meta, "title"); v != "" {
+ if v := kccommon.MetaString(p.Meta, "title"); v != "" {
m["title_kwd"] = v
}
if pageType != "" {
m["entity_type_kwd"] = pageType
}
- if v := kccommon.NormalizeWikiTopicPath(metaString(p.Meta, "topic")); v != "" {
+ if v := kccommon.NormalizeWikiTopicPath(kccommon.MetaString(p.Meta, "topic")); v != "" {
m["topic_kwd"] = v
}
- if v := metaString(p.Meta, "plan_group"); v != "" {
+ if v := kccommon.MetaString(p.Meta, "plan_group"); v != "" {
m["plan_group_kwd"] = v
}
- if v := metaString(p.Meta, "generation"); v != "" {
+ if v := kccommon.MetaString(p.Meta, "generation"); v != "" {
m["generation_kwd"] = v
}
- if v := metaString(p.Meta, "summary"); v != "" {
+ if v := kccommon.MetaString(p.Meta, "summary"); v != "" {
m["summary_with_weight"] = v
}
- if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {
+ if v := kccommon.MetaStringSlice(p.Meta, "entity_names"); len(v) > 0 {
m["entity_names_kwd"] = v
}
- if v := metaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {
+ if v := kccommon.MetaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {
m["related_kb_pages_kwd"] = v
}
- if v := metaStringSlice(p.Meta, "outlinks"); len(v) > 0 {
+ if v := kccommon.MetaStringSlice(p.Meta, "outlinks"); len(v) > 0 {
m["outlinks_kwd"] = v
}
// Persist the merged product's embedding under the dimension-suffixed column
@@ -1123,7 +1123,7 @@ func (w engineWriter) StripMergedSources(ctx context.Context, tenant, kb string,
if id == "" {
continue
}
- src := metaStringSlice(c, "source_doc_ids")
+ src := kccommon.MetaStringSlice(c, "source_doc_ids")
kept := make([]string, 0, len(src))
changed := false
for _, d := range src {
@@ -1204,7 +1204,7 @@ func mergedInputHash(products []kccommon.Product) string {
seen := make(map[string]struct{}, 16)
docs := make([]string, 0, 16)
for _, p := range products {
- for _, d := range metaStringSlice(p.Meta, "source_doc_ids") {
+ for _, d := range kccommon.MetaStringSlice(p.Meta, "source_doc_ids") {
if _, ok := seen[d]; ok {
continue
}
@@ -1216,57 +1216,6 @@ func mergedInputHash(products []kccommon.Product) string {
return hashStr(strings.Join(docs, "\n"))
}
-// metaString extracts a string from a map value, tolerating a missing or
-// non-string entry.
-func metaString(m map[string]any, key string) string {
- if v, ok := m[key].(string); ok {
- return v
- }
- return ""
-}
-
-// firstStringSlice extracts a []string from an engine row value, tolerating both
-// []string and []any forms; it returns nil when the value is not a string slice.
-func firstStringSlice(v any) []string {
- switch s := v.(type) {
- case []string:
- return s
- case []any:
- out := make([]string, 0, len(s))
- for _, x := range s {
- if str, ok := x.(string); ok {
- out = append(out, str)
- }
- }
- return out
- }
- return nil
-}
-
-func metaStringSlice(m map[string]any, key string) []string {
- return firstStringSlice(m[key])
-}
-
-// metaInt extracts an integer from a map value that may be boxed as float64
-// (JSON number), int64, string, or a typed int — the engine/JSON round-trip does
-// not guarantee a single numeric type.
-func metaInt(m map[string]any, key string) (int64, bool) {
- switch v := m[key].(type) {
- case int64:
- return v, true
- case int:
- return int64(v), true
- case float64:
- return int64(v), true
- case string:
- var n int64
- if _, err := fmt.Sscanf(v, "%d", &n); err == nil {
- return n, true
- }
- }
- return 0, false
-}
-
// metaFloat extracts a float64 from a map value that may be boxed as float64,
// int64, int, or string — the engine/JSON round-trip does not guarantee a
// single numeric type. Used to recover create_timestamp_flt so the reader can
@@ -1288,6 +1237,24 @@ func metaFloat(m map[string]any, key string) (float64, bool) {
return 0, false
}
+// firstStringSlice extracts a []string from an engine row value, tolerating both
+// []string and []any forms; it returns nil when the value is not a string slice.
+func firstStringSlice(v any) []string {
+ switch s := v.(type) {
+ case []string:
+ return s
+ case []any:
+ out := make([]string, 0, len(s))
+ for _, x := range s {
+ if str, ok := x.(string); ok {
+ out = append(out, str)
+ }
+ }
+ return out
+ }
+ return nil
+}
+
// KwdToVariant maps a stored compilation kind to its execution variant.
func KwdToVariant(kwd string) (kccommon.Variant, error) {
switch kwd {
@@ -1477,7 +1444,7 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, k
break
}
for _, row := range result.Chunks {
- slug := metaString(row, "slug_kwd")
+ slug := kccommon.MetaString(row, "slug_kwd")
if slug == "" {
continue
}
@@ -1485,13 +1452,13 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, k
if page.Slug == "" {
page.Slug = slug
page.PageType = types.WikiPageCategory(row)
- page.Title = metaString(row, "title_kwd")
- page.Summary = metaString(row, "summary_with_weight")
+ page.Title = kccommon.MetaString(row, "title_kwd")
+ page.Summary = kccommon.MetaString(row, "summary_with_weight")
}
- page.Aliases = unionStrs(page.Aliases, metaStringSlice(row, "entity_names_kwd"))
- page.Outlinks = unionStrs(page.Outlinks, metaStringSlice(row, "outlinks_kwd"))
- page.SourceDocIDs = unionStrs(page.SourceDocIDs, metaStringSlice(row, "source_doc_ids"))
- page.SourceChunkIDs = unionStrs(page.SourceChunkIDs, metaStringSlice(row, "source_chunk_ids"))
+ page.Aliases = unionStrs(page.Aliases, kccommon.MetaStringSlice(row, "entity_names_kwd"))
+ page.Outlinks = unionStrs(page.Outlinks, kccommon.MetaStringSlice(row, "outlinks_kwd"))
+ page.SourceDocIDs = unionStrs(page.SourceDocIDs, kccommon.MetaStringSlice(row, "source_doc_ids"))
+ page.SourceChunkIDs = unionStrs(page.SourceChunkIDs, kccommon.MetaStringSlice(row, "source_chunk_ids"))
bySlug[slug] = page
}
if len(result.Chunks) < batchSize {
@@ -1673,21 +1640,21 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string
// Filter map is being ignored and wiki_entity rows leak into the page
// projection (which then has nothing under compile_kwd=wiki_page).
for _, c := range res.Chunks {
- kwdSeen[metaString(c, "compile_kwd")]++
+ kwdSeen[kccommon.MetaString(c, "compile_kwd")]++
}
if len(res.Chunks) == 0 {
break
}
for _, c := range res.Chunks {
p := wikiPageProjection{
- Slug: metaString(c, "slug_kwd"),
+ Slug: kccommon.MetaString(c, "slug_kwd"),
PageType: types.WikiPageCategory(c),
- Title: metaString(c, "title_kwd"),
- Aliases: metaStringSlice(c, "entity_names_kwd"),
- Summary: metaString(c, "summary_with_weight"),
- Outlinks: metaStringSlice(c, "outlinks_kwd"),
- SourceDocIDs: metaStringSlice(c, "source_doc_ids"),
- SourceChunkIDs: metaStringSlice(c, "source_chunk_ids"),
+ Title: kccommon.MetaString(c, "title_kwd"),
+ Aliases: kccommon.MetaStringSlice(c, "entity_names_kwd"),
+ Summary: kccommon.MetaString(c, "summary_with_weight"),
+ Outlinks: kccommon.MetaStringSlice(c, "outlinks_kwd"),
+ SourceDocIDs: kccommon.MetaStringSlice(c, "source_doc_ids"),
+ SourceChunkIDs: kccommon.MetaStringSlice(c, "source_chunk_ids"),
}
out = append(out, p)
}