RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(infinity,nav): restore the navigation tree and document structure graph (#20339)
qinling02104d1fa96a8d9e
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 00fa7865ea..dae7635a7b 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -431,6 +431,14 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface// Insert chunks to dataset_, err = table.Insert(insertChunks)if err != nil {+ common.Warn("Infinity insert failed",+ zap.String("tableName", tableName),+ zap.Int("rows", len(insertChunks)),+ zap.Strings("rowFields", rowFieldNames(insertChunks)),+ zap.Error(err))+ if isConnectionLevelError(err) {+ e.client.dropConnectionFor(db, "InsertChunks")+ }return nil, fmt.Errorf("failed to insert chunks to dataset: %w", err)}@@ -592,6 +600,13 @@ func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interfaccommon.Info(fmt.Sprintf("INFINITY update: table=%s, filter=%s, newValue=%v", tableName, filter, newValue))_, err = table.Update(filter, newValue)if err != nil {+ common.Warn("Infinity update failed",+ zap.String("tableName", tableName),+ zap.String("filter", filter),+ zap.Error(err))+ if isConnectionLevelError(err) {+ e.client.dropConnectionFor(db, "UpdateChunks")+ }return fmt.Errorf("failed to update chunks: %w", err)}@@ -746,6 +761,13 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfacdelResp, err := table.Delete(filter)if err != nil {+ common.Warn("Infinity delete failed",+ zap.String("tableName", tableName),+ zap.String("filter", filter),+ zap.Error(err))+ if isConnectionLevelError(err) {+ e.client.dropConnectionFor(db, "DeleteChunks")+ }return 0, fmt.Errorf("failed to delete: %w", err)}@@ -839,6 +861,14 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S}}+ // Infinity binds a "_score" projection to SCORE(), which it allows only with+ // MATCH TEXT/TENSOR or Fusion — otherwise 3013. Elasticsearch tolerates+ // "_score" for every shape, so callers pass it unconditionally (the nav+ // cluster descent and navScan do): drop theirs and re-add the one form this+ // shape accepts. calculateScores() materializes "_score" for readers either+ // way.+ outputColumns = stripScorePseudoColumns(outputColumns)+if hasTextMatch || hasVectorMatch {if hasTextMatch {outputColumns = append(outputColumns, "score()")@@ -864,23 +894,6 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SoutputColumns = append(outputColumns, "row_id()")}- // Strip score pseudo-columns when there's no match expression — Infinity- // rejects SCORE()/SCORE_FACTORS() without MATCH TEXT/TENSOR/Fusion with- // "InfinityException(3013)". This protects callers (e.g. the no-match- // fallback in retrieval.go) that reuse a SelectFields list containing- // "_score" across both matched and unmatched queries.- if !hasTextMatch && !hasVectorMatch {- filtered := outputColumns[:0]- for _, c := range outputColumns {- switch c {- case "_score", "SCORE", "score()", "similarity()":- continue- }- filtered = append(filtered, c)- }- outputColumns = filtered- }-outputColumns = convertSelectFields(outputColumns, isSkillIndex)if hasVectorMatch && matchDense != nil && matchDense.VectorColumnName != "" {outputColumns = append(outputColumns, matchDense.VectorColumnName)@@ -930,6 +943,13 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.Svar allResults []map[string]interface{}totalHits := int64(0)+ // queryErr keeps the first per-table failure. A failed table query used to be+ // swallowed (warn + continue), which turned schema/query mismatches into+ // silently empty results — a partial-wildcard projection ("q_*_vec", 3013)+ // made the knowledge-compile reader see no products, so the dataset+ // navigation tree silently never materialized. Failures stay tolerable while+ // another table answered, but surface when nothing did.+ var queryErr errorfor _, indexName := range req.IndexNames {var tableNames []string@@ -989,28 +1009,50 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.Sfor _, tableName := range tableNames {tbl, err := db.GetTable(tableName)if err != nil {+ // A dataset whose table does not exist yet is normal (tables are+ // created on first write): skip it. A transport failure leaves the+ // socket unusable, so it is recorded like the failures below.+ if isConnectionLevelError(err) {+ e.client.dropConnectionFor(db, "Search")+ if queryErr == nil {+ queryErr = fmt.Errorf("open table %s: %w", tableName, err)+ }+ }continue}- tableFilterParts := append([]string(nil), filterParts...)- if len(filterCopy) > 0 {- var tableColumns map[string]struct {- Type string- Default interface{}- }- if hasJSONListFilter(filterCopy) {- tableColumns, err = loadTableColumns(tbl)- if err != nil {- common.Warn("Failed to load Infinity columns for JSON-list filter",- zap.String("tableName", tableName), zap.Error(err))- continue+ // Resolve partial-wildcard projections ("q_*_vec") against this table's+ // real columns: Infinity parses "q_*_vec" as the arithmetic expression+ // "q_" * "_vec" and fails to bind it (3013), while Elasticsearch expands+ // the pattern in _source. Embedding dims differ per table, so this is+ // decided per table. A bare "*" is left alone: Infinity supports it.+ wildcardSelect := hasPartialWildcard(outputColumns)+ var tableColumns map[string]struct {+ Type string+ Default interface{}+ }+ if wildcardSelect || (len(filterCopy) > 0 && hasJSONListFilter(filterCopy)) {+ tableColumns, err = loadTableColumns(tbl)+ if err != nil {+ common.Warn("Failed to load Infinity columns for search",+ zap.String("tableName", tableName), zap.Error(err))+ if queryErr == nil {+ queryErr = fmt.Errorf("load columns for %s: %w", tableName, err)}+ continue}+ }+ tableOutputColumns := outputColumns+ if wildcardSelect {+ tableOutputColumns = expandSelectWildcards(outputColumns, tableColumns)+ }+ tableFilterParts := append([]string(nil), filterParts...)+ if len(filterCopy) > 0 {if condition := equivalentConditionToStr(filterCopy, tableColumns); condition != "" {tableFilterParts = append(tableFilterParts, condition)}}filterStr := strings.Join(tableFilterParts, " AND ")- table := tbl.Output(outputColumns)+ table := tbl.Output(tableOutputColumns)var textFields []stringif matchText != nil && len(matchText.Fields) > 0 {@@ -1200,12 +1242,25 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S// Execute querydf, err := table.ToDataFrame()if err != nil {+ // Log the full request, not just its shape: a failing search used+ // to be indistinguishable from any other, which made a recurring+ // 7018/EOF impossible to attribute to a caller.common.Warn("Infinity query failed",zap.String("tableName", tableName),+ zap.String("filter", filterStr),+ zap.Strings("selectFields", tableOutputColumns),+ zap.Int("limit", pageSize),+ zap.Int("offset", offset),zap.Bool("hasTextMatch", hasTextMatch),zap.Bool("hasVectorMatch", hasVectorMatch),zap.Bool("hasFusion", fusionExpr != nil),zap.Error(err))+ if isConnectionLevelError(err) {+ e.client.dropConnectionFor(db, "Search")+ }+ if queryErr == nil {+ queryErr = fmt.Errorf("query table %s: %w", tableName, err)+ }continue}@@ -1213,6 +1268,17 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SsearchChunks := make([]map[string]interface{}, 0)for colName, colData := range df.ColumnData {for i, val := range colData {+ // A null cell must not materialize a row of its own. Infinity+ // can return a single all-null cell for a projection it+ // matched nothing for (observed with a full-text filter plus a+ // json-list projection: source_chunk_ids came back as a nil+ // slice while the other columns were empty), and fabricating+ // one empty row out of it leaks phantom documents to callers —+ // GetWikiGraph then read an empty slug_kwd off it and issued+ // filter_fulltext('from_kwd', ''), which Infinity rejects.+ if isEmptyEngineCell(val) {+ continue+ }for len(searchChunks) <= i {searchChunks = append(searchChunks, make(map[string]interface{}))}@@ -1220,6 +1286,12 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S}}+ // Infinity returns a Json/Array column as ONE cell holding one+ // length-prefixed segment per row, so the loop above parks the whole+ // column on row 0. Redistribute it before applyFieldMappings renames+ // anything.+ realignJSONColumnCells(searchChunks, df.ColumnData)+// Apply field name mapping and row_id handling// Skill index uses different schema// so we skip the document-specific field mappings@@ -1289,6 +1361,12 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SallResults = allResults[:pageSize]}+ // Every table failed: return the error instead of an empty result set, so a+ // backend-specific query mismatch cannot masquerade as "no data".+ if len(allResults) == 0 && totalHits == 0 && queryErr != nil {+ return nil, fmt.Errorf("infinity search failed on all tables: %w", queryErr)+ }+common.Debug("Search in Infinity completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits))return &types.SearchResult{@@ -2180,6 +2258,138 @@ func convertSelectFields(output []string, isSkillIndex ...bool) []string {return result}+// rowFieldNames returns the sorted field names of the first row, so a failed+// insert logs which columns were sent without dumping values (a vector column+// would make the log unusable).+func rowFieldNames(rows []map[string]interface{}) []string {+ if len(rows) == 0 {+ return nil+ }+ names := make([]string, 0, len(rows[0]))+ for k := range rows[0] {+ names = append(names, k)+ }+ sort.Strings(names)+ return names+}++// isEmptyEngineCell reports whether a cell Infinity returned carries no value:+// an untyped null, or a nil slice/map/pointer. Infinity can hand back such a cell+// for a projection it found no row for, so it must not turn into a row by itself.+func isEmptyEngineCell(val interface{}) bool {+ if val == nil {+ return true+ }+ rv := reflect.ValueOf(val)+ switch rv.Kind() {+ case reflect.Slice, reflect.Map, reflect.Ptr, reflect.Interface:+ return rv.IsNil()+ }+ return false+}++// scorePseudoColumns are the score columns a caller may put in a SelectFields+// list. Elasticsearch returns "_score" for every query shape, so callers pass it+// unconditionally; Infinity only accepts the SCORE()/SIMILARITY() function form,+// and only together with a compatible MATCH.+var scorePseudoColumns = map[string]bool{+ "_score": true,+ "SCORE": true,+ "score()": true,+ "similarity()": true,+ "SIMILARITY": true,+}++// stripScorePseudoColumns removes caller-supplied score columns so Search can+// add back the single form Infinity accepts for the query shape it is about to+// run (none for a filter-only query, similarity() for MATCH VECTOR only,+// score() when a MATCH TEXT/TENSOR or Fusion is present).+func stripScorePseudoColumns(fields []string) []string {+ out := make([]string, 0, len(fields))+ for _, f := range fields {+ if scorePseudoColumns[f] {+ continue+ }+ out = append(out, f)+ }+ return out+}++// hasPartialWildcard reports whether any requested column contains a wildcard+// without being the bare "*" Infinity reads as "all columns".+func hasPartialWildcard(fields []string) bool {+ for _, f := range fields {+ if f != "*" && strings.Contains(f, "*") {+ return true+ }+ }+ return false+}++// expandSelectWildcards replaces partial wildcards ("q_*_vec" — the only+// pattern RAGFlow uses) with the table columns they match. Infinity cannot+// project a partial wildcard: it parses "q_*_vec" as the arithmetic expression+// "q_" * "_vec" and fails to bind it ("Fail to bind the expression: q_", 3013),+// whereas Elasticsearch expands the same pattern in _source. Embedding dims+// differ per dataset table, so the expansion is resolved per table. A pattern+// matching no column is dropped, so one unmatched pattern cannot fail the query.+func expandSelectWildcards(fields []string, columns map[string]struct {+ Type string+ Default interface{}+}) []string {+ names := make([]string, 0, len(columns))+ for name := range columns {+ names = append(names, name)+ }+ sort.Strings(names)++ out := make([]string, 0, len(fields))+ seen := make(map[string]bool, len(fields))+ appendField := func(f string) {+ if f == "" || seen[f] {+ return+ }+ seen[f] = true+ out = append(out, f)+ }+ for _, f := range fields {+ if f == "*" || !strings.Contains(f, "*") {+ appendField(f)+ continue+ }+ for _, name := range names {+ if wildcardMatchColumn(f, name) {+ appendField(name)+ }+ }+ }+ return out+}++// wildcardMatchColumn matches a column name against a "*"-wildcard pattern,+// where "*" stands for any run of characters.+func wildcardMatchColumn(pattern, name string) bool {+ parts := strings.Split(pattern, "*")+ if len(parts) == 1 {+ return pattern == name+ }+ if !strings.HasPrefix(name, parts[0]) {+ return false+ }+ rest := name[len(parts[0]):]+ for _, part := range parts[1 : len(parts)-1] {+ if part == "" {+ continue+ }+ idx := strings.Index(rest, part)+ if idx < 0 {+ return false+ }+ rest = rest[idx+len(part):]+ }+ return strings.HasSuffix(rest, parts[len(parts)-1])+}+// convertMatchingField converts field names for matching// For regular document indices: maps _tks/_kwd fields to column@index_name format// For skill indices: maps raw field names to column@index_name format@@ -2333,46 +2543,51 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map// JSON-list fields use member containment. Legacy varchar columns keep// the old full-text behavior.if fieldJSONList(k) && tableColumns != nil {- jsonConditions := jsonListFilterConditions(k, v, tableColumns)- if len(jsonConditions) > 0 {- cond = append(cond, "("+strings.Join(jsonConditions, " OR ")+")")+ if jsonConditions := jsonListFilterConditions(k, v, tableColumns); len(jsonConditions) > 0 {+ cond = append(cond, joinBalanced(jsonConditions, " OR "))}continue}- // Handle keyword fields (using full-text filter)+ // Handle keyword fields (exact match, or full-text for the single-token+ // values Infinity can tokenize — see keywordFilterCondition)if fieldKeyword(k) {- // For keyword fields, values are always treated as strings for filter_fulltext+ // For keyword fields, values are always treated as strings. Blank+ // entries are dropped: Infinity rejects a filter with an empty query+ // ("Trying to match: on fields: <column> failed", 3052) and fails+ // the whole statement, so one empty element in an IN-list built from+ // optional values (e.g. GetWikiGraph's from_kwd list when an entity+ // has no slug) must not poison the condition.+ var inCond []string+ appendKeywordCond := func(raw string) {+ if strings.TrimSpace(raw) == "" {+ return+ }+ inCond = append(inCond, keywordFilterCondition(k, raw))+ }switch val := v.(type) {case []string:- var inCond []stringfor _, item := range val {- inCond = append(inCond, fmt.Sprintf("filter_fulltext('%s', '%s')",- convertMatchingField(k), escapeFilterValue(item)))- }- if len(inCond) > 0 {- cond = append(cond, "("+strings.Join(inCond, " or ")+")")+ appendKeywordCond(item)}case []interface{}:- var inCond []stringfor _, item := range val {if s, ok := item.(string); ok {- inCond = append(inCond, fmt.Sprintf("filter_fulltext('%s', '%s')",- convertMatchingField(k), escapeFilterValue(s)))+ appendKeywordCond(s)} else {- inCond = append(inCond, fmt.Sprintf("filter_fulltext('%s', '%s')",- convertMatchingField(k), escapeFilterValue(fmt.Sprintf("%v", item))))+ appendKeywordCond(fmt.Sprintf("%v", item))}}- if len(inCond) > 0 {- cond = append(cond, "("+strings.Join(inCond, " or ")+")")- }case string:- cond = append(cond, fmt.Sprintf("filter_fulltext('%s', '%s')",- convertMatchingField(k), escapeFilterValue(val)))+ appendKeywordCond(val)default:- cond = append(cond, fmt.Sprintf("filter_fulltext('%s', '%s')",- convertMatchingField(k), escapeFilterValue(fmt.Sprintf("%v", v))))+ appendKeywordCond(fmt.Sprintf("%v", v))+ }+ if len(inCond) > 0 {+ // Balanced, not a flat "a or b or ... " chain: a long keyword list+ // (22 wiki slugs) otherwise exceeds Infinity's Thrift expression+ // depth and kills the connection (see joinBalanced).+ cond = append(cond, joinBalanced(inCond, " or "))}continue}@@ -2479,6 +2694,25 @@ func decodeJSONFields(chunk map[string]interface{}) {listField := fieldJSONList(fieldName)text, ok := value.(string)if !ok {+ // Infinity hands a Json/Array column back as the raw+ // length-prefixed multi-value byte blob (see+ // decodeInfinityJSONBytes) rather than the JSON text a legacily+ // typed varchar column returns, so decode that shape too. Without+ // it every reader of such a column saw raw bytes: the structure+ // graph could not read compilation_template_ids off the entity /+ // relation rows, so it could not bucket them per template — it fell+ // back to a synthetic "legacy" bucket with zero relations and the+ // navigation tree's document level stayed flat instead of nesting+ // into its entity tree.+ if raw, isBytes := value.([]byte); isBytes {+ if decoded, ok := decodeInfinityJSONBytes(raw); ok {+ if listField {+ decoded = asJSONList(decoded)+ }+ chunk[fieldName] = decoded+ continue+ }+ }if listField {chunk[fieldName] = asJSONList(value)}@@ -2511,6 +2745,130 @@ func decodeJSONFields(chunk map[string]interface{}) {}}+// splitLengthPrefixedSegments splits Infinity's `[int32 little-endian+// length][payload]` framing into its payloads, each the JSON text of one value.+// parseLengthPrefixedJSON (metadata.go) unpacks the same framing for the+// metadata column.+func splitLengthPrefixedSegments(data []byte) [][]byte {+ var out [][]byte+ for offset := 0; offset+4 <= len(data); {+ length := int(uint32(data[offset]) |+ uint32(data[offset+1])<<8 |+ uint32(data[offset+2])<<16 |+ uint32(data[offset+3])<<24)+ offset += 4+ if length < 0 || offset+length > len(data) {+ break+ }+ out = append(out, data[offset:offset+length])+ offset += length+ }+ return out+}++// realignJSONColumnCells redistributes a Json/Array column that Infinity returns+// as ONE cell holding one length-prefixed segment per returned row, in row+// order (the quirk realignMetaFieldsColumn documents for the metadata column).+// Without it the column-oriented loop in Search parks the whole column on row 0+// and rows 1..N lose it: a five-row entity/relation read carried+// compilation_template_ids on exactly one row, so the structure graph could not+// bucket rows per template and answered with a synthetic "legacy" template with+// zero relations.+//+// Each row receives its own segment as JSON text, which decodeJSONFields then+// decodes through the ordinary string path. The blob must frame into exactly one+// valid JSON value per row, so a column that is already aligned (one cell per+// row) or any other byte shape is left untouched.+func realignJSONColumnCells(rows []map[string]interface{}, columns map[string][]interface{}) {+ if len(rows) < 2 {+ return+ }+ for colName, cells := range columns {+ if !fieldJSON(colName) || len(cells) != 1 {+ continue+ }+ raw, isBytes := cells[0].([]byte)+ if !isBytes {+ continue+ }+ segments := splitLengthPrefixedSegments(raw)+ if len(segments) != len(rows) {+ continue+ }+ valid := true+ for _, segment := range segments {+ var value interface{}+ if err := json.Unmarshal(segment, &value); err != nil {+ valid = false+ break+ }+ }+ if !valid {+ continue+ }+ for i := range rows {+ rows[i][colName] = string(segments[i])+ }+ }+}++// decodeInfinityJSONBytes decodes a Json/Array column value that arrived as raw+// length-prefixed bytes rather than JSON text: the shape of a single-value cell+// (a one-row result, or a value realignJSONColumnCells left alone). Every segment+// must parse as JSON — anything else reports ok=false so the caller keeps its+// previous behavior. Array segments concatenate (duplicates dropped, so a+// repeated value cannot fabricate a second identical entry); a non-array segment+// replaces the accumulated value.+func decodeInfinityJSONBytes(raw []byte) (interface{}, bool) {+ var (+ items []interface{}+ scalar interface{}+ )+ segments := splitLengthPrefixedSegments(raw)+ if len(segments) == 0 {+ return nil, false+ }+ for _, segment := range segments {+ var value interface{}+ if err := json.Unmarshal(segment, &value); err != nil {+ return nil, false+ }+ if arr, isArray := value.([]interface{}); isArray {+ items = appendJSONUnique(items, arr)+ continue+ }+ scalar = value+ }+ if len(items) > 0 {+ return items, true+ }+ return scalar, true+}++// appendJSONUnique appends the elements of add that dst does not already hold,+// comparing them by their JSON encoding (Infinity stores scalars here, so the+// encoding is a stable identity for a value).+func appendJSONUnique(dst, add []interface{}) []interface{} {+ for _, item := range add {+ key, err := json.Marshal(item)+ if err != nil {+ dst = append(dst, item)+ continue+ }+ dup := false+ for _, existing := range dst {+ if other, err := json.Marshal(existing); err == nil && string(other) == string(key) {+ dup = true+ break+ }+ }+ if !dup {+ dst = append(dst, item)+ }+ }+ return dst+}+// asJSONList normalizes a list-column value the way Python's parse_json_list// does (infinity_conn.py:938-944): a slice passes through unchanged, a nil value// becomes [] and any other scalar is wrapped in a single-element list.@@ -2700,8 +3058,20 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]intercase "docnm_kwd":d["docnm"] = vcase "title_kwd":+ // title_kwd is a column of its own in the Infinity mapping: the+ // compiled wiki/nav rows carry their label there, and nav keys its+ // cluster updates on it (title_kwd = <cluster name>). Folding it away+ // into docnm (Python's chunk-title alias, infinity_conn.py:522-527)+ // wrote ONLY docnm, so on Infinity every reader of title_kwd saw ""+ // while ES returned the field verbatim: nav labels degraded to the+ // description's first line and every cluster lookup matched nothing,+ // so each document opened a new root cluster instead of merging.+ // Keep the column, and still mirror it into docnm when the row has no+ // docnm_kwd, preserving the chunk-title search behavior.+ title := utility.ConvertToString(v)+ d["title_kwd"] = titleif _, exists := chunk["docnm_kwd"]; !exists {- d["docnm"] = utility.ConvertToString(v)+ d["docnm"] = title}case "title_sm_tks":if _, exists := chunk["docnm_kwd"]; !exists {diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 5c38ca22b1..69afb7f302 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -15,6 +15,7 @@package infinityimport (+ "encoding/binary""encoding/json""os""reflect"@@ -503,3 +504,290 @@ func TestJSONColumnsMatchCaseInsensitively(t *testing.T) {t.Errorf("nil Source_Doc_IDs = %#v, want %#v", row["Source_Doc_IDs"], want)}}++// TestTransformChunkFieldsKeepsTitleColumn pins that title_kwd survives as its+// own column. The compiled wiki/nav rows carry their label only in title_kwd and+// nav keys every cluster update on it (title_kwd = <cluster name>); folding the+// field into docnm — Python's chunk-title alias — wrote ONLY docnm, so on+// Infinity every reader of title_kwd saw "" while ES returned the field verbatim+// (nav labels degraded to the description's first line and cluster merges never+// matched, so each document opened a new root cluster).+func TestTransformChunkFieldsKeepsTitleColumn(t *testing.T) {+ got := transformChunkFields(map[string]interface{}{"title_kwd": "忠臣进谏与托孤遗志 27577543"}, nil)+ if want := "忠臣进谏与托孤遗志 27577543"; got["title_kwd"] != want {+ t.Errorf("title_kwd = %#v, want %q", got["title_kwd"], want)+ }+ // The chunk-title alias is still written when the row carries no docnm_kwd.+ if want := "忠臣进谏与托孤遗志 27577543"; got["docnm"] != want {+ t.Errorf("docnm = %#v, want %q", got["docnm"], want)+ }++ // With docnm_kwd present, the alias must not overwrite the document name,+ // and title_kwd must still keep the row's own label.+ got = transformChunkFields(map[string]interface{}{+ "docnm_kwd": "出师表.txt",+ "title_kwd": "wiki page title",+ }, nil)+ if want := "出师表.txt"; got["docnm"] != want {+ t.Errorf("docnm = %#v, want %q", got["docnm"], want)+ }+ if want := "wiki page title"; got["title_kwd"] != want {+ t.Errorf("title_kwd = %#v, want %q", got["title_kwd"], want)+ }+}++// TestStripScorePseudoColumns pins that a caller's ES-style score columns are+// dropped: Infinity rejects "_score" unless a MATCH TEXT/TENSOR or Fusion is+// present (3013), and Search re-adds the one form its query shape accepts.+func TestStripScorePseudoColumns(t *testing.T) {+ got := stripScorePseudoColumns([]string{"type_kwd", "_score", "SCORE", "similarity()", "title_kwd"})+ want := []string{"type_kwd", "title_kwd"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("stripScorePseudoColumns = %#v, want %#v", got, want)+ }++ // A filter-only query keeps no score column at all.+ if got := stripScorePseudoColumns([]string{"_score"}); len(got) != 0 {+ t.Fatalf("filter-only score columns = %#v, want empty", got)+ }+}++func TestHasPartialWildcard(t *testing.T) {+ cases := []struct {+ fields []string+ want bool+ }{+ {[]string{"id", "title_kwd"}, false},+ {[]string{"*"}, false}, // Infinity reads a bare "*" as "all columns"+ {[]string{"id", "q_*_vec"}, true},+ {[]string{"q_*_vec", "*"}, true},+ }+ for _, c := range cases {+ if got := hasPartialWildcard(c.fields); got != c.want {+ t.Errorf("hasPartialWildcard(%#v) = %v, want %v", c.fields, got, c.want)+ }+ }+}++// TestExpandSelectWildcards pins the workaround for the partial wildcard the+// compile reader asks for: Infinity parses "q_*_vec" as the expression+// "q_" * "_vec" and fails to bind it (3013), so it is expanded into the table's+// real vector column(s) and dropped when the table has none.+func TestExpandSelectWildcards(t *testing.T) {+ cols := map[string]struct {+ Type string+ Default interface{}+ }{+ "id": {Type: "varchar"},+ "q_1024_vec": {Type: "vector,1024,float"},+ "content": {Type: "varchar"},+ }++ if got, want := expandSelectWildcards([]string{"id", "q_*_vec", "title_kwd"}, cols),+ []string{"id", "q_1024_vec", "title_kwd"}; !reflect.DeepEqual(got, want) {+ t.Errorf("expandSelectWildcards = %#v, want %#v", got, want)+ }++ // A bare "*" is left untouched: Infinity supports it.+ if got, want := expandSelectWildcards([]string{"*"}, cols), []string{"*"}; !reflect.DeepEqual(got, want) {+ t.Errorf("bare wildcard = %#v, want %#v", got, want)+ }++ // A pattern matching nothing is dropped instead of being sent verbatim.+ if got, want := expandSelectWildcards([]string{"id", "missing_*_col"}, cols), []string{"id"}; !reflect.DeepEqual(got, want) {+ t.Errorf("unmatched wildcard = %#v, want %#v", got, want)+ }++ // Resolving to an already listed column must not duplicate it.+ if got, want := expandSelectWildcards([]string{"q_1024_vec", "q_*_vec"}, cols), []string{"q_1024_vec"}; !reflect.DeepEqual(got, want) {+ t.Errorf("duplicate wildcard = %#v, want %#v", got, want)+ }++ // Columns with no vector column at all: the wildcard disappears, the rest+ // survives (a query may legitimately target a table without embeddings).+ got := expandSelectWildcards([]string{"id", "q_*_vec"}, map[string]struct {+ Type string+ Default interface{}+ }{"id": {Type: "varchar"}})+ if want := []string{"id"}; !reflect.DeepEqual(got, want) {+ t.Errorf("table without vector column = %#v, want %#v", got, want)+ }+}++func TestWildcardMatchColumn(t *testing.T) {+ cases := []struct {+ pattern, name string+ want bool+ }{+ {"q_*_vec", "q_1024_vec", true},+ {"q_*_vec", "q_768_vec", true},+ {"q_*_vec", "content", false},+ {"q_*_vec", "q_1024_vec_x", false},+ {"*_vec", "q_1024_vec", true},+ {"q_*", "q_1024_vec", true},+ {"foo", "foo", true},+ {"foo", "bar", false},+ {"a*b*c", "aXbYc", true},+ {"a*b*c", "ac", false},+ }+ for _, c := range cases {+ if got := wildcardMatchColumn(c.pattern, c.name); got != c.want {+ t.Errorf("wildcardMatchColumn(%q, %q) = %v, want %v", c.pattern, c.name, got, c.want)+ }+ }+}++// lengthPrefixedJSON builds Infinity's raw Json/Array column encoding: one+// `[int32 little-endian length][payload]` segment per stored value.+func lengthPrefixedJSON(parts ...string) []byte {+ out := make([]byte, 0, 64)+ for _, p := range parts {+ var l [4]byte+ binary.LittleEndian.PutUint32(l[:], uint32(len(p)))+ out = append(out, l[:]...)+ out = append(out, p...)+ }+ return out+}++// TestDecodeJSONFieldsLengthPrefixedBytes pins the read path for the Json/Array+// columns Infinity returns as length-prefixed bytes instead of JSON text — the+// shape the structure graph reads compilation_template_ids from.+func TestDecodeJSONFieldsLengthPrefixedBytes(t *testing.T) {+ row := map[string]interface{}{+ // One real value after three empty writes (the observed shape).+ "compilation_template_ids": lengthPrefixedJSON("[]", "[]", "[]", `["03b36295dcb34f9e811d8073b21e7f2f"]`),+ "source_chunk_ids": lengthPrefixedJSON("[]", "[]", "[]", `["d1962d8d1cf69ab6","1313e71f7c7c9cca"]`),+ // Every write empty: the column reads back as an empty list.+ "doc_ids_kwd": lengthPrefixedJSON("[]", "[]", "[]", "[]", "[]", "[]"),+ }+ decodeJSONFields(row)++ if want := []interface{}{"03b36295dcb34f9e811d8073b21e7f2f"}; !reflect.DeepEqual(row["compilation_template_ids"], want) {+ t.Errorf("compilation_template_ids = %#v, want %#v", row["compilation_template_ids"], want)+ }+ if want := []interface{}{"d1962d8d1cf69ab6", "1313e71f7c7c9cca"}; !reflect.DeepEqual(row["source_chunk_ids"], want) {+ t.Errorf("source_chunk_ids = %#v, want %#v", row["source_chunk_ids"], want)+ }+ if want := []interface{}{}; !reflect.DeepEqual(row["doc_ids_kwd"], want) {+ t.Errorf("doc_ids_kwd = %#v, want %#v", row["doc_ids_kwd"], want)+ }++ // A value written twice must not fabricate a second identical bucket.+ dup := map[string]interface{}{+ "compilation_template_ids": lengthPrefixedJSON(`["t1"]`, `["t1","t2"]`),+ }+ decodeJSONFields(dup)+ if want := []interface{}{"t1", "t2"}; !reflect.DeepEqual(dup["compilation_template_ids"], want) {+ t.Errorf("duplicate segments = %#v, want %#v", dup["compilation_template_ids"], want)+ }++ // A non-array segment (an object) on a list column keeps the list shape+ // (Python's parse_json_list wraps a scalar), and on a plain JSON column it+ // decodes to the object itself.+ listObj := map[string]interface{}{+ "compilation_template_ids": lengthPrefixedJSON(`{"kind":"tree"}`),+ }+ decodeJSONFields(listObj)+ wrapped, ok := listObj["compilation_template_ids"].([]interface{})+ if !ok || len(wrapped) != 1 {+ t.Fatalf("object segment on a list column = %#v, want a one-element list", listObj["compilation_template_ids"])+ }+ if m, ok := wrapped[0].(map[string]interface{}); !ok || m["kind"] != "tree" {+ t.Errorf("object segment element = %#v, want the decoded map", wrapped[0])+ }++ plainObj := map[string]interface{}{+ "source_chunk_hashes": lengthPrefixedJSON(`{"a":"b"}`),+ }+ decodeJSONFields(plainObj)+ if m, ok := plainObj["source_chunk_hashes"].(map[string]interface{}); !ok || m["a"] != "b" {+ t.Errorf("object segment on a JSON column = %#v, want the decoded map", plainObj["source_chunk_hashes"])+ }+}++func TestDecodeInfinityJSONBytesRejectsOtherShapes(t *testing.T) {+ if _, ok := decodeInfinityJSONBytes([]byte("not-length-prefixed")); ok {+ t.Error("plain text bytes must not be accepted as a Json column blob")+ }+ if _, ok := decodeInfinityJSONBytes([]byte{1, 2}); ok {+ t.Error("too-short bytes must not be accepted")+ }+ // A truncated final segment decodes the complete prefix only.+ blob := lengthPrefixedJSON(`["a"]`)+ blob = append(blob, 0, 0, 0, 9, 'x')+ value, ok := decodeInfinityJSONBytes(blob)+ if !ok {+ t.Fatal("complete leading segment must still decode")+ }+ if want := []interface{}{"a"}; !reflect.DeepEqual(value, want) {+ t.Errorf("truncated blob = %#v, want %#v", value, want)+ }+}++// TestRealignJSONColumnCells pins the redistribution of a Json/Array column that+// Infinity returns as one cell holding one segment per row: without it only row 0+// keeps the column, so the structure graph sees one entity row per read.+func TestRealignJSONColumnCells(t *testing.T) {+ columns := map[string][]interface{}{+ "id": {"a", "b", "c"},+ "knowledge_graph_kwd": {"entity", "entity", "relation"},+ "compilation_template_ids": {lengthPrefixedJSON(`["t1"]`, `[]`, `["t2"]`)},+ }+ rows := []map[string]interface{}{{}, {}, {}}+ realignJSONColumnCells(rows, columns)++ wantText := []string{`["t1"]`, `[]`, `["t2"]`}+ for i, want := range wantText {+ if got := rows[i]["compilation_template_ids"]; got != want {+ t.Errorf("row %d segment = %#v, want %q", i, got, want)+ }+ }+ // Decoding each row's redistributed text yields that row's own value.+ decodeJSONFields(rows[0])+ decodeJSONFields(rows[1])+ decodeJSONFields(rows[2])+ if want := []interface{}{"t1"}; !reflect.DeepEqual(rows[0]["compilation_template_ids"], want) {+ t.Errorf("row 0 = %#v, want %#v", rows[0]["compilation_template_ids"], want)+ }+ if want := []interface{}{}; !reflect.DeepEqual(rows[1]["compilation_template_ids"], want) {+ t.Errorf("row 1 = %#v, want %#v", rows[1]["compilation_template_ids"], want)+ }+ if want := []interface{}{"t2"}; !reflect.DeepEqual(rows[2]["compilation_template_ids"], want) {+ t.Errorf("row 2 = %#v, want %#v", rows[2]["compilation_template_ids"], want)+ }++ // A blob whose segment count does not match the row count is left alone, and+ // a column that already has one cell per row is untouched.+ odd := []map[string]interface{}{{}, {}}+ realignJSONColumnCells(odd, map[string][]interface{}{+ "compilation_template_ids": {lengthPrefixedJSON(`["t1"]`, `[]`, `["t2"]`)},+ })+ if len(odd[0]) != 0 {+ t.Errorf("mismatched segment count must be left untouched, got %#v", odd[0])+ }+ aligned := []map[string]interface{}{{}, {}}+ realignJSONColumnCells(aligned, map[string][]interface{}{+ "compilation_template_ids": {`["t1"]`, `["t2"]`},+ })+ if len(aligned[0]) != 0 {+ t.Errorf("an aligned column must be left untouched, got %#v", aligned[0])+ }+ // A single row keeps the cell as it came: decodeInfinityJSONBytes handles it.+ single := []map[string]interface{}{{}}+ realignJSONColumnCells(single, map[string][]interface{}{+ "compilation_template_ids": {lengthPrefixedJSON(`["t1"]`)},+ })+ if len(single[0]) != 0 {+ t.Errorf("a single-row result must be left untouched, got %#v", single[0])+ }+ // Segments that are not JSON are not per-row values: leave the column alone+ // rather than copying the framing into every row.+ notJSON := []map[string]interface{}{{}, {}}+ realignJSONColumnCells(notJSON, map[string][]interface{}{+ "compilation_template_ids": {lengthPrefixedJSON("t1", "t2")},+ })+ if len(notJSON[0]) != 0 {+ t.Errorf("non-JSON segments must be left untouched, got %#v", notJSON[0])+ }+}diff --git a/internal/engine/infinity/client.go b/internal/engine/infinity/client.goindex 7da66805bd..6b1d732e68 100644--- a/internal/engine/infinity/client.go+++ b/internal/engine/infinity/client.go@@ -26,6 +26,7 @@ import ("reflect""strconv""strings"+ "sync""sync/atomic""time"@@ -45,6 +46,12 @@ type infinityClient struct {getDatabaseSeq atomic.Uint64getDatabaseInflight atomic.Int64+ // dbConns maps the *infinity.Database handed to a caller back to the pooled+ // connection it was checked out from, so a call that fails at the transport+ // level can drop that exact socket (see dropConnectionFor). Entries live only+ // for the duration of a checkout.+ dbConns sync.Map+// Original URI from config, used by RunSQL to extract the host.hostURI string@@ -314,10 +321,15 @@ func (c *infinityClient) checkoutDatabase(ctx context.Context, caller string) (*zap.Duration("elapsed", elapsed),zap.Error(err),)+ if isConnectionLevelError(err) {+ dropConnection(conn, caller)+ }release()return nil, nil, err}+ c.dbConns.Store(db, conn)wrappedRelease := func() {+ c.dbConns.Delete(db)inflight := c.getDatabaseInflight.Add(-1)elapsed := time.Since(start)common.Info("Infinity GetDatabase done",@@ -333,6 +345,66 @@ func (c *infinityClient) checkoutDatabase(ctx context.Context, caller string) (*return db, wrappedRelease, nil}+// dropConnectionFor closes the pooled connection behind a checked-out database+// handle after a transport-level failure. Such a socket still reports+// IsConnected(), so Put() would return it to the pool and desync the next+// caller's Thrift stream — observed as "InfinityException(7018, ... EOF)".+// Disconnecting first makes Put() evict it instead. caller names the operation+// that failed, for the log.+func (c *infinityClient) dropConnectionFor(db *infinity.Database, caller string) {+ if c == nil || db == nil {+ return+ }+ raw, ok := c.dbConns.Load(db)+ if !ok {+ return+ }+ conn, _ := raw.(*infinity.InfinityConnection)+ dropConnection(conn, caller)+}++// dropConnection disconnects a connection whose call failed mid-flight so the+// pool evicts it. Errors are logged and ignored: the socket is already suspect.+func dropConnection(conn *infinity.InfinityConnection, caller string) {+ if conn == nil || !conn.IsConnected() {+ return+ }+ common.Warn("Infinity connection dropped after a transport-level failure",+ zap.String("caller", caller),+ zap.String("conn_ptr", fmt.Sprintf("%p", conn)))+ if _, err := conn.Disconnect(); err != nil {+ common.Warn("Infinity connection disconnect failed",+ zap.String("caller", caller), zap.Error(err))+ }+}++// isConnectionLevelError reports whether a failed Infinity call may have left the+// socket in an unknown state (the reply was not fully read), in which case the+// connection must not be reused. Infinity's own error codes for this are 7018+// ("Failed to execute query: EOF") plus the thrift/transport failures below.+func isConnectionLevelError(err error) bool {+ if err == nil {+ return false+ }+ msg := strings.ToLower(err.Error())+ for _, marker := range []string{+ "eof",+ "connection is dead",+ "connection reset",+ "broken pipe",+ "i/o timeout",+ "deadline exceeded",+ "tprotocolexception",+ "transport is not open",+ "no route to host",+ } {+ if strings.Contains(msg, marker) {+ return true+ }+ }+ return false+}+// Engine Infinity engine implementation using Go SDKtype Engine struct {config config.InfinityConfigdiff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.goindex eb28450bc9..9a4c551d5b 100644--- a/internal/engine/infinity/common.go+++ b/internal/engine/infinity/common.go@@ -23,6 +23,7 @@ import ("fmt""ragflow/internal/common""strings"+ "unicode"infinity "github.com/infiniflow/infinity-go-sdk"@@ -158,6 +159,45 @@ func fieldKeyword(fieldName string) bool {return false}+// keywordFilterCondition renders a filter for a *_kwd column.+//+// Infinity declares these columns as analyzed varchars, so filter_fulltext()+// tokenizes the query and a multi-token value matches NOTHING — while every+// navigation cluster name is multi-token, so nav's child lookup (parent_kwd=…),+// its cluster updates and cleanupEmptyCluster all silently matched no row.+// Elasticsearch matches a plain *_kwd column exactly (keyword + term/terms), so+// a whitespace-bearing value becomes `col = 'value'`: it matched no row before,+// so this cannot narrow an existing match, and single-token values keep+// filter_fulltext for the legacy ###-joined columns (see fieldJSONList).+//+// tag_kwd and toc_kwd are the exception: convertMatchingField turns them into a+// full-text index reference ("tag_kwd@ft_tag_kwd_whitespace__"), which is not a+// column and cannot be compared with `=` (Infinity rejects the whole statement).+// Their cells hold several ###-joined values, and that index is what gives them+// the membership match ES gets from its keyword array, so a multi-token value is+// matched as a quoted phrase through it.+func keywordFilterCondition(field string, value string) string {+ escaped := escapeFilterValue(value)+ multiToken := strings.ContainsFunc(value, unicode.IsSpace)+ indexRef := convertMatchingField(field)++ if indexRef == field {+ if multiToken {+ return fmt.Sprintf("%s = '%s'", field, escaped)+ }+ return fmt.Sprintf("filter_fulltext('%s', '%s')", field, escaped)+ }+ if !multiToken {+ return fmt.Sprintf("filter_fulltext('%s', '%s')", indexRef, escaped)+ }+ if strings.Contains(value, `"`) {+ // Cannot be phrased: compare the raw column so the value still matches a+ // single-valued cell instead of failing the statement.+ return fmt.Sprintf("%s = '%s'", field, escaped)+ }+ return fmt.Sprintf("filter_fulltext('%s', '\"%s\"')", indexRef, escaped)+}+// fieldJSON reports fields stored as Infinity JSON columns. The Infinity Go// SDK expects JSON columns as encoded strings, not Go slices or maps. Column// names are matched case-insensitively so the write path (transformChunkFields)@@ -188,6 +228,36 @@ func fieldJSONList(fieldName string) bool {}}+// joinBalanced renders a boolean chain as a balanced tree instead of the flat,+// left-deep chain strings.Join produces. Infinity's filter string is parsed by+// the SDK into a Thrift expression tree (infinity-go-sdk expression_parser.go)+// and sent as that tree, so a left-deep chain of N terms nests N levels deep: a+// filter with 22 OR'ed filter_fulltext(...) clauses (the wiki contribution and+// graph reads build exactly that) made Infinity abort the connection with+//+// Thrift: ... TProtocolException: Exceeded depth limit+//+// which our callers then see as "InfinityException(7018, Failed to execute+// query: EOF)". A balanced tree keeps the depth at log2(N) — 22 terms pass+// against a live Infinity, verified — while matching exactly the same rows.+// The parenthesization is boolean-equivalent to the flat chain (a single term+// needs no parentheses).+func joinBalanced(parts []string, op string) string {+ switch len(parts) {+ case 0:+ return ""+ case 1:+ // Keep the single-element form parenthesized: callers (and tests) relied+ // on a self-contained group, e.g. "(1=0)" for a never-matching list.+ return "(" + parts[0] + ")"+ case 2:+ return "(" + parts[0] + op + parts[1] + ")"+ default:+ mid := (len(parts) + 1) / 2+ return "(" + joinBalanced(parts[:mid], op) + op + joinBalanced(parts[mid:], op) + ")"+ }+}+func jsonListFilterConditions(fieldName string, value interface{}, tableColumns map[string]struct {Type stringDefault interface{}@@ -333,38 +403,47 @@ func buildFilterFromCondition(condition map[string]interface{}, tableColumns map// JSON-list fields use member containment. Legacy tables stored these// columns as ###-joined varchar values, so retain their full-text fallback.if fieldJSONList(k) && tableColumns != nil {- jsonConditions := jsonListFilterConditions(k, v, tableColumns)- if len(jsonConditions) > 0 {- conditions = append(conditions, "("+strings.Join(jsonConditions, " OR ")+")")+ if jsonConditions := jsonListFilterConditions(k, v, tableColumns); len(jsonConditions) > 0 {+ conditions = append(conditions, joinBalanced(jsonConditions, " OR "))}continue}- // Handle keyword fields -> filter_fulltext with converted field name+ // Handle keyword fields -> exact match, or filter_fulltext for the+ // single-token values Infinity can tokenize (see keywordFilterCondition).if fieldKeyword(k) {var orConds []string- addFullText := func(item string) {- item = strings.ReplaceAll(item, "'", "''")- orConds = append(orConds, fmt.Sprintf("filter_fulltext('%s', '%s')", convertMatchingField(k), item))+ addKeyword := func(item string) {+ // Blank entries are dropped exactly as the search path drops them:+ // Infinity rejects an empty full-text query ("Trying to match: on+ // fields: <column> failed", 3052) and fails the whole+ // UpdateChunks/DeleteChunks statement, so one empty element in a+ // list built from optional values must not poison it. A condition+ // left with no clause at all is refused by the callers' "1=1"+ // guard, so this cannot widen an update or delete.+ if strings.TrimSpace(item) == "" {+ return+ }+ orConds = append(orConds, keywordFilterCondition(k, item))}switch val := v.(type) {case []string:for _, item := range val {- addFullText(item)+ addKeyword(item)}case []interface{}:for _, item := range val {- addFullText(fmt.Sprintf("%v", item))+ addKeyword(fmt.Sprintf("%v", item))}case string:- addFullText(val)+ addKeyword(val)default:- addFullText(fmt.Sprintf("%v", val))+ addKeyword(fmt.Sprintf("%v", val))}if len(orConds) > 0 {- conditions = append(conditions, "("+strings.Join(orConds, " OR ")+")")+ conditions = append(conditions, joinBalanced(orConds, " OR "))}continue}diff --git a/internal/engine/infinity/meta_filter.go b/internal/engine/infinity/meta_filter.goindex cca63b17cb..6f16ae02d7 100644--- a/internal/engine/infinity/meta_filter.go+++ b/internal/engine/infinity/meta_filter.go@@ -158,10 +158,10 @@ func (t *MetaFilterTranslator) translateIn(key string, value interface{}, flt mavar conditions []stringif len(stringParts) > 0 {- conditions = append(conditions, "("+strings.Join(stringParts, " OR ")+")")+ conditions = append(conditions, joinBalanced(stringParts, " OR "))}if len(numParts) > 0 {- conditions = append(conditions, "("+strings.Join(numParts, " OR ")+")")+ conditions = append(conditions, joinBalanced(numParts, " OR "))}return "(" + strings.Join(conditions, " OR ") + ")"@@ -263,8 +263,13 @@ func BuildInfinityFilter(filters []map[string]interface{}, logic string) (stringif logic == "or" {joiner = " OR "}+ if len(fragments) == 0 {+ return "()", nil+ }- return "(" + strings.Join(fragments, joiner) + ")", nil+ // Balanced so a long OR chain (many metadata values) cannot exceed Infinity's+ // Thrift expression depth (see joinBalanced).+ return joinBalanced(fragments, joiner), nil}// IsPushdownSupported checks if all filters can be pushed downdiff --git a/internal/engine/infinity/metadata.go b/internal/engine/infinity/metadata.goindex 8456707153..65441cb8da 100644--- a/internal/engine/infinity/metadata.go+++ b/internal/engine/infinity/metadata.go@@ -176,7 +176,10 @@ func (e *Engine) InsertMetadata(ctx context.Context, metadata []map[string]interkbID := fmt.Sprintf("'%s'", strings.ReplaceAll(fmt.Sprintf("%v", m["kb_id"]), "'", "''"))idList[i] = fmt.Sprintf("(id = %s AND kb_id = %s)", docID, kbID)}- filter := strings.Join(idList, " OR ")+ // Balanced, not a flat chain: a bulk delete of many metadata rows would+ // otherwise nest N levels deep and trip Infinity's Thrift expression depth+ // limit (see joinBalanced).+ filter := joinBalanced(idList, " OR ")common.Debug(fmt.Sprintf("Deleting existing metadata with filter: %s", filter))delResp, delErr := table.Delete(filter)if delErr != nil {diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.goindex 9c3be5faa8..951132e11c 100644--- a/internal/engine/infinity/sql_test.go+++ b/internal/engine/infinity/sql_test.go@@ -17,9 +17,12 @@package infinityimport (+ "errors"+ "fmt""os""path/filepath""reflect"+ "strings""testing")@@ -425,6 +428,172 @@ func TestBuildFilterFromCondition_StringSliceIDPreservesScope(t *testing.T) {}}+// TestEquivalentConditionToStrSkipsBlankKeywordValues pins that a blank entry in+// a keyword (filter_fulltext) condition is dropped instead of being rendered as+// filter_fulltext('<col>', ”): Infinity rejects that with+//+// 3052 Trying to match: on fields: <col> failed+//+// and fails the whole statement, which is exactly what GetWikiGraph hit when an+// entity row carried no slug.+func TestEquivalentConditionToStrSkipsBlankKeywordValues(t *testing.T) {+ got := equivalentConditionToStr(map[string]interface{}{+ "from_kwd": []string{"", " ", "topic/出师表"},+ }, nil)+ want := `(filter_fulltext('from_kwd', 'topic/出师表'))`+ if got != want {+ t.Errorf("blank keyword filter = %q, want %q", got, want)+ }++ // All entries blank: the column contributes no clause at all.+ if got := equivalentConditionToStr(map[string]interface{}{"from_kwd": []string{"", ""}}, nil); got != "" {+ t.Errorf("all-blank keyword filter = %q, want no clause", got)+ }+}++// maxParenDepth returns the deepest nesting of parentheses in an expression.+func maxParenDepth(expr string) int {+ depth, max := 0, 0+ for _, r := range expr {+ switch r {+ case '(':+ depth+++ if depth > max {+ max = depth+ }+ case ')':+ depth--+ }+ }+ return max+}++// TestJoinBalanced pins the depth-bounded boolean rendering. Infinity parses the+// filter string into a Thrift expression tree, so a left-deep chain of N terms+// nests N levels deep: the wiki contribution read's 22 OR'ed filter_fulltext+// clauses made Infinity abort the connection ("TProtocolException: Exceeded depth+// limit"), which surfaced as 7018/EOF. A balanced tree must keep every term and+// only log2(N) nesting.+func TestJoinBalanced(t *testing.T) {+ if got := joinBalanced(nil, " OR "); got != "" {+ t.Errorf("joinBalanced(nil) = %q, want empty", got)+ }+ if got := joinBalanced([]string{"a"}, " OR "); got != "(a)" {+ t.Errorf("single term = %q, want %q", got, "(a)")+ }+ if got := joinBalanced([]string{"a", "b"}, " OR "); got != "(a OR b)" {+ t.Errorf("two terms = %q, want %q", got, "(a OR b)")+ }++ parts := make([]string, 0, 22)+ for i := 0; i < 22; i++ {+ parts = append(parts, fmt.Sprintf("f%02d", i))+ }+ expr := joinBalanced(parts, " or ")+ for _, p := range parts {+ if !strings.Contains(expr, p) {+ t.Fatalf("expression dropped %s: %s", p, expr)+ }+ }+ // Paren depth of the rendered string mirrors the depth of the expression tree+ // the SDK builds from it; a flat chain would be sent N levels deep.+ if maxParenDepth(expr) > 6 {+ t.Errorf("balanced depth = %d, want <= ceil(log2(22))+1 = 6: %s", maxParenDepth(expr), expr)+ }+}++// TestEquivalentConditionToStrBalancesLongKeywordLists pins that the keyword+// branch uses the balanced join: a 22-element slug list is exactly the request+// that killed the connection in production.+func TestEquivalentConditionToStrBalancesLongKeywordLists(t *testing.T) {+ values := make([]string, 0, 22)+ for i := 0; i < 22; i++ {+ values = append(values, fmt.Sprintf("entity/person/p%d", i))+ }+ got := equivalentConditionToStr(map[string]interface{}{"slug_kwd": values}, nil)+ for _, v := range values {+ if !strings.Contains(got, "filter_fulltext('slug_kwd', '"+v+"')") {+ t.Fatalf("condition dropped %s: %s", v, got)+ }+ }+ if depth := maxParenDepth(got); depth > 6 {+ t.Errorf("paren depth = %d, want <= 6 for 22 values: %s", depth, got)+ }+}++// TestIsConnectionLevelError pins which failures mark a pooled connection as+// unusable: a cut-short reply desyncs the Thrift stream, and the next caller on+// that socket is the one the server rejects with "Exceeded depth limit" (seen as+// 7018/EOF). Semantic Infinity errors must NOT drop the connection.+func TestIsConnectionLevelError(t *testing.T) {+ drop := []string{+ "InfinityException(7018, Failed to execute query: EOF)",+ "InfinityException(6003, Connection is dead, removed from pool)",+ "read tcp 127.0.0.1:33924->127.0.0.1:23817: read: connection reset by peer",+ "write tcp: broken pipe",+ "dial tcp 127.0.0.1:23817: i/o timeout",+ "context deadline exceeded",+ "TProtocolException: Exceeded depth limit",+ }+ for _, msg := range drop {+ if !isConnectionLevelError(errors.New(msg)) {+ t.Errorf("isConnectionLevelError(%q) = false, want true", msg)+ }+ }+ keep := []string{+ "",+ "InfinityException(3052, Failed to execute query: Trying to match: on fields: from_kwd failed)",+ "InfinityException(3013, Failed to execute query: Fail to bind the expression: q_)",+ "InfinityException(7001, Table not found)",+ }+ for _, msg := range keep {+ var err error+ if msg != "" {+ err = errors.New(msg)+ }+ if isConnectionLevelError(err) {+ t.Errorf("isConnectionLevelError(%q) = true, want false", msg)+ }+ }+}++// TestRowFieldNames pins the insert-failure log payload: sorted column names of+// the first row (values are never logged — a vector column would flood the log).+func TestRowFieldNames(t *testing.T) {+ got := rowFieldNames([]map[string]interface{}{{"title_kwd": "a", "available_int": 0, "q_1024_vec": []float64{1}}})+ want := []string{"available_int", "q_1024_vec", "title_kwd"}+ if !reflect.DeepEqual(got, want) {+ t.Errorf("rowFieldNames = %#v, want %#v", got, want)+ }+ if got := rowFieldNames(nil); got != nil {+ t.Errorf("rowFieldNames(nil) = %#v, want nil", got)+ }+}++// TestIsEmptyEngineCell pins the row-materialization guard: Infinity can return a+// null cell (untyped nil or a nil slice) for a projection it found no row for, and+// such a cell must not turn into a row of its own — GetWikiGraph used to read an+// empty slug off that phantom row and poison the relation filter.+func TestIsEmptyEngineCell(t *testing.T) {+ cases := []struct {+ name string+ value interface{}+ want bool+ }{+ {"untyped nil", nil, true},+ {"nil slice", []uint8(nil), true},+ {"nil []interface{}", []interface{}(nil), true},+ {"empty slice", []interface{}{}, false},+ {"empty string", "", false},+ {"value", "doc-1", false},+ }+ for _, c := range cases {+ if got := isEmptyEngineCell(c.value); got != c.want {+ t.Errorf("isEmptyEngineCell(%s) = %v, want %v", c.name, got, c.want)+ }+ }+}+func TestBuildFilterFromConditionJSONListMembership(t *testing.T) {jsonColumns := map[string]struct {Type string@@ -469,3 +638,111 @@ func TestBuildFilterFromConditionMissingJSONListColumnNeverMatches(t *testing.T)t.Errorf("missing JSON-list column condition = %q, want %q", got, want)}}++// TestKeywordFilterConditionExactForWhitespaceValues pins the fix that makes a+// *_kwd filter with a multi-token value match at all. Infinity tokenizes a+// filter_fulltext() query, so a whitespace-bearing value matched NOTHING (nav's+// parent_kwd child lookup, cluster updates and cleanup all no-op'd); the ES+// columns are keywords and match exactly.+func TestKeywordFilterConditionExactForWhitespaceValues(t *testing.T) {+ cases := []struct {+ name, field, value, want string+ }{+ {"cluster name", "parent_kwd", "Imperial Memorial and Governance Advice 59cbfbef",+ `parent_kwd = 'Imperial Memorial and Governance Advice 59cbfbef'`},+ {"CJK cluster name with suffix", "title_kwd", "忠臣进谏与托孤遗志 27577543",+ `title_kwd = '忠臣进谏与托孤遗志 27577543'`},+ {"doc leaf label", "title_kwd", "出师表中的忠诚与北伐决心 1234",+ `title_kwd = '出师表中的忠诚与北伐决心 1234'`},+ // Single-token values keep the full-text rendering: legacy ###-joined+ // varchar columns still rely on token matching.+ {"single token", "type_kwd", "nav_cluster", `filter_fulltext('type_kwd', 'nav_cluster')`},+ {"root parent", "parent_kwd", "root", `filter_fulltext('parent_kwd', 'root')`},+ // Any Unicode space counts: a full-width space tokenizes like an ASCII+ // one and would hit the same silent no-match.+ {"ideographic space", "title_kwd", "忠臣进谏 与托孤遗志",+ `title_kwd = '忠臣进谏 与托孤遗志'`},+ // Fields convertMatchingField maps to a full-text index reference are NOT+ // columns: `= ` on the reference makes Infinity reject the whole statement,+ // so a multi-token value is phrased through the index instead (that index+ // is what gives tag/toc the membership match ES gets from its array).+ {"tag multi-token", "tag_kwd", "machine learning",+ `filter_fulltext('tag_kwd@ft_tag_kwd_whitespace__', '"machine learning"')`},+ {"toc multi-token", "toc_kwd", "第 一章",+ `filter_fulltext('toc_kwd@ft_toc_kwd_whitespace__', '"第 一章"')`},+ {"tag single token", "tag_kwd", "nlp",+ `filter_fulltext('tag_kwd@ft_tag_kwd_whitespace__', 'nlp')`},+ // A value that cannot be phrased compares the raw column, which is at+ // least addressable (the index reference is not).+ {"tag with a quote", "tag_kwd", `say "hi" now`,+ `tag_kwd = 'say "hi" now'`},+ // A *_kwd field with no index variant stays on the raw column.+ {"source_id", "source_id", "a b", `source_id = 'a b'`},+ // A quote stays escaped in both renderings.+ {"quoted multi-token", "title_kwd", "O'Brien cluster", `title_kwd = 'O''Brien cluster'`},+ {"quoted single token", "type_kwd", "o'brien", `filter_fulltext('type_kwd', 'o''brien')`},+ }+ for _, c := range cases {+ if got := keywordFilterCondition(c.field, c.value); got != c.want {+ t.Errorf("%s: keywordFilterCondition(%q, %q) = %q, want %q",+ c.name, c.field, c.value, got, c.want)+ }+ }+}++// TestKeywordFilterRenderingBothPaths pins that the search path+// (equivalentConditionToStr) and the update/delete path+// (buildFilterFromCondition) render a cluster-name filter identically: a drift+// would write a doc's parent_kwd while the cluster's own update matched nothing.+func TestKeywordFilterRenderingBothPaths(t *testing.T) {+ const name = "Imperial Memorial and Governance Advice 59cbfbef"++ wantParent := `(parent_kwd = '` + name + `')`+ parentCond := map[string]interface{}{"parent_kwd": []string{name}}+ if got := equivalentConditionToStr(parentCond, nil); got != wantParent {+ t.Errorf("search path parent_kwd = %q, want %q", got, wantParent)+ }+ if got := buildFilterFromCondition(parentCond, nil); got != wantParent {+ t.Errorf("update/delete path parent_kwd = %q, want %q", got, wantParent)+ }++ // The cluster's own filter (merge description, append doc, cleanup) uses+ // title_kwd; ListClusters keeps filtering parent_kwd='root', which stays on+ // the full-text path because it is a single token.+ wantTitle := `(title_kwd = '` + name + `')`+ titleCond := map[string]interface{}{"title_kwd": []string{name}}+ if got := equivalentConditionToStr(titleCond, nil); got != wantTitle {+ t.Errorf("search path title_kwd = %q, want %q", got, wantTitle)+ }+ if got := buildFilterFromCondition(titleCond, nil); got != wantTitle {+ t.Errorf("update/delete path title_kwd = %q, want %q", got, wantTitle)+ }+ if got := equivalentConditionToStr(map[string]interface{}{"parent_kwd": []string{"root"}}, nil); got !=+ `(filter_fulltext('parent_kwd', 'root'))` {+ t.Errorf("root clusters must keep the single-token rendering, got %q", got)+ }++ // Blank entries are dropped by BOTH paths. Infinity rejects an empty+ // full-text query ("Trying to match: on fields: <column> failed", 3052) and+ // fails the whole statement — including UpdateChunks/DeleteChunks, which the+ // update path feeds.+ blankMixed := map[string]interface{}{"from_kwd": []string{"", " ", "x"}}+ wantMixed := `(filter_fulltext('from_kwd', 'x'))`+ if got := equivalentConditionToStr(blankMixed, nil); got != wantMixed {+ t.Errorf("search path with blank entries = %q, want %q", got, wantMixed)+ }+ if got := buildFilterFromCondition(blankMixed, nil); got != wantMixed {+ t.Errorf("update/delete path with blank entries = %q, want %q", got, wantMixed)+ }++ // Every entry blank: the search path contributes no clause, while the update+ // path yields "1=1" so the caller's unconstrained-statement guard refuses it+ // instead of running an unscoped update or delete.+ allBlank := map[string]interface{}{"from_kwd": []string{"", " "}}+ if got := equivalentConditionToStr(allBlank, nil); got != "" {+ t.Errorf("search path with only blank entries = %q, want no clause", got)+ }+ if got := buildFilterFromCondition(allBlank, nil); got != "1=1" {+ t.Errorf("update/delete path with only blank entries = %q, want '1=1'", got)+ }+}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 81123c81f5..97c424ca6d 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -129,8 +129,9 @@ var compiledSelectFields = []string{// LoadDocProducts reconstructs products with an empty Vector, so merged rows// carry no embedding and the dataset-level KNN dedup (SearchSimilar on// available_int=1 + q_<dim>_vec) can never match an existing wiki page — the-// graph keeps accumulating cross-run duplicates. ES accepts the wildcard in-// both _source includes and the fields parameter.+// graph keeps accumulating cross-run duplicates. ES takes the wildcard verbatim+// in _source; the Infinity engine expands it against the table's real columns+// because Infinity's SQL binder rejects a partial wildcard (3013).var wikiSelectFields = []string{"page_type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd","entity_names_kwd", "summary_with_weight",diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 4aee8ab937..dbc62a0e46 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -620,10 +620,7 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datif err != nil {return nil, err}- fromKwds := make([]string, 0, len(entityChunks))- for _, c := range entityChunks {- fromKwds = append(fromKwds, firstStringValue(c["slug_kwd"]))- }+ fromKwds := wikiEntitySlugs(entityChunks)graph := &WikiGraph{Entities: []WikiGraphEntity{},Relations: []WikiGraphRelation{},@@ -689,6 +686,21 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datreturn graph, nil}+// wikiEntitySlugs returns the entity slugs the graph's relation lookup filters+// on. Rows without a slug are skipped: a blank value would render as+// filter_fulltext('from_kwd', ”), which Infinity rejects with 3052 ("Trying to+// match: on fields: from_kwd failed") and fails the whole graph request, while+// ES happily returns an empty result.+func wikiEntitySlugs(entityChunks []map[string]interface{}) []string {+ slugs := make([]string, 0, len(entityChunks))+ for _, c := range entityChunks {+ if slug := firstStringValue(c["slug_kwd"]); slug != "" {+ slugs = append(slugs, slug)+ }+ }+ return slugs+}+// WikiAlteration is the alteration summary for a dataset's wiki artifacts.type WikiAlteration struct {Removed int `json:"removed"`diff --git a/internal/service/dataset_artifact_topic_test.go b/internal/service/dataset_artifact_topic_test.goindex 05bb39d496..8bc5efcac5 100644--- a/internal/service/dataset_artifact_topic_test.go+++ b/internal/service/dataset_artifact_topic_test.go@@ -28,6 +28,32 @@ func TestAggregateWikiTopicItemsUsesMaterializedPaths(t *testing.T) {}}+// TestWikiEntitySlugsSkipsBlankSlugs pins the GetWikiGraph guard: an entity row+// without a slug must not reach the relation filter, because the engine renders+// it as filter_fulltext('from_kwd', ”) and Infinity fails the whole query with+// "Trying to match: on fields: from_kwd failed" (3052).+func TestWikiEntitySlugsSkipsBlankSlugs(t *testing.T) {+ got := wikiEntitySlugs([]map[string]interface{}{+ {"slug_kwd": "topic/出师表"},+ {"slug_kwd": ""},+ {"slug_kwd": nil},+ {}, // phantom row: the engine projected nothing+ {"slug_kwd": "person/诸葛亮"},+ })+ want := []string{"topic/出师表", "person/诸葛亮"}+ if len(got) != len(want) {+ t.Fatalf("wikiEntitySlugs = %#v, want %#v", got, want)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("wikiEntitySlugs = %#v, want %#v", got, want)+ }+ }+ if slugs := wikiEntitySlugs(nil); len(slugs) != 0 {+ t.Errorf("wikiEntitySlugs(nil) = %#v, want empty", slugs)+ }+}+func TestAggregateWikiTopicItemsIgnoresPathCase(t *testing.T) {items := aggregateWikiTopicItems([]map[string]interface{}{{"topic_kwd": "Knowledge/Core"},diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.goindex bfe0ad8c53..2addd09374 100644--- a/internal/service/nlp/datasetnav.go+++ b/internal/service/nlp/datasetnav.go@@ -59,10 +59,11 @@ const ()// navNodeSelectFields are the columns every nav tree read needs: the label-// (title_kwd), the Python-era name column, the parent edge, the payload-// (description), and the leaf's document id.+// (title_kwd, plus docnm_kwd for rows Infinity folded the label into), the+// Python-era name column, the parent edge, the payload (description), and the+// leaf's document id.var navNodeSelectFields = []string{- "name", "title_kwd", "parent_kwd", "depth_int",+ "name", "title_kwd", "docnm_kwd", "parent_kwd", "depth_int","content_with_weight", "doc_count_int", "type_kwd", "doc_id",}@@ -155,6 +156,11 @@ func (s *NavService) navSearch(ctx context.Context, tenantID, kbID string, filteSelectFields: selectFields,Filter: merged,MatchExprs: matchExprs,+ // Nav rows carry available_int=0, and Infinity adds available_int=1 to+ // every MATCH query unless the caller opts out (shouldDefaultAvailableFilter):+ // nav's own vector/keyword reads would otherwise see none of its rows. ES+ // never filters available_int; the compile_kwd filter scopes the read.+ IncludeUnavailable: true,}res, err := de.Search(ctx, req)if err != nil {@@ -338,13 +344,26 @@ func (s *NavService) navNodesFromRows(rows []map[string]interface{}) []nav.NavNoreturn nodes}+// navRowLabel returns the nav row's stored label: the value parent_kwd+// references and the tree renders. Go's writer stores it in title_kwd, Python's+// in name. Rows persisted before Infinity kept title_kwd as a column of its own+// carry it only in docnm (see engine/infinity transformChunkFields), which reads+// back as docnm_kwd; without that fallback the API answered the description's+// first line as a cluster name, so the tree's child lookup by name (the frontend+// expands with GET /navigation/<name>/children) matched no row.+func navRowLabel(row map[string]interface{}) string {+ for _, col := range []string{"name", "title_kwd", "docnm_kwd"} {+ if v := firstStringValue(row[col]); v != "" {+ return v+ }+ }+ return ""+}+// navClusterRowKey is the key a cluster's children reference through parent_kwd:// Python's writer stores it in `name`, Go's in `title_kwd`.func navClusterRowKey(row map[string]interface{}) string {- if name := firstStringValue(row["name"]); name != "" {- return name- }- return firstStringValue(row["title_kwd"])+ return navRowLabel(row)}// navDocRowKey / navClusterKey namespace the two row identities so a cluster and@@ -377,10 +396,7 @@ func navKeywordExpressions(keywords string) []interface{} {// is written. An explicit readable `name` column (Python's writer stores one)// wins when present.func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string) nav.NavNode {- name := firstStringValue(row["title_kwd"])- if n := firstStringValue(row["name"]); n != "" {- name = n- }+ name := navRowLabel(row)// A raw id (doc_id or "cluster_<hash>") is not a human-readable name; fall// back to a title derived from the payload description. Cluster names are the// child-lookup key (parent_kwd references them verbatim), so they must stay@@ -526,7 +542,7 @@ func (s *NavService) Search(ctx context.Context, tenantID, kbID, query string, efunc (s *NavService) navScan(ctx context.Context, tenantID, kbID string, filter map[string]interface{}, f64 []float64, limit int, allowed map[string]struct{}) ([]nav.NavHit, error) {chunks, _, err := s.navSearch(ctx, tenantID, kbID,filter,- []string{"type_kwd", "title_kwd", "doc_id", "doc_ids_kwd", "_score"}, 0, limit,+ []string{"type_kwd", "title_kwd", "docnm_kwd", "doc_id", "doc_ids_kwd", "_score"}, 0, limit,[]interface{}{&types.MatchDenseExpr{VectorColumnName: fmt.Sprintf("q_%d_vec", len(f64)),EmbeddingData: f64,@@ -542,7 +558,7 @@ func (s *NavService) navScan(ctx context.Context, tenantID, kbID string, filterfor _, c := range chunks {h := nav.NavHit{Type: firstStringValue(c["type_kwd"]),- Name: firstStringValue(c["title_kwd"]),+ Name: navRowLabel(c),DocID: firstStringValue(c["doc_id"]),DocIDs: firstStringSlice(c["doc_ids_kwd"]),}@@ -647,7 +663,7 @@ func (s *NavService) SummariesByDocIDs(ctx context.Context, tenantID, kbID strin"type_kwd": []string{"nav_doc"},"doc_id": docIDs,}),- []string{"type_kwd", "name", "title_kwd", "content_with_weight", "doc_id"},+ []string{"type_kwd", "name", "title_kwd", "docnm_kwd", "content_with_weight", "doc_id"},0, len(docIDs), nil)if err != nil {return map[string]string{}@@ -669,12 +685,9 @@ func (s *NavService) SummariesByDocIDs(ctx context.Context, tenantID, kbID strin}if summary == "" {// No description in the payload: fall back to the stored label- // (readable name, else title_kwd), rejecting raw ids — a 32-hex id or- // "cluster_<hash>" is not a human label.- summary = firstStringValue(c["name"])- if summary == "" {- summary = firstStringValue(c["title_kwd"])- }+ // (readable name, else title_kwd/docnm), rejecting raw ids — a+ // 32-hex id or "cluster_<hash>" is not a human label.+ summary = navRowLabel(c)if graphIsRawID(summary) {summary = ""}@@ -726,7 +739,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"type_kwd": []string{nav.TypeNavDoc},"doc_id": []string{in.DocID},}),- []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd"}, 0, 1, nil)+ []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd", "docnm_kwd"}, 0, 1, nil)if err != nil {return err}@@ -737,7 +750,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error// summary's first line, or the model-facing readers would keep seeing a// file name. The tokenized text is (re)built when it is missing.title := navDocTitle(in.Summary)- titleStale := firstStringValue(existing[0]["title_kwd"]) != title+ titleStale := navRowLabel(existing[0]) != titleltksStale := firstStringValue(existing[0]["content_ltks"]) == "" ||firstStringValue(existing[0]["content_sm_ltks"]) == ""if payload, ok := existing[0]["content_with_weight"].(string); ok {@@ -924,6 +937,48 @@ func navClusterID(tenantID, kbID, name string) string {return "dataset_nav_cluster_" + contentHash8(tenantID+"\x00"+kbID+"\x00"+name)}+// navClusterCondition addresses one nav cluster row by its deterministic row id,+// the way Python's dataset_nav does (every cluster update/delete resolves+// _nav_cluster_id and read-modify-writes that row). The label is not usable as a+// key: it is an LLM-written name that can share words with another cluster, and+// Infinity cannot match a multi-token value on these columns at all (see+// engine/infinity keywordFilterCondition). The row id is always single-token.+func navClusterCondition(tenantID, kbID, clusterName string) map[string]interface{} {+ return navFilter(map[string]interface{}{+ "id": []string{navClusterID(tenantID, kbID, clusterName)},+ })+}++// navChildRehomeCondition builds the condition that moves one split child under+// its new parent, or nil when the child cannot be addressed at all.+//+// The row id is preferred: the label is empty for rows the Infinity write path+// folded into docnm, and two children can share a summary's first line, so a+// label filter can miss the row or hit a sibling. A nav_doc leaf without a row id+// is still identifiable by its document. Nothing else is safe — with blank+// keywords dropped, a type-only condition would match every row of that type in+// the dataset.+func navChildRehomeCondition(child map[string]interface{}, kbID string) map[string]interface{} {+ typ := firstStringValue(child["type_kwd"])+ if typ == "" {+ return nil+ }+ cond := map[string]interface{}{+ "compile_kwd": []string{navCompileKwd},+ "kb_id": kbID,+ }+ if rowID := firstStringValue(child["id"]); rowID != "" {+ cond["id"] = []string{rowID}+ return cond+ }+ if docID := firstStringValue(child["doc_id"]); typ == "nav_doc" && docID != "" {+ cond["type_kwd"] = []string{typ}+ cond["doc_id"] = []string{docID}+ return cond+ }+ return nil+}+// cleanupEmptyCluster deletes a nav_cluster that holds no docs and no child// clusters, then recurses to its parent (cascade, mirroring Python// _cleanup_empty_cluster). It is called by RemoveDoc after deleting a nav_doc so@@ -937,7 +992,7 @@ func (s *NavService) cleanupEmptyCluster(ctx context.Context, tenantID, kbID, clreturn err}chunks, _, err := s.navSearch(ctx, tenantID, kbID,- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ navClusterCondition(tenantID, kbID, clusterName),[]string{"id", "parent_kwd", "doc_count_int", "doc_ids_kwd"}, 0, 1, nil)if err != nil {return err@@ -994,7 +1049,7 @@ func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string,"type_kwd": []string{"nav_cluster"},"parent_kwd": []string{parent},}),- []string{"title_kwd", "_score"}, 0, 1,+ []string{"title_kwd", "docnm_kwd", "_score"}, 0, 1,[]interface{}{&types.MatchDenseExpr{VectorColumnName: fmt.Sprintf("q_%d_vec", len(f64)),EmbeddingData: f64,@@ -1009,7 +1064,7 @@ func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string,if len(chunks) == 0 {break}- name := firstStringValue(chunks[0]["title_kwd"])+ name := navRowLabel(chunks[0])sim := rowScore(chunks[0])// Adopt-and-overwrite, mirroring Python _find_best_cluster: the result is// the DEEPEST cluster reached while the descent keeps clearing@@ -1058,7 +1113,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus// engine column q_<dim>_vec, e.g. from the doc being appended) is selected so// the split siblings can inherit a representative vector for KNN routing; an// empty vecCol skips it (callers without a known dim).- selectFields := []string{"id", "doc_id", "doc_count_int", "doc_ids_kwd", "title_kwd", "type_kwd"}+ selectFields := []string{"id", "doc_id", "doc_count_int", "doc_ids_kwd", "title_kwd", "docnm_kwd", "type_kwd"}if vecCol != "" {selectFields = append(selectFields, vecCol)}@@ -1092,7 +1147,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus// held docs; select id + doc_ids_kwd so the split can delete by the// engine-assigned row id and inherit the cluster's directly-held documents.clusterChunks, _, err := s.navSearch(ctx, tenantID, kbID,- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ navClusterCondition(tenantID, kbID, clusterName),[]string{"id", "doc_count_int", "doc_ids_kwd", "parent_kwd", "depth_int"}, 0, 1, nil)if err != nil {return err@@ -1121,23 +1176,23 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clusids []stringvec []float32}{}+ // Every child must be addressable before the first write. The rehome points+ // children at splitA/splitB, and those rows are only inserted after the+ // original cluster is deleted, so a child that cannot be rehomed would be left+ // hanging off a name no query can reach — refuse the split instead of+ // half-moving the subtree.+ for _, child := range children {+ if navChildRehomeCondition(child, kbID) == nil {+ return fmt.Errorf("datasetnav: refusing to split cluster %q: child %q carries no row id, document id or type",+ clusterName, navRowLabel(child))+ }+ }for i, child := range children {acc := &accAif i%2 == 1 {acc = &accB}- title := firstStringValue(child["title_kwd"])typ := firstStringValue(child["type_kwd"])- childID := firstStringValue(child["id"])- if title == "" && childID == "" {- continue- }- if typ == "" {- // The engine returned a child without a type discriminator; skip it- // so we never rehome with an empty-type filter that matches nothing- // (leaving the child orphaned under a deleted cluster name).- continue- }// Aggregate this child's doc count + doc ids into the target split.if c := intValue(child["doc_count_int"]); c > 0 {acc.count += c@@ -1150,16 +1205,15 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, cluschildDocIDs = append(childDocIDs, firstStringValue(child["doc_id"]))}acc.ids = appendUnique(acc.ids, childDocIDs)- // Rehome the child by its row id (precise) rather than an empty-safe- // type+title filter, so the UpdateChunks filter always matches.- upd := map[string]interface{}{"parent_kwd": accTarget(i, splitA, splitB)}- if err := de.UpdateChunks(ctx,- map[string]interface{}{- "compile_kwd": []string{navCompileKwd},- "type_kwd": []string{typ},- "title_kwd": []string{title},- "kb_id": kbID,- }, upd, s.navIndexName(tenantID), kbID); err != nil {+ childCond := navChildRehomeCondition(child, kbID)+ if childCond == nil {+ // Unreachable after the pre-pass above; kept so a future caller+ // cannot fall through to a condition that matches the whole dataset.+ return fmt.Errorf("datasetnav: refusing to rehome %s child %q", typ, navRowLabel(child))+ }+ if err := de.UpdateChunks(ctx, childCond,+ map[string]interface{}{"parent_kwd": accTarget(i, splitA, splitB)},+ s.navIndexName(tenantID), kbID); err != nil {return err}}@@ -1312,7 +1366,7 @@ func (s *NavService) mergeClusterDescription(ctx context.Context, de engine.DocEreturn nil}rows, _, err := s.navSearch(ctx, tenantID, kbID,- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ navClusterCondition(tenantID, kbID, clusterName),[]string{"content_with_weight"}, 0, 1, nil)if err != nil {return err@@ -1351,12 +1405,7 @@ func (s *NavService) mergeClusterDescription(ctx context.Context, de engine.DocE}}return de.UpdateChunks(ctx,- map[string]interface{}{- "compile_kwd": []string{navCompileKwd},- "type_kwd": []string{"nav_cluster"},- "title_kwd": []string{clusterName},- "kb_id": kbID,- }, updates, s.navIndexName(tenantID), kbID)+ navClusterCondition(tenantID, kbID, clusterName), updates, s.navIndexName(tenantID), kbID)}// llmMergeDescription merges a set of source descriptions into one via the@@ -1460,7 +1509,7 @@ func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string// representative vector; pass "" when the caller has no known dimension.func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID, vecCol string) error {chunks, _, err := s.navSearch(ctx, tenantID, kbID,- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ navClusterCondition(tenantID, kbID, clusterName),[]string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)if err != nil {return err@@ -1491,17 +1540,14 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngineif !found {count++}- // Pin the update to the nav_cluster row only: a regular chunk sharing the- // same title_kwd must never be clobbered. The read-modify-write here is- // expected to run under a per-dataset lock held by the UpsertDoc caller;- // without it, concurrent appends to the same cluster can lose updates.+ // The condition addresses the cluster by its row id, so no other row can be+ // caught by the update — neither a regular chunk that happens to carry a+ // similar title nor a second cluster whose readable name shares words with+ // this one. The read-modify-write here is expected to run under a per-dataset+ // lock held by the UpsertDoc caller; without it, concurrent appends to the+ // same cluster can lose updates.if err := de.UpdateChunks(ctx,- map[string]interface{}{- "compile_kwd": []string{navCompileKwd},- "type_kwd": []string{"nav_cluster"},- "title_kwd": []string{clusterName},- "kb_id": kbID,- },+ navClusterCondition(tenantID, kbID, clusterName),map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},s.navIndexName(tenantID), kbID); err != nil {return err@@ -1602,7 +1648,7 @@ func (s *NavService) findClusterContainingDoc(ctx context.Context, tenantID, kbIres, err := de.Search(ctx, &types.SearchRequest{IndexNames: []string{s.navIndexName(tenantID)},KbIDs: []string{kbID},- SelectFields: []string{"title_kwd", "doc_ids_kwd"},+ SelectFields: []string{"title_kwd", "docnm_kwd", "doc_ids_kwd"},Filter: map[string]interface{}{"type_kwd": "nav_cluster", "compile_kwd": []string{navCompileKwd}},Offset: offset,Limit: pageSize,@@ -1613,7 +1659,7 @@ func (s *NavService) findClusterContainingDoc(ctx context.Context, tenantID, kbIfor _, row := range res.Chunks {for _, d := range firstStringSlice(row["doc_ids_kwd"]) {if d == docID {- return firstStringValue(row["title_kwd"]), nil+ return navRowLabel(row), nil}}}@@ -1632,7 +1678,7 @@ func (s *NavService) removeDocFromCluster(ctx context.Context, tenantID, kbID, creturn err}chunks, _, err := s.navSearch(ctx, tenantID, kbID,- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),+ navClusterCondition(tenantID, kbID, clusterName),[]string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)if err != nil {return err@@ -1654,12 +1700,7 @@ func (s *NavService) removeDocFromCluster(ctx context.Context, tenantID, kbID, ccount--}return de.UpdateChunks(ctx,- map[string]interface{}{- "compile_kwd": []string{navCompileKwd},- "type_kwd": []string{"nav_cluster"},- "title_kwd": []string{clusterName},- "kb_id": kbID,- },+ navClusterCondition(tenantID, kbID, clusterName),map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},s.navIndexName(tenantID), kbID)}diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.goindex 5a70733828..f900aa272b 100644--- a/internal/service/nlp/datasetnav_test.go+++ b/internal/service/nlp/datasetnav_test.go@@ -21,6 +21,10 @@ import (type memNavEngine struct {rows []map[string]interface{}nextID int++ // searches records every SearchRequest so a test can pin the request shape+ // (e.g. that nav reads opt out of the retrieval-only available_int=1 default).+ searches []*types.SearchRequest}func newMemNavEngine() *memNavEngine { return &memNavEngine{} }@@ -28,8 +32,16 @@ func newMemNavEngine() *memNavEngine { return &memNavEngine{} }func (m *memNavEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _ string, datasetID string) ([]string, error) {ids := make([]string, 0, len(chunks))for _, c := range chunks {- m.nextID++- id := "nav" + strconvItoa(m.nextID)+ // A caller-provided row id is kept, exactly as the real engines do it+ // (elasticsearch: chunk["id"] becomes the document id; infinity: the id+ // column is written verbatim). Nav addresses its cluster rows by the+ // deterministic id they were written with, so generating a fresh one here+ // would make every update/delete miss.+ id, _ := c["id"].(string)+ if id == "" {+ m.nextID+++ id = "nav" + strconvItoa(m.nextID)+ }cp := make(map[string]interface{}, len(c)+2)for k, v := range c {cp[k] = v@@ -68,6 +80,7 @@ func (m *memNavEngine) DeleteChunks(_ context.Context, cond map[string]interface}func (m *memNavEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ m.searches = append(m.searches, req)var matched []map[string]interface{}// This engine implements the DENSE leg only. A lexical match expression is// deliberately a no-op rather than a dense one: treating a text expression@@ -1414,8 +1427,13 @@ func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {// Seed the overfull cluster directly: one nav_cluster (parent=root sentinel,// depth 1) carrying 55 nav_doc children.idx := sTestNavIndex(ns)+ // The seeded rows carry the row ids production writes: nav addresses its+ // clusters by id (navClusterID, exactly like Python's _nav_cluster_id), so a+ // fixture without them would leave the locate/update/delete steps matching no+ // row at all.rows := []map[string]interface{}{{+ "id": navClusterID("t1", "kb1", clusterName),"compile_kwd": navCompileKwd,"available_int": 0,"type_kwd": "nav_cluster",@@ -1428,6 +1446,7 @@ func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {}for i := 0; i < 55; i++ {rows = append(rows, map[string]interface{}{+ "id": navDocID("t1", "kb1", fmt.Sprintf("d%02d", i)),"compile_kwd": navCompileKwd,"available_int": 0,"type_kwd": "nav_doc",@@ -1695,3 +1714,237 @@ func intValAny(v any) int {}return 0}++// TestNavService_ReadsIncludeUnavailableRows pins that nav reads opt out of the+// retrieval-only available_int=1 default Infinity applies to every MATCH query:+// nav rows carry available_int=0, so without it the cluster descent and navScan+// would see an empty tree. ES never filters available_int.+func TestNavService_ReadsIncludeUnavailableRows(t *testing.T) {+ eng := newMemNavEngine()+ ns := newTestNav(eng)+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", "aaa")); err != nil {+ t.Fatal(err)+ }+ if len(eng.rows) == 0 {+ t.Fatal("UpsertDoc wrote no nav row")+ }+ if v := eng.rows[0]["available_int"]; v != 0 {+ t.Fatalf("nav row available_int = %#v, want 0", v)+ }++ if _, _, err := ns.ListClusters(t.Context(), "t1", "kb1", "", 0, 10); err != nil {+ t.Fatal(err)+ }+ if len(eng.searches) == 0 {+ t.Fatal("ListClusters issued no search")+ }+ for i, req := range eng.searches {+ if !req.IncludeUnavailable {+ t.Errorf("search %d did not opt out of the available_int=1 default (SelectFields=%#v)", i, req.SelectFields)+ }+ }+}++// TestNavService_MultiTokenClusterUpdatesByRowID pins the write path for a+// cluster whose label is multi-token and whose title_kwd is empty. Such a row is+// addressable only by its deterministic row id: a name filter on Infinity matched+// nothing, so the membership/count update and the cleanup lookup silently+// no-op'd.+func TestNavService_MultiTokenClusterUpdatesByRowID(t *testing.T) {+ const clusterName = "Imperial Memorial and Governance Advice 59cbfbef"+ clusterRowID := navClusterID("t1", "kb1", clusterName)++ eng := newMemNavEngine()+ eng.rows = append(eng.rows,+ map[string]interface{}{+ "id": clusterRowID,+ "kb_id": "kb1", "compile_kwd": navCompileKwd, "type_kwd": "nav_cluster",+ "available_int": 0,+ // The legacy Infinity layout: the label only reached docnm.+ "title_kwd": "", "docnm_kwd": clusterName,+ "parent_kwd": navRootParent, "depth_int": 1,+ "doc_count_int": 2, "doc_ids_kwd": []string{"d1", "d2"},+ "content_with_weight": `{"description":"two memorials"}`,+ },+ map[string]interface{}{+ "id": navDocID("t1", "kb1", "d1"),+ "kb_id": "kb1", "compile_kwd": navCompileKwd, "type_kwd": "nav_doc",+ "available_int": 0, "title_kwd": "memorial of d1",+ "parent_kwd": clusterName, "depth_int": 2,+ "doc_id": "d1", "doc_count_int": 1,+ })+ ns := newTestNav(eng)++ // The read side must answer the stored key so the frontend can expand it.+ clusters, _, err := ns.ListClusters(context.Background(), "t1", "kb1", "", 0, 10)+ if err != nil {+ t.Fatalf("ListClusters: %v", err)+ }+ if len(clusters) != 1 || clusters[0].Name != clusterName {+ t.Fatalf("clusters = %#v, want the stored key %q", clusters, clusterName)+ }++ if err := ns.RemoveDoc(context.Background(), "t1", "kb1", "d1"); err != nil {+ t.Fatalf("RemoveDoc: %v", err)+ }++ row := navRowByID(eng, clusterRowID)+ if row == nil {+ t.Fatal("cluster row vanished")+ }+ if got := intValAny(row["doc_count_int"]); got != 1 {+ t.Errorf("cluster doc_count_int = %d, want 1 (the update must reach a row whose title_kwd is empty)", got)+ }+ if ids := firstStringSlice(row["doc_ids_kwd"]); len(ids) != 1 || ids[0] != "d2" {+ t.Errorf("cluster doc_ids_kwd = %v, want [d2]", ids)+ }+}++// TestNavService_MaybeSplitClusterRefusesUnidentifiableChild pins the guard that+// dropping blank keyword values made necessary: a child with neither a row id nor+// a document id must abort the split. Falling back to a label would leave a+// type-only condition, and with the blank label dropped before rendering that+// condition reparents every row of that type in the dataset instead of failing.+func TestNavService_MaybeSplitClusterRefusesUnidentifiableChild(t *testing.T) {+ const clusterName = "overfull_no_ids"+ eng := newMemNavEngine()+ eng.rows = append(eng.rows, map[string]interface{}{+ "id": navClusterID("t1", "kb1", clusterName), "kb_id": "kb1",+ "compile_kwd": navCompileKwd, "type_kwd": "nav_cluster",+ "title_kwd": clusterName, "docnm_kwd": clusterName,+ "parent_kwd": navRootParent, "depth_int": 1,+ "doc_count_int": navMaxDocsPerCluster + 1, "doc_ids_kwd": []string{},+ })+ // Directly seeded (not via InsertChunks) so they carry NO row id, and no+ // doc_id either: every fallback the rehome could use is unavailable.+ for i := 0; i <= navMaxDocsPerCluster; i++ {+ eng.rows = append(eng.rows, map[string]interface{}{+ "kb_id": "kb1", "compile_kwd": navCompileKwd, "type_kwd": "nav_doc",+ "title_kwd": fmt.Sprintf("kid %02d", i), "parent_kwd": clusterName,+ "depth_int": 2, "doc_count_int": 1,+ })+ }+ ns := newTestNav(eng)++ if err := ns.maybeSplitCluster(context.Background(), "t1", "kb1", clusterName, ""); err == nil {+ t.Fatal("a child without a row id or document id must abort the split")+ }+ // Nothing may be reparented, and the original cluster must still be there.+ reparented := 0+ originalPresent := false+ for _, row := range eng.rows {+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] == clusterName {+ originalPresent = true+ }+ if p, _ := row["parent_kwd"].(string); strings.HasSuffix(p, ":A") || strings.HasSuffix(p, ":B") {+ reparented+++ }+ }+ if reparented != 0 {+ t.Errorf("%d rows were reparented by a type-only condition", reparented)+ }+ if !originalPresent {+ t.Error("the original cluster must not be deleted when the split aborts")+ }+}++// TestNavService_MaybeSplitClusterValidatesEveryChildFirst pins the pre-pass: an+// unaddressable child anywhere in the list must stop the split before the first+// write. The rehome points children at splitA/splitB, whose rows are inserted+// only after the original cluster is deleted, so aborting mid-loop would leave+// the already-moved children hanging off a name no query can reach.+func TestNavService_MaybeSplitClusterValidatesEveryChildFirst(t *testing.T) {+ const clusterName = "overfull_second_child_broken"+ eng := newMemNavEngine()+ eng.rows = append(eng.rows, map[string]interface{}{+ "id": navClusterID("t1", "kb1", clusterName), "kb_id": "kb1",+ "compile_kwd": navCompileKwd, "type_kwd": "nav_cluster",+ "title_kwd": clusterName, "docnm_kwd": clusterName,+ "parent_kwd": navRootParent, "depth_int": 1,+ "doc_count_int": navMaxDocsPerCluster + 1, "doc_ids_kwd": []string{},+ })+ // The first child IS addressable, so a loop that validated lazily would have+ // reparented it before reaching the broken child below.+ eng.rows = append(eng.rows, map[string]interface{}{+ "id": navDocID("t1", "kb1", "d0"), "kb_id": "kb1", "compile_kwd": navCompileKwd,+ "type_kwd": "nav_doc", "title_kwd": "good child", "doc_id": "d0",+ "parent_kwd": clusterName, "depth_int": 2, "doc_count_int": 1,+ })+ for i := 1; i <= navMaxDocsPerCluster; i++ {+ eng.rows = append(eng.rows, map[string]interface{}{+ "kb_id": "kb1", "compile_kwd": navCompileKwd, "type_kwd": "nav_doc",+ "title_kwd": fmt.Sprintf("broken %02d", i), "parent_kwd": clusterName,+ "depth_int": 2, "doc_count_int": 1,+ })+ }+ ns := newTestNav(eng)++ if err := ns.maybeSplitCluster(context.Background(), "t1", "kb1", clusterName, ""); err == nil {+ t.Fatal("an unaddressable child must abort the split")+ }+ for _, row := range eng.rows {+ if p, _ := row["parent_kwd"].(string); strings.HasSuffix(p, ":A") || strings.HasSuffix(p, ":B") {+ t.Fatalf("row %v was reparented before every child had been validated", row["id"])+ }+ }+ if navRowByID(eng, navClusterID("t1", "kb1", clusterName)) == nil {+ t.Error("the original cluster must not be deleted when the split aborts")+ }+}++// navRowByID returns the stored row with the given id, or nil.+func navRowByID(eng *memNavEngine, id string) map[string]interface{} {+ for _, r := range eng.rows {+ if s, _ := r["id"].(string); s == id {+ return r+ }+ }+ return nil+}++// TestNavService_LabelsFallBackToDocnm pins the label read for rows persisted+// before Infinity kept title_kwd as its own column (the label reached docnm+// only). The name must be the STORED key, not a display fallback: the frontend+// looks children up with it, so a description-derived name left the tree flat.+func TestNavService_LabelsFallBackToDocnm(t *testing.T) {+ const clusterName = "忠臣进谏与托孤遗志 27577543"+ eng := newMemNavEngine()+ eng.rows = append(eng.rows,+ map[string]interface{}{+ "id": "dataset_nav_cluster_1", "compile_kwd": navCompileKwd, "type_kwd": "nav_cluster",+ "kb_id": "kb1", "available_int": 0, "title_kwd": "", "docnm_kwd": clusterName,+ "parent_kwd": "root", "depth_int": 0, "doc_count_int": 1, "doc_ids_kwd": []string{"d1"},+ },+ map[string]interface{}{+ "id": "dataset_nav_doc_1", "compile_kwd": navCompileKwd, "type_kwd": "nav_doc",+ "kb_id": "kb1", "available_int": 0, "title_kwd": "", "docnm_kwd": "出师表中的忠诚与北伐决心",+ "parent_kwd": clusterName, "depth_int": 1, "doc_count_int": 1, "doc_id": "d1",+ })+ ns := newTestNav(eng)++ clusters, _, err := ns.ListClusters(context.Background(), "t1", "kb1", "", 0, 10)+ if err != nil {+ t.Fatalf("ListClusters: %v", err)+ }+ if len(clusters) != 1 {+ t.Fatalf("clusters = %#v, want exactly one root cluster", clusters)+ }+ if clusters[0].Name != clusterName {+ t.Fatalf("cluster name = %q, want the stored key %q", clusters[0].Name, clusterName)+ }+ if !clusters[0].HasChildren {+ t.Error("cluster must expose has_children so the frontend can expand it")+ }++ // The frontend expands with the name it was given.+ children, _, err := ns.ListChildren(context.Background(), "t1", "kb1", clusters[0].Name, "", 0, 10)+ if err != nil {+ t.Fatalf("ListChildren: %v", err)+ }+ if len(children) != 1 {+ t.Fatalf("children = %#v, want the doc leaf", children)+ }+ if children[0].Name != "出师表中的忠诚与北伐决心" || children[0].DocID != "d1" {+ t.Fatalf("child = %#v, want the doc leaf labelled from docnm_kwd", children[0])+ }+}