Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(infinity,nav): restore the navigation tree and document structure graph (#20339)

qinling02104d1fa96a8d9e
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 00fa7865ea..dae7635a7b 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -431,6 +431,14 @@ func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface
// Insert chunks to dataset
_, err = table.Insert(insertChunks)
if err != nil {
+ common.Warn("Infinity insert failed",
+ zap.String("tableName", tableName),
+ zap.Int("rows", len(insertChunks)),
+ zap.Strings("rowFields", rowFieldNames(insertChunks)),
+ zap.Error(err))
+ if isConnectionLevelError(err) {
+ e.client.dropConnectionFor(db, "InsertChunks")
+ }
return nil, fmt.Errorf("failed to insert chunks to dataset: %w", err)
}
@@ -592,6 +600,13 @@ func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interfac
common.Info(fmt.Sprintf("INFINITY update: table=%s, filter=%s, newValue=%v", tableName, filter, newValue))
_, err = table.Update(filter, newValue)
if err != nil {
+ common.Warn("Infinity update failed",
+ zap.String("tableName", tableName),
+ zap.String("filter", filter),
+ zap.Error(err))
+ if isConnectionLevelError(err) {
+ e.client.dropConnectionFor(db, "UpdateChunks")
+ }
return fmt.Errorf("failed to update chunks: %w", err)
}
@@ -746,6 +761,13 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
delResp, err := table.Delete(filter)
if err != nil {
+ common.Warn("Infinity delete failed",
+ zap.String("tableName", tableName),
+ zap.String("filter", filter),
+ zap.Error(err))
+ if isConnectionLevelError(err) {
+ e.client.dropConnectionFor(db, "DeleteChunks")
+ }
return 0, fmt.Errorf("failed to delete: %w", err)
}
@@ -839,6 +861,14 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
}
}
+ // Infinity binds a "_score" projection to SCORE(), which it allows only with
+ // MATCH TEXT/TENSOR or Fusion — otherwise 3013. Elasticsearch tolerates
+ // "_score" for every shape, so callers pass it unconditionally (the nav
+ // cluster descent and navScan do): drop theirs and re-add the one form this
+ // shape accepts. calculateScores() materializes "_score" for readers either
+ // way.
+ outputColumns = stripScorePseudoColumns(outputColumns)
+
if hasTextMatch || hasVectorMatch {
if hasTextMatch {
outputColumns = append(outputColumns, "score()")
@@ -864,23 +894,6 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
outputColumns = append(outputColumns, "row_id()")
}
- // Strip score pseudo-columns when there's no match expression — Infinity
- // rejects SCORE()/SCORE_FACTORS() without MATCH TEXT/TENSOR/Fusion with
- // "InfinityException(3013)". This protects callers (e.g. the no-match
- // fallback in retrieval.go) that reuse a SelectFields list containing
- // "_score" across both matched and unmatched queries.
- if !hasTextMatch && !hasVectorMatch {
- filtered := outputColumns[:0]
- for _, c := range outputColumns {
- switch c {
- case "_score", "SCORE", "score()", "similarity()":
- continue
- }
- filtered = append(filtered, c)
- }
- outputColumns = filtered
- }
-
outputColumns = convertSelectFields(outputColumns, isSkillIndex)
if hasVectorMatch && matchDense != nil && matchDense.VectorColumnName != "" {
outputColumns = append(outputColumns, matchDense.VectorColumnName)
@@ -930,6 +943,13 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
var allResults []map[string]interface{}
totalHits := int64(0)
+ // queryErr keeps the first per-table failure. A failed table query used to be
+ // swallowed (warn + continue), which turned schema/query mismatches into
+ // silently empty results — a partial-wildcard projection ("q_*_vec", 3013)
+ // made the knowledge-compile reader see no products, so the dataset
+ // navigation tree silently never materialized. Failures stay tolerable while
+ // another table answered, but surface when nothing did.
+ var queryErr error
for _, indexName := range req.IndexNames {
var tableNames []string
@@ -989,28 +1009,50 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
for _, tableName := range tableNames {
tbl, err := db.GetTable(tableName)
if err != nil {
+ // A dataset whose table does not exist yet is normal (tables are
+ // created on first write): skip it. A transport failure leaves the
+ // socket unusable, so it is recorded like the failures below.
+ if isConnectionLevelError(err) {
+ e.client.dropConnectionFor(db, "Search")
+ if queryErr == nil {
+ queryErr = fmt.Errorf("open table %s: %w", tableName, err)
+ }
+ }
continue
}
- tableFilterParts := append([]string(nil), filterParts...)
- if len(filterCopy) > 0 {
- var tableColumns map[string]struct {
- Type string
- Default interface{}
- }
- if hasJSONListFilter(filterCopy) {
- tableColumns, err = loadTableColumns(tbl)
- if err != nil {
- common.Warn("Failed to load Infinity columns for JSON-list filter",
- zap.String("tableName", tableName), zap.Error(err))
- continue
+ // Resolve partial-wildcard projections ("q_*_vec") against this table's
+ // real columns: Infinity parses "q_*_vec" as the arithmetic expression
+ // "q_" * "_vec" and fails to bind it (3013), while Elasticsearch expands
+ // the pattern in _source. Embedding dims differ per table, so this is
+ // decided per table. A bare "*" is left alone: Infinity supports it.
+ wildcardSelect := hasPartialWildcard(outputColumns)
+ var tableColumns map[string]struct {
+ Type string
+ Default interface{}
+ }
+ if wildcardSelect || (len(filterCopy) > 0 && hasJSONListFilter(filterCopy)) {
+ tableColumns, err = loadTableColumns(tbl)
+ if err != nil {
+ common.Warn("Failed to load Infinity columns for search",
+ zap.String("tableName", tableName), zap.Error(err))
+ if queryErr == nil {
+ queryErr = fmt.Errorf("load columns for %s: %w", tableName, err)
}
+ continue
}
+ }
+ tableOutputColumns := outputColumns
+ if wildcardSelect {
+ tableOutputColumns = expandSelectWildcards(outputColumns, tableColumns)
+ }
+ tableFilterParts := append([]string(nil), filterParts...)
+ if len(filterCopy) > 0 {
if condition := equivalentConditionToStr(filterCopy, tableColumns); condition != "" {
tableFilterParts = append(tableFilterParts, condition)
}
}
filterStr := strings.Join(tableFilterParts, " AND ")
- table := tbl.Output(outputColumns)
+ table := tbl.Output(tableOutputColumns)
var textFields []string
if matchText != nil && len(matchText.Fields) > 0 {
@@ -1200,12 +1242,25 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
// Execute query
df, err := table.ToDataFrame()
if err != nil {
+ // Log the full request, not just its shape: a failing search used
+ // to be indistinguishable from any other, which made a recurring
+ // 7018/EOF impossible to attribute to a caller.
common.Warn("Infinity query failed",
zap.String("tableName", tableName),
+ zap.String("filter", filterStr),
+ zap.Strings("selectFields", tableOutputColumns),
+ zap.Int("limit", pageSize),
+ zap.Int("offset", offset),
zap.Bool("hasTextMatch", hasTextMatch),
zap.Bool("hasVectorMatch", hasVectorMatch),
zap.Bool("hasFusion", fusionExpr != nil),
zap.Error(err))
+ if isConnectionLevelError(err) {
+ e.client.dropConnectionFor(db, "Search")
+ }
+ if queryErr == nil {
+ queryErr = fmt.Errorf("query table %s: %w", tableName, err)
+ }
continue
}
@@ -1213,6 +1268,17 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
searchChunks := make([]map[string]interface{}, 0)
for colName, colData := range df.ColumnData {
for i, val := range colData {
+ // A null cell must not materialize a row of its own. Infinity
+ // can return a single all-null cell for a projection it
+ // matched nothing for (observed with a full-text filter plus a
+ // json-list projection: source_chunk_ids came back as a nil
+ // slice while the other columns were empty), and fabricating
+ // one empty row out of it leaks phantom documents to callers —
+ // GetWikiGraph then read an empty slug_kwd off it and issued
+ // filter_fulltext('from_kwd', ''), which Infinity rejects.
+ if isEmptyEngineCell(val) {
+ continue
+ }
for len(searchChunks) <= i {
searchChunks = append(searchChunks, make(map[string]interface{}))
}
@@ -1220,6 +1286,12 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
}
}
+ // Infinity returns a Json/Array column as ONE cell holding one
+ // length-prefixed segment per row, so the loop above parks the whole
+ // column on row 0. Redistribute it before applyFieldMappings renames
+ // anything.
+ realignJSONColumnCells(searchChunks, df.ColumnData)
+
// Apply field name mapping and row_id handling
// Skill index uses different schema
// so we skip the document-specific field mappings
@@ -1289,6 +1361,12 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
allResults = allResults[:pageSize]
}
+ // Every table failed: return the error instead of an empty result set, so a
+ // backend-specific query mismatch cannot masquerade as "no data".
+ if len(allResults) == 0 && totalHits == 0 && queryErr != nil {
+ return nil, fmt.Errorf("infinity search failed on all tables: %w", queryErr)
+ }
+
common.Debug("Search in Infinity completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits))
return &types.SearchResult{
@@ -2180,6 +2258,138 @@ func convertSelectFields(output []string, isSkillIndex ...bool) []string {
return result
}
+// rowFieldNames returns the sorted field names of the first row, so a failed
+// insert logs which columns were sent without dumping values (a vector column
+// would make the log unusable).
+func rowFieldNames(rows []map[string]interface{}) []string {
+ if len(rows) == 0 {
+ return nil
+ }
+ names := make([]string, 0, len(rows[0]))
+ for k := range rows[0] {
+ names = append(names, k)
+ }
+ sort.Strings(names)
+ return names
+}
+
+// isEmptyEngineCell reports whether a cell Infinity returned carries no value:
+// an untyped null, or a nil slice/map/pointer. Infinity can hand back such a cell
+// for a projection it found no row for, so it must not turn into a row by itself.
+func isEmptyEngineCell(val interface{}) bool {
+ if val == nil {
+ return true
+ }
+ rv := reflect.ValueOf(val)
+ switch rv.Kind() {
+ case reflect.Slice, reflect.Map, reflect.Ptr, reflect.Interface:
+ return rv.IsNil()
+ }
+ return false
+}
+
+// scorePseudoColumns are the score columns a caller may put in a SelectFields
+// list. Elasticsearch returns "_score" for every query shape, so callers pass it
+// unconditionally; Infinity only accepts the SCORE()/SIMILARITY() function form,
+// and only together with a compatible MATCH.
+var scorePseudoColumns = map[string]bool{
+ "_score": true,
+ "SCORE": true,
+ "score()": true,
+ "similarity()": true,
+ "SIMILARITY": true,
+}
+
+// stripScorePseudoColumns removes caller-supplied score columns so Search can
+// add back the single form Infinity accepts for the query shape it is about to
+// run (none for a filter-only query, similarity() for MATCH VECTOR only,
+// score() when a MATCH TEXT/TENSOR or Fusion is present).
+func stripScorePseudoColumns(fields []string) []string {
+ out := make([]string, 0, len(fields))
+ for _, f := range fields {
+ if scorePseudoColumns[f] {
+ continue
+ }
+ out = append(out, f)
+ }
+ return out
+}
+
+// hasPartialWildcard reports whether any requested column contains a wildcard
+// without being the bare "*" Infinity reads as "all columns".
+func hasPartialWildcard(fields []string) bool {
+ for _, f := range fields {
+ if f != "*" && strings.Contains(f, "*") {
+ return true
+ }
+ }
+ return false
+}
+
+// expandSelectWildcards replaces partial wildcards ("q_*_vec" — the only
+// pattern RAGFlow uses) with the table columns they match. Infinity cannot
+// project a partial wildcard: it parses "q_*_vec" as the arithmetic expression
+// "q_" * "_vec" and fails to bind it ("Fail to bind the expression: q_", 3013),
+// whereas Elasticsearch expands the same pattern in _source. Embedding dims
+// differ per dataset table, so the expansion is resolved per table. A pattern
+// matching no column is dropped, so one unmatched pattern cannot fail the query.
+func expandSelectWildcards(fields []string, columns map[string]struct {
+ Type string
+ Default interface{}
+}) []string {
+ names := make([]string, 0, len(columns))
+ for name := range columns {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+
+ out := make([]string, 0, len(fields))
+ seen := make(map[string]bool, len(fields))
+ appendField := func(f string) {
+ if f == "" || seen[f] {
+ return
+ }
+ seen[f] = true
+ out = append(out, f)
+ }
+ for _, f := range fields {
+ if f == "*" || !strings.Contains(f, "*") {
+ appendField(f)
+ continue
+ }
+ for _, name := range names {
+ if wildcardMatchColumn(f, name) {
+ appendField(name)
+ }
+ }
+ }
+ return out
+}
+
+// wildcardMatchColumn matches a column name against a "*"-wildcard pattern,
+// where "*" stands for any run of characters.
+func wildcardMatchColumn(pattern, name string) bool {
+ parts := strings.Split(pattern, "*")
+ if len(parts) == 1 {
+ return pattern == name
+ }
+ if !strings.HasPrefix(name, parts[0]) {
+ return false
+ }
+ rest := name[len(parts[0]):]
+ for _, part := range parts[1 : len(parts)-1] {
+ if part == "" {
+ continue
+ }
+ idx := strings.Index(rest, part)
+ if idx < 0 {
+ return false
+ }
+ rest = rest[idx+len(part):]
+ }
+ return strings.HasSuffix(rest, parts[len(parts)-1])
+}
+
// convertMatchingField converts field names for matching
// For regular document indices: maps _tks/_kwd fields to column@index_name format
// For skill indices: maps raw field names to column@index_name format
@@ -2333,46 +2543,51 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map
// JSON-list fields use member containment. Legacy varchar columns keep
// the old full-text behavior.
if fieldJSONList(k) && tableColumns != nil {
- jsonConditions := jsonListFilterConditions(k, v, tableColumns)
- if len(jsonConditions) > 0 {
- cond = append(cond, "("+strings.Join(jsonConditions, " OR ")+")")
+ if jsonConditions := jsonListFilterConditions(k, v, tableColumns); len(jsonConditions) > 0 {
+ cond = append(cond, joinBalanced(jsonConditions, " OR "))
}
continue
}
- // Handle keyword fields (using full-text filter)
+ // Handle keyword fields (exact match, or full-text for the single-token
+ // values Infinity can tokenize — see keywordFilterCondition)
if fieldKeyword(k) {
- // For keyword fields, values are always treated as strings for filter_fulltext
+ // For keyword fields, values are always treated as strings. Blank
+ // entries are dropped: Infinity rejects a filter with an empty query
+ // ("Trying to match: on fields: <column> failed", 3052) and fails
+ // the whole statement, so one empty element in an IN-list built from
+ // optional values (e.g. GetWikiGraph's from_kwd list when an entity
+ // has no slug) must not poison the condition.
+ var inCond []string
+ appendKeywordCond := func(raw string) {
+ if strings.TrimSpace(raw) == "" {
+ return
+ }
+ inCond = append(inCond, keywordFilterCondition(k, raw))
+ }
switch val := v.(type) {
case []string:
- var inCond []string
for _, item := range val {
- inCond = append(inCond, fmt.Sprintf("filter_fulltext('%s', '%s')",
- convertMatchingField(k), escapeFilterValue(item)))
- }
- if len(inCond) > 0 {
- cond = append(cond, "("+strings.Join(inCond, " or ")+")")
+ appendKeywordCond(item)
}
case []interface{}:
- var inCond []string
for _, item := range val {
if s, ok := item.(string); ok {
- inCond = append(inCond, fmt.Sprintf("filter_fulltext('%s', '%s')",
- convertMatchingField(k), escapeFilterValue(s)))
+ appendKeywordCond(s)
} else {
- inCond = append(inCond, fmt.Sprintf("filter_fulltext('%s', '%s')",
- convertMatchingField(k), escapeFilterValue(fmt.Sprintf("%v", item))))
+ appendKeywordCond(fmt.Sprintf("%v", item))
}
}
- if len(inCond) > 0 {
- cond = append(cond, "("+strings.Join(inCond, " or ")+")")
- }
case string:
- cond = append(cond, fmt.Sprintf("filter_fulltext('%s', '%s')",
- convertMatchingField(k), escapeFilterValue(val)))
+ appendKeywordCond(val)
default:
- cond = append(cond, fmt.Sprintf("filter_fulltext('%s', '%s')",
- convertMatchingField(k), escapeFilterValue(fmt.Sprintf("%v", v))))
+ appendKeywordCond(fmt.Sprintf("%v", v))
+ }
+ if len(inCond) > 0 {
+ // Balanced, not a flat "a or b or ... " chain: a long keyword list
+ // (22 wiki slugs) otherwise exceeds Infinity's Thrift expression
+ // depth and kills the connection (see joinBalanced).
+ cond = append(cond, joinBalanced(inCond, " or "))
}
continue
}
@@ -2479,6 +2694,25 @@ func decodeJSONFields(chunk map[string]interface{}) {
listField := fieldJSONList(fieldName)
text, ok := value.(string)
if !ok {
+ // Infinity hands a Json/Array column back as the raw
+ // length-prefixed multi-value byte blob (see
+ // decodeInfinityJSONBytes) rather than the JSON text a legacily
+ // typed varchar column returns, so decode that shape too. Without
+ // it every reader of such a column saw raw bytes: the structure
+ // graph could not read compilation_template_ids off the entity /
+ // relation rows, so it could not bucket them per template — it fell
+ // back to a synthetic "legacy" bucket with zero relations and the
+ // navigation tree's document level stayed flat instead of nesting
+ // into its entity tree.
+ if raw, isBytes := value.([]byte); isBytes {
+ if decoded, ok := decodeInfinityJSONBytes(raw); ok {
+ if listField {
+ decoded = asJSONList(decoded)
+ }
+ chunk[fieldName] = decoded
+ continue
+ }
+ }
if listField {
chunk[fieldName] = asJSONList(value)
}
@@ -2511,6 +2745,130 @@ func decodeJSONFields(chunk map[string]interface{}) {
}
}
+// splitLengthPrefixedSegments splits Infinity's `[int32 little-endian
+// length][payload]` framing into its payloads, each the JSON text of one value.
+// parseLengthPrefixedJSON (metadata.go) unpacks the same framing for the
+// metadata column.
+func splitLengthPrefixedSegments(data []byte) [][]byte {
+ var out [][]byte
+ for offset := 0; offset+4 <= len(data); {
+ length := int(uint32(data[offset]) |
+ uint32(data[offset+1])<<8 |
+ uint32(data[offset+2])<<16 |
+ uint32(data[offset+3])<<24)
+ offset += 4
+ if length < 0 || offset+length > len(data) {
+ break
+ }
+ out = append(out, data[offset:offset+length])
+ offset += length
+ }
+ return out
+}
+
+// realignJSONColumnCells redistributes a Json/Array column that Infinity returns
+// as ONE cell holding one length-prefixed segment per returned row, in row
+// order (the quirk realignMetaFieldsColumn documents for the metadata column).
+// Without it the column-oriented loop in Search parks the whole column on row 0
+// and rows 1..N lose it: a five-row entity/relation read carried
+// compilation_template_ids on exactly one row, so the structure graph could not
+// bucket rows per template and answered with a synthetic "legacy" template with
+// zero relations.
+//
+// Each row receives its own segment as JSON text, which decodeJSONFields then
+// decodes through the ordinary string path. The blob must frame into exactly one
+// valid JSON value per row, so a column that is already aligned (one cell per
+// row) or any other byte shape is left untouched.
+func realignJSONColumnCells(rows []map[string]interface{}, columns map[string][]interface{}) {
+ if len(rows) < 2 {
+ return
+ }
+ for colName, cells := range columns {
+ if !fieldJSON(colName) || len(cells) != 1 {
+ continue
+ }
+ raw, isBytes := cells[0].([]byte)
+ if !isBytes {
+ continue
+ }
+ segments := splitLengthPrefixedSegments(raw)
+ if len(segments) != len(rows) {
+ continue
+ }
+ valid := true
+ for _, segment := range segments {
+ var value interface{}
+ if err := json.Unmarshal(segment, &value); err != nil {
+ valid = false
+ break
+ }
+ }
+ if !valid {
+ continue
+ }
+ for i := range rows {
+ rows[i][colName] = string(segments[i])
+ }
+ }
+}
+
+// decodeInfinityJSONBytes decodes a Json/Array column value that arrived as raw
+// length-prefixed bytes rather than JSON text: the shape of a single-value cell
+// (a one-row result, or a value realignJSONColumnCells left alone). Every segment
+// must parse as JSON — anything else reports ok=false so the caller keeps its
+// previous behavior. Array segments concatenate (duplicates dropped, so a
+// repeated value cannot fabricate a second identical entry); a non-array segment
+// replaces the accumulated value.
+func decodeInfinityJSONBytes(raw []byte) (interface{}, bool) {
+ var (
+ items []interface{}
+ scalar interface{}
+ )
+ segments := splitLengthPrefixedSegments(raw)
+ if len(segments) == 0 {
+ return nil, false
+ }
+ for _, segment := range segments {
+ var value interface{}
+ if err := json.Unmarshal(segment, &value); err != nil {
+ return nil, false
+ }
+ if arr, isArray := value.([]interface{}); isArray {
+ items = appendJSONUnique(items, arr)
+ continue
+ }
+ scalar = value
+ }
+ if len(items) > 0 {
+ return items, true
+ }
+ return scalar, true
+}
+
+// appendJSONUnique appends the elements of add that dst does not already hold,
+// comparing them by their JSON encoding (Infinity stores scalars here, so the
+// encoding is a stable identity for a value).
+func appendJSONUnique(dst, add []interface{}) []interface{} {
+ for _, item := range add {
+ key, err := json.Marshal(item)
+ if err != nil {
+ dst = append(dst, item)
+ continue
+ }
+ dup := false
+ for _, existing := range dst {
+ if other, err := json.Marshal(existing); err == nil && string(other) == string(key) {
+ dup = true
+ break
+ }
+ }
+ if !dup {
+ dst = append(dst, item)
+ }
+ }
+ return dst
+}
+
// asJSONList normalizes a list-column value the way Python's parse_json_list
// does (infinity_conn.py:938-944): a slice passes through unchanged, a nil value
// becomes [] and any other scalar is wrapped in a single-element list.
@@ -2700,8 +3058,20 @@ func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]inter
case "docnm_kwd":
d["docnm"] = v
case "title_kwd":
+ // title_kwd is a column of its own in the Infinity mapping: the
+ // compiled wiki/nav rows carry their label there, and nav keys its
+ // cluster updates on it (title_kwd = <cluster name>). Folding it away
+ // into docnm (Python's chunk-title alias, infinity_conn.py:522-527)
+ // wrote ONLY docnm, so on Infinity every reader of title_kwd saw ""
+ // while ES returned the field verbatim: nav labels degraded to the
+ // description's first line and every cluster lookup matched nothing,
+ // so each document opened a new root cluster instead of merging.
+ // Keep the column, and still mirror it into docnm when the row has no
+ // docnm_kwd, preserving the chunk-title search behavior.
+ title := utility.ConvertToString(v)
+ d["title_kwd"] = title
if _, exists := chunk["docnm_kwd"]; !exists {
- d["docnm"] = utility.ConvertToString(v)
+ d["docnm"] = title
}
case "title_sm_tks":
if _, exists := chunk["docnm_kwd"]; !exists {
diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.go
index 5c38ca22b1..69afb7f302 100644
--- a/internal/engine/infinity/chunk_transform_test.go
+++ b/internal/engine/infinity/chunk_transform_test.go
@@ -15,6 +15,7 @@
package infinity
import (
+ "encoding/binary"
"encoding/json"
"os"
"reflect"
@@ -503,3 +504,290 @@ func TestJSONColumnsMatchCaseInsensitively(t *testing.T) {
t.Errorf("nil Source_Doc_IDs = %#v, want %#v", row["Source_Doc_IDs"], want)
}
}
+
+// TestTransformChunkFieldsKeepsTitleColumn pins that title_kwd survives as its
+// own column. The compiled wiki/nav rows carry their label only in title_kwd and
+// nav keys every cluster update on it (title_kwd = <cluster name>); folding the
+// field into docnm — Python's chunk-title alias — wrote ONLY docnm, so on
+// Infinity every reader of title_kwd saw "" while ES returned the field verbatim
+// (nav labels degraded to the description's first line and cluster merges never
+// matched, so each document opened a new root cluster).
+func TestTransformChunkFieldsKeepsTitleColumn(t *testing.T) {
+ got := transformChunkFields(map[string]interface{}{"title_kwd": "忠臣进谏与托孤遗志 27577543"}, nil)
+ if want := "忠臣进谏与托孤遗志 27577543"; got["title_kwd"] != want {
+ t.Errorf("title_kwd = %#v, want %q", got["title_kwd"], want)
+ }
+ // The chunk-title alias is still written when the row carries no docnm_kwd.
+ if want := "忠臣进谏与托孤遗志 27577543"; got["docnm"] != want {
+ t.Errorf("docnm = %#v, want %q", got["docnm"], want)
+ }
+
+ // With docnm_kwd present, the alias must not overwrite the document name,
+ // and title_kwd must still keep the row's own label.
+ got = transformChunkFields(map[string]interface{}{
+ "docnm_kwd": "出师表.txt",
+ "title_kwd": "wiki page title",
+ }, nil)
+ if want := "出师表.txt"; got["docnm"] != want {
+ t.Errorf("docnm = %#v, want %q", got["docnm"], want)
+ }
+ if want := "wiki page title"; got["title_kwd"] != want {
+ t.Errorf("title_kwd = %#v, want %q", got["title_kwd"], want)
+ }
+}
+
+// TestStripScorePseudoColumns pins that a caller's ES-style score columns are
+// dropped: Infinity rejects "_score" unless a MATCH TEXT/TENSOR or Fusion is
+// present (3013), and Search re-adds the one form its query shape accepts.
+func TestStripScorePseudoColumns(t *testing.T) {
+ got := stripScorePseudoColumns([]string{"type_kwd", "_score", "SCORE", "similarity()", "title_kwd"})
+ want := []string{"type_kwd", "title_kwd"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("stripScorePseudoColumns = %#v, want %#v", got, want)
+ }
+
+ // A filter-only query keeps no score column at all.
+ if got := stripScorePseudoColumns([]string{"_score"}); len(got) != 0 {
+ t.Fatalf("filter-only score columns = %#v, want empty", got)
+ }
+}
+
+func TestHasPartialWildcard(t *testing.T) {
+ cases := []struct {
+ fields []string
+ want bool
+ }{
+ {[]string{"id", "title_kwd"}, false},
+ {[]string{"*"}, false}, // Infinity reads a bare "*" as "all columns"
+ {[]string{"id", "q_*_vec"}, true},
+ {[]string{"q_*_vec", "*"}, true},
+ }
+ for _, c := range cases {
+ if got := hasPartialWildcard(c.fields); got != c.want {
+ t.Errorf("hasPartialWildcard(%#v) = %v, want %v", c.fields, got, c.want)
+ }
+ }
+}
+
+// TestExpandSelectWildcards pins the workaround for the partial wildcard the
+// compile reader asks for: Infinity parses "q_*_vec" as the expression
+// "q_" * "_vec" and fails to bind it (3013), so it is expanded into the table's
+// real vector column(s) and dropped when the table has none.
+func TestExpandSelectWildcards(t *testing.T) {
+ cols := map[string]struct {
+ Type string
+ Default interface{}
+ }{
+ "id": {Type: "varchar"},
+ "q_1024_vec": {Type: "vector,1024,float"},
+ "content": {Type: "varchar"},
+ }
+
+ if got, want := expandSelectWildcards([]string{"id", "q_*_vec", "title_kwd"}, cols),
+ []string{"id", "q_1024_vec", "title_kwd"}; !reflect.DeepEqual(got, want) {
+ t.Errorf("expandSelectWildcards = %#v, want %#v", got, want)
+ }
+
+ // A bare "*" is left untouched: Infinity supports it.
+ if got, want := expandSelectWildcards([]string{"*"}, cols), []string{"*"}; !reflect.DeepEqual(got, want) {
+ t.Errorf("bare wildcard = %#v, want %#v", got, want)
+ }
+
+ // A pattern matching nothing is dropped instead of being sent verbatim.
+ if got, want := expandSelectWildcards([]string{"id", "missing_*_col"}, cols), []string{"id"}; !reflect.DeepEqual(got, want) {
+ t.Errorf("unmatched wildcard = %#v, want %#v", got, want)
+ }
+
+ // Resolving to an already listed column must not duplicate it.
+ if got, want := expandSelectWildcards([]string{"q_1024_vec", "q_*_vec"}, cols), []string{"q_1024_vec"}; !reflect.DeepEqual(got, want) {
+ t.Errorf("duplicate wildcard = %#v, want %#v", got, want)
+ }
+
+ // Columns with no vector column at all: the wildcard disappears, the rest
+ // survives (a query may legitimately target a table without embeddings).
+ got := expandSelectWildcards([]string{"id", "q_*_vec"}, map[string]struct {
+ Type string
+ Default interface{}
+ }{"id": {Type: "varchar"}})
+ if want := []string{"id"}; !reflect.DeepEqual(got, want) {
+ t.Errorf("table without vector column = %#v, want %#v", got, want)
+ }
+}
+
+func TestWildcardMatchColumn(t *testing.T) {
+ cases := []struct {
+ pattern, name string
+ want bool
+ }{
+ {"q_*_vec", "q_1024_vec", true},
+ {"q_*_vec", "q_768_vec", true},
+ {"q_*_vec", "content", false},
+ {"q_*_vec", "q_1024_vec_x", false},
+ {"*_vec", "q_1024_vec", true},
+ {"q_*", "q_1024_vec", true},
+ {"foo", "foo", true},
+ {"foo", "bar", false},
+ {"a*b*c", "aXbYc", true},
+ {"a*b*c", "ac", false},
+ }
+ for _, c := range cases {
+ if got := wildcardMatchColumn(c.pattern, c.name); got != c.want {
+ t.Errorf("wildcardMatchColumn(%q, %q) = %v, want %v", c.pattern, c.name, got, c.want)
+ }
+ }
+}
+
+// lengthPrefixedJSON builds Infinity's raw Json/Array column encoding: one
+// `[int32 little-endian length][payload]` segment per stored value.
+func lengthPrefixedJSON(parts ...string) []byte {
+ out := make([]byte, 0, 64)
+ for _, p := range parts {
+ var l [4]byte
+ binary.LittleEndian.PutUint32(l[:], uint32(len(p)))
+ out = append(out, l[:]...)
+ out = append(out, p...)
+ }
+ return out
+}
+
+// TestDecodeJSONFieldsLengthPrefixedBytes pins the read path for the Json/Array
+// columns Infinity returns as length-prefixed bytes instead of JSON text — the
+// shape the structure graph reads compilation_template_ids from.
+func TestDecodeJSONFieldsLengthPrefixedBytes(t *testing.T) {
+ row := map[string]interface{}{
+ // One real value after three empty writes (the observed shape).
+ "compilation_template_ids": lengthPrefixedJSON("[]", "[]", "[]", `["03b36295dcb34f9e811d8073b21e7f2f"]`),
+ "source_chunk_ids": lengthPrefixedJSON("[]", "[]", "[]", `["d1962d8d1cf69ab6","1313e71f7c7c9cca"]`),
+ // Every write empty: the column reads back as an empty list.
+ "doc_ids_kwd": lengthPrefixedJSON("[]", "[]", "[]", "[]", "[]", "[]"),
+ }
+ decodeJSONFields(row)
+
+ if want := []interface{}{"03b36295dcb34f9e811d8073b21e7f2f"}; !reflect.DeepEqual(row["compilation_template_ids"], want) {
+ t.Errorf("compilation_template_ids = %#v, want %#v", row["compilation_template_ids"], want)
+ }
+ if want := []interface{}{"d1962d8d1cf69ab6", "1313e71f7c7c9cca"}; !reflect.DeepEqual(row["source_chunk_ids"], want) {
+ t.Errorf("source_chunk_ids = %#v, want %#v", row["source_chunk_ids"], want)
+ }
+ if want := []interface{}{}; !reflect.DeepEqual(row["doc_ids_kwd"], want) {
+ t.Errorf("doc_ids_kwd = %#v, want %#v", row["doc_ids_kwd"], want)
+ }
+
+ // A value written twice must not fabricate a second identical bucket.
+ dup := map[string]interface{}{
+ "compilation_template_ids": lengthPrefixedJSON(`["t1"]`, `["t1","t2"]`),
+ }
+ decodeJSONFields(dup)
+ if want := []interface{}{"t1", "t2"}; !reflect.DeepEqual(dup["compilation_template_ids"], want) {
+ t.Errorf("duplicate segments = %#v, want %#v", dup["compilation_template_ids"], want)
+ }
+
+ // A non-array segment (an object) on a list column keeps the list shape
+ // (Python's parse_json_list wraps a scalar), and on a plain JSON column it
+ // decodes to the object itself.
+ listObj := map[string]interface{}{
+ "compilation_template_ids": lengthPrefixedJSON(`{"kind":"tree"}`),
+ }
+ decodeJSONFields(listObj)
+ wrapped, ok := listObj["compilation_template_ids"].([]interface{})
+ if !ok || len(wrapped) != 1 {
+ t.Fatalf("object segment on a list column = %#v, want a one-element list", listObj["compilation_template_ids"])
+ }
+ if m, ok := wrapped[0].(map[string]interface{}); !ok || m["kind"] != "tree" {
+ t.Errorf("object segment element = %#v, want the decoded map", wrapped[0])
+ }
+
+ plainObj := map[string]interface{}{
+ "source_chunk_hashes": lengthPrefixedJSON(`{"a":"b"}`),
+ }
+ decodeJSONFields(plainObj)
+ if m, ok := plainObj["source_chunk_hashes"].(map[string]interface{}); !ok || m["a"] != "b" {
+ t.Errorf("object segment on a JSON column = %#v, want the decoded map", plainObj["source_chunk_hashes"])
+ }
+}
+
+func TestDecodeInfinityJSONBytesRejectsOtherShapes(t *testing.T) {
+ if _, ok := decodeInfinityJSONBytes([]byte("not-length-prefixed")); ok {
+ t.Error("plain text bytes must not be accepted as a Json column blob")
+ }
+ if _, ok := decodeInfinityJSONBytes([]byte{1, 2}); ok {
+ t.Error("too-short bytes must not be accepted")
+ }
+ // A truncated final segment decodes the complete prefix only.
+ blob := lengthPrefixedJSON(`["a"]`)
+ blob = append(blob, 0, 0, 0, 9, 'x')
+ value, ok := decodeInfinityJSONBytes(blob)
+ if !ok {
+ t.Fatal("complete leading segment must still decode")
+ }
+ if want := []interface{}{"a"}; !reflect.DeepEqual(value, want) {
+ t.Errorf("truncated blob = %#v, want %#v", value, want)
+ }
+}
+
+// TestRealignJSONColumnCells pins the redistribution of a Json/Array column that
+// Infinity returns as one cell holding one segment per row: without it only row 0
+// keeps the column, so the structure graph sees one entity row per read.
+func TestRealignJSONColumnCells(t *testing.T) {
+ columns := map[string][]interface{}{
+ "id": {"a", "b", "c"},
+ "knowledge_graph_kwd": {"entity", "entity", "relation"},
+ "compilation_template_ids": {lengthPrefixedJSON(`["t1"]`, `[]`, `["t2"]`)},
+ }
+ rows := []map[string]interface{}{{}, {}, {}}
+ realignJSONColumnCells(rows, columns)
+
+ wantText := []string{`["t1"]`, `[]`, `["t2"]`}
+ for i, want := range wantText {
+ if got := rows[i]["compilation_template_ids"]; got != want {
+ t.Errorf("row %d segment = %#v, want %q", i, got, want)
+ }
+ }
+ // Decoding each row's redistributed text yields that row's own value.
+ decodeJSONFields(rows[0])
+ decodeJSONFields(rows[1])
+ decodeJSONFields(rows[2])
+ if want := []interface{}{"t1"}; !reflect.DeepEqual(rows[0]["compilation_template_ids"], want) {
+ t.Errorf("row 0 = %#v, want %#v", rows[0]["compilation_template_ids"], want)
+ }
+ if want := []interface{}{}; !reflect.DeepEqual(rows[1]["compilation_template_ids"], want) {
+ t.Errorf("row 1 = %#v, want %#v", rows[1]["compilation_template_ids"], want)
+ }
+ if want := []interface{}{"t2"}; !reflect.DeepEqual(rows[2]["compilation_template_ids"], want) {
+ t.Errorf("row 2 = %#v, want %#v", rows[2]["compilation_template_ids"], want)
+ }
+
+ // A blob whose segment count does not match the row count is left alone, and
+ // a column that already has one cell per row is untouched.
+ odd := []map[string]interface{}{{}, {}}
+ realignJSONColumnCells(odd, map[string][]interface{}{
+ "compilation_template_ids": {lengthPrefixedJSON(`["t1"]`, `[]`, `["t2"]`)},
+ })
+ if len(odd[0]) != 0 {
+ t.Errorf("mismatched segment count must be left untouched, got %#v", odd[0])
+ }
+ aligned := []map[string]interface{}{{}, {}}
+ realignJSONColumnCells(aligned, map[string][]interface{}{
+ "compilation_template_ids": {`["t1"]`, `["t2"]`},
+ })
+ if len(aligned[0]) != 0 {
+ t.Errorf("an aligned column must be left untouched, got %#v", aligned[0])
+ }
+ // A single row keeps the cell as it came: decodeInfinityJSONBytes handles it.
+ single := []map[string]interface{}{{}}
+ realignJSONColumnCells(single, map[string][]interface{}{
+ "compilation_template_ids": {lengthPrefixedJSON(`["t1"]`)},
+ })
+ if len(single[0]) != 0 {
+ t.Errorf("a single-row result must be left untouched, got %#v", single[0])
+ }
+ // Segments that are not JSON are not per-row values: leave the column alone
+ // rather than copying the framing into every row.
+ notJSON := []map[string]interface{}{{}, {}}
+ realignJSONColumnCells(notJSON, map[string][]interface{}{
+ "compilation_template_ids": {lengthPrefixedJSON("t1", "t2")},
+ })
+ if len(notJSON[0]) != 0 {
+ t.Errorf("non-JSON segments must be left untouched, got %#v", notJSON[0])
+ }
+}
diff --git a/internal/engine/infinity/client.go b/internal/engine/infinity/client.go
index 7da66805bd..6b1d732e68 100644
--- a/internal/engine/infinity/client.go
+++ b/internal/engine/infinity/client.go
@@ -26,6 +26,7 @@ import (
"reflect"
"strconv"
"strings"
+ "sync"
"sync/atomic"
"time"
@@ -45,6 +46,12 @@ type infinityClient struct {
getDatabaseSeq atomic.Uint64
getDatabaseInflight atomic.Int64
+ // dbConns maps the *infinity.Database handed to a caller back to the pooled
+ // connection it was checked out from, so a call that fails at the transport
+ // level can drop that exact socket (see dropConnectionFor). Entries live only
+ // for the duration of a checkout.
+ dbConns sync.Map
+
// Original URI from config, used by RunSQL to extract the host.
hostURI string
@@ -314,10 +321,15 @@ func (c *infinityClient) checkoutDatabase(ctx context.Context, caller string) (*
zap.Duration("elapsed", elapsed),
zap.Error(err),
)
+ if isConnectionLevelError(err) {
+ dropConnection(conn, caller)
+ }
release()
return nil, nil, err
}
+ c.dbConns.Store(db, conn)
wrappedRelease := func() {
+ c.dbConns.Delete(db)
inflight := c.getDatabaseInflight.Add(-1)
elapsed := time.Since(start)
common.Info("Infinity GetDatabase done",
@@ -333,6 +345,66 @@ func (c *infinityClient) checkoutDatabase(ctx context.Context, caller string) (*
return db, wrappedRelease, nil
}
+// dropConnectionFor closes the pooled connection behind a checked-out database
+// handle after a transport-level failure. Such a socket still reports
+// IsConnected(), so Put() would return it to the pool and desync the next
+// caller's Thrift stream — observed as "InfinityException(7018, ... EOF)".
+// Disconnecting first makes Put() evict it instead. caller names the operation
+// that failed, for the log.
+func (c *infinityClient) dropConnectionFor(db *infinity.Database, caller string) {
+ if c == nil || db == nil {
+ return
+ }
+ raw, ok := c.dbConns.Load(db)
+ if !ok {
+ return
+ }
+ conn, _ := raw.(*infinity.InfinityConnection)
+ dropConnection(conn, caller)
+}
+
+// dropConnection disconnects a connection whose call failed mid-flight so the
+// pool evicts it. Errors are logged and ignored: the socket is already suspect.
+func dropConnection(conn *infinity.InfinityConnection, caller string) {
+ if conn == nil || !conn.IsConnected() {
+ return
+ }
+ common.Warn("Infinity connection dropped after a transport-level failure",
+ zap.String("caller", caller),
+ zap.String("conn_ptr", fmt.Sprintf("%p", conn)))
+ if _, err := conn.Disconnect(); err != nil {
+ common.Warn("Infinity connection disconnect failed",
+ zap.String("caller", caller), zap.Error(err))
+ }
+}
+
+// isConnectionLevelError reports whether a failed Infinity call may have left the
+// socket in an unknown state (the reply was not fully read), in which case the
+// connection must not be reused. Infinity's own error codes for this are 7018
+// ("Failed to execute query: EOF") plus the thrift/transport failures below.
+func isConnectionLevelError(err error) bool {
+ if err == nil {
+ return false
+ }
+ msg := strings.ToLower(err.Error())
+ for _, marker := range []string{
+ "eof",
+ "connection is dead",
+ "connection reset",
+ "broken pipe",
+ "i/o timeout",
+ "deadline exceeded",
+ "tprotocolexception",
+ "transport is not open",
+ "no route to host",
+ } {
+ if strings.Contains(msg, marker) {
+ return true
+ }
+ }
+ return false
+}
+
// Engine Infinity engine implementation using Go SDK
type Engine struct {
config config.InfinityConfig
diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.go
index eb28450bc9..9a4c551d5b 100644
--- a/internal/engine/infinity/common.go
+++ b/internal/engine/infinity/common.go
@@ -23,6 +23,7 @@ import (
"fmt"
"ragflow/internal/common"
"strings"
+ "unicode"
infinity "github.com/infiniflow/infinity-go-sdk"
@@ -158,6 +159,45 @@ func fieldKeyword(fieldName string) bool {
return false
}
+// keywordFilterCondition renders a filter for a *_kwd column.
+//
+// Infinity declares these columns as analyzed varchars, so filter_fulltext()
+// tokenizes the query and a multi-token value matches NOTHING — while every
+// navigation cluster name is multi-token, so nav's child lookup (parent_kwd=…),
+// its cluster updates and cleanupEmptyCluster all silently matched no row.
+// Elasticsearch matches a plain *_kwd column exactly (keyword + term/terms), so
+// a whitespace-bearing value becomes `col = 'value'`: it matched no row before,
+// so this cannot narrow an existing match, and single-token values keep
+// filter_fulltext for the legacy ###-joined columns (see fieldJSONList).
+//
+// tag_kwd and toc_kwd are the exception: convertMatchingField turns them into a
+// full-text index reference ("tag_kwd@ft_tag_kwd_whitespace__"), which is not a
+// column and cannot be compared with `=` (Infinity rejects the whole statement).
+// Their cells hold several ###-joined values, and that index is what gives them
+// the membership match ES gets from its keyword array, so a multi-token value is
+// matched as a quoted phrase through it.
+func keywordFilterCondition(field string, value string) string {
+ escaped := escapeFilterValue(value)
+ multiToken := strings.ContainsFunc(value, unicode.IsSpace)
+ indexRef := convertMatchingField(field)
+
+ if indexRef == field {
+ if multiToken {
+ return fmt.Sprintf("%s = '%s'", field, escaped)
+ }
+ return fmt.Sprintf("filter_fulltext('%s', '%s')", field, escaped)
+ }
+ if !multiToken {
+ return fmt.Sprintf("filter_fulltext('%s', '%s')", indexRef, escaped)
+ }
+ if strings.Contains(value, `"`) {
+ // Cannot be phrased: compare the raw column so the value still matches a
+ // single-valued cell instead of failing the statement.
+ return fmt.Sprintf("%s = '%s'", field, escaped)
+ }
+ return fmt.Sprintf("filter_fulltext('%s', '\"%s\"')", indexRef, escaped)
+}
+
// fieldJSON reports fields stored as Infinity JSON columns. The Infinity Go
// SDK expects JSON columns as encoded strings, not Go slices or maps. Column
// names are matched case-insensitively so the write path (transformChunkFields)
@@ -188,6 +228,36 @@ func fieldJSONList(fieldName string) bool {
}
}
+// joinBalanced renders a boolean chain as a balanced tree instead of the flat,
+// left-deep chain strings.Join produces. Infinity's filter string is parsed by
+// the SDK into a Thrift expression tree (infinity-go-sdk expression_parser.go)
+// and sent as that tree, so a left-deep chain of N terms nests N levels deep: a
+// filter with 22 OR'ed filter_fulltext(...) clauses (the wiki contribution and
+// graph reads build exactly that) made Infinity abort the connection with
+//
+// Thrift: ... TProtocolException: Exceeded depth limit
+//
+// which our callers then see as "InfinityException(7018, Failed to execute
+// query: EOF)". A balanced tree keeps the depth at log2(N) — 22 terms pass
+// against a live Infinity, verified — while matching exactly the same rows.
+// The parenthesization is boolean-equivalent to the flat chain (a single term
+// needs no parentheses).
+func joinBalanced(parts []string, op string) string {
+ switch len(parts) {
+ case 0:
+ return ""
+ case 1:
+ // Keep the single-element form parenthesized: callers (and tests) relied
+ // on a self-contained group, e.g. "(1=0)" for a never-matching list.
+ return "(" + parts[0] + ")"
+ case 2:
+ return "(" + parts[0] + op + parts[1] + ")"
+ default:
+ mid := (len(parts) + 1) / 2
+ return "(" + joinBalanced(parts[:mid], op) + op + joinBalanced(parts[mid:], op) + ")"
+ }
+}
+
func jsonListFilterConditions(fieldName string, value interface{}, tableColumns map[string]struct {
Type string
Default interface{}
@@ -333,38 +403,47 @@ func buildFilterFromCondition(condition map[string]interface{}, tableColumns map
// JSON-list fields use member containment. Legacy tables stored these
// columns as ###-joined varchar values, so retain their full-text fallback.
if fieldJSONList(k) && tableColumns != nil {
- jsonConditions := jsonListFilterConditions(k, v, tableColumns)
- if len(jsonConditions) > 0 {
- conditions = append(conditions, "("+strings.Join(jsonConditions, " OR ")+")")
+ if jsonConditions := jsonListFilterConditions(k, v, tableColumns); len(jsonConditions) > 0 {
+ conditions = append(conditions, joinBalanced(jsonConditions, " OR "))
}
continue
}
- // Handle keyword fields -> filter_fulltext with converted field name
+ // Handle keyword fields -> exact match, or filter_fulltext for the
+ // single-token values Infinity can tokenize (see keywordFilterCondition).
if fieldKeyword(k) {
var orConds []string
- addFullText := func(item string) {
- item = strings.ReplaceAll(item, "'", "''")
- orConds = append(orConds, fmt.Sprintf("filter_fulltext('%s', '%s')", convertMatchingField(k), item))
+ addKeyword := func(item string) {
+ // Blank entries are dropped exactly as the search path drops them:
+ // Infinity rejects an empty full-text query ("Trying to match: on
+ // fields: <column> failed", 3052) and fails the whole
+ // UpdateChunks/DeleteChunks statement, so one empty element in a
+ // list built from optional values must not poison it. A condition
+ // left with no clause at all is refused by the callers' "1=1"
+ // guard, so this cannot widen an update or delete.
+ if strings.TrimSpace(item) == "" {
+ return
+ }
+ orConds = append(orConds, keywordFilterCondition(k, item))
}
switch val := v.(type) {
case []string:
for _, item := range val {
- addFullText(item)
+ addKeyword(item)
}
case []interface{}:
for _, item := range val {
- addFullText(fmt.Sprintf("%v", item))
+ addKeyword(fmt.Sprintf("%v", item))
}
case string:
- addFullText(val)
+ addKeyword(val)
default:
- addFullText(fmt.Sprintf("%v", val))
+ addKeyword(fmt.Sprintf("%v", val))
}
if len(orConds) > 0 {
- conditions = append(conditions, "("+strings.Join(orConds, " OR ")+")")
+ conditions = append(conditions, joinBalanced(orConds, " OR "))
}
continue
}
diff --git a/internal/engine/infinity/meta_filter.go b/internal/engine/infinity/meta_filter.go
index cca63b17cb..6f16ae02d7 100644
--- a/internal/engine/infinity/meta_filter.go
+++ b/internal/engine/infinity/meta_filter.go
@@ -158,10 +158,10 @@ func (t *MetaFilterTranslator) translateIn(key string, value interface{}, flt ma
var conditions []string
if len(stringParts) > 0 {
- conditions = append(conditions, "("+strings.Join(stringParts, " OR ")+")")
+ conditions = append(conditions, joinBalanced(stringParts, " OR "))
}
if len(numParts) > 0 {
- conditions = append(conditions, "("+strings.Join(numParts, " OR ")+")")
+ conditions = append(conditions, joinBalanced(numParts, " OR "))
}
return "(" + strings.Join(conditions, " OR ") + ")"
@@ -263,8 +263,13 @@ func BuildInfinityFilter(filters []map[string]interface{}, logic string) (string
if logic == "or" {
joiner = " OR "
}
+ if len(fragments) == 0 {
+ return "()", nil
+ }
- return "(" + strings.Join(fragments, joiner) + ")", nil
+ // Balanced so a long OR chain (many metadata values) cannot exceed Infinity's
+ // Thrift expression depth (see joinBalanced).
+ return joinBalanced(fragments, joiner), nil
}
// IsPushdownSupported checks if all filters can be pushed down
diff --git a/internal/engine/infinity/metadata.go b/internal/engine/infinity/metadata.go
index 8456707153..65441cb8da 100644
--- a/internal/engine/infinity/metadata.go
+++ b/internal/engine/infinity/metadata.go
@@ -176,7 +176,10 @@ func (e *Engine) InsertMetadata(ctx context.Context, metadata []map[string]inter
kbID := fmt.Sprintf("'%s'", strings.ReplaceAll(fmt.Sprintf("%v", m["kb_id"]), "'", "''"))
idList[i] = fmt.Sprintf("(id = %s AND kb_id = %s)", docID, kbID)
}
- filter := strings.Join(idList, " OR ")
+ // Balanced, not a flat chain: a bulk delete of many metadata rows would
+ // otherwise nest N levels deep and trip Infinity's Thrift expression depth
+ // limit (see joinBalanced).
+ filter := joinBalanced(idList, " OR ")
common.Debug(fmt.Sprintf("Deleting existing metadata with filter: %s", filter))
delResp, delErr := table.Delete(filter)
if delErr != nil {
diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.go
index 9c3be5faa8..951132e11c 100644
--- a/internal/engine/infinity/sql_test.go
+++ b/internal/engine/infinity/sql_test.go
@@ -17,9 +17,12 @@
package infinity
import (
+ "errors"
+ "fmt"
"os"
"path/filepath"
"reflect"
+ "strings"
"testing"
)
@@ -425,6 +428,172 @@ func TestBuildFilterFromCondition_StringSliceIDPreservesScope(t *testing.T) {
}
}
+// TestEquivalentConditionToStrSkipsBlankKeywordValues pins that a blank entry in
+// a keyword (filter_fulltext) condition is dropped instead of being rendered as
+// filter_fulltext('<col>', ”): Infinity rejects that with
+//
+// 3052 Trying to match: on fields: <col> failed
+//
+// and fails the whole statement, which is exactly what GetWikiGraph hit when an
+// entity row carried no slug.
+func TestEquivalentConditionToStrSkipsBlankKeywordValues(t *testing.T) {
+ got := equivalentConditionToStr(map[string]interface{}{
+ "from_kwd": []string{"", " ", "topic/出师表"},
+ }, nil)
+ want := `(filter_fulltext('from_kwd', 'topic/出师表'))`
+ if got != want {
+ t.Errorf("blank keyword filter = %q, want %q", got, want)
+ }
+
+ // All entries blank: the column contributes no clause at all.
+ if got := equivalentConditionToStr(map[string]interface{}{"from_kwd": []string{"", ""}}, nil); got != "" {
+ t.Errorf("all-blank keyword filter = %q, want no clause", got)
+ }
+}
+
+// maxParenDepth returns the deepest nesting of parentheses in an expression.
+func maxParenDepth(expr string) int {
+ depth, max := 0, 0
+ for _, r := range expr {
+ switch r {
+ case '(':
+ depth++
+ if depth > max {
+ max = depth
+ }
+ case ')':
+ depth--
+ }
+ }
+ return max
+}
+
+// TestJoinBalanced pins the depth-bounded boolean rendering. Infinity parses the
+// filter string into a Thrift expression tree, so a left-deep chain of N terms
+// nests N levels deep: the wiki contribution read's 22 OR'ed filter_fulltext
+// clauses made Infinity abort the connection ("TProtocolException: Exceeded depth
+// limit"), which surfaced as 7018/EOF. A balanced tree must keep every term and
+// only log2(N) nesting.
+func TestJoinBalanced(t *testing.T) {
+ if got := joinBalanced(nil, " OR "); got != "" {
+ t.Errorf("joinBalanced(nil) = %q, want empty", got)
+ }
+ if got := joinBalanced([]string{"a"}, " OR "); got != "(a)" {
+ t.Errorf("single term = %q, want %q", got, "(a)")
+ }
+ if got := joinBalanced([]string{"a", "b"}, " OR "); got != "(a OR b)" {
+ t.Errorf("two terms = %q, want %q", got, "(a OR b)")
+ }
+
+ parts := make([]string, 0, 22)
+ for i := 0; i < 22; i++ {
+ parts = append(parts, fmt.Sprintf("f%02d", i))
+ }
+ expr := joinBalanced(parts, " or ")
+ for _, p := range parts {
+ if !strings.Contains(expr, p) {
+ t.Fatalf("expression dropped %s: %s", p, expr)
+ }
+ }
+ // Paren depth of the rendered string mirrors the depth of the expression tree
+ // the SDK builds from it; a flat chain would be sent N levels deep.
+ if maxParenDepth(expr) > 6 {
+ t.Errorf("balanced depth = %d, want <= ceil(log2(22))+1 = 6: %s", maxParenDepth(expr), expr)
+ }
+}
+
+// TestEquivalentConditionToStrBalancesLongKeywordLists pins that the keyword
+// branch uses the balanced join: a 22-element slug list is exactly the request
+// that killed the connection in production.
+func TestEquivalentConditionToStrBalancesLongKeywordLists(t *testing.T) {
+ values := make([]string, 0, 22)
+ for i := 0; i < 22; i++ {
+ values = append(values, fmt.Sprintf("entity/person/p%d", i))
+ }
+ got := equivalentConditionToStr(map[string]interface{}{"slug_kwd": values}, nil)
+ for _, v := range values {
+ if !strings.Contains(got, "filter_fulltext('slug_kwd', '"+v+"')") {
+ t.Fatalf("condition dropped %s: %s", v, got)
+ }
+ }
+ if depth := maxParenDepth(got); depth > 6 {
+ t.Errorf("paren depth = %d, want <= 6 for 22 values: %s", depth, got)
+ }
+}
+
+// TestIsConnectionLevelError pins which failures mark a pooled connection as
+// unusable: a cut-short reply desyncs the Thrift stream, and the next caller on
+// that socket is the one the server rejects with "Exceeded depth limit" (seen as
+// 7018/EOF). Semantic Infinity errors must NOT drop the connection.
+func TestIsConnectionLevelError(t *testing.T) {
+ drop := []string{
+ "InfinityException(7018, Failed to execute query: EOF)",
+ "InfinityException(6003, Connection is dead, removed from pool)",
+ "read tcp 127.0.0.1:33924->127.0.0.1:23817: read: connection reset by peer",
+ "write tcp: broken pipe",
+ "dial tcp 127.0.0.1:23817: i/o timeout",
+ "context deadline exceeded",
+ "TProtocolException: Exceeded depth limit",
+ }
+ for _, msg := range drop {
+ if !isConnectionLevelError(errors.New(msg)) {
+ t.Errorf("isConnectionLevelError(%q) = false, want true", msg)
+ }
+ }
+ keep := []string{
+ "",
+ "InfinityException(3052, Failed to execute query: Trying to match: on fields: from_kwd failed)",
+ "InfinityException(3013, Failed to execute query: Fail to bind the expression: q_)",
+ "InfinityException(7001, Table not found)",
+ }
+ for _, msg := range keep {
+ var err error
+ if msg != "" {
+ err = errors.New(msg)
+ }
+ if isConnectionLevelError(err) {
+ t.Errorf("isConnectionLevelError(%q) = true, want false", msg)
+ }
+ }
+}
+
+// TestRowFieldNames pins the insert-failure log payload: sorted column names of
+// the first row (values are never logged — a vector column would flood the log).
+func TestRowFieldNames(t *testing.T) {
+ got := rowFieldNames([]map[string]interface{}{{"title_kwd": "a", "available_int": 0, "q_1024_vec": []float64{1}}})
+ want := []string{"available_int", "q_1024_vec", "title_kwd"}
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("rowFieldNames = %#v, want %#v", got, want)
+ }
+ if got := rowFieldNames(nil); got != nil {
+ t.Errorf("rowFieldNames(nil) = %#v, want nil", got)
+ }
+}
+
+// TestIsEmptyEngineCell pins the row-materialization guard: Infinity can return a
+// null cell (untyped nil or a nil slice) for a projection it found no row for, and
+// such a cell must not turn into a row of its own — GetWikiGraph used to read an
+// empty slug off that phantom row and poison the relation filter.
+func TestIsEmptyEngineCell(t *testing.T) {
+ cases := []struct {
+ name string
+ value interface{}
+ want bool
+ }{
+ {"untyped nil", nil, true},
+ {"nil slice", []uint8(nil), true},
+ {"nil []interface{}", []interface{}(nil), true},
+ {"empty slice", []interface{}{}, false},
+ {"empty string", "", false},
+ {"value", "doc-1", false},
+ }
+ for _, c := range cases {
+ if got := isEmptyEngineCell(c.value); got != c.want {
+ t.Errorf("isEmptyEngineCell(%s) = %v, want %v", c.name, got, c.want)
+ }
+ }
+}
+
func TestBuildFilterFromConditionJSONListMembership(t *testing.T) {
jsonColumns := map[string]struct {
Type string
@@ -469,3 +638,111 @@ func TestBuildFilterFromConditionMissingJSONListColumnNeverMatches(t *testing.T)
t.Errorf("missing JSON-list column condition = %q, want %q", got, want)
}
}
+
+// TestKeywordFilterConditionExactForWhitespaceValues pins the fix that makes a
+// *_kwd filter with a multi-token value match at all. Infinity tokenizes a
+// filter_fulltext() query, so a whitespace-bearing value matched NOTHING (nav's
+// parent_kwd child lookup, cluster updates and cleanup all no-op'd); the ES
+// columns are keywords and match exactly.
+func TestKeywordFilterConditionExactForWhitespaceValues(t *testing.T) {
+ cases := []struct {
+ name, field, value, want string
+ }{
+ {"cluster name", "parent_kwd", "Imperial Memorial and Governance Advice 59cbfbef",
+ `parent_kwd = 'Imperial Memorial and Governance Advice 59cbfbef'`},
+ {"CJK cluster name with suffix", "title_kwd", "忠臣进谏与托孤遗志 27577543",
+ `title_kwd = '忠臣进谏与托孤遗志 27577543'`},
+ {"doc leaf label", "title_kwd", "出师表中的忠诚与北伐决心 1234",
+ `title_kwd = '出师表中的忠诚与北伐决心 1234'`},
+ // Single-token values keep the full-text rendering: legacy ###-joined
+ // varchar columns still rely on token matching.
+ {"single token", "type_kwd", "nav_cluster", `filter_fulltext('type_kwd', 'nav_cluster')`},
+ {"root parent", "parent_kwd", "root", `filter_fulltext('parent_kwd', 'root')`},
+ // Any Unicode space counts: a full-width space tokenizes like an ASCII
+ // one and would hit the same silent no-match.
+ {"ideographic space", "title_kwd", "忠臣进谏 与托孤遗志",
+ `title_kwd = '忠臣进谏 与托孤遗志'`},
+ // Fields convertMatchingField maps to a full-text index reference are NOT
+ // columns: `= ` on the reference makes Infinity reject the whole statement,
+ // so a multi-token value is phrased through the index instead (that index
+ // is what gives tag/toc the membership match ES gets from its array).
+ {"tag multi-token", "tag_kwd", "machine learning",
+ `filter_fulltext('tag_kwd@ft_tag_kwd_whitespace__', '"machine learning"')`},
+ {"toc multi-token", "toc_kwd", "第 一章",
+ `filter_fulltext('toc_kwd@ft_toc_kwd_whitespace__', '"第 一章"')`},
+ {"tag single token", "tag_kwd", "nlp",
+ `filter_fulltext('tag_kwd@ft_tag_kwd_whitespace__', 'nlp')`},
+ // A value that cannot be phrased compares the raw column, which is at
+ // least addressable (the index reference is not).
+ {"tag with a quote", "tag_kwd", `say "hi" now`,
+ `tag_kwd = 'say "hi" now'`},
+ // A *_kwd field with no index variant stays on the raw column.
+ {"source_id", "source_id", "a b", `source_id = 'a b'`},
+ // A quote stays escaped in both renderings.
+ {"quoted multi-token", "title_kwd", "O'Brien cluster", `title_kwd = 'O''Brien cluster'`},
+ {"quoted single token", "type_kwd", "o'brien", `filter_fulltext('type_kwd', 'o''brien')`},
+ }
+ for _, c := range cases {
+ if got := keywordFilterCondition(c.field, c.value); got != c.want {
+ t.Errorf("%s: keywordFilterCondition(%q, %q) = %q, want %q",
+ c.name, c.field, c.value, got, c.want)
+ }
+ }
+}
+
+// TestKeywordFilterRenderingBothPaths pins that the search path
+// (equivalentConditionToStr) and the update/delete path
+// (buildFilterFromCondition) render a cluster-name filter identically: a drift
+// would write a doc's parent_kwd while the cluster's own update matched nothing.
+func TestKeywordFilterRenderingBothPaths(t *testing.T) {
+ const name = "Imperial Memorial and Governance Advice 59cbfbef"
+
+ wantParent := `(parent_kwd = '` + name + `')`
+ parentCond := map[string]interface{}{"parent_kwd": []string{name}}
+ if got := equivalentConditionToStr(parentCond, nil); got != wantParent {
+ t.Errorf("search path parent_kwd = %q, want %q", got, wantParent)
+ }
+ if got := buildFilterFromCondition(parentCond, nil); got != wantParent {
+ t.Errorf("update/delete path parent_kwd = %q, want %q", got, wantParent)
+ }
+
+ // The cluster's own filter (merge description, append doc, cleanup) uses
+ // title_kwd; ListClusters keeps filtering parent_kwd='root', which stays on
+ // the full-text path because it is a single token.
+ wantTitle := `(title_kwd = '` + name + `')`
+ titleCond := map[string]interface{}{"title_kwd": []string{name}}
+ if got := equivalentConditionToStr(titleCond, nil); got != wantTitle {
+ t.Errorf("search path title_kwd = %q, want %q", got, wantTitle)
+ }
+ if got := buildFilterFromCondition(titleCond, nil); got != wantTitle {
+ t.Errorf("update/delete path title_kwd = %q, want %q", got, wantTitle)
+ }
+ if got := equivalentConditionToStr(map[string]interface{}{"parent_kwd": []string{"root"}}, nil); got !=
+ `(filter_fulltext('parent_kwd', 'root'))` {
+ t.Errorf("root clusters must keep the single-token rendering, got %q", got)
+ }
+
+ // Blank entries are dropped by BOTH paths. Infinity rejects an empty
+ // full-text query ("Trying to match: on fields: <column> failed", 3052) and
+ // fails the whole statement — including UpdateChunks/DeleteChunks, which the
+ // update path feeds.
+ blankMixed := map[string]interface{}{"from_kwd": []string{"", " ", "x"}}
+ wantMixed := `(filter_fulltext('from_kwd', 'x'))`
+ if got := equivalentConditionToStr(blankMixed, nil); got != wantMixed {
+ t.Errorf("search path with blank entries = %q, want %q", got, wantMixed)
+ }
+ if got := buildFilterFromCondition(blankMixed, nil); got != wantMixed {
+ t.Errorf("update/delete path with blank entries = %q, want %q", got, wantMixed)
+ }
+
+ // Every entry blank: the search path contributes no clause, while the update
+ // path yields "1=1" so the caller's unconstrained-statement guard refuses it
+ // instead of running an unscoped update or delete.
+ allBlank := map[string]interface{}{"from_kwd": []string{"", " "}}
+ if got := equivalentConditionToStr(allBlank, nil); got != "" {
+ t.Errorf("search path with only blank entries = %q, want no clause", got)
+ }
+ if got := buildFilterFromCondition(allBlank, nil); got != "1=1" {
+ t.Errorf("update/delete path with only blank entries = %q, want '1=1'", got)
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index 81123c81f5..97c424ca6d 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -129,8 +129,9 @@ var compiledSelectFields = []string{
// LoadDocProducts reconstructs products with an empty Vector, so merged rows
// carry no embedding and the dataset-level KNN dedup (SearchSimilar on
// available_int=1 + q_<dim>_vec) can never match an existing wiki page — the
-// graph keeps accumulating cross-run duplicates. ES accepts the wildcard in
-// both _source includes and the fields parameter.
+// graph keeps accumulating cross-run duplicates. ES takes the wildcard verbatim
+// in _source; the Infinity engine expands it against the table's real columns
+// because Infinity's SQL binder rejects a partial wildcard (3013).
var wikiSelectFields = []string{
"page_type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd",
"entity_names_kwd", "summary_with_weight",
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index 4aee8ab937..dbc62a0e46 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -620,10 +620,7 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat
if err != nil {
return nil, err
}
- fromKwds := make([]string, 0, len(entityChunks))
- for _, c := range entityChunks {
- fromKwds = append(fromKwds, firstStringValue(c["slug_kwd"]))
- }
+ fromKwds := wikiEntitySlugs(entityChunks)
graph := &WikiGraph{
Entities: []WikiGraphEntity{},
Relations: []WikiGraphRelation{},
@@ -689,6 +686,21 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat
return graph, nil
}
+// wikiEntitySlugs returns the entity slugs the graph's relation lookup filters
+// on. Rows without a slug are skipped: a blank value would render as
+// filter_fulltext('from_kwd', ”), which Infinity rejects with 3052 ("Trying to
+// match: on fields: from_kwd failed") and fails the whole graph request, while
+// ES happily returns an empty result.
+func wikiEntitySlugs(entityChunks []map[string]interface{}) []string {
+ slugs := make([]string, 0, len(entityChunks))
+ for _, c := range entityChunks {
+ if slug := firstStringValue(c["slug_kwd"]); slug != "" {
+ slugs = append(slugs, slug)
+ }
+ }
+ return slugs
+}
+
// WikiAlteration is the alteration summary for a dataset's wiki artifacts.
type WikiAlteration struct {
Removed int `json:"removed"`
diff --git a/internal/service/dataset_artifact_topic_test.go b/internal/service/dataset_artifact_topic_test.go
index 05bb39d496..8bc5efcac5 100644
--- a/internal/service/dataset_artifact_topic_test.go
+++ b/internal/service/dataset_artifact_topic_test.go
@@ -28,6 +28,32 @@ func TestAggregateWikiTopicItemsUsesMaterializedPaths(t *testing.T) {
}
}
+// TestWikiEntitySlugsSkipsBlankSlugs pins the GetWikiGraph guard: an entity row
+// without a slug must not reach the relation filter, because the engine renders
+// it as filter_fulltext('from_kwd', ”) and Infinity fails the whole query with
+// "Trying to match: on fields: from_kwd failed" (3052).
+func TestWikiEntitySlugsSkipsBlankSlugs(t *testing.T) {
+ got := wikiEntitySlugs([]map[string]interface{}{
+ {"slug_kwd": "topic/出师表"},
+ {"slug_kwd": ""},
+ {"slug_kwd": nil},
+ {}, // phantom row: the engine projected nothing
+ {"slug_kwd": "person/诸葛亮"},
+ })
+ want := []string{"topic/出师表", "person/诸葛亮"}
+ if len(got) != len(want) {
+ t.Fatalf("wikiEntitySlugs = %#v, want %#v", got, want)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("wikiEntitySlugs = %#v, want %#v", got, want)
+ }
+ }
+ if slugs := wikiEntitySlugs(nil); len(slugs) != 0 {
+ t.Errorf("wikiEntitySlugs(nil) = %#v, want empty", slugs)
+ }
+}
+
func TestAggregateWikiTopicItemsIgnoresPathCase(t *testing.T) {
items := aggregateWikiTopicItems([]map[string]interface{}{
{"topic_kwd": "Knowledge/Core"},
diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.go
index bfe0ad8c53..2addd09374 100644
--- a/internal/service/nlp/datasetnav.go
+++ b/internal/service/nlp/datasetnav.go
@@ -59,10 +59,11 @@ const (
)
// navNodeSelectFields are the columns every nav tree read needs: the label
-// (title_kwd), the Python-era name column, the parent edge, the payload
-// (description), and the leaf's document id.
+// (title_kwd, plus docnm_kwd for rows Infinity folded the label into), the
+// Python-era name column, the parent edge, the payload (description), and the
+// leaf's document id.
var navNodeSelectFields = []string{
- "name", "title_kwd", "parent_kwd", "depth_int",
+ "name", "title_kwd", "docnm_kwd", "parent_kwd", "depth_int",
"content_with_weight", "doc_count_int", "type_kwd", "doc_id",
}
@@ -155,6 +156,11 @@ func (s *NavService) navSearch(ctx context.Context, tenantID, kbID string, filte
SelectFields: selectFields,
Filter: merged,
MatchExprs: matchExprs,
+ // Nav rows carry available_int=0, and Infinity adds available_int=1 to
+ // every MATCH query unless the caller opts out (shouldDefaultAvailableFilter):
+ // nav's own vector/keyword reads would otherwise see none of its rows. ES
+ // never filters available_int; the compile_kwd filter scopes the read.
+ IncludeUnavailable: true,
}
res, err := de.Search(ctx, req)
if err != nil {
@@ -338,13 +344,26 @@ func (s *NavService) navNodesFromRows(rows []map[string]interface{}) []nav.NavNo
return nodes
}
+// navRowLabel returns the nav row's stored label: the value parent_kwd
+// references and the tree renders. Go's writer stores it in title_kwd, Python's
+// in name. Rows persisted before Infinity kept title_kwd as a column of its own
+// carry it only in docnm (see engine/infinity transformChunkFields), which reads
+// back as docnm_kwd; without that fallback the API answered the description's
+// first line as a cluster name, so the tree's child lookup by name (the frontend
+// expands with GET /navigation/<name>/children) matched no row.
+func navRowLabel(row map[string]interface{}) string {
+ for _, col := range []string{"name", "title_kwd", "docnm_kwd"} {
+ if v := firstStringValue(row[col]); v != "" {
+ return v
+ }
+ }
+ return ""
+}
+
// navClusterRowKey is the key a cluster's children reference through parent_kwd:
// Python's writer stores it in `name`, Go's in `title_kwd`.
func navClusterRowKey(row map[string]interface{}) string {
- if name := firstStringValue(row["name"]); name != "" {
- return name
- }
- return firstStringValue(row["title_kwd"])
+ return navRowLabel(row)
}
// navDocRowKey / navClusterKey namespace the two row identities so a cluster and
@@ -377,10 +396,7 @@ func navKeywordExpressions(keywords string) []interface{} {
// is written. An explicit readable `name` column (Python's writer stores one)
// wins when present.
func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string) nav.NavNode {
- name := firstStringValue(row["title_kwd"])
- if n := firstStringValue(row["name"]); n != "" {
- name = n
- }
+ name := navRowLabel(row)
// A raw id (doc_id or "cluster_<hash>") is not a human-readable name; fall
// back to a title derived from the payload description. Cluster names are the
// child-lookup key (parent_kwd references them verbatim), so they must stay
@@ -526,7 +542,7 @@ func (s *NavService) Search(ctx context.Context, tenantID, kbID, query string, e
func (s *NavService) navScan(ctx context.Context, tenantID, kbID string, filter map[string]interface{}, f64 []float64, limit int, allowed map[string]struct{}) ([]nav.NavHit, error) {
chunks, _, err := s.navSearch(ctx, tenantID, kbID,
filter,
- []string{"type_kwd", "title_kwd", "doc_id", "doc_ids_kwd", "_score"}, 0, limit,
+ []string{"type_kwd", "title_kwd", "docnm_kwd", "doc_id", "doc_ids_kwd", "_score"}, 0, limit,
[]interface{}{&types.MatchDenseExpr{
VectorColumnName: fmt.Sprintf("q_%d_vec", len(f64)),
EmbeddingData: f64,
@@ -542,7 +558,7 @@ func (s *NavService) navScan(ctx context.Context, tenantID, kbID string, filter
for _, c := range chunks {
h := nav.NavHit{
Type: firstStringValue(c["type_kwd"]),
- Name: firstStringValue(c["title_kwd"]),
+ Name: navRowLabel(c),
DocID: firstStringValue(c["doc_id"]),
DocIDs: firstStringSlice(c["doc_ids_kwd"]),
}
@@ -647,7 +663,7 @@ func (s *NavService) SummariesByDocIDs(ctx context.Context, tenantID, kbID strin
"type_kwd": []string{"nav_doc"},
"doc_id": docIDs,
}),
- []string{"type_kwd", "name", "title_kwd", "content_with_weight", "doc_id"},
+ []string{"type_kwd", "name", "title_kwd", "docnm_kwd", "content_with_weight", "doc_id"},
0, len(docIDs), nil)
if err != nil {
return map[string]string{}
@@ -669,12 +685,9 @@ func (s *NavService) SummariesByDocIDs(ctx context.Context, tenantID, kbID strin
}
if summary == "" {
// No description in the payload: fall back to the stored label
- // (readable name, else title_kwd), rejecting raw ids — a 32-hex id or
- // "cluster_<hash>" is not a human label.
- summary = firstStringValue(c["name"])
- if summary == "" {
- summary = firstStringValue(c["title_kwd"])
- }
+ // (readable name, else title_kwd/docnm), rejecting raw ids — a
+ // 32-hex id or "cluster_<hash>" is not a human label.
+ summary = navRowLabel(c)
if graphIsRawID(summary) {
summary = ""
}
@@ -726,7 +739,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"type_kwd": []string{nav.TypeNavDoc},
"doc_id": []string{in.DocID},
}),
- []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd"}, 0, 1, nil)
+ []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd", "docnm_kwd"}, 0, 1, nil)
if err != nil {
return err
}
@@ -737,7 +750,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
// summary's first line, or the model-facing readers would keep seeing a
// file name. The tokenized text is (re)built when it is missing.
title := navDocTitle(in.Summary)
- titleStale := firstStringValue(existing[0]["title_kwd"]) != title
+ titleStale := navRowLabel(existing[0]) != title
ltksStale := firstStringValue(existing[0]["content_ltks"]) == "" ||
firstStringValue(existing[0]["content_sm_ltks"]) == ""
if payload, ok := existing[0]["content_with_weight"].(string); ok {
@@ -924,6 +937,48 @@ func navClusterID(tenantID, kbID, name string) string {
return "dataset_nav_cluster_" + contentHash8(tenantID+"\x00"+kbID+"\x00"+name)
}
+// navClusterCondition addresses one nav cluster row by its deterministic row id,
+// the way Python's dataset_nav does (every cluster update/delete resolves
+// _nav_cluster_id and read-modify-writes that row). The label is not usable as a
+// key: it is an LLM-written name that can share words with another cluster, and
+// Infinity cannot match a multi-token value on these columns at all (see
+// engine/infinity keywordFilterCondition). The row id is always single-token.
+func navClusterCondition(tenantID, kbID, clusterName string) map[string]interface{} {
+ return navFilter(map[string]interface{}{
+ "id": []string{navClusterID(tenantID, kbID, clusterName)},
+ })
+}
+
+// navChildRehomeCondition builds the condition that moves one split child under
+// its new parent, or nil when the child cannot be addressed at all.
+//
+// The row id is preferred: the label is empty for rows the Infinity write path
+// folded into docnm, and two children can share a summary's first line, so a
+// label filter can miss the row or hit a sibling. A nav_doc leaf without a row id
+// is still identifiable by its document. Nothing else is safe — with blank
+// keywords dropped, a type-only condition would match every row of that type in
+// the dataset.
+func navChildRehomeCondition(child map[string]interface{}, kbID string) map[string]interface{} {
+ typ := firstStringValue(child["type_kwd"])
+ if typ == "" {
+ return nil
+ }
+ cond := map[string]interface{}{
+ "compile_kwd": []string{navCompileKwd},
+ "kb_id": kbID,
+ }
+ if rowID := firstStringValue(child["id"]); rowID != "" {
+ cond["id"] = []string{rowID}
+ return cond
+ }
+ if docID := firstStringValue(child["doc_id"]); typ == "nav_doc" && docID != "" {
+ cond["type_kwd"] = []string{typ}
+ cond["doc_id"] = []string{docID}
+ return cond
+ }
+ return nil
+}
+
// cleanupEmptyCluster deletes a nav_cluster that holds no docs and no child
// clusters, then recurses to its parent (cascade, mirroring Python
// _cleanup_empty_cluster). It is called by RemoveDoc after deleting a nav_doc so
@@ -937,7 +992,7 @@ func (s *NavService) cleanupEmptyCluster(ctx context.Context, tenantID, kbID, cl
return err
}
chunks, _, err := s.navSearch(ctx, tenantID, kbID,
- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ navClusterCondition(tenantID, kbID, clusterName),
[]string{"id", "parent_kwd", "doc_count_int", "doc_ids_kwd"}, 0, 1, nil)
if err != nil {
return err
@@ -994,7 +1049,7 @@ func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string,
"type_kwd": []string{"nav_cluster"},
"parent_kwd": []string{parent},
}),
- []string{"title_kwd", "_score"}, 0, 1,
+ []string{"title_kwd", "docnm_kwd", "_score"}, 0, 1,
[]interface{}{&types.MatchDenseExpr{
VectorColumnName: fmt.Sprintf("q_%d_vec", len(f64)),
EmbeddingData: f64,
@@ -1009,7 +1064,7 @@ func (s *NavService) findBestCluster(ctx context.Context, tenantID, kbID string,
if len(chunks) == 0 {
break
}
- name := firstStringValue(chunks[0]["title_kwd"])
+ name := navRowLabel(chunks[0])
sim := rowScore(chunks[0])
// Adopt-and-overwrite, mirroring Python _find_best_cluster: the result is
// the DEEPEST cluster reached while the descent keeps clearing
@@ -1058,7 +1113,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
// engine column q_<dim>_vec, e.g. from the doc being appended) is selected so
// the split siblings can inherit a representative vector for KNN routing; an
// empty vecCol skips it (callers without a known dim).
- selectFields := []string{"id", "doc_id", "doc_count_int", "doc_ids_kwd", "title_kwd", "type_kwd"}
+ selectFields := []string{"id", "doc_id", "doc_count_int", "doc_ids_kwd", "title_kwd", "docnm_kwd", "type_kwd"}
if vecCol != "" {
selectFields = append(selectFields, vecCol)
}
@@ -1092,7 +1147,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
// held docs; select id + doc_ids_kwd so the split can delete by the
// engine-assigned row id and inherit the cluster's directly-held documents.
clusterChunks, _, err := s.navSearch(ctx, tenantID, kbID,
- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ navClusterCondition(tenantID, kbID, clusterName),
[]string{"id", "doc_count_int", "doc_ids_kwd", "parent_kwd", "depth_int"}, 0, 1, nil)
if err != nil {
return err
@@ -1121,23 +1176,23 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
ids []string
vec []float32
}{}
+ // Every child must be addressable before the first write. The rehome points
+ // children at splitA/splitB, and those rows are only inserted after the
+ // original cluster is deleted, so a child that cannot be rehomed would be left
+ // hanging off a name no query can reach — refuse the split instead of
+ // half-moving the subtree.
+ for _, child := range children {
+ if navChildRehomeCondition(child, kbID) == nil {
+ return fmt.Errorf("datasetnav: refusing to split cluster %q: child %q carries no row id, document id or type",
+ clusterName, navRowLabel(child))
+ }
+ }
for i, child := range children {
acc := &accA
if i%2 == 1 {
acc = &accB
}
- title := firstStringValue(child["title_kwd"])
typ := firstStringValue(child["type_kwd"])
- childID := firstStringValue(child["id"])
- if title == "" && childID == "" {
- continue
- }
- if typ == "" {
- // The engine returned a child without a type discriminator; skip it
- // so we never rehome with an empty-type filter that matches nothing
- // (leaving the child orphaned under a deleted cluster name).
- continue
- }
// Aggregate this child's doc count + doc ids into the target split.
if c := intValue(child["doc_count_int"]); c > 0 {
acc.count += c
@@ -1150,16 +1205,15 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
childDocIDs = append(childDocIDs, firstStringValue(child["doc_id"]))
}
acc.ids = appendUnique(acc.ids, childDocIDs)
- // Rehome the child by its row id (precise) rather than an empty-safe
- // type+title filter, so the UpdateChunks filter always matches.
- upd := map[string]interface{}{"parent_kwd": accTarget(i, splitA, splitB)}
- if err := de.UpdateChunks(ctx,
- map[string]interface{}{
- "compile_kwd": []string{navCompileKwd},
- "type_kwd": []string{typ},
- "title_kwd": []string{title},
- "kb_id": kbID,
- }, upd, s.navIndexName(tenantID), kbID); err != nil {
+ childCond := navChildRehomeCondition(child, kbID)
+ if childCond == nil {
+ // Unreachable after the pre-pass above; kept so a future caller
+ // cannot fall through to a condition that matches the whole dataset.
+ return fmt.Errorf("datasetnav: refusing to rehome %s child %q", typ, navRowLabel(child))
+ }
+ if err := de.UpdateChunks(ctx, childCond,
+ map[string]interface{}{"parent_kwd": accTarget(i, splitA, splitB)},
+ s.navIndexName(tenantID), kbID); err != nil {
return err
}
}
@@ -1312,7 +1366,7 @@ func (s *NavService) mergeClusterDescription(ctx context.Context, de engine.DocE
return nil
}
rows, _, err := s.navSearch(ctx, tenantID, kbID,
- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ navClusterCondition(tenantID, kbID, clusterName),
[]string{"content_with_weight"}, 0, 1, nil)
if err != nil {
return err
@@ -1351,12 +1405,7 @@ func (s *NavService) mergeClusterDescription(ctx context.Context, de engine.DocE
}
}
return de.UpdateChunks(ctx,
- map[string]interface{}{
- "compile_kwd": []string{navCompileKwd},
- "type_kwd": []string{"nav_cluster"},
- "title_kwd": []string{clusterName},
- "kb_id": kbID,
- }, updates, s.navIndexName(tenantID), kbID)
+ navClusterCondition(tenantID, kbID, clusterName), updates, s.navIndexName(tenantID), kbID)
}
// llmMergeDescription merges a set of source descriptions into one via the
@@ -1460,7 +1509,7 @@ func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string
// representative vector; pass "" when the caller has no known dimension.
func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID, vecCol string) error {
chunks, _, err := s.navSearch(ctx, tenantID, kbID,
- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ navClusterCondition(tenantID, kbID, clusterName),
[]string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)
if err != nil {
return err
@@ -1491,17 +1540,14 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine
if !found {
count++
}
- // Pin the update to the nav_cluster row only: a regular chunk sharing the
- // same title_kwd must never be clobbered. The read-modify-write here is
- // expected to run under a per-dataset lock held by the UpsertDoc caller;
- // without it, concurrent appends to the same cluster can lose updates.
+ // The condition addresses the cluster by its row id, so no other row can be
+ // caught by the update — neither a regular chunk that happens to carry a
+ // similar title nor a second cluster whose readable name shares words with
+ // this one. The read-modify-write here is expected to run under a per-dataset
+ // lock held by the UpsertDoc caller; without it, concurrent appends to the
+ // same cluster can lose updates.
if err := de.UpdateChunks(ctx,
- map[string]interface{}{
- "compile_kwd": []string{navCompileKwd},
- "type_kwd": []string{"nav_cluster"},
- "title_kwd": []string{clusterName},
- "kb_id": kbID,
- },
+ navClusterCondition(tenantID, kbID, clusterName),
map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},
s.navIndexName(tenantID), kbID); err != nil {
return err
@@ -1602,7 +1648,7 @@ func (s *NavService) findClusterContainingDoc(ctx context.Context, tenantID, kbI
res, err := de.Search(ctx, &types.SearchRequest{
IndexNames: []string{s.navIndexName(tenantID)},
KbIDs: []string{kbID},
- SelectFields: []string{"title_kwd", "doc_ids_kwd"},
+ SelectFields: []string{"title_kwd", "docnm_kwd", "doc_ids_kwd"},
Filter: map[string]interface{}{"type_kwd": "nav_cluster", "compile_kwd": []string{navCompileKwd}},
Offset: offset,
Limit: pageSize,
@@ -1613,7 +1659,7 @@ func (s *NavService) findClusterContainingDoc(ctx context.Context, tenantID, kbI
for _, row := range res.Chunks {
for _, d := range firstStringSlice(row["doc_ids_kwd"]) {
if d == docID {
- return firstStringValue(row["title_kwd"]), nil
+ return navRowLabel(row), nil
}
}
}
@@ -1632,7 +1678,7 @@ func (s *NavService) removeDocFromCluster(ctx context.Context, tenantID, kbID, c
return err
}
chunks, _, err := s.navSearch(ctx, tenantID, kbID,
- navFilter(map[string]interface{}{"type_kwd": []string{"nav_cluster"}, "title_kwd": []string{clusterName}}),
+ navClusterCondition(tenantID, kbID, clusterName),
[]string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)
if err != nil {
return err
@@ -1654,12 +1700,7 @@ func (s *NavService) removeDocFromCluster(ctx context.Context, tenantID, kbID, c
count--
}
return de.UpdateChunks(ctx,
- map[string]interface{}{
- "compile_kwd": []string{navCompileKwd},
- "type_kwd": []string{"nav_cluster"},
- "title_kwd": []string{clusterName},
- "kb_id": kbID,
- },
+ navClusterCondition(tenantID, kbID, clusterName),
map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},
s.navIndexName(tenantID), kbID)
}
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index 5a70733828..f900aa272b 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -21,6 +21,10 @@ import (
type memNavEngine struct {
rows []map[string]interface{}
nextID int
+
+ // searches records every SearchRequest so a test can pin the request shape
+ // (e.g. that nav reads opt out of the retrieval-only available_int=1 default).
+ searches []*types.SearchRequest
}
func newMemNavEngine() *memNavEngine { return &memNavEngine{} }
@@ -28,8 +32,16 @@ func newMemNavEngine() *memNavEngine { return &memNavEngine{} }
func (m *memNavEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, _ string, datasetID string) ([]string, error) {
ids := make([]string, 0, len(chunks))
for _, c := range chunks {
- m.nextID++
- id := "nav" + strconvItoa(m.nextID)
+ // A caller-provided row id is kept, exactly as the real engines do it
+ // (elasticsearch: chunk["id"] becomes the document id; infinity: the id
+ // column is written verbatim). Nav addresses its cluster rows by the
+ // deterministic id they were written with, so generating a fresh one here
+ // would make every update/delete miss.
+ id, _ := c["id"].(string)
+ if id == "" {
+ m.nextID++
+ id = "nav" + strconvItoa(m.nextID)
+ }
cp := make(map[string]interface{}, len(c)+2)
for k, v := range c {
cp[k] = v
@@ -68,6 +80,7 @@ func (m *memNavEngine) DeleteChunks(_ context.Context, cond map[string]interface
}
func (m *memNavEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ m.searches = append(m.searches, req)
var matched []map[string]interface{}
// This engine implements the DENSE leg only. A lexical match expression is
// deliberately a no-op rather than a dense one: treating a text expression
@@ -1414,8 +1427,13 @@ func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {
// Seed the overfull cluster directly: one nav_cluster (parent=root sentinel,
// depth 1) carrying 55 nav_doc children.
idx := sTestNavIndex(ns)
+ // The seeded rows carry the row ids production writes: nav addresses its
+ // clusters by id (navClusterID, exactly like Python's _nav_cluster_id), so a
+ // fixture without them would leave the locate/update/delete steps matching no
+ // row at all.
rows := []map[string]interface{}{
{
+ "id": navClusterID("t1", "kb1", clusterName),
"compile_kwd": navCompileKwd,
"available_int": 0,
"type_kwd": "nav_cluster",
@@ -1428,6 +1446,7 @@ func TestNavService_MaybeSplitCluster_SplitsOverfull(t *testing.T) {
}
for i := 0; i < 55; i++ {
rows = append(rows, map[string]interface{}{
+ "id": navDocID("t1", "kb1", fmt.Sprintf("d%02d", i)),
"compile_kwd": navCompileKwd,
"available_int": 0,
"type_kwd": "nav_doc",
@@ -1695,3 +1714,237 @@ func intValAny(v any) int {
}
return 0
}
+
+// TestNavService_ReadsIncludeUnavailableRows pins that nav reads opt out of the
+// retrieval-only available_int=1 default Infinity applies to every MATCH query:
+// nav rows carry available_int=0, so without it the cluster descent and navScan
+// would see an empty tree. ES never filters available_int.
+func TestNavService_ReadsIncludeUnavailableRows(t *testing.T) {
+ eng := newMemNavEngine()
+ ns := newTestNav(eng)
+ if err := ns.UpsertDoc(t.Context(), navUpsertInput("t1", "kb1", "d1", "aaa")); err != nil {
+ t.Fatal(err)
+ }
+ if len(eng.rows) == 0 {
+ t.Fatal("UpsertDoc wrote no nav row")
+ }
+ if v := eng.rows[0]["available_int"]; v != 0 {
+ t.Fatalf("nav row available_int = %#v, want 0", v)
+ }
+
+ if _, _, err := ns.ListClusters(t.Context(), "t1", "kb1", "", 0, 10); err != nil {
+ t.Fatal(err)
+ }
+ if len(eng.searches) == 0 {
+ t.Fatal("ListClusters issued no search")
+ }
+ for i, req := range eng.searches {
+ if !req.IncludeUnavailable {
+ t.Errorf("search %d did not opt out of the available_int=1 default (SelectFields=%#v)", i, req.SelectFields)
+ }
+ }
+}
+
+// TestNavService_MultiTokenClusterUpdatesByRowID pins the write path for a
+// cluster whose label is multi-token and whose title_kwd is empty. Such a row is
+// addressable only by its deterministic row id: a name filter on Infinity matched
+// nothing, so the membership/count update and the cleanup lookup silently
+// no-op'd.
+func TestNavService_MultiTokenClusterUpdatesByRowID(t *testing.T) {
+ const clusterName = "Imperial Memorial and Governance Advice 59cbfbef"
+ clusterRowID := navClusterID("t1", "kb1", clusterName)
+
+ eng := newMemNavEngine()
+ eng.rows = append(eng.rows,
+ map[string]interface{}{
+ "id": clusterRowID,
+ "kb_id": "kb1", "compile_kwd": navCompileKwd, "type_kwd": "nav_cluster",
+ "available_int": 0,
+ // The legacy Infinity layout: the label only reached docnm.
+ "title_kwd": "", "docnm_kwd": clusterName,
+ "parent_kwd": navRootParent, "depth_int": 1,
+ "doc_count_int": 2, "doc_ids_kwd": []string{"d1", "d2"},
+ "content_with_weight": `{"description":"two memorials"}`,
+ },
+ map[string]interface{}{
+ "id": navDocID("t1", "kb1", "d1"),
+ "kb_id": "kb1", "compile_kwd": navCompileKwd, "type_kwd": "nav_doc",
+ "available_int": 0, "title_kwd": "memorial of d1",
+ "parent_kwd": clusterName, "depth_int": 2,
+ "doc_id": "d1", "doc_count_int": 1,
+ })
+ ns := newTestNav(eng)
+
+ // The read side must answer the stored key so the frontend can expand it.
+ clusters, _, err := ns.ListClusters(context.Background(), "t1", "kb1", "", 0, 10)
+ if err != nil {
+ t.Fatalf("ListClusters: %v", err)
+ }
+ if len(clusters) != 1 || clusters[0].Name != clusterName {
+ t.Fatalf("clusters = %#v, want the stored key %q", clusters, clusterName)
+ }
+
+ if err := ns.RemoveDoc(context.Background(), "t1", "kb1", "d1"); err != nil {
+ t.Fatalf("RemoveDoc: %v", err)
+ }
+
+ row := navRowByID(eng, clusterRowID)
+ if row == nil {
+ t.Fatal("cluster row vanished")
+ }
+ if got := intValAny(row["doc_count_int"]); got != 1 {
+ t.Errorf("cluster doc_count_int = %d, want 1 (the update must reach a row whose title_kwd is empty)", got)
+ }
+ if ids := firstStringSlice(row["doc_ids_kwd"]); len(ids) != 1 || ids[0] != "d2" {
+ t.Errorf("cluster doc_ids_kwd = %v, want [d2]", ids)
+ }
+}
+
+// TestNavService_MaybeSplitClusterRefusesUnidentifiableChild pins the guard that
+// dropping blank keyword values made necessary: a child with neither a row id nor
+// a document id must abort the split. Falling back to a label would leave a
+// type-only condition, and with the blank label dropped before rendering that
+// condition reparents every row of that type in the dataset instead of failing.
+func TestNavService_MaybeSplitClusterRefusesUnidentifiableChild(t *testing.T) {
+ const clusterName = "overfull_no_ids"
+ eng := newMemNavEngine()
+ eng.rows = append(eng.rows, map[string]interface{}{
+ "id": navClusterID("t1", "kb1", clusterName), "kb_id": "kb1",
+ "compile_kwd": navCompileKwd, "type_kwd": "nav_cluster",
+ "title_kwd": clusterName, "docnm_kwd": clusterName,
+ "parent_kwd": navRootParent, "depth_int": 1,
+ "doc_count_int": navMaxDocsPerCluster + 1, "doc_ids_kwd": []string{},
+ })
+ // Directly seeded (not via InsertChunks) so they carry NO row id, and no
+ // doc_id either: every fallback the rehome could use is unavailable.
+ for i := 0; i <= navMaxDocsPerCluster; i++ {
+ eng.rows = append(eng.rows, map[string]interface{}{
+ "kb_id": "kb1", "compile_kwd": navCompileKwd, "type_kwd": "nav_doc",
+ "title_kwd": fmt.Sprintf("kid %02d", i), "parent_kwd": clusterName,
+ "depth_int": 2, "doc_count_int": 1,
+ })
+ }
+ ns := newTestNav(eng)
+
+ if err := ns.maybeSplitCluster(context.Background(), "t1", "kb1", clusterName, ""); err == nil {
+ t.Fatal("a child without a row id or document id must abort the split")
+ }
+ // Nothing may be reparented, and the original cluster must still be there.
+ reparented := 0
+ originalPresent := false
+ for _, row := range eng.rows {
+ if row["type_kwd"] == "nav_cluster" && row["title_kwd"] == clusterName {
+ originalPresent = true
+ }
+ if p, _ := row["parent_kwd"].(string); strings.HasSuffix(p, ":A") || strings.HasSuffix(p, ":B") {
+ reparented++
+ }
+ }
+ if reparented != 0 {
+ t.Errorf("%d rows were reparented by a type-only condition", reparented)
+ }
+ if !originalPresent {
+ t.Error("the original cluster must not be deleted when the split aborts")
+ }
+}
+
+// TestNavService_MaybeSplitClusterValidatesEveryChildFirst pins the pre-pass: an
+// unaddressable child anywhere in the list must stop the split before the first
+// write. The rehome points children at splitA/splitB, whose rows are inserted
+// only after the original cluster is deleted, so aborting mid-loop would leave
+// the already-moved children hanging off a name no query can reach.
+func TestNavService_MaybeSplitClusterValidatesEveryChildFirst(t *testing.T) {
+ const clusterName = "overfull_second_child_broken"
+ eng := newMemNavEngine()
+ eng.rows = append(eng.rows, map[string]interface{}{
+ "id": navClusterID("t1", "kb1", clusterName), "kb_id": "kb1",
+ "compile_kwd": navCompileKwd, "type_kwd": "nav_cluster",
+ "title_kwd": clusterName, "docnm_kwd": clusterName,
+ "parent_kwd": navRootParent, "depth_int": 1,
+ "doc_count_int": navMaxDocsPerCluster + 1, "doc_ids_kwd": []string{},
+ })
+ // The first child IS addressable, so a loop that validated lazily would have
+ // reparented it before reaching the broken child below.
+ eng.rows = append(eng.rows, map[string]interface{}{
+ "id": navDocID("t1", "kb1", "d0"), "kb_id": "kb1", "compile_kwd": navCompileKwd,
+ "type_kwd": "nav_doc", "title_kwd": "good child", "doc_id": "d0",
+ "parent_kwd": clusterName, "depth_int": 2, "doc_count_int": 1,
+ })
+ for i := 1; i <= navMaxDocsPerCluster; i++ {
+ eng.rows = append(eng.rows, map[string]interface{}{
+ "kb_id": "kb1", "compile_kwd": navCompileKwd, "type_kwd": "nav_doc",
+ "title_kwd": fmt.Sprintf("broken %02d", i), "parent_kwd": clusterName,
+ "depth_int": 2, "doc_count_int": 1,
+ })
+ }
+ ns := newTestNav(eng)
+
+ if err := ns.maybeSplitCluster(context.Background(), "t1", "kb1", clusterName, ""); err == nil {
+ t.Fatal("an unaddressable child must abort the split")
+ }
+ for _, row := range eng.rows {
+ if p, _ := row["parent_kwd"].(string); strings.HasSuffix(p, ":A") || strings.HasSuffix(p, ":B") {
+ t.Fatalf("row %v was reparented before every child had been validated", row["id"])
+ }
+ }
+ if navRowByID(eng, navClusterID("t1", "kb1", clusterName)) == nil {
+ t.Error("the original cluster must not be deleted when the split aborts")
+ }
+}
+
+// navRowByID returns the stored row with the given id, or nil.
+func navRowByID(eng *memNavEngine, id string) map[string]interface{} {
+ for _, r := range eng.rows {
+ if s, _ := r["id"].(string); s == id {
+ return r
+ }
+ }
+ return nil
+}
+
+// TestNavService_LabelsFallBackToDocnm pins the label read for rows persisted
+// before Infinity kept title_kwd as its own column (the label reached docnm
+// only). The name must be the STORED key, not a display fallback: the frontend
+// looks children up with it, so a description-derived name left the tree flat.
+func TestNavService_LabelsFallBackToDocnm(t *testing.T) {
+ const clusterName = "忠臣进谏与托孤遗志 27577543"
+ eng := newMemNavEngine()
+ eng.rows = append(eng.rows,
+ map[string]interface{}{
+ "id": "dataset_nav_cluster_1", "compile_kwd": navCompileKwd, "type_kwd": "nav_cluster",
+ "kb_id": "kb1", "available_int": 0, "title_kwd": "", "docnm_kwd": clusterName,
+ "parent_kwd": "root", "depth_int": 0, "doc_count_int": 1, "doc_ids_kwd": []string{"d1"},
+ },
+ map[string]interface{}{
+ "id": "dataset_nav_doc_1", "compile_kwd": navCompileKwd, "type_kwd": "nav_doc",
+ "kb_id": "kb1", "available_int": 0, "title_kwd": "", "docnm_kwd": "出师表中的忠诚与北伐决心",
+ "parent_kwd": clusterName, "depth_int": 1, "doc_count_int": 1, "doc_id": "d1",
+ })
+ ns := newTestNav(eng)
+
+ clusters, _, err := ns.ListClusters(context.Background(), "t1", "kb1", "", 0, 10)
+ if err != nil {
+ t.Fatalf("ListClusters: %v", err)
+ }
+ if len(clusters) != 1 {
+ t.Fatalf("clusters = %#v, want exactly one root cluster", clusters)
+ }
+ if clusters[0].Name != clusterName {
+ t.Fatalf("cluster name = %q, want the stored key %q", clusters[0].Name, clusterName)
+ }
+ if !clusters[0].HasChildren {
+ t.Error("cluster must expose has_children so the frontend can expand it")
+ }
+
+ // The frontend expands with the name it was given.
+ children, _, err := ns.ListChildren(context.Background(), "t1", "kb1", clusters[0].Name, "", 0, 10)
+ if err != nil {
+ t.Fatalf("ListChildren: %v", err)
+ }
+ if len(children) != 1 {
+ t.Fatalf("children = %#v, want the doc leaf", children)
+ }
+ if children[0].Name != "出师表中的忠诚与北伐决心" || children[0].DocID != "d1" {
+ t.Fatalf("child = %#v, want the doc leaf labelled from docnm_kwd", children[0])
+ }
+}