RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package infinity
import (
"context"
"encoding/json"
"errors"
"fmt"
"hash/fnv"
"maps"
"os"
"ragflow/internal/common"
"ragflow/internal/engine/types"
"ragflow/internal/utility"
"reflect"
"regexp"
"slices"
"sort"
"strconv"
"strings"
"sync"
"unicode"
"unicode/utf8"
infinity "github.com/infiniflow/infinity-go-sdk"
"go.uber.org/zap"
)
// ChinesePunctRegex splits on comma, semicolon, Chinese punctuations, and newlines
var ChinesePunctRegex = regexp.MustCompile(`[,,;;、\r\n]+`)
const (
pagerankAdjustRetryCount = 2
pagerankAdjustLockCount = 256
infinityMaxInsertRows = 8192
)
var pagerankAdjustLocks [pagerankAdjustLockCount]sync.Mutex
// CreateChunkStore creates a chunk table in Infinity
// baseName is the table name prefix (e.g., "ragflow_<tenant_id>")
// The full table name is built as "{baseName}_{datasetID}"
// For skill index (datasetID="skill"), tableName is just baseName and uses skill_infinity_mapping.json
// For memory message indexes (baseName prefixed "memory_"), uses message_infinity_mapping.json
// language is the dataset language, which selects the fulltext analyzer; see analyzerForLanguage.
func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID, language string) error {
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return fmt.Errorf("failed to get database: %w", err)
}
defer release()
return e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserID, language)
}
// memoryIndexPrefix marks per-memory message indexes (MemoryIndexName).
const memoryIndexPrefix = "memory_"
// isMemoryIndexName reports whether name is a memory message index.
func isMemoryIndexName(name string) bool {
return strings.HasPrefix(name, memoryIndexPrefix)
}
// chunkMappingFileName picks the schema an index is created from. Memory message
// tables need message_infinity_mapping.json: their rows carry user_id,
// message_id, ... which the chunk mapping does not declare, so creating them
// from the chunk mapping made every insert fail with 3013.
func chunkMappingFileName(baseName, datasetID, configured string) string {
switch {
case datasetID == "skill":
return "skill_infinity_mapping.json"
case isMemoryIndexName(baseName):
return "message_infinity_mapping.json"
default:
return configured
}
}
func (e *Engine) createChunkStoreWithDB(db *infinity.Database, baseName, datasetID string, vectorSize int, parserID, language string) error {
vecSize := vectorSize
// Determine table name and mapping file based on index type
var tableName string
var mappingFile string
tableName = buildChunkTableName(baseName, datasetID)
mappingFile = chunkMappingFileName(baseName, datasetID, e.mappingFileName)
common.Info("Creating table",
zap.String("tableName", tableName),
zap.String("mappingFile", mappingFile),
zap.Bool("memoryIndex", isMemoryIndexName(baseName)))
fpMapping, err := utility.FindConfFileInProject(mappingFile)
if err != nil {
return err
}
// Use configured schema
schemaData, err := os.ReadFile(*fpMapping)
if err != nil {
return fmt.Errorf("failed to read mapping file %s: %w", *fpMapping, err)
}
var schema orderedFields
if err := json.Unmarshal(schemaData, &schema); err != nil {
return fmt.Errorf("failed to parse mapping file: %w", err)
}
// Determine vector column name
vectorColName := fmt.Sprintf("q_%d_vec", vecSize)
// Check if table already exists
exists, err := e.tableExistsWithDB(db, tableName)
if err != nil {
return fmt.Errorf("failed to check if table exists: %w", err)
}
var table *infinity.Table
if exists {
// Table exists, open it and check if vector column needs to be added
common.Info("Table already exists, checking for vector column", zap.String("tableName", tableName))
table, err = db.GetTable(tableName)
if err != nil {
return fmt.Errorf("failed to open existing table %s: %w", tableName, err)
}
// Check if vector column exists (for embedding model changes)
colExists, err := e.columnExists(table, vectorColName)
if err != nil {
common.Warn("Failed to check column existence", zap.String("column", vectorColName), zap.Error(err))
}
// Add new vector column if it doesn't exist (handles embedding model change)
if !colExists {
common.Info("Adding new vector column for embedding model change", zap.String("column", vectorColName), zap.Int("size", vecSize))
addColSchema := infinity.TableSchema{
&infinity.ColumnDefinition{
Name: vectorColName,
DataType: fmt.Sprintf("vector,%d,float", vecSize),
},
}
if _, err := table.AddColumns(addColSchema); err != nil {
common.Error("Failed to add vector column "+vectorColName, err)
return fmt.Errorf("failed to add vector column %s: %w", vectorColName, err)
}
common.Info("Successfully added vector column", zap.String("column", vectorColName))
}
// Reconcile the parser-specific column on the existing-table path: the
// table may have been created ahead of its first table-parser document.
if parserID == "table" {
chunkDataExists, checkErr := e.columnExists(table, "chunk_data")
if checkErr != nil {
return fmt.Errorf("failed to check chunk_data column: %w", checkErr)
}
if err := e.ensureChunkDataColumn(table, chunkDataExists); err != nil {
return err
}
}
} else {
// Table doesn't exist, create it with vector column in the initial schema
common.Info(fmt.Sprintf("Creating table with vector column: %s with dimension %d", vectorColName, vecSize))
// Build column definitions (preserving JSON order)
var columns infinity.TableSchema
for _, fieldName := range schema.Keys {
fieldInfo := schema.Fields[fieldName]
col := infinity.ColumnDefinition{
Name: fieldName,
DataType: fieldInfo.Type,
Default: fieldInfo.Default,
}
columns = append(columns, &col)
}
// Add vector column
columns = append(columns, &infinity.ColumnDefinition{
Name: vectorColName,
DataType: fmt.Sprintf("vector,%d,float", vecSize),
})
// Add chunk_data column for table parser
if parserID == "table" {
columns = append(columns, &infinity.ColumnDefinition{
Name: "chunk_data",
DataType: "json",
Default: "{}",
})
}
// Create table
table, err = db.CreateTable(tableName, columns, infinity.ConflictTypeIgnore)
if err != nil {
return fmt.Errorf("failed to create table: %w", err)
}
common.Debug("Infinity created table", zap.String("tableName", tableName))
}
// Create HNSW index on vector column with unique name based on vector size
// Use unique index name to avoid conflict when embedding model changes
vectorIndexName := fmt.Sprintf("q_%d_vec_idx", vecSize)
_, err = table.CreateIndex(
vectorIndexName,
infinity.NewIndexInfo(vectorColName, infinity.IndexTypeHnsw, map[string]string{
"M": "16",
"ef_construction": "50",
"metric": "cosine",
"encode": "lvq",
}),
infinity.ConflictTypeIgnore,
"",
)
if err != nil {
return fmt.Errorf("failed to create HNSW index %s: %w", vectorIndexName, err)
}
common.Info("Created vector index", zap.String("indexName", vectorIndexName), zap.String("column", vectorColName))
// Create full-text indexes for varchar fields with analyzers. CreateTable
// above is ConflictTypeIgnore, so this may be an existing table: its indexes
// are kept as they are, and with them the analyzer it was created with.
for _, fieldName := range schema.Keys {
fieldInfo := schema.Fields[fieldName]
if fieldInfo.Type != "varchar" || fieldInfo.Analyzer == nil {
continue
}
analyzers := []string{}
switch a := fieldInfo.Analyzer.(type) {
case string:
analyzers = []string{a}
case []interface{}:
for _, v := range a {
if s, ok := v.(string); ok {
analyzers = append(analyzers, s)
}
}
}
for _, analyzer := range analyzers {
indexNameFt := fulltextIndexName(fieldName, analyzer)
_, err = table.CreateIndex(
indexNameFt,
infinity.NewIndexInfo(fieldName, infinity.IndexTypeFullText, map[string]string{"ANALYZER": analyzerForLanguage(analyzer, language)}),
infinity.ConflictTypeIgnore,
"",
)
if err != nil {
return fmt.Errorf("failed to create fulltext index %s: %w", indexNameFt, err)
}
}
}
// Create secondary indexes for fields with index_type
for _, fieldName := range schema.Keys {
fieldInfo := schema.Fields[fieldName]
if fieldInfo.IndexType == nil {
continue
}
indexTypeStr := ""
params := map[string]string{}
switch it := fieldInfo.IndexType.(type) {
case string:
indexTypeStr = it
case map[string]interface{}:
if t, ok := it["type"].(string); ok {
indexTypeStr = t
}
if card, ok := it["cardinality"].(string); ok {
params["cardinality"] = card
}
}
if indexTypeStr == "secondary" {
indexNameSec := fmt.Sprintf("sec_%s", fieldName)
_, err = table.CreateIndex(
indexNameSec,
infinity.NewIndexInfo(fieldName, infinity.IndexTypeSecondary, params),
infinity.ConflictTypeIgnore,
"",
)
if err != nil {
return fmt.Errorf("failed to create secondary index %s: %w", indexNameSec, err)
}
}
}
return nil
}
// parserIDFromChunks reports the parser that produced a batch: a chunk carrying
// chunk_data comes from the table parser (Python's create_idx(parser_id)).
func parserIDFromChunks(chunks []map[string]interface{}) string {
for _, chunk := range chunks {
if chunkData, ok := chunk["chunk_data"].(map[string]interface{}); ok && chunkData != nil {
return "table"
}
}
return ""
}
// ensureChunkDataColumn adds the column the table parser writes when a table
// created ahead of its first table-parser document lacks it. exists is the
// caller's already-known column existence.
func (e *Engine) ensureChunkDataColumn(table *infinity.Table, exists bool) error {
if exists {
return nil
}
if _, err := table.AddColumns(infinity.TableSchema{
&infinity.ColumnDefinition{Name: "chunk_data", DataType: "json", Default: "{}"},
}); err != nil {
return fmt.Errorf("failed to add chunk_data column: %w", err)
}
return nil
}
// InsertChunks inserts documents into a dataset table;
// Table name format: {baseName}_{datasetID}
// Auto-create the table if it doesn't exist
// Delete existing rows with matching IDs before insert
func (e *Engine) InsertChunks(ctx context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {
tableName := buildChunkTableName(baseName, datasetID)
common.Info("InfinityConnection.InsertChunks called", zap.String("tableName", tableName), zap.Int("chunkCount", len(chunks)))
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return nil, fmt.Errorf("failed to get database: %w", err)
}
defer release()
table, err := db.GetTable(tableName)
if err != nil {
// Table doesn't exist, try to create it
errMsg := strings.ToLower(err.Error())
if !strings.Contains(errMsg, "not found") && !strings.Contains(errMsg, "doesn't exist") {
return nil, fmt.Errorf("failed to get table %s: %w", tableName, err)
}
// Infer vector size from chunks
vectorSize := 0
vectorPattern := regexp.MustCompile(`q_(\d+)_vec`)
for _, chunk := range chunks {
for key := range chunk {
matches := vectorPattern.FindStringSubmatch(key)
if len(matches) >= 2 {
vectorSize, _ = strconv.Atoi(matches[1])
break
}
}
if vectorSize > 0 {
break
}
}
if vectorSize == 0 {
return nil, fmt.Errorf("cannot infer vector size from chunks")
}
// Create table. This is the path the ingestion pipeline actually takes,
// so the dataset language has to reach it here.
if err := e.createChunkStoreWithDB(db, baseName, datasetID, vectorSize, parserIDFromChunks(chunks), language); err != nil {
return nil, fmt.Errorf("failed to create table: %w", err)
}
table, err = db.GetTable(tableName)
if err != nil {
return nil, fmt.Errorf("failed to get table after creation: %w", err)
}
}
// Get embedding columns and their sizes
var embeddingCols [][2]interface{}
colsResp, err := table.ShowColumns()
if err != nil {
return nil, fmt.Errorf("failed to get columns: %w", err)
}
result, ok := colsResp.(*infinity.QueryResult)
if !ok {
return nil, fmt.Errorf("unexpected response type: %T", colsResp)
}
// ShowColumns returns a result set where Data contains arrays of column values
re := regexp.MustCompile(`Embedding\([a-z]+,(\d+)\)`)
hasChunkData := false
columns := make(map[string]bool)
if nameArr, ok := result.Data["name"]; ok {
if typeArr, ok := result.Data["type"]; ok {
for i := 0; i < len(nameArr); i++ {
colName, _ := nameArr[i].(string)
colType, _ := typeArr[i].(string)
columns[colName] = true
if colName == "chunk_data" {
hasChunkData = true
}
matches := re.FindStringSubmatch(colType)
if len(matches) >= 2 {
size, _ := strconv.Atoi(matches[1])
embeddingCols = append(embeddingCols, [2]interface{}{colName, size})
}
}
}
}
// Reconcile the parser column here too: an existing table is opened through
// GetTable and never reaches createChunkStoreWithDB. Best effort, like Python's
// ensure_columns — a column that stays missing surfaces on the insert below.
if parserIDFromChunks(chunks) == "table" {
if err := e.ensureChunkDataColumn(table, hasChunkData); err != nil {
common.Warn("Failed to add chunk_data column", zap.Error(err))
}
}
// Transform chunks using helper function
insertChunks := make([]map[string]interface{}, len(chunks))
memoryIndex := isMemoryIndexName(baseName)
for i, chunk := range chunks {
if memoryIndex {
// Memory rows keep the message mapping's columns; kb_id and
// doc_id are not message columns.
insertChunks[i] = shapeMemoryMessageRow(chunk, embeddingCols)
continue
}
insertChunks[i] = transformChunkFields(chunk, embeddingCols)
// kb_id is owned by the engine at the write boundary (mirrors ES
// chunk.go InsertChunks). The ingestion producer no longer stamps it,
// so the producer value (if any) is intentionally overridden here.
insertChunks[i]["kb_id"] = datasetID
}
if err := e.dropUndeclaredFields(tableName, insertChunks, columns); err != nil {
return nil, err
}
// Chunk ids are content-addressed, so one batch can repeat an id. Elasticsearch
// keeps a single row per _id; Infinity has no id uniqueness, so collapse to one row
// per id, last wins. Repeats across batches need nothing: the delete below covers them.
insertChunks = dedupeByChunkID(insertChunks)
// Delete existing rows with matching IDs
if len(insertChunks) > 0 {
idList := make([]string, len(insertChunks))
for i, chunk := range insertChunks {
// is a UUID produced by the document ingestion path
// (uuid.NewString), not user input. We single-quote it
// for Infinity SQL; UUIDs cannot contain single quotes
// by construction (RFC 4122 §3).
// codeql[go/unsafe-quoting] False positive: chunk["id"]
idList[i] = fmt.Sprintf("'%v'", chunk["id"])
}
filter := fmt.Sprintf("id IN (%s)", strings.Join(idList, ", "))
common.Debug(fmt.Sprintf("Deleting existing rows with filter: %s", filter))
delResp, delErr := table.Delete(filter)
if delErr != nil {
common.Warn(fmt.Sprintf("Failed to delete existing rows: %v", delErr))
} else {
common.Info(fmt.Sprintf("Deleted %d existing rows", delResp.DeletedRows))
}
}
// Insert chunks to dataset
for start := 0; start < len(insertChunks); start += infinityMaxInsertRows {
end := min(start+infinityMaxInsertRows, len(insertChunks))
batch := insertChunks[start:end]
if _, err = table.Insert(batch); err != nil {
common.Warn("Infinity insert failed",
zap.String("tableName", tableName),
zap.Int("rows", len(batch)),
zap.Strings("rowFields", rowFieldNames(batch)),
zap.Error(err))
if isConnectionLevelError(err) {
e.client.dropConnectionFor(db, "InsertChunks")
}
return nil, fmt.Errorf("failed to insert chunks to dataset: %w", err)
}
}
common.Info("InfinityConnection.InsertChunks result", zap.String("tableName", tableName), zap.Int("count", len(insertChunks)))
return []string{}, nil
}
// vectorColumnPattern matches the q_<dim>_vec columns the engine adds per
// embedding size; no mapping file declares them.
var vectorColumnPattern = regexp.MustCompile(`^q_\d+_vec$`)
// dedupeByChunkID keeps the last row of each chunk id, mirroring Elasticsearch, where
// a repeated _id overwrites the earlier document. Rows without an id are all kept.
func dedupeByChunkID(rows []map[string]interface{}) []map[string]interface{} {
if len(rows) < 2 {
return rows
}
lastAt := make(map[string]int, len(rows))
dupes := 0
for i, row := range rows {
id, ok := row["id"].(string)
if !ok || id == "" {
continue
}
if _, seen := lastAt[id]; seen {
dupes++
}
lastAt[id] = i
}
if dupes == 0 {
return rows
}
out := make([]map[string]interface{}, 0, len(rows)-dupes)
for i, row := range rows {
if id, ok := row["id"].(string); ok && id != "" && lastAt[id] != i {
continue
}
out = append(out, row)
}
return out
}
// dropUndeclaredFields removes each row key that is not a column of the table
// and that no mapping file declares, such as parser bookkeeping keys
// (slide_number, __outline__). Infinity fails the whole batch on one unknown
// column. A declared key that the table lacks is kept, so a stale table still
// fails the insert instead of losing that value.
func (e *Engine) dropUndeclaredFields(tableName string, rows []map[string]interface{}, columns map[string]bool) error {
var declared map[string]bool
dropped := make(map[string]bool)
for _, row := range rows {
for key := range row {
if columns[key] || key == "chunk_data" || vectorColumnPattern.MatchString(key) {
continue
}
if declared == nil {
var err error
if declared, err = declaredChunkColumns(e.mappingFileName); err != nil {
return err
}
}
if !declared[key] {
delete(row, key)
dropped[key] = true
}
}
}
if len(dropped) > 0 {
common.Warn("Dropping fields that are not Infinity columns",
zap.String("tableName", tableName),
zap.Strings("fields", slices.Sorted(maps.Keys(dropped))))
}
return nil
}
// declaredChunkColumns returns the columns of the chunk mapping and of the
// memory message mapping, because memory messages go through InsertChunks too.
func declaredChunkColumns(mappingFileName string) (map[string]bool, error) {
declared := make(map[string]bool)
for _, name := range []string{mappingFileName, "message_infinity_mapping.json"} {
path, err := utility.FindConfFileInProject(name)
if err != nil {
return nil, err
}
data, err := os.ReadFile(*path)
if err != nil {
return nil, fmt.Errorf("failed to read mapping file %s: %w", *path, err)
}
var schema map[string]json.RawMessage
if err := json.Unmarshal(data, &schema); err != nil {
return nil, fmt.Errorf("failed to parse mapping file %s: %w", *path, err)
}
for column := range schema {
declared[column] = true
}
}
return declared, nil
}
// UpdateChunks updates chunks in a dataset
// Table name format: {baseName}_{datasetID}
func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interface{}, newValue map[string]interface{}, baseName string, datasetID string) error {
tableName := buildChunkTableName(baseName, datasetID)
common.Info("InfinityConnection.UpdateChunks called", zap.String("tableName", tableName), zap.Any("condition", condition))
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return fmt.Errorf("failed to get database: %w", err)
}
defer release()
table, err := db.GetTable(tableName)
if err != nil {
// Tolerate missing table (mirrors Python's docStoreConn which
// silently returns False on a non-existent table).
errMsg := strings.ToLower(err.Error())
if strings.Contains(errMsg, "not found") || strings.Contains(errMsg, "doesn't exist") {
return nil
}
return fmt.Errorf("failed to get table %s: %w", tableName, err)
}
// Get table columns
clmns := make(map[string]struct {
Type string
Default interface{}
})
colsResp, err := table.ShowColumns()
if err != nil {
return fmt.Errorf("failed to get columns: %w", err)
}
result, ok := colsResp.(*infinity.QueryResult)
if ok {
if nameArr, ok := result.Data["name"]; ok {
if typeArr, ok := result.Data["type"]; ok {
if defArr, ok := result.Data["default"]; ok {
for i := 0; i < len(nameArr); i++ {
colName, _ := nameArr[i].(string)
colType, _ := typeArr[i].(string)
var colDefault interface{}
if i < len(defArr) {
colDefault = defArr[i]
}
clmns[colName] = struct {
Type string
Default interface{}
}{colType, colDefault}
}
}
}
}
}
// Build filter string from condition
filter := buildFilterFromCondition(condition, clmns)
if len(condition) > 0 && (filter == "" || filter == "1=1") {
// Every condition key was dropped (blank value, empty list, unknown
// column). table.Update("1=1", ...) would rewrite every row of the
// dataset, so refuse instead — mirrors DeleteChunks/DeleteMetadata.
return fmt.Errorf("INFINITY update aborted: non-empty condition yielded unconstrained filter on table %s", tableName)
}
// Process remove operation first
removeValue := make(map[string]interface{})
if removeData, ok := newValue["remove"].(map[string]interface{}); ok {
removeValue = removeData
}
delete(newValue, "remove")
// Transform new_value fields using helper function (no embeddings needed for update)
transformed := transformChunkFields(newValue, nil)
for k, v := range transformed {
newValue[k] = v
}
// Project the update the way an insert is projected, so the logical fields
// reach the message mapping's columns (status -> status_int). Without it the
// status API failed with 3013 "Column status doesn't exist".
if isMemoryIndexName(baseName) {
mapMemoryMessageStorageFields(newValue)
}
// Remove original fields that were transformed (they're now in transformed with new names/types)
// Also remove intermediate token fields that shouldn't be stored in Infinity
// This must match Python's delete list in infinity_conn.py
for _, key := range []string{"docnm_kwd", "title_tks", "title_sm_tks", "important_kwd", "important_tks",
"content_with_weight", "content_ltks", "content_sm_ltks", "authors_tks", "authors_sm_tks",
"question_kwd", "question_tks"} {
delete(newValue, key)
}
// Handle remove operations if any
if len(removeValue) > 0 {
colToRemove := make([]string, 0, len(removeValue))
for k := range removeValue {
colToRemove = append(colToRemove, k)
}
colToRemove = append(colToRemove, "id")
// Query rows to be updated
queryResult, err := table.Output(colToRemove).Filter(filter).ToResult()
if err != nil {
common.Warn(fmt.Sprintf("Failed to query rows for remove operation: %v", err))
} else {
qr, ok := queryResult.(*infinity.QueryResult)
if ok && len(qr.Data) > 0 {
// Get the id column and columns to remove
idCol := qr.Data["id"]
removeOpt := make(map[string]map[string][]string) // column -> value -> [ids]
for colName, colData := range qr.Data {
if colName == "id" {
continue
}
removeVal := removeValue[colName]
for i, id := range idCol {
if i < len(colData) {
existingVal := colData[i]
if removeStr, ok := removeVal.(string); ok {
// Split existing value by ### and remove the target value
if existingStr, ok := existingVal.(string); ok {
parts := strings.Split(existingStr, "###")
var newParts []string
for _, p := range parts {
if p != removeStr {
newParts = append(newParts, p)
}
}
if len(newParts) != len(parts) {
idStr := fmt.Sprintf("'%s'", escapeFilterValue(fmt.Sprintf("%v", id)))
if removeOpt[colName] == nil {
removeOpt[colName] = make(map[string][]string)
}
removeOpt[colName][strings.Join(newParts, "###")] = append(removeOpt[colName][strings.Join(newParts, "###")], idStr)
}
}
}
}
}
}
// Execute remove updates
for colName, valueToIDs := range removeOpt {
for newVal, ids := range valueToIDs {
idFilter := filter + " AND id IN (" + strings.Join(ids, ", ") + ")"
common.Info(fmt.Sprintf("INFINITY remove update: table=%s, idFilter=%s, column=%s, newValue=%v", tableName, idFilter, colName, newVal))
_, err := table.Update(idFilter, map[string]interface{}{colName: newVal})
if err != nil {
common.Warn(fmt.Sprintf("Failed to remove value from column %s: %v", colName, err))
}
}
}
}
}
}
// Execute the main update
common.Info(fmt.Sprintf("INFINITY update: table=%s, filter=%s, newValue=%v", tableName, filter, newValue))
_, err = table.Update(filter, newValue)
if err != nil {
common.Warn("Infinity update failed",
zap.String("tableName", tableName),
zap.String("filter", filter),
zap.Error(err))
if isConnectionLevelError(err) {
e.client.dropConnectionFor(db, "UpdateChunks")
}
return fmt.Errorf("failed to update chunks: %w", err)
}
common.Info("InfinityConnection.UpdateChunks completes", zap.String("tableName", tableName))
return nil
}
// AdjustChunkPagerank adjusts pagerank_fea and clamps it to [minWeight, maxWeight].
func (e *Engine) AdjustChunkPagerank(ctx context.Context, baseName, chunkID, datasetID string, delta, minWeight, maxWeight float64) error {
if baseName == "" {
return fmt.Errorf("index name cannot be empty")
}
if chunkID == "" {
return fmt.Errorf("chunk id cannot be empty")
}
if e.client == nil || e.client.pool == nil {
return fmt.Errorf("infinity client not initialized")
}
tableName := buildChunkTableName(baseName, datasetID)
lock := pagerankAdjustLock(tableName + ":" + chunkID)
lock.Lock()
defer lock.Unlock()
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return fmt.Errorf("failed to get database: %w", err)
}
defer release()
table, err := db.GetTable(tableName)
if err != nil {
return fmt.Errorf("failed to get table %s: %w", tableName, err)
}
var lastErr error
filter := fmt.Sprintf("id = '%s'", escapeFilterValue(chunkID))
for attempt := 0; attempt <= pagerankAdjustRetryCount; attempt++ {
if err := ctx.Err(); err != nil {
return err
}
result, err := table.Output([]string{common.PAGERANK_FLD, "row_id()"}).Filter(filter).ToResult()
if err != nil {
lastErr = err
continue
}
qr, ok := result.(*infinity.QueryResult)
if !ok {
return fmt.Errorf("unexpected query result type: %T", result)
}
rowID, ok := firstQueryInt64(qr.Data, "ROW_ID", "row_id()", "row_id")
if !ok {
return fmt.Errorf("%w: %s", types.ErrDocumentNotFound, chunkID)
}
currentWeight := 0.0
if currentValue, ok := firstQueryValue(qr.Data, common.PAGERANK_FLD); ok {
if weight, ok := coerceToFloat(currentValue); ok {
currentWeight = weight
}
}
nextWeight := currentWeight + delta
if nextWeight < minWeight {
nextWeight = minWeight
}
if nextWeight > maxWeight {
nextWeight = maxWeight
}
if floatsEqual(currentWeight, nextWeight) {
return nil
}
updateFilter := fmt.Sprintf("_row_id = %d AND %s = %s", rowID, common.PAGERANK_FLD, formatFilterFloat(currentWeight))
if _, err := table.Update(updateFilter, map[string]interface{}{common.PAGERANK_FLD: nextWeight}); err != nil {
lastErr = err
continue
}
verifyResult, err := table.Output([]string{common.PAGERANK_FLD}).Filter(filter).ToResult()
if err != nil {
lastErr = err
continue
}
verifyQR, ok := verifyResult.(*infinity.QueryResult)
if !ok {
return fmt.Errorf("unexpected query result type: %T", verifyResult)
}
if currentValue, ok := firstQueryValue(verifyQR.Data, common.PAGERANK_FLD); ok {
if actualWeight, ok := coerceToFloat(currentValue); ok && floatsEqual(actualWeight, nextWeight) {
return nil
}
}
lastErr = fmt.Errorf("pagerank update conflict")
}
return fmt.Errorf("failed to adjust chunk pagerank: %w", lastErr)
}
// DeleteChunks deletes chunks from a dataset table
// Table name format: {baseName}_{datasetID}
// condition specifies which chunks to delete
func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interface{}, baseName string, datasetID string) (int64, error) {
tableName := buildChunkTableName(baseName, datasetID)
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return 0, fmt.Errorf("failed to get database: %w", err)
}
defer release()
table, err := db.GetTable(tableName)
if err != nil {
common.Warn(fmt.Sprintf("Table %s does not exist, skipping delete", tableName))
return 0, nil
}
// Get table columns for building filter
clmns := make(map[string]struct {
Type string
Default interface{}
})
colsResp, err := table.ShowColumns()
if err != nil {
return 0, fmt.Errorf("failed to get columns: %w", err)
}
result, ok := colsResp.(*infinity.QueryResult)
if ok {
if nameArr, ok := result.Data["name"]; ok {
if typeArr, ok := result.Data["type"]; ok {
if defArr, ok := result.Data["default"]; ok {
for i := 0; i < len(nameArr); i++ {
colName, _ := nameArr[i].(string)
colType, _ := typeArr[i].(string)
var colDefault interface{}
if i < len(defArr) {
colDefault = defArr[i]
}
clmns[colName] = struct {
Type string
Default interface{}
}{colType, colDefault}
}
}
}
}
}
// Build filter from condition
filter := buildFilterFromCondition(condition, clmns)
if len(condition) > 0 && (filter == "" || filter == "1=1") {
return 0, fmt.Errorf("INFINITY delete aborted: non-empty condition yielded unconstrained filter on table %s", tableName)
}
delResp, err := table.Delete(filter)
if err != nil {
common.Warn("Infinity delete failed",
zap.String("tableName", tableName),
zap.String("filter", filter),
zap.Error(err))
if isConnectionLevelError(err) {
e.client.dropConnectionFor(db, "DeleteChunks")
}
return 0, fmt.Errorf("failed to delete: %w", err)
}
return delResp.DeletedRows, nil
}
// Search searches the Infinity engine for matching chunks.
// It supports three matching types: MatchTextExpr (full-text), MatchDenseExpr (vector), and FusionExpr (combined).
// If no match expressions are provided, Search relies solely on filter (e.g., doc_id, available_int) to find results.
func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
types.LogSearchRequest("Infinity", req)
if len(req.IndexNames) == 0 {
return nil, fmt.Errorf("index names cannot be empty")
}
// Get retrieval parameters with defaults
pageSize := req.Limit
if pageSize <= 0 {
pageSize = 30
}
offset := max(req.Offset, 0)
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return nil, fmt.Errorf("failed to get database: %w", err)
}
defer release()
isSkillIndex := false
for _, idx := range req.IndexNames {
if strings.HasPrefix(idx, "skill_") {
isSkillIndex = true
break
}
}
isMemoryIndex := false
for _, idx := range req.IndexNames {
if isMemoryIndexName(idx) {
isMemoryIndex = true
break
}
}
var outputColumns []string
if isSkillIndex {
outputColumns = []string{
"skill_id", "space_id", "folder_id", "name", "tags", "description", "content",
"version", "status", "create_time", "update_time",
}
outputColumns = convertSelectFields(outputColumns, true, false)
} else {
outputColumns = []string{
"id", "doc_id", "kb_id", "content_ltks", "content_with_weight",
"title_tks", "docnm_kwd", "img_id", "available_int", "important_kwd",
"position_int", "page_num_int", "top_int", "chunk_order_int",
"create_timestamp_flt", "knowledge_graph_kwd", "question_kwd", "question_tks",
"doc_type_kwd", "mom_id", "tag_kwd", "pagerank_fea", "tag_feas",
}
outputColumns = convertSelectFields(outputColumns, false, isMemoryIndex)
}
// Allow caller to override output columns (used by KG search, etc.)
if len(req.SelectFields) > 0 {
outputColumns = convertSelectFields(req.SelectFields, false, isMemoryIndex)
}
hasTextMatch := false
hasVectorMatch := false
var matchText *types.MatchTextExpr
var matchDense *types.MatchDenseExpr
if req.MatchExprs != nil && len(req.MatchExprs) > 0 {
for _, expr := range req.MatchExprs {
if expr == nil {
continue
}
switch e := expr.(type) {
case string:
if e != "" {
hasTextMatch = true
matchText = &types.MatchTextExpr{
MatchingText: e,
TopN: pageSize,
}
}
case *types.MatchTextExpr:
if e.MatchingText != "" {
hasTextMatch = true
matchText = e
}
case *types.MatchDenseExpr:
if len(e.EmbeddingData) > 0 {
hasVectorMatch = true
matchDense = e
}
}
}
}
// Infinity binds a "_score" projection to SCORE(), which it allows only with
// MATCH TEXT/TENSOR or Fusion — otherwise 3013. Elasticsearch tolerates
// "_score" for every shape, so callers pass it unconditionally (the nav
// cluster descent and navScan do): drop theirs and re-add the one form this
// shape accepts. calculateScores() materializes "_score" for readers either
// way.
outputColumns = stripScorePseudoColumns(outputColumns)
if hasTextMatch || hasVectorMatch {
if hasTextMatch {
outputColumns = append(outputColumns, "score()")
}
// similarity() is only allowed by Infinity when there is ONLY MATCH VECTOR.
// When both text and vector matches exist (hybrid search with Fusion),
// only score() is valid — Fusion produces a unified SCORE column.
if hasVectorMatch && !hasTextMatch {
outputColumns = append(outputColumns, "similarity()")
}
// Skill and memory indexes have no pagerank_fea / tag_feas columns
if !isSkillIndex && !isMemoryIndex {
if !slices.Contains(outputColumns, common.PAGERANK_FLD) {
outputColumns = append(outputColumns, common.PAGERANK_FLD)
}
if !slices.Contains(outputColumns, common.TAG_FLD) {
outputColumns = append(outputColumns, common.TAG_FLD)
}
}
}
if !slices.Contains(outputColumns, "row_id") && !slices.Contains(outputColumns, "row_id()") {
outputColumns = append(outputColumns, "row_id()")
}
outputColumns = convertSelectFields(outputColumns, isSkillIndex, isMemoryIndex)
if hasVectorMatch && matchDense != nil && matchDense.VectorColumnName != "" {
outputColumns = append(outputColumns, matchDense.VectorColumnName)
}
var filterParts []string
if hasTextMatch || hasVectorMatch {
// Memory tables have no available_int/status; their conditions come in
// through the filter below.
if !isMemoryIndex {
if req.Filter != nil {
if availInt, ok := req.Filter["available_int"]; ok {
filterParts = append(filterParts, fmt.Sprintf("available_int=%v", availInt))
} else if status, ok := req.Filter["status"]; ok {
filterParts = append(filterParts, fmt.Sprintf("status='%s'", status))
} else if shouldDefaultAvailableFilter(req, isSkillIndex) {
filterParts = append(filterParts, "available_int=1")
}
} else if shouldDefaultAvailableFilter(req, isSkillIndex) {
filterParts = append(filterParts, "available_int=1")
}
}
}
// Build the backend-independent portion of the filter. JSON-list predicates
// are completed per table below because legacy tables stored those columns
// as varchar while current tables use JSON.
var filterCopy map[string]interface{}
if req.Filter != nil {
filterCopy = make(map[string]interface{})
for k, v := range req.Filter {
if k != "kb_id" {
filterCopy[k] = v
}
}
}
orderBy := req.OrderBy
mergeMemoryOrder := isMemoryIndex && !hasTextMatch && !hasVectorMatch && orderBy != nil && len(orderBy.Fields) > 0
var rankFeature map[string]float64
if req.RankFeature != nil {
rankFeature = req.RankFeature
}
var fusionExpr *types.FusionExpr
if len(req.MatchExprs) > 2 {
if fe, ok := req.MatchExprs[2].(*types.FusionExpr); ok {
fusionExpr = fe
}
}
var allResults []map[string]interface{}
totalHits := int64(0)
// queryErr keeps the first per-table failure. A failed table query used to be
// swallowed (warn + continue), which turned schema/query mismatches into
// silently empty results — a partial-wildcard projection ("q_*_vec", 3013)
// made the knowledge-compile reader see no products, so the dataset
// navigation tree silently never materialized. Failures stay tolerable while
// another table answered, but surface when nothing did.
var queryErr error
for _, indexName := range req.IndexNames {
var tableNames []string
if strings.HasPrefix(indexName, "ragflow_doc_meta_") {
tableNames = []string{indexName}
} else {
kbIDs := req.KbIDs
if len(kbIDs) == 0 {
kbIDs = []string{""}
}
for _, kbID := range kbIDs {
if kbID == "" {
tableNames = append(tableNames, indexName)
} else {
tableNames = append(tableNames, fmt.Sprintf("%s_%s", indexName, kbID))
}
}
}
// minMatch comes from matchText.ExtraOptions when set (Python parity).
// Mirrors rag/utils/infinity_conn.py which reads
// matchExpr.extra_options.get("minimum_should_match", 0.0) — for the
// English (non-Chinese) path, the Go Question() builder omits
// minimum_should_match, so the default is 0.0 to match Python's
// effective 0% threshold for English queries.
minMatch := 0.0
var questionText string
var vectorData []float64
textTopN := pageSize
var originalQuery string
if matchText != nil {
questionText = matchText.MatchingText
textTopN = int(matchText.TopN)
if matchText.ExtraOptions != nil {
if oq, ok := matchText.ExtraOptions["original_query"].(string); ok {
originalQuery = oq
}
if v, ok := matchText.ExtraOptions["minimum_should_match"]; ok {
switch x := v.(type) {
case float64:
minMatch = x
case int:
minMatch = float64(x)
case string:
s := strings.TrimSuffix(x, "%")
if pct, err := strconv.Atoi(s); err == nil {
minMatch = float64(pct) / 100
}
}
}
}
}
if matchDense != nil {
vectorData = matchDense.EmbeddingData
}
for _, tableName := range tableNames {
tbl, err := db.GetTable(tableName)
if err != nil {
// A dataset whose table does not exist yet is normal (tables are
// created on first write): skip it. A transport failure leaves the
// socket unusable, so it is recorded like the failures below.
if isConnectionLevelError(err) {
e.client.dropConnectionFor(db, "Search")
if queryErr == nil {
queryErr = fmt.Errorf("open table %s: %w", tableName, err)
}
}
continue
}
// Resolve partial-wildcard projections ("q_*_vec") against this table's
// real columns: Infinity parses "q_*_vec" as the arithmetic expression
// "q_" * "_vec" and fails to bind it (3013), while Elasticsearch expands
// the pattern in _source. Embedding dims differ per table, so this is
// decided per table. A bare "*" is left alone: Infinity supports it.
wildcardSelect := hasPartialWildcard(outputColumns)
var tableColumns map[string]struct {
Type string
Default interface{}
}
if wildcardSelect || (len(filterCopy) > 0 && hasJSONListFilter(filterCopy)) {
tableColumns, err = loadTableColumns(tbl)
if err != nil {
common.Warn("Failed to load Infinity columns for search",
zap.String("tableName", tableName), zap.Error(err))
if queryErr == nil {
queryErr = fmt.Errorf("load columns for %s: %w", tableName, err)
}
continue
}
}
tableOutputColumns := outputColumns
if mergeMemoryOrder {
tableOutputColumns = unionSortColumns(tableOutputColumns, orderBy)
}
if wildcardSelect {
tableOutputColumns = expandSelectWildcards(tableOutputColumns, tableColumns)
}
tableFilterParts := append([]string(nil), filterParts...)
if len(filterCopy) > 0 {
if condition := equivalentConditionToStr(filterCopy, tableColumns, isMemoryIndex); condition != "" {
tableFilterParts = append(tableFilterParts, condition)
}
}
filterStr := strings.Join(tableFilterParts, " AND ")
table := tbl.Output(tableOutputColumns)
var textFields []string
if matchText != nil && len(matchText.Fields) > 0 {
textFields = matchText.Fields
} else if isSkillIndex {
textFields = []string{
"name^10",
"tags^5",
"description^3",
"content^1",
}
} else {
textFields = []string{
"title_tks^10",
"title_sm_tks^5",
"important_kwd^30",
"important_tks^20",
"question_tks^20",
"content_ltks^2",
"content_sm_ltks",
}
}
// Convert field names for Infinity
var convertedFields []string
for _, f := range textFields {
cf := convertMatchingField(f)
convertedFields = append(convertedFields, cf)
}
fields := strings.Join(convertedFields, ",")
hasTextMatch := questionText != ""
hasVectorMatch := len(vectorData) > 0
// Add text match if question is provided
if hasTextMatch {
extraOptions := map[string]string{
"minimum_should_match": common.FormatMinimumShouldMatchPercent(minMatch),
}
if filterStr != "" {
extraOptions["filter"] = filterStr
}
if rankFeature != nil {
var rankFeaturesList []string
for featureName, weight := range rankFeature {
rankFeaturesList = append(rankFeaturesList, fmt.Sprintf("%s^%s^%.0f", common.TAG_FLD, featureName, weight))
}
if len(rankFeaturesList) > 0 {
extraOptions["rank_features"] = strings.Join(rankFeaturesList, ",")
}
}
if originalQuery != "" {
extraOptions["original_query"] = originalQuery
}
table = table.MatchText(fields, questionText, textTopN, extraOptions)
common.Debug(fmt.Sprintf(
"MatchTextExpr:\n"+
" fields=%s\n"+
" matching_text=%s\n"+
" topn=%d\n"+
" extra_options=%v",
fields, questionText, textTopN, extraOptions,
))
}
// Add vector match if provided
if hasVectorMatch {
vecFieldName := fmt.Sprintf("q_%d_vec", len(vectorData))
dataType := "float"
distanceType := "cosine"
if matchDense != nil {
if matchDense.VectorColumnName != "" {
vecFieldName = matchDense.VectorColumnName
}
if matchDense.EmbeddingDataType != "" {
dataType = matchDense.EmbeddingDataType
}
if matchDense.DistanceType != "" {
distanceType = matchDense.DistanceType
}
}
vectorTopN := pageSize
if matchDense != nil && matchDense.TopN > 0 {
vectorTopN = int(matchDense.TopN)
}
denseFilterStr := filterStr
if denseFilterStr == "" && !isMemoryIndex && shouldDefaultAvailableFilter(req, isSkillIndex) {
denseFilterStr = "available_int=1"
}
if hasTextMatch {
fieldsStr := strings.Join(convertedFields, ",")
// Escape single quotes in user-controlled questionText
// before splicing into the filter_fulltext() call.
// fieldsStr is sourced from a fixed allowlist (see
// textFields above) and is not user-controlled.
safeQuery := strings.ReplaceAll(questionText, "'", "''")
safeFields := strings.ReplaceAll(fieldsStr, "'", "''")
filterFulltext := fmt.Sprintf("filter_fulltext('%s', '%s')", safeFields, safeQuery)
if denseFilterStr != "" {
denseFilterStr = fmt.Sprintf("(%s) AND %s", denseFilterStr, filterFulltext)
} else {
denseFilterStr = filterFulltext
}
}
threshold := "0.0"
if matchDense != nil && matchDense.ExtraOptions != nil {
if sim, ok := matchDense.ExtraOptions["similarity"].(float64); ok {
threshold = fmt.Sprintf("%g", sim)
} else if s, ok := matchDense.ExtraOptions["threshold"].(string); ok {
threshold = s
}
}
extraOptions := map[string]string{
"threshold": threshold,
"filter": denseFilterStr,
}
common.Debug("MatchDense for hybrid search",
zap.String("fieldName", vecFieldName),
zap.String("distanceType", distanceType),
zap.Int("topN", vectorTopN),
zap.Bool("hasFusion", fusionExpr != nil))
table = table.MatchDense(vecFieldName, vectorData, dataType, distanceType, vectorTopN, extraOptions)
}
// Add fusion (for text + vector combination)
if hasTextMatch && hasVectorMatch && fusionExpr != nil {
fusionMethod := fusionExpr.Method
fusionTopK := fusionExpr.TopN
if fusionTopK == 0 {
fusionTopK = pageSize
}
fusionParams := map[string]interface{}{
"normalize": "atan",
}
if fusionExpr.FusionParams != nil {
for k, v := range fusionExpr.FusionParams {
fusionParams[k] = v
}
}
common.Debug("Applying Fusion for hybrid search",
zap.String("method", fusionMethod),
zap.Int("topN", fusionTopK),
zap.Any("params", fusionParams))
table = table.Fusion(fusionMethod, fusionTopK, fusionParams)
}
// Add order_by if provided
if orderBy != nil && len(orderBy.Fields) > 0 {
var sortFields [][2]interface{}
for _, orderField := range orderBy.Fields {
sortType := infinity.SortTypeAsc
if orderField.Type == types.SortDesc {
sortType = infinity.SortTypeDesc
}
sortFields = append(sortFields, [2]interface{}{orderField.Field, sortType})
}
table = table.Sort(sortFields)
}
// Add filter when there's no text/vector match (like metadata queries)
if !hasTextMatch && !hasVectorMatch && filterStr != "" {
common.Debug(fmt.Sprintf("Adding filter for no-match query: %s", filterStr))
table = table.Filter(filterStr)
}
// Set limit and offset
if mergeMemoryOrder {
table = table.Limit(pageSize + offset)
} else {
table = table.Limit(pageSize)
if offset > 0 {
table = table.Offset(offset)
}
}
// Request total_hits_count from Infinity
table = table.Option(map[string]interface{}{"total_hits_count": true})
// Execute query
df, err := table.ToDataFrame()
if err != nil {
// Log the full request, not just its shape: a failing search used
// to be indistinguishable from any other, which made a recurring
// 7018/EOF impossible to attribute to a caller.
common.Warn("Infinity query failed",
zap.String("tableName", tableName),
zap.String("filter", filterStr),
zap.Strings("selectFields", tableOutputColumns),
zap.Int("limit", pageSize),
zap.Int("offset", offset),
zap.Bool("hasTextMatch", hasTextMatch),
zap.Bool("hasVectorMatch", hasVectorMatch),
zap.Bool("hasFusion", fusionExpr != nil),
zap.Error(err))
if isConnectionLevelError(err) {
e.client.dropConnectionFor(db, "Search")
}
if queryErr == nil {
queryErr = fmt.Errorf("query table %s: %w", tableName, err)
}
continue
}
// Convert DataFrame to chunks format (column-oriented to row-oriented)
searchChunks := chunksFromColumnData(df.ColumnData)
// Apply field name mapping and row_id handling
// Skill index uses different schema
// so we skip the document-specific field mappings
if isMemoryIndex {
// Memory rows come back under the message mapping's storage
// column names; rename them to the logical fields the memory
// service reads (mirrors the Elasticsearch engine).
normalizeMemoryMessageRows(searchChunks)
} else if !isSkillIndex {
applyFieldMappings(searchChunks)
} else {
// For skill index, only handle ROW_ID -> row_id() mapping
for _, chunk := range searchChunks {
if val, ok := chunk["ROW_ID"]; ok {
chunk["row_id()"] = val
delete(chunk, "ROW_ID")
}
}
}
// Parse total_hits_count from ExtraInfo
var tableTotal int64
if df.ExtraInfo != "" {
var extraResult map[string]interface{}
if err := json.Unmarshal([]byte(df.ExtraInfo), &extraResult); err == nil {
if count, ok := extraResult["total_hits_count"].(float64); ok {
tableTotal = int64(count)
}
}
}
// Log every query shape at debug level: which table, which filter, whether
// a match was applied, and the row/total counts. A query that silently
// spans the dataset is only visible here.
common.Debug("Infinity search",
zap.String("table", tableName),
zap.String("filter", filterStr),
zap.Bool("hasTextMatch", hasTextMatch),
zap.Bool("hasVectorMatch", hasVectorMatch),
zap.Int("rows", len(searchChunks)),
zap.Int64("total", tableTotal))
searchResult := &types.SearchResult{
Chunks: searchChunks,
Total: tableTotal,
}
allResults = append(allResults, searchResult.Chunks...)
totalHits += searchResult.Total
}
}
if hasTextMatch || hasVectorMatch {
scoreColumn := ""
if hasTextMatch && hasVectorMatch {
scoreColumn = "SCORE"
} else if hasTextMatch {
scoreColumn = "SCORE"
} else if hasVectorMatch {
scoreColumn = "SIMILARITY"
}
pagerankField := common.PAGERANK_FLD
if isSkillIndex || isMemoryIndex {
pagerankField = "" // Skill and memory indexes have no pagerank field
}
allResults = calculateScores(allResults, scoreColumn, pagerankField)
allResults = sortByScore(allResults, len(allResults))
}
if mergeMemoryOrder {
sortChunksByFields(allResults, orderBy)
allResults = allResults[min(offset, len(allResults)):]
}
if len(allResults) > pageSize {
allResults = allResults[:pageSize]
}
// Every table failed: return the error instead of an empty result set, so a
// backend-specific query mismatch cannot masquerade as "no data".
if len(allResults) == 0 && totalHits == 0 && queryErr != nil {
return nil, fmt.Errorf("infinity search failed on all tables: %w", queryErr)
}
common.Debug("Search in Infinity completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits))
return &types.SearchResult{
Chunks: allResults,
Total: totalHits,
}, nil
}
func shouldDefaultAvailableFilter(req *types.SearchRequest, isSkillIndex bool) bool {
return !isSkillIndex && !req.IncludeUnavailable
}
const (
// regexpSearchDefaultLimit caps a regexp search, matching the Elasticsearch
// engine so grep_chunks returns the same page on either backend.
regexpSearchDefaultLimit = 30
// regexpSearchReturnAllLimit is the ceiling for list_chunks' deep read.
regexpSearchReturnAllLimit = 10000
)
// SearchByRegexp is a regex-match-only search over chunk content for the agent's
// grep_chunks/list_chunks tools, mirroring the Elasticsearch engine. The pattern goes
// down as regex(content, ...), which the content column's sparse gram index narrows by
// the pattern's mandatory literals; Infinity evaluates it with RE2, so the pattern is
// used verbatim (no ".*(...).*" wrapper) and "(?i)" carries case folding.
func (e *Engine) SearchByRegexp(ctx context.Context, req *types.RegexpSearchRequest) (*types.SearchResult, error) {
if req == nil {
return nil, fmt.Errorf("regexp search request cannot be nil")
}
if strings.TrimSpace(req.Pattern) == "" {
return nil, fmt.Errorf("regexp pattern cannot be empty")
}
tenantID := strings.TrimSpace(req.TenantID)
if tenantID == "" {
return nil, fmt.Errorf("tenant id cannot be empty")
}
// One index per tenant; a list would fan out to unrelated tables.
if strings.Contains(tenantID, ",") {
return nil, fmt.Errorf("regexp search supports exactly one tenant")
}
// Chunk tables are per dataset: an unscoped request would read the bare
// ragflow_<tenant>, which holds no chunks, and report an empty result.
scoped := false
for _, kbID := range req.KbIDs {
if strings.TrimSpace(kbID) != "" {
scoped = true
break
}
}
if !scoped {
return nil, fmt.Errorf("regexp search requires at least one dataset id")
}
maxResults := regexpSearchDefaultLimit
if req.ReturnAll {
maxResults = regexpSearchReturnAllLimit
}
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return nil, fmt.Errorf("failed to get database: %w", err)
}
defer release()
// Chunk tables are per dataset: ragflow_<tenant>_<datasetID>.
baseName := fmt.Sprintf("ragflow_%s", tenantID)
tableNames := make([]string, 0, len(req.KbIDs))
for _, kbID := range req.KbIDs {
if kbID == "" {
tableNames = append(tableNames, baseName)
} else {
tableNames = append(tableNames, fmt.Sprintf("%s_%s", baseName, kbID))
}
}
outputColumns := []string{
"id", "doc_id", "kb_id", "content_ltks", "content_with_weight",
"title_tks", "docnm_kwd", "img_id", "available_int", "important_kwd",
"position_int", "page_num_int", "top_int", "chunk_order_int",
"create_timestamp_flt", "knowledge_graph_kwd", "question_kwd", "question_tks",
"doc_type_kwd", "mom_id", "tag_kwd", "pagerank_fea", "tag_feas",
}
if len(req.SelectFields) > 0 {
outputColumns = append([]string(nil), req.SelectFields...)
}
// The merge at the end restores the order Go-side, so the sort keys must be
// projected even when the caller did not ask for them.
outputColumns = unionSortColumns(outputColumns, req.Sort)
// No MATCH is applied, so Infinity accepts no score column to project.
outputColumns = stripScorePseudoColumns(outputColumns)
outputColumns = convertSelectFields(outputColumns, false, false)
// Same filter for every dataset table; kb_id is implied by the table name.
filterStr := buildRegexpFilter(req.Filter, req.Pattern)
var allResults []map[string]interface{}
totalHits := int64(0)
// A missing table is normal (created on first write); other failures surface
// only if no table answered.
var queryErr error
for _, tableName := range tableNames {
tbl, err := db.GetTable(tableName)
if err != nil {
// A dataset table that does not exist yet is normal (tables are created on
// first write); any other open failure is real and must reach the caller.
errMsg := strings.ToLower(err.Error())
tableMissing := strings.Contains(errMsg, "not found") || strings.Contains(errMsg, "doesn't exist")
if isConnectionLevelError(err) {
e.client.dropConnectionFor(db, "SearchByRegexp")
}
if !tableMissing && queryErr == nil {
queryErr = fmt.Errorf("open table %s: %w", tableName, err)
}
continue
}
table := tbl.Output(outputColumns).Filter(filterStr)
if req.Sort != nil && len(req.Sort.Fields) > 0 {
var sortFields [][2]interface{}
for _, orderField := range req.Sort.Fields {
sortType := infinity.SortTypeAsc
if orderField.Type == types.SortDesc {
sortType = infinity.SortTypeDesc
}
sortFields = append(sortFields, [2]interface{}{orderField.Field, sortType})
}
table = table.Sort(sortFields)
}
table = table.Limit(maxResults)
table = table.Option(map[string]interface{}{"total_hits_count": true})
df, err := table.ToDataFrame()
if err != nil {
common.Warn("Infinity regexp query failed",
zap.String("tableName", tableName),
zap.String("filter", filterStr),
zap.Int("limit", maxResults),
zap.Error(err))
if isConnectionLevelError(err) {
e.client.dropConnectionFor(db, "SearchByRegexp")
}
if queryErr == nil {
queryErr = fmt.Errorf("query table %s: %w", tableName, err)
}
continue
}
chunks := chunksFromColumnData(df.ColumnData)
applyFieldMappings(chunks)
var tableTotal int64
if df.ExtraInfo != "" {
var extraResult map[string]interface{}
if err := json.Unmarshal([]byte(df.ExtraInfo), &extraResult); err == nil {
if count, ok := extraResult["total_hits_count"].(float64); ok {
tableTotal = int64(count)
}
}
}
common.Debug("Infinity regexp search",
zap.String("table", tableName),
zap.String("filter", filterStr),
zap.Int("rows", len(chunks)),
zap.Int64("total", tableTotal))
allResults = append(allResults, chunks...)
totalHits += tableTotal
}
// limit() applies per table, so merge the tables' pages into the requested order
// and cap once, as the Elasticsearch engine does after its single-index query.
// The cap must follow the sort, and Total stays the pre-cap count.
sortChunksByFields(allResults, req.Sort)
if len(allResults) > maxResults {
allResults = allResults[:maxResults]
}
// All tables failed: surface the error rather than an empty result.
if len(allResults) == 0 && totalHits == 0 && queryErr != nil {
return nil, fmt.Errorf("infinity regexp search failed on all tables: %w", queryErr)
}
return &types.SearchResult{
Chunks: allResults,
Total: totalHits,
}, nil
}
// buildRegexpFilter renders the regexp search filter: the caller's scope (minus
// kb_id, already implied by the per-dataset table name) AND regex(content, ...)
// with the case-insensitive "(?i)" the grep contract requires.
func buildRegexpFilter(filter map[string]interface{}, pattern string) string {
filterCopy := map[string]interface{}{}
for k, v := range filter {
if k == "kb_id" {
continue
}
filterCopy[k] = v
}
parts := make([]string, 0, 2)
if len(filterCopy) > 0 {
if condition := equivalentConditionToStr(filterCopy, nil); condition != "" {
parts = append(parts, condition)
}
}
parts = append(parts, infinity.RegexFilter("content", "(?i)"+pattern))
return strings.Join(parts, " AND ")
}
// unionSortColumns appends the requested sort keys to a projection, preserving the
// caller's column order: the engine orders the merged rows itself.
func unionSortColumns(columns []string, sort *types.OrderByExpr) []string {
if sort == nil || len(sort.Fields) == 0 {
return columns
}
seen := make(map[string]struct{}, len(columns)+len(sort.Fields))
for _, column := range columns {
seen[column] = struct{}{}
}
for _, field := range sort.Fields {
if field.Field == "" {
continue
}
if _, ok := seen[field.Field]; ok {
continue
}
seen[field.Field] = struct{}{}
columns = append(columns, field.Field)
}
return columns
}
// sortChunksByFields orders merged rows by the requested fields, ascending unless a
// field asks for descending, mirroring the Elasticsearch engine's sortByFields: a
// missing field sorts first, numbers numerically, strings lexicographically.
// page_num_int arrives hex-decoded as an int slice, compared on its leading element.
func sortChunksByFields(chunks []map[string]interface{}, expr *types.OrderByExpr) {
if expr == nil || len(expr.Fields) == 0 {
return
}
sort.SliceStable(chunks, func(i, j int) bool {
for _, field := range expr.Fields {
vi, oki := chunks[i][field.Field]
vj, okj := chunks[j][field.Field]
if !oki || !okj {
if !oki && !okj {
continue
}
// Missing field sorts before a present one.
return !oki
}
less, equal := compareOrderValues(vi, vj)
if equal {
continue
}
if field.Type == types.SortDesc {
return !less
}
return less
}
return false
})
}
// compareOrderValues compares two raw field values: numbers numerically, strings
// lexicographically, anything else by text. An unset key (the empty column default, as
// a page number that was never set) ranks before a set one in either argument order, so
// the pair cannot order differently by comparison direction; equality is reported
// separately, so only a real tie falls through to the next sort field.
func compareOrderValues(a, b interface{}) (less, equal bool) {
fa, okA := numericOrderValue(a)
fb, okB := numericOrderValue(b)
if okA != okB {
return !okA, false
}
if okA {
return fa < fb, fa == fb
}
sa, okSA := a.(string)
sb, okSB := b.(string)
if okSA && okSB {
return sa < sb, sa == sb
}
ta, tb := fmt.Sprintf("%v", a), fmt.Sprintf("%v", b)
return ta < tb, ta == tb
}
// numericOrderValue reads a scalar order key, unwrapping the int slice
// applyFieldMappings leaves behind for hex-encoded columns.
func numericOrderValue(v interface{}) (float64, bool) {
switch n := v.(type) {
case int:
return float64(n), true
case int64:
return float64(n), true
case float64:
return n, true
case float32:
return float64(n), true
case []int:
if len(n) > 0 {
return float64(n[0]), true
}
case []interface{}:
if len(n) > 0 {
return numericOrderValue(n[0])
}
}
return 0, false
}
// GetChunk gets a chunk by ID
func (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datasetIDs []string) (interface{}, error) {
if e.client == nil || e.client.pool == nil {
return nil, fmt.Errorf("infinity client not initialized")
}
common.Info("Infinity get chunk start",
zap.String("chunkID", chunkID),
zap.String("tableName", tableName),
zap.Strings("datasetIDs", datasetIDs))
// Build list of table names to search
tableNames := make([]string, 0, len(datasetIDs))
for _, datasetID := range datasetIDs {
tableNames = append(tableNames, fmt.Sprintf("%s_%s", tableName, datasetID))
}
// Try each table and collect results from all tables
db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")
if err != nil {
return nil, fmt.Errorf("failed to get database: %w", err)
}
defer release()
// Collect chunks from all tables (same as Python's concat_dataframes)
allChunks := make(map[string]map[string]interface{})
for _, tblName := range tableNames {
table, err := db.GetTable(tblName)
if err != nil {
if isMemoryIndexName(tableName) {
var infinityErr *infinity.InfinityException
if !errors.As(err, &infinityErr) || infinity.ErrorCode(infinityErr.ErrorCode) != infinity.ErrorCodeTableNotExist {
return nil, fmt.Errorf("failed to open message table: %w", err)
}
}
continue
}
// Query with filter for the specific chunk ID
filter := fmt.Sprintf("id = '%s'", chunkID)
result, err := table.Output([]string{"*"}).Filter(filter).ToResult()
if err != nil {
if isMemoryIndexName(tableName) {
return nil, fmt.Errorf("failed to read message: %w", err)
}
continue
}
qr, ok := result.(*infinity.QueryResult)
if !ok {
if isMemoryIndexName(tableName) {
return nil, fmt.Errorf("unexpected message query result %T", result)
}
continue
}
if len(qr.Data) == 0 {
continue
}
// Convert to chunk format
chunks := make([]map[string]interface{}, 0)
for colName, colData := range qr.Data {
for i, val := range colData {
for len(chunks) <= i {
chunks = append(chunks, make(map[string]interface{}))
}
chunks[i][colName] = val
}
}
// Merge chunks into allChunks (by id), keeping first non-empty value
for _, chunk := range chunks {
if idVal, ok := chunk["id"].(string); ok {
if existing, exists := allChunks[idVal]; exists {
// Merge: keep first non-empty value for each field
for k, v := range chunk {
if _, has := existing[k]; !has || (utility.IsEmpty(existing[k]) && !utility.IsEmpty(v)) {
existing[k] = v
}
}
} else {
allChunks[idVal] = chunk
}
}
}
}
// Get the chunk by chunkID
chunk, found := allChunks[chunkID]
if !found {
return nil, nil
}
common.Debug("infinity get chunk", zap.String("chunkID", chunkID), zap.Any("tables", tableNames))
decodeJSONFields(chunk)
decodeVarcharJSONFields(chunk)
// Memory rows come back under the message mapping's storage columns, which
// the chunk field mappings below would mangle.
if isMemoryIndexName(tableName) {
normalizeMemoryMessageRows([]map[string]interface{}{chunk})
return chunk, nil
}
// Apply field mappings (same as in GetFields)
// docnm -> docnm_kwd, title_tks, title_sm_tks
if val, ok := chunk["docnm"].(string); ok {
chunk["docnm_kwd"] = val
chunk["title_tks"] = val
chunk["title_sm_tks"] = val
}
// content -> content_with_weight, content_ltks, content_sm_ltks
if val, ok := chunk["content"].(string); ok {
chunk["content_with_weight"] = val
chunk["content_ltks"] = val
chunk["content_sm_ltks"] = val
}
// important_keywords -> important_kwd (split by comma), important_tks
if val, ok := chunk["important_keywords"].(string); ok {
if val == "" {
chunk["important_kwd"] = []interface{}{}
} else {
parts := strings.Split(val, ",")
chunk["important_kwd"] = parts
}
chunk["important_tks"] = val
} else {
chunk["important_kwd"] = []interface{}{}
chunk["important_tks"] = []interface{}{}
}
// questions -> question_kwd (split by newline), question_tks
if val, ok := chunk["questions"].(string); ok {
if val == "" {
chunk["question_kwd"] = []interface{}{}
} else {
parts := strings.Split(val, "\n")
chunk["question_kwd"] = parts
}
chunk["question_tks"] = val
} else {
chunk["question_kwd"] = []interface{}{}
chunk["question_tks"] = []interface{}{}
}
if posVal, ok := chunk["position_int"].(string); ok {
chunk["position_int"] = utility.ConvertHexToPositionIntArray(posVal)
} else {
chunk["position_int"] = []interface{}{}
}
if val, ok := chunk["tag_kwd"].(string); ok {
chunk["tag_kwd"], _ = splitKeywordList("tag_kwd", val)
}
return chunk, nil
}
// splitKeywordList reads a "###"-joined keyword column back as a list.
// transformChunkFields always joins tag_kwd, so it reads back as a list even
// with one tag or none; any other keyword column is split only when it holds
// several values, since most of them are scalar.
func splitKeywordList(field, value string) ([]interface{}, bool) {
if field != "tag_kwd" && !strings.Contains(value, "###") {
return nil, false
}
list := []interface{}{}
for part := range strings.SplitSeq(value, "###") {
if part != "" {
list = append(list, part)
}
}
return list, true
}
// chunksFromColumnData converts an Infinity column-oriented DataFrame into the
// row-oriented chunk maps the engine returns. A null cell must not become a row
// of its own (Infinity returns one all-null cell for a projection that matched
// nothing, and a phantom row leaks to callers); a Json/Array column arrives
// parked on row 0, so realignJSONColumnCells spreads it before applyFieldMappings.
func chunksFromColumnData(columnData map[string][]interface{}) []map[string]interface{} {
rows := make([]map[string]interface{}, 0)
for colName, colData := range columnData {
for i, val := range colData {
if isEmptyEngineCell(val) {
continue
}
for len(rows) <= i {
rows = append(rows, make(map[string]interface{}))
}
rows[i][colName] = val
}
}
realignJSONColumnCells(rows, columnData)
return rows
}
// applyFieldMappings applies field mappings to chunks (side-effect only).
// Used by Search() to mutate chunks with derived fields before returning.
func applyFieldMappings(chunks []map[string]interface{}) {
for _, chunk := range chunks {
decodeJSONFields(chunk)
// docnm -> docnm_kwd, title_tks, title_sm_tks
if val, ok := chunk["docnm"].(string); ok {
chunk["docnm_kwd"] = val
chunk["title_tks"] = val
chunk["title_sm_tks"] = val
}
// important_keywords -> important_kwd (split by comma/semicolon/Chinese punctuations), important_tks
if val, ok := chunk["important_keywords"].(string); ok {
if val == "" {
chunk["important_kwd"] = []interface{}{}
} else {
parts := ChinesePunctRegex.Split(val, -1)
chunk["important_kwd"] = parts
}
chunk["important_tks"] = val
} else {
chunk["important_kwd"] = []interface{}{}
chunk["important_tks"] = []interface{}{}
}
// questions -> question_kwd (split by newline), question_tks
if val, ok := chunk["questions"].(string); ok {
if val == "" {
chunk["question_kwd"] = []interface{}{}
} else {
parts := strings.Split(val, "\n")
chunk["question_kwd"] = parts
}
chunk["question_tks"] = val
} else {
chunk["question_kwd"] = []interface{}{}
chunk["question_tks"] = []interface{}{}
}
// content -> content_with_weight, content_ltks, content_sm_ltks
if val, ok := chunk["content"].(string); ok {
chunk["content_with_weight"] = val
chunk["content_ltks"] = val
chunk["content_sm_ltks"] = val
}
// authors -> authors_tks, authors_sm_tks
if val, ok := chunk["authors"].(string); ok {
chunk["authors_tks"] = val
chunk["authors_sm_tks"] = val
}
if val, ok := chunk["message_type_kwd"]; ok {
chunk["message_type"] = val
}
if val, ok := chunk["status_int"]; ok {
chunk["status"] = memoryMessageStatusBool(val)
}
// position_int: convert from hex string to array format (grouped by 5)
if val, ok := chunk["position_int"].(string); ok {
chunk["position_int"] = utility.ConvertHexToPositionIntArray(val)
}
// Convert page_num_int and top_int from hex string to array
for _, colName := range []string{"page_num_int", "top_int"} {
if val, ok := chunk[colName].(string); ok && val != "" {
chunk[colName] = utility.ConvertHexToIntArray(val)
}
}
// Post-process: convert nil/empty values to empty slices for array-like fields
// and split _kwd fields by "###" (except knowledge_graph_kwd, docnm_kwd, important_kwd, question_kwd)
kwdNoSplit := map[string]bool{
"knowledge_graph_kwd": true, "docnm_kwd": true,
"important_kwd": true, "question_kwd": true,
}
arrayFields := []string{
"important_kwd", "important_tks", "question_tks",
"question_kwd", "authors_tks", "authors_sm_tks", "title_tks",
"title_sm_tks", "content_ltks", "content_sm_ltks", "tag_kwd",
}
for _, colName := range arrayFields {
val, ok := chunk[colName]
if !ok || val == nil || val == "" {
chunk[colName] = []interface{}{}
} else if !kwdNoSplit[colName] {
if strVal, ok := val.(string); ok {
if list, split := splitKeywordList(colName, strVal); split {
chunk[colName] = list
}
}
}
}
// Handle row_id mapping - Infinity returns "ROW_ID" but we use "row_id()"
if val, ok := chunk["ROW_ID"]; ok {
chunk["row_id()"] = val
delete(chunk, "ROW_ID")
}
}
}
func memoryMessageStatusBool(value interface{}) bool {
switch v := value.(type) {
case bool:
return v
// int32 is what the thrift path returns for the mapping's integer columns;
// without it a stored status_int=1 fell through to the default and every
// message rendered as disabled.
case int:
return v != 0
case int8:
return v != 0
case int16:
return v != 0
case int32:
return v != 0
case int64:
return v != 0
case uint:
return v != 0
case uint8:
return v != 0
case uint16:
return v != 0
case uint32:
return v != 0
case uint64:
return v != 0
case float32:
return v != 0
case float64:
return v != 0
case json.Number:
n, err := v.Int64()
return err == nil && n != 0
case string:
return v != "" && v != "0" && !strings.EqualFold(v, "false")
default:
return false
}
}
// memoryMessageStatusInt renders a logical status as the mapping's status_int.
func memoryMessageStatusInt(value interface{}) int {
if memoryMessageStatusBool(value) {
return 1
}
return 0
}
// mapMemoryMessageStorageFields renames the logical message fields onto the
// message mapping's storage columns.
func mapMemoryMessageStorageFields(row map[string]interface{}) {
if val, ok := row["message_type"]; ok {
row["message_type_kwd"] = val
delete(row, "message_type")
}
if val, ok := row["status"]; ok {
row["status_int"] = memoryMessageStatusInt(val)
delete(row, "status")
}
}
// normalizeMemoryMessageRows renames the message mapping's storage columns back
// to the logical fields the memory service reads.
func normalizeMemoryMessageRows(rows []map[string]interface{}) {
for _, row := range rows {
for field, value := range row {
if vectorColumnPattern.MatchString(field) {
row["content_embed"] = value
delete(row, field)
}
}
if val, ok := row["message_type_kwd"]; ok {
row["message_type"] = val
delete(row, "message_type_kwd")
}
if val, ok := row["status_int"]; ok {
row["status"] = memoryMessageStatusBool(val)
delete(row, "status_int")
}
}
}
// shapeMemoryMessageRow prepares one raw memory message for insert. The rename
// onto the message mapping's storage columns (message_type -> message_type_kwd,
// status -> status_int) is transformChunkFields'; this drops the chunk-only
// doc_id/kb_id keys and the unset (nil) fields, so the column DEFAULT applies.
// A NULL cell makes the comparison NULL, which drops the row from
// `must_not exists forget_at` and reads back as a truthy "null" (showing the
// message as disabled). Infinity 0.7.4 has no IS NULL / isnull() / coalesce()
// to rescue it (3069/3030).
func shapeMemoryMessageRow(chunk map[string]interface{}, embeddingCols [][2]interface{}) map[string]interface{} {
row := transformChunkFields(chunk, embeddingCols)
delete(row, "doc_id")
delete(row, "kb_id")
for key, value := range row {
if value == nil {
delete(row, key)
}
}
return row
}
// GetFields extracts the requested fields from Infinity search results
func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) map[string]map[string]interface{} {
result := make(map[string]map[string]interface{})
// Python: if not fields, return {}
if len(fields) == 0 {
return result
}
if len(chunks) == 0 {
return result
}
// Build field set for lookup (Python lines 713-715)
fieldsAll := make(map[string]bool)
for _, f := range fields {
fieldsAll[f] = true
}
fieldsAll["id"] = true
// noneColumns is rebuilt per chunk inside the loop below. The
// per-chunk "missing → nil" map MUST be fresh for every iteration; if
// it's reused, the first chunk that contains a field removes it from
// the shared set, and later chunks missing that same field silently
// stop getting the nil placeholder, producing inconsistent shapes
// per document.
// Check if important_kwd is needed (for empty_count handling)
needImportantKwdEmptyCount := fieldsAll["important_kwd"]
for _, chunk := range chunks {
decodeJSONFields(chunk)
// Build column map for case-insensitive lookup (Python line 747)
columnMap := make(map[string]string)
for k := range chunk {
columnMap[strings.ToLower(k)] = k
}
// Apply field mappings first (to get derived fields)
// docnm -> docnm_kwd, title_tks, title_sm_tks (Python lines 716-719)
// Note: Python checks "docnm" in res.columns regardless of whether fields were requested
if val, ok := chunk["docnm"].(string); ok {
if fieldsAll["docnm_kwd"] {
chunk["docnm_kwd"] = val
}
if fieldsAll["title_tks"] {
chunk["title_tks"] = val
}
if fieldsAll["title_sm_tks"] {
chunk["title_sm_tks"] = val
}
}
// important_keywords -> important_kwd (split by comma), important_tks (Python lines 720-732)
// Python: v.split(",") if v else [] — empty string yields empty list
if fieldsAll["important_kwd"] || fieldsAll["important_tks"] {
if val, ok := chunk["important_keywords"].(string); ok && val != "" {
if fieldsAll["important_kwd"] {
if needImportantKwdEmptyCount {
// Check for important_kwd_empty_count (Python lines 722-728)
if emptyCountVal, hasEmptyCount := chunk["important_kwd_empty_count"]; hasEmptyCount {
tokens := strings.Split(val, ",")
var emptyCount int
switch v := emptyCountVal.(type) {
case float64:
emptyCount = int(v)
case int:
emptyCount = v
case string:
emptyCount, _ = strconv.Atoi(v)
}
kwdList := make([]interface{}, 0, len(tokens)+emptyCount)
for _, t := range tokens {
kwdList = append(kwdList, t)
}
for i := 0; i < emptyCount; i++ {
kwdList = append(kwdList, "")
}
chunk["important_kwd"] = kwdList
} else {
parts := strings.Split(val, ",")
kwdList := make([]interface{}, len(parts))
for i, p := range parts {
kwdList[i] = p
}
chunk["important_kwd"] = kwdList
}
} else {
parts := strings.Split(val, ",")
kwdList := make([]interface{}, len(parts))
for i, p := range parts {
kwdList[i] = p
}
chunk["important_kwd"] = kwdList
}
}
if fieldsAll["important_tks"] {
chunk["important_tks"] = val
}
} else {
if fieldsAll["important_kwd"] {
chunk["important_kwd"] = []interface{}{}
}
if fieldsAll["important_tks"] {
chunk["important_tks"] = []interface{}{}
}
}
}
// questions -> question_kwd (split by newline), question_tks (Python lines 733-737)
// Python: v.splitlines() — empty string yields empty list
if fieldsAll["question_kwd"] || fieldsAll["question_tks"] {
if val, ok := chunk["questions"].(string); ok && val != "" {
if fieldsAll["question_kwd"] {
parts := strings.Split(val, "\n")
qList := make([]interface{}, len(parts))
for i, p := range parts {
qList[i] = p
}
chunk["question_kwd"] = qList
}
if fieldsAll["question_tks"] {
chunk["question_tks"] = val
}
} else {
if fieldsAll["question_kwd"] {
chunk["question_kwd"] = []interface{}{}
}
if fieldsAll["question_tks"] {
chunk["question_tks"] = []interface{}{}
}
}
}
// content -> content_with_weight, content_ltks, content_sm_ltks (Python lines 738-741)
if fieldsAll["content_with_weight"] || fieldsAll["content_ltks"] || fieldsAll["content_sm_ltks"] {
if val, ok := chunk["content"].(string); ok {
if fieldsAll["content_with_weight"] {
chunk["content_with_weight"] = val
}
if fieldsAll["content_ltks"] {
chunk["content_ltks"] = val
}
if fieldsAll["content_sm_ltks"] {
chunk["content_sm_ltks"] = val
}
}
}
// authors -> authors_tks, authors_sm_tks (Python lines 742-745)
if fieldsAll["authors_tks"] || fieldsAll["authors_sm_tks"] {
if val, ok := chunk["authors"].(string); ok {
if fieldsAll["authors_tks"] {
chunk["authors_tks"] = val
}
if fieldsAll["authors_sm_tks"] {
chunk["authors_sm_tks"] = val
}
}
}
// Post-process fields matching Python lines 758-780
// This single loop processes all column transformations in Python order
kwdNoSplit := map[string]bool{
"knowledge_graph_kwd": true, "docnm_kwd": true,
"important_kwd": true, "question_kwd": true,
}
for field, val := range chunk {
fieldLower := strings.ToLower(field)
// field_keyword: split by "###" (Python lines 760-761)
needsSplit := false
if fieldLower == "source_id" {
needsSplit = true
} else if strings.HasSuffix(fieldLower, "_kwd") && !kwdNoSplit[fieldLower] {
needsSplit = true
}
if needsSplit {
if strVal, ok := val.(string); ok {
if list, split := splitKeywordList(fieldLower, strVal); split {
chunk[field] = list
}
}
continue
}
// _feas: JSON parse (Python lines 762-763)
if strings.HasSuffix(fieldLower, "_feas") {
if strVal, ok := val.(string); ok && strVal != "" {
var parsed interface{}
if err := json.Unmarshal([]byte(strVal), &parsed); err == nil {
chunk[field] = parsed
}
} else {
chunk[field] = map[string]interface{}{}
}
continue
}
// chunk_data: JSON parse (Python lines 764-766)
if fieldLower == "chunk_data" {
if strVal, ok := val.(string); ok && strVal != "" {
var parsed interface{}
if err := json.Unmarshal([]byte(strVal), &parsed); err == nil {
chunk[field] = parsed
}
} else if val == nil {
// Keep nil
}
continue
}
// position_int: hex decode with grouping by 5 (Python lines 767-776)
if fieldLower == "position_int" && fieldsAll[fieldLower] {
// If already converted to slice by applyFieldMappings, skip
if _, isSlice := val.([]interface{}); isSlice {
continue
}
// applyFieldMappings returns [][]int, check that too
if _, isIntSlice := val.([][]int); isIntSlice {
continue
}
if strVal, ok := val.(string); ok && strVal != "" {
chunk[field] = utility.ConvertHexToPositionIntArray(strVal)
} else {
chunk[field] = []interface{}{}
}
continue
}
// page_num_int, top_int: hex decode (Python lines 777-778)
if (fieldLower == "page_num_int" || fieldLower == "top_int") && fieldsAll[fieldLower] {
// If already converted to slice by applyFieldMappings, skip
if _, isSlice := val.([]interface{}); isSlice {
continue
}
// applyFieldMappings returns []int, check that too
if _, isIntSlice := val.([]int); isIntSlice {
continue
}
if strVal, ok := val.(string); ok && strVal != "" {
chunk[field] = utility.ConvertHexToIntArray(strVal)
} else {
chunk[field] = []interface{}{}
}
continue
}
}
// Handle row_id mapping (Python lines 748-750)
if fieldsAll["row_id()"] {
if lowerKey, ok := columnMap["row_id"]; ok {
chunk["row_id()"] = chunk[lowerKey]
}
}
// Delete base columns after mapping (Python lines 781-783)
for _, col := range []string{"docnm", "important_keywords", "questions", "content", "authors"} {
delete(chunk, col)
}
// Build result map keyed by id
if idVal, ok := chunk["id"].(string); ok {
fieldMap := make(map[string]interface{})
// Rebuild noneColumns for this chunk so that fields missing
// from THIS chunk get a nil placeholder. Reusing a set across
// chunks would let the first chunk's contents permanently
// remove keys, leaving later chunks with inconsistent shapes.
noneColumns := make(map[string]bool, len(fieldsAll))
for f := range fieldsAll {
noneColumns[strings.ToLower(f)] = true
}
for field, value := range chunk {
if fieldsAll[field] {
fieldMap[field] = value
delete(noneColumns, strings.ToLower(field))
}
}
// Set none_columns to None (Python lines 784-785)
for col := range noneColumns {
fieldMap[col] = nil
}
result[idVal] = fieldMap
}
}
return result
}
// GetAggregation aggregates chunk values by field name.
// Input: [{"docnm_kwd": "docA"}, {"docnm_kwd": "docA"}, {"docnm_kwd": "docB"}]
//
// GetAggregation(chunks, "docnm_kwd") returns:
//
// [{"key": "docA", "count": 2}, {"key": "docB", "count": 1}]
//
// For tag_kwd field, splits values by "###" separator.
// For other fields, uses comma separation.
func (e *Engine) GetAggregation(chunks []map[string]interface{}, fieldName string) []map[string]interface{} {
if len(chunks) == 0 {
return []map[string]interface{}{}
}
// Check if field exists in first chunk
hasField := false
for _, chunk := range chunks {
if _, ok := chunk[fieldName]; ok {
hasField = true
break
}
}
if !hasField {
return []map[string]interface{}{}
}
// Count occurrences
tagCounts := make(map[string]int)
for _, chunk := range chunks {
value, ok := chunk[fieldName]
if !ok || value == nil {
continue
}
// Handle string value
if valueStr, ok := value.(string); ok {
if valueStr == "" {
continue
}
var tags []string
// Split by "###" for tag_kwd field
if fieldName == "tag_kwd" && strings.Contains(valueStr, "###") {
for tag := range strings.SplitSeq(valueStr, "###") {
tag = strings.TrimSpace(tag)
if tag != "" {
tags = append(tags, tag)
}
}
} else {
// Fallback to comma separation
for tag := range strings.SplitSeq(valueStr, ",") {
tag = strings.TrimSpace(tag)
if tag != "" {
tags = append(tags, tag)
}
}
}
for _, tag := range tags {
tagCounts[tag]++
}
continue
}
// Handle list value
if valueList, ok := value.([]interface{}); ok {
for _, item := range valueList {
if itemStr, ok := item.(string); ok {
tag := strings.TrimSpace(itemStr)
if tag != "" {
tagCounts[tag]++
}
}
}
}
}
if len(tagCounts) == 0 {
return []map[string]interface{}{}
}
// Convert to slice and sort by count descending
type tagCountPair struct {
tag string
count int
}
pairs := make([]tagCountPair, 0, len(tagCounts))
for tag, count := range tagCounts {
pairs = append(pairs, tagCountPair{tag, count})
}
sort.Slice(pairs, func(i, j int) bool {
return pairs[i].count > pairs[j].count
})
// Convert to []map[string]interface{} directly
result := make([]map[string]interface{}, len(pairs))
for i, p := range pairs {
result[i] = map[string]interface{}{"key": p.tag, "count": p.count}
}
return result
}
// GetChunkIDs extracts chunk IDs from Infinity search results.
func (e *Engine) GetChunkIDs(chunks []map[string]interface{}) []string {
ids := make([]string, 0, len(chunks))
for _, chunk := range chunks {
if id, ok := chunk["id"].(string); ok {
ids = append(ids, id)
}
}
return ids
}
// GetHighlight returns highlighted text for search results.
// Matches keywords in text and wraps them with <em> tags.
func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string {
result := make(map[string]string)
if fieldName == "content_with_weight" && !hasInfinityHighlightField(chunks, fieldName) {
fieldName = "content"
}
pattern := compileInfinityHighlightPattern(keywords)
for _, chunk := range chunks {
id, ok := chunk["id"].(string)
if !ok || id == "" {
continue
}
txt, ok := chunk[fieldName].(string)
if !ok {
continue
}
if pattern != nil {
txt = pattern.ReplaceAllStringFunc(txt, func(match string) string {
return "<em>" + match + "</em>"
})
}
result[id] = txt
}
return result
}
func hasInfinityHighlightField(chunks []map[string]interface{}, fieldName string) bool {
for _, chunk := range chunks {
if _, ok := chunk[fieldName]; ok {
return true
}
}
return false
}
func compileInfinityHighlightPattern(keywords []string) *regexp.Regexp {
nonEmpty := make([]string, 0, len(keywords))
for _, keyword := range keywords {
if keyword != "" {
nonEmpty = append(nonEmpty, keyword)
}
}
if len(nonEmpty) == 0 {
return nil
}
sort.SliceStable(nonEmpty, func(i, j int) bool {
return utf8.RuneCountInString(nonEmpty[i]) > utf8.RuneCountInString(nonEmpty[j])
})
parts := make([]string, len(nonEmpty))
for i, keyword := range nonEmpty {
parts[i] = regexp.QuoteMeta(keyword)
if isLatinKeyword(keyword) {
parts[i] += `\p{Latin}*`
}
}
return regexp.MustCompile("(?i)" + strings.Join(parts, "|"))
}
func isLatinKeyword(keyword string) bool {
for _, r := range keyword {
if !unicode.In(r, unicode.Latin) {
return false
}
}
return keyword != ""
}
// KNNScores for Infinity - since Infinity normalizes scores during fusion,
// we just need to return a result structure that GetScores can parse.
// This matches Python's approach where Infinity doesn't use the two-pass KNN.
func (e *Engine) KNNScores(ctx context.Context, chunks []map[string]interface{}, queryVector []float64, topK int) (map[string]interface{}, error) {
if len(chunks) == 0 {
return nil, nil
}
// Build a result structure that GetScores can parse
// For Infinity, scores are already in _score field from the first search
result := make(map[string]interface{})
hitList := make([]interface{}, 0, len(chunks))
for _, chunk := range chunks {
if id, ok := chunk["id"].(string); ok {
hit := map[string]interface{}{
"_id": id,
"_score": chunk["_score"],
}
hitList = append(hitList, hit)
}
}
result["hits"] = map[string]interface{}{
"hits": hitList,
}
return result, nil
}
// GetScores extracts similarity scores from KNN search result.
// For Infinity, it parses the result from KNNScores and extracts _score values.
func (e *Engine) GetScores(knnResult map[string]interface{}) map[string]float64 {
scores := make(map[string]float64)
hits, ok := knnResult["hits"].(map[string]interface{})
if !ok {
return scores
}
hitList, ok := hits["hits"].([]interface{})
if !ok {
return scores
}
for _, h := range hitList {
hit, ok := h.(map[string]interface{})
if !ok {
continue
}
docID, ok := hit["_id"].(string)
if !ok {
continue
}
scoreVal := hit["_score"]
if scoreVal == nil {
scores[docID] = 0.0
continue
}
score, ok := scoreVal.(float64)
if !ok {
scores[docID] = 0.0
continue
}
scores[docID] = score
}
return scores
}
// convertSelectFields converts field names to Infinity format
// isSkillIndex indicates if this is a skill index (uses skill_id instead of id)
// isMemoryIndex maps the logical "status" onto the message mapping's status_int.
//
// Does NOT mutate the input slice — callers (e.g. retrieval.go) reuse the same
// SelectFields list both for Search() and GetFields(); mutating it would
// replace logical names like "content_with_weight" with their Infinity column
// names ("content"), breaking GetFields's field-presence checks.
func convertSelectFields(output []string, isSkillIndex, isMemoryIndex bool) []string {
fieldMapping := map[string]string{
"docnm_kwd": "docnm",
"title_tks": "docnm",
"title_sm_tks": "docnm",
"important_kwd": "important_keywords",
"important_tks": "important_keywords",
"question_kwd": "questions",
"question_tks": "questions",
"content_with_weight": "content",
"content_ltks": "content",
"content_sm_ltks": "content",
"authors_tks": "authors",
"authors_sm_tks": "authors",
"message_type": "message_type_kwd",
}
if isMemoryIndex {
fieldMapping["status"] = "status_int"
}
skillIndex := isSkillIndex
// Copy + map without mutating the caller's slice.
mapped := make([]string, len(output))
needEmptyCount := false
for i, field := range output {
if field == "important_kwd" {
needEmptyCount = true
}
if newField, ok := fieldMapping[field]; ok {
mapped[i] = newField
} else {
mapped[i] = field
}
}
// Remove duplicates
seen := make(map[string]bool)
result := []string{}
for _, f := range mapped {
if f != "" && !seen[f] {
seen[f] = true
result = append(result, f)
}
}
// Add id and empty count if needed
// For skill index, use skill_id instead of id
idField := "id"
if skillIndex {
idField = "skill_id"
}
if !slices.Contains(result, idField) {
result = append([]string{idField}, result...)
}
if needEmptyCount {
result = append(result, "important_kwd_empty_count")
}
return result
}
// rowFieldNames returns the sorted field names of the first row, so a failed
// insert logs which columns were sent without dumping values (a vector column
// would make the log unusable).
func rowFieldNames(rows []map[string]interface{}) []string {
if len(rows) == 0 {
return nil
}
names := make([]string, 0, len(rows[0]))
for k := range rows[0] {
names = append(names, k)
}
sort.Strings(names)
return names
}
// isEmptyEngineCell reports whether a cell Infinity returned carries no value:
// an untyped null, or a nil slice/map/pointer. Infinity can hand back such a cell
// for a projection it found no row for, so it must not turn into a row by itself.
func isEmptyEngineCell(val interface{}) bool {
if val == nil {
return true
}
rv := reflect.ValueOf(val)
switch rv.Kind() {
case reflect.Slice, reflect.Map, reflect.Ptr, reflect.Interface:
return rv.IsNil()
}
return false
}
// scorePseudoColumns are the score columns a caller may put in a SelectFields
// list. Elasticsearch returns "_score" for every query shape, so callers pass it
// unconditionally; Infinity only accepts the SCORE()/SIMILARITY() function form,
// and only together with a compatible MATCH.
var scorePseudoColumns = map[string]bool{
"_score": true,
"SCORE": true,
"score()": true,
"similarity()": true,
"SIMILARITY": true,
}
// stripScorePseudoColumns removes caller-supplied score columns so Search can
// add back the single form Infinity accepts for the query shape it is about to
// run (none for a filter-only query, similarity() for MATCH VECTOR only,
// score() when a MATCH TEXT/TENSOR or Fusion is present).
func stripScorePseudoColumns(fields []string) []string {
out := make([]string, 0, len(fields))
for _, f := range fields {
if scorePseudoColumns[f] {
continue
}
out = append(out, f)
}
return out
}
// hasPartialWildcard reports whether any requested column contains a wildcard
// without being the bare "*" Infinity reads as "all columns".
func hasPartialWildcard(fields []string) bool {
for _, f := range fields {
if f != "*" && strings.Contains(f, "*") {
return true
}
}
return false
}
// expandSelectWildcards replaces partial wildcards ("q_*_vec" — the only
// pattern RAGFlow uses) with the table columns they match. Infinity cannot
// project a partial wildcard: it parses "q_*_vec" as the arithmetic expression
// "q_" * "_vec" and fails to bind it ("Fail to bind the expression: q_", 3013),
// whereas Elasticsearch expands the same pattern in _source. Embedding dims
// differ per dataset table, so the expansion is resolved per table. A pattern
// matching no column is dropped, so one unmatched pattern cannot fail the query.
func expandSelectWildcards(fields []string, columns map[string]struct {
Type string
Default interface{}
}) []string {
names := make([]string, 0, len(columns))
for name := range columns {
names = append(names, name)
}
sort.Strings(names)
out := make([]string, 0, len(fields))
seen := make(map[string]bool, len(fields))
appendField := func(f string) {
if f == "" || seen[f] {
return
}
seen[f] = true
out = append(out, f)
}
for _, f := range fields {
if f == "*" || !strings.Contains(f, "*") {
appendField(f)
continue
}
for _, name := range names {
if wildcardMatchColumn(f, name) {
appendField(name)
}
}
}
return out
}
// wildcardMatchColumn matches a column name against a "*"-wildcard pattern,
// where "*" stands for any run of characters.
func wildcardMatchColumn(pattern, name string) bool {
parts := strings.Split(pattern, "*")
if len(parts) == 1 {
return pattern == name
}
if !strings.HasPrefix(name, parts[0]) {
return false
}
rest := name[len(parts[0]):]
for _, part := range parts[1 : len(parts)-1] {
if part == "" {
continue
}
idx := strings.Index(rest, part)
if idx < 0 {
return false
}
rest = rest[idx+len(part):]
}
return strings.HasSuffix(rest, parts[len(parts)-1])
}
// convertMatchingField converts field names for matching
// For regular document indices: maps _tks/_kwd fields to column@index_name format
// For skill indices: maps raw field names to column@index_name format
// Infinity requires column@index_name when a column has multiple full-text indexes
func convertMatchingField(fieldWeightStr string) string {
// Split on ^ to get field name
parts := strings.Split(fieldWeightStr, "^")
field := parts[0]
// Field name conversion
fieldMapping := map[string]string{
"docnm_kwd": "docnm@ft_docnm_rag_coarse",
"title_tks": "docnm@ft_docnm_rag_coarse",
"title_sm_tks": "docnm@ft_docnm_rag_fine",
"important_kwd": "important_keywords@ft_important_keywords_rag_coarse",
"important_tks": "important_keywords@ft_important_keywords_rag_fine",
"question_kwd": "questions@ft_questions_rag_coarse",
"question_tks": "questions@ft_questions_rag_fine",
"content_with_weight": "content@ft_content_rag_coarse",
"content_ltks": "content@ft_content_rag_coarse",
"content_sm_ltks": "content@ft_content_rag_fine",
"authors_tks": "authors@ft_authors_rag_coarse",
"authors_sm_tks": "authors@ft_authors_rag_fine",
"tag_kwd": "tag_kwd@ft_tag_kwd_whitespace__",
"toc_kwd": "toc_kwd@ft_toc_kwd_whitespace__",
// Skill index fields
"name": "name@ft_name_rag_coarse",
"tags": "tags@ft_tags_rag_coarse",
"description": "description@ft_description_rag_coarse",
"content": "content@ft_content_rag_coarse",
}
if newField, ok := fieldMapping[field]; ok {
parts[0] = newField
}
return strings.Join(parts, "^")
}
// escapeFilterValue escapes single quotes for filter values
func escapeFilterValue(s string) string {
return strings.ReplaceAll(s, "'", "''")
}
func firstQueryValue(data map[string][]interface{}, columns ...string) (interface{}, bool) {
for _, column := range columns {
values, ok := data[column]
if ok && len(values) > 0 {
return values[0], true
}
}
return nil, false
}
func firstQueryInt64(data map[string][]interface{}, columns ...string) (int64, bool) {
value, ok := firstQueryValue(data, columns...)
if !ok {
return 0, false
}
switch v := value.(type) {
case int:
return int64(v), true
case int8:
return int64(v), true
case int16:
return int64(v), true
case int32:
return int64(v), true
case int64:
return v, true
case uint:
if uint64(v) <= ^uint64(0)>>1 {
return int64(v), true
}
case uint8:
return int64(v), true
case uint16:
return int64(v), true
case uint32:
return int64(v), true
case uint64:
if v <= ^uint64(0)>>1 {
return int64(v), true
}
case float32:
return int64(v), true
case float64:
return int64(v), true
case string:
parsed, err := strconv.ParseInt(v, 10, 64)
if err == nil {
return parsed, true
}
}
return 0, false
}
func pagerankAdjustLock(key string) *sync.Mutex {
hash := fnv.New32a()
_, _ = hash.Write([]byte(key))
return &pagerankAdjustLocks[hash.Sum32()%pagerankAdjustLockCount]
}
func formatFilterFloat(value float64) string {
return strconv.FormatFloat(value, 'f', -1, 64)
}
func floatsEqual(a, b float64) bool {
diff := a - b
if diff < 0 {
diff = -diff
}
return diff < 1e-9
}
func buildGraphTypeFilterCondition(value interface{}) string {
var values []string
switch val := value.(type) {
case []string:
values = val
case []interface{}:
for _, item := range val {
values = append(values, fmt.Sprintf("%v", item))
}
case string:
values = []string{val}
default:
values = []string{fmt.Sprintf("%v", value)}
}
conditions := make([]string, 0, len(values)*2)
for _, value := range values {
if strings.TrimSpace(value) == "" {
continue
}
conditions = append(conditions,
keywordFilterCondition("type_kwd", value),
fmt.Sprintf("knowledge_graph_kwd='%s'", escapeFilterValue(value)),
)
}
return joinBalanced(conditions, " OR ")
}
// equivalentConditionToStr renders a condition map as an Infinity filter
// string. memoryIndex marks a memory message index, whose source_id column is
// an integer rather than a keyword.
func equivalentConditionToStr(condition map[string]interface{}, tableColumns map[string]struct {
Type string
Default interface{}
}, memoryIndex ...bool) string {
return equivalentConditionToStrRaw(types.CompilationFilter(condition), tableColumns, memoryIndex...)
}
func equivalentConditionToStrRaw(condition map[string]interface{}, tableColumns map[string]struct {
Type string
Default interface{}
}, memoryIndex ...bool) string {
if len(condition) == 0 {
return ""
}
memIndex := false
if len(memoryIndex) > 0 {
memIndex = memoryIndex[0]
}
var cond []string
for k, v := range condition {
if k == "and" || k == "or" {
var children []string
for _, child := range types.FilterClauses(v) {
if expr := equivalentConditionToStrRaw(child, tableColumns, memIndex); expr != "" {
children = append(children, "("+expr+")")
}
}
operator := " AND "
if k == "or" {
operator = " OR "
}
if len(children) > 0 {
cond = append(cond, joinBalanced(children, operator))
}
continue
}
if k == "_id" || utility.IsEmpty(v) {
continue
}
if memIndex && k == "session_id_contains" {
if text, ok := v.(string); ok {
pattern := strings.NewReplacer("'", `\x27`, `"`, `\x22`).Replace(regexp.QuoteMeta(text))
cond = append(cond, infinity.RegexFilter("session_id", pattern))
}
continue
}
// Handle must_not specially
if k == "must_not" {
if m, ok := v.(map[string]interface{}); ok {
if expr := equivalentConditionToStrRaw(m, tableColumns, memIndex); expr != "" {
cond = append(cond, "NOT ("+expr+")")
}
}
continue
}
// Handle exists specially (without table schema, use string comparison).
// A NULL cell yields NULL here, so writers store the column default
// instead (see shapeMemoryMessageRow).
if k == "exists" {
cond = append(cond, fmt.Sprintf("%v!=''", v))
continue
}
if k == "knowledge_graph_kwd" {
if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" {
cond = append(cond, graphTypeCondition)
}
continue
}
// JSON-list fields use member containment. Legacy varchar columns keep
// the old full-text behavior.
if fieldJSONList(k) && tableColumns != nil {
if jsonConditions := jsonListFilterConditions(k, v, tableColumns); len(jsonConditions) > 0 {
cond = append(cond, joinBalanced(jsonConditions, " OR "))
}
continue
}
// Handle keyword fields (exact match, or full-text for the single-token
// values Infinity can tokenize — see keywordFilterCondition)
keywordField := fieldKeyword(k)
if memIndex && k == "source_id" {
// The mapping's source_id is an integer, not a keyword column, so
// it cannot use the keyword branch's filter_fulltext().
keywordField = false
}
if keywordField {
// For keyword fields, values are always treated as strings. Blank
// entries are dropped: Infinity rejects a filter with an empty query
// ("Trying to match: on fields: <column> failed", 3052) and fails
// the whole statement, so one empty element in an IN-list built from
// optional values (e.g. GetWikiGraph's from_kwd list when an entity
// has no slug) must not poison the condition.
var inCond []string
appendKeywordCond := func(raw string) {
if strings.TrimSpace(raw) == "" {
return
}
inCond = append(inCond, keywordFilterCondition(k, raw))
}
switch val := v.(type) {
case []string:
for _, item := range val {
appendKeywordCond(item)
}
case []interface{}:
for _, item := range val {
if s, ok := item.(string); ok {
appendKeywordCond(s)
} else {
appendKeywordCond(fmt.Sprintf("%v", item))
}
}
case string:
appendKeywordCond(val)
default:
appendKeywordCond(fmt.Sprintf("%v", v))
}
if len(inCond) > 0 {
// Balanced, not a flat "a or b or ... " chain: a long keyword list
// (22 wiki slugs) otherwise exceeds Infinity's Thrift expression
// depth and kills the connection (see joinBalanced).
cond = append(cond, joinBalanced(inCond, " or "))
}
continue
}
if k == "message_type" {
k = "message_type_kwd"
} else if k == "status" {
k = "status_int"
}
// Handle list values (mixed types - strings get quotes, numbers don't)
if list, ok := v.([]interface{}); ok && len(list) > 0 {
var strItems, numItems []string
for _, item := range list {
if s, ok := item.(string); ok {
strItems = append(strItems, fmt.Sprintf("'%s'", escapeFilterValue(s)))
} else if n, ok := item.(int); ok {
numItems = append(numItems, strconv.Itoa(n))
} else if n, ok := item.(int64); ok {
numItems = append(numItems, strconv.FormatInt(n, 10))
} else if f, ok := item.(float64); ok {
numItems = append(numItems, strconv.FormatFloat(f, 'f', -1, 64))
} else if s, ok := item.(fmt.Stringer); ok {
strItems = append(strItems, fmt.Sprintf("'%s'", escapeFilterValue(s.String())))
} else {
strItems = append(strItems, fmt.Sprintf("'%s'", escapeFilterValue(fmt.Sprintf("%v", item))))
}
}
if len(strItems) > 0 {
if len(strItems) == 1 {
cond = append(cond, fmt.Sprintf("%s=%s", k, strItems[0]))
} else {
cond = append(cond, fmt.Sprintf("%s IN (%s)", k, strings.Join(strItems, ", ")))
}
}
if len(numItems) > 0 {
if len(numItems) == 1 {
cond = append(cond, fmt.Sprintf("%s=%s", k, numItems[0]))
} else {
cond = append(cond, fmt.Sprintf("%s IN (%s)", k, strings.Join(numItems, ", ")))
}
}
continue
}
if list, ok := v.([]string); ok && len(list) > 0 {
if len(list) == 1 {
cond = append(cond, fmt.Sprintf("%s='%s'", k, escapeFilterValue(list[0])))
} else {
var items []string
for _, item := range list {
items = append(items, fmt.Sprintf("'%s'", escapeFilterValue(item)))
}
cond = append(cond, fmt.Sprintf("%s IN (%s)", k, strings.Join(items, ", ")))
}
continue
}
if list, ok := v.([]int); ok && len(list) > 0 {
if len(list) == 1 {
cond = append(cond, fmt.Sprintf("%s=%d", k, list[0]))
} else {
var strs []string
for _, n := range list {
strs = append(strs, strconv.Itoa(n))
}
cond = append(cond, fmt.Sprintf("%s IN (%s)", k, strings.Join(strs, ", ")))
}
continue
}
// Handle numeric values (no quotes)
if utility.IsNumericValue(v) {
cond = append(cond, fmt.Sprintf("%s=%v", k, v))
continue
}
// Handle string values (with quotes and escaping)
if str, ok := v.(string); ok {
cond = append(cond, fmt.Sprintf("%s='%s'", k, escapeFilterValue(str)))
continue
}
// Fallback: treat as string
cond = append(cond, fmt.Sprintf("%s='%s'", k, escapeFilterValue(fmt.Sprintf("%v", v))))
}
if len(cond) == 0 {
return ""
}
return strings.Join(cond, " AND ")
}
// decodeJSONFields converts Infinity JSON-column values back into their Go
// representation, mirroring Python's read-side parse_json_list for the list
// columns (infinity_conn.py:937-947): a slice passes through, nil becomes [], a
// JSON non-list value is wrapped, a legacy "###" string is split, and object
// columns such as source_chunk_hashes keep their shape.
func decodeJSONFields(chunk map[string]interface{}) {
for fieldName, value := range chunk {
if !fieldJSON(fieldName) {
continue
}
listField := fieldJSONList(fieldName)
text, ok := value.(string)
if !ok {
// Infinity hands a Json/Array column back as the raw
// length-prefixed multi-value byte blob (see
// decodeInfinityJSONBytes) rather than the JSON text a legacily
// typed varchar column returns, so decode that shape too. Without
// it every reader of such a column saw raw bytes: the structure
// graph could not read compilation_template_ids off the entity /
// relation rows, so it could not bucket them per template — it fell
// back to a synthetic "legacy" bucket with zero relations and the
// navigation tree's document level stayed flat instead of nesting
// into its entity tree.
if raw, isBytes := value.([]byte); isBytes {
if decoded, ok := decodeInfinityJSONBytes(raw); ok {
if listField {
decoded = asJSONList(decoded)
}
chunk[fieldName] = decoded
continue
}
}
if listField {
chunk[fieldName] = asJSONList(value)
}
continue
}
if text == "" {
if listField {
chunk[fieldName] = []interface{}{}
}
continue
}
var decoded interface{}
if err := json.Unmarshal([]byte(text), &decoded); err == nil {
if listField {
decoded = asJSONList(decoded)
}
chunk[fieldName] = decoded
continue
}
if listField {
parts := strings.Split(text, "###")
decodedList := make([]interface{}, 0, len(parts))
for _, part := range parts {
if part != "" {
decodedList = append(decodedList, part)
}
}
chunk[fieldName] = decodedList
}
}
}
// decodeVarcharJSONFields parses the JSON text transformChunkFields writes into
// fieldVarcharJSON columns back into a map; empty or nil becomes {}.
func decodeVarcharJSONFields(chunk map[string]interface{}) {
for fieldName, value := range chunk {
if !fieldVarcharJSON(fieldName) {
continue
}
text, ok := value.(string)
if !ok {
if value == nil {
chunk[fieldName] = map[string]interface{}{}
}
continue
}
if text == "" {
chunk[fieldName] = map[string]interface{}{}
continue
}
var decoded interface{}
if err := json.Unmarshal([]byte(text), &decoded); err == nil {
chunk[fieldName] = decoded
}
}
}
// splitLengthPrefixedSegments splits Infinity's `[int32 little-endian
// length][payload]` framing into its payloads, each the JSON text of one value.
// parseLengthPrefixedJSON (metadata.go) unpacks the same framing for the
// metadata column.
func splitLengthPrefixedSegments(data []byte) [][]byte {
var out [][]byte
for offset := 0; offset+4 <= len(data); {
length := int(uint32(data[offset]) |
uint32(data[offset+1])<<8 |
uint32(data[offset+2])<<16 |
uint32(data[offset+3])<<24)
offset += 4
if length < 0 || offset+length > len(data) {
break
}
out = append(out, data[offset:offset+length])
offset += length
}
return out
}
// realignJSONColumnCells redistributes a Json/Array column that Infinity returns
// as ONE cell holding one length-prefixed segment per returned row, in row
// order (the quirk realignMetaFieldsColumn documents for the metadata column).
// Without it the column-oriented loop in Search parks the whole column on row 0
// and rows 1..N lose it: a five-row entity/relation read carried
// compilation_template_ids on exactly one row, so the structure graph could not
// bucket rows per template and answered with a synthetic "legacy" template with
// zero relations.
//
// Each row receives its own segment as JSON text, which decodeJSONFields then
// decodes through the ordinary string path. The blob must frame into exactly one
// valid JSON value per row, so a column that is already aligned (one cell per
// row) or any other byte shape is left untouched.
func realignJSONColumnCells(rows []map[string]interface{}, columns map[string][]interface{}) {
if len(rows) < 2 {
return
}
for colName, cells := range columns {
if !fieldJSON(colName) || len(cells) != 1 {
continue
}
raw, isBytes := cells[0].([]byte)
if !isBytes {
continue
}
segments := splitLengthPrefixedSegments(raw)
if len(segments) != len(rows) {
continue
}
valid := true
for _, segment := range segments {
var value interface{}
if err := json.Unmarshal(segment, &value); err != nil {
valid = false
break
}
}
if !valid {
continue
}
for i := range rows {
rows[i][colName] = string(segments[i])
}
}
}
// decodeInfinityJSONBytes decodes a Json/Array column value that arrived as raw
// length-prefixed bytes rather than JSON text: the shape of a single-value cell
// (a one-row result, or a value realignJSONColumnCells left alone). Every segment
// must parse as JSON — anything else reports ok=false so the caller keeps its
// previous behavior. Array segments concatenate (duplicates dropped, so a
// repeated value cannot fabricate a second identical entry); a non-array segment
// replaces the accumulated value.
func decodeInfinityJSONBytes(raw []byte) (interface{}, bool) {
var (
items []interface{}
scalar interface{}
)
segments := splitLengthPrefixedSegments(raw)
if len(segments) == 0 {
return nil, false
}
for _, segment := range segments {
var value interface{}
if err := json.Unmarshal(segment, &value); err != nil {
return nil, false
}
if arr, isArray := value.([]interface{}); isArray {
items = appendJSONUnique(items, arr)
continue
}
scalar = value
}
if len(items) > 0 {
return items, true
}
return scalar, true
}
// appendJSONUnique appends the elements of add that dst does not already hold,
// comparing them by their JSON encoding (Infinity stores scalars here, so the
// encoding is a stable identity for a value).
func appendJSONUnique(dst, add []interface{}) []interface{} {
for _, item := range add {
key, err := json.Marshal(item)
if err != nil {
dst = append(dst, item)
continue
}
dup := false
for _, existing := range dst {
if other, err := json.Marshal(existing); err == nil && string(other) == string(key) {
dup = true
break
}
}
if !dup {
dst = append(dst, item)
}
}
return dst
}
// asJSONList normalizes a list-column value the way Python's parse_json_list
// does (infinity_conn.py:938-944): a slice passes through unchanged, a nil value
// becomes [] and any other scalar is wrapped in a single-element list.
func asJSONList(value interface{}) interface{} {
if value == nil {
return []interface{}{}
}
if rv := reflect.ValueOf(value); rv.Kind() == reflect.Slice || rv.Kind() == reflect.Array {
return value
}
return []interface{}{value}
}
// calculateScores calculates _score = score_column + pagerank
func calculateScores(chunks []map[string]interface{}, scoreColumn, pagerankField string) []map[string]interface{} {
for i := range chunks {
score := 0.0
if scoreVal, ok := chunks[i][scoreColumn]; ok {
if f, ok := common.ToFloat(scoreVal); ok {
score += f
}
}
if pagerankField != "" {
if prVal, ok := chunks[i][pagerankField]; ok {
if f, ok := common.ToFloat(prVal); ok {
score += f
}
}
}
chunks[i]["_score"] = score
}
return chunks
}
// sortByScore sorts by _score descending and limits
func sortByScore(chunks []map[string]interface{}, limit int) []map[string]interface{} {
if len(chunks) == 0 {
return chunks
}
// Sort by _score descending
sort.Slice(chunks, func(i, j int) bool {
scoreI := getChunkScore(chunks[i])
scoreJ := getChunkScore(chunks[j])
return scoreI > scoreJ
})
// Limit
if len(chunks) > limit && limit > 0 {
chunks = chunks[:limit]
}
return chunks
}
// getChunkScore extracts the score from a chunk
func getChunkScore(chunk map[string]interface{}) float64 {
if v, ok := chunk["_score"].(float64); ok {
return v
}
if v, ok := chunk["SCORE"].(float64); ok {
return v
}
if v, ok := chunk["SIMILARITY"].(float64); ok {
return v
}
return 0.0
}
// numericValue reports whether v is a number the hex encoder accepts.
func numericValue(v interface{}) (interface{}, bool) {
switch n := v.(type) {
case int, int64, float64:
return n, true
}
return nil, false
}
// numericRow appends one flat row of numbers: a JSON-decoded []interface{}, or
// the Go-typed []int / []int64 / []float64 a chunk built in process carries.
func numericRow(row interface{}, out *[]interface{}) bool {
if vals, ok := row.([]interface{}); ok {
for _, item := range vals {
n, isNum := numericValue(item)
if !isNum {
return false
}
*out = append(*out, n)
}
return true
}
switch vals := row.(type) {
case []int:
for _, n := range vals {
*out = append(*out, n)
}
case []int64:
for _, n := range vals {
*out = append(*out, n)
}
case []float64:
for _, n := range vals {
*out = append(*out, n)
}
default:
return false
}
return true
}
// numericSlice flattens a numeric slice — or a slice of numeric slices — into the
// flat []interface{} the hex encoder takes. Two shapes reach the engine: values
// decoded from JSON arrive as []interface{} (numbers as float64), while a chunk
// built IN PROCESS carries Go's typed slices (the ingestion pipeline's
// addPDFPositions emits []int / [][]int). Both must be encoded, because these
// columns are VARCHAR holding the hex form: handing Infinity the typed slice
// makes it try to store an int64 tensor and fail with
// "Not support to convert Tensor(int64,5) to Varchar" (InfinityException 3049).
// A scalar or a non-numeric value reports ok=false, so the caller keeps its
// pass-through instead of hex-encoding something it cannot parse back.
func numericSlice(v interface{}) ([]interface{}, bool) {
out := make([]interface{}, 0, 5)
switch vals := v.(type) {
case []interface{}:
// JSON shape: plain numbers, and/or position rows that are slices too.
for _, item := range vals {
if n, ok := numericValue(item); ok {
out = append(out, n)
continue
}
if !numericRow(item, &out) {
return nil, false
}
}
case []int:
for _, n := range vals {
out = append(out, n)
}
case []int64:
for _, n := range vals {
out = append(out, n)
}
case []float64:
for _, n := range vals {
out = append(out, n)
}
case [][]int:
for _, row := range vals {
if !numericRow(row, &out) {
return nil, false
}
}
case [][]int64:
for _, row := range vals {
if !numericRow(row, &out) {
return nil, false
}
}
case [][]float64:
for _, row := range vals {
if !numericRow(row, &out) {
return nil, false
}
}
default:
return nil, false
}
return out, true
}
// transformChunkFields converts chunk field names to Infinity format.
// Converts internal field names (like docnm_kwd) to Infinity column names (docnm).
// Also handles:
// - kb_id: extracts first element if it's a list
// - position_int, page_num_int, top_int: converts arrays to hex strings
// - fieldJSON / fieldJSONList: json column -> JSON string (Infinity json columns)
// - tag_kwd: joins with ### separator
// - question_kwd: joins with newline separator
// - other *_kwd keyword fields: lists join with ###
// - chunk_data / extra: dict -> JSON string (chunk-table varchar columns)
// - Missing embeddings filled with zeros if embeddingCols provided
func transformChunkFields(chunk map[string]interface{}, embeddingCols [][2]interface{}) map[string]interface{} {
d := make(map[string]interface{})
for k, v := range chunk {
switch k {
case "docnm_kwd":
d["docnm"] = v
case "title_kwd":
// title_kwd is a column of its own in the Infinity mapping: the
// compiled wiki/nav rows carry their label there, and nav keys its
// cluster updates on it (title_kwd = <cluster name>). Folding it away
// into docnm (Python's chunk-title alias, infinity_conn.py:522-527)
// wrote ONLY docnm, so on Infinity every reader of title_kwd saw ""
// while ES returned the field verbatim: nav labels degraded to the
// description's first line and every cluster lookup matched nothing,
// so each document opened a new root cluster instead of merging.
// Keep the column, and still mirror it into docnm when the row has no
// docnm_kwd, preserving the chunk-title search behavior.
title := utility.ConvertToString(v)
d["title_kwd"] = title
if _, exists := chunk["docnm_kwd"]; !exists {
d["docnm"] = title
}
case "title_sm_tks":
if _, exists := chunk["docnm_kwd"]; !exists {
d["docnm"] = utility.ConvertToString(v)
}
case "important_kwd":
// Python: list2str(tokens, ",") plus the count of the empty entries
// (infinity_conn.py:528-535). The extractor and the tokenizer hand
// over a Go-NATIVE []string, which the old []interface{}-only branch
// fed to ConvertToString — writing "[a b]" into important_keywords
// and leaving important_kwd_empty_count unset.
parts := utility.ConvertToStringSlice(v)
tokens := make([]string, 0, len(parts))
emptyCount := 0
for _, str := range parts {
if str == "" {
emptyCount++
continue
}
tokens = append(tokens, str)
}
d["important_keywords"] = strings.Join(tokens, ",")
d["important_kwd_empty_count"] = emptyCount
case "important_tks":
if _, exists := chunk["important_kwd"]; !exists {
d["important_keywords"] = v
}
case "content_with_weight":
d["content"] = v
case "content_ltks":
if _, exists := chunk["content_with_weight"]; !exists {
d["content"] = v
}
case "content_sm_ltks":
if _, exists := chunk["content_with_weight"]; !exists {
d["content"] = v
}
case "authors_tks":
d["authors"] = v
case "authors_sm_tks":
if _, exists := chunk["authors_tks"]; !exists {
d["authors"] = v
}
case "question_kwd":
d["questions"] = strings.Join(utility.ConvertToStringSlice(v), "\n")
case "tag_kwd":
d["tag_kwd"] = strings.Join(utility.ConvertToStringSlice(v), "###")
case "message_type":
d["message_type_kwd"] = v
case "status":
switch status := v.(type) {
case bool:
if status {
d["status_int"] = 1
} else {
d["status_int"] = 0
}
default:
d["status_int"] = v
}
case "question_tks":
if _, exists := chunk["question_kwd"]; !exists {
d["questions"] = utility.ConvertToString(v)
}
case "tenant_id":
// Infinity isolates tenants by table name (ragflow_<tenant>), so
// tenant_id is an internal routing field, not a table column.
continue
case "kb_id":
// 1. First check if it's a string
if str, ok := v.(string); ok {
d["kb_id"] = str
} else if list := utility.ConvertToStringSlice(v); len(list) > 0 {
// 2. If it's a list, convert and take the first element
d["kb_id"] = list[0]
} else {
// 3. Otherwise assign v directly
d["kb_id"] = v
}
case "position_int", "page_num_int", "top_int":
// Python flattens the position rows and hex-encodes every number
// (infinity_conn.py: `[num for row in v for num in row]`, "%08x").
// The input may be Go-NATIVE: the ingestion pipeline's addPDFPositions
// emits []int / [][]int, and those used to miss the
// []interface{}-only branch and reach Infinity as a raw tensor
// ("Not support to convert Tensor(int64,5) to Varchar", 3049).
if nums, ok := numericSlice(v); ok {
d[k] = utility.ConvertIntArrayToHex(nums)
} else {
d[k] = v
}
case "chunk_data":
d["chunk_data"] = utility.ConvertMapToJSONString(v)
case "extra":
// Python json-encodes a dict-valued `extra` (infinity_conn.py:572-581);
// raw, the SDK reads the map as a sparse-vector literal and rejects the
// string keys. meta_fields is not a chunk column — InsertMetadata
// encodes it on the doc-meta path (infinity/metadata.go).
d[k] = utility.ConvertMapToJSONString(v)
default:
if fieldJSON(k) {
d[k] = encodeJSONField(k, v)
} else if fieldVarcharJSON(k) {
jsonBytes, _ := json.Marshal(v)
d[k] = string(jsonBytes)
} else if fieldKeyword(k) {
// keyword fields with list values -> ### joined; accept the
// Go-native []string as well as []interface{}.
switch list := v.(type) {
case []string:
d[k] = strings.Join(list, "###")
case []interface{}:
d[k] = strings.Join(utility.ConvertToStringSlice(list), "###")
default:
d[k] = v
}
} else {
d[k] = v
}
}
}
// Remove intermediate token fields
for _, key := range []string{"docnm_kwd", "title_tks", "title_sm_tks", "important_kwd", "important_tks",
"content_with_weight", "content_ltks", "content_sm_ltks", "authors_tks", "authors_sm_tks",
"question_kwd", "question_tks"} {
delete(d, key)
}
// Fill missing embedding columns with zeros if embedding info provided
for _, ec := range embeddingCols {
name, ok1 := ec[0].(string)
size, ok2 := ec[1].(int)
if !ok1 || !ok2 {
continue
}
if _, exists := d[name]; !exists {
zeros := make([]float64, size)
for i := range zeros {
zeros[i] = 0
}
d[name] = zeros
}
}
return d
}
// encodeJSONField renders a json column value as the string Infinity stores. The
// Go SDK cannot encode a []string constant (3058), so a list is JSON-dumped; a
// non-list value falls back to the column default ("[]" for the json-list columns,
// infinity_conn.py:557-558; "{}" for the object one). Strings pass through, so
// re-transforming a row stays idempotent.
func encodeJSONField(fieldName string, value interface{}) interface{} {
listField := fieldJSONList(fieldName)
fallback := "{}"
if listField {
fallback = "[]"
}
if text, ok := value.(string); ok {
if text == "" {
return fallback
}
return text
}
if value == nil {
return fallback
}
if listField {
if rv := reflect.ValueOf(value); rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array {
return fallback
}
}
encoded, err := json.Marshal(value)
if err != nil {
return fallback
}
return string(encoded)
}
// DropChunkStore drops a chunk table from Infinity
func (e *Engine) DropChunkStore(ctx context.Context, baseName, datasetID string) error {
return e.dropTable(ctx, buildChunkTableName(baseName, datasetID))
}
// ChunkStoreExists checks if a chunk table exists in Infinity
func (e *Engine) ChunkStoreExists(ctx context.Context, baseName, datasetID string) (bool, error) {
return e.tableExists(ctx, buildChunkTableName(baseName, datasetID))
}