Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

12.3 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package vastbase

import (
	"encoding/json"
	"fmt"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

var placeholderRegex = regexp.MustCompile(`\$(\d+)`)

// shiftPlaceholders renumbers the $n placeholders of a built clause so it can
// be embedded after clauses that already consumed the first offset args.
func shiftPlaceholders(clause string, offset int) string {
	if offset == 0 {
		return clause
	}
	return placeholderRegex.ReplaceAllStringFunc(clause, func(match string) string {
		number, _ := strconv.Atoi(match[1:])
		return fmt.Sprintf("$%d", number+offset)
	})
}

// filterBuilder accumulates WHERE parts with a running $n counter; args order
// is bound to placeholder order by construction.
type filterBuilder struct {
	parts []string
	args  []interface{}
}

// raw appends a pre-built SQL fragment to the conjunction.
func (b *filterBuilder) raw(part string) {
	b.parts = append(b.parts, part)
}

// placeholder binds a value and returns its $n reference.
func (b *filterBuilder) placeholder(value interface{}) string {
	b.args = append(b.args, value)
	return fmt.Sprintf("$%d", len(b.args))
}

// join renders the accumulated parts as one AND chain, with a trivially
// true predicate for the empty case.
func (b *filterBuilder) join() string {
	if len(b.parts) == 0 {
		return "1=1"
	}
	return strings.Join(b.parts, " AND ")
}

// buildFilter translates an ES-style condition map into a parameterized WHERE
// clause, porting the Python connector's equivalent_condition_to_str:
//
//   - falsy values are dropped (the Python code skipped them too),
//   - keyword columns (source_id, *_kwd except docnm_kwd/knowledge_graph_kwd)
//     store ###-joined lists, so their term/terms conditions become LIKE
//     containment tests on the separator-padded column. Dropping them instead
//     would collapse a write filter to its dataset scope and let an update or
//     delete hit every row of the dataset,
//   - a column missing from the live schema matches no row (1=0), like an
//     unmapped ES field; a nil columns map means "unknown schema" and allows
//     every column,
//   - exists on a text column excludes DEFAULT ” rows, because unwritten text
//     columns hold ” rather than NULL; must_not.exists is the mirror image.
//
// Memory tables skip the keyword translation: their keyword-named columns
// (message_type_kwd, source_id) store plain scalars.
//
// Unlike the Python connector, kb_id is NOT skipped: Go tables are shared per
// baseName with kb_id/memory_id as row-level dataset discriminators, so the
// scope condition must reach the SQL.
func buildFilter(condition map[string]interface{}, kind string, columns map[string]columnMeta) (string, []interface{}, error) {
	if _, ok := condition["_id"]; ok {
		return "", nil, fmt.Errorf("vastbase: _id is not a filterable column, use id")
	}
	filter := &filterBuilder{}
	for _, rawKey := range sortedKeys(condition) {
		key := rawKey
		if kind == "memory" {
			key = mapMemoryField(key)
		} else if kind == "skill" && key == "id" {
			key = "skill_id"
		}
		value := condition[rawKey]
		if isEmptyFilterValue(value) {
			continue
		}
		if fieldKeyword(key) && kind != "memory" {
			filter.appendKeywordTerms(key, value, columns)
			continue
		}
		switch key {
		case "exists":
			column, ok := value.(string)
			if !ok {
				continue
			}
			if kind == "memory" {
				column = mapMemoryField(column)
			}
			filter.appendExists(column, columns)
		case "must_not":
			object, ok := value.(map[string]interface{})
			if !ok {
				continue
			}
			column, ok := object["exists"].(string)
			if !ok {
				continue
			}
			if kind == "memory" {
				column = mapMemoryField(column)
			}
			meta, known := columns[column]
			if !known {
				// A column the table lacks is NULL on every row, so the
				// must_not clause holds everywhere.
				continue
			}
			quoted := quoteIdent(column)
			if isTextColumn(meta.dataType) {
				filter.raw("(" + quoted + " IS NULL OR " + quoted + " = '')")
			} else {
				filter.raw(quoted + " IS NULL")
			}
		case "metadata_filtering_conditions":
			if err := filter.appendMetadataFilteringConditions(value); err != nil {
				return "", nil, err
			}
		default:
			if columns != nil {
				if _, known := columns[key]; !known {
					filter.raw("1=0")
					continue
				}
			}
			if values, ok := interfaceSlice(value); ok {
				placeholders := make([]string, 0, len(values))
				for _, item := range values {
					placeholders = append(placeholders, filter.placeholder(item))
				}
				filter.raw(quoteIdent(key) + " IN (" + strings.Join(placeholders, ", ") + ")")
			} else {
				filter.raw(quoteIdent(key) + " = " + filter.placeholder(value))
			}
		}
	}
	return filter.join(), filter.args, nil
}

// appendExists renders an exists/does-not-exist filter against a column,
// treating never-stored fields as absent on every row (ES dynamic-mapping
// parity).
func (b *filterBuilder) appendExists(column string, columns map[string]columnMeta) {
	meta, known := columns[column]
	if !known {
		// ES dynamic-mapping parity: a field this table never stored exists
		// on no row.
		b.raw("1=0")
		return
	}
	quoted := quoteIdent(column)
	if isTextColumn(meta.dataType) {
		b.raw("(" + quoted + " IS NOT NULL AND " + quoted + " <> '')")
		return
	}
	b.raw(quoted + " IS NOT NULL")
}

// likeEscaper neutralizes LIKE metacharacters so a bound value matches
// literally; backslash is the default LIKE escape character.
var likeEscaper = strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)

// appendKeywordTerms translates a term/terms condition on a ###-joined keyword
// column into LIKE containment tests: padding the column with separators makes
// first and last list elements match and prevents substring false positives
// ("nav_doc" must not match a stored "nav_doc_2" element). A terms list ORs
// one test per value, matching the ES any-of semantics. The pattern is bound
// as a parameter, so only the value's own LIKE metacharacters need escaping.
func (b *filterBuilder) appendKeywordTerms(column string, value interface{}, columns map[string]columnMeta) {
	if columns != nil {
		if _, known := columns[column]; !known {
			// ES dynamic-mapping parity: a field this table never stored exists
			// on no row.
			b.raw("1=0")
			return
		}
	}
	values := []interface{}{value}
	if list, ok := interfaceSlice(value); ok {
		values = list
	}
	padded := "('" + keywordSeparator + "' || " + quoteIdent(column) + " || '" + keywordSeparator + "')"
	parts := make([]string, 0, len(values))
	for _, item := range values {
		text := stringValue(item)
		if text == "" {
			continue
		}
		pattern := "%" + keywordSeparator + likeEscaper.Replace(text) + keywordSeparator + "%"
		parts = append(parts, padded+" LIKE "+b.placeholder(pattern))
	}
	if len(parts) == 0 {
		// Only empty-string values remain. The decode side drops empty
		// elements, so no stored row can ever contain one.
		b.raw("1=0")
		return
	}
	b.raw("(" + strings.Join(parts, " OR ") + ")")
}

// metadataJSON reads the metadata text column as jsonb, mapping rows the
// writer left at DEFAULT ” to an empty object so the cast never fails the
// whole query.
const metadataJSON = `CASE WHEN "metadata" LIKE '{%' THEN "metadata"::jsonb ELSE '{}'::jsonb END`

// metadataText builds the jsonb string extraction for one metadata key.
func (b *filterBuilder) metadataText(path string) string {
	return "(" + metadataJSON + ") ->> " + b.placeholder(path)
}

// metadataCast wraps the extracted text in a guarded cast: rows whose value
// does not parse as the target type yield NULL (no match) instead of failing
// the query, matching the silent-coercion behavior of the MySQL JSON path.
func (b *filterBuilder) metadataCast(path, castType, pattern string) string {
	text := b.metadataText(path)
	return "(CASE WHEN " + text + " ~ '" + pattern + "' THEN (" + text + ")::" + castType + " ELSE NULL END)"
}

// appendMetadataFilteringConditions adds the post-filter conditions from
// the metadata filtering option, joined as a conjunction.
func (b *filterBuilder) appendMetadataFilteringConditions(raw interface{}) error {
	filter, ok := raw.(map[string]interface{})
	if !ok {
		return nil
	}
	conditions, ok := interfaceSlice(filter["conditions"])
	if !ok || len(conditions) == 0 {
		return nil
	}
	logic := strings.ToUpper(stringValue(filter["logical_operator"]))
	if logic == "" {
		logic = "AND"
	}
	if logic != "AND" && logic != "OR" {
		return fmt.Errorf("vastbase: unsupported metadata logical operator: %s", logic)
	}
	parts := make([]string, 0, len(conditions))
	for _, item := range conditions {
		condition, ok := item.(map[string]interface{})
		if !ok {
			continue
		}
		name := stringValue(condition["name"])
		operator := stringValue(condition["comparison_operator"])
		if name == "" || operator == "" || strings.ContainsAny(name, `"\\`) {
			continue
		}
		// A plain top-level key, not a MySQL "$.name" JSON path: the jsonb
		// ->>/-> text operand is a literal key lookup, and no metadata object
		// carries a key literally named "$.name".
		path := name
		value := condition["value"]
		switch operator {
		case "is", "is not":
			comparison := "="
			if operator == "is not" {
				comparison = "<>"
			}
			parts = append(parts, b.metadataText(path)+" "+comparison+" "+b.placeholder(value))
		case "=", "≠", ">", "<", "≥", "≤":
			comparison := map[string]string{"=": "=", "≠": "<>", ">": ">", "<": "<", "≥": ">=", "≤": "<="}[operator]
			parts = append(parts, b.metadataCast(path, "numeric", `^-?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?$`)+
				" "+comparison+" "+b.placeholder(value))
		case "before", "after":
			comparison := "<"
			if operator == "after" {
				comparison = ">"
			}
			parts = append(parts, b.metadataCast(path, "timestamp", `^[0-9]{4}-[0-9]{2}-[0-9]{2}([ T][0-9]{2}:[0-9]{2}(:[0-9]{2})?)?$`)+
				" "+comparison+" "+b.placeholder(value))
		case "contains", "not contains":
			encoded, err := json.Marshal(value)
			if err != nil {
				continue
			}
			part := "(" + metadataJSON + ") -> " + b.placeholder(path) + " @> " + b.placeholder(string(encoded)) + "::jsonb"
			if operator == "not contains" {
				part = "NOT " + part
			}
			parts = append(parts, part)
		case "start with":
			parts = append(parts, b.metadataText(path)+" LIKE "+b.placeholder(stringValue(value)+"%"))
		case "end with":
			parts = append(parts, b.metadataText(path)+" LIKE "+b.placeholder("%"+stringValue(value)))
		case "empty", "not empty":
			text := b.metadataText(path)
			empty := "(" + text + " IS NULL OR " + text + " = '' OR " + text + " = '[]' OR " + text + " = '{}')"
			if operator == "not empty" {
				empty = "(" + text + " IS NOT NULL AND " + text + " != '' AND " + text + " != '[]' AND " + text + " != '{}')"
			}
			parts = append(parts, empty)
		}
	}
	if len(parts) == 0 {
		return nil
	}
	b.raw("(" + strings.Join(parts, " "+logic+" ") + ")")
	return nil
}

// isEmptyFilterValue reports whether a filter value means "match nothing":
// nil, empty string/collection, false, or zero.
func isEmptyFilterValue(value interface{}) bool {
	if value == nil {
		return true
	}
	rv := reflect.ValueOf(value)
	switch rv.Kind() {
	case reflect.String, reflect.Array, reflect.Slice, reflect.Map:
		return rv.Len() == 0
	case reflect.Bool:
		return !rv.Bool()
	// Python truthiness also drops numeric zero: the connector's
	// `if ... or not v: continue` skipped 0 and 0.0 conditions.
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return rv.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return rv.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return rv.Float() == 0
	}
	return false
}

// sortedKeys lists a map's keys in sorted order for deterministic SQL.
func sortedKeys(source map[string]interface{}) []string {
	keys := make([]string, 0, len(source))
	for key := range source {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	return keys
}