Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chat): always conjoin the authorized kb_id filter to SQL retrieval (#21866)

ZZZZY42da9923e531
diff --git a/internal/engine/elasticsearch/sql.go b/internal/engine/elasticsearch/sql.go
index 1d869c8d3a..358237d5ce 100644
--- a/internal/engine/elasticsearch/sql.go
+++ b/internal/engine/elasticsearch/sql.go
@@ -86,7 +86,7 @@ func Preprocess(sql string) string {
// RunSQL posts SQL to `/_sql`, translates the response into chunk-shaped maps.
// Returns (nil, nil) on empty rows; (nil, error) when retries exhausted.
-func (e *Engine) RunSQL(ctx context.Context, tableName string, sqlText string, kbIDs []string, format string) ([]map[string]interface{}, error) {
+func (e *Engine) RunSQL(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error) {
if e == nil || e.client == nil {
return nil, fmt.Errorf("Elasticsearch RunSQL: client not initialized")
}
diff --git a/internal/engine/elasticsearch/sql_test.go b/internal/engine/elasticsearch/sql_test.go
index 1c280392ee..89ba21c016 100644
--- a/internal/engine/elasticsearch/sql_test.go
+++ b/internal/engine/elasticsearch/sql_test.go
@@ -100,7 +100,7 @@ func TestRunSQL_NoFilterAdded(t *testing.T) {
e := newTestEngine(t, srv.URL)
ctx := t.Context()
- rows, err := e.RunSQL(ctx, "ragflow_t1", "SELECT doc_id FROM ragflow_t1", nil, "json")
+ rows, err := e.RunSQL(ctx, "SELECT doc_id FROM ragflow_t1", "json")
if err != nil {
t.Fatalf("RunSQL: %v", err)
}
@@ -135,7 +135,7 @@ func TestRunSQL_WhitespaceNormalizedAndPercentStripped(t *testing.T) {
// Input SQL has multiple backticks/spaces and trailing % characters.
in := "SELECT doc_id FROM `ragflow_t1` WHERE count > 0 %"
- _, err := e.RunSQL(ctx, "ragflow_t1", in, nil, "json")
+ _, err := e.RunSQL(ctx, in, "json")
if err != nil {
t.Fatalf("RunSQL: %v", err)
}
@@ -198,7 +198,7 @@ func TestRunSQL_PerAttemptTimeout(t *testing.T) {
done := make(chan result, 1)
go func() {
start := time.Now()
- _, err := e.RunSQL(ctx, "ragflow_t1", "SELECT 1", nil, "json")
+ _, err := e.RunSQL(ctx, "SELECT 1", "json")
done <- result{elapsed: time.Since(start), err: err}
}()
@@ -255,7 +255,7 @@ func TestRunSQL_RetryOnTimeoutThenSucceed(t *testing.T) {
e := newTestEngine(t, srv.URL)
ctx := t.Context()
- rows, err := e.RunSQL(ctx, "ragflow_t1", "SELECT 1", nil, "json")
+ rows, err := e.RunSQL(ctx, "SELECT 1", "json")
if err != nil {
t.Fatalf("RunSQL: %v", err)
}
@@ -290,7 +290,7 @@ func TestRunSQL_NonTimeoutErrorSurfacesImmediately(t *testing.T) {
e := newTestEngine(t, srv.URL)
ctx := t.Context()
- _, err := e.RunSQL(ctx, "ragflow_t1", "SELECT bad", nil, "json")
+ _, err := e.RunSQL(ctx, "SELECT bad", "json")
if err == nil {
t.Fatalf("RunSQL: got nil error, want error")
}
@@ -316,7 +316,7 @@ func TestRunSQL_RequestBodyHasFetchSizeAndFormat(t *testing.T) {
e := newTestEngine(t, srv.URL)
ctx := t.Context()
- if _, err := e.RunSQL(ctx, "ragflow_t1", "SELECT 1", nil, "json"); err != nil {
+ if _, err := e.RunSQL(ctx, "SELECT 1", "json"); err != nil {
t.Fatalf("RunSQL: %v", err)
}
cap.mu.Lock()
@@ -345,7 +345,7 @@ func TestRunSQL_EmptyRowsReturnsNilNil(t *testing.T) {
e := newTestEngine(t, srv.URL)
ctx := t.Context()
- rows, err := e.RunSQL(ctx, "ragflow_t1", "SELECT doc_id FROM ragflow_t1", nil, "json")
+ rows, err := e.RunSQL(ctx, "SELECT doc_id FROM ragflow_t1", "json")
if err != nil {
t.Fatalf("RunSQL: %v", err)
}
@@ -365,7 +365,7 @@ func TestRunSQL_PostsToSQLPath(t *testing.T) {
e := newTestEngine(t, srv.URL)
ctx := t.Context()
- if _, err := e.RunSQL(ctx, "ragflow_t1", "SELECT 1", nil, "json"); err != nil {
+ if _, err := e.RunSQL(ctx, "SELECT 1", "json"); err != nil {
t.Fatalf("RunSQL: %v", err)
}
cap.mu.Lock()
diff --git a/internal/engine/engine.go b/internal/engine/engine.go
index cf7455801f..e1b09b408f 100644
--- a/internal/engine/engine.go
+++ b/internal/engine/engine.go
@@ -77,7 +77,7 @@ type DocEngine interface {
GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string
// RunSQL runs a SQL query
- RunSQL(ctx context.Context, tableName string, sqlText string, kbIDs []string, format string) ([]map[string]interface{}, error)
+ RunSQL(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error)
GetChunkIDs(chunks []map[string]interface{}) []string
KNNScores(ctx context.Context, chunks []map[string]interface{}, queryVector []float64, topK int) (map[string]interface{}, error)
diff --git a/internal/engine/infinity/sql.go b/internal/engine/infinity/sql.go
index e98a676c90..ed1bb7fd3b 100644
--- a/internal/engine/infinity/sql.go
+++ b/internal/engine/infinity/sql.go
@@ -286,7 +286,7 @@ func resolvePsqlHostPort(hostURI string, postgresPort int) (host, port string) {
// RunSQL implements the SQL retrieval path: preprocess, rewrite aliases,
// run psql subprocess, parse output.
-func (e *Engine) RunSQL(ctx context.Context, tableName string, sqlText string, kbIDs []string, _ string) ([]map[string]interface{}, error) {
+func (e *Engine) RunSQL(ctx context.Context, sqlText string, _ string) ([]map[string]interface{}, error) {
if e == nil || e.client == nil {
return nil, fmt.Errorf("infinity RunSQL: client not initialized")
}
diff --git a/internal/engine/oceanbase/sql.go b/internal/engine/oceanbase/sql.go
index 8935d49eea..64623476b4 100644
--- a/internal/engine/oceanbase/sql.go
+++ b/internal/engine/oceanbase/sql.go
@@ -30,12 +30,7 @@ var (
)
// RunSQL executes the read-only SQL produced by the chat SQL-retrieval flow.
-func (e *Engine) RunSQL(ctx context.Context, tableName, sqlText string, kbIDs []string, format string) ([]map[string]interface{}, error) {
- if tableName != "" {
- if err := validateIdentifier(tableName); err != nil {
- return nil, err
- }
- }
+func (e *Engine) RunSQL(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error) {
normalized := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(sqlText), ";"))
normalized = strings.ReplaceAll(normalized, "`", "")
normalized = jsonExtractStringPattern.ReplaceAllString(normalized, "JSON_UNQUOTE(JSON_EXTRACT($1, $2))")
diff --git a/internal/engine/serenedb/serenedb_test.go b/internal/engine/serenedb/serenedb_test.go
index 9725bf4309..928ad28136 100644
--- a/internal/engine/serenedb/serenedb_test.go
+++ b/internal/engine/serenedb/serenedb_test.go
@@ -438,7 +438,7 @@ func TestRunSQLRejectsMutationStatementsBeforeExecution(t *testing.T) {
defer db.Close()
engine := &serenedbEngine{db: db}
- _, err = engine.RunSQL(context.Background(), "chunks", query, nil, "json")
+ _, err = engine.RunSQL(context.Background(), query, "json")
if err == nil {
t.Fatalf("RunSQL(%q) error = %v, want read-only SELECT rejection before query execution", query, err)
}
@@ -461,7 +461,7 @@ func TestRunSQLUsesReadOnlyTransaction(t *testing.T) {
mock.ExpectCommit()
engine := &serenedbEngine{db: db}
- rows, err := engine.RunSQL(context.Background(), "chunks", "SELECT 1", nil, "json")
+ rows, err := engine.RunSQL(context.Background(), "SELECT 1", "json")
if err != nil {
t.Fatalf("RunSQL: %v", err)
}
diff --git a/internal/engine/serenedb/sql.go b/internal/engine/serenedb/sql.go
index ea852f4857..2db437f906 100644
--- a/internal/engine/serenedb/sql.go
+++ b/internal/engine/serenedb/sql.go
@@ -215,7 +215,7 @@ func runSQLHasLimit(tokens []runSQLToken) bool {
// RunSQL executes a text-to-SQL query directly against SereneDB and returns the
// rows as maps. SereneDB speaks SQL natively, so unlike the Infinity engine no
// psql subprocess or field-alias rewrite is needed.
-func (e *serenedbEngine) RunSQL(ctx context.Context, tableName string, sqlText string, kbIDs []string, format string) ([]map[string]interface{}, error) {
+func (e *serenedbEngine) RunSQL(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error) {
query, err := boundRunSQL(sqlText, runSQLDefaultLimit)
if err != nil {
return nil, fmt.Errorf("serenedb: run sql: %w", err)
diff --git a/internal/engine/vastbase/sql.go b/internal/engine/vastbase/sql.go
index 9fa547318a..319e2bb082 100644
--- a/internal/engine/vastbase/sql.go
+++ b/internal/engine/vastbase/sql.go
@@ -33,12 +33,7 @@ var sqlLimitPattern = regexp.MustCompile(`(?i)\blimit\b`)
// (WITH d AS (DELETE FROM ... RETURNING *) SELECT * FROM d) also starts with
// WITH, so the statement runs inside a READ ONLY transaction the engine
// always rolls back — the server itself rejects writes on that path.
-func (e *Engine) RunSQL(ctx context.Context, tableName, sqlText string, kbIDs []string, format string) ([]map[string]interface{}, error) {
- if tableName != "" {
- if err := validateIdentifier(tableName); err != nil {
- return nil, err
- }
- }
+func (e *Engine) RunSQL(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error) {
normalized := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(sqlText), ";"))
if strings.Contains(normalized, ";") {
return nil, fmt.Errorf("multiple statements are not allowed")
diff --git a/internal/engine/vastbase/sql_test.go b/internal/engine/vastbase/sql_test.go
index a3b39f9ec1..4812726605 100644
--- a/internal/engine/vastbase/sql_test.go
+++ b/internal/engine/vastbase/sql_test.go
@@ -92,8 +92,7 @@ func TestRunSQLExecutesInReadOnlyRolledBackTransaction(t *testing.T) {
WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow("c1"))
mock.ExpectRollback()
- rows, err := engine.RunSQL(context.Background(), "t1",
- `SELECT "id" FROM "t1" WHERE kb_id = 'kb-1'`, []string{"kb-1"}, "")
+ rows, err := engine.RunSQL(context.Background(), `SELECT "id" FROM "t1" WHERE kb_id = 'kb-1'`, "")
if err != nil {
t.Fatal(err)
}
@@ -120,8 +119,7 @@ func TestRunSQLDataModifyingCTEFailsInsideReadOnlyTransaction(t *testing.T) {
WillReturnError(fmt.Errorf("cannot execute DELETE in a read-only transaction"))
mock.ExpectRollback()
- if _, err := engine.RunSQL(context.Background(), "t1",
- `WITH d AS (DELETE FROM "t1" RETURNING *) SELECT * FROM d`, nil, ""); err == nil {
+ if _, err := engine.RunSQL(context.Background(), `WITH d AS (DELETE FROM "t1" RETURNING *) SELECT * FROM d`, ""); err == nil {
t.Fatal("data-modifying CTE must fail inside the read-only transaction")
}
if err := mock.ExpectationsWereMet(); err != nil {
@@ -142,15 +140,12 @@ func TestRunSQLRejectsMalformedInputBeforeTouchingTheDatabase(t *testing.T) {
defer db.Close()
engine := newEngineWithDB(config.VastbaseConfig{DBCompatibility: compatPG}, db)
- if _, err := engine.RunSQL(context.Background(), "bad;table", "SELECT 1", nil, ""); err == nil {
- t.Fatal("invalid table names must be rejected")
- }
for _, bad := range []string{
"DELETE FROM t1",
"UPDATE t1 SET x = 1",
"SELECT 1; DELETE FROM t1",
} {
- if _, err := engine.RunSQL(context.Background(), "t1", bad, nil, ""); err == nil {
+ if _, err := engine.RunSQL(context.Background(), bad, ""); err == nil {
t.Fatalf("must reject %q", bad)
}
}
diff --git a/internal/ingestion/knowledge_compile/fake_engine_test.go b/internal/ingestion/knowledge_compile/fake_engine_test.go
index 3dbd726301..e0f694698f 100644
--- a/internal/ingestion/knowledge_compile/fake_engine_test.go
+++ b/internal/ingestion/knowledge_compile/fake_engine_test.go
@@ -108,7 +108,7 @@ func (f *fakeEngine) GetAggregation([]map[string]interface{}, string) []map[stri
func (f *fakeEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
return nil
}
-func (f *fakeEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
+func (f *fakeEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (f *fakeEngine) GetChunkIDs([]map[string]interface{}) []string { return nil }
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 14969c4f50..dd7be9d48a 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -3508,20 +3508,22 @@ func (s *ChatPipelineService) useSQL(
}
}
- // Step 1.5: inject the kb_id WHERE filter for ES / OS / OceanBase.
- // No-op for Infinity (the table name already encodes the KB scope).
- // Mirrors add_kb_filter at dialog_service.py:992-1021, called from
- // get_table right after normalize_sql.
+ // Step 1.5: constrain the SQL to the authorized kb_ids for ES / OS /
+ // OceanBase. No-op for Infinity (the table name already encodes the KB
+ // scope). When the scope cannot be enforced safely, skip SQL retrieval so
+ // the caller falls back to vector search instead of running an unfiltered
+ // query.
if filtered, ok := addKBFilter(sqlText, engineName, kbs); ok {
sqlText = filtered
} else {
- common.Warn("SQL retrieval: invalid kb_id UUID; SQL will run unfiltered")
+ common.Warn("SQL retrieval: SQL cannot be scoped to the authorized datasets; falling back")
+ return nil, nil
}
// Step 2: try to execute. On failure, repair once with the
// engine-specific execution-error prompt so the LLM regenerates
// correctly (Flow B at dialog_service.py:1164-1205).
- rows, execErr := docEngine.RunSQL(ctx, tableName, sqlText, kbIDStrings(kbs), "json")
+ rows, execErr := docEngine.RunSQL(ctx, sqlText, "json")
if execErr != nil {
common.Debug("SQL retrieval: initial execution failed, attempting repair",
zap.String("sql", sqlText), zap.Error(execErr))
@@ -3533,10 +3535,12 @@ func (s *ChatPipelineService) useSQL(
return nil, nil
}
// Re-apply the kb filter after the LLM-driven repair.
- if filtered, ok := addKBFilter(repaired, engineName, kbs); ok {
- repaired = filtered
+ filtered, ok := addKBFilter(repaired, engineName, kbs)
+ if !ok {
+ common.Warn("SQL retrieval: repaired SQL cannot be scoped to the authorized datasets; falling back")
+ return nil, nil
}
- rows, execErr = docEngine.RunSQL(ctx, tableName, repaired, kbIDStrings(kbs), "json")
+ rows, execErr = docEngine.RunSQL(ctx, filtered, "json")
if execErr != nil {
common.Warn("SQL retrieval: repaired SQL also failed", zap.Error(execErr))
return nil, nil
@@ -3563,19 +3567,23 @@ func (s *ChatPipelineService) useSQL(
ctx, chatModel, sysPrompt, tableName, question, sqlText, expectedCol, engineName, fieldMap,
)
if repairErr == nil && repaired != "" {
- // Re-apply the kb filter after the LLM-driven repair.
- if filtered, ok := addKBFilter(repaired, engineName, kbs); ok {
- repaired = filtered
- }
- repairedRows, repairedErr := docEngine.RunSQL(ctx, tableName, repaired, kbIDStrings(kbs), "json")
- if repairedErr == nil && len(repairedRows) > 0 && hasSourceColumns(repairedRows) {
- common.Debug("SQL retrieval: missing-columns repair succeeded",
- zap.String("sql", repaired))
- rows = repairedRows
- sqlText = repaired
+ // Re-apply the kb filter after the LLM-driven repair. The current
+ // rows are already scope-safe, so when the repaired SQL cannot be
+ // scoped we keep them instead of running an unfiltered query.
+ filtered, ok := addKBFilter(repaired, engineName, kbs)
+ if !ok {
+ common.Warn("SQL retrieval: repaired SQL cannot be scoped to the authorized datasets; keeping the scoped result")
} else {
- common.Warn("SQL retrieval: missing-columns repair did not yield source columns; using best-effort answer",
- zap.String("sql", repaired))
+ repairedRows, repairedErr := docEngine.RunSQL(ctx, filtered, "json")
+ if repairedErr == nil && len(repairedRows) > 0 && hasSourceColumns(repairedRows) {
+ common.Debug("SQL retrieval: missing-columns repair succeeded",
+ zap.String("sql", filtered))
+ rows = repairedRows
+ sqlText = filtered
+ } else {
+ common.Warn("SQL retrieval: missing-columns repair did not yield source columns; using best-effort answer",
+ zap.String("sql", filtered))
+ }
}
} else if repairErr != nil {
common.Warn("SQL retrieval: missing-columns repair failed; using best-effort answer",
@@ -3634,24 +3642,258 @@ func isValidUUID(s string) bool {
return uuidRe.MatchString(s)
}
-// addKBFilter injects a validated kb_id WHERE filter into sqlText for
-// ES / OS / OceanBase engines. Infinity is a no-op because the table
-// name already encodes the KB scope. Mirrors dialog_service.py:992-1021.
+// scalarWord is one unquoted SQL word plus its parenthesis depth.
+type scalarWord struct {
+ word string
+ start, end int
+ depth int
+}
+
+func isScalarWordStart(c byte) bool {
+ return c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= 0x80
+}
+
+func isScalarWordByte(c byte) bool {
+ return isScalarWordStart(c) || c >= '0' && c <= '9' || c == '$'
+}
+
+// scalarDollarQuoteDelimiter returns the delimiter of a PostgreSQL dollar-quoted
+// string starting at start (either "$$" or "$tag$"). ok is false when start does
+// not open one, e.g. for the "$1" bind placeholder.
+func scalarDollarQuoteDelimiter(sqlText string, start int) (string, bool) {
+ i := start + 1
+ if i < len(sqlText) && sqlText[i] == '$' {
+ return "$$", true
+ }
+ if i >= len(sqlText) || !isScalarWordStart(sqlText[i]) {
+ return "", false
+ }
+ for i < len(sqlText) && (isScalarWordStart(sqlText[i]) || sqlText[i] >= '0' && sqlText[i] <= '9') {
+ i++
+ }
+ if i >= len(sqlText) || sqlText[i] != '$' {
+ return "", false
+ }
+ return sqlText[start : i+1], true
+}
+
+// sqlScanDialect selects the lexical differences between the engines
+// addKBFilter runs against. The scan never guesses: a construct it cannot skip
+// exactly makes it fail, which callers treat as "do not execute".
+type sqlScanDialect struct {
+ // backslashEscapes enables MySQL-family backslash escapes in quoted strings.
+ backslashEscapes bool
+ // hashLineComments treats "#" as a line comment (MySQL/OceanBase). It is
+ // refused rather than skipped because a "#" would swallow any filter the
+ // wrapper appends on the same line.
+ hashLineComments bool
+ // executableComments refuses "/*! ... */" (MySQL/OceanBase), whose contents
+ // the server executes and which could therefore hide a set operation.
+ executableComments bool
+ // dollarQuotes enables $$...$$ and $tag$...$tag$ literals (PostgreSQL).
+ dollarQuotes bool
+ // nestedBlockComments enables PostgreSQL's nested "/* /* */ */".
+ nestedBlockComments bool
+}
+
+func sqlScanDialectFor(engineName string) sqlScanDialect {
+ switch {
+ case engine.IsOceanBaseFamily(engineName):
+ return sqlScanDialect{backslashEscapes: true, hashLineComments: true, executableComments: true}
+ case engine.IsVastbase(engineName), engineName == string(engine.EngineSereneDB):
+ return sqlScanDialect{dollarQuotes: true, nestedBlockComments: true}
+ default:
+ // Elasticsearch / OpenSearch / unknown: standard literals only.
+ return sqlScanDialect{}
+ }
+}
+
+// scanScalarWords tokenizes sqlText, skipping quoted strings, quoted
+// identifiers, dollar-quoted strings, and comments. It records only unquoted
+// words plus their parenthesis depth, which is all that is needed to locate the
+// top-level WHERE clause without being fooled by keywords inside literals or
+// comments.
+//
+// lineComments lists the byte ranges of line comments (from "--" to the
+// terminating newline or end of input). Callers use it to avoid appending SQL
+// onto a commented-out line.
+//
+// Parentheses must balance: a ')' at depth 0 or a positive depth at the end
+// would let a predicate escape the wrapping parentheses, so both make the scan
+// fail. Unterminated literals and comments also fail so callers can fail closed
+// instead of guessing.
+func scanScalarWords(sqlText string, dialect sqlScanDialect) (words []scalarWord, lineComments [][2]int, ok bool) {
+ words = make([]scalarWord, 0, len(sqlText)/4)
+ depth := 0
+ for i := 0; i < len(sqlText); {
+ c := sqlText[i]
+ if c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
+ i++
+ continue
+ }
+ if c == '-' && i+1 < len(sqlText) && sqlText[i+1] == '-' {
+ // MySQL-family SQL treats "--" as a comment only when followed by
+ // whitespace or a control character. Otherwise it is SQL syntax.
+ if dialect.hashLineComments && (i+2 >= len(sqlText) || sqlText[i+2] > ' ') {
+ return nil, nil, false
+ }
+ start := i
+ i += 2
+ for i < len(sqlText) && sqlText[i] != '\n' && sqlText[i] != '\r' {
+ i++
+ }
+ lineComments = append(lineComments, [2]int{start, i})
+ continue
+ }
+ if dialect.hashLineComments && c == '#' {
+ // A "#" comment runs to end of line and would swallow anything the
+ // wrapper appends after it. Refuse instead of trying to place the
+ // filter around it.
+ return nil, nil, false
+ }
+ if c == '/' && i+1 < len(sqlText) && sqlText[i+1] == '*' {
+ if dialect.executableComments && i+2 < len(sqlText) && sqlText[i+2] == '!' {
+ // Executable comment: the server runs its contents, so the scan
+ // would not see statements (e.g. a UNION) hidden inside it.
+ return nil, nil, false
+ }
+ i += 2
+ commentDepth := 1
+ for i < len(sqlText) && commentDepth > 0 {
+ if i+1 < len(sqlText) && sqlText[i] == '/' && sqlText[i+1] == '*' {
+ if !dialect.nestedBlockComments {
+ return nil, nil, false
+ }
+ commentDepth++
+ i += 2
+ continue
+ }
+ if i+1 < len(sqlText) && sqlText[i] == '*' && sqlText[i+1] == '/' {
+ commentDepth--
+ i += 2
+ continue
+ }
+ i++
+ }
+ if commentDepth != 0 {
+ return nil, nil, false
+ }
+ continue
+ }
+ if c == '\'' || c == '"' || c == '`' {
+ quote := c
+ escapes := quote != '`' && dialect.backslashEscapes
+ if dialect.dollarQuotes && quote == '\'' && i > 0 &&
+ (sqlText[i-1] == 'E' || sqlText[i-1] == 'e') &&
+ (i == 1 || !isScalarWordByte(sqlText[i-2])) {
+ escapes = true // PostgreSQL E'...' escape string.
+ }
+ i++
+ closed := false
+ for i < len(sqlText) {
+ if sqlText[i] == '\\' && i+1 < len(sqlText) {
+ // OceanBase's SQL mode may make backslash an ordinary
+ // character. Refuse quote boundaries that depend on it.
+ if dialect.backslashEscapes && sqlText[i+1] == quote {
+ return nil, nil, false
+ }
+ if escapes {
+ i += 2
+ continue
+ }
+ // PostgreSQL's plain-string behavior depends on the server
+ // setting. Refuse ambiguous quote escapes in either mode.
+ if dialect.dollarQuotes && quote == '\'' && sqlText[i+1] == '\'' {
+ return nil, nil, false
+ }
+ }
+ if sqlText[i] == quote {
+ if i+1 < len(sqlText) && sqlText[i+1] == quote {
+ i += 2
+ continue
+ }
+ i++
+ closed = true
+ break
+ }
+ i++
+ }
+ if !closed {
+ return nil, nil, false
+ }
+ continue
+ }
+ if dialect.dollarQuotes && c == '$' {
+ if delim, isQuote := scalarDollarQuoteDelimiter(sqlText, i); isQuote {
+ bodyStart := i + len(delim)
+ closeOffset := strings.Index(sqlText[bodyStart:], delim)
+ if closeOffset < 0 {
+ return nil, nil, false
+ }
+ i = bodyStart + closeOffset + len(delim)
+ continue
+ }
+ }
+ if c == '(' {
+ depth++
+ i++
+ continue
+ }
+ if c == ')' {
+ // A ')' without a matching '(' means the predicate would close the
+ // wrapper early and let OR escape the injected scope.
+ if depth == 0 {
+ return nil, nil, false
+ }
+ depth--
+ i++
+ continue
+ }
+ if isScalarWordStart(c) {
+ start := i
+ i++
+ for i < len(sqlText) && isScalarWordByte(sqlText[i]) {
+ i++
+ }
+ words = append(words, scalarWord{
+ word: strings.ToUpper(sqlText[start:i]),
+ start: start,
+ end: i,
+ depth: depth,
+ })
+ continue
+ }
+ i++
+ }
+ if depth != 0 {
+ return nil, nil, false
+ }
+ return words, lineComments, true
+}
+
+// addKBFilter constrains sqlText to the caller's authorized kb_ids. Infinity is
+// a no-op because its table name already encodes the KB scope.
//
-// Returns the (possibly modified) SQL and a boolean indicating whether
-// all kb_ids passed UUID validation. When validation fails, the SQL is
-// returned unchanged — the engine will likely reject the un-filtered
-// query, triggering the repair path (Python's `_assert_valid_uuid` raises
-// ValueError, which `get_table`'s try/except catches and routes to the
-// repair flow).
+// The authorized filter is always ANDed onto the top-level predicate, with the
+// model's own predicate wrapped in parentheses. This means a model-written
+// `kb_id = '<other>'` is neutralized rather than trusted: the resulting
+// predicate is a contradiction and yields zero rows. The filter is never
+// skipped just because the text already mentions kb_id.
//
-// If the SQL already has a WHERE clause with `kb_id =`, the filter is
-// not duplicated. Otherwise a fresh WHERE is appended, or `kb_id = '...'
-// AND` is prepended to an existing WHERE.
+// Returns the rewritten SQL and whether the result is safe to execute. It
+// returns false when a kb_id is not a UUID, or when the statement shape cannot
+// be scoped safely (set operations, unterminated literals/comments). Callers
+// must not execute the SQL when it is false.
func addKBFilter(sqlText, engineName string, kbs []*entity.Knowledgebase) (string, bool) {
if engineName == "infinity" || len(kbs) == 0 {
return sqlText, true
}
+ // ES and OceanBase remove backticks after this rewrite. Refuse them so
+ // engine preprocessing cannot expose SQL syntax hidden from the scan.
+ if strings.Contains(sqlText, "`") &&
+ (engine.IsOceanBaseFamily(engineName) || engineName == "elasticsearch" || engineName == "opensearch") {
+ return sqlText, false
+ }
// Validate all kb_ids as UUIDs.
for _, kb := range kbs {
@@ -3672,20 +3914,98 @@ func addKBFilter(sqlText, engineName string, kbs []*entity.Knowledgebase) (strin
kbFilter = "(" + strings.Join(parts, " OR ") + ")"
}
- lower := strings.ToLower(sqlText)
- if !strings.Contains(lower, "where ") {
- // No WHERE clause: append one. Honor ORDER BY if present.
- if oIdx := strings.Index(lower, "order by"); oIdx >= 0 {
- sqlText = sqlText[:oIdx] + " WHERE " + kbFilter + " order by " + sqlText[oIdx+len("order by"):]
- } else {
- sqlText += " WHERE " + kbFilter
+ // ES removes '%' before execution. Do the same before locating SQL clauses
+ // so that removal cannot create comments after the filter is injected.
+ if engineName == "elasticsearch" || engineName == "opensearch" {
+ sqlText = strings.ReplaceAll(sqlText, "%", "")
+ }
+
+ words, lineComments, ok := scanScalarWords(sqlText, sqlScanDialectFor(engineName))
+ if !ok {
+ return sqlText, false
+ }
+
+ // Set operations cannot be scoped from the first arm alone: the second arm
+ // would bypass the injected predicate. Refuse instead of guessing.
+ for _, w := range words {
+ if w.depth == 0 && (w.word == "UNION" || w.word == "INTERSECT" || w.word == "EXCEPT") {
+ return sqlText, false
}
- } else if !strings.Contains(lower, "kb_id =") && !strings.Contains(lower, "kb_id=") {
- // Has WHERE but no kb_id: insert "kb_id = '...' AND" after WHERE.
- whereRe := regexp.MustCompile(`(?i)\bwhere\b `)
- sqlText = whereRe.ReplaceAllString(sqlText, "where "+kbFilter+" and ")
}
- return sqlText, true
+
+ whereIdx := -1
+ for i, w := range words {
+ if w.depth == 0 && w.word == "WHERE" {
+ whereIdx = i
+ break
+ }
+ }
+
+ // The predicate ends at the first top-level trailing clause after WHERE, or
+ // at end of statement. Comments after WHERE stay inside the predicate: they
+ // may sit mid-expression (`1=1 -- note\n OR 1=1`), and cutting there would
+ // leave the trailing OR outside the wrapper and make it always true.
+ endPos := len(sqlText)
+ scanFrom := 0
+ if whereIdx >= 0 {
+ scanFrom = whereIdx + 1
+ }
+ for i := scanFrom; i < len(words); i++ {
+ w := words[i]
+ if w.depth != 0 {
+ continue
+ }
+ switch w.word {
+ case "GROUP", "HAVING", "ORDER", "LIMIT", "OFFSET":
+ endPos = w.start
+ }
+ if endPos != len(sqlText) {
+ break
+ }
+ }
+
+ // A line comment running to the end of its line swallows anything appended
+ // on that line. When the insertion point falls at the end of such a
+ // comment, terminate the comment with a newline first so the injected text
+ // starts on a fresh line.
+ lineCommentSwallows := func(pos int) bool {
+ for _, lc := range lineComments {
+ if pos > lc[0] && pos <= lc[1] {
+ return true
+ }
+ }
+ return false
+ }
+
+ if whereIdx < 0 {
+ head := strings.TrimRight(sqlText[:endPos], " \t\r\n")
+ sep := " WHERE "
+ if lineCommentSwallows(len(head)) {
+ sep = "\nWHERE "
+ }
+ tail := strings.TrimSpace(sqlText[endPos:])
+ if tail != "" {
+ return head + sep + kbFilter + " " + tail, true
+ }
+ return head + sep + kbFilter, true
+ }
+
+ whereWord := words[whereIdx]
+ predicateRaw := sqlText[whereWord.end:endPos]
+ predicate := strings.TrimSpace(predicateRaw)
+ if predicate == "" {
+ return sqlText, false
+ }
+ closeParen := ")"
+ if lineCommentSwallows(whereWord.end + len(strings.TrimRight(predicateRaw, " \t\r\n"))) {
+ closeParen = "\n)"
+ }
+ rewritten := sqlText[:whereWord.start] +
+ "WHERE (" + predicate + closeParen + " AND (" + kbFilter + ")"
+ if tail := strings.TrimSpace(sqlText[endPos:]); tail != "" {
+ rewritten += " " + tail
+ }
+ return rewritten, true
}
// generateSQL calls the chat model to produce a SQL SELECT.
@@ -4113,35 +4433,57 @@ func extractSourceColumnIndexes(rows []map[string]interface{}) (docIDIdx, docNam
return
}
-// WHERE-clause extraction for the aggregate secondary fetch.
-// Mirrors dialog_service.py:1321.
-var whereClauseRe = regexp.MustCompile(`(?i)\bwhere\b(.+?)(?:\bgroup by\b|\border by\b|\blimit\b|$)`)
-
-// limitClauseRe detects whether a SQL already has a LIMIT clause.
-var limitClauseRe = regexp.MustCompile(`(?i)\blimit\b`)
-
-// buildChunkFetchSQL extracts the WHERE clause from the original SQL
-// and constructs a secondary SQL to fetch source chunks. Mirrors
-// dialog_service.py:1327-1331. Returns ("", false) when no WHERE is
-// present. The `multiKB` flag controls whether `kb_id` is included
+// buildChunkFetchSQL extracts the top-level WHERE predicate from the original
+// SQL and constructs a secondary query to fetch source chunks. Returns
+// ("", false) when scanning fails or no predicate is present.
+// The `multiKB` flag controls whether `kb_id` is included
// in the SELECT list (single-kb queries don't need it because the
// caller already knows the kb_id).
-func buildChunkFetchSQL(originalSQL, tableName, expectedCol string, multiKB bool) (string, bool) {
- m := whereClauseRe.FindStringSubmatch(originalSQL)
- if len(m) < 2 {
+func buildChunkFetchSQL(originalSQL, tableName, expectedCol string, multiKB bool, engineName string) (string, bool) {
+ words, lineComments, ok := scanScalarWords(originalSQL, sqlScanDialectFor(engineName))
+ if !ok {
+ return "", false
+ }
+ start, end := -1, len(originalSQL)
+predicate:
+ for _, word := range words {
+ if word.depth != 0 {
+ continue
+ }
+ if start < 0 {
+ if word.word == "WHERE" {
+ start = word.end
+ }
+ continue
+ }
+ switch word.word {
+ case "GROUP", "HAVING", "ORDER", "LIMIT", "OFFSET":
+ end = word.start
+ break predicate
+ }
+ }
+ if start < 0 {
+ return "", false
+ }
+ where := strings.TrimSpace(originalSQL[start:end])
+ if where == "" {
return "", false
}
- where := strings.TrimSpace(m[1])
kbCol := ""
if multiKB {
kbCol = ", kb_id"
}
sql := fmt.Sprintf("select doc_id, %s%s from %s where %s",
expectedCol, kbCol, tableName, where)
- if !limitClauseRe.MatchString(sql) {
- sql += " limit 20"
+ limitSeparator := " "
+ predicateEnd := len(strings.TrimRight(originalSQL[:end], " \t\r\n"))
+ for _, comment := range lineComments {
+ if predicateEnd > comment[0] && predicateEnd <= comment[1] {
+ limitSeparator = "\n"
+ break
+ }
}
- return sql, true
+ return sql + limitSeparator + "limit 20", true
}
// toIfaceSlice converts a []map[string]interface{} to []interface{} for
@@ -4186,14 +4528,28 @@ func (s *ChatPipelineService) fetchAggregateChunks(
return nil, nil
}
- chunksSQL, ok := buildChunkFetchSQL(originalSQL, tableName, expectedCol, multiKB)
+ engineName := docEngine.GetType()
+ chunksSQL, ok := buildChunkFetchSQL(originalSQL, tableName, expectedCol, multiKB, engineName)
if !ok {
common.Debug("SQL retrieval: aggregate secondary fetch skipped (no WHERE clause)",
zap.String("sql", originalSQL))
return nil, nil
}
- rows, err := docEngine.RunSQL(ctx, tableName, chunksSQL, kbIDs, "json")
+ if len(kbIDs) == 0 {
+ return nil, nil
+ }
+ kbs := make([]*entity.Knowledgebase, len(kbIDs))
+ for i, kbID := range kbIDs {
+ kbs[i] = &entity.Knowledgebase{ID: kbID}
+ }
+ chunksSQL, ok = addKBFilter(chunksSQL, engineName, kbs)
+ if !ok {
+ common.Warn("SQL retrieval: aggregate secondary fetch cannot be scoped to the authorized datasets; skipping")
+ return nil, nil
+ }
+
+ rows, err := docEngine.RunSQL(ctx, chunksSQL, "json")
if err != nil {
common.Warn("SQL retrieval: aggregate secondary fetch failed",
zap.String("sql", chunksSQL), zap.Error(err))
diff --git a/internal/service/chat_pipeline_sql_scope_test.go b/internal/service/chat_pipeline_sql_scope_test.go
new file mode 100644
index 0000000000..1437cd9a49
--- /dev/null
+++ b/internal/service/chat_pipeline_sql_scope_test.go
@@ -0,0 +1,566 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
+)
+
+const (
+ scopeAllowedKB = "11111111-1111-1111-1111-111111111111"
+ scopeVictimKB = "22222222-2222-2222-2222-222222222222"
+)
+
+func mustContainScope(t *testing.T, got, want string) {
+ t.Helper()
+ if !strings.Contains(got, want) {
+ t.Fatalf("expected %q in rewritten SQL:\n%s", want, got)
+ }
+}
+
+// TestAddKBFilterNeutralizesModelSuppliedKB checks the core invariant: the
+// authorized filter is always conjoined, so a model-written kb_id for another
+// dataset becomes a contradiction that returns no rows.
+func TestAddKBFilterNeutralizesModelSuppliedKB(t *testing.T) {
+ sql := "SELECT doc_id, docnm_kwd FROM ragflow_tenant_b WHERE kb_id = '" + scopeVictimKB + "'"
+ got, ok := addKBFilter(sql, "elasticsearch", []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected the statement to be safely scoped, got ok=false")
+ }
+ mustContainScope(t, got, "kb_id = '"+scopeAllowedKB+"'")
+ mustContainScope(t, got, "WHERE (kb_id = '"+scopeVictimKB+"')")
+ mustContainScope(t, got, ") AND (kb_id = '"+scopeAllowedKB+"')")
+}
+
+// TestAddKBFilterPrecedence checks that an OR in the model's predicate cannot
+// escape the injected scope.
+func TestAddKBFilterPrecedence(t *testing.T) {
+ got, ok := addKBFilter("SELECT doc_id FROM t WHERE title = 'x' OR 1=1", "elasticsearch",
+ []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ mustContainScope(t, got, "WHERE (title = 'x' OR 1=1) AND (kb_id = '"+scopeAllowedKB+"')")
+}
+
+// TestAddKBFilterNoWhere checks that a statement without WHERE gets the scope
+// inserted before trailing clauses.
+func TestAddKBFilterNoWhere(t *testing.T) {
+ got, ok := addKBFilter("SELECT doc_id FROM t ORDER BY x", "elasticsearch",
+ []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ mustContainScope(t, got, "WHERE kb_id = '"+scopeAllowedKB+"' ORDER BY x")
+}
+
+// TestAddKBFilterTrailingClauses checks the predicate ends before GROUP BY so
+// the injected scope does not land inside the grouping clause.
+func TestAddKBFilterTrailingClauses(t *testing.T) {
+ got, ok := addKBFilter("SELECT doc_id FROM t WHERE x=1 GROUP BY g ORDER BY o LIMIT 5", "elasticsearch",
+ []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ mustContainScope(t, got, "WHERE (x=1) AND (kb_id = '"+scopeAllowedKB+"') GROUP BY g ORDER BY o LIMIT 5")
+}
+
+// TestAddKBFilterSubqueryAndCTE checks only the top-level predicate is wrapped.
+func TestAddKBFilterSubqueryAndCTE(t *testing.T) {
+ got, ok := addKBFilter("SELECT doc_id FROM t WHERE a IN (SELECT b FROM u WHERE c=1)", "elasticsearch",
+ []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ mustContainScope(t, got, "WHERE (a IN (SELECT b FROM u WHERE c=1)) AND (kb_id = '"+scopeAllowedKB+"')")
+
+ got, ok = addKBFilter("WITH x AS (SELECT doc_id FROM t WHERE y=1) SELECT doc_id FROM x WHERE z=2", "elasticsearch",
+ []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ mustContainScope(t, got, "WHERE (z=2) AND (kb_id = '"+scopeAllowedKB+"')")
+ mustContainScope(t, got, "(SELECT doc_id FROM t WHERE y=1)")
+}
+
+// TestAddKBFilterIgnoresKeywordsInsideLiterals checks a quoted keyword cannot
+// be mistaken for the real clause.
+func TestAddKBFilterIgnoresKeywordsInsideLiterals(t *testing.T) {
+ got, ok := addKBFilter("SELECT doc_id FROM t WHERE title = 'where kb_id ='", "elasticsearch",
+ []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ mustContainScope(t, got, "WHERE (title = 'where kb_id =') AND (kb_id = '"+scopeAllowedKB+"')")
+
+ got, ok = addKBFilter("SELECT doc_id FROM t /* where */ WHERE a=1 -- kb_id =", "elasticsearch",
+ []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ // The line comment stays inside the wrapper and is terminated by a newline
+ // so the conjoined filter is not swallowed by it.
+ if !strings.Contains(got, "WHERE (a=1 -- kb_id =\n) AND (kb_id = '"+scopeAllowedKB+"')") {
+ t.Fatalf("trailing line comment must stay inside the wrapper, got:\n%s", got)
+ }
+}
+
+// TestAddKBFilterMultiKB checks multiple authorized datasets are ORed inside
+// the conjoined filter.
+func TestAddKBFilterMultiKB(t *testing.T) {
+ got, ok := addKBFilter("SELECT doc_id FROM t WHERE title='x'", "elasticsearch",
+ []*entity.Knowledgebase{{ID: scopeAllowedKB}, {ID: scopeVictimKB}})
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ mustContainScope(t, got, "AND ((kb_id = '"+scopeAllowedKB+"' OR kb_id = '"+scopeVictimKB+"'))")
+}
+
+// TestAddKBFilterRejectsUnscopableShapes checks fail-closed behavior: shapes
+// where the injected predicate cannot cover every arm must be refused.
+func TestAddKBFilterRejectsUnscopableShapes(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ for _, sql := range []string{
+ "SELECT doc_id FROM t1 UNION SELECT doc_id FROM t2",
+ "SELECT doc_id FROM t1 INTERSECT SELECT doc_id FROM t2",
+ "SELECT doc_id FROM t1 EXCEPT SELECT doc_id FROM t2",
+ "SELECT doc_id FROM t WHERE title = 'unterminated",
+ "SELECT doc_id FROM t WHERE a=1 /* unterminated",
+ // Unbalanced parentheses would let the predicate escape the wrapping
+ // parens: `WHERE (1=1) OR (1=1) AND (...)`, which is true for every row.
+ "SELECT doc_id FROM t WHERE 1=1) OR (1=1",
+ "SELECT doc_id FROM t WHERE (1=1",
+ "SELECT doc_id FROM t WHERE ((a=1)",
+ } {
+ if got, ok := addKBFilter(sql, "elasticsearch", kbs); ok {
+ t.Fatalf("SECURITY: unscopable SQL was accepted: %q -> %q", sql, got)
+ }
+ }
+}
+
+func TestAddKBFilterRejectsAmbiguousBackslashQuotes(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ // The newline ends the server's -- comment. If the scanner mistakes \' for
+ // an escaped quote, its wrapper makes the first OR arm unscoped.
+ sql := "SELECT doc_id FROM t WHERE a = 'x\\' OR 1=1) OR (1=1 --'\n AND b = 'y'"
+ for _, engineName := range []string{"elasticsearch", "opensearch", "vastbase", "serenedb"} {
+ if got, ok := addKBFilter(sql, engineName, kbs); ok {
+ t.Fatalf("SECURITY: %s accepted ambiguous quote: %q", engineName, got)
+ }
+ }
+ for _, engineName := range []string{"oceanbase", "seekdb"} {
+ for _, sql := range []string{
+ "SELECT doc_id FROM t WHERE a = 'x\\'y'",
+ "SELECT doc_id FROM t WHERE a = \"x\\\"y\"",
+ "SELECT doc_id FROM t WHERE a = 'x\\' OR 1=1) OR (1=1 -- '\n AND b = 'y'",
+ } {
+ if got, ok := addKBFilter(sql, engineName, kbs); ok {
+ t.Fatalf("%s accepted quote with mode-dependent escape: %q", engineName, got)
+ }
+ }
+ }
+
+ // PostgreSQL E-strings have explicit escape semantics. Doubled quotes have
+ // the same boundary with or without MySQL's NO_BACKSLASH_ESCAPES mode.
+ for _, tc := range []struct {
+ engineName string
+ sql string
+ }{
+ {"vastbase", "SELECT doc_id FROM t WHERE a = E'x\\'y'"},
+ {"serenedb", "SELECT doc_id FROM t WHERE a = E'x\\'y'"},
+ {"oceanbase", "SELECT doc_id FROM t WHERE a = 'x''y'"},
+ {"seekdb", "SELECT doc_id FROM t WHERE a = 'x''y'"},
+ } {
+ got, ok := addKBFilter(tc.sql, tc.engineName, kbs)
+ if !ok {
+ t.Fatalf("%s rejected valid quoted literal: %q", tc.engineName, got)
+ }
+ mustContainScope(t, got, "AND (kb_id = '"+scopeAllowedKB+"')")
+ }
+}
+
+// TestAddKBFilterRejectsMySQLCommentSyntax checks the comment syntaxes OceanBase
+// inherits from MySQL, which the "--"/"/* */" scan does not cover:
+// - "#" runs to end of line and would swallow the appended filter.
+// - "/*! ... */" is executable, so it can hide a set operation from the scan.
+//
+// Both are refused for the OceanBase family and left alone for other engines.
+func TestAddKBFilterRejectsMySQLCommentSyntax(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ shapes := []string{
+ "SELECT doc_id FROM t # note",
+ "SELECT doc_id FROM t WHERE a=1 # note",
+ "SELECT doc_id FROM t /*! UNION SELECT doc_id FROM t */",
+ "SELECT doc_id FROM t WHERE a=1 /*!50000 OR 1=1 */",
+ "SELECT doc_id FROM t WHERE 1--1) OR (1=1",
+ "SELECT doc_id FROM t WHERE 1--",
+ }
+ for _, engineName := range []string{"oceanbase", "seekdb"} {
+ for _, sql := range shapes {
+ if got, ok := addKBFilter(sql, engineName, kbs); ok {
+ t.Fatalf("SECURITY: %s accepted MySQL-only comment syntax: %q -> %q", engineName, sql, got)
+ }
+ }
+ }
+ if got, ok := addKBFilter("SELECT doc_id FROM t WHERE a=1 -- note", "oceanbase", kbs); !ok {
+ t.Fatalf("oceanbase rejected a valid line comment: %q", got)
+ }
+
+ // PostgreSQL-family engines treat "#" as an operator, so the same text is
+ // not a comment there and must keep working.
+ if got, ok := addKBFilter("SELECT doc_id FROM t WHERE a # b", "vastbase", kbs); !ok {
+ t.Fatalf("vastbase rejected '#' operator: %q", got)
+ }
+}
+
+// TestAddKBFilterSkipsDollarQuotedLiterals checks PostgreSQL dollar-quoted
+// strings are treated as literals. Otherwise a clause keyword inside one is
+// mistaken for the end of the predicate and the injected filter lands inside
+// the string, where the server treats it as data.
+func TestAddKBFilterSkipsDollarQuotedLiterals(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+
+ // The bypass from the review. With $$ recognised as a literal, the text
+ // inside is data, so the trailing ")" leaves an unbalanced parenthesis and
+ // the statement is refused instead of being rewritten into valid SQL.
+ if got, ok := addKBFilter("SELECT doc_id FROM t WHERE title = $$ORDER BY($$) OR 1=1", "vastbase", kbs); ok {
+ t.Fatalf("SECURITY: unbalanced statement made valid by the rewrite: %q", got)
+ }
+
+ // The valid variant must be scoped, not split: the clause keyword inside
+ // the literal must not be taken as the end of the predicate.
+ got, ok := addKBFilter("SELECT doc_id FROM t WHERE title = $$ORDER BY$$ OR 1=1", "vastbase", kbs)
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ if !strings.Contains(got, "WHERE (title = $$ORDER BY$$ OR 1=1) AND (kb_id = '"+scopeAllowedKB+"')") {
+ t.Fatalf("dollar-quoted literal was split by the wrapper, got:\n%s", got)
+ }
+
+ // Tagged dollar quotes.
+ got, ok = addKBFilter("SELECT doc_id FROM t WHERE title = $tag$ ORDER BY $tag$ OR 1=1", "serenedb", kbs)
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ if !strings.Contains(got, "WHERE (title = $tag$ ORDER BY $tag$ OR 1=1) AND (kb_id = '"+scopeAllowedKB+"')") {
+ t.Fatalf("tagged dollar quote was split, got:\n%s", got)
+ }
+
+ // An unterminated dollar quote must fail closed.
+ if got, ok := addKBFilter("SELECT doc_id FROM t WHERE title = $$unterminated", "vastbase", kbs); ok {
+ t.Fatalf("SECURITY: unterminated dollar quote accepted: %q", got)
+ }
+
+ // "$1" is a bind placeholder, not a dollar quote; the scan must not treat
+ // it as one and swallow the rest of the statement.
+ if got, ok := addKBFilter("SELECT doc_id FROM t WHERE a = $1 AND b = 2", "vastbase", kbs); !ok {
+ t.Fatalf("placeholder rejected: %q", got)
+ } else if !strings.Contains(got, "AND (kb_id = '"+scopeAllowedKB+"')") {
+ t.Fatalf("filter missing for placeholder query:\n%s", got)
+ }
+}
+
+func TestAddKBFilterNonASCIIDollarQuoteTags(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ for _, engineName := range []string{"vastbase", "serenedb"} {
+ if got, ok := addKBFilter("SELECT doc_id FROM t WHERE title = $é$ ORDER BY ($é$ OR 1=1)", engineName, kbs); ok {
+ t.Fatalf("%s accepted an unbalanced predicate: %q", engineName, got)
+ }
+ got, ok := addKBFilter("SELECT doc_id FROM t WHERE title = $é$ ORDER BY ($é$ OR 1=1", engineName, kbs)
+ if !ok {
+ t.Fatalf("%s rejected a balanced dollar quote: %q", engineName, got)
+ }
+ mustContainScope(t, got, "WHERE (title = $é$ ORDER BY ($é$ OR 1=1) AND (kb_id = '"+scopeAllowedKB+"')")
+ }
+}
+
+func TestAddKBFilterNonASCIIIdentifiersDoNotOpenDollarQuotes(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ sql := "SELECT doc_id FROM t WHERE é$x$ = 1 OR 1=1) OR (1=1 AND b$x$ = 2"
+ for _, engineName := range []string{"vastbase", "serenedb"} {
+ if got, ok := addKBFilter(sql, engineName, kbs); ok {
+ t.Fatalf("%s accepted an unbalanced predicate: %q", engineName, got)
+ }
+ }
+}
+
+// The engine removes backticks after filtering, which can turn a quoted
+// identifier into SQL syntax that escapes the added predicate.
+func TestAddKBFilterRejectsBackticks(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ for _, engineName := range []string{"elasticsearch", "opensearch", "oceanbase", "seekdb"} {
+ if got, ok := addKBFilter("SELECT doc_id FROM t WHERE `order by` = 1", engineName, kbs); ok {
+ t.Fatalf("%s accepted backticks removed by its engine: %q", engineName, got)
+ }
+ }
+ if got, ok := addKBFilter("SELECT doc_id FROM t WHERE title = 'a`b'", "vastbase", kbs); !ok {
+ t.Fatalf("vastbase rejected a backtick inside a string: %q", got)
+ }
+}
+
+func TestAddKBFilterScopesAfterElasticsearchPercentRemoval(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ for _, engineName := range []string{"elasticsearch", "opensearch"} {
+ got, ok := addKBFilter("SELECT doc_id FROM t -%- ORDER BY doc_id", engineName, kbs)
+ if !ok || !strings.Contains(got, "-- ORDER BY doc_id\nWHERE kb_id = '"+scopeAllowedKB+"'") {
+ t.Fatalf("%s line comment did not keep the filter live: %q", engineName, got)
+ }
+ got, ok = addKBFilter("SELECT doc_id FROM t /%* ORDER BY doc_id *%/", engineName, kbs)
+ if !ok || !strings.Contains(got, "/* ORDER BY doc_id */ WHERE kb_id = '"+scopeAllowedKB+"'") {
+ t.Fatalf("%s block comment did not keep the filter live: %q", engineName, got)
+ }
+ }
+}
+
+// A nested block comment can keep the added WHERE inside the server's comment
+// even though the scanner sees ORDER BY after the inner comment closes.
+func TestAddKBFilterRejectsNestedBlockCommentsOutsidePostgres(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ sql := "SELECT doc_id FROM t /* outer /* inner */ ORDER BY x */"
+ for _, engineName := range []string{"elasticsearch", "opensearch", "oceanbase", "seekdb"} {
+ if got, ok := addKBFilter(sql, engineName, kbs); ok {
+ t.Fatalf("%s accepted nested block comment: %q", engineName, got)
+ }
+ }
+}
+
+// TestAddKBFilterSupportsNestedBlockComments checks PostgreSQL nested block
+// comments are consumed exactly; the old scan stopped at the first "*/" and
+// then read the comment body as SQL.
+func TestAddKBFilterSupportsNestedBlockComments(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ got, ok := addKBFilter("SELECT doc_id FROM t /* a /* b */ c */ WHERE x=1", "vastbase", kbs)
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ if !strings.Contains(got, "WHERE (x=1) AND (kb_id = '"+scopeAllowedKB+"')") {
+ t.Fatalf("nested block comment mishandled, got:\n%s", got)
+ }
+}
+
+// TestAddKBFilterWrapsPredicateAcrossLineComments checks a `--` comment cannot
+// cut the predicate in two. Everything from the model's predicate must stay
+// inside the wrapper, including any OR that follows on the next line, so the
+// authorization filter (ANDed outside) always constrains the whole predicate.
+func TestAddKBFilterWrapsPredicateAcrossLineComments(t *testing.T) {
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+
+ // The review bypass: OR after the comment used to escape the wrapper.
+ // Wrapped output is `WHERE (1=1 -- note\nOR 1=1) AND (kb_id = 'A')`, so the
+ // trailing OR can no longer make the predicate always true.
+ got, ok := addKBFilter("SELECT doc_id FROM t WHERE 1=1 -- note\nOR 1=1", "elasticsearch", kbs)
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ if !strings.Contains(got, "-- note\nOR 1=1) AND (kb_id = '"+scopeAllowedKB+"')") {
+ t.Fatalf("OR after the comment escaped the wrapper, got:\n%s", got)
+ }
+
+ // No WHERE: a comment before ORDER BY must not swallow the injected filter.
+ got, ok = addKBFilter("SELECT doc_id FROM t -- note\nORDER BY x", "elasticsearch", kbs)
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ if !strings.Contains(got, "WHERE kb_id = '"+scopeAllowedKB+"'") {
+ t.Fatalf("filter missing, got:\n%s", got)
+ }
+ if strings.Index(got, "WHERE kb_id") < strings.Index(got, "-- note") {
+ t.Fatalf("filter injected inside the comment, got:\n%s", got)
+ }
+
+ // Comment between the predicate and LIMIT: the filter must still apply.
+ got, ok = addKBFilter("SELECT doc_id FROM t WHERE 1=1 --\nLIMIT 5", "elasticsearch", kbs)
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ if !strings.Contains(got, ") AND (kb_id = '"+scopeAllowedKB+"')") {
+ t.Fatalf("filter not conjoined after the wrapped predicate, got:\n%s", got)
+ }
+ if strings.Index(got, scopeAllowedKB) < strings.Index(got, "--") {
+ t.Fatalf("filter landed inside the comment, got:\n%s", got)
+ }
+
+ // A block comment in the middle of the predicate stays inside the wrapper.
+ got, ok = addKBFilter("SELECT doc_id FROM t WHERE a=1 /* x */ OR b=2 GROUP BY g", "elasticsearch", kbs)
+ if !ok {
+ t.Fatalf("expected ok=true")
+ }
+ if !strings.Contains(got, "WHERE (a=1 /* x */ OR b=2) AND (kb_id = '"+scopeAllowedKB+"') GROUP BY g") {
+ t.Fatalf("block comment handling changed, got:\n%s", got)
+ }
+}
+
+// TestAddKBFilterRejectsInvalidKBID checks a non-UUID id fails closed instead
+// of producing an unfiltered query.
+func TestAddKBFilterRejectsInvalidKBID(t *testing.T) {
+ if got, ok := addKBFilter("SELECT doc_id FROM t", "elasticsearch",
+ []*entity.Knowledgebase{{ID: "kb-private"}}); ok {
+ t.Fatalf("SECURITY: invalid kb id accepted: %q", got)
+ }
+}
+
+// TestAddKBFilterInfinityNoop pins that Infinity keeps relying on its
+// per-dataset table name, as before.
+func TestAddKBFilterInfinityNoop(t *testing.T) {
+ const sql = "SELECT doc_id FROM ragflow_t_kb"
+ got, ok := addKBFilter(sql, "infinity", []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok || got != sql {
+ t.Fatalf("infinity must stay a no-op, got %q ok=%v", got, ok)
+ }
+}
+
+// TestBuildChunkFetchSQLInheritsScope checks the aggregate secondary fetch
+// builds on the already-scoped SQL, so it does not re-introduce a
+// model-written kb_id. useSQL passes the filtered sqlText here.
+func TestBuildChunkFetchSQLInheritsScope(t *testing.T) {
+ scoped, ok := addKBFilter(
+ "SELECT COUNT(*) FROM ragflow_tenant_a WHERE kb_id = '"+scopeVictimKB+"'",
+ "elasticsearch", []*entity.Knowledgebase{{ID: scopeAllowedKB}})
+ if !ok {
+ t.Fatalf("expected the input to be scoped")
+ }
+ got, found := buildChunkFetchSQL(scoped, "ragflow_tenant_a", "docnm_kwd", false, "elasticsearch")
+ if !found {
+ t.Fatalf("expected a secondary fetch SQL")
+ }
+ mustContainScope(t, got, "kb_id = '"+scopeAllowedKB+"'")
+ if !strings.Contains(got, "AND (kb_id = '"+scopeAllowedKB+"')") {
+ t.Fatalf("secondary fetch lost the authorized filter:\n%s", got)
+ }
+}
+
+func TestBuildChunkFetchSQLTopLevelPredicate(t *testing.T) {
+ for _, tc := range []struct {
+ name, sql, want string
+ }{
+ {"quoted alias", `SELECT COUNT(*) AS "where 1=1 --", COUNT(*) AS n FROM t WHERE kb_id = 'allowed'`, "select doc_id, docnm_kwd from t where kb_id = 'allowed' limit 20"},
+ {"alias without predicate", `SELECT COUNT(*) AS "where 1=1 --", COUNT(*) AS n FROM t`, ""},
+ {"nested predicate only", "SELECT COUNT(*) FROM (SELECT * FROM t WHERE x = 1) s", ""},
+ {"nested and outer predicates", "SELECT COUNT(*) FROM (SELECT * FROM t WHERE x = 1) s WHERE x = 2", "select doc_id, docnm_kwd from t where x = 2 limit 20"},
+ {"keywords in literal", "SELECT COUNT(*) FROM t WHERE title = 'group by order by limit' ORDER BY n", "select doc_id, docnm_kwd from t where title = 'group by order by limit' limit 20"},
+ {"multiline predicate", "SELECT COUNT(*) FROM t WHERE x = 1 -- note\n OR x = 2 GROUP BY x HAVING COUNT(*) > 1", "select doc_id, docnm_kwd from t where x = 1 -- note\n OR x = 2 limit 20"},
+ {"trailing comment", "SELECT COUNT(*) FROM t WHERE x = 1 -- note", "select doc_id, docnm_kwd from t where x = 1 -- note\nlimit 20"},
+ {"having and offset", "SELECT COUNT(*) FROM t WHERE x = 1 GROUP BY x HAVING COUNT(*) > 1 OFFSET 5", "select doc_id, docnm_kwd from t where x = 1 limit 20"},
+ {"unterminated literal", "SELECT COUNT(*) FROM t WHERE title = 'open", ""},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got, ok := buildChunkFetchSQL(tc.sql, "t", "docnm_kwd", false, "elasticsearch")
+ if got != tc.want || ok != (tc.want != "") {
+ t.Fatalf("buildChunkFetchSQL() = %q, %v; want %q, %v", got, ok, tc.want, tc.want != "")
+ }
+ })
+ }
+}
+
+func TestAggregateReferencesStayInAuthorizedKB(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = sqlDB.Close() })
+ sqlDB.SetMaxOpenConns(1)
+ if err := db.Exec("CREATE TABLE t (doc_id TEXT, docnm_kwd TEXT, kb_id TEXT)").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO t VALUES (?, ?, ?), (?, ?, ?)",
+ "allowed-doc", "Allowed", scopeAllowedKB, "victim-doc", "Victim", scopeVictimKB).Error; err != nil {
+ t.Fatal(err)
+ }
+ docEngine := &sqlFakeEngine{
+ engineType: "elasticsearch",
+ runSQL: func(ctx context.Context, sqlText string) ([]map[string]interface{}, error) {
+ var rows []map[string]interface{}
+ err := db.WithContext(ctx).Raw(sqlText).Scan(&rows).Error
+ return rows, err
+ },
+ }
+ s := &ChatPipelineService{}
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
+ t.Run("quoted alias in aggregate answer", func(t *testing.T) {
+ scoped, ok := addKBFilter(`SELECT COUNT(*) AS "where 1=1 --", COUNT(*) AS n FROM t`, "elasticsearch", kbs)
+ if !ok {
+ t.Fatal("aggregate query could not be scoped")
+ }
+ rows, err := docEngine.RunSQL(t.Context(), scoped, "json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, ref := s.buildSQLReference(t.Context(), docEngine, "t", scoped, rows, "", "elasticsearch", kbs, nil, true)
+ chunks, _ := ref["chunks"].([]map[string]interface{})
+ if len(chunks) != 1 || chunks[0]["document_id"] != "allowed-doc" {
+ t.Fatalf("references = %#v, want only allowed-doc", ref)
+ }
+ })
+ t.Run("secondary query reapplies scope", func(t *testing.T) {
+ chunks, _ := s.fetchAggregateChunks(t.Context(), docEngine, "t", "SELECT COUNT(*) FROM t WHERE 1=1", "docnm_kwd", []string{scopeAllowedKB})
+ if len(chunks) != 1 || chunks[0]["doc_id"] != "allowed-doc" {
+ t.Fatalf("chunks = %#v, want only allowed-doc", chunks)
+ }
+ })
+ t.Run("multi KB secondary query", func(t *testing.T) {
+ chunks, _ := s.fetchAggregateChunks(t.Context(), docEngine, "t", "SELECT COUNT(*) FROM t WHERE doc_id = 'allowed-doc'", "docnm_kwd", []string{scopeAllowedKB, scopeVictimKB})
+ if len(chunks) != 1 || chunks[0]["doc_id"] != "allowed-doc" || chunks[0]["kb_id"] != scopeAllowedKB {
+ t.Fatalf("chunks = %#v, want allowed-doc with its KB", chunks)
+ }
+ })
+ for _, kbIDs := range [][]string{nil, {"invalid"}} {
+ t.Run("unscopable query skips execution", func(t *testing.T) {
+ var calls []string
+ docEngine := &sqlFakeEngine{engineType: "elasticsearch", sqlCalls: &calls}
+ kbs := make([]*entity.Knowledgebase, len(kbIDs))
+ for i, id := range kbIDs {
+ kbs[i] = &entity.Knowledgebase{ID: id}
+ }
+ answer, ref := s.buildSQLReference(t.Context(), docEngine, "t", "SELECT COUNT(*) AS n, 'total' AS label FROM t WHERE 1=1",
+ []map[string]interface{}{{"n": 42, "label": "total"}}, "", "elasticsearch", kbs, nil, true)
+ chunks, _ := ref["chunks"].([]map[string]interface{})
+ if len(calls) != 0 || len(chunks) != 0 || !strings.Contains(answer, "42") {
+ t.Fatalf("calls = %v, answer = %q, references = %#v; want aggregate answer without secondary execution", calls, answer, ref)
+ }
+ })
+ }
+}
+
+func TestBuildChunkFetchSQLDialect(t *testing.T) {
+ for _, tc := range []struct {
+ engineName, sql, want string
+ }{
+ {"vastbase", "SELECT COUNT(*) FROM t WHERE title = $tag$where limit$tag$ ORDER BY n", "select doc_id, docnm_kwd from t where title = $tag$where limit$tag$ limit 20"},
+ {"serenedb", `SELECT COUNT(*) FROM t WHERE title = E'it\'s where limit' ORDER BY n`, `select doc_id, docnm_kwd from t where title = E'it\'s where limit' limit 20`},
+ {"oceanbase", "SELECT COUNT(*) FROM t WHERE x = 1 # comment", ""},
+ {"infinity", "SELECT COUNT(*) FROM t WHERE x = 1", "select doc_id, docnm from t where x = 1 limit 20"},
+ } {
+ t.Run(tc.engineName, func(t *testing.T) {
+ expectedCol := expectedDocNameColumn(tc.engineName)
+ got, ok := buildChunkFetchSQL(tc.sql, "t", expectedCol, false, tc.engineName)
+ if got != tc.want || ok != (tc.want != "") {
+ t.Fatalf("buildChunkFetchSQL() = %q, %v; want %q, %v", got, ok, tc.want, tc.want != "")
+ }
+ })
+ }
+}
diff --git a/internal/service/chat_pipeline_test.go b/internal/service/chat_pipeline_test.go
index bda6a93ace..1ea228b2aa 100644
--- a/internal/service/chat_pipeline_test.go
+++ b/internal/service/chat_pipeline_test.go
@@ -1207,11 +1207,8 @@ func TestBuildChunkFetchSQL(t *testing.T) {
wantFound: true,
},
{
- // Python's regex is non-greedy, so WHERE-clause extraction
- // stops at the first occurrence of ORDER BY / LIMIT / GROUP BY.
- // Python's subsequent SQL string is then
- // "select doc_id, ... from t where {where}", which DROPS
- // the order by / limit suffixes. Go matches this behavior.
+ // The secondary fetch drops the aggregate query's trailing clauses
+ // and applies its own limit.
name: "WHERE + ORDER BY + LIMIT 5 (suffixes dropped, no extra limit)",
sql: "select * from t where x = 1 order by y limit 5",
multiKB: false,
@@ -1248,7 +1245,7 @@ func TestBuildChunkFetchSQL(t *testing.T) {
if tc.name == "Infinity expectedCol is docnm (not _kwd)" {
expectedCol = "docnm"
}
- gotSQL, gotFound := buildChunkFetchSQL(tc.sql, "t", expectedCol, tc.multiKB)
+ gotSQL, gotFound := buildChunkFetchSQL(tc.sql, "t", expectedCol, tc.multiKB, "elasticsearch")
if gotFound != tc.wantFound {
t.Errorf("found = %v, want %v", gotFound, tc.wantFound)
}
@@ -1335,13 +1332,13 @@ type sqlFakeEngine struct {
sqlCalls *[]string
rowsBySQL map[string][]map[string]interface{}
errBySQL map[string]error
- runSQL func(ctx context.Context, table, sqlText string, kbIDs []string) ([]map[string]interface{}, error)
+ runSQL func(ctx context.Context, sqlText string) ([]map[string]interface{}, error)
}
func (f *sqlFakeEngine) GetType() string { return f.engineType }
-func (f *sqlFakeEngine) RunSQL(ctx context.Context, table, sqlText string, kbIDs []string, format string) ([]map[string]interface{}, error) {
+func (f *sqlFakeEngine) RunSQL(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error) {
if f.runSQL != nil {
- return f.runSQL(ctx, table, sqlText, kbIDs)
+ return f.runSQL(ctx, sqlText)
}
if f.sqlCalls != nil {
*f.sqlCalls = append(*f.sqlCalls, sqlText)
@@ -1378,7 +1375,7 @@ func TestFetchAggregateChunks_SkipsInfinityMultiKB(t *testing.T) {
// TestFetchAggregateChunks_SingleKBSuccess verifies the secondary fetch
// path populates chunks and doc_aggs correctly.
func TestFetchAggregateChunks_SingleKBSuccess(t *testing.T) {
- chunksSQL := "select doc_id, docnm_kwd from t where x = 1 limit 20"
+ chunksSQL := "select doc_id, docnm_kwd from t WHERE (x = 1) AND (kb_id = '" + scopeAllowedKB + "') limit 20"
sqlEngine := &sqlFakeEngine{
engineType: "elasticsearch",
rowsBySQL: map[string][]map[string]interface{}{
@@ -1393,7 +1390,7 @@ func TestFetchAggregateChunks_SingleKBSuccess(t *testing.T) {
chunks, docAggs := s.fetchAggregateChunks(
t.Context(), sqlEngine, "t",
"select count(*) from t where x = 1",
- "docnm_kwd", []string{"kb_a"},
+ "docnm_kwd", []string{scopeAllowedKB},
)
if len(chunks) != 3 {
t.Fatalf("chunks len = %d, want 3", len(chunks))
@@ -1411,8 +1408,8 @@ func TestFetchAggregateChunks_SingleKBSuccess(t *testing.T) {
}
// Single-kb: each chunk gets kb_id from the dialog's kb list.
for i, c := range chunks {
- if c["kb_id"] != "kb_a" {
- t.Errorf("chunks[%d].kb_id = %v, want kb_a", i, c["kb_id"])
+ if c["kb_id"] != scopeAllowedKB {
+ t.Errorf("chunks[%d].kb_id = %v, want %s", i, c["kb_id"], scopeAllowedKB)
}
}
}
@@ -1434,9 +1431,11 @@ func TestFetchAggregateChunks_NoWhereClause(t *testing.T) {
// TestFetchAggregateChunks_RunSQLError verifies graceful failure.
func TestFetchAggregateChunks_RunSQLError(t *testing.T) {
+ called := false
sqlEngine := &sqlFakeEngine{
engineType: "elasticsearch",
- runSQL: func(ctx context.Context, table, sqlText string, kbIDs []string) ([]map[string]interface{}, error) {
+ runSQL: func(ctx context.Context, sqlText string) ([]map[string]interface{}, error) {
+ called = true
return nil, fmt.Errorf("engine boom")
},
}
@@ -1444,8 +1443,11 @@ func TestFetchAggregateChunks_RunSQLError(t *testing.T) {
chunks, docAggs := s.fetchAggregateChunks(
t.Context(), sqlEngine, "t",
"select count(*) from t where x = 1",
- "docnm_kwd", []string{"kb_a"},
+ "docnm_kwd", []string{scopeAllowedKB},
)
+ if !called {
+ t.Fatal("expected the secondary query to execute")
+ }
if chunks != nil || docAggs != nil {
t.Errorf("expected nil on RunSQL error, got %v / %v", chunks, docAggs)
}
@@ -1523,7 +1525,7 @@ func TestBuildSQLReference_AggregateMissingSourceColumnsSecondaryFetch(t *testin
rows := []map[string]interface{}{
{"count": 42.0, "label": "total"},
}
- chunksSQL := "select doc_id, docnm_kwd from t where x = 1 limit 20"
+ chunksSQL := "select doc_id, docnm_kwd from t WHERE (x = 1) AND (kb_id = '" + scopeAllowedKB + "') limit 20"
sqlEngine := &sqlFakeEngine{
engineType: "elasticsearch",
rowsBySQL: map[string][]map[string]interface{}{
@@ -1532,7 +1534,7 @@ func TestBuildSQLReference_AggregateMissingSourceColumnsSecondaryFetch(t *testin
},
},
}
- kbs := []*entity.Knowledgebase{{ID: "kb_a"}}
+ kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
s := &ChatPipelineService{}
ans, ref := s.buildSQLReference(
t.Context(), sqlEngine, "t",
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 5b8e732d9f..f6dacfa686 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -2237,7 +2237,7 @@ func (e *parseTestDocEngine) GetHighlight([]map[string]interface{}, []string, st
return nil
}
-func (e *parseTestDocEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
+func (e *parseTestDocEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
@@ -2645,7 +2645,7 @@ func (m *switchChunksEngineMock) GetAggregation([]map[string]interface{}, string
func (m *switchChunksEngineMock) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
return nil
}
-func (m *switchChunksEngineMock) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
+func (m *switchChunksEngineMock) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (m *switchChunksEngineMock) GetChunkIDs([]map[string]interface{}) []string { return nil }
diff --git a/internal/service/dataset/fake_doc_engine_test.go b/internal/service/dataset/fake_doc_engine_test.go
index 6c98f3de85..71d338f2f7 100644
--- a/internal/service/dataset/fake_doc_engine_test.go
+++ b/internal/service/dataset/fake_doc_engine_test.go
@@ -67,7 +67,7 @@ func (fakeChatDocEngine) GetAggregation([]map[string]interface{}, string) []map[
func (fakeChatDocEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
return nil
}
-func (fakeChatDocEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
+func (fakeChatDocEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (fakeChatDocEngine) GetChunkIDs([]map[string]interface{}) []string { return nil }
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index 225c3e97e8..f9a1959a9f 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -231,7 +231,7 @@ func (fakeChatDocEngine) GetAggregation([]map[string]interface{}, string) []map[
func (fakeChatDocEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
return nil
}
-func (fakeChatDocEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
+func (fakeChatDocEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (fakeChatDocEngine) GetChunkIDs([]map[string]interface{}) []string {
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index fc17df3cdc..67cb6152fc 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -176,7 +176,7 @@ func (m *memNavEngine) GetAggregation([]map[string]interface{}, string) []map[st
func (m *memNavEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
return nil
}
-func (m *memNavEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
+func (m *memNavEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (m *memNavEngine) GetChunkIDs([]map[string]interface{}) []string { return nil }
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index b35f71b7a9..1ee01de5c4 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -909,7 +909,7 @@ func (e *retrievalCountEngine) GetHighlight([]map[string]interface{}, []string,
e.highlightCalls++
return e.highlights
}
-func (e *retrievalCountEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
+func (e *retrievalCountEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (e *retrievalCountEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []string, []map[string]interface{}, string) []string {
diff --git a/internal/service/test_helpers_test.go b/internal/service/test_helpers_test.go
index 04fb5cb184..03957c1b31 100644
--- a/internal/service/test_helpers_test.go
+++ b/internal/service/test_helpers_test.go
@@ -205,7 +205,7 @@ func (fakeChatDocEngine) GetAggregation([]map[string]interface{}, string) []map[
func (fakeChatDocEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
return nil
}
-func (fakeChatDocEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
+func (fakeChatDocEngine) RunSQL(context.Context, string, string) ([]map[string]interface{}, error) {
return nil, nil
}
func (fakeChatDocEngine) GetChunkIDs([]map[string]interface{}) []string { return nil }