Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Enforce CLI SQL row limits and report truncation https://github.com/siyuan-note/siyuan/issues/20376

Daniele3616307dfe8
diff --git a/kernel/cli/cmd/sql.go b/kernel/cli/cmd/sql.go
index 992ff95b03..72ac357b93 100644
--- a/kernel/cli/cmd/sql.go
+++ b/kernel/cli/cmd/sql.go
@@ -45,7 +45,7 @@ var sqlCmd = &cobra.Command{
if err := sql.CheckReadonlyStatement(stmt); err != nil {
return err
}
- rows, err := sql.Query(stmt, limit)
+ rows, info, err := sql.QueryWithRowLimitInfo(stmt, limit)
if err != nil {
return err
}
@@ -57,6 +57,9 @@ var sqlCmd = &cobra.Command{
default:
printSQLResult(rows)
}
+ if info.Truncated {
+ fmt.Fprintf(cmd.ErrOrStderr(), "Results truncated to %d rows; use LIMIT and OFFSET to paginate.\n", info.Limit)
+ }
return nil
},
}
diff --git a/kernel/cli/cmd/sql_test.go b/kernel/cli/cmd/sql_test.go
new file mode 100644
index 0000000000..8c6b5d7a25
--- /dev/null
+++ b/kernel/cli/cmd/sql_test.go
@@ -0,0 +1,89 @@
+//go:build fts5
+
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/sql"
+ "github.com/siyuan-note/siyuan/kernel/util"
+ "github.com/spf13/cobra"
+)
+
+func TestSQLCommandRowLimit(t *testing.T) {
+ const childEnv = "SIYUAN_TEST_CLI_SQL_LIMIT"
+ if os.Getenv(childEnv) != "1" {
+ cmd := exec.Command(os.Args[0], "-test.run=^TestSQLCommandRowLimit$", "-test.timeout=30s")
+ cmd.Env = append(os.Environ(), childEnv+"=1")
+ if output, err := cmd.CombinedOutput(); err != nil {
+ t.Fatalf("SQL command regression failed: %v\n%s", err, output)
+ }
+ return
+ }
+ root := t.TempDir()
+ util.DBPath = filepath.Join(root, "siyuan.db")
+ util.HistoryDBPath = filepath.Join(root, "history.db")
+ util.AssetContentDBPath = filepath.Join(root, "asset_content.db")
+ util.BlockTreeDBPath = filepath.Join(root, "blocktree.db")
+ util.QueueDir = filepath.Join(root, "queue")
+ sql.InitDatabase(true)
+ sql.InitHistoryDatabase(true)
+ sql.InitAssetContentDatabase(true)
+ t.Cleanup(sql.CloseDatabase)
+ const query = "WITH RECURSIVE items(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM items WHERE n<105) SELECT n FROM items"
+ for _, tc := range []struct {
+ name, statement, format string
+ limit, count int
+ truncated bool
+ }{
+ {"default JSON", query, "json", 100, 100, true},
+ {"explicit SQL limit", query + " LIMIT 5", "json", 3, 3, true},
+ {"equal limit", query + " LIMIT 3", "json", 3, 3, false},
+ {"small result", query + " LIMIT 2", "json", 3, 2, false},
+ {"table", query, "table", 3, 3, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ command := &cobra.Command{}
+ command.Flags().Int("limit", tc.limit, "")
+ var warning bytes.Buffer
+ command.SetErr(&warning)
+ output, err := os.CreateTemp(root, "output-")
+ if err != nil {
+ t.Fatal(err)
+ }
+ previousStdout := os.Stdout
+ os.Stdout = output
+ defer func() { os.Stdout = previousStdout }()
+ outputFormat = tc.format
+ err = sqlCmd.RunE(command, []string{tc.statement})
+ os.Stdout = previousStdout
+ if closeErr := output.Close(); closeErr != nil {
+ t.Fatal(closeErr)
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err := os.ReadFile(output.Name())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if tc.format == "json" {
+ var rows []json.RawMessage
+ if err = json.Unmarshal(data, &rows); err != nil || len(rows) != tc.count {
+ t.Fatalf("invalid or unbounded JSON: %s %v", data, err)
+ }
+ } else if !strings.Contains(string(data), "3 row(s)") {
+ t.Fatalf("unexpected table summary: %s", data)
+ }
+ if strings.Contains(warning.String(), "truncated") != tc.truncated {
+ t.Fatalf("unexpected truncation warning: %q", warning.String())
+ }
+ })
+ }
+}
diff --git a/kernel/sql/block_query.go b/kernel/sql/block_query.go
index c0cc74f230..2d6263b05d 100644
--- a/kernel/sql/block_query.go
+++ b/kernel/sql/block_query.go
@@ -575,6 +575,10 @@ func queryRawStmt(stmt string, limit int) (ret []map[string]any, err error) {
}
func queryRawStmtWithLimitInfo(stmt string, limit int, info *QueryLimitInfo) (ret []map[string]any, err error) {
+ return queryRawStmtRows(stmt, limit, info, false)
+}
+
+func queryRawStmtRows(stmt string, limit int, info *QueryLimitInfo, enforceLimit bool) (ret []map[string]any, err error) {
rows, err := query(stmt)
if err != nil {
if strings.Contains(err.Error(), "syntax error") {
@@ -616,7 +620,7 @@ func queryRawStmtWithLimitInfo(stmt string, limit int, info *QueryLimitInfo) (re
ret = append(ret, m)
count++
- if noLimit && (limit < count || (info != nil && limit == count)) {
+ if (noLimit || enforceLimit) && count >= limit {
break
}
}
diff --git a/kernel/sql/query_limit.go b/kernel/sql/query_limit.go
index b0a67cc00f..6b4382cc89 100644
--- a/kernel/sql/query_limit.go
+++ b/kernel/sql/query_limit.go
@@ -6,12 +6,33 @@ import (
"strings"
)
-// QueryLimitInfo 描述服务端默认限制;Limit 为零表示使用 SQL 自身的限制。
+// QueryLimitInfo 描述查询行数限制;Limit 为零表示使用 SQL 自身的限制。
type QueryLimitInfo struct {
Limit int `json:"limit"`
Truncated bool `json:"truncated"`
}
+// QueryWithRowLimitInfo 在 SQL 自身限制之外封顶返回行数,并多读取一行判断是否截断。
+// 直接执行原始语句,保留注释、递归查询和解析器不支持的 SQLite 语法。
+func QueryWithRowLimitInfo(stmt string, limit int) (rows []map[string]any, info QueryLimitInfo, err error) {
+ if limit < 1 || limit == math.MaxInt {
+ return nil, info, errors.New("invalid SQL row limit")
+ }
+ info.Limit = limit
+ rows, err = queryRawStmtRows(stmt, limit+1, nil, true)
+ if err != nil {
+ return nil, QueryLimitInfo{}, err
+ }
+ if len(rows) > limit {
+ info.Truncated = true
+ rows = rows[:limit]
+ }
+ if rows == nil {
+ rows = []map[string]any{}
+ }
+ return
+}
+
// QueryWithLimitInfo 在同一次查询中多读取一行,判断默认限制是否截断了结果。
func QueryWithLimitInfo(stmt string, limit int) (rows []map[string]any, info QueryLimitInfo, err error) {
if limit < 1 || limit == math.MaxInt {
diff --git a/kernel/sql/query_limit_test.go b/kernel/sql/query_limit_test.go
index 18524dde10..fe89a279ac 100644
--- a/kernel/sql/query_limit_test.go
+++ b/kernel/sql/query_limit_test.go
@@ -2,9 +2,62 @@ package sql
import (
gosql "database/sql"
+ "math"
"testing"
)
+func TestQueryRowLimit(t *testing.T) {
+ testDB, err := gosql.Open("sqlite3_extended", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ testDB.SetMaxOpenConns(1)
+ t.Cleanup(func() { _ = testDB.Close() })
+ previousDB := db
+ db = testDB
+ t.Cleanup(func() { db = previousDB })
+ if _, err = testDB.Exec("CREATE TABLE items (n INTEGER, content TEXT); INSERT INTO items VALUES (1, 'a'), (2, 'a'), (3, 'a'), (4, 'a'), (5, 'a')"); err != nil {
+ t.Fatal(err)
+ }
+ for _, tc := range []struct {
+ stmt string
+ count int
+ truncated bool
+ }{
+ {"SELECT n FROM items ORDER BY n", 3, true},
+ {"SELECT n FROM items WHERE n <= 3 ORDER BY n", 3, false},
+ {"SELECT n FROM items WHERE n < 0", 0, false},
+ {"SELECT n FROM items ORDER BY n LIMIT 5", 3, true},
+ {"SELECT n FROM items ORDER BY n LIMIT -1", 3, true},
+ {"SELECT n FROM items ORDER BY n LIMIT 2", 2, false},
+ {"SELECT n FROM items ORDER BY n LIMIT 3 OFFSET 3", 2, false},
+ {"SELECT n FROM items UNION SELECT 6 ORDER BY n", 3, true},
+ {"WITH x AS (SELECT n FROM items LIMIT 5) SELECT n FROM x ORDER BY n", 3, true},
+ {"SELECT n FROM items -- comment\nORDER BY n", 3, true},
+ {`SELECT n FROM items WHERE content LIKE '%a%' ESCAPE '\' ORDER BY n`, 3, true},
+ } {
+ rows, info, queryErr := QueryWithRowLimitInfo(tc.stmt, 3)
+ if queryErr != nil || rows == nil || len(rows) != tc.count || info.Limit != 3 || info.Truncated != tc.truncated {
+ t.Fatalf("%s: rows=%v info=%+v err=%v", tc.stmt, rows, info, queryErr)
+ }
+ if tc.count > 0 && rows[0]["n"] != int64(1) && tc.stmt != "SELECT n FROM items ORDER BY n LIMIT 3 OFFSET 3" {
+ t.Fatalf("query order changed: %s: %v", tc.stmt, rows)
+ }
+ }
+ for _, limit := range []int{0, -1, math.MaxInt} {
+ if _, _, err = QueryWithRowLimitInfo("SELECT 1", limit); err == nil {
+ t.Fatalf("invalid limit accepted: %d", limit)
+ }
+ }
+ if _, _, err = QueryWithRowLimitInfo("SELECT abs(-9223372036854775808)", 3); err == nil {
+ t.Fatal("row iteration error was ignored")
+ }
+ rows, err := Query(`SELECT n FROM items WHERE content LIKE '%a%' ESCAPE '\' ORDER BY n`, 3)
+ if err != nil || len(rows) != 3 {
+ t.Fatalf("raw fallback exceeded default limit: %v %v", rows, err)
+ }
+}
+
func TestQueryWithLimitInfo(t *testing.T) {
testDB, err := gosql.Open("sqlite3_extended", ":memory:")
if err != nil {