An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:bug: Enforce CLI SQL row limits and report truncation https://github.com/siyuan-note/siyuan/issues/20376
Daniele3616307dfe8
diff --git a/kernel/cli/cmd/sql.go b/kernel/cli/cmd/sql.goindex 992ff95b03..72ac357b93 100644--- a/kernel/cli/cmd/sql.go+++ b/kernel/cli/cmd/sql.go@@ -45,7 +45,7 @@ var sqlCmd = &cobra.Command{if err := sql.CheckReadonlyStatement(stmt); err != nil {return err}- rows, err := sql.Query(stmt, limit)+ rows, info, err := sql.QueryWithRowLimitInfo(stmt, limit)if err != nil {return err}@@ -57,6 +57,9 @@ var sqlCmd = &cobra.Command{default:printSQLResult(rows)}+ if info.Truncated {+ fmt.Fprintf(cmd.ErrOrStderr(), "Results truncated to %d rows; use LIMIT and OFFSET to paginate.\n", info.Limit)+ }return nil},}diff --git a/kernel/cli/cmd/sql_test.go b/kernel/cli/cmd/sql_test.gonew file mode 100644index 0000000000..8c6b5d7a25--- /dev/null+++ b/kernel/cli/cmd/sql_test.go@@ -0,0 +1,89 @@+//go:build fts5++package cmd++import (+ "bytes"+ "encoding/json"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"++ "github.com/siyuan-note/siyuan/kernel/sql"+ "github.com/siyuan-note/siyuan/kernel/util"+ "github.com/spf13/cobra"+)++func TestSQLCommandRowLimit(t *testing.T) {+ const childEnv = "SIYUAN_TEST_CLI_SQL_LIMIT"+ if os.Getenv(childEnv) != "1" {+ cmd := exec.Command(os.Args[0], "-test.run=^TestSQLCommandRowLimit$", "-test.timeout=30s")+ cmd.Env = append(os.Environ(), childEnv+"=1")+ if output, err := cmd.CombinedOutput(); err != nil {+ t.Fatalf("SQL command regression failed: %v\n%s", err, output)+ }+ return+ }+ root := t.TempDir()+ util.DBPath = filepath.Join(root, "siyuan.db")+ util.HistoryDBPath = filepath.Join(root, "history.db")+ util.AssetContentDBPath = filepath.Join(root, "asset_content.db")+ util.BlockTreeDBPath = filepath.Join(root, "blocktree.db")+ util.QueueDir = filepath.Join(root, "queue")+ sql.InitDatabase(true)+ sql.InitHistoryDatabase(true)+ sql.InitAssetContentDatabase(true)+ t.Cleanup(sql.CloseDatabase)+ const query = "WITH RECURSIVE items(n) AS (SELECT 1 UNION ALL SELECT n+1 FROM items WHERE n<105) SELECT n FROM items"+ for _, tc := range []struct {+ name, statement, format string+ limit, count int+ truncated bool+ }{+ {"default JSON", query, "json", 100, 100, true},+ {"explicit SQL limit", query + " LIMIT 5", "json", 3, 3, true},+ {"equal limit", query + " LIMIT 3", "json", 3, 3, false},+ {"small result", query + " LIMIT 2", "json", 3, 2, false},+ {"table", query, "table", 3, 3, true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ command := &cobra.Command{}+ command.Flags().Int("limit", tc.limit, "")+ var warning bytes.Buffer+ command.SetErr(&warning)+ output, err := os.CreateTemp(root, "output-")+ if err != nil {+ t.Fatal(err)+ }+ previousStdout := os.Stdout+ os.Stdout = output+ defer func() { os.Stdout = previousStdout }()+ outputFormat = tc.format+ err = sqlCmd.RunE(command, []string{tc.statement})+ os.Stdout = previousStdout+ if closeErr := output.Close(); closeErr != nil {+ t.Fatal(closeErr)+ }+ if err != nil {+ t.Fatal(err)+ }+ data, err := os.ReadFile(output.Name())+ if err != nil {+ t.Fatal(err)+ }+ if tc.format == "json" {+ var rows []json.RawMessage+ if err = json.Unmarshal(data, &rows); err != nil || len(rows) != tc.count {+ t.Fatalf("invalid or unbounded JSON: %s %v", data, err)+ }+ } else if !strings.Contains(string(data), "3 row(s)") {+ t.Fatalf("unexpected table summary: %s", data)+ }+ if strings.Contains(warning.String(), "truncated") != tc.truncated {+ t.Fatalf("unexpected truncation warning: %q", warning.String())+ }+ })+ }+}diff --git a/kernel/sql/block_query.go b/kernel/sql/block_query.goindex c0cc74f230..2d6263b05d 100644--- a/kernel/sql/block_query.go+++ b/kernel/sql/block_query.go@@ -575,6 +575,10 @@ func queryRawStmt(stmt string, limit int) (ret []map[string]any, err error) {}func queryRawStmtWithLimitInfo(stmt string, limit int, info *QueryLimitInfo) (ret []map[string]any, err error) {+ return queryRawStmtRows(stmt, limit, info, false)+}++func queryRawStmtRows(stmt string, limit int, info *QueryLimitInfo, enforceLimit bool) (ret []map[string]any, err error) {rows, err := query(stmt)if err != nil {if strings.Contains(err.Error(), "syntax error") {@@ -616,7 +620,7 @@ func queryRawStmtWithLimitInfo(stmt string, limit int, info *QueryLimitInfo) (reret = append(ret, m)count++- if noLimit && (limit < count || (info != nil && limit == count)) {+ if (noLimit || enforceLimit) && count >= limit {break}}diff --git a/kernel/sql/query_limit.go b/kernel/sql/query_limit.goindex b0a67cc00f..6b4382cc89 100644--- a/kernel/sql/query_limit.go+++ b/kernel/sql/query_limit.go@@ -6,12 +6,33 @@ import ("strings")-// QueryLimitInfo 描述服务端默认限制;Limit 为零表示使用 SQL 自身的限制。+// QueryLimitInfo 描述查询行数限制;Limit 为零表示使用 SQL 自身的限制。type QueryLimitInfo struct {Limit int `json:"limit"`Truncated bool `json:"truncated"`}+// QueryWithRowLimitInfo 在 SQL 自身限制之外封顶返回行数,并多读取一行判断是否截断。+// 直接执行原始语句,保留注释、递归查询和解析器不支持的 SQLite 语法。+func QueryWithRowLimitInfo(stmt string, limit int) (rows []map[string]any, info QueryLimitInfo, err error) {+ if limit < 1 || limit == math.MaxInt {+ return nil, info, errors.New("invalid SQL row limit")+ }+ info.Limit = limit+ rows, err = queryRawStmtRows(stmt, limit+1, nil, true)+ if err != nil {+ return nil, QueryLimitInfo{}, err+ }+ if len(rows) > limit {+ info.Truncated = true+ rows = rows[:limit]+ }+ if rows == nil {+ rows = []map[string]any{}+ }+ return+}+// QueryWithLimitInfo 在同一次查询中多读取一行,判断默认限制是否截断了结果。func QueryWithLimitInfo(stmt string, limit int) (rows []map[string]any, info QueryLimitInfo, err error) {if limit < 1 || limit == math.MaxInt {diff --git a/kernel/sql/query_limit_test.go b/kernel/sql/query_limit_test.goindex 18524dde10..fe89a279ac 100644--- a/kernel/sql/query_limit_test.go+++ b/kernel/sql/query_limit_test.go@@ -2,9 +2,62 @@ package sqlimport (gosql "database/sql"+ "math""testing")+func TestQueryRowLimit(t *testing.T) {+ testDB, err := gosql.Open("sqlite3_extended", ":memory:")+ if err != nil {+ t.Fatal(err)+ }+ testDB.SetMaxOpenConns(1)+ t.Cleanup(func() { _ = testDB.Close() })+ previousDB := db+ db = testDB+ t.Cleanup(func() { db = previousDB })+ if _, err = testDB.Exec("CREATE TABLE items (n INTEGER, content TEXT); INSERT INTO items VALUES (1, 'a'), (2, 'a'), (3, 'a'), (4, 'a'), (5, 'a')"); err != nil {+ t.Fatal(err)+ }+ for _, tc := range []struct {+ stmt string+ count int+ truncated bool+ }{+ {"SELECT n FROM items ORDER BY n", 3, true},+ {"SELECT n FROM items WHERE n <= 3 ORDER BY n", 3, false},+ {"SELECT n FROM items WHERE n < 0", 0, false},+ {"SELECT n FROM items ORDER BY n LIMIT 5", 3, true},+ {"SELECT n FROM items ORDER BY n LIMIT -1", 3, true},+ {"SELECT n FROM items ORDER BY n LIMIT 2", 2, false},+ {"SELECT n FROM items ORDER BY n LIMIT 3 OFFSET 3", 2, false},+ {"SELECT n FROM items UNION SELECT 6 ORDER BY n", 3, true},+ {"WITH x AS (SELECT n FROM items LIMIT 5) SELECT n FROM x ORDER BY n", 3, true},+ {"SELECT n FROM items -- comment\nORDER BY n", 3, true},+ {`SELECT n FROM items WHERE content LIKE '%a%' ESCAPE '\' ORDER BY n`, 3, true},+ } {+ rows, info, queryErr := QueryWithRowLimitInfo(tc.stmt, 3)+ if queryErr != nil || rows == nil || len(rows) != tc.count || info.Limit != 3 || info.Truncated != tc.truncated {+ t.Fatalf("%s: rows=%v info=%+v err=%v", tc.stmt, rows, info, queryErr)+ }+ if tc.count > 0 && rows[0]["n"] != int64(1) && tc.stmt != "SELECT n FROM items ORDER BY n LIMIT 3 OFFSET 3" {+ t.Fatalf("query order changed: %s: %v", tc.stmt, rows)+ }+ }+ for _, limit := range []int{0, -1, math.MaxInt} {+ if _, _, err = QueryWithRowLimitInfo("SELECT 1", limit); err == nil {+ t.Fatalf("invalid limit accepted: %d", limit)+ }+ }+ if _, _, err = QueryWithRowLimitInfo("SELECT abs(-9223372036854775808)", 3); err == nil {+ t.Fatal("row iteration error was ignored")+ }+ rows, err := Query(`SELECT n FROM items WHERE content LIKE '%a%' ESCAPE '\' ORDER BY n`, 3)+ if err != nil || len(rows) != 3 {+ t.Fatalf("raw fallback exceeded default limit: %v %v", rows, err)+ }+}+func TestQueryWithLimitInfo(t *testing.T) {testDB, err := gosql.Open("sqlite3_extended", ":memory:")if err != nil {