Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Guard span queries against incompatible results https://github.com/siyuan-note/siyuan/issues/20824

Daniel9da1576b88af
diff --git a/kernel/sql/span.go b/kernel/sql/span.go
index 5a04b6dde1..3b2d2190ca 100644
--- a/kernel/sql/span.go
+++ b/kernel/sql/span.go
@@ -28,6 +28,7 @@ import (
// 标签候选条件使用确定性函数,部分索引不受搜索的 LIKE 大小写设置影响。
const tagSpanIndexPredicate = "instr(lower(type), 'tag') > 0"
const tagSpanPredicate = tagSpanIndexPredicate + " AND type LIKE '%tag%'"
+const spanSelectColumns = "id, block_id, root_id, box, path, content, markdown, type, ial"
func escapeLikePattern(s string) string {
var b strings.Builder
@@ -71,18 +72,18 @@ func QueryImageSpanMarkdowns() (ret []string, err error) {
}
func SelectSpansRawStmt(stmt string, limit int) (ret []*Span) {
- return selectSpansRawStmt(stmt, limit, func(stmt string) (*sql.Rows, error) {
+ return selectSpansRawStmt(stmt, limit, "SelectSpansRawStmt", func(stmt string) (*sql.Rows, error) {
return query(stmt)
})
}
func SelectSpansRawStmtInBox(stmt string, limit int, boxID string) (ret []*Span) {
- return selectSpansRawStmt(stmt, limit, func(stmt string) (*sql.Rows, error) {
+ return selectSpansRawStmt(stmt, limit, "SelectSpansRawStmtInBox", func(stmt string) (*sql.Rows, error) {
return queryForBox(boxID, stmt)
})
}
-func selectSpansRawStmt(stmt string, limit int, queryFn func(string) (*sql.Rows, error)) (ret []*Span) {
+func selectSpansRawStmt(stmt string, limit int, source string, queryFn func(string) (*sql.Rows, error)) (ret []*Span) {
parsedStmt, err := sqlparser.Parse(stmt)
if err != nil {
//logging.LogErrorf("select [%s] failed: %s", stmt, err)
@@ -116,34 +117,24 @@ func selectSpansRawStmt(stmt string, limit int, queryFn func(string) (*sql.Rows,
logging.LogWarnf("sql query [%s] failed: %s", stmt, err)
return
}
- defer rows.Close()
- for rows.Next() {
- span := scanSpanRows(rows)
- ret = append(ret, span)
- }
- return
+ return scanSpans(rows, source)
}
func QueryTagSpansByLabel(label string) (ret []*Span) {
var stmt string
var args []any
if "" != label {
- stmt = "SELECT * FROM spans WHERE " + tagSpanPredicate + " AND content LIKE ? ESCAPE '\\' GROUP BY block_id"
+ stmt = "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate + " AND content LIKE ? ESCAPE '\\' GROUP BY block_id"
args = append(args, "%"+escapeLikePattern(label)+"%")
} else {
- stmt = "SELECT * FROM spans WHERE " + tagSpanPredicate + " AND content = '' GROUP BY block_id"
+ stmt = "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate + " AND content = '' GROUP BY block_id"
}
rows, err := query(stmt, args...)
if err != nil {
logging.LogErrorf("sql query failed: %s", err)
return
}
- defer rows.Close()
- for rows.Next() {
- span := scanSpanRows(rows)
- ret = append(ret, span)
- }
- return
+ return scanSpans(rows, "QueryTagSpansByLabel")
}
func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) {
@@ -152,30 +143,25 @@ func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) {
var stmt string
var args []any
if len(keywords) == 0 {
- stmt = "SELECT * FROM spans WHERE " + tagSpanPredicate + " AND content != '' GROUP BY markdown LIMIT " + strconv.Itoa(limit)
+ stmt = "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate + " AND content != '' GROUP BY markdown LIMIT " + strconv.Itoa(limit)
} else {
var likes []string
for _, k := range keywords {
likes = append(likes, "content LIKE ? ESCAPE '\\'")
args = append(args, "%"+escapeLikePattern(k)+"%")
}
- stmt = "SELECT * FROM spans WHERE " + tagSpanPredicate + " AND (" + strings.Join(likes, " AND ") + ") GROUP BY markdown LIMIT " + strconv.Itoa(limit)
+ stmt = "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate + " AND (" + strings.Join(likes, " AND ") + ") GROUP BY markdown LIMIT " + strconv.Itoa(limit)
}
rows, err := query(stmt, args...)
if err != nil {
logging.LogErrorf("sql query failed: %s", err)
return
}
- defer rows.Close()
- for rows.Next() {
- span := scanSpanRows(rows)
- ret = append(ret, span)
- }
- return
+ return scanSpans(rows, "QueryTagSpansByKeyword")
}
func QueryTagSpans(p string) (ret []*Span) {
- stmt := "SELECT * FROM spans WHERE " + tagSpanPredicate
+ stmt := "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate
var args []any
if "" != p {
stmt += " AND path = ?"
@@ -186,20 +172,39 @@ func QueryTagSpans(p string) (ret []*Span) {
logging.LogErrorf("sql query failed: %s", err)
return
}
- defer rows.Close()
- for rows.Next() {
- span := scanSpanRows(rows)
- ret = append(ret, span)
- }
- return
+ return scanSpans(rows, "QueryTagSpans")
}
-func scanSpanRows(rows *sql.Rows) (ret *Span) {
- var span Span
- if err := rows.Scan(&span.ID, &span.BlockID, &span.RootID, &span.Box, &span.Path, &span.Content, &span.Markdown, &span.Type, &span.IAL); err != nil {
- logging.LogErrorf("query scan field failed: %s", err)
+func scanSpans(rows *sql.Rows, source string) (ret []*Span) {
+ defer rows.Close()
+ columns, err := rows.Columns()
+ if err != nil {
+ logging.LogErrorf("span query [%s] columns failed: %s", source, err)
+ return
+ }
+ // 自定义查询仍按 Span 的九个字段位置读取,不改写 SQL 或猜测字段映射。
+ if len(columns) != 9 {
+ logging.LogErrorf("span query [%s] expected 9 columns, got %d", source, len(columns))
return
}
- ret = &span
+ var firstScanErr error
+ var skipped int
+ for rows.Next() {
+ var span Span
+ if err = rows.Scan(&span.ID, &span.BlockID, &span.RootID, &span.Box, &span.Path, &span.Content, &span.Markdown, &span.Type, &span.IAL); err != nil {
+ if firstScanErr == nil {
+ firstScanErr = err
+ }
+ skipped++
+ continue
+ }
+ ret = append(ret, &span)
+ }
+ if firstScanErr != nil {
+ logging.LogErrorf("span query [%s] skipped %d invalid rows: %s", source, skipped, firstScanErr)
+ }
+ if err = rows.Err(); err != nil {
+ logging.LogErrorf("span query [%s] iteration failed: %s", source, err)
+ }
return
}
diff --git a/kernel/sql/span_test.go b/kernel/sql/span_test.go
new file mode 100644
index 0000000000..05d34315f0
--- /dev/null
+++ b/kernel/sql/span_test.go
@@ -0,0 +1,216 @@
+// SiYuan - From thought to insight, with agents
+// Copyright (c) 2020-present, b3log.org
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package sql
+
+import (
+ gosql "database/sql"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "text/template"
+
+ "github.com/siyuan-note/logging"
+)
+
+func TestTagSpanQueriesIgnoreExtraColumns(t *testing.T) {
+ for _, extended := range []bool{false, true} {
+ name := "standard"
+ if extended {
+ name = "extra columns"
+ }
+ t.Run(name, func(t *testing.T) {
+ testDB := useSpanTestDB(t)
+ if extended {
+ for _, column := range []string{"extra1", "extra2", "extra3"} {
+ if _, err := testDB.Exec("ALTER TABLE spans ADD COLUMN " + column + " TEXT"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ for _, query := range tagSpanTestQueries() {
+ t.Run(query.name, func(t *testing.T) {
+ assertSpanTestResult(t, query.run())
+ })
+ }
+ if _, err := testDB.Exec("UPDATE spans SET content = ''"); err != nil {
+ t.Fatal(err)
+ }
+ if spans := QueryTagSpansByLabel(""); len(spans) != 1 || spans[0] == nil || spans[0].ID != "span" || spans[0].Content != "" {
+ t.Errorf("unexpected empty-label results: %#v", spans)
+ }
+ })
+ }
+}
+
+func TestSpanQueriesSkipInvalidRows(t *testing.T) {
+ testDB := useSpanTestDB(t)
+ // 无效行分别位于有效行之前和之后,不能污染返回值或中断剩余行扫描。
+ if _, err := testDB.Exec("INSERT INTO spans SELECT 'a-invalid', 'a-invalid', root_id, box, path, content, '#invalid-a#', type, NULL FROM spans; INSERT INTO spans SELECT 'z-invalid', 'z-invalid', root_id, box, path, content, '#invalid-z#', type, NULL FROM spans WHERE id = 'span'"); err != nil {
+ t.Fatal(err)
+ }
+ logPath := useSpanTestLog(t)
+ queries := append(tagSpanTestQueries(), struct {
+ name string
+ run func() []*Span
+ }{"SelectSpansRawStmt", func() []*Span {
+ return SelectSpansRawStmt("SELECT * FROM spans ORDER BY id", 100)
+ }})
+ for _, query := range queries {
+ t.Run(query.name, func(t *testing.T) {
+ assertSpanTestResult(t, query.run())
+ })
+ }
+ data, err := os.ReadFile(logPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := strings.Count(string(data), "span query ["); got != len(queries) {
+ t.Fatalf("want one scan diagnostic per query, got %d: %s", got, data)
+ }
+}
+
+func TestSpanTemplateQueriesValidateProjection(t *testing.T) {
+ for _, scope := range []string{"global", "encrypted box"} {
+ t.Run(scope, func(t *testing.T) {
+ var testDB *gosql.DB
+ var boxIDs []string
+ source := "SelectSpansRawStmt"
+ if scope == "global" {
+ testDB = useSpanTestDB(t)
+ } else {
+ var boxID string
+ testDB, boxID = useEncryptedQueryTestDB(t)
+ createSpanTestTable(t, testDB)
+ boxIDs = []string{boxID}
+ source += "InBox"
+ }
+ if _, err := testDB.Exec("INSERT INTO spans SELECT 'second', block_id, root_id, box, path, content, markdown, type, ial FROM spans"); err != nil {
+ t.Fatal(err)
+ }
+ logPath := useSpanTestLog(t)
+ funcs := template.FuncMap{}
+ SQLTemplateFuncs(&funcs, boxIDs...)
+ querySpans := funcs["querySpans"].(func(string, ...string) []*Span)
+ assertSpanTestResult(t, querySpans("SELECT * FROM spans WHERE id = 'span' LIMIT ?", "1"))
+ for _, stmt := range []string{
+ "SELECT spans.*, 'private-query-marker' AS extra1, 2 AS extra2, 3 AS extra3 FROM spans",
+ "SELECT spans.*, 1 AS extra1, 2 AS extra2, 3 AS extra3 FROM spans WHERE 0",
+ "SELECT id FROM spans",
+ } {
+ if got := querySpans(stmt); len(got) != 0 {
+ t.Errorf("incompatible projection returned span entries: %#v", got)
+ }
+ }
+ assertSpanTestResult(t, querySpans("SELECT * FROM spans WHERE id = 'span'"))
+ data, err := os.ReadFile(logPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ log := strings.ReplaceAll(string(data), "\r\n", "\n")
+ if strings.Count(log, "span query ["+source+"]") != 3 ||
+ strings.Count(log, "expected 9 columns, got 12\n") != 2 ||
+ strings.Count(log, "expected 9 columns, got 1\n") != 1 {
+ t.Errorf("missing source/column diagnostics or repeated per-row logs: %s", log)
+ }
+ if strings.Contains(log, "private-query-marker") {
+ t.Error("projection diagnostic includes SQL content")
+ }
+ })
+ }
+}
+
+func TestSpanQueryReportsIterationError(t *testing.T) {
+ testDB := useSpanTestDB(t)
+ if _, err := testDB.Exec("INSERT INTO spans SELECT 'z-error', block_id, root_id, box, path, content, markdown, type, ial FROM spans"); err != nil {
+ t.Fatal(err)
+ }
+ logPath := useSpanTestLog(t)
+ // 第二行计算失败应留下诊断,第一行仍保持既有的部分结果语义。
+ spans := SelectSpansRawStmt("SELECT id, block_id, root_id, box, path, content, markdown, type, CASE WHEN id = 'span' THEN ial ELSE abs(-9223372036854775808) END FROM spans ORDER BY rowid", 100)
+ assertSpanTestResult(t, spans)
+ data, err := os.ReadFile(logPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(data), "span query [SelectSpansRawStmt] iteration failed") || !strings.Contains(string(data), "integer overflow") {
+ t.Fatalf("missing iteration error diagnostic: %s", data)
+ }
+}
+
+func useSpanTestDB(t *testing.T) *gosql.DB {
+ t.Helper()
+ testDB, err := gosql.Open("sqlite3_extended", ":memory:")
+ if err != nil {
+ t.Fatal(err)
+ }
+ testDB.SetMaxOpenConns(1)
+ previous := db
+ db = testDB
+ t.Cleanup(func() {
+ db = previous
+ testDB.Close()
+ })
+ createSpanTestTable(t, testDB)
+ return testDB
+}
+
+func createSpanTestTable(t *testing.T, testDB *gosql.DB) {
+ t.Helper()
+ for _, stmt := range []string{
+ "CREATE TABLE spans (id, block_id, root_id, box, path, content, markdown, type, ial)",
+ "INSERT INTO spans VALUES ('span', 'block', 'root', 'box', '/doc.sy', 'tag', '#tag#', 'tag strong', '')",
+ } {
+ if _, err := testDB.Exec(stmt); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func tagSpanTestQueries() []struct {
+ name string
+ run func() []*Span
+} {
+ return []struct {
+ name string
+ run func() []*Span
+ }{
+ {"QueryTagSpans all", func() []*Span { return QueryTagSpans("") }},
+ {"QueryTagSpans path", func() []*Span { return QueryTagSpans("/doc.sy") }},
+ {"QueryTagSpansByLabel", func() []*Span { return QueryTagSpansByLabel("tag") }},
+ {"QueryTagSpansByKeyword all", func() []*Span { return QueryTagSpansByKeyword("", 100) }},
+ {"QueryTagSpansByKeyword", func() []*Span { return QueryTagSpansByKeyword("tag", 100) }},
+ }
+}
+
+func assertSpanTestResult(t *testing.T, spans []*Span) {
+ t.Helper()
+ want := &Span{ID: "span", BlockID: "block", RootID: "root", Box: "box", Path: "/doc.sy", Content: "tag", Markdown: "#tag#", Type: "tag strong"}
+ if len(spans) != 1 || !reflect.DeepEqual(spans[0], want) {
+ t.Errorf("unexpected spans: %#v, want %#v", spans, want)
+ }
+}
+
+func useSpanTestLog(t *testing.T) string {
+ t.Helper()
+ previous := logging.LogPath
+ logPath := filepath.Join(t.TempDir(), "span.log")
+ logging.SetLogPath(logPath)
+ t.Cleanup(func() { logging.SetLogPath(previous) })
+ return logPath
+}
diff --git a/kernel/sql/tag_index_test.go b/kernel/sql/tag_index_test.go
index 86ad4e29c8..42c6db197d 100644
--- a/kernel/sql/tag_index_test.go
+++ b/kernel/sql/tag_index_test.go
@@ -70,7 +70,7 @@ func TestTagSpanIndexPreservesQueries(t *testing.T) {
t.Fatalf("%s: unexpected query results: %v", setting, after)
}
for _, suffix := range []string{"", " AND path = '/one.sy'", " AND content = '' GROUP BY block_id", " AND content != '' GROUP BY markdown LIMIT 16"} {
- plan := queryPlanDetails(t, database, "SELECT * FROM spans WHERE "+tagSpanPredicate+suffix)
+ plan := queryPlanDetails(t, database, "SELECT "+spanSelectColumns+" FROM spans WHERE "+tagSpanPredicate+suffix)
if !strings.Contains(plan, "idx_spans_tag_path") {
t.Fatalf("tag query does not use partial index: %s", plan)
}