An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:bug: Guard span queries against incompatible results https://github.com/siyuan-note/siyuan/issues/20824
Daniel9da1576b88af
diff --git a/kernel/sql/span.go b/kernel/sql/span.goindex 5a04b6dde1..3b2d2190ca 100644--- a/kernel/sql/span.go+++ b/kernel/sql/span.go@@ -28,6 +28,7 @@ import (// 标签候选条件使用确定性函数,部分索引不受搜索的 LIKE 大小写设置影响。const tagSpanIndexPredicate = "instr(lower(type), 'tag') > 0"const tagSpanPredicate = tagSpanIndexPredicate + " AND type LIKE '%tag%'"+const spanSelectColumns = "id, block_id, root_id, box, path, content, markdown, type, ial"func escapeLikePattern(s string) string {var b strings.Builder@@ -71,18 +72,18 @@ func QueryImageSpanMarkdowns() (ret []string, err error) {}func SelectSpansRawStmt(stmt string, limit int) (ret []*Span) {- return selectSpansRawStmt(stmt, limit, func(stmt string) (*sql.Rows, error) {+ return selectSpansRawStmt(stmt, limit, "SelectSpansRawStmt", func(stmt string) (*sql.Rows, error) {return query(stmt)})}func SelectSpansRawStmtInBox(stmt string, limit int, boxID string) (ret []*Span) {- return selectSpansRawStmt(stmt, limit, func(stmt string) (*sql.Rows, error) {+ return selectSpansRawStmt(stmt, limit, "SelectSpansRawStmtInBox", func(stmt string) (*sql.Rows, error) {return queryForBox(boxID, stmt)})}-func selectSpansRawStmt(stmt string, limit int, queryFn func(string) (*sql.Rows, error)) (ret []*Span) {+func selectSpansRawStmt(stmt string, limit int, source string, queryFn func(string) (*sql.Rows, error)) (ret []*Span) {parsedStmt, err := sqlparser.Parse(stmt)if err != nil {//logging.LogErrorf("select [%s] failed: %s", stmt, err)@@ -116,34 +117,24 @@ func selectSpansRawStmt(stmt string, limit int, queryFn func(string) (*sql.Rows,logging.LogWarnf("sql query [%s] failed: %s", stmt, err)return}- defer rows.Close()- for rows.Next() {- span := scanSpanRows(rows)- ret = append(ret, span)- }- return+ return scanSpans(rows, source)}func QueryTagSpansByLabel(label string) (ret []*Span) {var stmt stringvar args []anyif "" != label {- stmt = "SELECT * FROM spans WHERE " + tagSpanPredicate + " AND content LIKE ? ESCAPE '\\' GROUP BY block_id"+ stmt = "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate + " AND content LIKE ? ESCAPE '\\' GROUP BY block_id"args = append(args, "%"+escapeLikePattern(label)+"%")} else {- stmt = "SELECT * FROM spans WHERE " + tagSpanPredicate + " AND content = '' GROUP BY block_id"+ stmt = "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate + " AND content = '' GROUP BY block_id"}rows, err := query(stmt, args...)if err != nil {logging.LogErrorf("sql query failed: %s", err)return}- defer rows.Close()- for rows.Next() {- span := scanSpanRows(rows)- ret = append(ret, span)- }- return+ return scanSpans(rows, "QueryTagSpansByLabel")}func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) {@@ -152,30 +143,25 @@ func QueryTagSpansByKeyword(keyword string, limit int) (ret []*Span) {var stmt stringvar args []anyif len(keywords) == 0 {- stmt = "SELECT * FROM spans WHERE " + tagSpanPredicate + " AND content != '' GROUP BY markdown LIMIT " + strconv.Itoa(limit)+ stmt = "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate + " AND content != '' GROUP BY markdown LIMIT " + strconv.Itoa(limit)} else {var likes []stringfor _, k := range keywords {likes = append(likes, "content LIKE ? ESCAPE '\\'")args = append(args, "%"+escapeLikePattern(k)+"%")}- stmt = "SELECT * FROM spans WHERE " + tagSpanPredicate + " AND (" + strings.Join(likes, " AND ") + ") GROUP BY markdown LIMIT " + strconv.Itoa(limit)+ stmt = "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicate + " AND (" + strings.Join(likes, " AND ") + ") GROUP BY markdown LIMIT " + strconv.Itoa(limit)}rows, err := query(stmt, args...)if err != nil {logging.LogErrorf("sql query failed: %s", err)return}- defer rows.Close()- for rows.Next() {- span := scanSpanRows(rows)- ret = append(ret, span)- }- return+ return scanSpans(rows, "QueryTagSpansByKeyword")}func QueryTagSpans(p string) (ret []*Span) {- stmt := "SELECT * FROM spans WHERE " + tagSpanPredicate+ stmt := "SELECT " + spanSelectColumns + " FROM spans WHERE " + tagSpanPredicatevar args []anyif "" != p {stmt += " AND path = ?"@@ -186,20 +172,39 @@ func QueryTagSpans(p string) (ret []*Span) {logging.LogErrorf("sql query failed: %s", err)return}- defer rows.Close()- for rows.Next() {- span := scanSpanRows(rows)- ret = append(ret, span)- }- return+ return scanSpans(rows, "QueryTagSpans")}-func scanSpanRows(rows *sql.Rows) (ret *Span) {- var span Span- if err := rows.Scan(&span.ID, &span.BlockID, &span.RootID, &span.Box, &span.Path, &span.Content, &span.Markdown, &span.Type, &span.IAL); err != nil {- logging.LogErrorf("query scan field failed: %s", err)+func scanSpans(rows *sql.Rows, source string) (ret []*Span) {+ defer rows.Close()+ columns, err := rows.Columns()+ if err != nil {+ logging.LogErrorf("span query [%s] columns failed: %s", source, err)+ return+ }+ // 自定义查询仍按 Span 的九个字段位置读取,不改写 SQL 或猜测字段映射。+ if len(columns) != 9 {+ logging.LogErrorf("span query [%s] expected 9 columns, got %d", source, len(columns))return}- ret = &span+ var firstScanErr error+ var skipped int+ for rows.Next() {+ var span Span+ if err = rows.Scan(&span.ID, &span.BlockID, &span.RootID, &span.Box, &span.Path, &span.Content, &span.Markdown, &span.Type, &span.IAL); err != nil {+ if firstScanErr == nil {+ firstScanErr = err+ }+ skipped+++ continue+ }+ ret = append(ret, &span)+ }+ if firstScanErr != nil {+ logging.LogErrorf("span query [%s] skipped %d invalid rows: %s", source, skipped, firstScanErr)+ }+ if err = rows.Err(); err != nil {+ logging.LogErrorf("span query [%s] iteration failed: %s", source, err)+ }return}diff --git a/kernel/sql/span_test.go b/kernel/sql/span_test.gonew file mode 100644index 0000000000..05d34315f0--- /dev/null+++ b/kernel/sql/span_test.go@@ -0,0 +1,216 @@+// SiYuan - From thought to insight, with agents+// Copyright (c) 2020-present, b3log.org+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as published by+// the Free Software Foundation, either version 3 of the License, or+// (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++package sql++import (+ gosql "database/sql"+ "os"+ "path/filepath"+ "reflect"+ "strings"+ "testing"+ "text/template"++ "github.com/siyuan-note/logging"+)++func TestTagSpanQueriesIgnoreExtraColumns(t *testing.T) {+ for _, extended := range []bool{false, true} {+ name := "standard"+ if extended {+ name = "extra columns"+ }+ t.Run(name, func(t *testing.T) {+ testDB := useSpanTestDB(t)+ if extended {+ for _, column := range []string{"extra1", "extra2", "extra3"} {+ if _, err := testDB.Exec("ALTER TABLE spans ADD COLUMN " + column + " TEXT"); err != nil {+ t.Fatal(err)+ }+ }+ }+ for _, query := range tagSpanTestQueries() {+ t.Run(query.name, func(t *testing.T) {+ assertSpanTestResult(t, query.run())+ })+ }+ if _, err := testDB.Exec("UPDATE spans SET content = ''"); err != nil {+ t.Fatal(err)+ }+ if spans := QueryTagSpansByLabel(""); len(spans) != 1 || spans[0] == nil || spans[0].ID != "span" || spans[0].Content != "" {+ t.Errorf("unexpected empty-label results: %#v", spans)+ }+ })+ }+}++func TestSpanQueriesSkipInvalidRows(t *testing.T) {+ testDB := useSpanTestDB(t)+ // 无效行分别位于有效行之前和之后,不能污染返回值或中断剩余行扫描。+ if _, err := testDB.Exec("INSERT INTO spans SELECT 'a-invalid', 'a-invalid', root_id, box, path, content, '#invalid-a#', type, NULL FROM spans; INSERT INTO spans SELECT 'z-invalid', 'z-invalid', root_id, box, path, content, '#invalid-z#', type, NULL FROM spans WHERE id = 'span'"); err != nil {+ t.Fatal(err)+ }+ logPath := useSpanTestLog(t)+ queries := append(tagSpanTestQueries(), struct {+ name string+ run func() []*Span+ }{"SelectSpansRawStmt", func() []*Span {+ return SelectSpansRawStmt("SELECT * FROM spans ORDER BY id", 100)+ }})+ for _, query := range queries {+ t.Run(query.name, func(t *testing.T) {+ assertSpanTestResult(t, query.run())+ })+ }+ data, err := os.ReadFile(logPath)+ if err != nil {+ t.Fatal(err)+ }+ if got := strings.Count(string(data), "span query ["); got != len(queries) {+ t.Fatalf("want one scan diagnostic per query, got %d: %s", got, data)+ }+}++func TestSpanTemplateQueriesValidateProjection(t *testing.T) {+ for _, scope := range []string{"global", "encrypted box"} {+ t.Run(scope, func(t *testing.T) {+ var testDB *gosql.DB+ var boxIDs []string+ source := "SelectSpansRawStmt"+ if scope == "global" {+ testDB = useSpanTestDB(t)+ } else {+ var boxID string+ testDB, boxID = useEncryptedQueryTestDB(t)+ createSpanTestTable(t, testDB)+ boxIDs = []string{boxID}+ source += "InBox"+ }+ if _, err := testDB.Exec("INSERT INTO spans SELECT 'second', block_id, root_id, box, path, content, markdown, type, ial FROM spans"); err != nil {+ t.Fatal(err)+ }+ logPath := useSpanTestLog(t)+ funcs := template.FuncMap{}+ SQLTemplateFuncs(&funcs, boxIDs...)+ querySpans := funcs["querySpans"].(func(string, ...string) []*Span)+ assertSpanTestResult(t, querySpans("SELECT * FROM spans WHERE id = 'span' LIMIT ?", "1"))+ for _, stmt := range []string{+ "SELECT spans.*, 'private-query-marker' AS extra1, 2 AS extra2, 3 AS extra3 FROM spans",+ "SELECT spans.*, 1 AS extra1, 2 AS extra2, 3 AS extra3 FROM spans WHERE 0",+ "SELECT id FROM spans",+ } {+ if got := querySpans(stmt); len(got) != 0 {+ t.Errorf("incompatible projection returned span entries: %#v", got)+ }+ }+ assertSpanTestResult(t, querySpans("SELECT * FROM spans WHERE id = 'span'"))+ data, err := os.ReadFile(logPath)+ if err != nil {+ t.Fatal(err)+ }+ log := strings.ReplaceAll(string(data), "\r\n", "\n")+ if strings.Count(log, "span query ["+source+"]") != 3 ||+ strings.Count(log, "expected 9 columns, got 12\n") != 2 ||+ strings.Count(log, "expected 9 columns, got 1\n") != 1 {+ t.Errorf("missing source/column diagnostics or repeated per-row logs: %s", log)+ }+ if strings.Contains(log, "private-query-marker") {+ t.Error("projection diagnostic includes SQL content")+ }+ })+ }+}++func TestSpanQueryReportsIterationError(t *testing.T) {+ testDB := useSpanTestDB(t)+ if _, err := testDB.Exec("INSERT INTO spans SELECT 'z-error', block_id, root_id, box, path, content, markdown, type, ial FROM spans"); err != nil {+ t.Fatal(err)+ }+ logPath := useSpanTestLog(t)+ // 第二行计算失败应留下诊断,第一行仍保持既有的部分结果语义。+ spans := SelectSpansRawStmt("SELECT id, block_id, root_id, box, path, content, markdown, type, CASE WHEN id = 'span' THEN ial ELSE abs(-9223372036854775808) END FROM spans ORDER BY rowid", 100)+ assertSpanTestResult(t, spans)+ data, err := os.ReadFile(logPath)+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(data), "span query [SelectSpansRawStmt] iteration failed") || !strings.Contains(string(data), "integer overflow") {+ t.Fatalf("missing iteration error diagnostic: %s", data)+ }+}++func useSpanTestDB(t *testing.T) *gosql.DB {+ t.Helper()+ testDB, err := gosql.Open("sqlite3_extended", ":memory:")+ if err != nil {+ t.Fatal(err)+ }+ testDB.SetMaxOpenConns(1)+ previous := db+ db = testDB+ t.Cleanup(func() {+ db = previous+ testDB.Close()+ })+ createSpanTestTable(t, testDB)+ return testDB+}++func createSpanTestTable(t *testing.T, testDB *gosql.DB) {+ t.Helper()+ for _, stmt := range []string{+ "CREATE TABLE spans (id, block_id, root_id, box, path, content, markdown, type, ial)",+ "INSERT INTO spans VALUES ('span', 'block', 'root', 'box', '/doc.sy', 'tag', '#tag#', 'tag strong', '')",+ } {+ if _, err := testDB.Exec(stmt); err != nil {+ t.Fatal(err)+ }+ }+}++func tagSpanTestQueries() []struct {+ name string+ run func() []*Span+} {+ return []struct {+ name string+ run func() []*Span+ }{+ {"QueryTagSpans all", func() []*Span { return QueryTagSpans("") }},+ {"QueryTagSpans path", func() []*Span { return QueryTagSpans("/doc.sy") }},+ {"QueryTagSpansByLabel", func() []*Span { return QueryTagSpansByLabel("tag") }},+ {"QueryTagSpansByKeyword all", func() []*Span { return QueryTagSpansByKeyword("", 100) }},+ {"QueryTagSpansByKeyword", func() []*Span { return QueryTagSpansByKeyword("tag", 100) }},+ }+}++func assertSpanTestResult(t *testing.T, spans []*Span) {+ t.Helper()+ want := &Span{ID: "span", BlockID: "block", RootID: "root", Box: "box", Path: "/doc.sy", Content: "tag", Markdown: "#tag#", Type: "tag strong"}+ if len(spans) != 1 || !reflect.DeepEqual(spans[0], want) {+ t.Errorf("unexpected spans: %#v, want %#v", spans, want)+ }+}++func useSpanTestLog(t *testing.T) string {+ t.Helper()+ previous := logging.LogPath+ logPath := filepath.Join(t.TempDir(), "span.log")+ logging.SetLogPath(logPath)+ t.Cleanup(func() { logging.SetLogPath(previous) })+ return logPath+}diff --git a/kernel/sql/tag_index_test.go b/kernel/sql/tag_index_test.goindex 86ad4e29c8..42c6db197d 100644--- a/kernel/sql/tag_index_test.go+++ b/kernel/sql/tag_index_test.go@@ -70,7 +70,7 @@ func TestTagSpanIndexPreservesQueries(t *testing.T) {t.Fatalf("%s: unexpected query results: %v", setting, after)}for _, suffix := range []string{"", " AND path = '/one.sy'", " AND content = '' GROUP BY block_id", " AND content != '' GROUP BY markdown LIMIT 16"} {- plan := queryPlanDetails(t, database, "SELECT * FROM spans WHERE "+tagSpanPredicate+suffix)+ plan := queryPlanDetails(t, database, "SELECT "+spanSelectColumns+" FROM spans WHERE "+tagSpanPredicate+suffix)if !strings.Contains(plan, "idx_spans_tag_path") {t.Fatalf("tag query does not use partial index: %s", plan)}