Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:perf: Avoid unrelated block scans in document-only search https://github.com/siyuan-note/siyuan/issues/20310

Danielce0c8bdae2d3
diff --git a/kernel/model/search.go b/kernel/model/search.go
index 2926f2dfe4..f5c41580ce 100644
--- a/kernel/model/search.go
+++ b/kernel/model/search.go
@@ -2148,6 +2148,10 @@ func buildTypeFilter(types, subTypes map[string]bool, alias ...string) string {
}
}
+ if len(simpleTypes) == 1 && simpleTypes[0] == "d" && len(clauses) == 0 {
+ // 文档类型的等值条件与文档部分索引的筛选条件保持一致。
+ return "(" + prefix + "type = 'd')"
+ }
if 0 < len(simpleTypes) {
clauses = append([]string{prefix + "type IN (" + sqlQuoteJoin(simpleTypes) + ")"}, clauses...)
}
diff --git a/kernel/model/search_document_test.go b/kernel/model/search_document_test.go
new file mode 100644
index 0000000000..c1367f18f2
--- /dev/null
+++ b/kernel/model/search_document_test.go
@@ -0,0 +1,156 @@
+package model
+
+import (
+ gosql "database/sql"
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/conf"
+)
+
+func TestDocumentTypeFilterFromSettings(t *testing.T) {
+ setSearchCaseSensitive(t, false)
+ Conf.Search = &conf.Search{Document: true, Mindmap: new(false), MindmapItem: new(false), CustomBlock: new(false)}
+ if got := buildTypeFilter(nil, nil); got != "(type = 'd')" {
+ t.Fatalf("document-only settings: %s", got)
+ }
+ Conf.Search.Paragraph = true
+ if got := buildTypeFilter(nil, nil); got != "(type IN ('d','p'))" {
+ t.Fatalf("mixed settings: %s", got)
+ }
+}
+
+func documentSearchRows(t *testing.T, database *gosql.DB, statement string, args []any) [][]any {
+ t.Helper()
+ rows, err := database.Query(statement, args...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer rows.Close()
+ columns, err := rows.Columns()
+ if err != nil {
+ t.Fatal(err)
+ }
+ var result [][]any
+ for rows.Next() {
+ values := make([]any, len(columns))
+ pointers := make([]any, len(columns))
+ for i := range values {
+ pointers[i] = &values[i]
+ }
+ if err = rows.Scan(pointers...); err != nil {
+ t.Fatal(err)
+ }
+ for i, value := range values {
+ if bytes, ok := value.([]byte); ok {
+ values[i] = string(bytes)
+ }
+ }
+ result = append(result, values)
+ }
+ if err = rows.Err(); err != nil {
+ t.Fatal(err)
+ }
+ return result
+}
+
+func TestDocumentSearchPartialIndexResults(t *testing.T) {
+ for _, sensitive := range []bool{false, true} {
+ t.Run(fmt.Sprintf("case=%t", sensitive), func(t *testing.T) {
+ setSearchCaseSensitive(t, sensitive)
+ database := newSearchHPathTestDB(t)
+ for _, ddl := range []string{
+ "CREATE INDEX idx_blocks_root_id ON blocks(root_id)",
+ "CREATE INDEX idx_blocks_doc_hpath ON blocks(hpath) WHERE type = 'd'",
+ "CREATE INDEX idx_blocks_recent_d ON blocks(updated DESC) WHERE type = 'd'",
+ } {
+ if _, err := database.Exec(ddl); err != nil {
+ t.Fatal(err)
+ }
+ }
+ setting := "OFF"
+ if sensitive {
+ setting = "ON"
+ }
+ if _, err := database.Exec("PRAGMA case_sensitive_like = " + setting); err != nil {
+ t.Fatal(err)
+ }
+ for i, block := range [][5]string{
+ {"release", "release", "/Projects/Release", "release notes", "d"},
+ {"upper", "upper", "/Projects/Upper", "RELEASE NOTES", "d"},
+ {"path", "path", "/Projects/release/notes", "path match", "d"},
+ {"body", "path", "/Projects/release/notes", "body exclusive", "p"},
+ {"quoted", "quoted", "/Other", "author's release", "d"},
+ {"comma", "comma", "/Other", "boundary", "d"},
+ {"comma", "comma", "/Other", "marker", "d"},
+ {"nul", "nul", "/Other", "release\x00notes", "d"},
+ } {
+ insertSearchHPathTestBlock(t, database, block[0], block[1], block[2], block[3], block[4])
+ if _, err := database.Exec("UPDATE blocks SET created = ?, updated = ?, sort = ? WHERE rowid = ?",
+ fmt.Sprintf("202610100000%02d", i), fmt.Sprintf("202610100000%02d", 20-i), i, i+1); err != nil {
+ t.Fatal(err)
+ }
+ }
+ type comparison struct {
+ statement string
+ args []any
+ rows [][]any
+ }
+ var comparisons []comparison
+ // 对照结果在补建索引之前读取,包含完整投影、统计值及分页顺序。
+ for _, hpath := range []bool{false, true} {
+ for _, query := range []string{"release notes", "body exclusive", "author's release", "boundary,marker", "release %", "_ notes", "missing words"} {
+ keywords := strings.Split(strings.ReplaceAll(query, "'", "''"), " ")
+ for order := 0; order <= 7; order++ {
+ for page := 1; page <= 3; page++ {
+ for _, scope := range []struct {
+ box, path, ignore string
+ args []any
+ }{
+ {},
+ {box: " AND box = ?", path: " AND path LIKE ?", args: []any{"20260729120000-box000", "/release%"}},
+ {ignore: " AND id != 'release'"},
+ } {
+ args := append(append([]any{}, scope.args...), scope.args...)
+ oldStatement := buildDocumentSearchStatement(query, keywords, "(type IN ('d'))", scope.box, scope.path, scope.ignore, order, page, 2, hpath)
+ newStatement := buildDocumentSearchStatement(query, keywords, buildTypeFilter(map[string]bool{"document": true}, nil), scope.box, scope.path, scope.ignore, order, page, 2, hpath)
+ want := documentSearchRows(t, database, oldStatement, args)
+ if query == "release notes" && page == 1 && scope.box == "" && scope.ignore == "" && len(want) == 0 {
+ t.Fatal("matching document missing from baseline")
+ }
+ comparisons = append(comparisons, comparison{newStatement, args, want})
+ }
+ }
+ }
+ }
+ }
+ if _, err := database.Exec("CREATE INDEX idx_blocks_doc_root_id ON blocks(root_id) WHERE type = 'd'"); err != nil {
+ t.Fatal(err)
+ }
+ for _, c := range comparisons {
+ if got := documentSearchRows(t, database, c.statement, c.args); !reflect.DeepEqual(got, c.rows) {
+ t.Fatalf("search result changed: got %v, want %v; query=%s", got, c.rows, c.statement)
+ }
+ }
+ for _, hpath := range []bool{false, true} {
+ statement := buildDocumentSearchStatement("release notes", []string{"release", "notes"}, buildTypeFilter(map[string]bool{"document": true}, nil), "", "", "", 0, 1, 32, hpath)
+ plan := documentSearchRows(t, database, "EXPLAIN QUERY PLAN "+statement, nil)
+ indexed := 0
+ for _, row := range plan {
+ detail := row[3].(string)
+ if strings.Contains(detail, "USING INDEX idx_blocks_doc_root_id") {
+ indexed++
+ }
+ if strings.Contains(detail, "SCAN blocks USING INDEX idx_blocks_root_id") {
+ t.Fatalf("document search scans unrelated blocks: %v", plan)
+ }
+ }
+ if indexed < 2 {
+ t.Fatalf("aggregation and result lookup must use the document index: %v", plan)
+ }
+ }
+ })
+ }
+}
diff --git a/kernel/model/search_subtypes_test.go b/kernel/model/search_subtypes_test.go
index 592ff76f0b..a3b6f2da9c 100644
--- a/kernel/model/search_subtypes_test.go
+++ b/kernel/model/search_subtypes_test.go
@@ -32,6 +32,10 @@ func TestBuildTypeFilterSubTypes(t *testing.T) {
alias string
want string
}{
+ {"document", map[string]bool{"document": true}, nil, "", "(type = 'd')"},
+ {"documentAlias", map[string]bool{"document": true}, map[string]bool{"h2": true}, "b.", "(b.type = 'd')"},
+ {"documentAndParagraph", map[string]bool{"document": true, "paragraph": true}, nil, "", "(type IN ('d','p'))"},
+ {"documentAndHeading", map[string]bool{"document": true, "heading": true}, map[string]bool{"h2": true}, "", "(type IN ('d') OR (type = 'h' AND subtype IN ('h2')))"},
{"heading", map[string]bool{"heading": true}, map[string]bool{"h2": true}, "", "((type = 'h' AND subtype IN ('h2')))"},
{"list", map[string]bool{"list": true, "listItem": true}, map[string]bool{"list:o": true}, "", "(type IN ('i') OR (type = 'l' AND subtype IN ('o')))"},
{"listItem", map[string]bool{"list": true, "listItem": true}, map[string]bool{"listItem:t": true}, "", "(type IN ('l') OR (type = 'i' AND subtype IN ('t')))"},
diff --git a/kernel/sql/database.go b/kernel/sql/database.go
index e435c4cf80..9637414b6b 100644
--- a/kernel/sql/database.go
+++ b/kernel/sql/database.go
@@ -119,8 +119,8 @@ func initDatabase(forceRebuild bool) {
if util.DatabaseVer == getDatabaseVer() {
// 老库版本一致但缺少新加的列时,做幂等迁移(不升 DatabaseVer,避免全库重建丢失已嵌入向量)
migrateBlockEmbeddingsSchema()
- if err := ensureBlocksDocHPathIndex(db); err != nil {
- logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document hpath index failed: %s", err)
+ if err := ensureBlocksDocIndexes(db); err != nil {
+ logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document indexes failed: %s", err)
}
if err := ensureRefsDefIndexes(db); err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create refs definition indexes failed: %s", err)
@@ -207,8 +207,8 @@ func initDBTables() {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create index [idx_blocks_root_id_id_hash] failed: %s", err)
}
- if err = ensureBlocksDocHPathIndex(db); err != nil {
- logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document hpath index failed: %s", err)
+ if err = ensureBlocksDocIndexes(db); err != nil {
+ logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document indexes failed: %s", err)
}
if err = ensureRecentUpdatedBlocksIndexes(db); err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create recent updated block indexes failed: %s", err)
@@ -1448,8 +1448,14 @@ func batchUpdateHPath(tx *sql.Tx, tree *parse.Tree, context map[string]any) (err
return
}
-func ensureBlocksDocHPathIndex(database *sql.DB) (err error) {
- _, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_blocks_doc_hpath ON blocks(hpath) WHERE type = 'd'")
+// 文档部分索引分别支持路径匹配和按根文档聚合,仅包含文档行。
+func ensureBlocksDocIndexes(database *sql.DB) (err error) {
+ for _, column := range []string{"hpath", "root_id"} {
+ if _, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_blocks_doc_" + column +
+ " ON blocks(" + column + ") WHERE type = 'd'"); err != nil {
+ return
+ }
+ }
return
}
@@ -2140,7 +2146,7 @@ func initEncryptedDBTables(boxDB *sql.DB) (err error) {
return
}
}
- if err = ensureBlocksDocHPathIndex(boxDB); err != nil {
+ if err = ensureBlocksDocIndexes(boxDB); err != nil {
return
}
if err = ensureRecentUpdatedBlocksIndexes(boxDB); err != nil {
diff --git a/kernel/sql/document_index_fts5_test.go b/kernel/sql/document_index_fts5_test.go
new file mode 100644
index 0000000000..43fd785921
--- /dev/null
+++ b/kernel/sql/document_index_fts5_test.go
@@ -0,0 +1,57 @@
+//go:build fts5
+
+package sql
+
+import (
+ "fmt"
+ "reflect"
+ "testing"
+)
+
+func TestDocumentPartialIndexesMigration(t *testing.T) {
+ for _, encrypted := range []bool{false, true} {
+ t.Run(fmt.Sprintf("encryptedSchema=%t", encrypted), func(t *testing.T) {
+ database := newDeletePathTestDB(t, encrypted)
+ for _, column := range []string{"hpath", "root_id"} {
+ var ddl string
+ if err := database.QueryRow("SELECT sql FROM sqlite_master WHERE name = ?", "idx_blocks_doc_"+column).Scan(&ddl); err != nil {
+ t.Fatal(err)
+ }
+ if want := "CREATE INDEX idx_blocks_doc_" + column + " ON blocks(" + column + ") WHERE type = 'd'"; ddl != want {
+ t.Fatalf("index definition: got %s, want %s", ddl, want)
+ }
+ }
+ execDeletePathTestSQL(t, database, "INSERT INTO blocks (id, root_id, content, type) VALUES "+
+ "('root', 'root', 'document', 'd'), ('child', 'root', 'paragraph', 'p'), ('unknown', 'root', 'unknown', NULL)")
+ before := deletePathTestIDs(t, database, "SELECT id || ':' || content FROM blocks ORDER BY rowid")
+ execDeletePathTestSQL(t, database, "DROP INDEX idx_blocks_doc_root_id")
+ // 在已有表上补建索引,重复运行不得改变文档内容。
+ for range 2 {
+ if err := ensureBlocksDocIndexes(database); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if after := deletePathTestIDs(t, database, "SELECT id || ':' || content FROM blocks ORDER BY rowid"); !reflect.DeepEqual(after, before) {
+ t.Fatalf("migration changed rows: %v", after)
+ }
+ assertDocuments := func(want []string) {
+ t.Helper()
+ got := deletePathTestIDs(t, database, "SELECT id FROM blocks INDEXED BY idx_blocks_doc_root_id WHERE type = 'd' ORDER BY id")
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("indexed documents: got %v, want %v", got, want)
+ }
+ }
+ assertDocuments([]string{"root"})
+ execDeletePathTestSQL(t, database, "UPDATE blocks SET type = 'd', root_id = 'child' WHERE id = 'child'")
+ assertDocuments([]string{"child", "root"})
+ var indexedRoot string
+ if err := database.QueryRow("SELECT root_id FROM blocks INDEXED BY idx_blocks_doc_root_id WHERE type = 'd' AND root_id = 'child'").Scan(&indexedRoot); err != nil || indexedRoot != "child" {
+ t.Fatalf("root lookup after type change: %q, %v", indexedRoot, err)
+ }
+ execDeletePathTestSQL(t, database, "UPDATE blocks SET type = 'p' WHERE id = 'root'")
+ assertDocuments([]string{"child"})
+ execDeletePathTestSQL(t, database, "DELETE FROM blocks WHERE id = 'child'")
+ assertDocuments(nil)
+ })
+ }
+}
diff --git a/kernel/sql/document_index_sqlcipher_test.go b/kernel/sql/document_index_sqlcipher_test.go
new file mode 100644
index 0000000000..ae8fdb2780
--- /dev/null
+++ b/kernel/sql/document_index_sqlcipher_test.go
@@ -0,0 +1,83 @@
+//go:build fts5 && (sqlcipher || libsqlcipher)
+
+package sql
+
+import (
+ "bytes"
+ "os"
+ "reflect"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestDocumentPartialIndexesEncryptedReopen(t *testing.T) {
+ previousTemp, previousDB, previousEncrypted := util.TempDir, db, IsEncryptedBoxFn
+ util.TempDir = t.TempDir()
+ const box = "document-partial-index-test"
+ db = newDeletePathTestDB(t, false)
+ IsEncryptedBoxFn = func(id string) bool {
+ return id == box || previousEncrypted != nil && previousEncrypted(id)
+ }
+ t.Cleanup(func() {
+ CloseEncryptedDB(box)
+ util.TempDir, db, IsEncryptedBoxFn = previousTemp, previousDB, previousEncrypted
+ })
+ execDeletePathTestSQL(t, db, "INSERT INTO blocks (id, root_id, type, content) VALUES ('decoy', 'decoy', 'd', 'plaintext')")
+ key := bytes.Repeat([]byte{37}, 32)
+ if err := OpenEncryptedDB(box, key); err != nil {
+ t.Fatal(err)
+ }
+ database := GetEncryptedDB(box)
+ var version string
+ if err := database.QueryRow("PRAGMA cipher_version").Scan(&version); err != nil || version == "" {
+ t.Fatalf("SQLCipher required: %q, %v", version, err)
+ }
+ execDeletePathTestSQL(t, database, "INSERT INTO blocks (id, root_id, type, content) VALUES "+
+ "('encrypted', 'encrypted', 'd', 'private title'), ('body', 'encrypted', 'p', 'private body')")
+ const contents = "SELECT id || ':' || content FROM blocks ORDER BY rowid"
+ want := deletePathTestIDs(t, database, contents)
+ // 保存只有路径索引的密文库,通过正式打开流程补建根文档索引。
+ execDeletePathTestSQL(t, database, "DROP INDEX idx_blocks_doc_root_id")
+ CloseEncryptedDB(box)
+ readCiphertext := func() []byte {
+ t.Helper()
+ data, err := os.ReadFile(util.EncryptedDBPath(box))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if bytes.HasPrefix(data, []byte("SQLite format 3")) {
+ t.Fatal("encrypted database has a plaintext header")
+ }
+ return data
+ }
+ original := readCiphertext()
+ if rows, err := queryForBox(box, "SELECT id FROM blocks"); err == nil {
+ if rows != nil {
+ rows.Close()
+ }
+ t.Fatal("locked database query succeeded")
+ }
+ if err := OpenEncryptedDB(box, bytes.Repeat([]byte{38}, 32)); err == nil {
+ t.Fatal("wrong key accepted")
+ }
+ if GetEncryptedDB(box) != nil || !bytes.Equal(readCiphertext(), original) {
+ t.Fatal("failed authentication changed or registered the database")
+ }
+ for range 2 {
+ if err := OpenEncryptedDB(box, key); err != nil {
+ t.Fatal(err)
+ }
+ database = GetEncryptedDB(box)
+ if got := deletePathTestIDs(t, database, contents); !reflect.DeepEqual(got, want) {
+ t.Fatalf("encrypted migration changed content: %v", got)
+ }
+ if got := deletePathTestIDs(t, database, "SELECT id FROM blocks INDEXED BY idx_blocks_doc_root_id WHERE type = 'd'"); !reflect.DeepEqual(got, []string{"encrypted"}) {
+ t.Fatalf("encrypted document lookup: %v", got)
+ }
+ CloseEncryptedDB(box)
+ }
+ if got := deletePathTestIDs(t, db, "SELECT content FROM blocks"); !reflect.DeepEqual(got, []string{"plaintext"}) {
+ t.Fatalf("plaintext database changed: %v", got)
+ }
+}