An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:perf: Avoid unrelated block scans in document-only search https://github.com/siyuan-note/siyuan/issues/20310
Danielce0c8bdae2d3
diff --git a/kernel/model/search.go b/kernel/model/search.goindex 2926f2dfe4..f5c41580ce 100644--- a/kernel/model/search.go+++ b/kernel/model/search.go@@ -2148,6 +2148,10 @@ func buildTypeFilter(types, subTypes map[string]bool, alias ...string) string {}}+ if len(simpleTypes) == 1 && simpleTypes[0] == "d" && len(clauses) == 0 {+ // 文档类型的等值条件与文档部分索引的筛选条件保持一致。+ return "(" + prefix + "type = 'd')"+ }if 0 < len(simpleTypes) {clauses = append([]string{prefix + "type IN (" + sqlQuoteJoin(simpleTypes) + ")"}, clauses...)}diff --git a/kernel/model/search_document_test.go b/kernel/model/search_document_test.gonew file mode 100644index 0000000000..c1367f18f2--- /dev/null+++ b/kernel/model/search_document_test.go@@ -0,0 +1,156 @@+package model++import (+ gosql "database/sql"+ "fmt"+ "reflect"+ "strings"+ "testing"++ "github.com/siyuan-note/siyuan/kernel/conf"+)++func TestDocumentTypeFilterFromSettings(t *testing.T) {+ setSearchCaseSensitive(t, false)+ Conf.Search = &conf.Search{Document: true, Mindmap: new(false), MindmapItem: new(false), CustomBlock: new(false)}+ if got := buildTypeFilter(nil, nil); got != "(type = 'd')" {+ t.Fatalf("document-only settings: %s", got)+ }+ Conf.Search.Paragraph = true+ if got := buildTypeFilter(nil, nil); got != "(type IN ('d','p'))" {+ t.Fatalf("mixed settings: %s", got)+ }+}++func documentSearchRows(t *testing.T, database *gosql.DB, statement string, args []any) [][]any {+ t.Helper()+ rows, err := database.Query(statement, args...)+ if err != nil {+ t.Fatal(err)+ }+ defer rows.Close()+ columns, err := rows.Columns()+ if err != nil {+ t.Fatal(err)+ }+ var result [][]any+ for rows.Next() {+ values := make([]any, len(columns))+ pointers := make([]any, len(columns))+ for i := range values {+ pointers[i] = &values[i]+ }+ if err = rows.Scan(pointers...); err != nil {+ t.Fatal(err)+ }+ for i, value := range values {+ if bytes, ok := value.([]byte); ok {+ values[i] = string(bytes)+ }+ }+ result = append(result, values)+ }+ if err = rows.Err(); err != nil {+ t.Fatal(err)+ }+ return result+}++func TestDocumentSearchPartialIndexResults(t *testing.T) {+ for _, sensitive := range []bool{false, true} {+ t.Run(fmt.Sprintf("case=%t", sensitive), func(t *testing.T) {+ setSearchCaseSensitive(t, sensitive)+ database := newSearchHPathTestDB(t)+ for _, ddl := range []string{+ "CREATE INDEX idx_blocks_root_id ON blocks(root_id)",+ "CREATE INDEX idx_blocks_doc_hpath ON blocks(hpath) WHERE type = 'd'",+ "CREATE INDEX idx_blocks_recent_d ON blocks(updated DESC) WHERE type = 'd'",+ } {+ if _, err := database.Exec(ddl); err != nil {+ t.Fatal(err)+ }+ }+ setting := "OFF"+ if sensitive {+ setting = "ON"+ }+ if _, err := database.Exec("PRAGMA case_sensitive_like = " + setting); err != nil {+ t.Fatal(err)+ }+ for i, block := range [][5]string{+ {"release", "release", "/Projects/Release", "release notes", "d"},+ {"upper", "upper", "/Projects/Upper", "RELEASE NOTES", "d"},+ {"path", "path", "/Projects/release/notes", "path match", "d"},+ {"body", "path", "/Projects/release/notes", "body exclusive", "p"},+ {"quoted", "quoted", "/Other", "author's release", "d"},+ {"comma", "comma", "/Other", "boundary", "d"},+ {"comma", "comma", "/Other", "marker", "d"},+ {"nul", "nul", "/Other", "release\x00notes", "d"},+ } {+ insertSearchHPathTestBlock(t, database, block[0], block[1], block[2], block[3], block[4])+ if _, err := database.Exec("UPDATE blocks SET created = ?, updated = ?, sort = ? WHERE rowid = ?",+ fmt.Sprintf("202610100000%02d", i), fmt.Sprintf("202610100000%02d", 20-i), i, i+1); err != nil {+ t.Fatal(err)+ }+ }+ type comparison struct {+ statement string+ args []any+ rows [][]any+ }+ var comparisons []comparison+ // 对照结果在补建索引之前读取,包含完整投影、统计值及分页顺序。+ for _, hpath := range []bool{false, true} {+ for _, query := range []string{"release notes", "body exclusive", "author's release", "boundary,marker", "release %", "_ notes", "missing words"} {+ keywords := strings.Split(strings.ReplaceAll(query, "'", "''"), " ")+ for order := 0; order <= 7; order++ {+ for page := 1; page <= 3; page++ {+ for _, scope := range []struct {+ box, path, ignore string+ args []any+ }{+ {},+ {box: " AND box = ?", path: " AND path LIKE ?", args: []any{"20260729120000-box000", "/release%"}},+ {ignore: " AND id != 'release'"},+ } {+ args := append(append([]any{}, scope.args...), scope.args...)+ oldStatement := buildDocumentSearchStatement(query, keywords, "(type IN ('d'))", scope.box, scope.path, scope.ignore, order, page, 2, hpath)+ newStatement := buildDocumentSearchStatement(query, keywords, buildTypeFilter(map[string]bool{"document": true}, nil), scope.box, scope.path, scope.ignore, order, page, 2, hpath)+ want := documentSearchRows(t, database, oldStatement, args)+ if query == "release notes" && page == 1 && scope.box == "" && scope.ignore == "" && len(want) == 0 {+ t.Fatal("matching document missing from baseline")+ }+ comparisons = append(comparisons, comparison{newStatement, args, want})+ }+ }+ }+ }+ }+ if _, err := database.Exec("CREATE INDEX idx_blocks_doc_root_id ON blocks(root_id) WHERE type = 'd'"); err != nil {+ t.Fatal(err)+ }+ for _, c := range comparisons {+ if got := documentSearchRows(t, database, c.statement, c.args); !reflect.DeepEqual(got, c.rows) {+ t.Fatalf("search result changed: got %v, want %v; query=%s", got, c.rows, c.statement)+ }+ }+ for _, hpath := range []bool{false, true} {+ statement := buildDocumentSearchStatement("release notes", []string{"release", "notes"}, buildTypeFilter(map[string]bool{"document": true}, nil), "", "", "", 0, 1, 32, hpath)+ plan := documentSearchRows(t, database, "EXPLAIN QUERY PLAN "+statement, nil)+ indexed := 0+ for _, row := range plan {+ detail := row[3].(string)+ if strings.Contains(detail, "USING INDEX idx_blocks_doc_root_id") {+ indexed+++ }+ if strings.Contains(detail, "SCAN blocks USING INDEX idx_blocks_root_id") {+ t.Fatalf("document search scans unrelated blocks: %v", plan)+ }+ }+ if indexed < 2 {+ t.Fatalf("aggregation and result lookup must use the document index: %v", plan)+ }+ }+ })+ }+}diff --git a/kernel/model/search_subtypes_test.go b/kernel/model/search_subtypes_test.goindex 592ff76f0b..a3b6f2da9c 100644--- a/kernel/model/search_subtypes_test.go+++ b/kernel/model/search_subtypes_test.go@@ -32,6 +32,10 @@ func TestBuildTypeFilterSubTypes(t *testing.T) {alias stringwant string}{+ {"document", map[string]bool{"document": true}, nil, "", "(type = 'd')"},+ {"documentAlias", map[string]bool{"document": true}, map[string]bool{"h2": true}, "b.", "(b.type = 'd')"},+ {"documentAndParagraph", map[string]bool{"document": true, "paragraph": true}, nil, "", "(type IN ('d','p'))"},+ {"documentAndHeading", map[string]bool{"document": true, "heading": true}, map[string]bool{"h2": true}, "", "(type IN ('d') OR (type = 'h' AND subtype IN ('h2')))"},{"heading", map[string]bool{"heading": true}, map[string]bool{"h2": true}, "", "((type = 'h' AND subtype IN ('h2')))"},{"list", map[string]bool{"list": true, "listItem": true}, map[string]bool{"list:o": true}, "", "(type IN ('i') OR (type = 'l' AND subtype IN ('o')))"},{"listItem", map[string]bool{"list": true, "listItem": true}, map[string]bool{"listItem:t": true}, "", "(type IN ('l') OR (type = 'i' AND subtype IN ('t')))"},diff --git a/kernel/sql/database.go b/kernel/sql/database.goindex e435c4cf80..9637414b6b 100644--- a/kernel/sql/database.go+++ b/kernel/sql/database.go@@ -119,8 +119,8 @@ func initDatabase(forceRebuild bool) {if util.DatabaseVer == getDatabaseVer() {// 老库版本一致但缺少新加的列时,做幂等迁移(不升 DatabaseVer,避免全库重建丢失已嵌入向量)migrateBlockEmbeddingsSchema()- if err := ensureBlocksDocHPathIndex(db); err != nil {- logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document hpath index failed: %s", err)+ if err := ensureBlocksDocIndexes(db); err != nil {+ logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document indexes failed: %s", err)}if err := ensureRefsDefIndexes(db); err != nil {logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create refs definition indexes failed: %s", err)@@ -207,8 +207,8 @@ func initDBTables() {logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create index [idx_blocks_root_id_id_hash] failed: %s", err)}- if err = ensureBlocksDocHPathIndex(db); err != nil {- logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document hpath index failed: %s", err)+ if err = ensureBlocksDocIndexes(db); err != nil {+ logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document indexes failed: %s", err)}if err = ensureRecentUpdatedBlocksIndexes(db); err != nil {logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create recent updated block indexes failed: %s", err)@@ -1448,8 +1448,14 @@ func batchUpdateHPath(tx *sql.Tx, tree *parse.Tree, context map[string]any) (errreturn}-func ensureBlocksDocHPathIndex(database *sql.DB) (err error) {- _, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_blocks_doc_hpath ON blocks(hpath) WHERE type = 'd'")+// 文档部分索引分别支持路径匹配和按根文档聚合,仅包含文档行。+func ensureBlocksDocIndexes(database *sql.DB) (err error) {+ for _, column := range []string{"hpath", "root_id"} {+ if _, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_blocks_doc_" + column ++ " ON blocks(" + column + ") WHERE type = 'd'"); err != nil {+ return+ }+ }return}@@ -2140,7 +2146,7 @@ func initEncryptedDBTables(boxDB *sql.DB) (err error) {return}}- if err = ensureBlocksDocHPathIndex(boxDB); err != nil {+ if err = ensureBlocksDocIndexes(boxDB); err != nil {return}if err = ensureRecentUpdatedBlocksIndexes(boxDB); err != nil {diff --git a/kernel/sql/document_index_fts5_test.go b/kernel/sql/document_index_fts5_test.gonew file mode 100644index 0000000000..43fd785921--- /dev/null+++ b/kernel/sql/document_index_fts5_test.go@@ -0,0 +1,57 @@+//go:build fts5++package sql++import (+ "fmt"+ "reflect"+ "testing"+)++func TestDocumentPartialIndexesMigration(t *testing.T) {+ for _, encrypted := range []bool{false, true} {+ t.Run(fmt.Sprintf("encryptedSchema=%t", encrypted), func(t *testing.T) {+ database := newDeletePathTestDB(t, encrypted)+ for _, column := range []string{"hpath", "root_id"} {+ var ddl string+ if err := database.QueryRow("SELECT sql FROM sqlite_master WHERE name = ?", "idx_blocks_doc_"+column).Scan(&ddl); err != nil {+ t.Fatal(err)+ }+ if want := "CREATE INDEX idx_blocks_doc_" + column + " ON blocks(" + column + ") WHERE type = 'd'"; ddl != want {+ t.Fatalf("index definition: got %s, want %s", ddl, want)+ }+ }+ execDeletePathTestSQL(t, database, "INSERT INTO blocks (id, root_id, content, type) VALUES "++ "('root', 'root', 'document', 'd'), ('child', 'root', 'paragraph', 'p'), ('unknown', 'root', 'unknown', NULL)")+ before := deletePathTestIDs(t, database, "SELECT id || ':' || content FROM blocks ORDER BY rowid")+ execDeletePathTestSQL(t, database, "DROP INDEX idx_blocks_doc_root_id")+ // 在已有表上补建索引,重复运行不得改变文档内容。+ for range 2 {+ if err := ensureBlocksDocIndexes(database); err != nil {+ t.Fatal(err)+ }+ }+ if after := deletePathTestIDs(t, database, "SELECT id || ':' || content FROM blocks ORDER BY rowid"); !reflect.DeepEqual(after, before) {+ t.Fatalf("migration changed rows: %v", after)+ }+ assertDocuments := func(want []string) {+ t.Helper()+ got := deletePathTestIDs(t, database, "SELECT id FROM blocks INDEXED BY idx_blocks_doc_root_id WHERE type = 'd' ORDER BY id")+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("indexed documents: got %v, want %v", got, want)+ }+ }+ assertDocuments([]string{"root"})+ execDeletePathTestSQL(t, database, "UPDATE blocks SET type = 'd', root_id = 'child' WHERE id = 'child'")+ assertDocuments([]string{"child", "root"})+ var indexedRoot string+ if err := database.QueryRow("SELECT root_id FROM blocks INDEXED BY idx_blocks_doc_root_id WHERE type = 'd' AND root_id = 'child'").Scan(&indexedRoot); err != nil || indexedRoot != "child" {+ t.Fatalf("root lookup after type change: %q, %v", indexedRoot, err)+ }+ execDeletePathTestSQL(t, database, "UPDATE blocks SET type = 'p' WHERE id = 'root'")+ assertDocuments([]string{"child"})+ execDeletePathTestSQL(t, database, "DELETE FROM blocks WHERE id = 'child'")+ assertDocuments(nil)+ })+ }+}diff --git a/kernel/sql/document_index_sqlcipher_test.go b/kernel/sql/document_index_sqlcipher_test.gonew file mode 100644index 0000000000..ae8fdb2780--- /dev/null+++ b/kernel/sql/document_index_sqlcipher_test.go@@ -0,0 +1,83 @@+//go:build fts5 && (sqlcipher || libsqlcipher)++package sql++import (+ "bytes"+ "os"+ "reflect"+ "testing"++ "github.com/siyuan-note/siyuan/kernel/util"+)++func TestDocumentPartialIndexesEncryptedReopen(t *testing.T) {+ previousTemp, previousDB, previousEncrypted := util.TempDir, db, IsEncryptedBoxFn+ util.TempDir = t.TempDir()+ const box = "document-partial-index-test"+ db = newDeletePathTestDB(t, false)+ IsEncryptedBoxFn = func(id string) bool {+ return id == box || previousEncrypted != nil && previousEncrypted(id)+ }+ t.Cleanup(func() {+ CloseEncryptedDB(box)+ util.TempDir, db, IsEncryptedBoxFn = previousTemp, previousDB, previousEncrypted+ })+ execDeletePathTestSQL(t, db, "INSERT INTO blocks (id, root_id, type, content) VALUES ('decoy', 'decoy', 'd', 'plaintext')")+ key := bytes.Repeat([]byte{37}, 32)+ if err := OpenEncryptedDB(box, key); err != nil {+ t.Fatal(err)+ }+ database := GetEncryptedDB(box)+ var version string+ if err := database.QueryRow("PRAGMA cipher_version").Scan(&version); err != nil || version == "" {+ t.Fatalf("SQLCipher required: %q, %v", version, err)+ }+ execDeletePathTestSQL(t, database, "INSERT INTO blocks (id, root_id, type, content) VALUES "++ "('encrypted', 'encrypted', 'd', 'private title'), ('body', 'encrypted', 'p', 'private body')")+ const contents = "SELECT id || ':' || content FROM blocks ORDER BY rowid"+ want := deletePathTestIDs(t, database, contents)+ // 保存只有路径索引的密文库,通过正式打开流程补建根文档索引。+ execDeletePathTestSQL(t, database, "DROP INDEX idx_blocks_doc_root_id")+ CloseEncryptedDB(box)+ readCiphertext := func() []byte {+ t.Helper()+ data, err := os.ReadFile(util.EncryptedDBPath(box))+ if err != nil {+ t.Fatal(err)+ }+ if bytes.HasPrefix(data, []byte("SQLite format 3")) {+ t.Fatal("encrypted database has a plaintext header")+ }+ return data+ }+ original := readCiphertext()+ if rows, err := queryForBox(box, "SELECT id FROM blocks"); err == nil {+ if rows != nil {+ rows.Close()+ }+ t.Fatal("locked database query succeeded")+ }+ if err := OpenEncryptedDB(box, bytes.Repeat([]byte{38}, 32)); err == nil {+ t.Fatal("wrong key accepted")+ }+ if GetEncryptedDB(box) != nil || !bytes.Equal(readCiphertext(), original) {+ t.Fatal("failed authentication changed or registered the database")+ }+ for range 2 {+ if err := OpenEncryptedDB(box, key); err != nil {+ t.Fatal(err)+ }+ database = GetEncryptedDB(box)+ if got := deletePathTestIDs(t, database, contents); !reflect.DeepEqual(got, want) {+ t.Fatalf("encrypted migration changed content: %v", got)+ }+ if got := deletePathTestIDs(t, database, "SELECT id FROM blocks INDEXED BY idx_blocks_doc_root_id WHERE type = 'd'"); !reflect.DeepEqual(got, []string{"encrypted"}) {+ t.Fatalf("encrypted document lookup: %v", got)+ }+ CloseEncryptedDB(box)+ }+ if got := deletePathTestIDs(t, db, "SELECT content FROM blocks"); !reflect.DeepEqual(got, []string{"plaintext"}) {+ t.Fatalf("plaintext database changed: %v", got)+ }+}