Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:zap: Index recent updated paragraphs and documents https://github.com/siyuan-note/siyuan/issues/20251

Daniel4caedcb31cb3
diff --git a/kernel/sql/database.go b/kernel/sql/database.go
index 798712dfa6..851b508ad3 100644
--- a/kernel/sql/database.go
+++ b/kernel/sql/database.go
@@ -134,6 +134,9 @@ func initDatabase(forceRebuild bool) {
if err := ensureBookmarkAttributesIndex(db); err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create bookmark attributes index failed: %s", err)
}
+ if err := ensureRecentUpdatedBlocksIndexes(db); err != nil {
+ logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create recent updated block indexes failed: %s", err)
+ }
if err := cleanupInvalidRefs(db); err != nil {
logging.LogErrorf("cleanup invalid refs failed: %s", err)
}
@@ -201,6 +204,9 @@ func initDBTables() {
if err = ensureBlocksDocHPathIndex(db); err != nil {
logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document hpath index failed: %s", err)
}
+ if err = ensureRecentUpdatedBlocksIndexes(db); err != nil {
+ logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create recent updated block indexes failed: %s", err)
+ }
if err = initFTSBlocks(); err != nil {
if isRecoverableDBFileError(err) {
@@ -1444,6 +1450,15 @@ func ensureBlocksDocHPathIndex(database *sql.DB) (err error) {
return
}
+// 最近更新查询只索引符合展示条件的段落和文档,按更新时间读取即可在达到限额后停止。
+func ensureRecentUpdatedBlocksIndexes(database *sql.DB) (err error) {
+ if _, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_blocks_recent_p ON blocks(updated DESC) WHERE type = 'p' AND length > 1"); err != nil {
+ return
+ }
+ _, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_blocks_recent_d ON blocks(updated DESC) WHERE type = 'd'")
+ return
+}
+
func ensureTagSpansIndex(database *sql.DB) (err error) {
_, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_spans_tag_path ON spans(path) WHERE " + tagSpanIndexPredicate)
return
@@ -2115,6 +2130,9 @@ func initEncryptedDBTables(boxDB *sql.DB) (err error) {
if err = ensureBlocksDocHPathIndex(boxDB); err != nil {
return
}
+ if err = ensureRecentUpdatedBlocksIndexes(boxDB); err != nil {
+ return
+ }
if err = ensureRefsDefIndexes(boxDB); err != nil {
return
}
diff --git a/kernel/sql/recent_index_fts5_test.go b/kernel/sql/recent_index_fts5_test.go
new file mode 100644
index 0000000000..bf97eeaf94
--- /dev/null
+++ b/kernel/sql/recent_index_fts5_test.go
@@ -0,0 +1,47 @@
+//go:build fts5
+
+package sql
+
+import "testing"
+
+func TestRecentUpdatedBlocksIndexDatabaseInitialization(t *testing.T) {
+ for _, encrypted := range []bool{false, true} {
+ database := newDeletePathTestDB(t, encrypted)
+ for _, name := range []string{"idx_blocks_recent_p", "idx_blocks_recent_d"} {
+ var count int
+ if err := database.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?", name).Scan(&count); err != nil {
+ t.Fatal(err)
+ }
+ if count != 1 {
+ t.Fatalf("encrypted=%v: index %s missing", encrypted, name)
+ }
+ }
+ if _, err := database.Exec("INSERT INTO blocks (id, type, length, updated) VALUES ('paragraph', 'p', 8, '20261008000000')"); err != nil {
+ t.Fatal(err)
+ }
+ if !encrypted {
+ if _, err := database.Exec("INSERT INTO block_embeddings (id, embedding) VALUES ('paragraph', x'010203')"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := ensureRecentUpdatedBlocksIndexes(database); err != nil {
+ t.Fatal(err)
+ }
+ var updated string
+ if err := database.QueryRow("SELECT updated FROM blocks WHERE id = 'paragraph'").Scan(&updated); err != nil {
+ t.Fatal(err)
+ }
+ if updated != "20261008000000" {
+ t.Fatal("index migration changed existing block data")
+ }
+ if !encrypted {
+ var embedding string
+ if err := database.QueryRow("SELECT hex(embedding) FROM block_embeddings WHERE id = 'paragraph'").Scan(&embedding); err != nil {
+ t.Fatal(err)
+ }
+ if embedding != "010203" {
+ t.Fatal("index migration changed existing embedding data")
+ }
+ }
+ }
+}
diff --git a/kernel/sql/recent_index_test.go b/kernel/sql/recent_index_test.go
new file mode 100644
index 0000000000..305ea20d57
--- /dev/null
+++ b/kernel/sql/recent_index_test.go
@@ -0,0 +1,81 @@
+package sql
+
+import (
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestRecentUpdatedBlocksIndexesPreserveResults(t *testing.T) {
+ database := createGraphTestBlocksTable(t)
+ previous := db
+ db = database
+ t.Cleanup(func() { db = previous })
+ insert := func(id, blockType, updated string, length int) {
+ t.Helper()
+ if _, err := database.Exec("INSERT INTO blocks VALUES (?, '', 'root', '', 'box', '/doc.sy', '/doc', '', '', '', '', 'content', '', 'markdown', ?, ?, '', '', 0, '2026', ?)", id, length, blockType, updated); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for _, blockType := range []string{"p", "d"} {
+ for i := 0; i < 40; i++ {
+ insert(fmt.Sprintf("%s-%02d", blockType, i), blockType, fmt.Sprintf("20261008%06d", i), 8)
+ }
+ }
+ insert("short", "p", "20991231235959", 1)
+ insert("heading", "h", "20991231235959", 8)
+ ids := func(blocks []*Block) []string {
+ ret := make([]string, 0, len(blocks))
+ for _, block := range blocks {
+ ret = append(ret, block.ID)
+ }
+ return ret
+ }
+ conditions := []string{"type = 'p' AND length > 1", "type = 'd'"}
+ var before [][]string
+ for _, condition := range conditions {
+ before = append(before, ids(SelectBlocksRawStmt("SELECT * FROM blocks WHERE "+condition+" ORDER BY updated DESC", 1, 16)))
+ before = append(before, ids(SelectBlocksRawStmt("SELECT * FROM blocks WHERE "+condition+" AND id NOT LIKE '%-39' ORDER BY updated DESC", 1, 16)))
+ }
+ for i := 0; i < 2; i++ {
+ if err := ensureRecentUpdatedBlocksIndexes(database); err != nil {
+ t.Fatal(err)
+ }
+ }
+ var after [][]string
+ for i, condition := range conditions {
+ for _, ignore := range []string{"", " AND id NOT LIKE '%-39'"} {
+ stmt := "SELECT * FROM blocks WHERE " + condition + ignore + " ORDER BY updated DESC"
+ after = append(after, ids(SelectBlocksRawStmt(stmt, 1, 16)))
+ index := "idx_blocks_recent_p"
+ if i == 1 {
+ index = "idx_blocks_recent_d"
+ }
+ plan := queryPlanDetails(t, database, stmt+" LIMIT 16 OFFSET 0")
+ if !strings.Contains(plan, index) || strings.Contains(plan, "TEMP B-TREE") {
+ t.Fatalf("recent query scans or sorts without its index: %s", plan)
+ }
+ }
+ }
+ if !reflect.DeepEqual(before, after) || len(after[0]) != 16 || after[0][0] != "p-39" || after[2][0] != "d-39" {
+ t.Fatalf("recent results changed: before=%v after=%v", before, after)
+ }
+ // 更新时间和展示条件改变后,索引同步反映新结果,无需失效缓存。
+ if _, err := database.Exec("UPDATE blocks SET updated = '20991231235959' WHERE id IN ('p-00', 'd-00')"); err != nil {
+ t.Fatal(err)
+ }
+ for i, condition := range conditions {
+ results := ids(SelectBlocksRawStmt("SELECT * FROM blocks WHERE "+condition+" ORDER BY updated DESC", 1, 16))
+ if results[0] != []string{"p-00", "d-00"}[i] {
+ t.Fatalf("updated block missing from recent results: %v", results)
+ }
+ }
+ if _, err := database.Exec("UPDATE blocks SET length = 1 WHERE id = 'p-00'"); err != nil {
+ t.Fatal(err)
+ }
+ results := ids(SelectBlocksRawStmt("SELECT * FROM blocks WHERE type = 'p' AND length > 1 ORDER BY updated DESC", 1, 16))
+ if results[0] != "p-39" {
+ t.Fatalf("short paragraph still in recent results: %v", results)
+ }
+}