An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:zap: Index recent updated paragraphs and documents https://github.com/siyuan-note/siyuan/issues/20251
Daniel4caedcb31cb3
diff --git a/kernel/sql/database.go b/kernel/sql/database.goindex 798712dfa6..851b508ad3 100644--- a/kernel/sql/database.go+++ b/kernel/sql/database.go@@ -134,6 +134,9 @@ func initDatabase(forceRebuild bool) {if err := ensureBookmarkAttributesIndex(db); err != nil {logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create bookmark attributes index failed: %s", err)}+ if err := ensureRecentUpdatedBlocksIndexes(db); err != nil {+ logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create recent updated block indexes failed: %s", err)+ }if err := cleanupInvalidRefs(db); err != nil {logging.LogErrorf("cleanup invalid refs failed: %s", err)}@@ -201,6 +204,9 @@ func initDBTables() {if err = ensureBlocksDocHPathIndex(db); err != nil {logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create document hpath index failed: %s", err)}+ if err = ensureRecentUpdatedBlocksIndexes(db); err != nil {+ logging.LogFatalf(logging.ExitCodeUnavailableDatabase, "create recent updated block indexes failed: %s", err)+ }if err = initFTSBlocks(); err != nil {if isRecoverableDBFileError(err) {@@ -1444,6 +1450,15 @@ func ensureBlocksDocHPathIndex(database *sql.DB) (err error) {return}+// 最近更新查询只索引符合展示条件的段落和文档,按更新时间读取即可在达到限额后停止。+func ensureRecentUpdatedBlocksIndexes(database *sql.DB) (err error) {+ if _, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_blocks_recent_p ON blocks(updated DESC) WHERE type = 'p' AND length > 1"); err != nil {+ return+ }+ _, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_blocks_recent_d ON blocks(updated DESC) WHERE type = 'd'")+ return+}+func ensureTagSpansIndex(database *sql.DB) (err error) {_, err = database.Exec("CREATE INDEX IF NOT EXISTS idx_spans_tag_path ON spans(path) WHERE " + tagSpanIndexPredicate)return@@ -2115,6 +2130,9 @@ func initEncryptedDBTables(boxDB *sql.DB) (err error) {if err = ensureBlocksDocHPathIndex(boxDB); err != nil {return}+ if err = ensureRecentUpdatedBlocksIndexes(boxDB); err != nil {+ return+ }if err = ensureRefsDefIndexes(boxDB); err != nil {return}diff --git a/kernel/sql/recent_index_fts5_test.go b/kernel/sql/recent_index_fts5_test.gonew file mode 100644index 0000000000..bf97eeaf94--- /dev/null+++ b/kernel/sql/recent_index_fts5_test.go@@ -0,0 +1,47 @@+//go:build fts5++package sql++import "testing"++func TestRecentUpdatedBlocksIndexDatabaseInitialization(t *testing.T) {+ for _, encrypted := range []bool{false, true} {+ database := newDeletePathTestDB(t, encrypted)+ for _, name := range []string{"idx_blocks_recent_p", "idx_blocks_recent_d"} {+ var count int+ if err := database.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?", name).Scan(&count); err != nil {+ t.Fatal(err)+ }+ if count != 1 {+ t.Fatalf("encrypted=%v: index %s missing", encrypted, name)+ }+ }+ if _, err := database.Exec("INSERT INTO blocks (id, type, length, updated) VALUES ('paragraph', 'p', 8, '20261008000000')"); err != nil {+ t.Fatal(err)+ }+ if !encrypted {+ if _, err := database.Exec("INSERT INTO block_embeddings (id, embedding) VALUES ('paragraph', x'010203')"); err != nil {+ t.Fatal(err)+ }+ }+ if err := ensureRecentUpdatedBlocksIndexes(database); err != nil {+ t.Fatal(err)+ }+ var updated string+ if err := database.QueryRow("SELECT updated FROM blocks WHERE id = 'paragraph'").Scan(&updated); err != nil {+ t.Fatal(err)+ }+ if updated != "20261008000000" {+ t.Fatal("index migration changed existing block data")+ }+ if !encrypted {+ var embedding string+ if err := database.QueryRow("SELECT hex(embedding) FROM block_embeddings WHERE id = 'paragraph'").Scan(&embedding); err != nil {+ t.Fatal(err)+ }+ if embedding != "010203" {+ t.Fatal("index migration changed existing embedding data")+ }+ }+ }+}diff --git a/kernel/sql/recent_index_test.go b/kernel/sql/recent_index_test.gonew file mode 100644index 0000000000..305ea20d57--- /dev/null+++ b/kernel/sql/recent_index_test.go@@ -0,0 +1,81 @@+package sql++import (+ "fmt"+ "reflect"+ "strings"+ "testing"+)++func TestRecentUpdatedBlocksIndexesPreserveResults(t *testing.T) {+ database := createGraphTestBlocksTable(t)+ previous := db+ db = database+ t.Cleanup(func() { db = previous })+ insert := func(id, blockType, updated string, length int) {+ t.Helper()+ if _, err := database.Exec("INSERT INTO blocks VALUES (?, '', 'root', '', 'box', '/doc.sy', '/doc', '', '', '', '', 'content', '', 'markdown', ?, ?, '', '', 0, '2026', ?)", id, length, blockType, updated); err != nil {+ t.Fatal(err)+ }+ }+ for _, blockType := range []string{"p", "d"} {+ for i := 0; i < 40; i++ {+ insert(fmt.Sprintf("%s-%02d", blockType, i), blockType, fmt.Sprintf("20261008%06d", i), 8)+ }+ }+ insert("short", "p", "20991231235959", 1)+ insert("heading", "h", "20991231235959", 8)+ ids := func(blocks []*Block) []string {+ ret := make([]string, 0, len(blocks))+ for _, block := range blocks {+ ret = append(ret, block.ID)+ }+ return ret+ }+ conditions := []string{"type = 'p' AND length > 1", "type = 'd'"}+ var before [][]string+ for _, condition := range conditions {+ before = append(before, ids(SelectBlocksRawStmt("SELECT * FROM blocks WHERE "+condition+" ORDER BY updated DESC", 1, 16)))+ before = append(before, ids(SelectBlocksRawStmt("SELECT * FROM blocks WHERE "+condition+" AND id NOT LIKE '%-39' ORDER BY updated DESC", 1, 16)))+ }+ for i := 0; i < 2; i++ {+ if err := ensureRecentUpdatedBlocksIndexes(database); err != nil {+ t.Fatal(err)+ }+ }+ var after [][]string+ for i, condition := range conditions {+ for _, ignore := range []string{"", " AND id NOT LIKE '%-39'"} {+ stmt := "SELECT * FROM blocks WHERE " + condition + ignore + " ORDER BY updated DESC"+ after = append(after, ids(SelectBlocksRawStmt(stmt, 1, 16)))+ index := "idx_blocks_recent_p"+ if i == 1 {+ index = "idx_blocks_recent_d"+ }+ plan := queryPlanDetails(t, database, stmt+" LIMIT 16 OFFSET 0")+ if !strings.Contains(plan, index) || strings.Contains(plan, "TEMP B-TREE") {+ t.Fatalf("recent query scans or sorts without its index: %s", plan)+ }+ }+ }+ if !reflect.DeepEqual(before, after) || len(after[0]) != 16 || after[0][0] != "p-39" || after[2][0] != "d-39" {+ t.Fatalf("recent results changed: before=%v after=%v", before, after)+ }+ // 更新时间和展示条件改变后,索引同步反映新结果,无需失效缓存。+ if _, err := database.Exec("UPDATE blocks SET updated = '20991231235959' WHERE id IN ('p-00', 'd-00')"); err != nil {+ t.Fatal(err)+ }+ for i, condition := range conditions {+ results := ids(SelectBlocksRawStmt("SELECT * FROM blocks WHERE "+condition+" ORDER BY updated DESC", 1, 16))+ if results[0] != []string{"p-00", "d-00"}[i] {+ t.Fatalf("updated block missing from recent results: %v", results)+ }+ }+ if _, err := database.Exec("UPDATE blocks SET length = 1 WHERE id = 'p-00'"); err != nil {+ t.Fatal(err)+ }+ results := ids(SelectBlocksRawStmt("SELECT * FROM blocks WHERE type = 'p' AND length > 1 ORDER BY updated DESC", 1, 16))+ if results[0] != "p-39" {+ t.Fatalf("short paragraph still in recent results: %v", results)+ }+}