Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:zap: Avoid unnecessary database collection work on unrelated edits https://github.com/siyuan-note/siyuan/issues/20782

Daniel2ebf849f69b4
diff --git a/kernel/av/automation.go b/kernel/av/automation.go
index 06c097dfb1..ef34d71650 100644
--- a/kernel/av/automation.go
+++ b/kernel/av/automation.go
@@ -47,6 +47,25 @@ type AutomationBlockAttribute struct {
Value string `json:"value,omitempty"`
}
+func (source *AutomationBlockSource) MatchesDocument(notebookID, documentID, documentPath string) bool {
+ if source == nil {
+ return false
+ }
+ if slices.Contains(source.NotebookIDs, notebookID) {
+ return true
+ }
+ for _, id := range source.DocumentIDs {
+ if documentID == id {
+ return true
+ }
+ if _, suffix, found := strings.Cut(documentPath, "/"+id+"/"); found &&
+ (source.DocumentDepth == nil || strings.Count(suffix, "/")+1 <= *source.DocumentDepth) {
+ return true
+ }
+ }
+ return false
+}
+
// IsAutomationBlockType 校验主类型或主类型与子类型的组合,提示块允许自定义类型名称。
func IsAutomationBlockType(value string) bool {
typ, sub, qualified := strings.Cut(value, ":")
diff --git a/kernel/av/collection_index.go b/kernel/av/collection_index.go
index 443597a3fa..d7b04af592 100644
--- a/kernel/av/collection_index.go
+++ b/kernel/av/collection_index.go
@@ -2,9 +2,11 @@ package av
import (
"encoding/json"
+ "iter"
"maps"
"os"
"path/filepath"
+ "slices"
"sort"
"strings"
"sync"
@@ -14,56 +16,181 @@ import (
"github.com/siyuan-note/siyuan/kernel/util"
)
-// 索引只保存数据库 ID,不保留加密笔记本的规则、属性或正文。同步清理缓存后重新发现规则。
+type collectionBoxIndex struct {
+ loaded bool
+ views map[string][]*AutomationBlockSource
+ snapshot *BlockCollectionIndex
+}
+
+// 索引只保留收集范围,不保留属性条件、赋值或正文;加密笔记本锁定时清理其范围。
var collectionIndex = struct {
- sync.Mutex
+ sync.RWMutex
directory string
generation uint64
- boxes map[string]map[string]bool
- loaded map[string]bool
-}{boxes: map[string]map[string]bool{}, loaded: map[string]bool{}}
+ boxes map[string]*collectionBoxIndex
+}{boxes: map[string]*collectionBoxIndex{}}
+
+// BlockCollectionIndex 是按加密边界共享的不可变范围快照,事务可复用到缓存失效或范围改变。
+type BlockCollectionIndex struct {
+ boxID string
+ ids []string
+ scopes map[string][]*AutomationBlockSource
+}
+
+func collectionIndexCurrentLocked() bool {
+ return collectionIndex.directory == util.DataDir && collectionIndex.generation == cache.GetAVCacheGeneration()
+}
func resetCollectionIndexLocked() {
- generation := cache.GetAVCacheGeneration()
- if collectionIndex.directory != util.DataDir || collectionIndex.generation != generation {
- collectionIndex.directory, collectionIndex.generation = util.DataDir, generation
- collectionIndex.boxes, collectionIndex.loaded = map[string]map[string]bool{}, map[string]bool{}
+ if !collectionIndexCurrentLocked() {
+ collectionIndex.directory, collectionIndex.generation = util.DataDir, cache.GetAVCacheGeneration()
+ collectionIndex.boxes = map[string]*collectionBoxIndex{}
}
}
-func hasBlockCollection(config *AutomationConfig) bool {
- if config != nil {
- for _, rule := range config.Rules {
- if rule != nil && rule.Enabled && rule.Trigger == "block" {
- return true
- }
+// ClearBlockCollectionIndex 在持有笔记本写锁时移除已解密的范围,避免锁定后保留文档 ID。
+func ClearBlockCollectionIndex(boxID string) {
+ collectionIndex.Lock()
+ defer collectionIndex.Unlock()
+ delete(collectionIndex.boxes, boxID)
+}
+
+// BlockCollectionIndexKnownEmpty 只确认当前代次已经扫描过的空索引,未知状态仍需首次扫描。
+func BlockCollectionIndexKnownEmpty(boxID string) bool {
+ collectionIndex.RLock()
+ defer collectionIndex.RUnlock()
+ box := collectionIndex.boxes[boxID]
+ return collectionIndexCurrentLocked() && box != nil && box.loaded && len(box.snapshot.ids) == 0
+}
+
+func (index *BlockCollectionIndex) Current() bool {
+ collectionIndex.RLock()
+ defer collectionIndex.RUnlock()
+ box := collectionIndex.boxes[index.boxID]
+ return collectionIndexCurrentLocked() && box != nil && box.snapshot == index
+}
+
+func (index *BlockCollectionIndex) Empty() bool { return len(index.ids) == 0 }
+
+func (index *BlockCollectionIndex) MatchesView(id, notebookID, documentID, documentPath string) bool {
+ for _, scope := range index.scopes[id] {
+ if scope == nil || scope.MatchesDocument(notebookID, documentID, documentPath) {
+ return true
}
}
return false
}
+func (index *BlockCollectionIndex) MatchesDocument(notebookID, documentID, documentPath string) bool {
+ for _, id := range index.ids {
+ if index.MatchesView(id, notebookID, documentID, documentPath) {
+ return true
+ }
+ }
+ return false
+}
+
+// ViewIDsForDocument 只遍历范围匹配的数据库,不为每个文档复制、排序 ID。
+func (index *BlockCollectionIndex) ViewIDsForDocument(notebookID, documentID, documentPath string) iter.Seq[string] {
+ return func(yield func(string) bool) {
+ for _, id := range index.ids {
+ if index.MatchesView(id, notebookID, documentID, documentPath) && !yield(id) {
+ return
+ }
+ }
+ }
+}
+
+func collectionScopes(config *AutomationConfig) (ret []*AutomationBlockSource) {
+ if config == nil {
+ return
+ }
+ unknownSpec := config.ValidateSpec() != nil
+ for _, rule := range config.Rules {
+ if rule == nil || !rule.Enabled || rule.Trigger != "block" {
+ continue
+ }
+ // 未知语义不能按已知范围排除,交给完整读取器报告兼容性错误。
+ if unknownSpec || rule.BlockSource == nil {
+ return []*AutomationBlockSource{nil}
+ }
+ source := rule.BlockSource
+ scope := &AutomationBlockSource{NotebookIDs: slices.Clone(source.NotebookIDs), DocumentIDs: slices.Clone(source.DocumentIDs)}
+ if source.DocumentDepth != nil {
+ depth := *source.DocumentDepth
+ scope.DocumentDepth = &depth
+ }
+ ret = append(ret, scope)
+ }
+ return
+}
+
+func equalCollectionScopes(left, right []*AutomationBlockSource) bool {
+ return slices.EqualFunc(left, right, func(a, b *AutomationBlockSource) bool {
+ if a == nil || b == nil {
+ return a == b
+ }
+ return slices.Equal(a.NotebookIDs, b.NotebookIDs) && slices.Equal(a.DocumentIDs, b.DocumentIDs) &&
+ (a.DocumentDepth == nil && b.DocumentDepth == nil || a.DocumentDepth != nil && b.DocumentDepth != nil && *a.DocumentDepth == *b.DocumentDepth)
+ })
+}
+
+func collectionSnapshot(boxID string, views map[string][]*AutomationBlockSource) *BlockCollectionIndex {
+ ret := &BlockCollectionIndex{boxID: boxID, scopes: map[string][]*AutomationBlockSource{}}
+ for id, scopes := range views {
+ if len(scopes) > 0 {
+ ret.ids = append(ret.ids, id)
+ ret.scopes[id] = scopes
+ }
+ }
+ sort.Strings(ret.ids)
+ return ret
+}
+
func rememberCollectionView(view *AttributeView, boxID string) {
+ scopes := collectionScopes(view.Automations)
collectionIndex.Lock()
defer collectionIndex.Unlock()
resetCollectionIndexLocked()
- if collectionIndex.boxes[boxID] == nil {
- collectionIndex.boxes[boxID] = map[string]bool{}
+ box := collectionIndex.boxes[boxID]
+ if box == nil {
+ box = &collectionBoxIndex{views: map[string][]*AutomationBlockSource{}}
+ collectionIndex.boxes[boxID] = box
+ }
+ previous := box.views[view.ID]
+ box.views[view.ID] = scopes
+ if box.loaded && !equalCollectionScopes(previous, scopes) {
+ box.snapshot = collectionSnapshot(boxID, box.views)
}
- collectionIndex.boxes[boxID][view.ID] = hasBlockCollection(view.Automations)
}
-// BlockCollectionViewIDsInBox 首次访问时发现规则,此后保存数据库直接更新索引,不逐次扫描目录。
-func BlockCollectionViewIDsInBox(boxID string) ([]string, error) {
- collectionIndex.Lock()
- resetCollectionIndexLocked()
- loaded, generation, directory := collectionIndex.loaded[boxID], collectionIndex.generation, collectionIndex.directory
- collectionIndex.Unlock()
- if !loaded {
- entries, readErr := os.ReadDir(filepath.Join(util.DataDir, boxID, "storage", "av"))
+// BlockCollectionIndexInBox 首次完整读取并认证数据库,此后保存数据库直接更新范围快照。
+func BlockCollectionIndexInBox(boxID string) (*BlockCollectionIndex, error) {
+ for {
+ release, err := holdAVBoxReadLock(boxID)
+ if err != nil {
+ return nil, err
+ }
+ collectionIndex.Lock()
+ resetCollectionIndexLocked()
+ box := collectionIndex.boxes[boxID]
+ if box == nil {
+ box = &collectionBoxIndex{views: map[string][]*AutomationBlockSource{}}
+ collectionIndex.boxes[boxID] = box
+ }
+ generation, directory := collectionIndex.generation, collectionIndex.directory
+ snapshot := box.snapshot
+ collectionIndex.Unlock()
+ release()
+ if snapshot != nil {
+ return snapshot, nil
+ }
+
+ entries, readErr := os.ReadDir(filepath.Join(directory, boxID, "storage", "av"))
if readErr != nil && !os.IsNotExist(readErr) {
return nil, readErr
}
- ids := map[string]bool{}
+ views := map[string][]*AutomationBlockSource{}
for _, entry := range entries {
id := strings.TrimSuffix(entry.Name(), ".json")
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(id) {
@@ -74,31 +201,37 @@ func BlockCollectionViewIDsInBox(boxID string) ([]string, error) {
return nil, readErr
}
var header struct {
+ Spec int `json:"spec"`
Automations *AutomationConfig `json:"automations"`
}
if readErr = json.Unmarshal(data, &header); readErr != nil {
return nil, readErr
}
- ids[id] = hasBlockCollection(header.Automations)
+ views[id] = collectionScopes(header.Automations)
+ if header.Spec > CurrentSpec && len(views[id]) > 0 {
+ views[id] = []*AutomationBlockSource{nil}
+ }
+ }
+ // 读取期间不递归持有笔记本读锁;发布时再次检查锁定状态和缓存代次。
+ release, err = holdAVBoxReadLock(boxID)
+ if err != nil {
+ return nil, err
}
collectionIndex.Lock()
resetCollectionIndexLocked()
- if generation != collectionIndex.generation || directory != collectionIndex.directory {
+ if generation != collectionIndex.generation || directory != collectionIndex.directory || collectionIndex.boxes[boxID] != box {
collectionIndex.Unlock()
- return BlockCollectionViewIDsInBox(boxID)
+ release()
+ continue
}
- maps.Copy(ids, collectionIndex.boxes[boxID])
- collectionIndex.boxes[boxID], collectionIndex.loaded[boxID] = ids, true
- collectionIndex.Unlock()
- }
- collectionIndex.Lock()
- defer collectionIndex.Unlock()
- ret := []string{}
- for id, enabled := range collectionIndex.boxes[boxID] {
- if enabled {
- ret = append(ret, id)
+ maps.Copy(views, box.views)
+ box.views, box.loaded = views, true
+ if box.snapshot == nil {
+ box.snapshot = collectionSnapshot(boxID, views)
}
+ snapshot = box.snapshot
+ collectionIndex.Unlock()
+ release()
+ return snapshot, nil
}
- sort.Strings(ret)
- return ret, nil
}
diff --git a/kernel/av/collection_index_test.go b/kernel/av/collection_index_test.go
new file mode 100644
index 0000000000..da4031f2e7
--- /dev/null
+++ b/kernel/av/collection_index_test.go
@@ -0,0 +1,253 @@
+package av
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "slices"
+ "testing"
+
+ "github.com/88250/lute/ast"
+ "github.com/siyuan-note/siyuan/kernel/cache"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func setupCollectionIndexTest(t *testing.T) {
+ t.Helper()
+ oldData := util.DataDir
+ util.DataDir = t.TempDir()
+ cache.ClearAVCache()
+ t.Cleanup(func() {
+ util.DataDir = oldData
+ cache.ClearAVCache()
+ collectionIndex.Lock()
+ resetCollectionIndexLocked()
+ collectionIndex.Unlock()
+ })
+}
+
+func writeCollectionIndexView(t *testing.T, view *AttributeView) {
+ t.Helper()
+ data, err := json.Marshal(view)
+ if err != nil {
+ t.Fatal(err)
+ }
+ path := attributeViewDataPathByBox(view.ID, "")
+ if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(path, data, 0600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func collectionIndexTestView() *AttributeView {
+ return &AttributeView{ID: ast.NewNodeID(), Automations: &AutomationConfig{Spec: 5, Rules: []*AutomationRule{{
+ Enabled: true, Trigger: "block", BlockSource: &AutomationBlockSource{DocumentIDs: []string{ast.NewNodeID()}, DocumentDepth: new(1)},
+ }}}}
+}
+
+func TestBlockCollectionIndexEmptyAndInvalidation(t *testing.T) {
+ setupCollectionIndexTest(t)
+ if BlockCollectionIndexKnownEmpty("") {
+ t.Fatal("unknown index was treated as empty")
+ }
+ index, err := BlockCollectionIndexInBox("")
+ if err != nil || !index.Empty() || !BlockCollectionIndexKnownEmpty("") {
+ t.Fatal("empty directory was not cached", err)
+ }
+ if allocations := testing.AllocsPerRun(100, func() {
+ if !BlockCollectionIndexKnownEmpty("") {
+ t.Fatal("lost empty cache")
+ }
+ }); allocations != 0 {
+ t.Fatalf("empty hot path allocated: %g", allocations)
+ }
+ view := collectionIndexTestView()
+ writeCollectionIndexView(t, view)
+ cache.ClearAVCache()
+ if index.Current() || BlockCollectionIndexKnownEmpty("") {
+ t.Fatal("cache generation did not invalidate empty snapshot")
+ }
+ index, err = BlockCollectionIndexInBox("")
+ if err != nil || !slices.Equal(index.ids, []string{view.ID}) {
+ t.Fatal("cold discovery missed persisted rule", err)
+ }
+ oldData := util.DataDir
+ util.DataDir = t.TempDir()
+ if index.Current() || BlockCollectionIndexKnownEmpty("") {
+ t.Fatal("workspace switch retained old index")
+ }
+ other, err := BlockCollectionIndexInBox("")
+ util.DataDir = oldData
+ if err != nil || !other.Empty() {
+ t.Fatal("workspace index was not isolated", err)
+ }
+}
+
+func TestBlockCollectionIndexScopesAndSnapshots(t *testing.T) {
+ setupCollectionIndexTest(t)
+ view := collectionIndexTestView()
+ source := view.Automations.Rules[0].BlockSource
+ parent, child, notebook := source.DocumentIDs[0], ast.NewNodeID(), ast.NewNodeID()
+ writeCollectionIndexView(t, view)
+ index, err := BlockCollectionIndexInBox("")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, test := range []struct {
+ id, path string
+ want bool
+ }{
+ {parent, "/" + parent + ".sy", true},
+ {child, "/" + parent + "/" + child + ".sy", true},
+ {child, "/" + parent + "/" + ast.NewNodeID() + "/" + child + ".sy", false},
+ {child, "/" + child + ".sy", false},
+ } {
+ if got := index.MatchesDocument(notebook, test.id, test.path); got != test.want {
+ t.Fatalf("document scope %s: %v, want %v", test.path, got, test.want)
+ }
+ }
+ view.Name = "Updated rows and display only"
+ rememberCollectionView(view, "")
+ if !index.Current() {
+ t.Fatal("unchanged scope invalidated shared snapshot")
+ }
+ rememberCollectionView(&AttributeView{ID: ast.NewNodeID()}, "")
+ if !index.Current() {
+ t.Fatal("database without rules invalidated shared snapshot")
+ }
+ source.NotebookIDs = []string{notebook}
+ rememberCollectionView(view, "")
+ updated, err := BlockCollectionIndexInBox("")
+ if err != nil || index.Current() || !updated.MatchesDocument(notebook, child, "/"+child+".sy") {
+ t.Fatal("scope update did not invalidate snapshot", err)
+ }
+ if index.MatchesDocument(notebook, child, "/"+child+".sy") {
+ t.Fatal("saved scope mutated earlier snapshot")
+ }
+ source.NotebookIDs[0] = ast.NewNodeID()
+ if !updated.MatchesDocument(notebook, child, "/"+child+".sy") {
+ t.Fatal("caller mutated cached scope")
+ }
+ if allocations := testing.AllocsPerRun(100, func() {
+ for id := range updated.ViewIDsForDocument(notebook, child, "/child.sy") {
+ if id != view.ID {
+ t.Fatal("unexpected view")
+ }
+ }
+ }); allocations != 0 {
+ t.Fatalf("candidate iteration allocated: %g", allocations)
+ }
+ view.Automations.Rules[0].Enabled = false
+ rememberCollectionView(view, "")
+ if updated.Current() || !BlockCollectionIndexKnownEmpty("") {
+ t.Fatal("disabling last rule did not update empty fast path")
+ }
+ view.Automations.Rules[0].Enabled = true
+ view.Automations.Spec = 7
+ rememberCollectionView(view, "")
+ unknown, err := BlockCollectionIndexInBox("")
+ if err != nil || !unknown.MatchesDocument("unrelated", child, "/child.sy") {
+ t.Fatal("unknown specification was silently excluded", err)
+ }
+ ClearBlockCollectionIndex("")
+ if unknown.Current() || BlockCollectionIndexKnownEmpty("") {
+ t.Fatal("explicit clear retained snapshot")
+ }
+}
+
+func TestBlockCollectionIndexCorruptionFailsClosed(t *testing.T) {
+ setupCollectionIndexTest(t)
+ view := collectionIndexTestView()
+ writeCollectionIndexView(t, view)
+ path := attributeViewDataPathByBox(view.ID, "")
+ corrupt := []byte(`{"automations":`)
+ if err := os.WriteFile(path, corrupt, 0600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := BlockCollectionIndexInBox(""); err == nil || BlockCollectionIndexKnownEmpty("") {
+ t.Fatal("corrupt database was treated as empty")
+ }
+ retained, err := os.ReadFile(path)
+ if err != nil || !bytes.Equal(retained, corrupt) {
+ t.Fatal("index discovery changed corrupt source", err)
+ }
+ view.Spec = CurrentSpec + 1
+ writeCollectionIndexView(t, view)
+ index, err := BlockCollectionIndexInBox("")
+ if err != nil || !index.MatchesDocument("unrelated", "unrelated", "/unrelated.sy") {
+ t.Fatal("unknown database specification was silently excluded", err)
+ }
+}
+
+func TestBlockCollectionIndexEncryptedAuthenticationAndLock(t *testing.T) {
+ setupCollectionIndexTest(t)
+ boxID := ast.NewNodeID()
+ view := collectionIndexTestView()
+ oldDEK, oldEncrypted, oldUnlocked := AVDEKProvider, AVIsEncryptedBox, AVIsBoxUnlocked
+ oldAcquire, oldRelease := AVLockAcquire, AVLockRelease
+ key, unlocked := bytes.Repeat([]byte{0x71}, 32), true
+ AVDEKProvider = func(string) ([]byte, error) { return key, nil }
+ AVIsEncryptedBox = func(id string) bool { return id == boxID }
+ AVIsBoxUnlocked = func(string) bool { return unlocked }
+ AVLockAcquire, AVLockRelease = func(string) {}, func(string) {}
+ SetAVBoxID(view.ID, boxID)
+ t.Cleanup(func() {
+ SetAVBoxID(view.ID, "")
+ AVDEKProvider, AVIsEncryptedBox, AVIsBoxUnlocked = oldDEK, oldEncrypted, oldUnlocked
+ AVLockAcquire, AVLockRelease = oldAcquire, oldRelease
+ })
+ plain, _ := json.Marshal(view)
+ ciphertext, err := encryptAVData(boxID, view.ID, plain)
+ if err != nil {
+ t.Fatal(err)
+ }
+ path := attributeViewDataPathByBox(view.ID, boxID)
+ if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(path, ciphertext, 0600); err != nil {
+ t.Fatal(err)
+ }
+ // 扫描过程中发布的新范围优先于扫描开始时的文件内容。
+ AVDEKProvider = func(string) ([]byte, error) {
+ updated := collectionIndexTestView()
+ updated.ID = view.ID
+ updated.Automations.Rules[0].BlockSource.NotebookIDs = []string{boxID}
+ rememberCollectionView(updated, boxID)
+ return key, nil
+ }
+ index, err := BlockCollectionIndexInBox(boxID)
+ if err != nil || !index.MatchesDocument(boxID, "other", "/other.sy") {
+ t.Fatal("concurrent saved scope was lost", err)
+ }
+ ClearBlockCollectionIndex(boxID)
+ unlocked = false
+ if index.Current() {
+ t.Fatal("locked boundary retained plaintext scope")
+ }
+ if _, err = BlockCollectionIndexInBox(boxID); err == nil {
+ t.Fatal("locked boundary returned cached rules")
+ }
+ unlocked = true
+ AVDEKProvider = func(string) ([]byte, error) { unlocked = false; return key, nil }
+ if _, err = BlockCollectionIndexInBox(boxID); err == nil || BlockCollectionIndexKnownEmpty(boxID) {
+ t.Fatal("scan published a scope after locking")
+ }
+ unlocked = true
+ AVDEKProvider = func(string) ([]byte, error) { return key, nil }
+ corrupt := bytes.Clone(ciphertext)
+ corrupt[len(corrupt)-1] ^= 1
+ if err = os.WriteFile(path, corrupt, 0600); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = BlockCollectionIndexInBox(boxID); err == nil || BlockCollectionIndexKnownEmpty(boxID) {
+ t.Fatal("unauthenticated rules were accepted")
+ }
+ retained, err := os.ReadFile(path)
+ if err != nil || !bytes.Equal(retained, corrupt) {
+ t.Fatal("authentication failure changed ciphertext", err)
+ }
+}
diff --git a/kernel/model/attribute_view_collection.go b/kernel/model/attribute_view_collection.go
index 86306e07a8..8ca6a28e74 100644
--- a/kernel/model/attribute_view_collection.go
+++ b/kernel/model/attribute_view_collection.go
@@ -25,6 +25,7 @@ import (
)
type attributeViewCollectionState struct {
+ indexes map[string]*av.BlockCollectionIndex
checked map[string]bool
before map[string]*parse.Tree
views map[string]*av.AttributeView
@@ -34,22 +35,7 @@ type attributeViewCollectionState struct {
}
func collectionScopeMatches(source *av.AutomationBlockSource, tree *parse.Tree) bool {
- if source == nil || tree == nil {
- return false
- }
- if slices.Contains(source.NotebookIDs, tree.Box) {
- return true
- }
- for _, id := range source.DocumentIDs {
- if tree.ID == id {
- return true
- }
- if _, suffix, found := strings.Cut(tree.Path, "/"+id+"/"); found &&
- (source.DocumentDepth == nil || strings.Count(suffix, "/")+1 <= *source.DocumentDepth) {
- return true
- }
- }
- return false
+ return tree != nil && source.MatchesDocument(tree.Box, tree.ID, tree.Path)
}
func collectionBlockConditionsMatch(source *av.AutomationBlockSource, node *ast.Node, attributes []func(*ast.Node) bool) bool {
@@ -302,7 +288,7 @@ func (tx *Transaction) validateBlockCollectionRule(view *av.AttributeView, rule
func (tx *Transaction) collectionState() *attributeViewCollectionState {
if tx.attributeViewCollections == nil {
tx.attributeViewCollections = &attributeViewCollectionState{checked: map[string]bool{}, before: map[string]*parse.Tree{},
- views: map[string]*av.AttributeView{}, requested: map[string]map[string][]string{}}
+ views: map[string]*av.AttributeView{}, requested: map[string]map[string][]string{}, indexes: map[string]*av.BlockCollectionIndex{}}
}
return tx.attributeViewCollections
}
@@ -323,20 +309,26 @@ func (tx *Transaction) captureCollectionTree(tree *parse.Tree) (captureErr error
if !observesBlocks {
return nil
}
- state := tx.collectionState()
defer func() {
if captureErr != nil {
- state.err = captureErr
+ tx.collectionState().err = captureErr
}
}()
- if state.running || state.checked[tree.ID] {
+ state := tx.attributeViewCollections
+ if state != nil && (state.running || state.checked[tree.ID] || state.err != nil) {
return state.err
}
- state.checked[tree.ID] = true
boxID := ""
if IsEncryptedBox(tree.Box) {
boxID = tree.Box
}
+ var index *av.BlockCollectionIndex
+ if state != nil {
+ index = state.indexes[boxID]
+ }
+ if index == nil && av.BlockCollectionIndexKnownEmpty(boxID) {
+ return nil
+ }
if boxID != "" {
if tx.attributeViewRollback == nil {
tx.attributeViewRollback = &attributeViewRollback{views: map[string]*av.AttributeView{}, trees: map[string]*parse.Tree{}, leases: map[string]bool{}}
@@ -348,14 +340,27 @@ func (tx *Transaction) captureCollectionTree(tree *parse.Tree) (captureErr error
tx.attributeViewRollback.leases[boxID] = true
}
}
- ids, err := av.BlockCollectionViewIDsInBox(boxID)
- if err != nil {
- return err
+ if index == nil || !index.Current() {
+ var err error
+ index, err = av.BlockCollectionIndexInBox(boxID)
+ if err != nil {
+ return err
+ }
+ if index.Empty() {
+ return nil
+ }
+ state = tx.collectionState()
+ state.indexes[boxID] = index
+ }
+ state.checked[tree.ID] = true
+ if !index.MatchesDocument(tree.Box, tree.ID, tree.Path) {
+ return nil
}
matched := false
- for _, id := range ids {
+ for id := range index.ViewIDsForDocument(tree.Box, tree.ID, tree.Path) {
view := state.views[boxID+"/"+id]
if view == nil {
+ var err error
view, err = av.ParseAttributeViewForIndexInBox(id, boxID)
if errors.Is(err, av.ErrViewNotFound) {
continue
@@ -729,6 +734,7 @@ func (tx *Transaction) rollbackCollectionCreatedTrees() {
// 属性 API 仍不生成编辑器撤销记录,但与自动收集共同提交;没有收集规则时沿用普通属性路径。
func performCollectionAttributeOperations(attrs []map[string]any, batch bool) (bool, error) {
before := map[string]*parse.Tree{}
+ indexes := map[string]*av.BlockCollectionIndex{}
active := false
for _, item := range attrs {
id := item["id"].(string)
@@ -741,11 +747,18 @@ func performCollectionAttributeOperations(attrs []map[string]any, batch bool) (b
if IsEncryptedBox(tree.Box) {
boxID = tree.Box
}
- ids, err := av.BlockCollectionViewIDsInBox(boxID)
- if err != nil {
- return false, err
+ index := indexes[boxID]
+ if index == nil && av.BlockCollectionIndexKnownEmpty(boxID) {
+ continue
+ }
+ if index == nil || !index.Current() {
+ index, err = av.BlockCollectionIndexInBox(boxID)
+ if err != nil {
+ return false, err
+ }
+ indexes[boxID] = index
}
- active = active || len(ids) > 0
+ active = active || index.MatchesDocument(tree.Box, tree.ID, tree.Path)
}
if !active {
return false, nil
diff --git a/kernel/model/attribute_view_collection_index_fts5_test.go b/kernel/model/attribute_view_collection_index_fts5_test.go
new file mode 100644
index 0000000000..c6048ff862
--- /dev/null
+++ b/kernel/model/attribute_view_collection_index_fts5_test.go
@@ -0,0 +1,93 @@
+//go:build fts5
+
+package model
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/88250/lute/ast"
+ "github.com/siyuan-note/siyuan/kernel/av"
+ "github.com/siyuan-note/siyuan/kernel/cache"
+ "github.com/siyuan-note/siyuan/kernel/treenode"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestBlockCollectionIndexSkipsStateWithoutRules(t *testing.T) {
+ fixture, view, _ := setupBlockCollectionTest(t)
+ view.Automations.Rules[0].Enabled = false
+ if err := av.SaveAttributeView(view); err != nil {
+ t.Fatal(err)
+ }
+ cache.ClearAVCache()
+ tree, err := LoadTreeByBlockID(fixture.sourceID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ op := &Operation{Action: "update", ID: tree.ID}
+ for range 2 {
+ tx := &Transaction{fromAPI: true, DoOperations: []*Operation{op}}
+ if err = tx.captureCollectionTree(tree); err != nil || tx.attributeViewCollections != nil || tx.attributeViewAutomations != nil {
+ t.Fatal("empty rule set allocated collection state", err)
+ }
+ }
+ view.Automations.Rules[0].Enabled = true
+ if err = av.SaveAttributeView(view); err != nil {
+ t.Fatal(err)
+ }
+ tx := &Transaction{fromAPI: true, luteEngine: util.NewLute(), DoOperations: []*Operation{op}}
+ if err = tx.captureCollectionTree(tree); err != nil || tx.attributeViewAutomations == nil || tx.attributeViewCollections.before[tree.ID] == nil {
+ t.Fatal("enabling rule did not capture source snapshot", err)
+ }
+}
+
+func TestBlockCollectionIndexFiltersBeforeFullParse(t *testing.T) {
+ fixture, view, _ := setupBlockCollectionTest(t)
+ // 范围头部有效,但完整数据库字段无法解码,用于验证无关编辑不会走完整解析。
+ data, err := json.Marshal(map[string]any{"id": view.ID, "spec": av.PlainTextSpec,
+ "automations": view.Automations, "keyValues": "invalid full database body"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ path := filepath.Join(util.DataDir, "storage", "av", view.ID+".json")
+ if err = os.WriteFile(path, data, 0600); err != nil {
+ t.Fatal(err)
+ }
+ cache.ClearAVCache()
+ otherID := ast.NewNodeID()
+ tree, err := LoadTreeByBlockID(fixture.sourceID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ unrelated := *tree
+ unrelated.ID, unrelated.Path = otherID, "/"+otherID+".sy"
+ tx := &Transaction{fromAPI: true, DoOperations: []*Operation{{Action: "update", ID: otherID}}}
+ if err = tx.captureCollectionTree(&unrelated); err != nil {
+ t.Fatal("unrelated edit parsed full database", err)
+ }
+ index := tx.attributeViewCollections.indexes[""]
+ unrelated.ID = ast.NewNodeID()
+ unrelated.Path = "/" + unrelated.ID + ".sy"
+ if err = tx.captureCollectionTree(&unrelated); err != nil || tx.attributeViewCollections.indexes[""] != index ||
+ len(tx.attributeViewCollections.indexes) != 1 || len(tx.attributeViewCollections.views) != 0 || tx.attributeViewAutomations != nil {
+ t.Fatal("multiple unrelated documents did not reuse boundary snapshot", err)
+ }
+ if err = tx.captureCollectionTree(tree); err == nil || tx.attributeViewCollections.err == nil {
+ t.Fatal("matching document suppressed full database error")
+ }
+ if handled, attrErr := performCollectionAttributeOperations([]map[string]any{{
+ "id": fixture.targetID, "attrs": map[string]string{"custom-state": "active"},
+ }}, false); attrErr != nil || handled {
+ t.Fatal("unrelated attribute API forced collection transaction", attrErr)
+ }
+ createdID := ast.NewNodeID()
+ created := treenode.NewTree(fixture.box.ID, "/"+createdID+".sy", "/Unrelated", "Unrelated")
+ if err = performCreateDocTransaction(created, false); err != nil {
+ t.Fatal("unrelated creation parsed full database", err)
+ }
+ if _, err = LoadTreeByBlockID(createdID); err != nil {
+ t.Fatal("unrelated creation did not commit", err)
+ }
+}
diff --git a/kernel/model/crypto.go b/kernel/model/crypto.go
index 9579dc4b17..438b0e6f06 100644
--- a/kernel/model/crypto.go
+++ b/kernel/model/crypto.go
@@ -41,6 +41,7 @@ import (
"github.com/88250/lute/ast"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
+ "github.com/siyuan-note/siyuan/kernel/av"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
@@ -1574,6 +1575,7 @@ func lockBoxWithPreparationHeld(boxID string, prepare func()) {
// lockBoxHeld 在已持有 box 写锁的前提下执行该 box 的锁定清理(不含全局缓存刷新)。
func lockBoxHeld(boxID string) {
+ av.ClearBlockCollectionIndex(boxID)
// 此时已排空在途操作并持有 box 写锁,可以安全清除该笔记本的明文 HEIF 预览缓存。
heif.ClearMemoryCache(boxID)
LivePhotoPreviews.Clear(boxID)
diff --git a/kernel/model/file.go b/kernel/model/file.go
index 6a293d23f1..38faa6a70b 100644
--- a/kernel/model/file.go
+++ b/kernel/model/file.go
@@ -2547,11 +2547,15 @@ func performCreateDocTransaction(tree *parse.Tree, syncWrite bool) (err error) {
if IsEncryptedBox(tree.Box) {
boundary = tree.Box
}
- collections, err := av.BlockCollectionViewIDsInBox(boundary)
- if err != nil {
- return err
+ collects := false
+ if !av.BlockCollectionIndexKnownEmpty(boundary) {
+ index, indexErr := av.BlockCollectionIndexInBox(boundary)
+ if indexErr != nil {
+ return indexErr
+ }
+ collects = index.MatchesDocument(tree.Box, tree.ID, tree.Path)
}
- if syncWrite || len(collections) > 0 {
+ if syncWrite || collects {
if err = PerformTxSync(transaction); nil != err {
// 事务在写文件前会更新块树,失败时清理未落盘文档的索引。
treenode.RemoveBlockTreesByRootID(tree.Box, tree.ID)