An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:bug: Isolate cache replacement admission keys https://github.com/siyuan-note/siyuan/issues/20496
Daniel843c67114c83
diff --git a/kernel/cache/av.go b/kernel/cache/av.goindex b621d31b76..8273c5ecc1 100644--- a/kernel/cache/av.go+++ b/kernel/cache/av.go@@ -41,8 +41,9 @@ type avSearchDataEntry struct {version uint64}-var avCacheKeys = map[string]map[string]struct{}{}-var avCacheKeysLock sync.Mutex+// 逻辑键与准入键由 avSearchDataCacheLock 一并保护;准入序号在清理后也不复用。+var avCacheKeys = map[string]map[string]uint64{}+var avCacheSequence uint64var avDataVersion uint64@@ -69,7 +70,7 @@ func GetAVDataWithVersionInBox(avID, boxID string) (raw []byte, version uint64,avSearchDataCacheLock.RLock()defer avSearchDataCacheLock.RUnlock()key := avCacheKey(avID, boxID)- v, _ := avCache.Get(key)+ v, _ := avCache.Get(avCacheKeys[avID][key])if nil == v {return}@@ -85,7 +86,7 @@ func GetAVNormalizedDataInBox(avID, boxID string, version uint64) (data []byte,avSearchDataCacheLock.RLock()defer avSearchDataCacheLock.RUnlock()key := avCacheKey(avID, boxID)- value, found := avCache.Get(key)+ value, found := avCache.Get(avCacheKeys[avID][key])if !found || value == nil || avDataVersions[key] != version {return}@@ -101,8 +102,6 @@ func SetAVNormalizedDataInBox(avID, boxID string, version uint64, raw, normalizeif raw == nil || normalized == nil || version == 0 {return false}- // 等待原始数据的异步准入,避免两次新条目准入互相覆盖派生副本。- avCache.Wait()avSearchDataCacheLock.Lock()defer avSearchDataCacheLock.Unlock()key := avCacheKey(avID, boxID)@@ -117,9 +116,25 @@ func SetAVNormalizedDataInBox(avID, boxID string, version uint64, raw, normalizeif cost > avCache.MaxCost() {return false}- // 按新条目重新准入,使派生字节也参与容量淘汰,而不是直接扩大已驻留条目。- avCache.Del(key)- return avCache.Set(key, &avCacheEntry{raw: raw, version: version, normalized: normalized}, cost)+ return replaceAVCacheLocked(avID, key, &avCacheEntry{raw: raw, version: version, normalized: normalized}, cost)+}++// replaceAVCacheLocked 使用独立准入键,让增长的条目参与淘汰,并隔离旧条目的异步写入及删除。+// 调用方必须持有 avSearchDataCacheLock。+func replaceAVCacheLocked(avID, key string, entry *avCacheEntry, cost int64) bool {+ keys := avCacheKeys[avID]+ if keys == nil {+ keys = map[string]uint64{}+ avCacheKeys[avID] = keys+ }+ previous := keys[key]+ avCacheSequence+++ cacheKey := avCacheSequence+ keys[key] = cacheKey+ if previous != 0 {+ avCache.Del(previous)+ }+ return avCache.Set(cacheKey, entry, cost)}func EnsureAVDataVersionInBox(avID, boxID string) (version uint64) {@@ -151,29 +166,21 @@ func setAVDataInBox(avID, boxID string, raw []byte) (version uint64) {if raw == nil {return}+ avSearchDataCacheLock.Lock()+ defer avSearchDataCacheLock.Unlock()key := avCacheKey(avID, boxID)- version = invalidateAVSearchDataByKey(key)- avCache.Set(key, &avCacheEntry{raw: raw, version: version}, int64(len(raw)))-- avCacheKeysLock.Lock()- defer avCacheKeysLock.Unlock()- keys := avCacheKeys[avID]- if keys == nil {- keys = map[string]struct{}{}- avCacheKeys[avID] = keys- }- keys[key] = struct{}{}+ version = invalidateAVSearchDataByKeyLocked(key)+ replaceAVCacheLocked(avID, key, &avCacheEntry{raw: raw, version: version}, int64(len(raw)))return}func RemoveAVDataInBox(avID, boxID string) {+ avSearchDataCacheLock.Lock()+ defer avSearchDataCacheLock.Unlock()key := avCacheKey(avID, boxID)- avCache.Del(key)- invalidateAVSearchDataByKey(key)-- avCacheKeysLock.Lock()- defer avCacheKeysLock.Unlock()+ invalidateAVSearchDataByKeyLocked(key)if keys := avCacheKeys[avID]; keys != nil {+ avCache.Del(keys[key])delete(keys, key)if len(keys) == 0 {delete(avCacheKeys, avID)@@ -182,30 +189,25 @@ func RemoveAVDataInBox(avID, boxID string) {}func RemoveAVData(avID string) {- avCacheKeysLock.Lock()+ avSearchDataCacheLock.Lock()+ defer avSearchDataCacheLock.Unlock()keys := avCacheKeys[avID]delete(avCacheKeys, avID)- avCacheKeysLock.Unlock()-- avCache.Del(avCacheKey(avID, ""))- for key := range keys {- avCache.Del(key)- invalidateAVSearchDataByKey(key)+ for key, cacheKey := range keys {+ avCache.Del(cacheKey)+ invalidateAVSearchDataByKeyLocked(key)}- invalidateAVSearchDataByKey(avCacheKey(avID, ""))+ invalidateAVSearchDataByKeyLocked(avCacheKey(avID, ""))}func ClearAVCache() {+ avSearchDataCacheLock.Lock()+ defer avSearchDataCacheLock.Unlock()avCacheGeneration.Add(1)- avCacheKeysLock.Lock()- avCacheKeys = map[string]map[string]struct{}{}- avCacheKeysLock.Unlock()+ avCacheKeys = map[string]map[string]uint64{}avCache.Clear()-- avSearchDataCacheLock.Lock()avDataVersions = map[string]uint64{}avSearchDataCache = map[string]*avSearchDataEntry{}- avSearchDataCacheLock.Unlock()}func GetAVCacheGeneration() uint64 {@@ -239,12 +241,10 @@ func SetAVSearchDataInBox(avID, boxID string, version uint64, data any) (ok boolreturn true}-func invalidateAVSearchDataByKey(key string) (version uint64) {- avSearchDataCacheLock.Lock()+func invalidateAVSearchDataByKeyLocked(key string) (version uint64) {avDataVersion++version = avDataVersionavDataVersions[key] = versiondelete(avSearchDataCache, key)- avSearchDataCacheLock.Unlock()return}diff --git a/kernel/cache/av_test.go b/kernel/cache/av_test.goindex 0dd292f63b..0eb886a31b 100644--- a/kernel/cache/av_test.go+++ b/kernel/cache/av_test.go@@ -192,3 +192,72 @@ func TestAVCacheRemovalKeepsNotebookIsolation(t *testing.T) {}}}++func TestAVCacheReplacementRemainsInvalidatable(t *testing.T) {+ ClearAVCache()+ t.Cleanup(ClearAVCache)+ const id = "replacement"+ var version uint64+ for i := 1; i <= 64; i++ {+ version = SetAVDataWithVersionInBox(id, "box", bytes.Repeat([]byte("x"), i))+ }+ avCache.Wait()+ raw, current, ok := GetAVDataWithVersionInBox(id, "box")+ if !ok || len(raw) != 64 || current != version {+ t.Fatal("latest raw replacement was lost")+ }+ for i := 1; i <= 64; i++ {+ if !SetAVNormalizedDataInBox(id, "box", version, raw, bytes.Repeat([]byte("n"), i)) {+ t.Fatal("normalized replacement was dropped")+ }+ }+ avCache.Wait()+ if normalized, ok := GetAVNormalizedDataInBox(id, "box", version); !ok || len(normalized) != 64 {+ t.Fatal("latest normalized replacement was lost")+ }+ RemoveAVData(id)+ avCache.Wait()+ if _, ok := GetAVDataInBox(id, "box"); ok || len(avCacheKeys) != 0 {+ t.Fatal("replacement survived invalidation")+ }+ if SetAVNormalizedDataInBox(id, "box", version, raw, []byte("stale")) {+ t.Fatal("stale normalization survived invalidation")+ }+}++func TestAVCacheGrowingReplacementRespectsBudget(t *testing.T) {+ for _, normalized := range []bool{false, true} {+ t.Run(map[bool]string{false: "raw", true: "normalized"}[normalized], func(t *testing.T) {+ ClearAVCache()+ previous := avCache.MaxCost()+ avCache.UpdateMaxCost(1600)+ t.Cleanup(func() { ClearAVCache(); avCache.UpdateMaxCost(previous) })+ raw := bytes.Repeat([]byte("x"), 400)+ var version uint64+ for _, id := range []string{"a", "b", "c"} {+ version = SetAVDataWithVersionInBox(id, "box", raw)+ avCache.Wait()+ if _, ok := GetAVDataInBox(id, "box"); !ok {+ t.Fatal("initial data was not admitted")+ }+ }+ if normalized {+ SetAVNormalizedDataInBox("c", "box", version, raw, bytes.Repeat([]byte("n"), 900))+ } else {+ SetAVDataInBox("c", "box", bytes.Repeat([]byte("x"), 1300))+ }+ avCache.Wait()+ used := 0+ for _, id := range []string{"a", "b", "c"} {+ if data, current, ok := GetAVDataWithVersionInBox(id, "box"); ok {+ used += len(data)+ derived, _ := GetAVNormalizedDataInBox(id, "box", current)+ used += len(derived)+ }+ }+ if used > 1600 {+ t.Fatalf("growing replacement exceeded budget: %d > 1600", used)+ }+ })+ }+}diff --git a/kernel/sql/cache.go b/kernel/sql/cache.goindex 983d8e1dbd..5ef2b38a5b 100644--- a/kernel/sql/cache.go+++ b/kernel/sql/cache.go@@ -55,10 +55,12 @@ var blockCache, _ = ristretto.NewCache(&ristretto.Config{var blockCacheKeys = map[string]map[string]*blockCacheEntry{}var blockCacheKeysMu sync.Mutex+var blockCacheSequence uint64type blockCacheEntry struct {- key string- block *Block+ key string+ cacheKey uint64+ block *Block}// blockCacheCost 估算块结构、字符串和失效索引的驻留字节数,内部条目成本由缓存另行计入。@@ -111,11 +113,24 @@ func putBlockCache(block *Block) {removeBlockCache(cloned.ID)return}- // 删除后重新准入,让变大的条目也接受字节预算检查。- blockCache.Del(key)entry := &blockCacheEntry{key: key, block: cloned}- addBlockCacheKey(cloned.ID, key, entry)- if !blockCache.Set(key, entry, cost) {+ // 每次刷新使用独立准入键,保留字节预算检查,并隔离旧条目的异步删除。+ blockCacheKeysMu.Lock()+ blockCacheSequence+++ entry.cacheKey = blockCacheSequence+ keys := blockCacheKeys[cloned.ID]+ if keys == nil {+ keys = map[string]*blockCacheEntry{}+ blockCacheKeys[cloned.ID] = keys+ }+ previous := keys[key]+ keys[key] = entry+ accepted := blockCache.Set(entry.cacheKey, entry, cost)+ blockCacheKeysMu.Unlock()+ if previous != nil {+ blockCache.Del(previous.cacheKey)+ }+ if !accepted {removeBlockCacheKey(cloned.ID, key, entry)}}@@ -129,7 +144,13 @@ func getBlockCacheInBox(id, boxID string) (ret *Block) {return}- b, _ := blockCache.Get(blockCacheKey(id, boxID))+ blockCacheKeysMu.Lock()+ defer blockCacheKeysMu.Unlock()+ entry := blockCacheKeys[id][blockCacheKey(id, boxID)]+ if entry == nil {+ return+ }+ b, _ := blockCache.Get(entry.cacheKey)if nil != b {if entry, ok := b.(*blockCacheEntry); ok {ret = entry.block@@ -143,23 +164,12 @@ func removeBlockCache(id string) {keys := blockCacheKeys[id]delete(blockCacheKeys, id)blockCacheKeysMu.Unlock()- for key := range keys {- blockCache.Del(key)+ for _, entry := range keys {+ blockCache.Del(entry.cacheKey)}removeRefCacheByDefID(id)}-func addBlockCacheKey(id, key string, entry *blockCacheEntry) {- blockCacheKeysMu.Lock()- defer blockCacheKeysMu.Unlock()- keys := blockCacheKeys[id]- if keys == nil {- keys = map[string]*blockCacheEntry{}- blockCacheKeys[id] = keys- }- keys[key] = entry-}-func removeBlockCacheKey(id, key string, entry *blockCacheEntry) {blockCacheKeysMu.Lock()defer blockCacheKeysMu.Unlock()diff --git a/kernel/sql/cache_budget_test.go b/kernel/sql/cache_budget_test.goindex 6f738a0e71..d0f91afeeb 100644--- a/kernel/sql/cache_budget_test.go+++ b/kernel/sql/cache_budget_test.go@@ -50,3 +50,31 @@ func TestBlockCacheReplacementRemainsInvalidatable(t *testing.T) {t.Fatal("replacement survived invalidation")}}++func TestBlockCacheGrowingReplacementRespectsBudget(t *testing.T) {+ previousDisabled, previousCost := cacheDisabled, blockCache.MaxCost()+ cacheDisabled = false+ ClearCache()+ t.Cleanup(func() { ClearCache(); cacheDisabled = previousDisabled; blockCache.UpdateMaxCost(previousCost) })+ ids := []string{"a", "b", "c"}+ budget := 3*blockCacheCost("a", &Block{ID: "a", Content: strings.Repeat("x", 64)}) + 512+ blockCache.UpdateMaxCost(budget)+ for _, id := range ids {+ putBlockCache(&Block{ID: id, Content: strings.Repeat("x", 64)})+ blockCache.Wait()+ if getBlockCache(id) == nil {+ t.Fatal("initial block was not admitted")+ }+ }+ putBlockCache(&Block{ID: "b", Content: strings.Repeat("x", int(budget/2))})+ blockCache.Wait()+ var used int64+ for _, id := range ids {+ if block := getBlockCache(id); block != nil {+ used += blockCacheCost(id, block)+ }+ }+ if used > budget {+ t.Fatalf("growing replacement exceeded budget: %d > %d", used, budget)+ }+}