Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Isolate cache replacement admission keys https://github.com/siyuan-note/siyuan/issues/20496

Daniel843c67114c83
diff --git a/kernel/cache/av.go b/kernel/cache/av.go
index b621d31b76..8273c5ecc1 100644
--- a/kernel/cache/av.go
+++ b/kernel/cache/av.go
@@ -41,8 +41,9 @@ type avSearchDataEntry struct {
version uint64
}
-var avCacheKeys = map[string]map[string]struct{}{}
-var avCacheKeysLock sync.Mutex
+// 逻辑键与准入键由 avSearchDataCacheLock 一并保护;准入序号在清理后也不复用。
+var avCacheKeys = map[string]map[string]uint64{}
+var avCacheSequence uint64
var avDataVersion uint64
@@ -69,7 +70,7 @@ func GetAVDataWithVersionInBox(avID, boxID string) (raw []byte, version uint64,
avSearchDataCacheLock.RLock()
defer avSearchDataCacheLock.RUnlock()
key := avCacheKey(avID, boxID)
- v, _ := avCache.Get(key)
+ v, _ := avCache.Get(avCacheKeys[avID][key])
if nil == v {
return
}
@@ -85,7 +86,7 @@ func GetAVNormalizedDataInBox(avID, boxID string, version uint64) (data []byte,
avSearchDataCacheLock.RLock()
defer avSearchDataCacheLock.RUnlock()
key := avCacheKey(avID, boxID)
- value, found := avCache.Get(key)
+ value, found := avCache.Get(avCacheKeys[avID][key])
if !found || value == nil || avDataVersions[key] != version {
return
}
@@ -101,8 +102,6 @@ func SetAVNormalizedDataInBox(avID, boxID string, version uint64, raw, normalize
if raw == nil || normalized == nil || version == 0 {
return false
}
- // 等待原始数据的异步准入,避免两次新条目准入互相覆盖派生副本。
- avCache.Wait()
avSearchDataCacheLock.Lock()
defer avSearchDataCacheLock.Unlock()
key := avCacheKey(avID, boxID)
@@ -117,9 +116,25 @@ func SetAVNormalizedDataInBox(avID, boxID string, version uint64, raw, normalize
if cost > avCache.MaxCost() {
return false
}
- // 按新条目重新准入,使派生字节也参与容量淘汰,而不是直接扩大已驻留条目。
- avCache.Del(key)
- return avCache.Set(key, &avCacheEntry{raw: raw, version: version, normalized: normalized}, cost)
+ return replaceAVCacheLocked(avID, key, &avCacheEntry{raw: raw, version: version, normalized: normalized}, cost)
+}
+
+// replaceAVCacheLocked 使用独立准入键,让增长的条目参与淘汰,并隔离旧条目的异步写入及删除。
+// 调用方必须持有 avSearchDataCacheLock。
+func replaceAVCacheLocked(avID, key string, entry *avCacheEntry, cost int64) bool {
+ keys := avCacheKeys[avID]
+ if keys == nil {
+ keys = map[string]uint64{}
+ avCacheKeys[avID] = keys
+ }
+ previous := keys[key]
+ avCacheSequence++
+ cacheKey := avCacheSequence
+ keys[key] = cacheKey
+ if previous != 0 {
+ avCache.Del(previous)
+ }
+ return avCache.Set(cacheKey, entry, cost)
}
func EnsureAVDataVersionInBox(avID, boxID string) (version uint64) {
@@ -151,29 +166,21 @@ func setAVDataInBox(avID, boxID string, raw []byte) (version uint64) {
if raw == nil {
return
}
+ avSearchDataCacheLock.Lock()
+ defer avSearchDataCacheLock.Unlock()
key := avCacheKey(avID, boxID)
- version = invalidateAVSearchDataByKey(key)
- avCache.Set(key, &avCacheEntry{raw: raw, version: version}, int64(len(raw)))
-
- avCacheKeysLock.Lock()
- defer avCacheKeysLock.Unlock()
- keys := avCacheKeys[avID]
- if keys == nil {
- keys = map[string]struct{}{}
- avCacheKeys[avID] = keys
- }
- keys[key] = struct{}{}
+ version = invalidateAVSearchDataByKeyLocked(key)
+ replaceAVCacheLocked(avID, key, &avCacheEntry{raw: raw, version: version}, int64(len(raw)))
return
}
func RemoveAVDataInBox(avID, boxID string) {
+ avSearchDataCacheLock.Lock()
+ defer avSearchDataCacheLock.Unlock()
key := avCacheKey(avID, boxID)
- avCache.Del(key)
- invalidateAVSearchDataByKey(key)
-
- avCacheKeysLock.Lock()
- defer avCacheKeysLock.Unlock()
+ invalidateAVSearchDataByKeyLocked(key)
if keys := avCacheKeys[avID]; keys != nil {
+ avCache.Del(keys[key])
delete(keys, key)
if len(keys) == 0 {
delete(avCacheKeys, avID)
@@ -182,30 +189,25 @@ func RemoveAVDataInBox(avID, boxID string) {
}
func RemoveAVData(avID string) {
- avCacheKeysLock.Lock()
+ avSearchDataCacheLock.Lock()
+ defer avSearchDataCacheLock.Unlock()
keys := avCacheKeys[avID]
delete(avCacheKeys, avID)
- avCacheKeysLock.Unlock()
-
- avCache.Del(avCacheKey(avID, ""))
- for key := range keys {
- avCache.Del(key)
- invalidateAVSearchDataByKey(key)
+ for key, cacheKey := range keys {
+ avCache.Del(cacheKey)
+ invalidateAVSearchDataByKeyLocked(key)
}
- invalidateAVSearchDataByKey(avCacheKey(avID, ""))
+ invalidateAVSearchDataByKeyLocked(avCacheKey(avID, ""))
}
func ClearAVCache() {
+ avSearchDataCacheLock.Lock()
+ defer avSearchDataCacheLock.Unlock()
avCacheGeneration.Add(1)
- avCacheKeysLock.Lock()
- avCacheKeys = map[string]map[string]struct{}{}
- avCacheKeysLock.Unlock()
+ avCacheKeys = map[string]map[string]uint64{}
avCache.Clear()
-
- avSearchDataCacheLock.Lock()
avDataVersions = map[string]uint64{}
avSearchDataCache = map[string]*avSearchDataEntry{}
- avSearchDataCacheLock.Unlock()
}
func GetAVCacheGeneration() uint64 {
@@ -239,12 +241,10 @@ func SetAVSearchDataInBox(avID, boxID string, version uint64, data any) (ok bool
return true
}
-func invalidateAVSearchDataByKey(key string) (version uint64) {
- avSearchDataCacheLock.Lock()
+func invalidateAVSearchDataByKeyLocked(key string) (version uint64) {
avDataVersion++
version = avDataVersion
avDataVersions[key] = version
delete(avSearchDataCache, key)
- avSearchDataCacheLock.Unlock()
return
}
diff --git a/kernel/cache/av_test.go b/kernel/cache/av_test.go
index 0dd292f63b..0eb886a31b 100644
--- a/kernel/cache/av_test.go
+++ b/kernel/cache/av_test.go
@@ -192,3 +192,72 @@ func TestAVCacheRemovalKeepsNotebookIsolation(t *testing.T) {
}
}
}
+
+func TestAVCacheReplacementRemainsInvalidatable(t *testing.T) {
+ ClearAVCache()
+ t.Cleanup(ClearAVCache)
+ const id = "replacement"
+ var version uint64
+ for i := 1; i <= 64; i++ {
+ version = SetAVDataWithVersionInBox(id, "box", bytes.Repeat([]byte("x"), i))
+ }
+ avCache.Wait()
+ raw, current, ok := GetAVDataWithVersionInBox(id, "box")
+ if !ok || len(raw) != 64 || current != version {
+ t.Fatal("latest raw replacement was lost")
+ }
+ for i := 1; i <= 64; i++ {
+ if !SetAVNormalizedDataInBox(id, "box", version, raw, bytes.Repeat([]byte("n"), i)) {
+ t.Fatal("normalized replacement was dropped")
+ }
+ }
+ avCache.Wait()
+ if normalized, ok := GetAVNormalizedDataInBox(id, "box", version); !ok || len(normalized) != 64 {
+ t.Fatal("latest normalized replacement was lost")
+ }
+ RemoveAVData(id)
+ avCache.Wait()
+ if _, ok := GetAVDataInBox(id, "box"); ok || len(avCacheKeys) != 0 {
+ t.Fatal("replacement survived invalidation")
+ }
+ if SetAVNormalizedDataInBox(id, "box", version, raw, []byte("stale")) {
+ t.Fatal("stale normalization survived invalidation")
+ }
+}
+
+func TestAVCacheGrowingReplacementRespectsBudget(t *testing.T) {
+ for _, normalized := range []bool{false, true} {
+ t.Run(map[bool]string{false: "raw", true: "normalized"}[normalized], func(t *testing.T) {
+ ClearAVCache()
+ previous := avCache.MaxCost()
+ avCache.UpdateMaxCost(1600)
+ t.Cleanup(func() { ClearAVCache(); avCache.UpdateMaxCost(previous) })
+ raw := bytes.Repeat([]byte("x"), 400)
+ var version uint64
+ for _, id := range []string{"a", "b", "c"} {
+ version = SetAVDataWithVersionInBox(id, "box", raw)
+ avCache.Wait()
+ if _, ok := GetAVDataInBox(id, "box"); !ok {
+ t.Fatal("initial data was not admitted")
+ }
+ }
+ if normalized {
+ SetAVNormalizedDataInBox("c", "box", version, raw, bytes.Repeat([]byte("n"), 900))
+ } else {
+ SetAVDataInBox("c", "box", bytes.Repeat([]byte("x"), 1300))
+ }
+ avCache.Wait()
+ used := 0
+ for _, id := range []string{"a", "b", "c"} {
+ if data, current, ok := GetAVDataWithVersionInBox(id, "box"); ok {
+ used += len(data)
+ derived, _ := GetAVNormalizedDataInBox(id, "box", current)
+ used += len(derived)
+ }
+ }
+ if used > 1600 {
+ t.Fatalf("growing replacement exceeded budget: %d > 1600", used)
+ }
+ })
+ }
+}
diff --git a/kernel/sql/cache.go b/kernel/sql/cache.go
index 983d8e1dbd..5ef2b38a5b 100644
--- a/kernel/sql/cache.go
+++ b/kernel/sql/cache.go
@@ -55,10 +55,12 @@ var blockCache, _ = ristretto.NewCache(&ristretto.Config{
var blockCacheKeys = map[string]map[string]*blockCacheEntry{}
var blockCacheKeysMu sync.Mutex
+var blockCacheSequence uint64
type blockCacheEntry struct {
- key string
- block *Block
+ key string
+ cacheKey uint64
+ block *Block
}
// blockCacheCost 估算块结构、字符串和失效索引的驻留字节数,内部条目成本由缓存另行计入。
@@ -111,11 +113,24 @@ func putBlockCache(block *Block) {
removeBlockCache(cloned.ID)
return
}
- // 删除后重新准入,让变大的条目也接受字节预算检查。
- blockCache.Del(key)
entry := &blockCacheEntry{key: key, block: cloned}
- addBlockCacheKey(cloned.ID, key, entry)
- if !blockCache.Set(key, entry, cost) {
+ // 每次刷新使用独立准入键,保留字节预算检查,并隔离旧条目的异步删除。
+ blockCacheKeysMu.Lock()
+ blockCacheSequence++
+ entry.cacheKey = blockCacheSequence
+ keys := blockCacheKeys[cloned.ID]
+ if keys == nil {
+ keys = map[string]*blockCacheEntry{}
+ blockCacheKeys[cloned.ID] = keys
+ }
+ previous := keys[key]
+ keys[key] = entry
+ accepted := blockCache.Set(entry.cacheKey, entry, cost)
+ blockCacheKeysMu.Unlock()
+ if previous != nil {
+ blockCache.Del(previous.cacheKey)
+ }
+ if !accepted {
removeBlockCacheKey(cloned.ID, key, entry)
}
}
@@ -129,7 +144,13 @@ func getBlockCacheInBox(id, boxID string) (ret *Block) {
return
}
- b, _ := blockCache.Get(blockCacheKey(id, boxID))
+ blockCacheKeysMu.Lock()
+ defer blockCacheKeysMu.Unlock()
+ entry := blockCacheKeys[id][blockCacheKey(id, boxID)]
+ if entry == nil {
+ return
+ }
+ b, _ := blockCache.Get(entry.cacheKey)
if nil != b {
if entry, ok := b.(*blockCacheEntry); ok {
ret = entry.block
@@ -143,23 +164,12 @@ func removeBlockCache(id string) {
keys := blockCacheKeys[id]
delete(blockCacheKeys, id)
blockCacheKeysMu.Unlock()
- for key := range keys {
- blockCache.Del(key)
+ for _, entry := range keys {
+ blockCache.Del(entry.cacheKey)
}
removeRefCacheByDefID(id)
}
-func addBlockCacheKey(id, key string, entry *blockCacheEntry) {
- blockCacheKeysMu.Lock()
- defer blockCacheKeysMu.Unlock()
- keys := blockCacheKeys[id]
- if keys == nil {
- keys = map[string]*blockCacheEntry{}
- blockCacheKeys[id] = keys
- }
- keys[key] = entry
-}
-
func removeBlockCacheKey(id, key string, entry *blockCacheEntry) {
blockCacheKeysMu.Lock()
defer blockCacheKeysMu.Unlock()
diff --git a/kernel/sql/cache_budget_test.go b/kernel/sql/cache_budget_test.go
index 6f738a0e71..d0f91afeeb 100644
--- a/kernel/sql/cache_budget_test.go
+++ b/kernel/sql/cache_budget_test.go
@@ -50,3 +50,31 @@ func TestBlockCacheReplacementRemainsInvalidatable(t *testing.T) {
t.Fatal("replacement survived invalidation")
}
}
+
+func TestBlockCacheGrowingReplacementRespectsBudget(t *testing.T) {
+ previousDisabled, previousCost := cacheDisabled, blockCache.MaxCost()
+ cacheDisabled = false
+ ClearCache()
+ t.Cleanup(func() { ClearCache(); cacheDisabled = previousDisabled; blockCache.UpdateMaxCost(previousCost) })
+ ids := []string{"a", "b", "c"}
+ budget := 3*blockCacheCost("a", &Block{ID: "a", Content: strings.Repeat("x", 64)}) + 512
+ blockCache.UpdateMaxCost(budget)
+ for _, id := range ids {
+ putBlockCache(&Block{ID: id, Content: strings.Repeat("x", 64)})
+ blockCache.Wait()
+ if getBlockCache(id) == nil {
+ t.Fatal("initial block was not admitted")
+ }
+ }
+ putBlockCache(&Block{ID: "b", Content: strings.Repeat("x", int(budget/2))})
+ blockCache.Wait()
+ var used int64
+ for _, id := range ids {
+ if block := getBlockCache(id); block != nil {
+ used += blockCacheCost(id, block)
+ }
+ }
+ if used > budget {
+ t.Fatalf("growing replacement exceeded budget: %d > %d", used, budget)
+ }
+}