Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:perf: Cache validated attribute view rich text normalization by data version https://github.com/siyuan-note/siyuan/issues/20356

Danieleffcebb6d80e
diff --git a/kernel/av/av.go b/kernel/av/av.go
index 57ec2c9e6c..0bf09536bf 100644
--- a/kernel/av/av.go
+++ b/kernel/av/av.go
@@ -868,7 +868,16 @@ func parseAttributeViewByPathInBoxWithOptions(avJSONPath, boxID string, resolveC
dataVersion = cache.SetAVDataWithVersionInBox(avID, boxID, data)
}
- ret, err = ParseAttributeViewData(avID, data)
+ if normalized, ok := cache.GetAVNormalizedDataInBox(avID, boxID, dataVersion); ok {
+ ret, err = parseAttributeViewData(avID, normalized, false)
+ } else {
+ ret, err = ParseAttributeViewData(avID, data)
+ if err == nil {
+ if normalized, marshalErr := json.Marshal(ret); marshalErr == nil {
+ cache.SetAVNormalizedDataInBox(avID, boxID, dataVersion, data, normalized)
+ }
+ }
+ }
if nil == err {
if resolveColors {
ret.ResolveDirectColors()
@@ -880,6 +889,10 @@ func parseAttributeViewByPathInBoxWithOptions(avJSONPath, boxID string, resolveC
// ParseAttributeViewData 解析已经完成解密认证的数据库数据,复用现有格式兼容与规范化处理。
func ParseAttributeViewData(avID string, data []byte) (ret *AttributeView, err error) {
+ return parseAttributeViewData(avID, data, true)
+}
+
+func parseAttributeViewData(avID string, data []byte, normalizeRichText bool) (ret *AttributeView, err error) {
ret = &AttributeView{RenderedViewables: map[string]Viewable{}}
if err = json.Unmarshal(data, ret); err != nil {
if strings.Contains(err.Error(), ".relation.contents of type av.Value") {
@@ -946,7 +959,7 @@ func ParseAttributeViewData(avID string, data []byte) (ret *AttributeView, err e
if nil == err {
err = ret.ValidateListLayouts()
}
- if nil == err {
+ if nil == err && normalizeRichText {
err = ret.NormalizeRichText()
}
if nil == err {
diff --git a/kernel/av/av_fix.go b/kernel/av/av_fix.go
index 397f08cab4..5da6260950 100644
--- a/kernel/av/av_fix.go
+++ b/kernel/av/av_fix.go
@@ -19,6 +19,7 @@ package av
import (
"time"
+ "github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/88250/lute/parse"
"github.com/siyuan-note/logging"
@@ -140,13 +141,17 @@ func (av *AttributeView) NormalizeRichText() (err error) {
richContent string
}
var normalized []normalizedText
+ var luteEngine *lute.Lute
av.visitPersistedValues(func(value *Value) {
if nil != err || nil == value.Text || !value.Text.IsRich() {
return
}
rich := *value.Text.Rich
+ if luteEngine == nil {
+ luteEngine = newValueTextRichLute()
+ }
var tree *parse.Tree
- if tree, err = NormalizeValueTextRich(&rich); nil == err {
+ if tree, err = normalizeValueTextRichWithLute(&rich, luteEngine); nil == err {
normalized = append(normalized, normalizedText{
value: value.Text,
content: valueTextRichPlainContent(tree),
diff --git a/kernel/av/normalized_cache_test.go b/kernel/av/normalized_cache_test.go
new file mode 100644
index 0000000000..3e9bafbbea
--- /dev/null
+++ b/kernel/av/normalized_cache_test.go
@@ -0,0 +1,110 @@
+package av
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/siyuan-note/siyuan/kernel/cache"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func normalizedRichCacheFixture(t testing.TB) (string, []byte) {
+ t.Helper()
+ const id = "20261010120000-cacheav"
+ oldData := util.DataDir
+ util.DataDir = t.TempDir()
+ t.Cleanup(func() { cache.RemoveAVData(id); util.DataDir = oldData })
+ view := &AttributeView{ID: id, Spec: RichTextSpec, Name: "Original", KeyValues: []*KeyValues{{
+ Key: &Key{ID: "text", Type: KeyTypeText},
+ }}}
+ for i := 0; i < 100; i++ {
+ view.KeyValues[0].Values = append(view.KeyValues[0].Values, &Value{Type: KeyTypeText, Text: &ValueText{
+ Content: "incorrect projection",
+ Rich: &ValueTextRich{Spec: ValueTextRichSpec, Format: ValueTextRichFormatKramdown, Content: "**bold**\n"},
+ }})
+ }
+ data, err := json.Marshal(view)
+ if err != nil {
+ t.Fatal(err)
+ }
+ file := GetAttributeViewDataPath(id)
+ if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(file, data, 0644); err != nil {
+ t.Fatal(err)
+ }
+ return file, data
+}
+
+func TestNormalizedAttributeViewCachePreservesIsolationAndValidation(t *testing.T) {
+ file, source := normalizedRichCacheFixture(t)
+ const id = "20261010120000-cacheav"
+ first, err := parseAttributeViewByPathInBoxWithOptions(file, "", false)
+ if err != nil || first.KeyValues[0].Values[0].Text.Content != "bold" {
+ t.Fatalf("first load failed: %+v, %v", first, err)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ _, version, ok := cache.GetAVDataWithVersionInBox(id, "")
+ if _, normalized := cache.GetAVNormalizedDataInBox(id, "", version); ok && normalized {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("normalized cache was not populated")
+ }
+ time.Sleep(time.Millisecond)
+ }
+ first.Name = "Mutated"
+ first.KeyValues[0].Values[0].Text.Content = "mutated"
+ first.KeyValues[0].Values[0].Text.Rich.Content = "mutated"
+ second, err := parseAttributeViewByPathInBoxWithOptions(file, "", false)
+ if err != nil || second.Name != "Original" || second.KeyValues[0].Values[0].Text.Content != "bold" {
+ t.Fatalf("render mutations escaped into the cache: %+v, %v", second, err)
+ }
+ if data, ok := cache.GetAVDataInBox(id, ""); !ok || !bytes.Equal(data, source) {
+ t.Fatal("normalization overwrote source cache bytes")
+ }
+ if data, err := os.ReadFile(file); err != nil || !bytes.Equal(data, source) {
+ t.Fatal("read normalization wrote source data")
+ }
+ second.KeyValues[0].Values[0].Text.Rich.Spec = 999
+ invalid, err := json.Marshal(second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(file, invalid, 0644); err != nil {
+ t.Fatal(err)
+ }
+ cache.RemoveAVData(id)
+ if _, err := parseAttributeViewByPathInBoxWithOptions(file, "", false); err == nil {
+ t.Fatal("invalidated data bypassed rich text validation")
+ }
+}
+
+func BenchmarkAttributeViewNormalizationCache(b *testing.B) {
+ file, data := normalizedRichCacheFixture(b)
+ if _, err := parseAttributeViewByPathInBoxWithOptions(file, "", false); err != nil {
+ b.Fatal(err)
+ }
+ b.Run("uncached", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ if _, err := ParseAttributeViewData("20261010120000-cacheav", data); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ b.Run("cached", func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ if _, err := parseAttributeViewByPathInBoxWithOptions(file, "", false); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+}
diff --git a/kernel/av/value.go b/kernel/av/value.go
index 898edd2f0c..ba363764c7 100644
--- a/kernel/av/value.go
+++ b/kernel/av/value.go
@@ -564,6 +564,10 @@ func parseValueTextRich(rich *ValueTextRich) (blockDOM string, tree *parse.Tree,
}
func parseValueTextRichWithImages(rich *ValueTextRich, images bool) (blockDOM string, tree *parse.Tree, err error) {
+ return parseValueTextRichWithLute(rich, images, newValueTextRichLute())
+}
+
+func parseValueTextRichWithLute(rich *ValueTextRich, images bool, luteEngine *lute.Lute) (blockDOM string, tree *parse.Tree, err error) {
if nil == rich {
return
}
@@ -576,7 +580,6 @@ func parseValueTextRichWithImages(rich *ValueTextRich, images bool) (blockDOM st
return
}
- luteEngine := newValueTextRichLute()
content, protectedStyleEntities, protectErr := protectValueTextRichKramdownStyleEntities(rich.Content)
if nil != protectErr {
err = protectErr
@@ -2171,12 +2174,16 @@ func newValueTextRichBackslashSentinel(content string) string {
// NormalizeValueTextRich 校验并将文本字段的富文本源规范化为 SiYuan Kramdown。
func NormalizeValueTextRich(rich *ValueTextRich) (tree *parse.Tree, err error) {
- _, tree, err = parseValueTextRich(rich)
+ return normalizeValueTextRichWithLute(rich, newValueTextRichLute())
+}
+
+func normalizeValueTextRichWithLute(rich *ValueTextRich, luteEngine *lute.Lute) (tree *parse.Tree, err error) {
+ _, tree, err = parseValueTextRichWithLute(rich, false, luteEngine)
if nil != err || nil == rich {
return tree, err
}
var normalized string
- if normalized, tree, err = normalizeValueTextRichTreeSource(tree); nil != err {
+ if normalized, tree, err = normalizeValueTextRichTreeSourceWithLute(tree, false, luteEngine); nil != err {
return nil, err
}
rich.Content = normalized
@@ -2188,13 +2195,16 @@ func normalizeValueTextRichTreeSource(tree *parse.Tree) (content string, normali
}
func normalizeValueTextRichTreeSourceWithImages(tree *parse.Tree, images bool) (content string, normalizedTree *parse.Tree, err error) {
+ return normalizeValueTextRichTreeSourceWithLute(tree, images, newValueTextRichLute())
+}
+
+func normalizeValueTextRichTreeSourceWithLute(tree *parse.Tree, images bool, luteEngine *lute.Lute) (content string, normalizedTree *parse.Tree, err error) {
normalizedTree = tree
previous := ""
for iteration := 0; iteration < 4; iteration++ {
if err = normalizeValueTextRichTreeStyles(normalizedTree, images); nil != err {
return "", nil, err
}
- luteEngine := newValueTextRichLute()
blockDOM := luteEngine.Tree2BlockDOM(normalizedTree, luteEngine.RenderOptions, luteEngine.ParseOptions)
content = valueTextRichBlockDOM2Kramdown(luteEngine, blockDOM)
if 0 < iteration && previous == content {
@@ -2204,7 +2214,7 @@ func normalizeValueTextRichTreeSourceWithImages(tree *parse.Tree, images bool) (
candidate := &ValueTextRich{
Spec: ValueTextRichSpec, Format: ValueTextRichFormatKramdown, Content: content,
}
- if _, normalizedTree, err = parseValueTextRichWithImages(candidate, images); nil != err {
+ if _, normalizedTree, err = parseValueTextRichWithLute(candidate, images, luteEngine); nil != err {
return "", nil, err
}
}
diff --git a/kernel/cache/av.go b/kernel/cache/av.go
index 72bac78035..af38eb7637 100644
--- a/kernel/cache/av.go
+++ b/kernel/cache/av.go
@@ -17,6 +17,7 @@
package cache
import (
+ "bytes"
"sync"
"sync/atomic"
@@ -30,8 +31,9 @@ var avCache, _ = ristretto.NewCache(&ristretto.Config{
})
type avCacheEntry struct {
- raw []byte
- version uint64
+ raw []byte
+ version uint64
+ normalized []byte
}
type avSearchDataEntry struct {
@@ -78,6 +80,48 @@ func GetAVDataWithVersionInBox(avID, boxID string) (raw []byte, version uint64,
return entry.raw, entry.version, true
}
+// GetAVNormalizedDataInBox 读取同一数据版本的规范化副本,共用原始数据的容量及失效边界。
+func GetAVNormalizedDataInBox(avID, boxID string, version uint64) (data []byte, ok bool) {
+ avSearchDataCacheLock.RLock()
+ defer avSearchDataCacheLock.RUnlock()
+ key := avCacheKey(avID, boxID)
+ value, found := avCache.Get(key)
+ if !found || value == nil || avDataVersions[key] != version {
+ return
+ }
+ entry := value.(*avCacheEntry)
+ if entry.version != version || entry.normalized == nil {
+ return
+ }
+ return entry.normalized, true
+}
+
+// SetAVNormalizedDataInBox 仅接纳已认证、已校验数据的同版本派生副本,不改变原始缓存字节。
+func SetAVNormalizedDataInBox(avID, boxID string, version uint64, raw, normalized []byte) bool {
+ if raw == nil || normalized == nil || version == 0 {
+ return false
+ }
+ // 等待原始数据的异步准入,避免两次新条目准入互相覆盖派生副本。
+ avCache.Wait()
+ avSearchDataCacheLock.Lock()
+ defer avSearchDataCacheLock.Unlock()
+ key := avCacheKey(avID, boxID)
+ if avDataVersions[key] != version {
+ return false
+ }
+ cost := int64(len(raw) + len(normalized))
+ if bytes.Equal(raw, normalized) {
+ normalized = raw
+ cost = int64(len(raw))
+ }
+ if cost > avCache.MaxCost() {
+ return false
+ }
+ // 按新条目重新准入,使派生字节也参与容量淘汰,而不是直接扩大已驻留条目。
+ avCache.Del(key)
+ return avCache.Set(key, &avCacheEntry{raw: raw, version: version, normalized: normalized}, cost)
+}
+
func EnsureAVDataVersionInBox(avID, boxID string) (version uint64) {
avSearchDataCacheLock.Lock()
defer avSearchDataCacheLock.Unlock()
diff --git a/kernel/cache/av_test.go b/kernel/cache/av_test.go
index e90b26221c..0dd292f63b 100644
--- a/kernel/cache/av_test.go
+++ b/kernel/cache/av_test.go
@@ -55,6 +55,53 @@ func TestAVSearchDataInvalidation(t *testing.T) {
}
}
+func TestAVNormalizedDataSharesVersionAndNotebookBoundary(t *testing.T) {
+ const id = "normalized-cache-test"
+ defer RemoveAVData(id)
+ raw, normalized := []byte("original"), []byte("normalized")
+ version := SetAVDataWithVersionInBox(id, "a", raw)
+ other := SetAVDataWithVersionInBox(id, "b", []byte("other"))
+ if !SetAVNormalizedDataInBox(id, "a", version, raw, normalized) {
+ t.Fatal("normalized data was not admitted")
+ }
+ avCache.Wait()
+ if got, ok := GetAVNormalizedDataInBox(id, "a", version); !ok || !bytes.Equal(got, normalized) {
+ t.Fatal("normalized data was not available")
+ }
+ if got, ok := GetAVDataInBox(id, "a"); !ok || !bytes.Equal(got, raw) {
+ t.Fatal("normalization changed the source cache")
+ }
+ if _, ok := GetAVNormalizedDataInBox(id, "b", other); ok {
+ t.Fatal("normalized plaintext crossed notebooks")
+ }
+ RemoveAVDataInBox(id, "a")
+ if _, ok := GetAVNormalizedDataInBox(id, "a", version); ok {
+ t.Fatal("removed source retained normalized data")
+ }
+ if SetAVNormalizedDataInBox(id, "a", version, raw, normalized) {
+ t.Fatal("stale normalization was admitted after invalidation")
+ }
+ if got, ok := GetAVDataInBox(id, "b"); !ok || string(got) != "other" {
+ t.Fatal("removal crossed notebook boundaries")
+ }
+}
+
+func TestAVNormalizedDataRespectsCacheBudget(t *testing.T) {
+ const id = "normalization-budget-test"
+ previous := avCache.MaxCost()
+ avCache.UpdateMaxCost(1024)
+ defer func() { RemoveAVData(id); avCache.UpdateMaxCost(previous) }()
+ raw := []byte("small source")
+ version := SetAVDataWithVersionInBox(id, "", raw)
+ avCache.Wait()
+ if SetAVNormalizedDataInBox(id, "", version, raw, bytes.Repeat([]byte("x"), 2048)) {
+ t.Fatal("oversize derived data exceeded cache budget")
+ }
+ if cached, ok := GetAVDataInBox(id, ""); !ok || !bytes.Equal(cached, raw) {
+ t.Fatal("rejected normalization removed the source cache")
+ }
+}
+
func TestAVSearchDataWithoutRawData(t *testing.T) {
const avID = "20260801120000-search"
const boxID = "20260801120000-box"
diff --git a/kernel/model/attribute_view_render.go b/kernel/model/attribute_view_render.go
index f7d5075fd2..0e034ca5f5 100644
--- a/kernel/model/attribute_view_render.go
+++ b/kernel/model/attribute_view_render.go
@@ -491,12 +491,6 @@ func renderAttributeViewWithTarget(blockID, avID, viewID, query string, page, pa
}
}
- // 诊断:AV 解析后的数据量
- blockKV := attrView.GetBlockKeyValues()
- if nil != blockKV {
- } else {
- }
-
viewable, err = renderAttributeView(attrView, blockID, viewID, "", query, page, pageSize, groupPaging, ignoreRows, writable, target, targetGroupID, calendarRanges...)
return
}