An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:perf: Cache validated attribute view rich text normalization by data version https://github.com/siyuan-note/siyuan/issues/20356
Danieleffcebb6d80e
diff --git a/kernel/av/av.go b/kernel/av/av.goindex 57ec2c9e6c..0bf09536bf 100644--- a/kernel/av/av.go+++ b/kernel/av/av.go@@ -868,7 +868,16 @@ func parseAttributeViewByPathInBoxWithOptions(avJSONPath, boxID string, resolveCdataVersion = cache.SetAVDataWithVersionInBox(avID, boxID, data)}- ret, err = ParseAttributeViewData(avID, data)+ if normalized, ok := cache.GetAVNormalizedDataInBox(avID, boxID, dataVersion); ok {+ ret, err = parseAttributeViewData(avID, normalized, false)+ } else {+ ret, err = ParseAttributeViewData(avID, data)+ if err == nil {+ if normalized, marshalErr := json.Marshal(ret); marshalErr == nil {+ cache.SetAVNormalizedDataInBox(avID, boxID, dataVersion, data, normalized)+ }+ }+ }if nil == err {if resolveColors {ret.ResolveDirectColors()@@ -880,6 +889,10 @@ func parseAttributeViewByPathInBoxWithOptions(avJSONPath, boxID string, resolveC// ParseAttributeViewData 解析已经完成解密认证的数据库数据,复用现有格式兼容与规范化处理。func ParseAttributeViewData(avID string, data []byte) (ret *AttributeView, err error) {+ return parseAttributeViewData(avID, data, true)+}++func parseAttributeViewData(avID string, data []byte, normalizeRichText bool) (ret *AttributeView, err error) {ret = &AttributeView{RenderedViewables: map[string]Viewable{}}if err = json.Unmarshal(data, ret); err != nil {if strings.Contains(err.Error(), ".relation.contents of type av.Value") {@@ -946,7 +959,7 @@ func ParseAttributeViewData(avID string, data []byte) (ret *AttributeView, err eif nil == err {err = ret.ValidateListLayouts()}- if nil == err {+ if nil == err && normalizeRichText {err = ret.NormalizeRichText()}if nil == err {diff --git a/kernel/av/av_fix.go b/kernel/av/av_fix.goindex 397f08cab4..5da6260950 100644--- a/kernel/av/av_fix.go+++ b/kernel/av/av_fix.go@@ -19,6 +19,7 @@ package avimport ("time"+ "github.com/88250/lute""github.com/88250/lute/ast""github.com/88250/lute/parse""github.com/siyuan-note/logging"@@ -140,13 +141,17 @@ func (av *AttributeView) NormalizeRichText() (err error) {richContent string}var normalized []normalizedText+ var luteEngine *lute.Luteav.visitPersistedValues(func(value *Value) {if nil != err || nil == value.Text || !value.Text.IsRich() {return}rich := *value.Text.Rich+ if luteEngine == nil {+ luteEngine = newValueTextRichLute()+ }var tree *parse.Tree- if tree, err = NormalizeValueTextRich(&rich); nil == err {+ if tree, err = normalizeValueTextRichWithLute(&rich, luteEngine); nil == err {normalized = append(normalized, normalizedText{value: value.Text,content: valueTextRichPlainContent(tree),diff --git a/kernel/av/normalized_cache_test.go b/kernel/av/normalized_cache_test.gonew file mode 100644index 0000000000..3e9bafbbea--- /dev/null+++ b/kernel/av/normalized_cache_test.go@@ -0,0 +1,110 @@+package av++import (+ "bytes"+ "encoding/json"+ "os"+ "path/filepath"+ "testing"+ "time"++ "github.com/siyuan-note/siyuan/kernel/cache"+ "github.com/siyuan-note/siyuan/kernel/util"+)++func normalizedRichCacheFixture(t testing.TB) (string, []byte) {+ t.Helper()+ const id = "20261010120000-cacheav"+ oldData := util.DataDir+ util.DataDir = t.TempDir()+ t.Cleanup(func() { cache.RemoveAVData(id); util.DataDir = oldData })+ view := &AttributeView{ID: id, Spec: RichTextSpec, Name: "Original", KeyValues: []*KeyValues{{+ Key: &Key{ID: "text", Type: KeyTypeText},+ }}}+ for i := 0; i < 100; i++ {+ view.KeyValues[0].Values = append(view.KeyValues[0].Values, &Value{Type: KeyTypeText, Text: &ValueText{+ Content: "incorrect projection",+ Rich: &ValueTextRich{Spec: ValueTextRichSpec, Format: ValueTextRichFormatKramdown, Content: "**bold**\n"},+ }})+ }+ data, err := json.Marshal(view)+ if err != nil {+ t.Fatal(err)+ }+ file := GetAttributeViewDataPath(id)+ if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(file, data, 0644); err != nil {+ t.Fatal(err)+ }+ return file, data+}++func TestNormalizedAttributeViewCachePreservesIsolationAndValidation(t *testing.T) {+ file, source := normalizedRichCacheFixture(t)+ const id = "20261010120000-cacheav"+ first, err := parseAttributeViewByPathInBoxWithOptions(file, "", false)+ if err != nil || first.KeyValues[0].Values[0].Text.Content != "bold" {+ t.Fatalf("first load failed: %+v, %v", first, err)+ }+ deadline := time.Now().Add(5 * time.Second)+ for {+ _, version, ok := cache.GetAVDataWithVersionInBox(id, "")+ if _, normalized := cache.GetAVNormalizedDataInBox(id, "", version); ok && normalized {+ break+ }+ if time.Now().After(deadline) {+ t.Fatal("normalized cache was not populated")+ }+ time.Sleep(time.Millisecond)+ }+ first.Name = "Mutated"+ first.KeyValues[0].Values[0].Text.Content = "mutated"+ first.KeyValues[0].Values[0].Text.Rich.Content = "mutated"+ second, err := parseAttributeViewByPathInBoxWithOptions(file, "", false)+ if err != nil || second.Name != "Original" || second.KeyValues[0].Values[0].Text.Content != "bold" {+ t.Fatalf("render mutations escaped into the cache: %+v, %v", second, err)+ }+ if data, ok := cache.GetAVDataInBox(id, ""); !ok || !bytes.Equal(data, source) {+ t.Fatal("normalization overwrote source cache bytes")+ }+ if data, err := os.ReadFile(file); err != nil || !bytes.Equal(data, source) {+ t.Fatal("read normalization wrote source data")+ }+ second.KeyValues[0].Values[0].Text.Rich.Spec = 999+ invalid, err := json.Marshal(second)+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(file, invalid, 0644); err != nil {+ t.Fatal(err)+ }+ cache.RemoveAVData(id)+ if _, err := parseAttributeViewByPathInBoxWithOptions(file, "", false); err == nil {+ t.Fatal("invalidated data bypassed rich text validation")+ }+}++func BenchmarkAttributeViewNormalizationCache(b *testing.B) {+ file, data := normalizedRichCacheFixture(b)+ if _, err := parseAttributeViewByPathInBoxWithOptions(file, "", false); err != nil {+ b.Fatal(err)+ }+ b.Run("uncached", func(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ if _, err := ParseAttributeViewData("20261010120000-cacheav", data); err != nil {+ b.Fatal(err)+ }+ }+ })+ b.Run("cached", func(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ if _, err := parseAttributeViewByPathInBoxWithOptions(file, "", false); err != nil {+ b.Fatal(err)+ }+ }+ })+}diff --git a/kernel/av/value.go b/kernel/av/value.goindex 898edd2f0c..ba363764c7 100644--- a/kernel/av/value.go+++ b/kernel/av/value.go@@ -564,6 +564,10 @@ func parseValueTextRich(rich *ValueTextRich) (blockDOM string, tree *parse.Tree,}func parseValueTextRichWithImages(rich *ValueTextRich, images bool) (blockDOM string, tree *parse.Tree, err error) {+ return parseValueTextRichWithLute(rich, images, newValueTextRichLute())+}++func parseValueTextRichWithLute(rich *ValueTextRich, images bool, luteEngine *lute.Lute) (blockDOM string, tree *parse.Tree, err error) {if nil == rich {return}@@ -576,7 +580,6 @@ func parseValueTextRichWithImages(rich *ValueTextRich, images bool) (blockDOM streturn}- luteEngine := newValueTextRichLute()content, protectedStyleEntities, protectErr := protectValueTextRichKramdownStyleEntities(rich.Content)if nil != protectErr {err = protectErr@@ -2171,12 +2174,16 @@ func newValueTextRichBackslashSentinel(content string) string {// NormalizeValueTextRich 校验并将文本字段的富文本源规范化为 SiYuan Kramdown。func NormalizeValueTextRich(rich *ValueTextRich) (tree *parse.Tree, err error) {- _, tree, err = parseValueTextRich(rich)+ return normalizeValueTextRichWithLute(rich, newValueTextRichLute())+}++func normalizeValueTextRichWithLute(rich *ValueTextRich, luteEngine *lute.Lute) (tree *parse.Tree, err error) {+ _, tree, err = parseValueTextRichWithLute(rich, false, luteEngine)if nil != err || nil == rich {return tree, err}var normalized string- if normalized, tree, err = normalizeValueTextRichTreeSource(tree); nil != err {+ if normalized, tree, err = normalizeValueTextRichTreeSourceWithLute(tree, false, luteEngine); nil != err {return nil, err}rich.Content = normalized@@ -2188,13 +2195,16 @@ func normalizeValueTextRichTreeSource(tree *parse.Tree) (content string, normali}func normalizeValueTextRichTreeSourceWithImages(tree *parse.Tree, images bool) (content string, normalizedTree *parse.Tree, err error) {+ return normalizeValueTextRichTreeSourceWithLute(tree, images, newValueTextRichLute())+}++func normalizeValueTextRichTreeSourceWithLute(tree *parse.Tree, images bool, luteEngine *lute.Lute) (content string, normalizedTree *parse.Tree, err error) {normalizedTree = treeprevious := ""for iteration := 0; iteration < 4; iteration++ {if err = normalizeValueTextRichTreeStyles(normalizedTree, images); nil != err {return "", nil, err}- luteEngine := newValueTextRichLute()blockDOM := luteEngine.Tree2BlockDOM(normalizedTree, luteEngine.RenderOptions, luteEngine.ParseOptions)content = valueTextRichBlockDOM2Kramdown(luteEngine, blockDOM)if 0 < iteration && previous == content {@@ -2204,7 +2214,7 @@ func normalizeValueTextRichTreeSourceWithImages(tree *parse.Tree, images bool) (candidate := &ValueTextRich{Spec: ValueTextRichSpec, Format: ValueTextRichFormatKramdown, Content: content,}- if _, normalizedTree, err = parseValueTextRichWithImages(candidate, images); nil != err {+ if _, normalizedTree, err = parseValueTextRichWithLute(candidate, images, luteEngine); nil != err {return "", nil, err}}diff --git a/kernel/cache/av.go b/kernel/cache/av.goindex 72bac78035..af38eb7637 100644--- a/kernel/cache/av.go+++ b/kernel/cache/av.go@@ -17,6 +17,7 @@package cacheimport (+ "bytes""sync""sync/atomic"@@ -30,8 +31,9 @@ var avCache, _ = ristretto.NewCache(&ristretto.Config{})type avCacheEntry struct {- raw []byte- version uint64+ raw []byte+ version uint64+ normalized []byte}type avSearchDataEntry struct {@@ -78,6 +80,48 @@ func GetAVDataWithVersionInBox(avID, boxID string) (raw []byte, version uint64,return entry.raw, entry.version, true}+// GetAVNormalizedDataInBox 读取同一数据版本的规范化副本,共用原始数据的容量及失效边界。+func GetAVNormalizedDataInBox(avID, boxID string, version uint64) (data []byte, ok bool) {+ avSearchDataCacheLock.RLock()+ defer avSearchDataCacheLock.RUnlock()+ key := avCacheKey(avID, boxID)+ value, found := avCache.Get(key)+ if !found || value == nil || avDataVersions[key] != version {+ return+ }+ entry := value.(*avCacheEntry)+ if entry.version != version || entry.normalized == nil {+ return+ }+ return entry.normalized, true+}++// SetAVNormalizedDataInBox 仅接纳已认证、已校验数据的同版本派生副本,不改变原始缓存字节。+func SetAVNormalizedDataInBox(avID, boxID string, version uint64, raw, normalized []byte) bool {+ if raw == nil || normalized == nil || version == 0 {+ return false+ }+ // 等待原始数据的异步准入,避免两次新条目准入互相覆盖派生副本。+ avCache.Wait()+ avSearchDataCacheLock.Lock()+ defer avSearchDataCacheLock.Unlock()+ key := avCacheKey(avID, boxID)+ if avDataVersions[key] != version {+ return false+ }+ cost := int64(len(raw) + len(normalized))+ if bytes.Equal(raw, normalized) {+ normalized = raw+ cost = int64(len(raw))+ }+ if cost > avCache.MaxCost() {+ return false+ }+ // 按新条目重新准入,使派生字节也参与容量淘汰,而不是直接扩大已驻留条目。+ avCache.Del(key)+ return avCache.Set(key, &avCacheEntry{raw: raw, version: version, normalized: normalized}, cost)+}+func EnsureAVDataVersionInBox(avID, boxID string) (version uint64) {avSearchDataCacheLock.Lock()defer avSearchDataCacheLock.Unlock()diff --git a/kernel/cache/av_test.go b/kernel/cache/av_test.goindex e90b26221c..0dd292f63b 100644--- a/kernel/cache/av_test.go+++ b/kernel/cache/av_test.go@@ -55,6 +55,53 @@ func TestAVSearchDataInvalidation(t *testing.T) {}}+func TestAVNormalizedDataSharesVersionAndNotebookBoundary(t *testing.T) {+ const id = "normalized-cache-test"+ defer RemoveAVData(id)+ raw, normalized := []byte("original"), []byte("normalized")+ version := SetAVDataWithVersionInBox(id, "a", raw)+ other := SetAVDataWithVersionInBox(id, "b", []byte("other"))+ if !SetAVNormalizedDataInBox(id, "a", version, raw, normalized) {+ t.Fatal("normalized data was not admitted")+ }+ avCache.Wait()+ if got, ok := GetAVNormalizedDataInBox(id, "a", version); !ok || !bytes.Equal(got, normalized) {+ t.Fatal("normalized data was not available")+ }+ if got, ok := GetAVDataInBox(id, "a"); !ok || !bytes.Equal(got, raw) {+ t.Fatal("normalization changed the source cache")+ }+ if _, ok := GetAVNormalizedDataInBox(id, "b", other); ok {+ t.Fatal("normalized plaintext crossed notebooks")+ }+ RemoveAVDataInBox(id, "a")+ if _, ok := GetAVNormalizedDataInBox(id, "a", version); ok {+ t.Fatal("removed source retained normalized data")+ }+ if SetAVNormalizedDataInBox(id, "a", version, raw, normalized) {+ t.Fatal("stale normalization was admitted after invalidation")+ }+ if got, ok := GetAVDataInBox(id, "b"); !ok || string(got) != "other" {+ t.Fatal("removal crossed notebook boundaries")+ }+}++func TestAVNormalizedDataRespectsCacheBudget(t *testing.T) {+ const id = "normalization-budget-test"+ previous := avCache.MaxCost()+ avCache.UpdateMaxCost(1024)+ defer func() { RemoveAVData(id); avCache.UpdateMaxCost(previous) }()+ raw := []byte("small source")+ version := SetAVDataWithVersionInBox(id, "", raw)+ avCache.Wait()+ if SetAVNormalizedDataInBox(id, "", version, raw, bytes.Repeat([]byte("x"), 2048)) {+ t.Fatal("oversize derived data exceeded cache budget")+ }+ if cached, ok := GetAVDataInBox(id, ""); !ok || !bytes.Equal(cached, raw) {+ t.Fatal("rejected normalization removed the source cache")+ }+}+func TestAVSearchDataWithoutRawData(t *testing.T) {const avID = "20260801120000-search"const boxID = "20260801120000-box"diff --git a/kernel/model/attribute_view_render.go b/kernel/model/attribute_view_render.goindex f7d5075fd2..0e034ca5f5 100644--- a/kernel/model/attribute_view_render.go+++ b/kernel/model/attribute_view_render.go@@ -491,12 +491,6 @@ func renderAttributeViewWithTarget(blockID, avID, viewID, query string, page, pa}}- // 诊断:AV 解析后的数据量- blockKV := attrView.GetBlockKeyValues()- if nil != blockKV {- } else {- }-viewable, err = renderAttributeView(attrView, blockID, viewID, "", query, page, pageSize, groupPaging, ignoreRows, writable, target, targetGroupID, calendarRanges...)return}