Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Accept UTF-8 BOM in attribute view definitions https://github.com/siyuan-note/siyuan/issues/21052

Daniel0ad0d284cd39
diff --git a/kernel/av/av.go b/kernel/av/av.go
index 5c6835bd16..6177be2cfd 100644
--- a/kernel/av/av.go
+++ b/kernel/av/av.go
@@ -619,6 +619,7 @@ func cacheAttributeViewData(attrView *AttributeView, boxID string, data []byte)
}
func parseAttributeViewSearchInfo(data []byte) (ret *AttributeViewSearchInfo, err error) {
+ data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
ret = &AttributeViewSearchInfo{}
if err = json.Unmarshal(data, ret); err != nil {
return
@@ -687,7 +688,7 @@ func loadAttributeViewSearchInfoInBox(avID, avJSONPath, avBoxID string) (ret *At
if ret, err = parseAttributeViewSearchInfo(data); err != nil {
logging.LogErrorf("unmarshal attribute view search info [%s] failed: %s", avID, err)
- return nil, err
+ return nil, fmt.Errorf("parse attribute view search info [%s] failed: %w", avJSONPath, err)
}
if cache.SetAVSearchDataInBox(avID, avBoxID, dataVersion, ret) {
return ret, nil
@@ -836,6 +837,11 @@ func ReadAttributeViewCustomColorUsageByPath(avJSONPath string) (avID string, in
}
func parseAttributeViewByPathInBoxWithOptions(avJSONPath, boxID string, resolveColors bool) (ret *AttributeView, err error) {
+ defer func() {
+ if err != nil {
+ err = fmt.Errorf("load attribute view [%s] failed: %w", avJSONPath, err)
+ }
+ }()
generation := cache.GetAVCacheGeneration()
if !filelock.IsExist(avJSONPath) {
err = ErrViewNotFound
@@ -905,6 +911,8 @@ func ParseAttributeViewData(avID string, data []byte) (ret *AttributeView, err e
}
func parseAttributeViewData(avID string, data []byte, normalizeRichText bool) (ret *AttributeView, err error) {
+ // 仅处理已认证明文的开头 BOM;缓存和条件写入保留原始字节以检测外部修改。
+ data = bytes.TrimPrefix(data, []byte("\xef\xbb\xbf"))
ret = &AttributeView{RenderedViewables: map[string]Viewable{}}
if err = json.Unmarshal(data, ret); err != nil {
if strings.Contains(err.Error(), ".relation.contents of type av.Value") {
diff --git a/kernel/av/bom_test.go b/kernel/av/bom_test.go
new file mode 100644
index 0000000000..b07a90ae03
--- /dev/null
+++ b/kernel/av/bom_test.go
@@ -0,0 +1,210 @@
+// SiYuan - https://github.com/siyuan-note/siyuan
+// Copyright (c) 2020-present, b3log.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+package av
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/cache"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestAttributeViewBOMReadAndSave(t *testing.T) {
+ for _, encrypted := range []bool{false, true} {
+ t.Run(map[bool]string{false: "ordinary", true: "encrypted"}[encrypted], func(t *testing.T) {
+ setupCollectionIndexTest(t)
+ boxID := ""
+ if encrypted {
+ boxID = "20261028120000-bombox0"
+ }
+ oldProvider, oldBoxes := AVDEKProvider, AVEncryptedBoxIDs
+ oldAcquire, oldRelease := AVLockAcquire, AVLockRelease
+ AVDEKProvider = func(string) ([]byte, error) { return bytes.Repeat([]byte{0x42}, 32), nil }
+ AVEncryptedBoxIDs = func() []string { return []string{boxID} }
+ AVLockAcquire, AVLockRelease = nil, nil
+ t.Cleanup(func() {
+ AVDEKProvider, AVEncryptedBoxIDs = oldProvider, oldBoxes
+ AVLockAcquire, AVLockRelease = oldAcquire, oldRelease
+ })
+ view := collectionIndexTestView()
+ view.Spec, view.Name = PlainTextSpec, "BOM database"
+ SetAVBoxID(view.ID, boxID)
+ t.Cleanup(func() { SetAVBoxID(view.ID, "") })
+ raw, err := json.Marshal(view)
+ if err != nil {
+ t.Fatal(err)
+ }
+ plain := append([]byte{0xef, 0xbb, 0xbf}, raw...)
+ disk := plain
+ if encrypted {
+ disk, err = encryptAVData(boxID, view.ID, plain)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ path := attributeViewDataPathByBox(view.ID, boxID)
+ if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(path, disk, 0600); err != nil {
+ t.Fatal(err)
+ }
+ for range 2 {
+ info, readErr := GetAttributeViewSearchInfoInBox(view.ID, boxID)
+ if readErr != nil || info == nil || info.Name != view.Name {
+ t.Fatalf("search: %+v, %v", info, readErr)
+ }
+ index, readErr := BlockCollectionIndexInBox(boxID)
+ if readErr != nil || index.Empty() {
+ t.Fatalf("collection: %+v, %v", index, readErr)
+ }
+ loaded, readErr := ParseAttributeViewForIndexInBox(view.ID, boxID)
+ if readErr != nil || loaded.Name != view.Name {
+ t.Fatalf("full parse: %+v, %v", loaded, readErr)
+ }
+ view = loaded
+ }
+ actual, err := os.ReadFile(path)
+ if err != nil || !bytes.Equal(actual, disk) {
+ t.Fatal("read changed source", err)
+ }
+ release := RecordSourceWrites(view, func() bool { return true }, func(_ string, before, after []byte) error {
+ if !bytes.Equal(before, disk) {
+ t.Fatal("recorder lost original bytes")
+ }
+ return nil
+ })
+ view.Name = "Saved database"
+ err = SaveAttributeView(view)
+ release()
+ if err != nil {
+ t.Fatal(err)
+ }
+ actual, err = os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if encrypted {
+ actual, err = decryptAVData(boxID, view.ID, actual)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ if bytes.HasPrefix(actual, []byte{0xef, 0xbb, 0xbf}) || !json.Valid(actual) {
+ t.Fatal("save did not produce BOM-free JSON")
+ }
+ cache.ClearAVCache()
+ loaded, err := ParseAttributeViewForIndexInBox(view.ID, boxID)
+ if err != nil || loaded.Name != view.Name {
+ t.Fatalf("saved data: %+v, %v", loaded, err)
+ }
+ if encrypted {
+ previous, readErr := decryptAVData(boxID, view.ID, disk)
+ if readErr != nil {
+ t.Fatal(readErr)
+ }
+ recovered, readErr := ParseAttributeViewData(view.ID, previous)
+ if readErr != nil || recovered.Name != "BOM database" {
+ t.Fatal("original ciphertext is not recoverable", readErr)
+ }
+ // 密文前的 BOM 不属于已认证明文,必须拒绝且保留源文件。
+ invalid := append([]byte{0xef, 0xbb, 0xbf}, disk...)
+ if err = os.WriteFile(path, invalid, 0600); err != nil {
+ t.Fatal(err)
+ }
+ cache.ClearAVCache()
+ if _, err = ParseAttributeViewForIndexInBox(view.ID, boxID); err == nil {
+ t.Fatal("accepted unauthenticated prefix")
+ }
+ actual, err = os.ReadFile(path)
+ if err != nil || !bytes.Equal(actual, invalid) {
+ t.Fatal("failed authentication changed source", err)
+ }
+ }
+ })
+ }
+}
+
+func TestAttributeViewBOMParsingBoundaries(t *testing.T) {
+ const id = "20261028120000-bomav00"
+ for _, data := range [][]byte{
+ []byte(`{"spec": 6, "name": "before\ufeffafter"}`),
+ []byte("\xef\xbb\xbf{\"spec\": 6, \"name\": \"before\\ufeffafter\"}"),
+ } {
+ view, err := ParseAttributeViewData(id, data)
+ if err != nil || view.Name != "before\ufeffafter" {
+ t.Fatal("valid content BOM was changed", err)
+ }
+ }
+ for _, data := range [][]byte{
+ []byte("\xef\xbb\xbf\xef\xbb\xbf{}"),
+ []byte(" \xef\xbb\xbf{}"),
+ []byte("\xef\xbb\xbf{broken"),
+ } {
+ if _, err := ParseAttributeViewData(id, data); err == nil {
+ t.Fatalf("accepted invalid JSON: %x", data)
+ }
+ }
+}
+
+func TestAttributeViewBOMConditionalSave(t *testing.T) {
+ setupCollectionIndexTest(t)
+ view := &AttributeView{ID: "20261028120000-bomav00", Spec: PlainTextSpec}
+ raw, err := json.Marshal(view)
+ if err != nil {
+ t.Fatal(err)
+ }
+ original := append([]byte{0xef, 0xbb, 0xbf}, raw...)
+ path := GetAttributeViewDataPath(view.ID)
+ if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(path, original, 0600); err != nil {
+ t.Fatal(err)
+ }
+ view, err = ParseAttributeView(view.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ view.Name = "Conditional save"
+ if err = SaveAttributeViewIfUnchanged(view, original); err != nil {
+ t.Fatal(err)
+ }
+ if err = SaveAttributeViewIfUnchanged(view, original); !errors.Is(err, util.ErrFileChanged) {
+ t.Fatal("stale write accepted", err)
+ }
+}
+
+func TestAttributeViewInvalidJSONReportsSource(t *testing.T) {
+ setupCollectionIndexTest(t)
+ const id = "20261028120000-bomav00"
+ path := GetAttributeViewDataPath(id)
+ if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+ t.Fatal(err)
+ }
+ invalid := []byte("\xef\xbb\xbf{broken")
+ if err := os.WriteFile(path, invalid, 0600); err != nil {
+ t.Fatal(err)
+ }
+ for name, read := range map[string]func() error{
+ "full": func() error { _, err := ParseAttributeView(id); return err },
+ "search": func() error { _, err := GetAttributeViewSearchInfoInBox(id, ""); return err },
+ "collection": func() error { _, err := BlockCollectionIndexInBox(""); return err },
+ } {
+ if err := read(); err == nil || !strings.Contains(err.Error(), id) {
+ t.Fatalf("%s omitted source: %v", name, err)
+ }
+ }
+ actual, err := os.ReadFile(path)
+ if err != nil || !bytes.Equal(actual, invalid) {
+ t.Fatal("invalid JSON was modified", err)
+ }
+}
diff --git a/kernel/av/collection_index.go b/kernel/av/collection_index.go
index e8bc9b07ec..a747a6ab22 100644
--- a/kernel/av/collection_index.go
+++ b/kernel/av/collection_index.go
@@ -5,7 +5,9 @@
package av
import (
+ "bytes"
"encoding/json"
+ "fmt"
"iter"
"maps"
"os"
@@ -207,14 +209,14 @@ func BlockCollectionIndexInBox(boxID string) (*BlockCollectionIndex, error) {
}
data, readErr := ReadAttributeViewDataInBox(id, boxID)
if readErr != nil {
- return nil, readErr
+ return nil, fmt.Errorf("read attribute view [%s] in box [%s] failed: %w", id, boxID, readErr)
}
var header struct {
Spec int `json:"spec"`
Automations *AutomationConfig `json:"automations"`
}
- if readErr = json.Unmarshal(data, &header); readErr != nil {
- return nil, readErr
+ if readErr = json.Unmarshal(bytes.TrimPrefix(data, []byte("\xef\xbb\xbf")), &header); readErr != nil {
+ return nil, fmt.Errorf("parse attribute view [%s] in box [%s] failed: %w", id, boxID, readErr)
}
views[id] = collectionScopes(header.Automations)
if header.Spec > CurrentSpec && len(views[id]) > 0 {
diff --git a/kernel/model/attribute_view_collection_bom_fts5_test.go b/kernel/model/attribute_view_collection_bom_fts5_test.go
new file mode 100644
index 0000000000..adddf46d5b
--- /dev/null
+++ b/kernel/model/attribute_view_collection_bom_fts5_test.go
@@ -0,0 +1,48 @@
+// SiYuan - https://github.com/siyuan-note/siyuan
+// Copyright (c) 2020-present, b3log.org
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+//go:build fts5
+
+package model
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/av"
+ "github.com/siyuan-note/siyuan/kernel/cache"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestBlockCollectionBOMDocumentEdits(t *testing.T) {
+ for _, warm := range []bool{false, true} {
+ t.Run(map[bool]string{false: "cold", true: "warm"}[warm], func(t *testing.T) {
+ fixture, view, _ := setupBlockCollectionTest(t)
+ path := filepath.Join(util.DataDir, "storage", "av", view.ID+".json")
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(path, append([]byte{0xef, 0xbb, 0xbf}, data...), 0600); err != nil {
+ t.Fatal(err)
+ }
+ cache.ClearAVCache()
+ if warm {
+ if _, err = av.BlockCollectionIndexInBox(""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ for _, tags := range []string{"todo", "todo,other"} {
+ if err = PerformTxSync(blockCollectionTags(t, fixture.sourceID, tags)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ loaded := readAttributeViewItemsTest(t, view.ID)
+ if loaded.GetBlockValueByBoundID(fixture.sourceID) == nil || len(loaded.GetBlockKeyValues().Values) != 1 {
+ t.Fatal("document edits did not preserve collection binding")
+ }
+ })
+ }
+}