Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Preserve documents after temporary read failures https://github.com/siyuan-note/siyuan/issues/20409

Danielce2e76ee4b12
diff --git a/kernel/filesys/doc_ial_test.go b/kernel/filesys/doc_ial_test.go
new file mode 100644
index 0000000000..1fe93859dd
--- /dev/null
+++ b/kernel/filesys/doc_ial_test.go
@@ -0,0 +1,91 @@
+package filesys
+
+import (
+ "bytes"
+ "errors"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+type failingDocIALReader struct {
+ err error
+}
+
+func TestDocIALEncryptedAuthentication(t *testing.T) {
+ originalDataDir, originalProvider := util.DataDir, DEKProvider
+ originalAcquire, originalRelease := DEKLockAcquire, DEKLockRelease
+ util.DataDir = t.TempDir()
+ dek := bytes.Repeat([]byte{19}, 32)
+ DEKProvider = func(string) ([]byte, error) { return bytes.Clone(dek), nil }
+ DEKLockAcquire, DEKLockRelease = nil, nil
+ t.Cleanup(func() {
+ util.DataDir, DEKProvider = originalDataDir, originalProvider
+ DEKLockAcquire, DEKLockRelease = originalAcquire, originalRelease
+ })
+ legacy, err := os.ReadFile("../treenode/testdata/table-cell-legacy.sy")
+ if err != nil {
+ t.Fatal(err)
+ }
+ const boxID = "20260908000000-box0001"
+ const document = "/20260908000000-root001.sy"
+ ciphertext, err := encryptDataWithDEK(boxID, document, legacy, dek)
+ if err != nil {
+ t.Fatal(err)
+ }
+ absPath := filepath.Join(util.DataDir, boxID, document)
+ if err = os.MkdirAll(filepath.Dir(absPath), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(absPath, ciphertext, 0600); err != nil {
+ t.Fatal(err)
+ }
+ if ial, readErr := DocIAL(absPath); readErr != nil || len(ial) == 0 {
+ t.Fatalf("supported encrypted document not readable: %v, %v", ial, readErr)
+ }
+ ciphertext[len(ciphertext)-1] ^= 1
+ if err = os.WriteFile(absPath, ciphertext, 0600); err != nil {
+ t.Fatal(err)
+ }
+ if ial, readErr := DocIAL(absPath); ial != nil || readErr == nil || errors.Is(readErr, ErrInvalidDocIAL) {
+ t.Fatalf("authentication failure misclassified: %v, %v", ial, readErr)
+ }
+ retained, err := os.ReadFile(absPath)
+ if err != nil || !bytes.Equal(retained, ciphertext) {
+ t.Fatal("failed authentication changed the original ciphertext")
+ }
+}
+
+func (r failingDocIALReader) Read([]byte) (int, error) {
+ return 0, r.err
+}
+
+func TestParseDocIALDistinguishesReadFailure(t *testing.T) {
+ readErr := errors.New("temporary read failure")
+ for _, reader := range []io.Reader{
+ failingDocIALReader{readErr},
+ io.MultiReader(strings.NewReader(`{"Type":"NodeDocument","Properties":{`), failingDocIALReader{readErr}),
+ } {
+ ial, err := parseDocIAL(reader)
+ if ial != nil || !errors.Is(err, readErr) || errors.Is(err, ErrInvalidDocIAL) {
+ t.Fatalf("read failure misclassified: ial=%v err=%v", ial, err)
+ }
+ }
+}
+
+func TestParseDocIALCorruptionAndProperties(t *testing.T) {
+ for _, data := range []string{`{"Type":"NodeDocument","broken":`, `{"Properties":{}}`, `{"Properties":{"title":`} {
+ ial, err := parseDocIAL(strings.NewReader(data))
+ if ial != nil || !errors.Is(err, ErrInvalidDocIAL) {
+ t.Fatalf("corruption misclassified: data=%q ial=%v err=%v", data, ial, err)
+ }
+ }
+ ial, err := parseDocIAL(strings.NewReader(`{"Properties":{"title":"A & B"},"Children":[]}`))
+ if err != nil || ial["title"] != "A & B" {
+ t.Fatalf("properties not decoded: ial=%v err=%v", ial, err)
+ }
+}
diff --git a/kernel/filesys/tree.go b/kernel/filesys/tree.go
index 537bb4a7a1..2c856de17b 100644
--- a/kernel/filesys/tree.go
+++ b/kernel/filesys/tree.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "io"
"maps"
"os"
"path/filepath"
@@ -395,14 +396,16 @@ func readDocIAL(absPath string, strict bool) (map[string]string, error) {
return doc.Properties, nil
}
-func DocIAL(absPath string) (ret map[string]string) {
+var ErrInvalidDocIAL = errors.New("invalid document properties")
+
+func DocIAL(absPath string) (ret map[string]string, err error) {
// 加密笔记本的 .sy 是密文,流式 jsoniter 解析无法处理,需先整体读+解密。
// 反推 boxID:路径形如 <DataDir>/<boxID>/...;非加密笔记本走原流式逻辑。
boxID := docIALBoxID(absPath)
if boxID != "" && DEKProvider != nil {
dek, encrypted, releaseCryptoLease, leaseErr := acquireCryptoLease(boxID)
if leaseErr != nil {
- return map[string]string{}
+ return nil, leaseErr
}
defer releaseCryptoLease()
if encrypted {
@@ -413,42 +416,49 @@ func DocIAL(absPath string) (ret map[string]string) {
if !errors.Is(readErr, os.ErrNotExist) {
logging.LogErrorf("read file [%s] failed: %s", absPath, readErr)
}
- return nil
+ return nil, readErr
}
relPath := filepath.ToSlash(strings.TrimPrefix(absPath, filepath.Join(util.DataDir, boxID)+string(os.PathSeparator)))
plain, decErr := decryptDataWithDEK(boxID, relPath, raw, dek)
if decErr != nil {
// 属性查询不返回未认证的数据;需要区分缺失与错误的补树流程使用 readParentDocIAL。
logging.LogErrorf("decrypt doc [%s] for IAL failed: %s", absPath, decErr)
- return map[string]string{}
- }
- iter := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, bytes.NewReader(plain), 512)
- for field := iter.ReadObject(); field != ""; field = iter.ReadObject() {
- if field == "Properties" {
- iter.ReadVal(&ret)
- break
- } else {
- iter.Skip()
- }
- }
- for k, v := range ret {
- ret[k] = html.UnescapeAttrVal(v)
+ return nil, decErr
}
- return
+ return parseDocIAL(bytes.NewReader(plain))
}
}
filelock.Lock(absPath)
+ defer filelock.Unlock(absPath)
file, err := os.Open(absPath)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
logging.LogErrorf("open file [%s] failed: %s", absPath, err)
}
- filelock.Unlock(absPath)
- return nil
+ return nil, err
+ }
+ defer file.Close()
+ return parseDocIAL(file)
+}
+
+// docIALReader 保留底层读取错误,避免将临时 I/O 失败当成 JSON 损坏。
+type docIALReader struct {
+ io.Reader
+ err error
+}
+
+func (r *docIALReader) Read(p []byte) (int, error) {
+ n, err := r.Reader.Read(p)
+ if err != nil && err != io.EOF {
+ r.err = err
}
+ return n, err
+}
- iter := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, file, 512)
+func parseDocIAL(reader io.Reader) (ret map[string]string, err error) {
+ r := &docIALReader{Reader: reader}
+ iter := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, r, 512)
for field := iter.ReadObject(); field != ""; field = iter.ReadObject() {
if field == "Properties" {
iter.ReadVal(&ret)
@@ -457,8 +467,12 @@ func DocIAL(absPath string) (ret map[string]string) {
iter.Skip()
}
}
- file.Close()
- filelock.Unlock(absPath)
+ if r.err != nil {
+ return nil, r.err
+ }
+ if (iter.Error != nil && iter.Error != io.EOF) || len(ret) == 0 {
+ return nil, ErrInvalidDocIAL
+ }
for k, v := range ret {
ret[k] = html.UnescapeAttrVal(v)
diff --git a/kernel/model/box_doc.go b/kernel/model/box_doc.go
index 935b3e6a11..3f179c25fe 100644
--- a/kernel/model/box_doc.go
+++ b/kernel/model/box_doc.go
@@ -289,7 +289,8 @@ func BoxDocSubFileCountForPublishAt(boxID, docPath string, publishAccess Publish
func boxDocSubFileCount(boxID, parentPath string, include func(docPath string) bool) int {
ret, _ := visibleDocCount(boxID, parentPath, func(p string) map[string]string {
- return filesys.DocIAL(filepath.Join(util.DataDir, boxID, p))
+ ial, _ := filesys.DocIAL(filepath.Join(util.DataDir, boxID, p))
+ return ial
}, include)
return ret
}
diff --git a/kernel/model/file.go b/kernel/model/file.go
index b068103791..c94967d588 100644
--- a/kernel/model/file.go
+++ b/kernel/model/file.go
@@ -162,7 +162,14 @@ func (box *Box) docIAL(p string) (ret map[string]string) {
}
filePath := filepath.Join(util.DataDir, box.ID, p)
- ret = filesys.DocIAL(filePath)
+ var readErr error
+ ret, readErr = filesys.DocIAL(filePath)
+ if readErr != nil && !errors.Is(readErr, filesys.ErrInvalidDocIAL) {
+ if !errors.Is(readErr, os.ErrNotExist) {
+ logging.LogWarnf("read document properties [%s] failed: %s", filePath, readErr)
+ }
+ return nil
+ }
if 1 > len(ret) {
// 目录枚举后文档可能已被删除,不将不存在的文件视为损坏。
if _, statErr := os.Stat(filePath); errors.Is(statErr, os.ErrNotExist) {
diff --git a/kernel/model/file_corrupted_test.go b/kernel/model/file_corrupted_test.go
index 4a58f28f4c..e9b4fbd3a0 100644
--- a/kernel/model/file_corrupted_test.go
+++ b/kernel/model/file_corrupted_test.go
@@ -56,3 +56,28 @@ func TestDocIALCorruptedFileRemainsRecoverable(t *testing.T) {
t.Fatalf("original corrupted bytes were not preserved: %q, %v", backup, err)
}
}
+
+func TestDocIALReadFailureDoesNotMoveData(t *testing.T) {
+ fixture := setupFileOperationTest(t)
+ originalWorkspace := util.WorkspaceDir
+ util.WorkspaceDir = t.TempDir()
+ t.Cleanup(func() { util.WorkspaceDir = originalWorkspace })
+ const document = "/20260718000009-abcdefg.sy"
+ filePath := filepath.Join(util.DataDir, fixture.box.ID, document)
+ if err := os.Mkdir(filePath, 0755); err != nil {
+ t.Fatal(err)
+ }
+ payload := filepath.Join(filePath, "original")
+ if err := os.WriteFile(payload, []byte("preserved"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if attrs := fixture.box.docIAL(document); attrs != nil {
+ t.Fatalf("unreadable document returned attributes: %v", attrs)
+ }
+ if raw, err := os.ReadFile(payload); err != nil || string(raw) != "preserved" {
+ t.Fatalf("unreadable data was moved: %q, %v", raw, err)
+ }
+ if _, err := os.Stat(filepath.Join(util.WorkspaceDir, "corrupted")); !os.IsNotExist(err) {
+ t.Fatalf("read failure created a corruption backup: %v", err)
+ }
+}
diff --git a/kernel/model/file_corrupted_windows_test.go b/kernel/model/file_corrupted_windows_test.go
new file mode 100644
index 0000000000..e3f8c5745f
--- /dev/null
+++ b/kernel/model/file_corrupted_windows_test.go
@@ -0,0 +1,51 @@
+package model
+
+import (
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/cache"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestDocIALSharingViolationRecovers(t *testing.T) {
+ fixture := setupFileOperationTest(t)
+ originalWorkspace := util.WorkspaceDir
+ util.WorkspaceDir = t.TempDir()
+ t.Cleanup(func() { util.WorkspaceDir = originalWorkspace })
+ filePath := filepath.Join(util.DataDir, fixture.box.ID, fixture.sourcePath)
+ cache.RemoveDocIAL(fixture.sourcePath)
+ ptr, err := syscall.UTF16PtrFromString(filePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ handle, err := syscall.CreateFile(ptr, syscall.GENERIC_READ, syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,
+ nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ closed := false
+ defer func() {
+ if !closed {
+ syscall.CloseHandle(handle)
+ }
+ }()
+ if _, err = os.Stat(filePath); err != nil {
+ t.Fatal(err)
+ }
+ if attrs := fixture.box.docIAL(fixture.sourcePath); attrs != nil {
+ t.Fatalf("unreadable file returned attributes: %v", attrs)
+ }
+ if _, err = os.Stat(filepath.Join(util.WorkspaceDir, "corrupted")); !os.IsNotExist(err) {
+ t.Fatalf("sharing violation created a backup: %v", err)
+ }
+ if err = syscall.CloseHandle(handle); err != nil {
+ t.Fatal(err)
+ }
+ closed = true
+ if attrs := fixture.box.docIAL(fixture.sourcePath); attrs["title"] != "Source" {
+ t.Fatalf("document did not recover after sharing violation: %v", attrs)
+ }
+}
diff --git a/kernel/model/file_test.go b/kernel/model/file_test.go
index 8adaab5e6a..ec58f608ea 100644
--- a/kernel/model/file_test.go
+++ b/kernel/model/file_test.go
@@ -133,7 +133,10 @@ func addFileOperationTestDoc(t *testing.T, fixture *fileOperationTestFixture, id
func TestListDocTreeUsesPathIDForInvalidPropertiesID(t *testing.T) {
fixture := setupFileOperationTest(t)
- ial := filesys.DocIAL(filepath.Join(util.DataDir, fixture.box.ID, fixture.sourcePath))
+ ial, err := filesys.DocIAL(filepath.Join(util.DataDir, fixture.box.ID, fixture.sourcePath))
+ if err != nil {
+ t.Fatal(err)
+ }
tests := []struct {
name string
id string
diff --git a/kernel/model/push_reload.go b/kernel/model/push_reload.go
index 828cf5f7d6..f6ce84258f 100644
--- a/kernel/model/push_reload.go
+++ b/kernel/model/push_reload.go
@@ -278,7 +278,7 @@ func refreshDocInfo0(tree *parse.Tree, size uint64) {
continue
}
- subDocIAL := filesys.DocIAL(filepath.Join(subDir, subFile.Name()))
+ subDocIAL, _ := filesys.DocIAL(filepath.Join(subDir, subFile.Name()))
if "true" == subDocIAL[DocHiddenAttr] {
continue
}