An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:bug: Preserve documents after temporary read failures https://github.com/siyuan-note/siyuan/issues/20409
Danielce2e76ee4b12
diff --git a/kernel/filesys/doc_ial_test.go b/kernel/filesys/doc_ial_test.gonew file mode 100644index 0000000000..1fe93859dd--- /dev/null+++ b/kernel/filesys/doc_ial_test.go@@ -0,0 +1,91 @@+package filesys++import (+ "bytes"+ "errors"+ "io"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/siyuan-note/siyuan/kernel/util"+)++type failingDocIALReader struct {+ err error+}++func TestDocIALEncryptedAuthentication(t *testing.T) {+ originalDataDir, originalProvider := util.DataDir, DEKProvider+ originalAcquire, originalRelease := DEKLockAcquire, DEKLockRelease+ util.DataDir = t.TempDir()+ dek := bytes.Repeat([]byte{19}, 32)+ DEKProvider = func(string) ([]byte, error) { return bytes.Clone(dek), nil }+ DEKLockAcquire, DEKLockRelease = nil, nil+ t.Cleanup(func() {+ util.DataDir, DEKProvider = originalDataDir, originalProvider+ DEKLockAcquire, DEKLockRelease = originalAcquire, originalRelease+ })+ legacy, err := os.ReadFile("../treenode/testdata/table-cell-legacy.sy")+ if err != nil {+ t.Fatal(err)+ }+ const boxID = "20260908000000-box0001"+ const document = "/20260908000000-root001.sy"+ ciphertext, err := encryptDataWithDEK(boxID, document, legacy, dek)+ if err != nil {+ t.Fatal(err)+ }+ absPath := filepath.Join(util.DataDir, boxID, document)+ if err = os.MkdirAll(filepath.Dir(absPath), 0700); err != nil {+ t.Fatal(err)+ }+ if err = os.WriteFile(absPath, ciphertext, 0600); err != nil {+ t.Fatal(err)+ }+ if ial, readErr := DocIAL(absPath); readErr != nil || len(ial) == 0 {+ t.Fatalf("supported encrypted document not readable: %v, %v", ial, readErr)+ }+ ciphertext[len(ciphertext)-1] ^= 1+ if err = os.WriteFile(absPath, ciphertext, 0600); err != nil {+ t.Fatal(err)+ }+ if ial, readErr := DocIAL(absPath); ial != nil || readErr == nil || errors.Is(readErr, ErrInvalidDocIAL) {+ t.Fatalf("authentication failure misclassified: %v, %v", ial, readErr)+ }+ retained, err := os.ReadFile(absPath)+ if err != nil || !bytes.Equal(retained, ciphertext) {+ t.Fatal("failed authentication changed the original ciphertext")+ }+}++func (r failingDocIALReader) Read([]byte) (int, error) {+ return 0, r.err+}++func TestParseDocIALDistinguishesReadFailure(t *testing.T) {+ readErr := errors.New("temporary read failure")+ for _, reader := range []io.Reader{+ failingDocIALReader{readErr},+ io.MultiReader(strings.NewReader(`{"Type":"NodeDocument","Properties":{`), failingDocIALReader{readErr}),+ } {+ ial, err := parseDocIAL(reader)+ if ial != nil || !errors.Is(err, readErr) || errors.Is(err, ErrInvalidDocIAL) {+ t.Fatalf("read failure misclassified: ial=%v err=%v", ial, err)+ }+ }+}++func TestParseDocIALCorruptionAndProperties(t *testing.T) {+ for _, data := range []string{`{"Type":"NodeDocument","broken":`, `{"Properties":{}}`, `{"Properties":{"title":`} {+ ial, err := parseDocIAL(strings.NewReader(data))+ if ial != nil || !errors.Is(err, ErrInvalidDocIAL) {+ t.Fatalf("corruption misclassified: data=%q ial=%v err=%v", data, ial, err)+ }+ }+ ial, err := parseDocIAL(strings.NewReader(`{"Properties":{"title":"A & B"},"Children":[]}`))+ if err != nil || ial["title"] != "A & B" {+ t.Fatalf("properties not decoded: ial=%v err=%v", ial, err)+ }+}diff --git a/kernel/filesys/tree.go b/kernel/filesys/tree.goindex 537bb4a7a1..2c856de17b 100644--- a/kernel/filesys/tree.go+++ b/kernel/filesys/tree.go@@ -21,6 +21,7 @@ import ("encoding/json""errors""fmt"+ "io""maps""os""path/filepath"@@ -395,14 +396,16 @@ func readDocIAL(absPath string, strict bool) (map[string]string, error) {return doc.Properties, nil}-func DocIAL(absPath string) (ret map[string]string) {+var ErrInvalidDocIAL = errors.New("invalid document properties")++func DocIAL(absPath string) (ret map[string]string, err error) {// 加密笔记本的 .sy 是密文,流式 jsoniter 解析无法处理,需先整体读+解密。// 反推 boxID:路径形如 <DataDir>/<boxID>/...;非加密笔记本走原流式逻辑。boxID := docIALBoxID(absPath)if boxID != "" && DEKProvider != nil {dek, encrypted, releaseCryptoLease, leaseErr := acquireCryptoLease(boxID)if leaseErr != nil {- return map[string]string{}+ return nil, leaseErr}defer releaseCryptoLease()if encrypted {@@ -413,42 +416,49 @@ func DocIAL(absPath string) (ret map[string]string) {if !errors.Is(readErr, os.ErrNotExist) {logging.LogErrorf("read file [%s] failed: %s", absPath, readErr)}- return nil+ return nil, readErr}relPath := filepath.ToSlash(strings.TrimPrefix(absPath, filepath.Join(util.DataDir, boxID)+string(os.PathSeparator)))plain, decErr := decryptDataWithDEK(boxID, relPath, raw, dek)if decErr != nil {// 属性查询不返回未认证的数据;需要区分缺失与错误的补树流程使用 readParentDocIAL。logging.LogErrorf("decrypt doc [%s] for IAL failed: %s", absPath, decErr)- return map[string]string{}- }- iter := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, bytes.NewReader(plain), 512)- for field := iter.ReadObject(); field != ""; field = iter.ReadObject() {- if field == "Properties" {- iter.ReadVal(&ret)- break- } else {- iter.Skip()- }- }- for k, v := range ret {- ret[k] = html.UnescapeAttrVal(v)+ return nil, decErr}- return+ return parseDocIAL(bytes.NewReader(plain))}}filelock.Lock(absPath)+ defer filelock.Unlock(absPath)file, err := os.Open(absPath)if err != nil {if !errors.Is(err, os.ErrNotExist) {logging.LogErrorf("open file [%s] failed: %s", absPath, err)}- filelock.Unlock(absPath)- return nil+ return nil, err+ }+ defer file.Close()+ return parseDocIAL(file)+}++// docIALReader 保留底层读取错误,避免将临时 I/O 失败当成 JSON 损坏。+type docIALReader struct {+ io.Reader+ err error+}++func (r *docIALReader) Read(p []byte) (int, error) {+ n, err := r.Reader.Read(p)+ if err != nil && err != io.EOF {+ r.err = err}+ return n, err+}- iter := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, file, 512)+func parseDocIAL(reader io.Reader) (ret map[string]string, err error) {+ r := &docIALReader{Reader: reader}+ iter := jsoniter.Parse(jsoniter.ConfigCompatibleWithStandardLibrary, r, 512)for field := iter.ReadObject(); field != ""; field = iter.ReadObject() {if field == "Properties" {iter.ReadVal(&ret)@@ -457,8 +467,12 @@ func DocIAL(absPath string) (ret map[string]string) {iter.Skip()}}- file.Close()- filelock.Unlock(absPath)+ if r.err != nil {+ return nil, r.err+ }+ if (iter.Error != nil && iter.Error != io.EOF) || len(ret) == 0 {+ return nil, ErrInvalidDocIAL+ }for k, v := range ret {ret[k] = html.UnescapeAttrVal(v)diff --git a/kernel/model/box_doc.go b/kernel/model/box_doc.goindex 935b3e6a11..3f179c25fe 100644--- a/kernel/model/box_doc.go+++ b/kernel/model/box_doc.go@@ -289,7 +289,8 @@ func BoxDocSubFileCountForPublishAt(boxID, docPath string, publishAccess Publishfunc boxDocSubFileCount(boxID, parentPath string, include func(docPath string) bool) int {ret, _ := visibleDocCount(boxID, parentPath, func(p string) map[string]string {- return filesys.DocIAL(filepath.Join(util.DataDir, boxID, p))+ ial, _ := filesys.DocIAL(filepath.Join(util.DataDir, boxID, p))+ return ial}, include)return ret}diff --git a/kernel/model/file.go b/kernel/model/file.goindex b068103791..c94967d588 100644--- a/kernel/model/file.go+++ b/kernel/model/file.go@@ -162,7 +162,14 @@ func (box *Box) docIAL(p string) (ret map[string]string) {}filePath := filepath.Join(util.DataDir, box.ID, p)- ret = filesys.DocIAL(filePath)+ var readErr error+ ret, readErr = filesys.DocIAL(filePath)+ if readErr != nil && !errors.Is(readErr, filesys.ErrInvalidDocIAL) {+ if !errors.Is(readErr, os.ErrNotExist) {+ logging.LogWarnf("read document properties [%s] failed: %s", filePath, readErr)+ }+ return nil+ }if 1 > len(ret) {// 目录枚举后文档可能已被删除,不将不存在的文件视为损坏。if _, statErr := os.Stat(filePath); errors.Is(statErr, os.ErrNotExist) {diff --git a/kernel/model/file_corrupted_test.go b/kernel/model/file_corrupted_test.goindex 4a58f28f4c..e9b4fbd3a0 100644--- a/kernel/model/file_corrupted_test.go+++ b/kernel/model/file_corrupted_test.go@@ -56,3 +56,28 @@ func TestDocIALCorruptedFileRemainsRecoverable(t *testing.T) {t.Fatalf("original corrupted bytes were not preserved: %q, %v", backup, err)}}++func TestDocIALReadFailureDoesNotMoveData(t *testing.T) {+ fixture := setupFileOperationTest(t)+ originalWorkspace := util.WorkspaceDir+ util.WorkspaceDir = t.TempDir()+ t.Cleanup(func() { util.WorkspaceDir = originalWorkspace })+ const document = "/20260718000009-abcdefg.sy"+ filePath := filepath.Join(util.DataDir, fixture.box.ID, document)+ if err := os.Mkdir(filePath, 0755); err != nil {+ t.Fatal(err)+ }+ payload := filepath.Join(filePath, "original")+ if err := os.WriteFile(payload, []byte("preserved"), 0644); err != nil {+ t.Fatal(err)+ }+ if attrs := fixture.box.docIAL(document); attrs != nil {+ t.Fatalf("unreadable document returned attributes: %v", attrs)+ }+ if raw, err := os.ReadFile(payload); err != nil || string(raw) != "preserved" {+ t.Fatalf("unreadable data was moved: %q, %v", raw, err)+ }+ if _, err := os.Stat(filepath.Join(util.WorkspaceDir, "corrupted")); !os.IsNotExist(err) {+ t.Fatalf("read failure created a corruption backup: %v", err)+ }+}diff --git a/kernel/model/file_corrupted_windows_test.go b/kernel/model/file_corrupted_windows_test.gonew file mode 100644index 0000000000..e3f8c5745f--- /dev/null+++ b/kernel/model/file_corrupted_windows_test.go@@ -0,0 +1,51 @@+package model++import (+ "os"+ "path/filepath"+ "syscall"+ "testing"++ "github.com/siyuan-note/siyuan/kernel/cache"+ "github.com/siyuan-note/siyuan/kernel/util"+)++func TestDocIALSharingViolationRecovers(t *testing.T) {+ fixture := setupFileOperationTest(t)+ originalWorkspace := util.WorkspaceDir+ util.WorkspaceDir = t.TempDir()+ t.Cleanup(func() { util.WorkspaceDir = originalWorkspace })+ filePath := filepath.Join(util.DataDir, fixture.box.ID, fixture.sourcePath)+ cache.RemoveDocIAL(fixture.sourcePath)+ ptr, err := syscall.UTF16PtrFromString(filePath)+ if err != nil {+ t.Fatal(err)+ }+ handle, err := syscall.CreateFile(ptr, syscall.GENERIC_READ, syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,+ nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0)+ if err != nil {+ t.Fatal(err)+ }+ closed := false+ defer func() {+ if !closed {+ syscall.CloseHandle(handle)+ }+ }()+ if _, err = os.Stat(filePath); err != nil {+ t.Fatal(err)+ }+ if attrs := fixture.box.docIAL(fixture.sourcePath); attrs != nil {+ t.Fatalf("unreadable file returned attributes: %v", attrs)+ }+ if _, err = os.Stat(filepath.Join(util.WorkspaceDir, "corrupted")); !os.IsNotExist(err) {+ t.Fatalf("sharing violation created a backup: %v", err)+ }+ if err = syscall.CloseHandle(handle); err != nil {+ t.Fatal(err)+ }+ closed = true+ if attrs := fixture.box.docIAL(fixture.sourcePath); attrs["title"] != "Source" {+ t.Fatalf("document did not recover after sharing violation: %v", attrs)+ }+}diff --git a/kernel/model/file_test.go b/kernel/model/file_test.goindex 8adaab5e6a..ec58f608ea 100644--- a/kernel/model/file_test.go+++ b/kernel/model/file_test.go@@ -133,7 +133,10 @@ func addFileOperationTestDoc(t *testing.T, fixture *fileOperationTestFixture, idfunc TestListDocTreeUsesPathIDForInvalidPropertiesID(t *testing.T) {fixture := setupFileOperationTest(t)- ial := filesys.DocIAL(filepath.Join(util.DataDir, fixture.box.ID, fixture.sourcePath))+ ial, err := filesys.DocIAL(filepath.Join(util.DataDir, fixture.box.ID, fixture.sourcePath))+ if err != nil {+ t.Fatal(err)+ }tests := []struct {name stringid stringdiff --git a/kernel/model/push_reload.go b/kernel/model/push_reload.goindex 828cf5f7d6..f6ce84258f 100644--- a/kernel/model/push_reload.go+++ b/kernel/model/push_reload.go@@ -278,7 +278,7 @@ func refreshDocInfo0(tree *parse.Tree, size uint64) {continue}- subDocIAL := filesys.DocIAL(filepath.Join(subDir, subFile.Name()))+ subDocIAL, _ := filesys.DocIAL(filepath.Join(subDir, subFile.Name()))if "true" == subDocIAL[DocHiddenAttr] {continue}