Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Preserve snapshot backups and track resource writes https://github.com/siyuan-note/siyuan/issues/20861

Daniel98311b3de8cb
diff --git a/kernel/api/repo_snapshot_test.go b/kernel/api/repo_snapshot_test.go
index 9aa1895748..3eec35356f 100644
--- a/kernel/api/repo_snapshot_test.go
+++ b/kernel/api/repo_snapshot_test.go
@@ -3,10 +3,12 @@ package api
import (
"bytes"
"encoding/json"
+ "mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
+ "strconv"
"strings"
"testing"
"time"
@@ -91,11 +93,47 @@ func TestSnapshotFullCheckHTTP(t *testing.T) {
if stable.Created || stable.ID != full.ID {
t.Fatal("unchanged full check created a snapshot")
}
+ var upload bytes.Buffer
+ writer := multipart.NewWriter(&upload)
+ for _, field := range [][2]string{{"path", "data/probe.txt"}, {"isDir", "false"}, {"modTime", strconv.FormatInt(stamp.UnixMilli(), 10)}} {
+ if err := writer.WriteField(field[0], field[1]); err != nil {
+ t.Fatal(err)
+ }
+ }
+ part, err := writer.CreateFormFile("file", "probe.txt")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = part.Write([]byte("LATER!")); err != nil {
+ t.Fatal(err)
+ }
+ if err = writer.Close(); err != nil {
+ t.Fatal(err)
+ }
+ engine.POST("/api/file/putFile", putFile)
+ request := httptest.NewRequest(http.MethodPost, "/api/file/putFile", &upload)
+ request.Header.Set("Content-Type", writer.FormDataContentType())
+ recorder := httptest.NewRecorder()
+ engine.ServeHTTP(recorder, request)
+ requireAPIContract(t, http.MethodPost, "/api/file/putFile", recorder)
+ var response struct {
+ Code int `json:"code"`
+ }
+ if err = json.Unmarshal(recorder.Body.Bytes(), &response); err != nil || response.Code != 0 {
+ t.Fatalf("putFile response: %s %v", recorder.Body.String(), err)
+ }
+ if info, statErr := os.Stat(file); statErr != nil || info.Size() != 6 || !info.ModTime().Equal(stamp) {
+ t.Fatalf("putFile did not preserve file metadata: %v %v", info, statErr)
+ }
+ uploaded := create(`{"memo":"after putFile","fullCheck":false}`)
+ if !uploaded.Created || uploaded.ID == full.ID {
+ t.Fatal("normal HTTP snapshot skipped a known same-attribute upload")
+ }
repo, err := dejavu.NewRepo(util.DataDir, util.RepoDir, util.HistoryDir, util.TempDir, "snapshot-test", "test", "test", model.Conf.Repo.Load().Key, nil, nil)
if err != nil {
t.Fatal(err)
}
- for _, version := range []struct{ id, content string }{{before.ID, "before"}, {full.ID, "AFTER!"}} {
+ for _, version := range []struct{ id, content string }{{before.ID, "before"}, {full.ID, "AFTER!"}, {uploaded.ID, "LATER!"}} {
index, err := repo.GetIndex(version.id)
if err != nil {
t.Fatal(err)
diff --git a/kernel/go.mod b/kernel/go.mod
index c328d03536..86a11b31d2 100644
--- a/kernel/go.mod
+++ b/kernel/go.mod
@@ -75,7 +75,7 @@ require (
github.com/sashabaranov/go-openai v1.42.0
github.com/shirou/gopsutil/v4 v4.26.7
github.com/siyuan-note/dataparser v0.0.0-20260115084335-b57cb8bc7c17
- github.com/siyuan-note/dejavu v0.0.0-20261024125433-e6c364e4ae88
+ github.com/siyuan-note/dejavu v0.0.0-20261024132346-de7e51e0ac64
github.com/siyuan-note/encryption v0.0.0-20260715062728-9cb8e9548044
github.com/siyuan-note/eventbus v0.0.0-20260530125927-d77c74260dce
github.com/siyuan-note/filelock v0.0.0-20260831011452-393425122aaa
diff --git a/kernel/go.sum b/kernel/go.sum
index 96e02b1fd0..7243ad452c 100644
--- a/kernel/go.sum
+++ b/kernel/go.sum
@@ -575,6 +575,8 @@ github.com/siyuan-note/dejavu v0.0.0-20261024124832-dca458f71a76 h1:PbGojPCJzLqT
github.com/siyuan-note/dejavu v0.0.0-20261024124832-dca458f71a76/go.mod h1:ixj6rxm+wkjoV8pBDGbuCyfMiPQh3MlxBwHG5OvtEQU=
github.com/siyuan-note/dejavu v0.0.0-20261024125433-e6c364e4ae88 h1:O46nhmeuNGJ4U+BafFtP9mQzSD45iM0656tYvQIwtxk=
github.com/siyuan-note/dejavu v0.0.0-20261024125433-e6c364e4ae88/go.mod h1:ixj6rxm+wkjoV8pBDGbuCyfMiPQh3MlxBwHG5OvtEQU=
+github.com/siyuan-note/dejavu v0.0.0-20261024132346-de7e51e0ac64 h1:/JfsgQx2c1Y8kzgSlLWL9OuvgkPq4w2odLGcKdYVhm4=
+github.com/siyuan-note/dejavu v0.0.0-20261024132346-de7e51e0ac64/go.mod h1:ixj6rxm+wkjoV8pBDGbuCyfMiPQh3MlxBwHG5OvtEQU=
github.com/siyuan-note/encryption v0.0.0-20260715062728-9cb8e9548044 h1:b1/Q4DSREDvGh1/NbzR9873EKrjakdoUtsNlghWxUOI=
github.com/siyuan-note/encryption v0.0.0-20260715062728-9cb8e9548044/go.mod h1:t6jXpJdrXzRsdQF+XGaw35f16JVZcDyIkk5/6//ywnY=
github.com/siyuan-note/eventbus v0.0.0-20260530125927-d77c74260dce h1:356bfC3pucxXuYpZIwaQ3viVCbq5RbRuvBhb4xtoBR0=
diff --git a/kernel/model/document_addressing_source.go b/kernel/model/document_addressing_source.go
index f93b44b58e..7b749439e7 100644
--- a/kernel/model/document_addressing_source.go
+++ b/kernel/model/document_addressing_source.go
@@ -86,7 +86,7 @@ func WriteFileWithDocumentAddressing(absPath string, data []byte) error {
return err
}
if !isDocumentAddressingSource(paths) {
- return filelock.WriteFile(absPath, data)
+ return util.WriteFileWithChangeTracking(absPath, data)
}
return filesys.MutateDocumentSources(documentSourcesFromPaths(paths), func() error {
return util.WriteFileWithPreflight(absPath, data, false, func(current []byte) error {
@@ -167,24 +167,6 @@ func CopyFileWithDocumentAddressing(source, destination string) error {
if !info.IsDir() {
return copyDocumentAddressingFile(source, destination)
}
- paths, err := documentAddressingDataPaths(destination)
- if err != nil {
- return err
- }
- containsDocuments := false
- for _, name := range paths {
- if runtime.GOOS == "windows" {
- name = strings.ToLower(name)
- }
- valid := true
- for _, part := range strings.Split(name, "/") {
- valid = valid && (name == "" || ast.IsNodeIDPattern(part))
- }
- containsDocuments = containsDocuments || valid
- }
- if !containsDocuments {
- return filelock.Copy(source, destination)
- }
if gulu.File.IsSubPath(source, destination) {
return gulu.ErrCopyToSub
}
@@ -229,7 +211,7 @@ func copyDocumentAddressingFile(source, destination string) error {
return err
}
if !isDocumentAddressingSource(paths) {
- return filelock.Copy(source, destination)
+ return util.CopyFileWithChangeTracking(source, destination)
}
data, err := filelock.ReadFile(source)
if err != nil {
diff --git a/kernel/model/repository.go b/kernel/model/repository.go
index 79394b25c6..01a5e01db9 100644
--- a/kernel/model/repository.go
+++ b/kernel/model/repository.go
@@ -264,7 +264,7 @@ func RollbackRepoSnapshotFile(fileID string) (err error) {
// 回滚快照时默认为当前数据创建一个快照
// When rolling back a snapshot, a snapshot is created for the current data by default https://github.com/siyuan-note/siyuan/issues/12470
FlushTxQueue()
- _, err = IndexRepo("Backup before checkout")
+ _, _, err = CreateRepoSnapshotWithFullCheck("Backup before checkout", true)
if err != nil {
logging.LogErrorf("index repository failed: %s", err)
util.PushClearProgress()
@@ -1461,14 +1461,6 @@ func checkoutRepo(id string) (err error) {
if err = processAssetDownloadRecovery(repo, true); err != nil {
return
}
- _, err = repo.Index("Backup before checkout", false, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
- if err != nil {
- logging.LogErrorf("index repository failed: %s", err)
- util.PushClearProgress()
- util.PushErrMsg(fmt.Sprintf(Conf.Language(140), err), 0)
- return
- }
-
err = checkoutRepoSnapshot(repo, id, func(checkoutErr error) {
release()
FullReindexDirect()
@@ -1493,6 +1485,12 @@ func checkoutRepo(id string) (err error) {
// checkoutRepoSnapshot 恢复失败前可能已有文件落盘,返回结果前同步更新索引、缓存和界面。
func checkoutRepoSnapshot(repo *dejavu.Repo, id string, refresh func(error)) error {
+ // 恢复会核对实际内容,备份也必须完整校验,保留随后可能被覆盖的当前数据。
+ if _, err := repo.IndexWithOptions("Backup before checkout", false,
+ map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress},
+ dejavu.IndexOptions{FullCheck: true}); err != nil {
+ return err
+ }
upserts, removes, err := repo.Checkout(id, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
themes, icons := appearanceChangedPackages(&dejavu.MergeResult{Upserts: upserts, Removes: removes})
refreshAppearancePackages(themes, icons)
diff --git a/kernel/model/repository_checkout_test.go b/kernel/model/repository_checkout_test.go
index 36e35aee47..95352a6afa 100644
--- a/kernel/model/repository_checkout_test.go
+++ b/kernel/model/repository_checkout_test.go
@@ -8,8 +8,95 @@ import (
"time"
"github.com/siyuan-note/dejavu"
+ "github.com/siyuan-note/siyuan/kernel/util"
)
+func TestCheckoutRepoSnapshotBacksUpPreservedAttributes(t *testing.T) {
+ for _, scope := range []string{"repository", "file"} {
+ t.Run(scope, func(t *testing.T) {
+ _, repo, _ := prepareAssetDownloadRepoTest(t)
+ file := filepath.Join(util.DataDir, "assets", "probe.txt")
+ if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
+ t.Fatal(err)
+ }
+ stamp := time.Unix(1700000100, 100000000)
+ write := func(content string) {
+ t.Helper()
+ if err := os.WriteFile(file, []byte(content), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(file, stamp, stamp); err != nil {
+ t.Fatal(err)
+ }
+ }
+ write("before")
+ id, _, err := CreateRepoSnapshot("original")
+ if err != nil {
+ t.Fatal(err)
+ }
+ original, err := repo.GetIndex(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ files, err := repo.GetFiles(original)
+ if err != nil {
+ t.Fatal(err)
+ }
+ write("AFTER!")
+ if scope == "repository" {
+ err = checkoutRepoSnapshot(repo, id, func(error) {})
+ } else {
+ fileID := ""
+ for _, item := range files {
+ if item.Path == "/assets/probe.txt" {
+ fileID = item.ID
+ }
+ }
+ if fileID == "" {
+ t.Fatal("original asset missing")
+ }
+ err = RollbackRepoSnapshotFile(fileID)
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ backup, err := repo.Latest()
+ if err != nil || backup.ID == id || backup.Memo != "Backup before checkout" {
+ t.Fatalf("current content was not backed up: %v %v", backup, err)
+ }
+ if content := repoSnapshotTestContent(t, repo, backup.ID, "/assets/probe.txt"); content != "AFTER!" {
+ t.Fatalf("backup omitted overwritten content: %q", content)
+ }
+ if content, readErr := os.ReadFile(file); readErr != nil || string(content) != "before" {
+ t.Fatalf("snapshot was not restored: %q %v", content, readErr)
+ }
+ })
+ }
+}
+
+func TestCheckoutRepoSnapshotStopsWhenBackupFails(t *testing.T) {
+ _, repo, _ := prepareAssetDownloadRepoTest(t)
+ id, _, err := CreateRepoSnapshot("original")
+ if err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(util.DataDir, "seed.txt")
+ if err = os.WriteFile(file, []byte("current"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(filepath.Join(repo.Path, "refs", "latest"), []byte("invalid"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ refreshed := false
+ err = checkoutRepoSnapshot(repo, id, func(error) { refreshed = true })
+ if err == nil || refreshed {
+ t.Fatalf("checkout continued after backup failure: refreshed=%v err=%v", refreshed, err)
+ }
+ if content, readErr := os.ReadFile(file); readErr != nil || string(content) != "current" {
+ t.Fatalf("failed backup allowed content replacement: %q %v", content, readErr)
+ }
+}
+
func TestCheckoutRepoSnapshotRefreshesPartialFailure(t *testing.T) {
previousConf := Conf
Conf = &AppConf{Lang: "en"}
diff --git a/kernel/model/repository_snapshot_test.go b/kernel/model/repository_snapshot_test.go
index a9e0383f93..31aa1f223c 100644
--- a/kernel/model/repository_snapshot_test.go
+++ b/kernel/model/repository_snapshot_test.go
@@ -71,6 +71,94 @@ func TestSnapshotFullCheckPreservedAttributes(t *testing.T) {
}
}
+func TestSnapshotKnownAssetChangesPreservedAttributes(t *testing.T) {
+ for _, operation := range []string{"write", "copy-file", "copy-directory"} {
+ t.Run(operation, func(t *testing.T) {
+ _, repo, _ := prepareAssetDownloadRepoTest(t)
+ assets := filepath.Join(util.DataDir, "assets")
+ if err := os.MkdirAll(assets, 0755); err != nil {
+ t.Fatal(err)
+ }
+ file := filepath.Join(assets, "probe.txt")
+ stamp := time.Unix(1700000100, 100000000)
+ if err := os.WriteFile(file, []byte("before"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chtimes(file, stamp, stamp); err != nil {
+ t.Fatal(err)
+ }
+ before, _, err := CreateRepoSnapshot("before asset change")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if operation == "write" {
+ if err = WriteFileWithDocumentAddressing(file, []byte("AFTER!")); err == nil {
+ err = os.Chtimes(file, stamp, stamp)
+ }
+ } else {
+ sourceDir := t.TempDir()
+ source := filepath.Join(sourceDir, "probe.txt")
+ if err = os.WriteFile(source, []byte("AFTER!"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if err = os.Chtimes(source, stamp, stamp); err != nil {
+ t.Fatal(err)
+ }
+ if operation == "copy-file" {
+ err = CopyFileWithDocumentAddressing(source, file)
+ } else {
+ err = CopyFileWithDocumentAddressing(sourceDir, assets)
+ }
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if info, statErr := os.Stat(file); statErr != nil || info.Size() != 6 || !info.ModTime().Equal(stamp) {
+ t.Fatalf("asset metadata was not preserved: %v %v", info, statErr)
+ }
+ IncSyncIfNeeded(file)
+ if changed, checkErr := CheckRepoSnapshot(); checkErr != nil || !changed {
+ t.Fatalf("known asset change was missed: changed=%v err=%v", changed, checkErr)
+ }
+ after, created, err := CreateRepoSnapshot("after asset change")
+ if err != nil || !created || before == after {
+ t.Fatalf("known asset change was not indexed: created=%v err=%v", created, err)
+ }
+ for id, expected := range map[string]string{before: "before", after: "AFTER!"} {
+ if actual := repoSnapshotTestContent(t, repo, id, "/assets/probe.txt"); actual != expected {
+ t.Fatalf("snapshot content: got %q, want %q", actual, expected)
+ }
+ }
+ if stable, created, indexErr := CreateRepoSnapshot("stable"); indexErr != nil || created || stable != after {
+ t.Fatalf("known asset change did not converge: created=%v err=%v", created, indexErr)
+ }
+ })
+ }
+}
+
+func repoSnapshotTestContent(t *testing.T, repo *dejavu.Repo, id, path string) string {
+ t.Helper()
+ index, err := repo.GetIndex(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ files, err := repo.GetFiles(index)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, file := range files {
+ if file.Path == path {
+ data, readErr := repo.OpenFile(file)
+ if readErr != nil {
+ t.Fatal(readErr)
+ }
+ return string(data)
+ }
+ }
+ t.Fatalf("file missing from snapshot: %s", path)
+ return ""
+}
+
func TestSearchRepoSnapshot(t *testing.T) {
_, partial, _ := prepareAssetDownloadRepoTest(t)
index, err := partial.Latest()
diff --git a/kernel/util/tracked_file.go b/kernel/util/tracked_file.go
new file mode 100644
index 0000000000..744b36f41d
--- /dev/null
+++ b/kernel/util/tracked_file.go
@@ -0,0 +1,37 @@
+package util
+
+import (
+ "github.com/88250/gulu"
+ "github.com/siyuan-note/dejavu"
+ "github.com/siyuan-note/filelock"
+)
+
+// WriteFileWithChangeTracking 在释放文件锁前标记写入,保留修改时间的后续操作也不会隐藏内容变化。
+func WriteFileWithChangeTracking(path string, data []byte) error {
+ filelock.Lock(path)
+ defer filelock.Unlock(path)
+ if err := gulu.File.WriteFileSafer(path, data, 0644); err != nil {
+ return err
+ }
+ dejavu.MarkFileChanged(path)
+ return nil
+}
+
+// CopyFileWithChangeTracking 复制单个文件并保留权限和时间,在目标文件锁内标记内容变化。
+func CopyFileWithChangeTracking(source, destination string) error {
+ first, second := source, destination
+ if first > second {
+ first, second = second, first
+ }
+ filelock.Lock(first)
+ defer filelock.Unlock(first)
+ if second != first {
+ filelock.Lock(second)
+ defer filelock.Unlock(second)
+ }
+ if err := gulu.File.CopyFile(source, destination); err != nil {
+ return err
+ }
+ dejavu.MarkFileChanged(destination)
+ return nil
+}
diff --git a/kernel/util/tracked_file_test.go b/kernel/util/tracked_file_test.go
new file mode 100644
index 0000000000..07df0d5fa4
--- /dev/null
+++ b/kernel/util/tracked_file_test.go
@@ -0,0 +1,52 @@
+package util
+
+import (
+ "bytes"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+func TestTrackedCopiesSerializeOppositeDirections(t *testing.T) {
+ dir := t.TempDir()
+ first, second := filepath.Join(dir, "first"), filepath.Join(dir, "second")
+ one, two := bytes.Repeat([]byte("a"), 4096), bytes.Repeat([]byte("b"), 4096)
+ if err := os.WriteFile(first, one, 0644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(second, two, 0644); err != nil {
+ t.Fatal(err)
+ }
+ start, done := make(chan struct{}), make(chan error, 2)
+ copyFiles := func(source, destination string) {
+ <-start
+ for i := 0; i < 20; i++ {
+ if err := CopyFileWithChangeTracking(source, destination); err != nil {
+ done <- err
+ return
+ }
+ }
+ done <- nil
+ }
+ go copyFiles(first, second)
+ go copyFiles(second, first)
+ close(start)
+ deadline := time.After(10 * time.Second)
+ for i := 0; i < 2; i++ {
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-deadline:
+ t.Fatal("opposite-direction copies deadlocked")
+ }
+ }
+ for _, name := range []string{first, second} {
+ content, err := os.ReadFile(name)
+ if err != nil || (!bytes.Equal(content, one) && !bytes.Equal(content, two)) {
+ t.Fatalf("concurrent copy produced incomplete content: size=%d err=%v", len(content), err)
+ }
+ }
+}