An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:bug: Preserve snapshot backups and track resource writes https://github.com/siyuan-note/siyuan/issues/20861
Daniel98311b3de8cb
diff --git a/kernel/api/repo_snapshot_test.go b/kernel/api/repo_snapshot_test.goindex 9aa1895748..3eec35356f 100644--- a/kernel/api/repo_snapshot_test.go+++ b/kernel/api/repo_snapshot_test.go@@ -3,10 +3,12 @@ package apiimport ("bytes""encoding/json"+ "mime/multipart""net/http""net/http/httptest""os""path/filepath"+ "strconv""strings""testing""time"@@ -91,11 +93,47 @@ func TestSnapshotFullCheckHTTP(t *testing.T) {if stable.Created || stable.ID != full.ID {t.Fatal("unchanged full check created a snapshot")}+ var upload bytes.Buffer+ writer := multipart.NewWriter(&upload)+ for _, field := range [][2]string{{"path", "data/probe.txt"}, {"isDir", "false"}, {"modTime", strconv.FormatInt(stamp.UnixMilli(), 10)}} {+ if err := writer.WriteField(field[0], field[1]); err != nil {+ t.Fatal(err)+ }+ }+ part, err := writer.CreateFormFile("file", "probe.txt")+ if err != nil {+ t.Fatal(err)+ }+ if _, err = part.Write([]byte("LATER!")); err != nil {+ t.Fatal(err)+ }+ if err = writer.Close(); err != nil {+ t.Fatal(err)+ }+ engine.POST("/api/file/putFile", putFile)+ request := httptest.NewRequest(http.MethodPost, "/api/file/putFile", &upload)+ request.Header.Set("Content-Type", writer.FormDataContentType())+ recorder := httptest.NewRecorder()+ engine.ServeHTTP(recorder, request)+ requireAPIContract(t, http.MethodPost, "/api/file/putFile", recorder)+ var response struct {+ Code int `json:"code"`+ }+ if err = json.Unmarshal(recorder.Body.Bytes(), &response); err != nil || response.Code != 0 {+ t.Fatalf("putFile response: %s %v", recorder.Body.String(), err)+ }+ if info, statErr := os.Stat(file); statErr != nil || info.Size() != 6 || !info.ModTime().Equal(stamp) {+ t.Fatalf("putFile did not preserve file metadata: %v %v", info, statErr)+ }+ uploaded := create(`{"memo":"after putFile","fullCheck":false}`)+ if !uploaded.Created || uploaded.ID == full.ID {+ t.Fatal("normal HTTP snapshot skipped a known same-attribute upload")+ }repo, err := dejavu.NewRepo(util.DataDir, util.RepoDir, util.HistoryDir, util.TempDir, "snapshot-test", "test", "test", model.Conf.Repo.Load().Key, nil, nil)if err != nil {t.Fatal(err)}- for _, version := range []struct{ id, content string }{{before.ID, "before"}, {full.ID, "AFTER!"}} {+ for _, version := range []struct{ id, content string }{{before.ID, "before"}, {full.ID, "AFTER!"}, {uploaded.ID, "LATER!"}} {index, err := repo.GetIndex(version.id)if err != nil {t.Fatal(err)diff --git a/kernel/go.mod b/kernel/go.modindex c328d03536..86a11b31d2 100644--- a/kernel/go.mod+++ b/kernel/go.mod@@ -75,7 +75,7 @@ require (github.com/sashabaranov/go-openai v1.42.0github.com/shirou/gopsutil/v4 v4.26.7github.com/siyuan-note/dataparser v0.0.0-20260115084335-b57cb8bc7c17- github.com/siyuan-note/dejavu v0.0.0-20261024125433-e6c364e4ae88+ github.com/siyuan-note/dejavu v0.0.0-20261024132346-de7e51e0ac64github.com/siyuan-note/encryption v0.0.0-20260715062728-9cb8e9548044github.com/siyuan-note/eventbus v0.0.0-20260530125927-d77c74260dcegithub.com/siyuan-note/filelock v0.0.0-20260831011452-393425122aaadiff --git a/kernel/go.sum b/kernel/go.sumindex 96e02b1fd0..7243ad452c 100644--- a/kernel/go.sum+++ b/kernel/go.sum@@ -575,6 +575,8 @@ github.com/siyuan-note/dejavu v0.0.0-20261024124832-dca458f71a76 h1:PbGojPCJzLqTgithub.com/siyuan-note/dejavu v0.0.0-20261024124832-dca458f71a76/go.mod h1:ixj6rxm+wkjoV8pBDGbuCyfMiPQh3MlxBwHG5OvtEQU=github.com/siyuan-note/dejavu v0.0.0-20261024125433-e6c364e4ae88 h1:O46nhmeuNGJ4U+BafFtP9mQzSD45iM0656tYvQIwtxk=github.com/siyuan-note/dejavu v0.0.0-20261024125433-e6c364e4ae88/go.mod h1:ixj6rxm+wkjoV8pBDGbuCyfMiPQh3MlxBwHG5OvtEQU=+github.com/siyuan-note/dejavu v0.0.0-20261024132346-de7e51e0ac64 h1:/JfsgQx2c1Y8kzgSlLWL9OuvgkPq4w2odLGcKdYVhm4=+github.com/siyuan-note/dejavu v0.0.0-20261024132346-de7e51e0ac64/go.mod h1:ixj6rxm+wkjoV8pBDGbuCyfMiPQh3MlxBwHG5OvtEQU=github.com/siyuan-note/encryption v0.0.0-20260715062728-9cb8e9548044 h1:b1/Q4DSREDvGh1/NbzR9873EKrjakdoUtsNlghWxUOI=github.com/siyuan-note/encryption v0.0.0-20260715062728-9cb8e9548044/go.mod h1:t6jXpJdrXzRsdQF+XGaw35f16JVZcDyIkk5/6//ywnY=github.com/siyuan-note/eventbus v0.0.0-20260530125927-d77c74260dce h1:356bfC3pucxXuYpZIwaQ3viVCbq5RbRuvBhb4xtoBR0=diff --git a/kernel/model/document_addressing_source.go b/kernel/model/document_addressing_source.goindex f93b44b58e..7b749439e7 100644--- a/kernel/model/document_addressing_source.go+++ b/kernel/model/document_addressing_source.go@@ -86,7 +86,7 @@ func WriteFileWithDocumentAddressing(absPath string, data []byte) error {return err}if !isDocumentAddressingSource(paths) {- return filelock.WriteFile(absPath, data)+ return util.WriteFileWithChangeTracking(absPath, data)}return filesys.MutateDocumentSources(documentSourcesFromPaths(paths), func() error {return util.WriteFileWithPreflight(absPath, data, false, func(current []byte) error {@@ -167,24 +167,6 @@ func CopyFileWithDocumentAddressing(source, destination string) error {if !info.IsDir() {return copyDocumentAddressingFile(source, destination)}- paths, err := documentAddressingDataPaths(destination)- if err != nil {- return err- }- containsDocuments := false- for _, name := range paths {- if runtime.GOOS == "windows" {- name = strings.ToLower(name)- }- valid := true- for _, part := range strings.Split(name, "/") {- valid = valid && (name == "" || ast.IsNodeIDPattern(part))- }- containsDocuments = containsDocuments || valid- }- if !containsDocuments {- return filelock.Copy(source, destination)- }if gulu.File.IsSubPath(source, destination) {return gulu.ErrCopyToSub}@@ -229,7 +211,7 @@ func copyDocumentAddressingFile(source, destination string) error {return err}if !isDocumentAddressingSource(paths) {- return filelock.Copy(source, destination)+ return util.CopyFileWithChangeTracking(source, destination)}data, err := filelock.ReadFile(source)if err != nil {diff --git a/kernel/model/repository.go b/kernel/model/repository.goindex 79394b25c6..01a5e01db9 100644--- a/kernel/model/repository.go+++ b/kernel/model/repository.go@@ -264,7 +264,7 @@ func RollbackRepoSnapshotFile(fileID string) (err error) {// 回滚快照时默认为当前数据创建一个快照// When rolling back a snapshot, a snapshot is created for the current data by default https://github.com/siyuan-note/siyuan/issues/12470FlushTxQueue()- _, err = IndexRepo("Backup before checkout")+ _, _, err = CreateRepoSnapshotWithFullCheck("Backup before checkout", true)if err != nil {logging.LogErrorf("index repository failed: %s", err)util.PushClearProgress()@@ -1461,14 +1461,6 @@ func checkoutRepo(id string) (err error) {if err = processAssetDownloadRecovery(repo, true); err != nil {return}- _, err = repo.Index("Backup before checkout", false, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})- if err != nil {- logging.LogErrorf("index repository failed: %s", err)- util.PushClearProgress()- util.PushErrMsg(fmt.Sprintf(Conf.Language(140), err), 0)- return- }-err = checkoutRepoSnapshot(repo, id, func(checkoutErr error) {release()FullReindexDirect()@@ -1493,6 +1485,12 @@ func checkoutRepo(id string) (err error) {// checkoutRepoSnapshot 恢复失败前可能已有文件落盘,返回结果前同步更新索引、缓存和界面。func checkoutRepoSnapshot(repo *dejavu.Repo, id string, refresh func(error)) error {+ // 恢复会核对实际内容,备份也必须完整校验,保留随后可能被覆盖的当前数据。+ if _, err := repo.IndexWithOptions("Backup before checkout", false,+ map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress},+ dejavu.IndexOptions{FullCheck: true}); err != nil {+ return err+ }upserts, removes, err := repo.Checkout(id, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})themes, icons := appearanceChangedPackages(&dejavu.MergeResult{Upserts: upserts, Removes: removes})refreshAppearancePackages(themes, icons)diff --git a/kernel/model/repository_checkout_test.go b/kernel/model/repository_checkout_test.goindex 36e35aee47..95352a6afa 100644--- a/kernel/model/repository_checkout_test.go+++ b/kernel/model/repository_checkout_test.go@@ -8,8 +8,95 @@ import ("time""github.com/siyuan-note/dejavu"+ "github.com/siyuan-note/siyuan/kernel/util")+func TestCheckoutRepoSnapshotBacksUpPreservedAttributes(t *testing.T) {+ for _, scope := range []string{"repository", "file"} {+ t.Run(scope, func(t *testing.T) {+ _, repo, _ := prepareAssetDownloadRepoTest(t)+ file := filepath.Join(util.DataDir, "assets", "probe.txt")+ if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {+ t.Fatal(err)+ }+ stamp := time.Unix(1700000100, 100000000)+ write := func(content string) {+ t.Helper()+ if err := os.WriteFile(file, []byte(content), 0644); err != nil {+ t.Fatal(err)+ }+ if err := os.Chtimes(file, stamp, stamp); err != nil {+ t.Fatal(err)+ }+ }+ write("before")+ id, _, err := CreateRepoSnapshot("original")+ if err != nil {+ t.Fatal(err)+ }+ original, err := repo.GetIndex(id)+ if err != nil {+ t.Fatal(err)+ }+ files, err := repo.GetFiles(original)+ if err != nil {+ t.Fatal(err)+ }+ write("AFTER!")+ if scope == "repository" {+ err = checkoutRepoSnapshot(repo, id, func(error) {})+ } else {+ fileID := ""+ for _, item := range files {+ if item.Path == "/assets/probe.txt" {+ fileID = item.ID+ }+ }+ if fileID == "" {+ t.Fatal("original asset missing")+ }+ err = RollbackRepoSnapshotFile(fileID)+ }+ if err != nil {+ t.Fatal(err)+ }+ backup, err := repo.Latest()+ if err != nil || backup.ID == id || backup.Memo != "Backup before checkout" {+ t.Fatalf("current content was not backed up: %v %v", backup, err)+ }+ if content := repoSnapshotTestContent(t, repo, backup.ID, "/assets/probe.txt"); content != "AFTER!" {+ t.Fatalf("backup omitted overwritten content: %q", content)+ }+ if content, readErr := os.ReadFile(file); readErr != nil || string(content) != "before" {+ t.Fatalf("snapshot was not restored: %q %v", content, readErr)+ }+ })+ }+}++func TestCheckoutRepoSnapshotStopsWhenBackupFails(t *testing.T) {+ _, repo, _ := prepareAssetDownloadRepoTest(t)+ id, _, err := CreateRepoSnapshot("original")+ if err != nil {+ t.Fatal(err)+ }+ file := filepath.Join(util.DataDir, "seed.txt")+ if err = os.WriteFile(file, []byte("current"), 0644); err != nil {+ t.Fatal(err)+ }+ if err = os.WriteFile(filepath.Join(repo.Path, "refs", "latest"), []byte("invalid"), 0644); err != nil {+ t.Fatal(err)+ }+ refreshed := false+ err = checkoutRepoSnapshot(repo, id, func(error) { refreshed = true })+ if err == nil || refreshed {+ t.Fatalf("checkout continued after backup failure: refreshed=%v err=%v", refreshed, err)+ }+ if content, readErr := os.ReadFile(file); readErr != nil || string(content) != "current" {+ t.Fatalf("failed backup allowed content replacement: %q %v", content, readErr)+ }+}+func TestCheckoutRepoSnapshotRefreshesPartialFailure(t *testing.T) {previousConf := ConfConf = &AppConf{Lang: "en"}diff --git a/kernel/model/repository_snapshot_test.go b/kernel/model/repository_snapshot_test.goindex a9e0383f93..31aa1f223c 100644--- a/kernel/model/repository_snapshot_test.go+++ b/kernel/model/repository_snapshot_test.go@@ -71,6 +71,94 @@ func TestSnapshotFullCheckPreservedAttributes(t *testing.T) {}}+func TestSnapshotKnownAssetChangesPreservedAttributes(t *testing.T) {+ for _, operation := range []string{"write", "copy-file", "copy-directory"} {+ t.Run(operation, func(t *testing.T) {+ _, repo, _ := prepareAssetDownloadRepoTest(t)+ assets := filepath.Join(util.DataDir, "assets")+ if err := os.MkdirAll(assets, 0755); err != nil {+ t.Fatal(err)+ }+ file := filepath.Join(assets, "probe.txt")+ stamp := time.Unix(1700000100, 100000000)+ if err := os.WriteFile(file, []byte("before"), 0644); err != nil {+ t.Fatal(err)+ }+ if err := os.Chtimes(file, stamp, stamp); err != nil {+ t.Fatal(err)+ }+ before, _, err := CreateRepoSnapshot("before asset change")+ if err != nil {+ t.Fatal(err)+ }+ if operation == "write" {+ if err = WriteFileWithDocumentAddressing(file, []byte("AFTER!")); err == nil {+ err = os.Chtimes(file, stamp, stamp)+ }+ } else {+ sourceDir := t.TempDir()+ source := filepath.Join(sourceDir, "probe.txt")+ if err = os.WriteFile(source, []byte("AFTER!"), 0644); err != nil {+ t.Fatal(err)+ }+ if err = os.Chtimes(source, stamp, stamp); err != nil {+ t.Fatal(err)+ }+ if operation == "copy-file" {+ err = CopyFileWithDocumentAddressing(source, file)+ } else {+ err = CopyFileWithDocumentAddressing(sourceDir, assets)+ }+ }+ if err != nil {+ t.Fatal(err)+ }+ if info, statErr := os.Stat(file); statErr != nil || info.Size() != 6 || !info.ModTime().Equal(stamp) {+ t.Fatalf("asset metadata was not preserved: %v %v", info, statErr)+ }+ IncSyncIfNeeded(file)+ if changed, checkErr := CheckRepoSnapshot(); checkErr != nil || !changed {+ t.Fatalf("known asset change was missed: changed=%v err=%v", changed, checkErr)+ }+ after, created, err := CreateRepoSnapshot("after asset change")+ if err != nil || !created || before == after {+ t.Fatalf("known asset change was not indexed: created=%v err=%v", created, err)+ }+ for id, expected := range map[string]string{before: "before", after: "AFTER!"} {+ if actual := repoSnapshotTestContent(t, repo, id, "/assets/probe.txt"); actual != expected {+ t.Fatalf("snapshot content: got %q, want %q", actual, expected)+ }+ }+ if stable, created, indexErr := CreateRepoSnapshot("stable"); indexErr != nil || created || stable != after {+ t.Fatalf("known asset change did not converge: created=%v err=%v", created, indexErr)+ }+ })+ }+}++func repoSnapshotTestContent(t *testing.T, repo *dejavu.Repo, id, path string) string {+ t.Helper()+ index, err := repo.GetIndex(id)+ if err != nil {+ t.Fatal(err)+ }+ files, err := repo.GetFiles(index)+ if err != nil {+ t.Fatal(err)+ }+ for _, file := range files {+ if file.Path == path {+ data, readErr := repo.OpenFile(file)+ if readErr != nil {+ t.Fatal(readErr)+ }+ return string(data)+ }+ }+ t.Fatalf("file missing from snapshot: %s", path)+ return ""+}+func TestSearchRepoSnapshot(t *testing.T) {_, partial, _ := prepareAssetDownloadRepoTest(t)index, err := partial.Latest()diff --git a/kernel/util/tracked_file.go b/kernel/util/tracked_file.gonew file mode 100644index 0000000000..744b36f41d--- /dev/null+++ b/kernel/util/tracked_file.go@@ -0,0 +1,37 @@+package util++import (+ "github.com/88250/gulu"+ "github.com/siyuan-note/dejavu"+ "github.com/siyuan-note/filelock"+)++// WriteFileWithChangeTracking 在释放文件锁前标记写入,保留修改时间的后续操作也不会隐藏内容变化。+func WriteFileWithChangeTracking(path string, data []byte) error {+ filelock.Lock(path)+ defer filelock.Unlock(path)+ if err := gulu.File.WriteFileSafer(path, data, 0644); err != nil {+ return err+ }+ dejavu.MarkFileChanged(path)+ return nil+}++// CopyFileWithChangeTracking 复制单个文件并保留权限和时间,在目标文件锁内标记内容变化。+func CopyFileWithChangeTracking(source, destination string) error {+ first, second := source, destination+ if first > second {+ first, second = second, first+ }+ filelock.Lock(first)+ defer filelock.Unlock(first)+ if second != first {+ filelock.Lock(second)+ defer filelock.Unlock(second)+ }+ if err := gulu.File.CopyFile(source, destination); err != nil {+ return err+ }+ dejavu.MarkFileChanged(destination)+ return nil+}diff --git a/kernel/util/tracked_file_test.go b/kernel/util/tracked_file_test.gonew file mode 100644index 0000000000..07df0d5fa4--- /dev/null+++ b/kernel/util/tracked_file_test.go@@ -0,0 +1,52 @@+package util++import (+ "bytes"+ "os"+ "path/filepath"+ "testing"+ "time"+)++func TestTrackedCopiesSerializeOppositeDirections(t *testing.T) {+ dir := t.TempDir()+ first, second := filepath.Join(dir, "first"), filepath.Join(dir, "second")+ one, two := bytes.Repeat([]byte("a"), 4096), bytes.Repeat([]byte("b"), 4096)+ if err := os.WriteFile(first, one, 0644); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(second, two, 0644); err != nil {+ t.Fatal(err)+ }+ start, done := make(chan struct{}), make(chan error, 2)+ copyFiles := func(source, destination string) {+ <-start+ for i := 0; i < 20; i++ {+ if err := CopyFileWithChangeTracking(source, destination); err != nil {+ done <- err+ return+ }+ }+ done <- nil+ }+ go copyFiles(first, second)+ go copyFiles(second, first)+ close(start)+ deadline := time.After(10 * time.Second)+ for i := 0; i < 2; i++ {+ select {+ case err := <-done:+ if err != nil {+ t.Fatal(err)+ }+ case <-deadline:+ t.Fatal("opposite-direction copies deadlocked")+ }+ }+ for _, name := range []string{first, second} {+ content, err := os.ReadFile(name)+ if err != nil || (!bytes.Equal(content, one) && !bytes.Equal(content, two)) {+ t.Fatalf("concurrent copy produced incomplete content: size=%d err=%v", len(content), err)+ }+ }+}