Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Recover interrupted duplicate document recreation https://github.com/siyuan-note/siyuan/issues/20929

Daniele05231b34116
diff --git a/kernel/filesys/tree_create_journal.go b/kernel/filesys/tree_create_journal.go
new file mode 100644
index 0000000000..75aecbc38b
--- /dev/null
+++ b/kernel/filesys/tree_create_journal.go
@@ -0,0 +1,47 @@
+package filesys
+
+import (
+ "fmt"
+ "github.com/88250/lute/parse"
+ "github.com/siyuan-note/siyuan/kernel/cache"
+ "github.com/siyuan-note/siyuan/kernel/treenode"
+ "github.com/siyuan-note/siyuan/kernel/util"
+ "os"
+)
+
+// CreateTreeWithJournal prepares the exact persisted bytes, then records recovery
+// before creating a new source. The destination must remain absent.
+func CreateTreeWithJournal(tree *parse.Tree, prepare func([]byte) error) (err error) {
+ unlock := lockDocumentSource()
+ defer unlock()
+ dek, _, release, err := acquireCryptoLease(tree.Box)
+ if err != nil {
+ return err
+ }
+ defer release()
+ data, filename, err := prepareWriteTree(tree)
+ if err != nil {
+ return err
+ }
+ disk, err := encryptDataWithDEK(tree.Box, tree.Path, data, dek)
+ if err != nil {
+ return err
+ }
+ if _, statErr := os.Lstat(filename); !os.IsNotExist(statErr) {
+ return fmt.Errorf("document creation target already exists or is inaccessible: %s", filename)
+ }
+ if err = prepare(disk); err != nil {
+ return err
+ }
+ pending, err := treenode.BeginDocumentMetadataWrite(tree)
+ if err != nil {
+ return err
+ }
+ defer finishDocumentMetadataWrite(tree, pending, &err)
+ if err = util.WriteFileIfUnchanged(filename, nil, disk); err != nil {
+ return err
+ }
+ cache.SetTreeDataInBox(tree.ID, tree.Box, data)
+ afterWriteTree(tree)
+ return nil
+}
diff --git a/kernel/model/conf.go b/kernel/model/conf.go
index 3e3e0b34b6..4b9f84bc0a 100644
--- a/kernel/model/conf.go
+++ b/kernel/model/conf.go
@@ -302,6 +302,10 @@ func InitConf() {
logging.LogFatalf(logging.ExitCodeFileSysErr, "recover incremental sort conf failed: %s", err)
return
}
+ if err := recoverDocumentSources(); err != nil {
+ logging.LogFatalf(logging.ExitCodeFileSysErr, "recover document sources failed: %s", err)
+ return
+ }
clearEncryptedExportTempOnBoot()
clearOldInstallPackages("")
if err := CleanupRetiredInstallResources(); err != nil {
@@ -1184,7 +1188,7 @@ func InitBoxes() {
}
box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
- if !initialized {
+ if !initialized || hasDocumentSourceRecovery(box.ID) {
indexBox(box.ID)
}
}
diff --git a/kernel/model/document_source_journal.go b/kernel/model/document_source_journal.go
new file mode 100644
index 0000000000..bbade2c01e
--- /dev/null
+++ b/kernel/model/document_source_journal.go
@@ -0,0 +1,286 @@
+package model
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path"
+ "path/filepath"
+ "slices"
+ "strings"
+
+ "github.com/88250/lute/ast"
+ "github.com/siyuan-note/siyuan/kernel/cache"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+// Recovery records retain exact persisted bytes, including ciphertext. They live
+// beside the durable SQL queue, outside synchronized notebook sources.
+type documentSourceFile struct {
+ Box string `json:"box"`
+ Path string `json:"path"`
+ Before []byte `json:"before"`
+ After []byte `json:"after"`
+}
+
+type documentSourceMove struct {
+ Box string `json:"box"`
+ From string `json:"from"`
+ To string `json:"to"`
+}
+
+type documentSourceJournal struct {
+ Version int `json:"version"`
+ ID string `json:"id"`
+ State string `json:"state"`
+ Files []*documentSourceFile `json:"files"`
+ Move *documentSourceMove `json:"move,omitempty"`
+ Recovered []string `json:"recovered,omitempty"`
+}
+
+func documentSourceJournalBase() string { return filepath.Join(util.QueueDir, "document-sources") }
+
+func documentSourcePath(box, p string, directory bool) (string, error) {
+ if !ast.IsNodeIDPattern(box) || p == "/" || p != path.Clean(p) || !strings.HasPrefix(p, "/") || strings.Contains(p, "\\") {
+ return "", errors.New("invalid document recovery path")
+ }
+ parts := strings.Split(strings.TrimPrefix(p, "/"), "/")
+ for i, part := range parts {
+ if !directory && i == len(parts)-1 {
+ if !strings.HasSuffix(part, ".sy") {
+ return "", errors.New("invalid document recovery source")
+ }
+ part = strings.TrimSuffix(part, ".sy")
+ }
+ if !ast.IsNodeIDPattern(part) {
+ return "", errors.New("invalid document recovery identity")
+ }
+ }
+ return filepath.Join(util.DataDir, box, filepath.FromSlash(p)), nil
+}
+
+func (j *documentSourceJournal) filename() string {
+ return filepath.Join(documentSourceJournalBase(), j.ID+".json")
+}
+
+func (j *documentSourceJournal) valid() bool {
+ if j.Version != 1 || !ast.IsNodeIDPattern(j.ID) || len(j.Files) == 0 ||
+ (j.State != "pending" && j.State != "committed" && j.State != "recovered") {
+ return false
+ }
+ seen := map[string]bool{}
+ for _, file := range j.Files {
+ if file == nil {
+ return false
+ }
+ name, err := documentSourcePath(file.Box, file.Path, false)
+ if err != nil || seen[name] {
+ return false
+ }
+ seen[name] = true
+ }
+ if j.Move != nil {
+ from, err := documentSourcePath(j.Move.Box, j.Move.From, true)
+ to, toErr := documentSourcePath(j.Move.Box, j.Move.To, true)
+ if err != nil || toErr != nil || from == to || filepath.Dir(from) != filepath.Dir(to) {
+ return false
+ }
+ }
+ return true
+}
+
+func (j *documentSourceJournal) save() error {
+ if util.QueueDir == "" || !j.valid() {
+ return errors.New("invalid document source recovery record")
+ }
+ data, err := json.Marshal(j)
+ if err != nil {
+ return err
+ }
+ digest := sha256.Sum256(data)
+ envelope, err := json.Marshal(struct {
+ Data json.RawMessage `json:"data"`
+ Digest string `json:"sha256"`
+ }{data, hex.EncodeToString(digest[:])})
+ if err != nil {
+ return err
+ }
+ if err = os.MkdirAll(documentSourceJournalBase(), 0700); err != nil {
+ return err
+ }
+ file, err := os.CreateTemp(documentSourceJournalBase(), ".source-*")
+ if err != nil {
+ return err
+ }
+ defer os.Remove(file.Name())
+ if err = file.Chmod(0600); err == nil {
+ _, err = file.Write(envelope)
+ }
+ if err == nil {
+ err = file.Sync()
+ }
+ err = errors.Join(err, file.Close())
+ if err != nil {
+ return err
+ }
+ return os.Rename(file.Name(), j.filename())
+}
+
+func readDocumentSourceJournals() ([]*documentSourceJournal, error) {
+ if util.QueueDir == "" {
+ return nil, nil
+ }
+ entries, err := os.ReadDir(documentSourceJournalBase())
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ var result []*documentSourceJournal
+ for _, entry := range entries {
+ if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") {
+ continue
+ }
+ data, err := os.ReadFile(filepath.Join(documentSourceJournalBase(), entry.Name()))
+ if err != nil {
+ return nil, err
+ }
+ var envelope struct {
+ Data json.RawMessage `json:"data"`
+ Digest string `json:"sha256"`
+ }
+ if json.Unmarshal(data, &envelope) != nil {
+ return nil, errors.New("corrupt document source recovery record")
+ }
+ digest := sha256.Sum256(envelope.Data)
+ j := &documentSourceJournal{}
+ if envelope.Digest != hex.EncodeToString(digest[:]) || json.Unmarshal(envelope.Data, j) != nil || !j.valid() || entry.Name() != j.ID+".json" {
+ return nil, errors.New("unsupported or corrupt document source recovery record")
+ }
+ result = append(result, j)
+ }
+ return result, nil
+}
+
+// rollback checks the entire batch before changing anything and conditionally restores
+// each source. Unexpected edits and conflicting directories preserve the record and data.
+func (j *documentSourceJournal) rollback() error {
+ for _, file := range j.Files {
+ name, _ := documentSourcePath(file.Box, file.Path, false)
+ current, err := os.ReadFile(name)
+ if os.IsNotExist(err) {
+ current, err = nil, nil
+ }
+ if err != nil {
+ return err
+ }
+ if !bytes.Equal(current, file.Before) && !bytes.Equal(current, file.After) {
+ return fmt.Errorf("%w: %s", util.ErrFileChanged, name)
+ }
+ }
+ if j.Move != nil {
+ from, _ := documentSourcePath(j.Move.Box, j.Move.From, true)
+ to, _ := documentSourcePath(j.Move.Box, j.Move.To, true)
+ fromInfo, fromErr := os.Lstat(from)
+ toInfo, toErr := os.Lstat(to)
+ if os.IsNotExist(fromErr) && toErr == nil && toInfo.IsDir() {
+ if err := os.Rename(to, from); err != nil {
+ return err
+ }
+ } else if fromErr != nil || !fromInfo.IsDir() || !os.IsNotExist(toErr) {
+ return errors.New("conflicting document recovery directories")
+ }
+ }
+ for _, file := range j.Files {
+ name, _ := documentSourcePath(file.Box, file.Path, false)
+ current, err := os.ReadFile(name)
+ if os.IsNotExist(err) {
+ current, err = nil, nil
+ }
+ if err != nil {
+ return err
+ }
+ if bytes.Equal(current, file.Before) {
+ continue
+ }
+ if file.Before == nil {
+ if err = removeIndexRepairSourceIfUnchanged(name, file.After); err != nil {
+ return err
+ }
+ } else if err = util.WriteFileIfUnchanged(name, current, file.Before); err != nil {
+ return err
+ }
+ cache.RemoveTreeDataInBox(util.GetTreeID(file.Path), file.Box)
+ cache.RemoveDocIALInBox(file.Path, file.Box)
+ }
+ j.State = "recovered"
+ return j.save()
+}
+
+func recoverDocumentSources() error {
+ entries, err := readDocumentSourceJournals()
+ if err != nil {
+ return err
+ }
+ for _, entry := range entries {
+ if entry.State == "pending" {
+ if err = entry.rollback(); err != nil {
+ return err
+ }
+ } else {
+ entry.State = "recovered"
+ if err = entry.save(); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func hasDocumentSourceRecovery(box string) bool {
+ entries, err := readDocumentSourceJournals()
+ if err != nil {
+ return true
+ }
+ for _, j := range entries {
+ for _, file := range j.Files {
+ if file.Box == box && !slices.Contains(j.Recovered, box) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func finishDocumentSourceRecovery(box string) error {
+ entries, err := readDocumentSourceJournals()
+ if err != nil {
+ return err
+ }
+ for _, j := range entries {
+ if j.State != "recovered" {
+ continue
+ }
+ if !slices.Contains(j.Recovered, box) {
+ j.Recovered = append(j.Recovered, box)
+ }
+ complete := true
+ for _, file := range j.Files {
+ complete = complete && slices.Contains(j.Recovered, file.Box)
+ }
+ if complete {
+ err = os.Remove(j.filename())
+ } else {
+ err = j.save()
+ }
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/kernel/model/document_source_journal_test.go b/kernel/model/document_source_journal_test.go
new file mode 100644
index 0000000000..5fbaf28843
--- /dev/null
+++ b/kernel/model/document_source_journal_test.go
@@ -0,0 +1,54 @@
+package model
+
+import (
+ "bytes"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/88250/lute/ast"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestDocumentSourceRecoveryPreservesConflictsAndCorruptRecords(t *testing.T) {
+ for _, corrupt := range []bool{false, true} {
+ t.Run(map[bool]string{false: "newer-source", true: "corrupt-record"}[corrupt], func(t *testing.T) {
+ fixture := setupFileOperationTest(t)
+ oldQueue := util.QueueDir
+ util.QueueDir = t.TempDir()
+ t.Cleanup(func() { util.QueueDir = oldQueue })
+ filename := filepath.Join(util.DataDir, fixture.box.ID, fixture.sourcePath)
+ original, err := os.ReadFile(filename)
+ if err != nil {
+ t.Fatal(err)
+ }
+ j := &documentSourceJournal{Version: 1, ID: ast.NewNodeID(), State: "pending", Files: []*documentSourceFile{
+ {Box: fixture.box.ID, Path: fixture.sourcePath, Before: original, After: []byte("expected replacement")},
+ }}
+ if err = j.save(); err != nil {
+ t.Fatal(err)
+ }
+ newer := append(append([]byte{}, original...), '\n')
+ if err = os.WriteFile(filename, newer, 0600); err != nil {
+ t.Fatal(err)
+ }
+ if corrupt {
+ if err = os.WriteFile(j.filename(), []byte(`{"data":{},"sha256":"corrupt"}`), 0600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ err = recoverDocumentSources()
+ if err == nil || (!corrupt && !errors.Is(err, util.ErrFileChanged)) {
+ t.Fatalf("unsafe recovery was not rejected: %v", err)
+ }
+ current, readErr := os.ReadFile(filename)
+ if readErr != nil || !bytes.Equal(current, newer) {
+ t.Fatal("unsafe recovery changed the source", readErr)
+ }
+ if _, statErr := os.Stat(j.filename()); statErr != nil {
+ t.Fatal("unsafe recovery discarded recovery evidence", statErr)
+ }
+ })
+ }
+}
diff --git a/kernel/model/index.go b/kernel/model/index.go
index ded97aa2c1..339ba362e7 100644
--- a/kernel/model/index.go
+++ b/kernel/model/index.go
@@ -307,6 +307,11 @@ func indexBox(boxID string) {
// 关联数据库和块
av.BatchUpsertBlockRel(avNodes)
+ if metadataReady {
+ if err := finishDocumentSourceRecovery(box.ID); err != nil {
+ logging.LogErrorf("complete document source recovery [%s] failed: %s", box.ID, err)
+ }
+ }
box.UpdateHistoryGenerated() // 初始化历史生成时间为当前时间
end := time.Now()
diff --git a/kernel/model/index_fix.go b/kernel/model/index_fix.go
index 9eba8145d2..fff28ae039 100644
--- a/kernel/model/index_fix.go
+++ b/kernel/model/index_fix.go
@@ -37,6 +37,7 @@ import (
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/av"
+ "github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
@@ -481,6 +482,10 @@ func saveIndexRepairHistory(tree *parse.Tree, original []byte) error {
}
func recreateTree(tree *parse.Tree, absPath string, original []byte) error {
+ return recreateTreeWithSteps(tree, absPath, original, nil)
+}
+
+func recreateTreeWithSteps(tree *parse.Tree, absPath string, original []byte, step func(string) error) (err error) {
FlushTxQueue()
flushLock.Lock()
defer flushLock.Unlock()
@@ -491,31 +496,114 @@ func recreateTree(tree *parse.Tree, absPath string, original []byte) error {
if !bytes.Equal(current, original) {
return fmt.Errorf("%w: %s", util.ErrFileChanged, absPath)
}
- // 删除关于该树的所有块树数据,后面会调用 fixBlockTreeByFileSys() 进行订正补全
- treenode.RemoveBlockTreesByPathPrefix(tree.Box, strings.TrimSuffix(tree.Path, ".sy"))
- treenode.RemoveBlockTreesByRootID(tree.Box, tree.ID)
-
- resetTree(tree, "", true)
- if _, err := filesys.WriteTree(tree); err != nil {
- logging.LogWarnf("write tree [%s] failed: %s", tree.Path, err)
+ prepared, snapshot, err := filesys.LoadTreeSnapshot(tree.Box, tree.Path, util.NewLute())
+ if err != nil {
return err
}
-
- if gulu.File.IsDir(strings.TrimSuffix(absPath, ".sy")) {
- // 重命名子文档文件夹
- from := strings.TrimSuffix(absPath, ".sy")
- to := filepath.Join(filepath.Dir(absPath), tree.ID)
- if renameErr := os.Rename(from, to); nil != renameErr {
- logging.LogWarnf("rename [%s] failed: %s", from, renameErr)
- return renameErr
+ if !bytes.Equal(snapshot, original) {
+ return util.ErrFileChanged
+ }
+ if err = saveIndexRepairHistory(tree, original); err != nil {
+ return err
+ }
+ oldPath := tree.Path
+ resetTreeData(prepared, "", true)
+ j := &documentSourceJournal{Version: 1, ID: ast.NewNodeID(), State: "pending", Files: []*documentSourceFile{
+ {Box: tree.Box, Path: oldPath, Before: original},
+ {Box: tree.Box, Path: prepared.Path},
+ }}
+ from := strings.TrimSuffix(absPath, ".sy")
+ to := filepath.Join(filepath.Dir(absPath), prepared.ID)
+ if info, statErr := os.Lstat(from); statErr == nil {
+ if !info.IsDir() {
+ return errors.New("invalid document child directory")
+ }
+ j.Move = &documentSourceMove{Box: tree.Box, From: strings.TrimSuffix(oldPath, ".sy"), To: strings.TrimSuffix(prepared.Path, ".sy")}
+ } else if !os.IsNotExist(statErr) {
+ return statErr
+ }
+ if _, statErr := os.Lstat(to); !os.IsNotExist(statErr) {
+ return errors.New("document repair destination directory already exists or is inaccessible")
+ }
+ committed, recorded := false, false
+ defer func() {
+ if !committed && recorded {
+ if rollbackErr := j.rollback(); rollbackErr != nil {
+ err = errors.Join(err, rollbackErr)
+ return
+ }
+ cache.RemoveTreeDataInBox(prepared.ID, prepared.Box)
+ cache.RemoveDocIALInBox(prepared.Path, prepared.Box)
+ treenode.RemoveBlockTreesByRootID(prepared.Box, prepared.ID)
+ if removeErr := os.Remove(j.filename()); removeErr != nil {
+ err = errors.Join(err, removeErr)
+ }
+ }
+ }()
+ checkpoint := func(stage string) error {
+ if step != nil {
+ return step(stage)
}
+ return nil
}
-
- if err := removeIndexRepairSourceIfUnchanged(absPath, original); err != nil {
- logging.LogWarnf("remove [%s] failed: %s", absPath, err)
+ err = filesys.CreateTreeWithJournal(prepared, func(data []byte) error {
+ j.Files[1].After = data
+ if saveErr := j.save(); saveErr != nil {
+ return saveErr
+ }
+ recorded = true
+ return checkpoint("prepared")
+ })
+ if err != nil {
return err
}
- return nil
+ if err = checkpoint("created"); err != nil {
+ return err
+ }
+ if j.Move != nil {
+ if err = os.Rename(from, to); err != nil {
+ return err
+ }
+ }
+ if err = checkpoint("children-moved"); err != nil {
+ return err
+ }
+ if err = removeIndexRepairSourceIfUnchanged(absPath, original); err != nil {
+ return err
+ }
+ if err = checkpoint("source-removed"); err != nil {
+ return err
+ }
+ j.State = "committed"
+ if err = j.save(); err != nil {
+ j.State = "pending"
+ return err
+ }
+ committed = true
+ *tree = *prepared
+ // Publish derived state only after the complete filesystem operation commits.
+ // A surviving committed record forces authenticated reindexing on the next mount.
+ sql.RemoveTreePathQueue(tree.Box, strings.TrimSuffix(oldPath, ".sy"))
+ treenode.RemoveBlockTreesByPathPrefix(tree.Box, strings.TrimSuffix(oldPath, ".sy"))
+ paths, err := duplicateDocTreePaths(tree.Box, tree.Path)
+ if err != nil {
+ return err
+ }
+ err = filesys.WalkTreeSnapshots(tree.Box, paths, func(current *parse.Tree) error {
+ boundary := ""
+ if IsEncryptedBox(current.Box) {
+ boundary = current.Box
+ }
+ if err := av.AddCopiedBlockRels(boundary, current.Root.ChildrenByType(ast.NodeAttributeView)); err != nil {
+ return err
+ }
+ sql.UpsertTreeQueue(current)
+ return treenode.ReplaceBlockTree(current)
+ })
+ if err == nil {
+ err = os.Remove(j.filename())
+ }
+ return err
}
func removeIndexRepairSourceIfUnchanged(absPath string, original []byte) error {
diff --git a/kernel/model/index_fix_recreate_fts5_test.go b/kernel/model/index_fix_recreate_fts5_test.go
new file mode 100644
index 0000000000..ce7576685a
--- /dev/null
+++ b/kernel/model/index_fix_recreate_fts5_test.go
@@ -0,0 +1,193 @@
+//go:build fts5
+
+package model
+
+import (
+ "bytes"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/88250/lute/ast"
+ "github.com/siyuan-note/siyuan/kernel/av"
+ "github.com/siyuan-note/siyuan/kernel/filesys"
+ "github.com/siyuan-note/siyuan/kernel/sql"
+ "github.com/siyuan-note/siyuan/kernel/treenode"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestIndexFixRootRecreationCompensatesEveryStage(t *testing.T) {
+ for _, failure := range []string{"prepared", "created", "children-moved", "source-removed", "success"} {
+ t.Run(failure, func(t *testing.T) {
+ fixture, view, _, _ := setupAttributeViewItemsTest(t, false)
+ isolateIndexRepairPaths(t)
+ source, err := LoadTreeByBlockID(fixture.sourceID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ node := &ast.Node{ID: ast.NewNodeID(), Type: ast.NodeAttributeView, AttributeViewID: view.ID, AttributeViewType: "table"}
+ node.SetIALAttr("id", node.ID)
+ source.Root.AppendChild(node)
+ if err = indexWriteTreeUpsertQueue(source); err != nil {
+ t.Fatal(err)
+ }
+ av.UpsertBlockRel(view.ID, node.ID)
+ child := addDocumentCollectionChild(t, source)
+ childBytes, err := os.ReadFile(filepath.Join(util.DataDir, child.Box, child.Path))
+ if err != nil {
+ t.Fatal(err)
+ }
+ oldID, oldPath := source.ID, source.Path
+ abs := filepath.Join(util.DataDir, source.Box, source.Path)
+ original, err := os.ReadFile(abs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ injected := errors.New("injected root recreation failure")
+ err = recreateTreeWithSteps(source, abs, original, func(stage string) error {
+ if stage == failure {
+ return injected
+ }
+ return nil
+ })
+ if failure != "success" {
+ if !errors.Is(err, injected) {
+ t.Fatalf("unexpected failure: %v", err)
+ }
+ if source.ID != oldID || source.Path != oldPath {
+ t.Fatal("failed recreation published new identity")
+ }
+ if got, readErr := os.ReadFile(abs); readErr != nil || !bytes.Equal(got, original) {
+ t.Fatal("failed recreation lost original source", readErr)
+ }
+ if got, readErr := os.ReadFile(filepath.Join(util.DataDir, child.Box, child.Path)); readErr != nil || !bytes.Equal(got, childBytes) {
+ t.Fatal("failed recreation lost original child directory", readErr)
+ }
+ if len(av.GetBlockRels()[view.ID]) != 1 {
+ t.Fatal("failed preparation published mirror relations")
+ }
+ } else {
+ if err != nil || source.ID == oldID {
+ t.Fatalf("root recreation failed: %v", err)
+ }
+ if _, statErr := os.Stat(abs); !os.IsNotExist(statErr) {
+ t.Fatal("committed recreation retained old source", statErr)
+ }
+ childPath := filepath.Join(filepath.Dir(source.Path), source.ID, child.ID+".sy")
+ if got, readErr := os.ReadFile(filepath.Join(util.DataDir, child.Box, childPath)); readErr != nil || !bytes.Equal(got, childBytes) {
+ t.Fatal("committed recreation changed child source", readErr)
+ }
+ sql.FlushQueue()
+ assertIndexRepairDocument(t, source)
+ if current := treenode.GetBlockTreeInExactBox(child.ID, child.Box); current == nil || filepath.Clean(current.Path) != filepath.Clean(childPath) {
+ t.Fatal("child index did not follow recreated parent")
+ }
+ }
+ journals, readErr := readDocumentSourceJournals()
+ if readErr != nil || len(journals) != 0 {
+ t.Fatalf("completed compensation retained pending recovery: %v, %v", journals, readErr)
+ }
+ })
+ }
+}
+
+func TestDocumentSourceRecoveryResumesInterruptedRecreation(t *testing.T) {
+ for _, stage := range []string{"created", "children-moved", "source-removed", "committed"} {
+ t.Run(stage, func(t *testing.T) {
+ fixture := setupStructureTransactionTest(t)
+ setupFoldTransactionDatabase(t, fixture)
+ isolateIndexRepairPaths(t)
+ source, original, err := filesys.LoadTreeSnapshot(fixture.box.ID, fixture.sourcePath, util.NewLute())
+ if err != nil {
+ t.Fatal(err)
+ }
+ child := addDocumentCollectionChild(t, source)
+ oldPath := source.Path
+ resetTreeData(source, "", true)
+ j := &documentSourceJournal{Version: 1, ID: ast.NewNodeID(), State: "pending", Files: []*documentSourceFile{
+ {Box: source.Box, Path: oldPath, Before: original}, {Box: source.Box, Path: source.Path},
+ }, Move: &documentSourceMove{Box: source.Box, From: oldPath[:len(oldPath)-3], To: source.Path[:len(source.Path)-3]}}
+ if err = filesys.CreateTreeWithJournal(source, func(data []byte) error { j.Files[1].After = data; return j.save() }); err != nil {
+ t.Fatal(err)
+ }
+ from, _ := documentSourcePath(j.Move.Box, j.Move.From, true)
+ to, _ := documentSourcePath(j.Move.Box, j.Move.To, true)
+ if stage != "created" {
+ if err = os.Rename(from, to); err != nil {
+ t.Fatal(err)
+ }
+ }
+ oldAbs := filepath.Join(util.DataDir, source.Box, oldPath)
+ if stage == "source-removed" || stage == "committed" {
+ if err = os.Remove(oldAbs); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if stage == "committed" {
+ j.State = "committed"
+ if err = j.save(); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err = recoverDocumentSources(); err != nil {
+ t.Fatal(err)
+ }
+ if !hasDocumentSourceRecovery(source.Box) {
+ t.Fatal("recovery did not retain the authenticated reindex requirement")
+ }
+ if stage != "committed" {
+ if got, readErr := os.ReadFile(oldAbs); readErr != nil || !bytes.Equal(got, original) {
+ t.Fatal("startup recovery lost original bytes", readErr)
+ }
+ if _, statErr := os.Stat(filepath.Join(from, child.ID+".sy")); statErr != nil {
+ t.Fatal("startup recovery lost child document", statErr)
+ }
+ if _, statErr := os.Stat(filepath.Join(util.DataDir, source.Box, source.Path)); !os.IsNotExist(statErr) {
+ t.Fatal("startup recovery left the uncommitted source", statErr)
+ }
+ } else if _, statErr := os.Stat(filepath.Join(to, child.ID+".sy")); statErr != nil {
+ t.Fatal("startup recovery undid a committed move", statErr)
+ }
+ // Mount-time indexing must finish before the recovery record can be discarded.
+ indexBox(source.Box)
+ sql.FlushQueue()
+ if hasDocumentSourceRecovery(source.Box) {
+ t.Fatal("authenticated indexing did not complete source recovery")
+ }
+ })
+ }
+}
+
+func TestIndexFixDuplicateRootPreservesCanonicalDocument(t *testing.T) {
+ fixture := setupStructureTransactionTest(t)
+ setupFoldTransactionDatabase(t, fixture)
+ isolateIndexRepairPaths(t)
+ source, original, err := filesys.LoadTreeSnapshot(fixture.box.ID, fixture.sourcePath, util.NewLute())
+ if err != nil {
+ t.Fatal(err)
+ }
+ duplicatePath := "/" + fixture.targetID + "/" + source.ID + ".sy"
+ duplicateAbs := filepath.Join(util.DataDir, source.Box, duplicatePath)
+ if err = os.MkdirAll(filepath.Dir(duplicateAbs), 0700); err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(duplicateAbs, original, 0600); err != nil {
+ t.Fatal(err)
+ }
+ if err = fixIndexPipeline(); err != nil {
+ t.Fatal(err)
+ }
+ canonical, _, err := filesys.ReadTreeSnapshot(source.Box, source.Path)
+ if err != nil || canonical.ID != source.ID {
+ t.Fatal("duplicate root repair changed canonical identity", err)
+ }
+ current, err := os.ReadFile(filepath.Join(util.DataDir, source.Box, source.Path))
+ if err != nil || !bytes.Equal(current, original) {
+ t.Fatal("duplicate root repair rewrote canonical source", err)
+ }
+ assertIndexRepairDocument(t, canonical)
+ if _, statErr := os.Stat(duplicateAbs); !os.IsNotExist(statErr) {
+ t.Fatal("duplicate root source was not replaced", statErr)
+ }
+}
diff --git a/kernel/model/index_fix_recreate_sqlcipher_test.go b/kernel/model/index_fix_recreate_sqlcipher_test.go
new file mode 100644
index 0000000000..c7344e63f1
--- /dev/null
+++ b/kernel/model/index_fix_recreate_sqlcipher_test.go
@@ -0,0 +1,66 @@
+//go:build fts5 && (sqlcipher || libsqlcipher)
+
+package model
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/siyuan-note/siyuan/kernel/filesys"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+func TestIndexFixEncryptedRootRecreation(t *testing.T) {
+ if !prepareStructuredTitlesCryptoTest(t) {
+ return
+ }
+ for _, failure := range []string{"created", "source-removed", "success"} {
+ t.Run(failure, func(t *testing.T) {
+ box := newStructuredTitlesEncryptedBox(t)
+ root := createStructuredTitlesEncryptedDoc(t, box, nil, "Root")
+ child := createStructuredTitlesEncryptedDoc(t, box, &root.ID, "Child")
+ tree, original, err := filesys.LoadTreeSnapshot(box, root.Path, util.NewLute())
+ if err != nil {
+ t.Fatal(err)
+ }
+ childData := structuredTitlesCiphertext(t, child)
+ injected := errors.New("injected encrypted root recreation failure")
+ err = recreateTreeWithSteps(tree, filepath.Join(util.DataDir, box, root.Path), original, func(stage string) error {
+ if stage == "created" {
+ journals, readErr := readDocumentSourceJournals()
+ if readErr != nil || len(journals) != 1 || !util.IsCiphertext(journals[0].Files[0].Before) || !util.IsCiphertext(journals[0].Files[1].After) {
+ t.Fatalf("encrypted recovery record did not retain ciphertext: %v", readErr)
+ }
+ }
+ if stage == failure {
+ return injected
+ }
+ return nil
+ })
+ if failure != "success" {
+ if !errors.Is(err, injected) {
+ t.Fatal(err)
+ }
+ assertEncryptedFileBytes(t, filepath.Join(util.DataDir, box, root.Path), original)
+ assertEncryptedFileBytes(t, filepath.Join(util.DataDir, box, child.Path), childData)
+ } else {
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, statErr := os.Stat(filepath.Join(util.DataDir, box, root.Path)); !os.IsNotExist(statErr) {
+ t.Fatal("encrypted recreation retained old source", statErr)
+ }
+ movedPath := "/" + tree.ID + "/" + child.ID + ".sy"
+ assertEncryptedFileBytes(t, filepath.Join(util.DataDir, box, movedPath), childData)
+ if _, _, err = filesys.ReadTreeSnapshot(box, tree.Path); err != nil {
+ t.Fatal("recreated ciphertext did not authenticate", err)
+ }
+ if _, _, err = filesys.ReadTreeSnapshot(box, movedPath); err != nil {
+ t.Fatal("moved child ciphertext did not authenticate", err)
+ }
+ }
+ })
+ }
+}
diff --git a/kernel/model/tree.go b/kernel/model/tree.go
index 8ec186159c..e18ac4f121 100644
--- a/kernel/model/tree.go
+++ b/kernel/model/tree.go
@@ -44,6 +44,17 @@ import (
)
func resetTree(tree *parse.Tree, titleSuffix string, removeAvBinding bool) {
+ resetTreeData(tree, titleSuffix, removeAvBinding)
+ ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
+ if entering && n.Type == ast.NodeAttributeView {
+ av.UpsertBlockRel(n.AttributeViewID, n.ID)
+ }
+ return ast.WalkContinue
+ })
+}
+
+// resetTreeData prepares identities without publishing database mirror side effects.
+func resetTreeData(tree *parse.Tree, titleSuffix string, removeAvBinding bool) {
oldRootID := tree.Root.ID
tree.ID = ast.NewNodeID()
tree.Root.ID = tree.ID
@@ -84,20 +95,6 @@ func resetTree(tree *parse.Tree, titleSuffix string, removeAvBinding bool) {
remapDuplicateDocTreeReferences(tree.Root, blockIDs)
- var attrViewIDs []string
- // 绑定镜像数据库
- ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
- if !entering {
- return ast.WalkContinue
- }
-
- if ast.NodeAttributeView == n.Type {
- av.UpsertBlockRel(n.AttributeViewID, n.ID)
- attrViewIDs = append(attrViewIDs, n.AttributeViewID)
- }
- return ast.WalkContinue
- })
-
if removeAvBinding {
// 清空文档绑定的数据库
tree.Root.RemoveIALAttr(av.NodeAttrNameAvs)