An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later
package model
import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"time"
"github.com/88250/lute/ast"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
)
type dataImportFile struct {
Path string `json:"path"`
Exists bool `json:"exists"`
Before notebookArchiveFile `json:"before"`
After notebookArchiveFile `json:"after"`
}
type dataImportOperation struct {
Version int `json:"version"`
State string `json:"state"`
Files []dataImportFile `json:"files"`
Dirs []string `json:"dirs"`
}
func dataImportFingerprint(p string) (notebookArchiveFile, error) {
file, err := os.Open(p)
if err != nil {
return notebookArchiveFile{}, err
}
defer file.Close()
return archiveFingerprint(file)
}
// copyDataImportFile 流式写入同目录临时文件,刷盘后替换,不把大资源整体载入内存。
func copyDataImportFile(source, target string) (err error) {
if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
return
}
input, err := os.Open(source)
if err != nil {
return
}
defer input.Close()
output, err := os.CreateTemp(filepath.Dir(target), ".data-import-*.tmp")
if err != nil {
return
}
defer func() {
output.Close()
os.Remove(output.Name())
}()
if _, err = io.Copy(output, input); err != nil {
return
}
if err = output.Sync(); err != nil {
return
}
if err = output.Close(); err != nil {
return
}
return os.Rename(output.Name(), target)
}
func dataImportTarget(name string) (string, error) {
if !archiveRelativePath(name) || strings.ToLower(name) == ".siyuan/master-password-migration.json" {
return "", errors.New(Conf.Language(422))
}
target := filepath.Join(util.DataDir, filepath.FromSlash(name))
if err := validateEncryptedCleanupPath(util.DataDir, target); err != nil {
return "", err
}
if info, err := os.Lstat(target); err == nil && !info.Mode().IsRegular() {
return "", errors.New(Conf.Language(422))
} else if err != nil && !os.IsNotExist(err) {
return "", err
}
return target, nil
}
// applyDataImport 先备份全部待覆盖文件,再登记提交日志;任何写入失败都补偿已安装的路径。
func applyDataImport(directory, install string, write func(string, string) error) error {
if err := checkDocumentAddressingCopy(install, ""); err != nil {
return err
}
files, err := archiveDirectoryFiles(install)
if err != nil {
return err
}
operation := &dataImportOperation{Version: 1, State: "committing", Files: []dataImportFile{}, Dirs: []string{}}
names := make([]string, 0, len(files))
for name := range files {
names = append(names, name)
}
sort.Strings(names)
dirs := map[string]bool{}
for _, name := range names {
target, targetErr := dataImportTarget(name)
if targetErr != nil {
return targetErr
}
entry := dataImportFile{Path: name, After: files[name]}
entry.Before, err = dataImportFingerprint(target)
if err == nil {
entry.Exists = true
if entry.Before == entry.After {
continue
}
backup := filepath.Join(directory, "rollback", filepath.FromSlash(name))
if err = copyDataImportFile(target, backup); err != nil {
return err
}
if actual, readErr := dataImportFingerprint(backup); readErr != nil || actual != entry.Before {
return errors.New(Conf.Language(422))
}
} else if !os.IsNotExist(err) {
return err
}
for parent := filepath.Dir(name); parent != "."; parent = filepath.Dir(parent) {
if _, statErr := os.Lstat(filepath.Join(util.DataDir, parent)); os.IsNotExist(statErr) {
dirs[filepath.ToSlash(parent)] = true
} else if statErr != nil {
return statErr
}
}
operation.Files = append(operation.Files, entry)
}
for directory := range dirs {
operation.Dirs = append(operation.Dirs, directory)
}
sort.Strings(operation.Dirs)
if err = writeDataImportJSON(filepath.Join(directory, "operation.json"), operation); err != nil {
return err
}
dataImportRecoveryRequired.Store(true)
for _, entry := range operation.Files {
target, targetErr := dataImportTarget(entry.Path)
if targetErr == nil {
actual, readErr := dataImportFingerprint(target)
if entry.Exists && (readErr != nil || actual != entry.Before) || !entry.Exists && !os.IsNotExist(readErr) {
targetErr = errors.New(Conf.Language(422))
}
}
if targetErr == nil {
targetErr = write(filepath.Join(install, filepath.FromSlash(entry.Path)), target)
}
if targetErr != nil {
return abortDataImport(directory, operation, targetErr)
}
}
operation.State = "committed"
if err = writeDataImportJSON(filepath.Join(directory, "operation.json"), operation); err != nil {
operation.State = "committing"
return abortDataImport(directory, operation, err)
}
dataImportRecoveryRequired.Store(false)
return nil
}
func abortDataImport(directory string, operation *dataImportOperation, cause error) error {
err := rollbackDataImport(directory, operation)
dataImportRecoveryRequired.Store(err != nil)
return errors.Join(cause, err)
}
func rollbackDataImport(directory string, operation *dataImportOperation) error {
for _, entry := range slices.Backward(operation.Files) {
target, err := dataImportTarget(entry.Path)
if err != nil {
return err
}
actual, readErr := dataImportFingerprint(target)
if entry.Exists && readErr == nil && actual == entry.Before || !entry.Exists && os.IsNotExist(readErr) {
continue
}
if readErr != nil && !os.IsNotExist(readErr) || readErr == nil && actual != entry.After {
return errors.New(Conf.Language(424))
}
if entry.Exists {
backup := filepath.Join(directory, "rollback", filepath.FromSlash(entry.Path))
if original, backupErr := dataImportFingerprint(backup); backupErr != nil || original != entry.Before {
return errors.New(Conf.Language(424))
}
if err = copyDataImportFile(backup, target); err != nil {
return err
}
} else if err = os.Remove(target); err != nil && !os.IsNotExist(err) {
return err
}
}
for _, name := range slices.Backward(operation.Dirs) {
if !archiveRelativePath(name) {
return errors.New(Conf.Language(424))
}
target := filepath.Join(util.DataDir, filepath.FromSlash(name))
if err := validateEncryptedCleanupPath(util.DataDir, target); err != nil {
return err
}
// 仅移除本次创建的空目录,不递归删除后来出现的文件。
if entries, err := os.ReadDir(target); err == nil && len(entries) == 0 {
if err = os.Remove(target); err != nil {
return err
}
} else if err != nil && !os.IsNotExist(err) {
return err
}
}
operation.State = "rolledback"
if err := writeDataImportJSON(filepath.Join(directory, "operation.json"), operation); err != nil {
return err
}
return nil
}
// recoverDataImports 在挂载和同步前补偿未完成的批次,成功批次的原件和恢复材料继续保留。
func recoverDataImports() (err error) {
defer func() { dataImportRecoveryRequired.Store(err != nil) }()
entries, err := os.ReadDir(dataImportBase())
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
for _, entry := range entries {
if !entry.IsDir() || !ast.IsNodeIDPattern(entry.Name()) {
return errors.New(Conf.Language(424))
}
directory, pathErr := dataImportDirectory(entry.Name())
if pathErr != nil {
return pathErr
}
data, readErr := os.ReadFile(filepath.Join(directory, "operation.json"))
if os.IsNotExist(readErr) {
continue
}
operation := &dataImportOperation{}
if readErr != nil || json.Unmarshal(data, operation) != nil || operation.Version != 1 ||
!slices.Contains([]string{"committing", "committed", "rolledback"}, operation.State) {
return errors.New(Conf.Language(424))
}
seen := map[string]bool{}
for _, file := range operation.Files {
if !archiveRelativePath(file.Path) || seen[strings.ToLower(file.Path)] || len(file.After.SHA256) != 64 ||
file.Exists && len(file.Before.SHA256) != 64 {
return errors.New(Conf.Language(424))
}
seen[strings.ToLower(file.Path)] = true
}
if operation.State == "committing" {
dataImportRecoveryRequired.Store(true)
if err = rollbackDataImport(directory, operation); err != nil {
return fmt.Errorf("%s: %w", Conf.Language(424), err)
}
}
}
if cleanupErr := cleanupExpiredDataImports(); cleanupErr != nil {
logging.LogWarnf("clean up expired Data.zip preparations failed: %s", cleanupErr)
}
return nil
}
// recoverDataImportsForPrepare 在独占写准入、同步和密钥锁下恢复,预检自身不持共享锁执行补偿写入。
func recoverDataImportsForPrepare() error {
dataImportAccess.Lock()
defer dataImportAccess.Unlock()
dataImportMu.Lock()
defer dataImportMu.Unlock()
lockSync()
defer unlockSync()
notebookCryptoMu.Lock()
defer notebookCryptoMu.Unlock()
return recoverDataImports()
}
func cleanupExpiredDataImports() error {
entries, err := os.ReadDir(dataImportBase())
if os.IsNotExist(err) {
return nil
}
if err != nil {
return err
}
for _, entry := range entries {
if !entry.IsDir() || !ast.IsNodeIDPattern(entry.Name()) {
continue
}
directory, stage, readErr := readDataImportStage(entry.Name())
if readErr != nil || time.Now().Unix()-stage.Created <= int64(dataImportTTL.Seconds()) {
continue
}
if err = os.RemoveAll(directory); err != nil {
return err
}
}
return nil
}