An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later
package model
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"encoding/base64"
"errors"
"fmt"
"math"
mathRand "math/rand"
"mime"
"net/url"
"os"
"path"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/88250/go-humanize"
"github.com/88250/gulu"
"github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/88250/lute/html"
"github.com/88250/lute/parse"
"github.com/88250/lute/render"
"github.com/emirpasic/gods/sets/hashset"
"github.com/siyuan-note/dataparser"
"github.com/siyuan-note/dejavu"
"github.com/siyuan-note/dejavu/cloud"
"github.com/siyuan-note/dejavu/entity"
"github.com/siyuan-note/encryption"
"github.com/siyuan-note/eventbus"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/av"
"github.com/siyuan-note/siyuan/kernel/bazaar"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/heif"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/tiffpreview"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
"github.com/studio-b12/gowebdav"
)
// AutoPurgeRepoJob 自动清理数据仓库 https://github.com/siyuan-note/siyuan/issues/13091
func AutoPurgeRepoJob() {
task.AppendTaskWithTimeout(task.RepoAutoPurge, 12*time.Hour, autoPurgeRepo, true)
}
var (
autoPurgeRepoAfterFirstSync = false
lastAutoPurgeRepo = time.Time{}
purgeCancelMu sync.Mutex
purgeCancel context.CancelFunc
)
func autoPurgeRepo(cron bool) {
confRepoReadSnapshot := Conf.LoadRepo()
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
if cron && !autoPurgeRepoAfterFirstSync {
return
}
if time.Since(lastAutoPurgeRepo) < 6*time.Hour {
return
}
autoPurgeRepoAfterFirstSync = true
defer func() {
lastAutoPurgeRepo = time.Now()
}()
if 1 > len(confRepoReadSnapshot.Key) {
return
}
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
return
}
now := time.Now()
dateGroupedIndexes := map[string][]*entity.Index{} // 按照日期分组
// 收集指定日期内需要保留的索引
var date string
page := 1
for {
indexes, pageCount, _, err := repo.GetIndexes(page, 512)
if nil != err {
logging.LogErrorf("get data repo index logs failed: %s", err)
return
}
if 1 > len(indexes) {
break
}
tooOld := false
for _, index := range indexes {
if now.UnixMilli()-index.Created <= int64(confRepoReadSnapshot.IndexRetentionDays)*24*60*60*1000 {
date = time.UnixMilli(index.Created).Format("2006-01-02")
if _, ok := dateGroupedIndexes[date]; !ok {
dateGroupedIndexes[date] = []*entity.Index{}
}
dateGroupedIndexes[date] = append(dateGroupedIndexes[date], index)
} else {
tooOld = true
break
}
}
if tooOld {
break
}
page++
if page > pageCount {
break
}
}
todayDate := now.Format("2006-01-02")
// 过滤出每日需要保留的索引
var retentionIndexIDs []string
for date, indexes := range dateGroupedIndexes {
if len(indexes) <= confRepoReadSnapshot.RetentionIndexesDaily || todayDate == date {
for _, index := range indexes {
retentionIndexIDs = append(retentionIndexIDs, index.ID)
}
continue
}
keepIndexes := hashset.New()
keepIndexes.Add(indexes[0]) // 每天最后一个固定保留
// 随机保留指定数量的索引
for i := 0; i < confRepoReadSnapshot.RetentionIndexesDaily*7; i++ {
keepIndexes.Add(indexes[mathRand.Intn(len(indexes)-1)])
if keepIndexes.Size() >= confRepoReadSnapshot.RetentionIndexesDaily {
break
}
}
for _, keepIndex := range keepIndexes.Values() {
retentionIndexIDs = append(retentionIndexIDs, keepIndex.(*entity.Index).ID)
}
}
retentionIndexIDs = gulu.Str.RemoveDuplicatedElem(retentionIndexIDs)
if 3 > len(retentionIndexIDs) {
logging.LogInfof("no index to purge [ellapsed=%.2fs]", time.Since(now).Seconds())
return
}
purgeCancelMu.Lock()
var ctx context.Context
ctx, purgeCancel = context.WithCancel(context.Background())
cancelCtx := ctx
purgeCancelMu.Unlock()
defer func() {
purgeCancelMu.Lock()
if nil != purgeCancel {
purgeCancel()
purgeCancel = nil
}
purgeCancelMu.Unlock()
}()
_, err = repo.Purge(cancelCtx, retentionIndexIDs...)
}
func cancelPurge() {
purgeCancelMu.Lock()
defer purgeCancelMu.Unlock()
if nil != purgeCancel {
purgeCancel()
purgeCancel = nil
}
}
func GetRepoFile(fileID string) (ret []byte, p string, err error) {
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
ret, file, err := readRepoFileWithAssets(fileID)
if err != nil {
return
}
ret, err = decryptRepoDataIfNeeded(ret, file.Path)
p = file.Path
return
}
// ResolveRepoFileBoxID 返回仓库文件路径中明确记录的加密笔记本 ID。
func ResolveRepoFileBoxID(fileID string) (boxID string, err error) {
if 1 > len(Conf.Repo.Load().Key) {
return "", errors.New(Conf.Language(26))
}
repo, err := newRepository()
if err != nil {
return "", err
}
file, err := repo.GetFile(fileID)
if err != nil {
return "", err
}
return encryptedBoxIDFromRepoPath(file.Path), nil
}
func encryptedBoxIDFromRepoPath(repoPath string) string {
repoPath = strings.TrimPrefix(filepath.ToSlash(repoPath), "/")
parts := strings.SplitN(repoPath, "/", 2)
if len(parts) == 2 && ast.IsNodeIDPattern(parts[0]) && IsEncryptedBox(parts[0]) {
return parts[0]
}
return ""
}
func RollbackRepoSnapshotFile(fileID string) (err error) {
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
data, file, err := readRepoFileWithAssets(fileID)
if err != nil {
return
}
dir, f := filepath.Split(file.Path)
tempRepoDiffDir := filepath.Join(util.TempDir, "repo", "rollback", dir)
if err = os.MkdirAll(tempRepoDiffDir, 0755); nil != err {
logging.LogErrorf("mkdir [%s] failed: %v", tempRepoDiffDir, err)
return
}
// 回滚快照时默认为当前数据创建一个快照
// When rolling back a snapshot, a snapshot is created for the current data by default https://github.com/siyuan-note/siyuan/issues/12470
FlushTxQueue()
_, _, err = CreateRepoSnapshotWithFullCheck("Backup before checkout", true)
if err != nil {
logging.LogErrorf("index repository failed: %s", err)
util.PushClearProgress()
util.PushErrMsg(fmt.Sprintf(Conf.Language(140), err), 0)
return
}
util.PushClearProgress()
from := filepath.Join(tempRepoDiffDir, f)
// 加密笔记本的快照数据是密文,写入临时文件前先解密
if strings.HasSuffix(file.Path, ".sy") {
data, err = decryptRepoDataIfNeeded(data, file.Path)
if err != nil {
return
}
}
if err = os.WriteFile(from, data, 0644); nil != err {
logging.LogErrorf("write file [%s] failed: %v", filepath.Join(tempRepoDiffDir, file.Path), err)
return
}
// 解密后的临时文件在函数返回时清理,避免加密文档明文残留在磁盘
defer os.Remove(from)
if strings.HasSuffix(file.Path, ".sy") {
boxID := strings.TrimPrefix(file.Path, "/")
flushLock.Lock()
defer flushLock.Unlock()
boxID = strings.Split(boxID, "/")[0]
origBoxID := boxID // 保留原始 boxID 用于加密边界校验
// 加密笔记本的快照回滚要求原笔记本已挂载:
// WriteTree 根据 tree.Box 判断是否加密落盘。若原笔记本未挂载导致
// getRollbackBox fallback 到普通 Rollback 笔记本,解密后的 .sy 将被 WriteTree
// 以明文落盘,违反加密笔记本"数据不跨边界"的约束。
if IsEncryptedBox(origBoxID) && nil == Conf.Box(origBoxID) {
logging.LogErrorf("rollback encrypted repo snapshot requires notebook [%s] to be mounted", origBoxID)
err = errors.New(Conf.Language(314))
return
}
var box *Box
var needResetTree bool
box, needResetTree, err = getRollbackBox(boxID)
if err != nil {
logging.LogErrorf("get rollback box [%s] failed: %s", boxID, err)
return
}
boxID = box.ID
var destPath, parentHPath string
rootID := util.GetTreeID(file.Path)
workingDoc := treenode.GetBlockTree(rootID)
if needResetTree {
workingDoc = nil
}
destPath, parentHPath, err = getRollbackDockPath(boxID, file.Path, workingDoc)
if err != nil {
return
}
tree, _ := loadTree(from, util.NewLute())
if nil == tree {
msg := fmt.Sprintf("no such file or directory: %s", from)
logging.LogError(msg)
err = errors.New(msg)
return
}
tree.Box = boxID
tree.Path = filepath.ToSlash(strings.TrimPrefix(destPath, util.DataDir+string(os.PathSeparator)+boxID))
tree.HPath = parentHPath + "/" + treenode.ReadDocumentTitle(tree.Root).Text
if needResetTree {
resetTree(tree, "", true)
}
// Publish the restored source before removing the working document or
// its derived indexes. A failed write must leave the current version intact.
if _, err = filesys.WriteTree(tree); err != nil {
return
}
invalidateDocumentHistory(tree.ID)
if nil != workingDoc && "d" == workingDoc.Type && workingDoc.Path != tree.Path {
workingDocPath := filepath.Join(util.DataDir, boxID, workingDoc.Path)
if err = filelock.Remove(workingDocPath); err != nil {
return
}
logging.LogInfof("removed working doc file [%s]", workingDocPath)
}
if nil != workingDoc {
treenode.RemoveBlockTreesByRootID(boxID, rootID)
}
sql.RemoveTreeQueue(boxID, rootID)
treenode.IndexBlockTree(tree)
sql.IndexTreeQueue(tree)
ReloadFiletree()
ReloadProtyle(rootID)
msg := fmt.Sprintf(Conf.Language(286), box.Name+tree.HPath)
util.PushMsg(msg, 7000)
} else {
to := filepath.Join(util.DataDir, file.Path)
if err = filelock.CopyNewtimes(from, to); nil != err {
logging.LogErrorf("copy file [%s] to [%s] failed: %s", from, to, err)
return
}
if strings.Contains(file.Path, "/storage/av/") && strings.HasSuffix(file.Path, ".json") {
avID := strings.TrimSuffix(filepath.Base(file.Path), ".json")
cache.RemoveAVData(avID)
queueExternalAttributeViewRefIndexByRepoPath(file.Path)
ReloadAttrView(avID)
}
msg := fmt.Sprintf(Conf.Language(286), to)
util.PushMsg(msg, 7000)
}
IncSync()
return
}
func OpenRepoSnapshotFile(fileID string) (title, content string, displayInText bool, updated int64, err error) {
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
data, file, err := readRepoFileWithAssets(fileID)
if err != nil {
return
}
updated = file.Updated
repoPath := strings.TrimPrefix(file.Path, "/")
repoPathParts := strings.SplitN(repoPath, "/", 2)
payloadBoxID := ""
if len(repoPathParts) == 2 && ast.IsNodeIDPattern(repoPathParts[0]) {
payloadBoxID = repoPathParts[0]
}
if (util.IsCiphertext(data) || bytes.HasPrefix(data, encryptedAssetMagic)) &&
(payloadBoxID == "" || !IsEncryptedBox(payloadBoxID)) {
err = errors.New("encrypted repository data is missing valid notebook context")
return
}
if strings.HasSuffix(file.Path, ".sy") {
// 加密笔记本的 .sy 在仓库里是密文,按路径提取 boxID 解密
data, err = decryptRepoDataIfNeeded(data, file.Path)
if err != nil {
return
}
luteEngine := NewLute()
var snapshotTree *parse.Tree
displayInText, snapshotTree, err = parseTreeInSnapshot(data, luteEngine)
if err != nil {
logging.LogErrorf("parse tree from snapshot file [%s] failed", fileID)
return
}
title = treenode.ReadDocumentTitle(snapshotTree.Root).Text
if !displayInText {
renderTree := &parse.Tree{Root: &ast.Node{Type: ast.NodeDocument}}
ast.Walk(snapshotTree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
n.RemoveIALAttr("heading-fold")
n.RemoveIALAttr("fold")
return ast.WalkContinue
})
var appends []*ast.Node
for n := snapshotTree.Root.FirstChild; nil != n; n = n.Next {
appends = append(appends, n)
}
for _, n := range appends {
renderTree.Root.AppendChild(n)
}
snapshotTree = renderTree
}
luteEngine.RenderOptions.ProtyleContenteditable = false
if displayInText {
util.PushMsg(Conf.Language(36), 5000)
formatRenderer := render.NewFormatRenderer(snapshotTree, luteEngine.RenderOptions, luteEngine.ParseOptions)
content = gulu.Str.FromBytes(formatRenderer.Render())
} else {
content = luteEngine.Tree2BlockDOM(snapshotTree, luteEngine.RenderOptions, luteEngine.ParseOptions)
}
} else {
displayInText = true
title = file.Path
// 加密 notebook 的 AV 定义在仓库里是密文,需先解密再展示
if strings.Contains(file.Path, "storage/av/") && strings.HasSuffix(file.Path, ".json") {
repoBoxID := ""
origPath := strings.TrimPrefix(file.Path, "/")
if parts := strings.SplitN(origPath, "/", 2); len(parts) >= 1 && ast.IsNodeIDPattern(parts[0]) {
repoBoxID = parts[0]
}
if repoBoxID != "" && IsEncryptedBox(repoBoxID) {
HoldBoxReadLock(repoBoxID)
defer ReleaseBoxReadLock(repoBoxID)
if dek, dekErr := GetDEKIfUnlocked(repoBoxID); dekErr == nil && dek != nil {
avID := strings.TrimSuffix(filepath.Base(file.Path), ".json")
if plainData, decErr := av.DecryptAVDataLocked(repoBoxID, avID, data); decErr == nil {
data = plainData
} else {
logging.LogWarnf("decrypt repo snapshot AV [%s] failed: %s", file.Path, decErr)
err = decErr
return
}
} else {
err = errors.New(Conf.Language(314))
return
}
}
}
if mimeType := mime.TypeByExtension(filepath.Ext(file.Path)); strings.HasPrefix(mimeType, "text/") || strings.Contains(mimeType, "json") {
// 如果是文本文件,直接返回文本内容
// All plain text formats are supported when comparing data snapshots https://github.com/siyuan-note/siyuan/issues/12975
content = gulu.Str.FromBytes(data)
} else {
if strings.Contains(file.Path, "assets/") { // 剔除笔记本级或者文档级资源文件路径前缀
// 先按原始路径确定加密笔记本身份并认证,再整理资源路径用于预览。
repoBoxID := ""
origPath := strings.TrimPrefix(file.Path, "/")
if parts := strings.SplitN(origPath, "/", 2); len(parts) >= 1 && ast.IsNodeIDPattern(parts[0]) {
repoBoxID = parts[0]
}
if repoBoxID != "" && IsEncryptedBox(repoBoxID) {
// 加密仓库快照的 HEIF 预览不落盘,保持与旧版普通文件展示行为一致。
if heif.IsPath(file.Path) {
return
}
HoldBoxReadLock(repoBoxID)
defer ReleaseBoxReadLock(repoBoxID)
// 加密 asset:尝试解密后预览,无法解密则 fail-closed
if dek, dekErr := GetDEKIfUnlocked(repoBoxID); dekErr == nil && dek != nil {
diskName := filepath.Base(file.Path)
plainData, decErr := DecryptAsset(repoBoxID, diskName, dek, data)
clear(dek)
if decErr == nil {
if tiffpreview.IsPath(file.Path) {
// TIFF 快照经认证后仍暂存原密文,HTTP 预览和下载在请求内存中解密。
clear(plainData)
} else {
data = plainData
}
} else {
logging.LogWarnf("decrypt repo snapshot asset [%s] failed: %s", file.Path, decErr)
err = decErr
return
}
} else {
err = errors.New(Conf.Language(314))
return
}
}
// 保留 boxID 前缀,确保 LockBox 清理和 serveRepoDiff 加密校验能命中
file.Path = path.Join(repoBoxID, file.Path[strings.Index(file.Path, "assets/"):])
if util.IsDisplayableAsset(file.Path) {
dir, f := filepath.Split(file.Path)
tempRepoDiffDir := filepath.Join(util.TempDir, "repo", "diff", dir)
if mkErr := os.MkdirAll(tempRepoDiffDir, 0755); nil != mkErr {
logging.LogErrorf("mkdir [%s] failed: %v", tempRepoDiffDir, mkErr)
} else {
if wrErr := os.WriteFile(filepath.Join(tempRepoDiffDir, f), data, 0644); nil != wrErr {
logging.LogErrorf("write file [%s] failed: %v", filepath.Join(tempRepoDiffDir, file.Path), wrErr)
}
}
content = path.Join("repo", "diff", file.Path)
}
} else {
content = file.Path
}
}
}
return
}
type LeftRightDiff struct {
LeftIndex *DiffIndex `json:"leftIndex"`
RightIndex *DiffIndex `json:"rightIndex"`
AddsLeft []*DiffFile `json:"addsLeft"`
UpdatesLeft []*DiffFile `json:"updatesLeft"`
UpdatesRight []*DiffFile `json:"updatesRight"`
RemovesRight []*DiffFile `json:"removesRight"`
}
type DiffFile struct {
FileID string `json:"fileID"`
IndexID string `json:"indexID"`
Title string `json:"title"`
Path string `json:"path"`
HPath string `json:"hPath,omitempty"`
HSize string `json:"hSize"`
Updated int64 `json:"updated"`
}
type RepoDocHistory struct {
FileID string `json:"fileID"`
IndexID string `json:"indexID"`
Title string `json:"title"`
HSize string `json:"hSize"`
Updated int64 `json:"updated"`
Snapshots []*DocHistorySnapshot `json:"snapshots"`
}
type DiffIndex struct {
ID string `json:"id"`
Created int64 `json:"created"`
}
func DiffRepoSnapshots(left, right string) (ret *LeftRightDiff, err error) {
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
repo, err := newRepository()
if err != nil {
return
}
diff, err := repo.DiffIndex(left, right)
if err != nil {
return
}
ret = &LeftRightDiff{
LeftIndex: &DiffIndex{
ID: diff.LeftIndex.ID,
Created: diff.LeftIndex.Created,
},
RightIndex: &DiffIndex{
ID: diff.RightIndex.ID,
Created: diff.RightIndex.Created,
},
}
luteEngine := NewLute()
for _, removeRight := range diff.RemovesRight {
title, _ := parseTitleInSnapshotForListing(removeRight, repo, luteEngine)
ret.AddsLeft = append(ret.AddsLeft, &DiffFile{
FileID: removeRight.ID,
Title: title,
Path: removeRight.Path,
HSize: humanize.BytesCustomCeil(uint64(removeRight.Size), 2),
Updated: removeRight.Updated,
})
}
if 1 > len(ret.AddsLeft) {
ret.AddsLeft = []*DiffFile{}
}
for _, addLeft := range diff.AddsLeft {
title, _ := parseTitleInSnapshotForListing(addLeft, repo, luteEngine)
ret.RemovesRight = append(ret.RemovesRight, &DiffFile{
FileID: addLeft.ID,
Title: title,
Path: addLeft.Path,
HSize: humanize.BytesCustomCeil(uint64(addLeft.Size), 2),
Updated: addLeft.Updated,
})
}
if 1 > len(ret.RemovesRight) {
ret.RemovesRight = []*DiffFile{}
}
for _, updateLeft := range diff.UpdatesLeft {
title, _ := parseTitleInSnapshotForListing(updateLeft, repo, luteEngine)
ret.UpdatesLeft = append(ret.UpdatesLeft, &DiffFile{
FileID: updateLeft.ID,
Title: title,
Path: updateLeft.Path,
HSize: humanize.BytesCustomCeil(uint64(updateLeft.Size), 2),
Updated: updateLeft.Updated,
})
}
if 1 > len(ret.UpdatesLeft) {
ret.UpdatesLeft = []*DiffFile{}
}
for _, updateRight := range diff.UpdatesRight {
title, _ := parseTitleInSnapshotForListing(updateRight, repo, luteEngine)
ret.UpdatesRight = append(ret.UpdatesRight, &DiffFile{
FileID: updateRight.ID,
Title: title,
Path: updateRight.Path,
HSize: humanize.BytesCustomCeil(uint64(updateRight.Size), 2),
Updated: updateRight.Updated,
})
}
if 1 > len(ret.UpdatesRight) {
ret.UpdatesRight = []*DiffFile{}
}
return
}
func parseTitleInSnapshotForListing(file *entity.File, repo *dejavu.Repo, luteEngine *lute.Lute) (title, rootID string) {
if file == nil {
return
}
if encryptedBoxIDFromRepoPath(file.Path) != "" {
return path.Base(file.Path), ""
}
title, rootID, err := parseTitleInSnapshot(file.ID, repo, luteEngine)
if err != nil || title == "" {
return path.Base(file.Path), ""
}
return
}
func parseTitleInSnapshot(fileID string, repo *dejavu.Repo, luteEngine *lute.Lute) (title, rootID string, err error) {
file, err := repo.GetFile(fileID)
if err != nil {
logging.LogErrorf("get file [%s] failed: %s", fileID, err)
return
}
title = path.Base(file.Path)
if strings.HasSuffix(file.Path, ".sy") {
var data []byte
data, err = repo.OpenFile(file)
if err != nil {
logging.LogErrorf("open file [%s] failed: %s", fileID, err)
return
}
// 加密笔记本的 .sy 在仓库里是密文,按路径提取 boxID 解密
data, err = decryptRepoDataIfNeeded(data, file.Path)
if err != nil {
return
}
var tree *parse.Tree
if err = treenode.CheckSpecJSON(data); nil != err {
return
}
tree, err = dataparser.ParseJSONWithoutFix(data, luteEngine.ParseOptions)
if err != nil {
logging.LogErrorf("parse file [%s] failed: %s", fileID, err)
return
}
title = treenode.ReadDocumentTitle(tree.Root).Text
rootID = tree.Root.ID
}
return
}
// decryptRepoDataIfNeeded 判断仓库数据是否属于加密笔记本,如果是则按路径类型分流解密。
// file.Path 格式:/<boxID>/...
// .sy → DecryptFile,assets/* → DecryptAsset,storage/av/*.json → av.DecryptAVData。
// 密文缺少有效路径上下文、笔记本未解锁或认证失败时返回错误,不允许调用方按明文继续处理。
func decryptRepoDataIfNeeded(data []byte, filePath string) ([]byte, error) {
relPath := strings.TrimPrefix(filePath, "/")
parts := strings.SplitN(relPath, "/", 2)
encryptedPayload := util.IsCiphertext(data) || bytes.HasPrefix(data, encryptedAssetMagic)
if len(parts) < 2 || !ast.IsNodeIDPattern(parts[0]) {
if encryptedPayload {
return nil, errors.New("encrypted repository data is missing notebook context")
}
return data, nil
}
boxID := parts[0]
if !IsEncryptedBox(boxID) {
if encryptedPayload {
return nil, fmt.Errorf("encrypted repository data has no matching notebook [%s]", boxID)
}
return data, nil
}
// 持读锁,防止 LockBox 在解密期间清 DEK/缓存
HoldBoxReadLock(boxID)
defer ReleaseBoxReadLock(boxID)
dek, err := GetDEKIfUnlocked(boxID)
if err != nil {
return nil, errors.New(Conf.Language(314))
}
boxRelPath := parts[1]
// 按路径类型分流
if strings.HasPrefix(boxRelPath, "assets/") {
diskName := filepath.Base(boxRelPath)
plain, decErr := DecryptAsset(boxID, diskName, dek, data)
if decErr != nil {
return nil, decErr
}
return plain, nil
}
if strings.HasPrefix(boxRelPath, "storage/av/") && strings.HasSuffix(boxRelPath, ".json") {
avID := strings.TrimSuffix(filepath.Base(boxRelPath), ".json")
plain, decErr := av.DecryptAVDataLocked(boxID, avID, data)
if decErr != nil {
return nil, decErr
}
return plain, nil
}
// .sy 和其他文件用 file 子密钥 + 相对路径 AAD
plain, decErr := DecryptFile(boxID, boxRelPath, dek, data)
if decErr != nil {
return nil, decErr
}
return plain, nil
}
func parseTreeInSnapshot(data []byte, luteEngine *lute.Lute) (isLargeDoc bool, tree *parse.Tree, err error) {
isLargeDoc = 1024*1024*1 <= len(data)
// 调用方必须先根据快照路径完成解密,解析层不处理密文。
if err = treenode.CheckSpecJSON(data); nil != err {
return
}
tree, err = dataparser.ParseJSONWithoutFix(data, luteEngine.ParseOptions)
if err != nil {
return
}
err = treenode.RefreshTableCellRichProjection(tree.Root)
return
}
func SearchRepoFile(keyword string, page int) (ret []*DiffFile, pageCount, totalCount int, err error) {
ret = []*DiffFile{}
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
repo, err := newRepository()
if err != nil {
return
}
files, fileIndexIDs, totalCount, pageCount, err := repo.SearchFile(keyword, page, 32)
if err != nil {
logging.LogErrorf("search repo file failed: %s", err)
return
}
if 1 > len(files) {
return
}
luteEngine := NewLute()
for _, file := range files {
title, rootID := parseTitleInSnapshotForListing(file, repo, luteEngine)
var hpath string
if "" != rootID && treenode.ExistBlockTree(rootID) {
hpath, _ = GetHPathByID(rootID)
} else {
hpath = file.Path
}
ret = append(ret, &DiffFile{
FileID: file.ID,
IndexID: fileIndexIDs[file.ID],
Title: title,
Path: file.Path,
HPath: hpath,
HSize: humanize.BytesCustomCeil(uint64(file.Size), 2),
Updated: file.Updated,
})
}
return
}
func GetRepoDocHistory(id string, page int) (ret []*RepoDocHistory, pageCount, totalCount int, err error) {
ret = []*RepoDocHistory{}
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
repo, err := newRepository()
if err != nil {
return
}
files, fileIndexIDs, totalCount, pageCount, err := repo.SearchFileByName(id+".sy", page, 32)
if err != nil {
logging.LogErrorf("get repo doc history failed: %s", err)
return
}
luteEngine := NewLute()
for _, file := range files {
title, _ := parseTitleInSnapshotForListing(file, repo, luteEngine)
ret = append(ret, &RepoDocHistory{
FileID: file.ID,
IndexID: fileIndexIDs[file.ID],
Title: title,
HSize: humanize.BytesCustomCeil(uint64(file.Size), 2),
Updated: file.Updated,
})
}
err = attachRepoDocHistorySnapshots(repo, ret)
return
}
func ExportRepoFile(id string) (exportPath string, err error) {
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
data, file, err := readRepoFileWithAssets(id)
if err != nil {
return
}
encryptedBoxID := encryptedBoxIDFromRepoPath(file.Path)
// 加密笔记本的 .sy 在仓库里是密文,按路径提取 boxID 解密
data, err = decryptRepoDataIfNeeded(data, file.Path)
if err != nil {
return
}
if encryptedBoxID != "" {
HoldBoxReadLock(encryptedBoxID)
defer ReleaseBoxReadLock(encryptedBoxID)
if _, dekErr := GetDEKIfUnlocked(encryptedBoxID); dekErr != nil {
err = errors.New(Conf.Language(314))
return
}
}
name := path.Base(file.Path)
exportRoot := filepath.Join(util.TempDir, "export", "repo")
managedKind := ""
if encryptedBoxID != "" {
var exportID string
exportID, err = newManagedEncryptedExportID()
if err != nil {
return
}
managedKind = path.Join("repo", exportID)
exportRoot = filepath.Join(util.TempDir, "export", encryptedBoxID, "repo", exportID)
defer func() {
if err != nil {
_ = os.RemoveAll(exportRoot)
}
}()
}
exportDir := exportRoot
// 如果是 .sy 文件需要打包为 .sy.zip 以便导入
var docTitle string
if strings.HasSuffix(file.Path, ".sy") {
var tree *parse.Tree
luteEngine := NewLute()
if err = treenode.CheckSpecJSON(data); nil != err {
return
}
tree, err = dataparser.ParseJSONWithoutFix(data, luteEngine.ParseOptions)
if err != nil {
logging.LogErrorf("parse file [%s] failed: %s", id, err)
return
}
docTitle = util.FilterFileName(treenode.ReadDocumentTitle(tree.Root).Text)
if docTitle == "" {
docTitle = strings.TrimSuffix(name, ".sy")
}
exportDir = filepath.Join(exportDir, docTitle)
}
if err = os.MkdirAll(exportDir, 0755); err != nil {
logging.LogErrorf("mkdir [%s] failed: %s", exportDir, err)
return
}
exportFilePath := filepath.Join(exportDir, name)
if err = os.WriteFile(exportFilePath, data, 0644); err != nil {
logging.LogErrorf("write file [%s] failed: %s", exportFilePath, err)
return
}
if strings.HasSuffix(file.Path, ".sy") {
zipPath := filepath.Join(exportRoot, docTitle+".sy.zip")
zip, zipErr := gulu.Zip.Create(zipPath)
if zipErr != nil {
logging.LogErrorf("create export .sy.zip [%s] failed: %s", exportDir, zipErr)
err = zipErr
return
}
zipClosed := false
defer func() {
if !zipClosed {
_ = zip.Close()
}
}()
if err = zip.AddDirectory(docTitle, exportDir); err != nil {
logging.LogErrorf("create export .sy.zip [%s] failed: %s", exportDir, err)
return
}
if err = zip.Close(); err != nil {
logging.LogErrorf("close export .sy.zip failed: %s", err)
return
}
zipClosed = true
_ = os.RemoveAll(exportDir)
if encryptedBoxID != "" {
exportPath = path.Join("/export", registerManagedEncryptedExport(encryptedBoxID, managedKind, zipPath))
} else {
exportPath = path.Join("/export/repo", url.PathEscape(filepath.Base(zipPath)))
}
return
}
if encryptedBoxID != "" {
exportPath = path.Join("/export", registerManagedEncryptedExport(encryptedBoxID, managedKind, exportFilePath))
} else {
exportPath = path.Join("/export/repo", url.PathEscape(name))
}
return
}
type Snapshot struct {
*dejavu.Log
Tags []string `json:"tags"`
TypesCount []*TypeCount `json:"typesCount"`
RequiresDownload bool `json:"requiresDownload"`
}
type TypeCount struct {
Type string `json:"type"`
Count int `json:"count"`
}
// SearchRepoSnapshot 按完整 ID 或至少 7 位前缀读取本地快照,复用列表的元数据和下载状态计算。
func SearchRepoSnapshot(id string, includeFiles bool) (ret []*Snapshot, pageCount, totalCount int, err error) {
ret = []*Snapshot{}
id = strings.ToLower(strings.TrimSpace(id))
if len(id) < 7 || len(id) > sha1.Size*2 || strings.Trim(id, "0123456789abcdef") != "" {
return ret, 0, 0, errors.New("invalid snapshot ID")
}
if includeFiles && len(id) != sha1.Size*2 {
return ret, 0, 0, errors.New("complete snapshot ID is required to include files")
}
if len(Conf.Repo.Load().Key) == 0 {
return ret, 0, 0, errors.New(Conf.Language(26))
}
repo, err := newRepository()
if err != nil {
return
}
ids := []string{id}
if len(id) < sha1.Size*2 {
entries, readErr := os.ReadDir(filepath.Join(repo.Path, "indexes"))
if os.IsNotExist(readErr) {
return ret, 0, 0, nil
}
if readErr != nil {
return ret, 0, 0, readErr
}
ids = nil
for _, entry := range entries {
name := entry.Name()
if !entry.IsDir() && len(name) == sha1.Size*2 && strings.HasPrefix(name, id) &&
strings.Trim(name, "0123456789abcdef") == "" {
ids = append(ids, name)
}
}
}
var logs []*dejavu.Log
for _, fullID := range ids {
index, readErr := repo.GetIndex(fullID)
if os.IsNotExist(readErr) || errors.Is(readErr, dejavu.ErrNotFoundIndex) {
continue
}
if readErr != nil {
return ret, 0, 0, readErr
}
files, readErr := repo.GetFiles(index)
if readErr != nil {
return ret, 0, 0, readErr
}
logs = append(logs, &dejavu.Log{
ID: index.ID, Memo: index.Memo, Created: index.Created,
HCreated: time.UnixMilli(index.Created).Format("2006-01-02 15:04:05"),
Files: files, Count: index.Count, Size: index.Size,
HSize: humanize.BytesCustomCeil(uint64(index.Size), 2),
SystemID: index.SystemID, SystemName: index.SystemName, SystemOS: index.SystemOS,
})
}
if len(logs) == 0 {
return ret, 0, 0, nil
}
sort.SliceStable(logs, func(i, j int) bool { return logs[i].Created > logs[j].Created })
ret = buildSnapshots(logs, includeFiles)
if err = attachSnapshotTags(repo, ret); err != nil {
return ret, 0, 0, err
}
return ret, 1, len(ret), nil
}
func GetRepoSnapshots(page int) (ret []*Snapshot, pageCount, totalCount int, err error) {
ret = []*Snapshot{}
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
repo, err := newRepository()
if err != nil {
return
}
logs, pageCount, totalCount, err := repo.GetIndexLogs(page, 32)
if err != nil {
if errors.Is(err, dejavu.ErrNotFoundIndex) {
logs = []*dejavu.Log{}
err = nil
return
}
logging.LogErrorf("get data repo index logs failed: %s", err)
return
}
ret = buildSnapshots(logs, false)
err = attachSnapshotTags(repo, ret)
if 1 > len(ret) {
ret = []*Snapshot{}
}
return
}
func attachSnapshotTags(repo *dejavu.Repo, snapshots []*Snapshot) error {
if len(snapshots) == 0 {
return nil
}
_, tags, err := localTaggedSnapshotIndexes(context.Background(), repo)
if err != nil {
return err
}
for _, snapshot := range snapshots {
if names := tags[snapshot.ID]; len(names) > 0 {
snapshot.Tags = names
}
}
return nil
}
func buildSnapshots(logs []*dejavu.Log, includeFiles bool) (ret []*Snapshot) {
chunkAvailability := map[string]bool{}
for _, l := range logs {
typesCount := statTypesByPath(l.Files)
requiresDownload := false
for _, file := range l.Files {
if repoFileNeedsDownloadWithCache(file, chunkAvailability) {
requiresDownload = true
break
}
}
if !includeFiles {
l.Files = nil // 列表仅返回概要,避免传输所有快照的文件列表。
}
ret = append(ret, &Snapshot{
Log: l,
Tags: []string{},
TypesCount: typesCount,
RequiresDownload: requiresDownload,
})
}
return
}
func statTypesByPath(files []*entity.File) (ret []*TypeCount) {
for _, f := range files {
ext := util.Ext(f.Path)
if "" == ext {
ext = "NoExt"
}
found := false
for _, tc := range ret {
if tc.Type == ext {
tc.Count++
found = true
break
}
}
if !found {
ret = append(ret, &TypeCount{Type: ext, Count: 1})
}
}
sort.Slice(ret, func(i, j int) bool { return ret[i].Count > ret[j].Count })
if 10 < len(ret) {
otherCount := 0
for _, tc := range ret[10:] {
otherCount += tc.Count
}
other := &TypeCount{
Type: "Other",
Count: otherCount,
}
ret = append(ret[:10], other)
}
return
}
func ImportRepoKey(base64Key string) (retKey string, err error) {
release := lockAssetSourceChange()
defer release()
if err = requireCompleteAssetDownloads(); err != nil {
return
}
util.PushMsg(Conf.Language(136), 3000)
retKey = gulu.Str.RemoveInvisible(base64Key)
retKey = strings.TrimSpace(retKey)
if 1 > len(retKey) {
err = errors.New(Conf.Language(142))
return
}
key, err := base64.StdEncoding.DecodeString(retKey)
if err != nil {
logging.LogErrorf("import data repo key failed: %s", err)
return "", errors.New(Conf.Language(157))
}
if 32 != len(key) {
return "", errors.New(Conf.Language(157))
}
if err = clearAssetDownloadState(); err != nil {
return
}
suspendLANSyncManager()
UpdateRepoConf(func(s *conf.Repo) { s.Key = key })
logging.LogInfof("imported repo key [%x]", sha1.Sum(Conf.Repo.Load().Key))
if err = os.RemoveAll(Conf.Repo.Load().GetSaveDir()); err != nil {
return
}
if err = os.MkdirAll(Conf.Repo.Load().GetSaveDir(), 0755); err != nil {
return
}
release()
initDataRepo()
refreshLANSyncManager()
return
}
func ResetRepo() (err error) {
release := lockAssetSourceChange()
defer release()
if err = requireCompleteAssetDownloads(); err != nil {
return
}
logging.LogInfof("resetting data repo...")
msgId := util.PushMsg(Conf.Language(144), 1000*60)
suspendLANSyncManager()
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
return
}
if err = repo.ClearAssetDownloadState(); err != nil {
return
}
if err = repo.Reset(); err != nil {
logging.LogErrorf("reset data repo failed: %s", err)
return
}
logging.LogInfof("reset data repo completed")
UpdateRepoConf(func(s *conf.Repo) { s.Key = nil })
UpdateSyncConf(func(s *conf.Sync) { s.Enabled = false })
refreshLANSyncManager()
util.PushUpdateMsg(msgId, Conf.Language(145), 3000)
task.AppendAsyncTaskWithDelay(task.ReloadUI, 2*time.Second, util.ReloadUI)
return
}
func PurgeCloud() (err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
if err = requireCompleteAssetDownloads(); err != nil {
return
}
msg := Conf.Language(223)
util.PushEndlessProgress(msg)
defer util.PushClearProgress()
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newCloudRepositoryWithAssetSourceLocked()
if err != nil {
return
}
stat, err := repo.PurgeCloud()
if err != nil {
return
}
deletedIndexes := stat.Indexes
deletedObjects := stat.Objects
deletedSize := humanize.BytesCustomCeil(uint64(stat.Size), 2)
msg = fmt.Sprintf(Conf.Language(232), deletedIndexes, deletedObjects, deletedSize)
util.PushMsg(msg, 7000)
return
}
func PurgeRepo() (err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
msg := Conf.Language(202)
util.PushEndlessProgress(msg)
defer util.PushClearProgress()
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
return
}
stat, err := repo.Purge(context.Background())
if err != nil {
return
}
deletedIndexes := stat.Indexes
deletedObjects := stat.Objects
deletedSize := humanize.BytesCustomCeil(uint64(stat.Size), 2)
msg = fmt.Sprintf(Conf.Language(203), deletedIndexes, deletedObjects, deletedSize)
util.PushMsg(msg, 7000)
return
}
func InitRepoKeyFromPassphrase(passphrase string) (err error) {
release := lockAssetSourceChange()
defer release()
if err = requireCompleteAssetDownloads(); err != nil {
return
}
passphrase = gulu.Str.RemoveInvisible(passphrase)
passphrase = strings.TrimSpace(passphrase)
if "" == passphrase {
return errors.New(Conf.Language(142))
}
if err = clearAssetDownloadState(); err != nil {
return
}
util.PushMsg(Conf.Language(136), 3000)
suspendLANSyncManager()
if err = os.RemoveAll(Conf.Repo.Load().GetSaveDir()); err != nil {
return
}
if err = os.MkdirAll(Conf.Repo.Load().GetSaveDir(), 0755); err != nil {
return
}
var key []byte
base64Data, base64Err := base64.StdEncoding.DecodeString(passphrase)
if nil == base64Err && 32 == len(base64Data) {
// 改进数据仓库 `通过密码生成密钥` https://github.com/siyuan-note/siyuan/issues/6782
logging.LogInfof("passphrase is base64 encoded, use it as key directly")
key = base64Data
} else {
salt := fmt.Sprintf("%x", sha256.Sum256([]byte(passphrase)))[:16]
key, err = encryption.KDF(passphrase, salt)
if err != nil {
logging.LogErrorf("init data repo key failed: %s", err)
return
}
}
UpdateRepoConf(func(s *conf.Repo) { s.Key = key })
logging.LogInfof("inited repo key [%x]", sha1.Sum(Conf.Repo.Load().Key))
release()
initDataRepo()
refreshLANSyncManager()
return
}
func InitRepoKey() (err error) {
release := lockAssetSourceChange()
defer release()
if err = requireCompleteAssetDownloads(); err != nil {
return
}
if err = clearAssetDownloadState(); err != nil {
return
}
util.PushMsg(Conf.Language(136), 3000)
suspendLANSyncManager()
if err = os.RemoveAll(Conf.Repo.Load().GetSaveDir()); err != nil {
return
}
if err = os.MkdirAll(Conf.Repo.Load().GetSaveDir(), 0755); err != nil {
return
}
randomBytes := make([]byte, 16)
_, err = rand.Read(randomBytes)
if err != nil {
return
}
password := string(randomBytes)
randomBytes = make([]byte, 16)
_, err = rand.Read(randomBytes)
if err != nil {
logging.LogErrorf("init data repo key failed: %s", err)
return
}
salt := string(randomBytes)
key, err := encryption.KDF(password, salt)
if err != nil {
logging.LogErrorf("init data repo key failed: %s", err)
return
}
UpdateRepoConf(func(s *conf.Repo) { s.Key = key })
logging.LogInfof("inited repo key [%x]", sha1.Sum(Conf.Repo.Load().Key))
release()
initDataRepo()
refreshLANSyncManager()
return
}
func initDataRepo() {
time.Sleep(1 * time.Second)
util.PushMsg(Conf.Language(138), 3000)
time.Sleep(1 * time.Second)
if _, initErr := IndexRepo("[Init] Init local data repo"); nil != initErr {
util.PushErrMsg(fmt.Sprintf(Conf.Language(140), initErr), 0)
}
}
func CheckoutRepo(id string) {
task.AppendTask(task.RepoCheckout, checkoutRepo, id)
}
func CheckoutRepoDirect(id string) error {
return checkoutRepo(id)
}
func checkoutRepo(id string) (err error) {
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
util.PushErrMsg(err.Error(), 7000)
return
}
FlushTxQueue()
// Keep synchronization serialized until the write guard and UI state are released.
lockSync()
defer unlockSync()
assetDownloadSourceMu.Lock()
var releaseSource sync.Once
release := func() { releaseSource.Do(assetDownloadSourceMu.Unlock) }
defer release()
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
logging.LogErrorf("new repository failed: %s", err)
util.PushErrMsg(Conf.Language(141), 7000)
return
}
if err = ensureAllSyncAssets(); err == nil {
err = ensureRepoSnapshotComplete(repo, id)
}
if err != nil {
util.PushErrMsg(err.Error(), 7000)
return
}
checkoutTask, err := prepareRepoCheckout(repo)
if err != nil {
util.PushErrMsg(err.Error(), 7000)
return err
}
defer checkoutTask.finish(repo, &err)
util.PushEndlessProgress(Conf.Language(63))
CloseWatchAssets()
defer WatchAssets()
CloseWatchEmojis()
defer WatchEmojis()
// 快照恢复期间暂停监听,完成后重新绑定实际目录。
CloseWatchThemes()
defer WatchThemes()
// 恢复快照时自动暂停同步,避免刚刚恢复后的数据又被同步覆盖
syncEnabled := Conf.Sync.Load().Enabled
UpdateSyncConf(func(s *conf.Sync) { s.Enabled = false })
if syncEnabled {
util.PushMsg(Conf.Language(134), 0)
}
// 回滚快照时默认为当前数据创建一个快照
// When rolling back a snapshot, a snapshot is created for the current data by default https://github.com/siyuan-note/siyuan/issues/12470
if err = processAssetDownloadRecovery(repo, true); err != nil {
return
}
err = checkoutRepoSnapshot(repo, id, func(checkoutErr error) {
release()
fullReindexWithTxFlush(false)
if checkoutErr != nil {
util.ReloadUI()
return
}
appendAgentRollbackEntries()
time.Sleep(time.Second)
task.AppendAsyncTaskWithDelay(task.ReloadUI, 1*time.Second, util.ReloadUI)
})
if err != nil {
logging.LogErrorf("checkout repository failed: %s", err)
util.PushClearProgress()
util.PushErrMsg(Conf.Language(141), 7000)
return
}
return
}
// prepareRepoCheckout saves every window before the backup and retains both write
// admission and transaction exclusion until the caller finishes its full reindex.
func prepareRepoCheckout(repo *dejavu.Repo) (*syncCheckoutTask, error) {
checkout := startSyncCheckout(repo)
if checkout == nil {
return nil, checkoutPrepareRetry()
}
// A full restore refreshes all indexes as one operation. Do not release the
// guard through incremental sync Applied callbacks or repeat a partial refresh.
repo.SetCheckoutHooks(dejavu.CheckoutHooks{
Context: checkout.ctx, Apply: checkout.prepareApply, Before: checkCheckoutSpace,
Preflight: documentAddressingPreflight(repo),
})
if err := checkout.prepareApply(); err != nil {
checkout.finish(repo, &err)
return nil, err
}
return checkout, nil
}
// checkoutRepoSnapshot 恢复失败前可能已有文件落盘,返回结果前同步更新索引、缓存和界面。
func checkoutRepoSnapshot(repo *dejavu.Repo, id string, refresh func(error)) error {
// 恢复会核对实际内容,备份也必须完整校验,保留随后可能被覆盖的当前数据。
if _, err := repo.IndexWithOptions("Backup before checkout", false,
map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress},
dejavu.IndexOptions{FullCheck: true}); err != nil {
return err
}
upserts, removes, err := repo.Checkout(id, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
themes, icons := appearanceChangedPackages(&dejavu.MergeResult{Upserts: upserts, Removes: removes})
refreshAppearancePackages(themes, icons)
refresh(err)
return err
}
func appendAgentRollbackEntries() {
pattern := filepath.Join(util.TempDir, "ai", "agent", "agentRollback_*.json")
markers, err := filepath.Glob(pattern)
if err != nil {
return
}
for _, markerPath := range markers {
data, err := os.ReadFile(markerPath)
if err != nil {
os.Remove(markerPath)
continue
}
var marker struct {
SessionID string `json:"sessionID"`
SnapshotID string `json:"snapshotID"`
}
if nil != gulu.JSON.UnmarshalJSON(data, &marker) {
os.Remove(markerPath)
continue
}
sessionPath := filepath.Join(util.DataDir, "storage", "ai", "agent", "sessions",
marker.SessionID, "session.json")
sessionData, err := os.ReadFile(sessionPath)
if err != nil {
os.Remove(markerPath)
continue
}
var session map[string]any
if nil != gulu.JSON.UnmarshalJSON(sessionData, &session) {
os.Remove(markerPath)
continue
}
entries, ok := session["entries"].([]any)
if !ok {
entries = make([]any, 0)
}
entry := map[string]any{
"type": "rollback",
"snapshotID": marker.SnapshotID,
}
entries = append(entries, entry)
session["entries"] = entries
newData, err := gulu.JSON.MarshalIndentJSON(session, "", "\t")
if err != nil {
os.Remove(markerPath)
continue
}
filelock.WriteFile(sessionPath, newData)
os.Remove(markerPath)
}
}
func DownloadCloudSnapshot(tag, id string) (err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newCloudRepositoryWithAssetSourceLocked()
if err != nil {
return
}
switch Conf.Sync.Load().Provider {
case conf.ProviderSiYuan:
if !IsSubscriber() {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
}
}
defer util.PushClearProgress()
var downloadFileCount, downloadChunkCount int
var downloadBytes int64
if "" == tag {
downloadFileCount, downloadChunkCount, downloadBytes, err = repo.DownloadIndex(id, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
} else {
downloadFileCount, downloadChunkCount, downloadBytes, err = repo.DownloadTagIndex(tag, id, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
}
if err != nil {
return
}
msg := fmt.Sprintf(Conf.Language(153), downloadFileCount, downloadChunkCount, humanize.BytesCustomCeil(uint64(downloadBytes), 2))
util.PushMsg(msg, 5000)
util.PushStatusBar(msg)
return
}
func UploadCloudSnapshot(tag, id string) (err error) {
confSyncSnapshot := Conf.LoadSync()
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newCloudRepositoryWithAssetSourceLocked()
if err != nil {
return
}
switch confSyncSnapshot.Provider {
case conf.ProviderSiYuan:
if !IsSubscriber() {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
}
}
util.PushEndlessProgress(Conf.Language(116))
defer util.PushClearProgress()
uploadFileCount, uploadChunkCount, uploadBytes, err := repo.UploadTagIndex(tag, id, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress})
if err != nil {
if errors.Is(err, dejavu.ErrCloudBackupCountExceeded) {
err = fmt.Errorf(Conf.Language(84), Conf.Language(154))
return
}
if conf.ProviderSiYuan == confSyncSnapshot.Provider && errors.Is(err, dejavu.ErrCloudStorageSizeExceeded) {
u := Conf.GetUser()
msg := fmt.Sprintf(Conf.Language(43), humanize.BytesCustomCeil(uint64(u.UserSiYuanRepoSize), 2))
if 2 == u.UserSiYuanSubscriptionPlan {
msg = fmt.Sprintf(Conf.Language(68), humanize.BytesCustomCeil(uint64(u.UserSiYuanRepoSize), 2))
}
err = fmt.Errorf(Conf.Language(84), msg)
return
}
handleCloudError(err)
err = fmt.Errorf(Conf.Language(84), formatRepoErrorMsg(err))
return
}
msg := fmt.Sprintf(Conf.Language(152), uploadFileCount, uploadChunkCount, humanize.BytesCustomCeil(uint64(uploadBytes), 2))
util.PushMsg(msg, 5000)
util.PushStatusBar(msg)
return
}
func RemoveCloudRepoTag(tag string) (err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newCloudRepositoryWithAssetSourceLocked()
if err != nil {
return
}
switch Conf.Sync.Load().Provider {
case conf.ProviderSiYuan:
if !IsSubscriber() {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
}
}
err = repo.RemoveCloudRepoTag(tag)
if err != nil {
return
}
return
}
func GetCloudRepoTagSnapshots() (ret []*dejavu.Log, err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
ret = []*dejavu.Log{}
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newCloudRepositoryWithAssetSourceLocked()
if err != nil {
return
}
switch Conf.Sync.Load().Provider {
case conf.ProviderSiYuan:
if !IsSubscriber() {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
}
}
logs, err := repo.GetCloudRepoTagLogsCached(&cloudRepoSnapshotCache, map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBar})
if err != nil {
return
}
ret = logs
if 1 > len(ret) {
ret = []*dejavu.Log{}
}
return
}
func GetCloudRepoSnapshots(page int) (ret []*dejavu.Log, pageCount, totalCount int, err error) {
return GetCloudRepoSnapshotsByTime(page, 0, 0)
}
func GetCloudRepoSnapshotsByTime(page int, startTime, endTime int64) (ret []*dejavu.Log, pageCount, totalCount int, err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
ret = []*dejavu.Log{}
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newCloudRepositoryWithAssetSourceLocked()
if err != nil {
return
}
switch Conf.Sync.Load().Provider {
case conf.ProviderSiYuan:
if !IsSubscriber() {
util.PushErrMsg(Conf.Language(29), 5000)
return
}
case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
if !IsPaidUser() {
util.PushErrMsg(Conf.Language(214), 5000)
return
}
}
if 1 > page {
page = 1
}
load, err := repo.NewCloudRepoLogLoader(&cloudRepoSnapshotCache)
if err != nil {
return
}
logs, pageCount, totalCount, err := cloudRepoSnapshotsByTime(page, startTime, endTime, load)
if err != nil {
return
}
ret = logs
if 1 > len(ret) {
ret = []*dejavu.Log{}
}
return
}
func GetTagSnapshots() (ret []*Snapshot, err error) {
ret = []*Snapshot{}
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
repo, err := newRepository()
if err != nil {
return
}
logs, err := repo.GetTagLogs()
if err != nil {
return
}
ret = buildSnapshots(logs, false)
err = attachSnapshotTags(repo, ret)
if 1 > len(ret) {
ret = []*Snapshot{}
}
return
}
func RemoveTagSnapshot(tag string) (err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
return
}
err = repo.RemoveTag(tag)
return
}
func TagSnapshot(id, name string) (err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
name = util.RemoveInvalid(name)
name = strings.TrimSpace(name)
if "" == name {
err = errors.New(Conf.Language(142))
return
}
if !gulu.File.IsValidFilename(name) {
err = errors.New(Conf.Language(151))
return
}
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
return
}
index, err := repo.GetIndex(id)
if err != nil {
return
}
if err = repo.AddTag(index.ID, name); err != nil {
msg := fmt.Sprintf("Add tag to data snapshot [%s] failed: %s", index.ID, err)
util.PushStatusBar(msg)
return
}
return
}
func IndexRepo(memo string) (id string, err error) {
id, _, err = CreateRepoSnapshot(memo)
return
}
func normalizeSnapshotMemo(memo string) string {
// 按行清理不可见字符,保留多行备注的换行。
lines := strings.Split(strings.ReplaceAll(memo, "\r\n", "\n"), "\n")
for i, line := range lines {
lines[i] = gulu.Str.RemoveInvisible(line)
}
memo = strings.TrimSpace(strings.Join(lines, "\n"))
if memo == "" {
return "Create manually"
}
return memo
}
func CreateRepoSnapshot(memo string) (id string, created bool, err error) {
return CreateRepoSnapshotWithFullCheck(memo, false)
}
// CreateRepoSnapshotWithFullCheck 允许本次快照核对所有本地已落盘文件的内容。
func CreateRepoSnapshotWithFullCheck(memo string, fullCheck bool) (id string, created bool, err error) {
if err = checkDataImportRecovery(); err != nil {
return
}
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
}
memo = normalizeSnapshotMemo(memo)
FlushTxQueue()
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
if err = checkRepositoryPasswordMigration(); err != nil {
return
}
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
return
}
needsDownload, err := repo.NeedsAssetDownloadsForIndex()
if err != nil {
return
}
if needsDownload {
if err = checkAssetDownloadAccess(); err != nil {
return
}
}
util.PushEndlessProgress(Conf.Language(143))
defer util.PushClearProgress()
start := time.Now()
if err = processAssetDownloadRecovery(repo, true); err != nil {
return
}
index, created, err := repo.IndexWithResultOptions(memo, true, map[string]any{
eventbus.CtxPushMsg: eventbus.CtxPushMsgToStatusBarAndProgress,
dejavu.CtxAssetDownloadsAllowed: checkAssetDownloadAccess() == nil,
}, dejavu.IndexOptions{FullCheck: fullCheck})
if err != nil {
util.PushStatusBar("Index data repo failed: " + html.EscapeString(err.Error()))
return
}
id = index.ID
elapsed := time.Since(start)
if created {
msg := fmt.Sprintf(Conf.Language(147), elapsed.Seconds())
util.PushStatusBar(msg)
util.PushMsg(msg, 5000)
} else {
msg := fmt.Sprintf(Conf.Language(148), elapsed.Seconds())
util.PushStatusBar(msg)
util.PushMsg(msg, 5000)
}
return
}
func CheckRepoSnapshot() (changed bool, err error) {
if len(Conf.Repo.Load().Key) == 0 {
return false, errors.New(Conf.Language(26))
}
FlushTxQueue()
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
return false, err
}
start := time.Now()
changed, err = repo.CheckSnapshot()
if err == nil && !changed {
util.PushMsg(fmt.Sprintf(Conf.Language(148), time.Since(start).Seconds()), 5000)
}
return
}
func SetRepoSnapshotMemo(id, memo string) error {
if len(Conf.Repo.Load().Key) == 0 {
return errors.New(Conf.Language(26))
}
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
repo, err := newRepositoryWithAssetSourceLocked()
if err != nil {
return err
}
return repo.SetSnapshotMemo(id, normalizeSnapshotMemo(memo))
}
var syncingFiles = sync.Map{}
func waitForSyncingStorages() {
for isSyncingStorages() {
time.Sleep(time.Second)
}
}
func isSyncingStorages() bool {
return isBootSyncing.Load()
}
func IsSyncingFile(rootID string) (ret bool) {
_, ret = syncingFiles.Load(rootID)
return
}
func syncStatusBarDisabled() bool {
return util.StatusBarCfg.Load().MsgDataSyncDisabled
}
func pushSyncStatusBar(msg string) {
if syncStatusBarDisabled() {
return
}
util.PushStatusBar(msg)
}
func newSyncContext() map[string]any {
pushTarget := eventbus.CtxPushMsgToStatusBar
if syncStatusBarDisabled() {
pushTarget = eventbus.CtxPushMsgToNone
}
return map[string]any{eventbus.CtxPushMsg: pushTarget}
}
func formatSyncRepoErrorMsg(err error) string {
if errors.Is(err, context.Canceled) {
return Conf.Language(433)
}
if conf.ProviderSiYuan == Conf.Sync.Load().Provider && errors.Is(err, dejavu.ErrCloudStorageSizeExceeded) {
u := Conf.GetUser()
if 2 == u.UserSiYuanSubscriptionPlan {
return fmt.Sprintf(Conf.Language(68), humanize.BytesCustomCeil(uint64(u.UserSiYuanRepoSize), 2))
}
return fmt.Sprintf(Conf.Language(43), humanize.BytesCustomCeil(uint64(u.UserSiYuanRepoSize), 2))
}
return fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(err))
}
func syncRepoDownload() (err error) {
if err = checkDataImportRecovery(); err != nil {
return
}
if 1 > len(Conf.Repo.Load().Key) {
planSyncAfter(fixSyncInterval)
msg := Conf.Language(26)
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
err = errors.New(msg)
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newSyncRepository()
if err != nil {
planSyncAfter(fixSyncInterval)
msg := fmt.Sprintf("sync repo failed: %s", err)
logging.LogError(msg)
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
return
}
logging.LogInfof("downloading data repo [device=%s, kernel=%s, provider=%d, mode=%s/%t]", Conf.System.Load().ID, KernelID, Conf.Sync.Load().Provider, "d", true)
passwordRecovery, err := preparePasswordRecoveryDownload(repo)
if err != nil {
return err
}
checkoutTask := startSyncCheckout(repo)
defer checkoutTask.finish(repo, &err)
defer finishAssetDownloadRecovery(repo, &err)
start := time.Now()
indexStart := time.Now()
_, _, err = indexRepoBeforeCloudSync0(repo)
indexElapsed := time.Since(indexStart)
if err != nil {
planSyncAfter(fixSyncInterval)
logging.LogErrorf("sync data repo download failed: %s", err)
msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(err))
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
return
}
beforeSyncPetals := getPetals()
syncContext := newSyncContext()
cloudStart := time.Now()
mergeResult, trafficStat, err := repo.SyncDownload(syncContext)
cloudElapsed := time.Since(cloudStart)
elapsed := time.Since(start)
if err != nil {
planSyncAfter(fixSyncInterval)
logging.LogErrorf("sync data repo download failed: %s", err)
msg := formatSyncRepoErrorMsg(err)
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
return
}
resetAutoSyncErrors()
pushSyncStatusBar(fmt.Sprintf(Conf.Language(149), elapsed.Seconds()))
UpdateSyncConf(func(s *conf.Sync) { s.Synced = util.CurrentTimeMillis() })
msg := fmt.Sprintf(Conf.Language(150), trafficStat.UploadFileCount, trafficStat.DownloadFileCount, trafficStat.UploadChunkCount, trafficStat.DownloadChunkCount, humanize.BytesCustomCeil(uint64(trafficStat.UploadBytes), 2), humanize.BytesCustomFloor(uint64(trafficStat.DownloadBytes+trafficStat.PeerDownloadBytes), 2))
msg = appendLANSyncTrafficStat(msg, trafficStat)
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
BootSyncSucc.Store(0)
calcPetalDiff(beforeSyncPetals, mergeResult)
postProcessStart := time.Now()
err = processAssetSyncMergeResult(repo, false, true, mergeResult, trafficStat, "d", elapsed)
if passwordRecovery {
// Reapply the retained envelopes to notebooks brought back by download.
// Final backup authentication still requires the user's new password.
notebookCryptoMu.Lock()
recoverMasterPasswordMigration()
notebookCryptoMu.Unlock()
}
logging.LogInfof("download data repo phases [index=%.2fs, cloud=%.2fs, post-process=%.2fs, total=%.2fs]",
indexElapsed.Seconds(), cloudElapsed.Seconds(), time.Since(postProcessStart).Seconds(), time.Since(start).Seconds())
return
}
func syncRepoUpload() (err error) {
if err = checkDataImportRecovery(); err != nil {
return
}
if 1 > len(Conf.Repo.Load().Key) {
planSyncAfter(fixSyncInterval)
msg := Conf.Language(26)
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
err = errors.New(msg)
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newSyncRepository()
if err != nil {
planSyncAfter(fixSyncInterval)
msg := fmt.Sprintf("sync repo failed: %s", err)
logging.LogError(msg)
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
return
}
logging.LogInfof("uploading data repo [device=%s, kernel=%s, provider=%d, mode=%s/%t]", Conf.System.Load().ID, KernelID, Conf.Sync.Load().Provider, "u", true)
checkoutTask := startSyncCheckout(repo)
defer checkoutTask.finish(repo, &err)
defer finishAssetDownloadRecovery(repo, &err)
start := time.Now()
indexStart := time.Now()
_, _, err = indexRepoBeforeCloudSync(repo)
indexElapsed := time.Since(indexStart)
if err != nil {
planSyncAfter(fixSyncInterval)
logging.LogErrorf("sync data repo upload failed: %s", err)
msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(err))
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
pushSyncStatusBar(msg)
if !errors.Is(err, context.Canceled) {
util.PushErrMsg(msg, 0)
}
return
}
syncContext := newSyncContext()
cloudStart := time.Now()
trafficStat, err := repo.SyncUpload(syncContext)
cloudElapsed := time.Since(cloudStart)
elapsed := time.Since(start)
if err != nil {
planSyncAfter(fixSyncInterval)
logging.LogErrorf("sync data repo upload failed: %s", err)
msg := formatSyncRepoErrorMsg(err)
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
pushSyncStatusBar(msg)
if !errors.Is(err, context.Canceled) {
util.PushErrMsg(msg, 0)
}
return
}
resetAutoSyncErrors()
pushSyncStatusBar(fmt.Sprintf(Conf.Language(149), elapsed.Seconds()))
UpdateSyncConf(func(s *conf.Sync) { s.Synced = util.CurrentTimeMillis() })
msg := fmt.Sprintf(Conf.Language(150), trafficStat.UploadFileCount, trafficStat.DownloadFileCount, trafficStat.UploadChunkCount, trafficStat.DownloadChunkCount, humanize.BytesCustomCeil(uint64(trafficStat.UploadBytes), 2), humanize.BytesCustomCeil(uint64(trafficStat.DownloadBytes+trafficStat.PeerDownloadBytes), 2))
msg = appendLANSyncTrafficStat(msg, trafficStat)
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
BootSyncSucc.Store(0)
postProcessStart := time.Now()
processSyncMergeResult(false, true, &dejavu.MergeResult{}, trafficStat, "u", elapsed)
notifyLANSyncCommit(repo)
logging.LogInfof("upload data repo phases [index=%.2fs, cloud=%.2fs, post-process=%.2fs, total=%.2fs]",
indexElapsed.Seconds(), cloudElapsed.Seconds(), time.Since(postProcessStart).Seconds(), time.Since(start).Seconds())
return
}
var isBootSyncing = atomic.Bool{}
func bootSyncRepo(complete func(error)) (scheduled bool, err error) {
if err = checkDataImportRecovery(); err != nil {
return
}
if 1 > len(Conf.Repo.Load().Key) {
autoSyncErrCount.Add(1)
planSyncAfter(fixSyncInterval)
msg := Conf.Language(26)
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
err = errors.New(msg)
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newSyncRepository()
if err != nil {
recordAutoSyncFailure(err)
planSyncAfter(fixSyncInterval)
msg := fmt.Sprintf("sync repo failed: %s", html.EscapeString(err.Error()))
logging.LogError(msg)
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
return
}
preparedSourceConf, err := buildCloudConf()
if err != nil {
return
}
preparedSourceScope := assetDownloadScope(Conf.Sync.Load().Provider, preparedSourceConf, Conf.Repo.Load().Key)
isBootSyncing.Store(true)
defer isBootSyncing.Store(false)
waitGroup := sync.WaitGroup{}
var indexErr error
waitGroup.Go(func() {
defer logging.Recover()
start := time.Now()
_, _, indexErr = indexRepoBeforeCloudSync(repo)
logging.LogInfof("boot index repo elapsed [%.2fs]", time.Since(start).Seconds())
})
var cloudLatest *entity.Index
var cloudLatestErr error
waitGroup.Go(func() {
defer logging.Recover()
start := time.Now()
syncContext := newSyncContext()
cloudLatest, cloudLatestErr = repo.GetCloudLatestFast(syncContext)
if nil != cloudLatestErr && !errors.Is(cloudLatestErr, cloud.ErrCloudObjectNotFound) {
logging.LogErrorf("download cloud latest failed: %s", cloudLatestErr)
}
logging.LogInfof("boot get cloud latest elapsed [%.2fs]", time.Since(start).Seconds())
})
waitGroup.Wait()
if nil != indexErr {
err = indexErr
} else if nil != cloudLatestErr && !errors.Is(cloudLatestErr, cloud.ErrCloudObjectNotFound) {
err = cloudLatestErr
}
var fetchedFiles []*entity.File
var prefetchTraffic *dejavu.DownloadTrafficStat
if nil == err {
start := time.Now()
syncContext := newSyncContext()
fetchedFiles, prefetchTraffic, err = repo.GetSyncCloudFilesWithTraffic(cloudLatest, syncContext)
logging.LogInfof("boot get sync cloud files elapsed [%.2fs]", time.Since(start).Seconds())
}
if err != nil {
recordAutoSyncFailure(err)
planSyncAfter(fixSyncInterval)
logging.LogErrorf("sync data repo failed: %s", err)
msg := formatSyncRepoErrorMsg(err)
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
BootSyncSucc.Store(1)
return
}
if 1 > len(fetchedFiles) {
cleanupSyncedBoxResiduals(nil)
}
if 0 < len(fetchedFiles) {
scheduled = true
go func() {
defer logging.Recover()
// 预取结束后不再用文件标记阻塞初始化,实际应用由 checkout 的保存确认和写保护负责。
if waitErr := waitForSyncUI(context.Background(), false, true); waitErr != nil {
complete(waitErr)
return
}
lockSync()
defer unlockSync()
syncErr := errSyncRemoteRequestSkipped
defer func() { complete(syncErr) }()
if !checkSync(true, false, false) {
return
}
confSyncSnapshot := Conf.LoadSync()
currentSourceConf, sourceErr := buildCloudConf()
if sourceErr != nil || preparedSourceScope != assetDownloadScope(confSyncSnapshot.Provider, currentSourceConf, Conf.Repo.Load().Key) {
logging.LogInfof("skip prepared boot sync after repository source changed")
return
}
repo, repoErr := newSyncRepository()
if repoErr != nil {
syncErr = repoErr
logging.LogErrorf("reopen prepared boot repository failed: %s", repoErr)
return
}
logging.LogInfof("syncing prepared boot data repo [device=%s, kernel=%s, provider=%d]", Conf.System.Load().ID, KernelID, confSyncSnapshot.Provider)
syncStart := time.Now()
indexStart := time.Now()
_, _, syncErr = indexRepoBeforeCloudSync(repo)
indexElapsed := time.Since(indexStart)
if nil == syncErr {
syncErr = syncIndexedRepoAfterBootWithDNSRetry(repo, syncStart, indexElapsed, prefetchTraffic)
}
if syncErr != nil {
logging.LogErrorf("boot background sync repo failed: %s", syncErr)
return
}
}()
}
return
}
func syncRepo(exit, byHand bool) (cloudPublished bool, err error) {
return syncRepoWithContext(context.Background(), exit, byHand)
}
func syncRepoWithContext(ctx context.Context, exit, byHand bool) (cloudPublished bool, err error) {
if err = ctx.Err(); err != nil {
return
}
if err = checkDataImportRecovery(); err != nil {
return
}
if 1 > len(Conf.Repo.Load().Key) {
autoSyncErrCount.Add(1)
planSyncAfter(fixSyncInterval)
msg := Conf.Language(26)
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
err = errors.New(msg)
return
}
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newSyncRepository()
if err != nil {
recordAutoSyncFailure(err)
planSyncAfter(fixSyncInterval)
msg := fmt.Sprintf("sync repo failed: %s", err)
logging.LogError(msg)
pushSyncStatusBar(msg)
util.PushErrMsg(msg, 0)
return
}
logging.LogInfof("syncing data repo [device=%s, kernel=%s, provider=%d, mode=%s/%t]", Conf.System.Load().ID, KernelID, Conf.Sync.Load().Provider, "a", byHand)
if !exit {
checkoutTask := startSyncCheckout(repo)
defer checkoutTask.finish(repo, &err)
} else {
hooks, finish := exitSyncHooks(repo, ctx)
repo.SetCheckoutHooks(hooks)
defer finish(&err)
}
start := time.Now()
indexStart := time.Now()
_, _, err = indexRepoBeforeCloudSync(repo)
indexElapsed := time.Since(indexStart)
if err != nil {
recordAutoSyncFailure(err)
planSyncAfter(fixSyncInterval)
logging.LogErrorf("sync data repo failed: %s", err)
msg := fmt.Sprintf(Conf.Language(80), formatRepoErrorMsg(err))
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
pushSyncStatusBar(msg)
if !errors.Is(err, context.Canceled) && (1 > autoSyncErrCount.Load() || byHand) {
util.PushErrMsg(msg, 0)
}
return
}
cloudPublished, err = syncIndexedRepo(repo, exit, byHand, start, indexElapsed, false, nil)
return
}
func syncIndexedRepo(repo *dejavu.Repo, exit, byHand bool, start time.Time, indexElapsed time.Duration, skipCloudPreflight bool, prefetchTraffic *dejavu.DownloadTrafficStat) (cloudPublished bool, err error) {
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
if !exit {
checkoutTask := startSyncCheckout(repo)
defer checkoutTask.finish(repo, &err)
defer finishAssetDownloadRecovery(repo, &err)
}
beforeSyncPetals := getPetals()
syncContext := newSyncContext()
if skipCloudPreflight {
// 启动同步已经读取过云端索引并预取了文件,锁内同步会再次校验最新版本。
syncContext["skipCloudPreflight"] = true
}
cloudStart := time.Now()
mergeResult, trafficStat, cloudPublished, err := syncRepoWithPublication(repo, syncContext)
cloudElapsed := time.Since(cloudStart)
elapsed := time.Since(start)
if err != nil {
recordAutoSyncFailure(err)
planSyncAfter(fixSyncInterval)
logging.LogErrorf("sync data repo failed: %s", err)
msg := formatSyncRepoErrorMsg(err)
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
pushSyncStatusBar(msg)
if !errors.Is(err, context.Canceled) && (1 > autoSyncErrCount.Load() || byHand) {
util.PushErrMsg(msg, 0)
}
return
}
if nil != prefetchTraffic {
trafficStat.DownloadFileCount += prefetchTraffic.DownloadFileCount
trafficStat.DownloadBytes += prefetchTraffic.DownloadBytes
trafficStat.PeerDownloadFileCount += prefetchTraffic.PeerDownloadFileCount
trafficStat.PeerDownloadBytes += prefetchTraffic.PeerDownloadBytes
trafficStat.PeerFallbackCount += prefetchTraffic.PeerFallbackCount
}
resetAutoSyncErrors()
pushSyncStatusBar(fmt.Sprintf(Conf.Language(149), elapsed.Seconds()))
UpdateSyncConf(func(s *conf.Sync) { s.Synced = util.CurrentTimeMillis() })
msg := fmt.Sprintf(Conf.Language(150), trafficStat.UploadFileCount, trafficStat.DownloadFileCount, trafficStat.UploadChunkCount, trafficStat.DownloadChunkCount, humanize.BytesCustomCeil(uint64(trafficStat.UploadBytes), 2), humanize.BytesCustomCeil(uint64(trafficStat.DownloadBytes+trafficStat.PeerDownloadBytes), 2))
msg = appendLANSyncTrafficStat(msg, trafficStat)
UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
calcPetalDiff(beforeSyncPetals, mergeResult)
postProcessStart := time.Now()
err = processAssetSyncMergeResult(repo, exit, byHand, mergeResult, trafficStat, "a", elapsed)
if nil == err && cloudPublished {
notifyLANSyncCommit(repo)
}
postProcessElapsed := time.Since(postProcessStart)
logging.LogInfof("sync data repo phases [index=%.2fs, cloud=%.2fs, post-process=%.2fs, total=%.2fs]",
indexElapsed.Seconds(), cloudElapsed.Seconds(), postProcessElapsed.Seconds(), time.Since(start).Seconds())
if !exit {
go func() {
// 首次数据同步执行完成后再执行索引订正 Index fixing should not be performed before data synchronization https://github.com/siyuan-note/siyuan/issues/10761
checkIndex()
// 索引订正结束后执行数据仓库清理 Automatic purge for local data repo https://github.com/siyuan-note/siyuan/issues/13091
autoPurgeRepo(false)
}()
}
return
}
func syncIndexedRepoAfterBootWithDNSRetry(repo *dejavu.Repo, start time.Time, indexElapsed time.Duration, prefetchTraffic *dejavu.DownloadTrafficStat) (err error) {
_, err = syncIndexedRepo(repo, false, false, start, indexElapsed, true, prefetchTraffic)
if nil != err && flushAndRetryOnDNSError(err) {
_, err = syncRepo(false, false)
}
return
}
func calcPetalDiff(beforeSyncPetals []*Petal, mergeResult *dejavu.MergeResult) {
var upsertPetals, removePetals []string
afterSyncPetals := getPetals()
for _, afterSyncPetal := range afterSyncPetals {
if beforeSyncPetal := getPetalByName(afterSyncPetal.Name, beforeSyncPetals); nil != beforeSyncPetal {
a, _ := gulu.JSON.MarshalJSON(afterSyncPetal)
b, _ := gulu.JSON.MarshalJSON(beforeSyncPetal)
if !bytes.Equal(a, b) {
upsertPetals = append(upsertPetals, afterSyncPetal.Name)
}
} else {
upsertPetals = append(upsertPetals, afterSyncPetal.Name)
}
}
for _, beforeSyncPetal := range beforeSyncPetals {
if nil == getPetalByName(beforeSyncPetal.Name, afterSyncPetals) {
removePetals = append(removePetals, beforeSyncPetal.Name)
}
}
mergeResult.UpsertPetals = gulu.Str.RemoveDuplicatedElem(upsertPetals)
mergeResult.RemovePetals = gulu.Str.RemoveDuplicatedElem(removePetals)
}
func processSyncMergeResult(exit, byHand bool, mergeResult *dejavu.MergeResult, trafficStat *dejavu.TrafficStat, mode string, elapsed time.Duration, immediateIndex ...bool) {
reloadSyncedDAV(mergeResult, &calendars, &contacts)
confSyncSnapshot := Conf.LoadSync()
logging.LogInfof("synced data repo [device=%s, kernel=%s, provider=%d, mode=%s/%t, ufc=%d, dfc=%d, ucc=%d, dcc=%d, ub=%s, db=%s, pfc=%d, pcc=%d, pb=%s, pf=%d] in [%.2fs], merge result [conflicts=%d, upserts=%d, removes=%d]\n\n",
Conf.System.Load().ID, KernelID, confSyncSnapshot.Provider, mode, byHand,
trafficStat.UploadFileCount, trafficStat.DownloadFileCount, trafficStat.UploadChunkCount, trafficStat.DownloadChunkCount, humanize.BytesCustomCeil(uint64(trafficStat.UploadBytes), 2), humanize.BytesCustomCeil(uint64(trafficStat.DownloadBytes), 2),
trafficStat.PeerDownloadFileCount, trafficStat.PeerDownloadChunkCount, humanize.BytesCustomCeil(uint64(trafficStat.PeerDownloadBytes), 2), trafficStat.PeerFallbackCount,
elapsed.Seconds(),
mergeResult.ConflictCount(), len(mergeResult.Upserts), len(mergeResult.Removes))
themes, icons := appearanceChangedPackages(mergeResult)
refreshAppearancePackages(themes, icons)
var needReloadFiletree bool
conflictCount := mergeResult.ConflictCount()
if 0 < conflictCount || mergeResult.HasHistory() {
luteEngine := util.NewLute()
if 0 < conflictCount && confSyncSnapshot.GenerateConflictDoc {
// 云端同步发生冲突时生成副本 https://github.com/siyuan-note/siyuan/issues/5687
conflictCopyFiles := mergeResult.ConflictCopyFiles()
for _, file := range conflictCopyFiles {
if !strings.HasSuffix(file.Path, ".sy") {
continue
}
parts := strings.Split(file.Path[1:], "/")
if 2 > len(parts) {
continue
}
boxID := parts[0]
absPath := filepath.Join(util.TempDir, "repo", "sync", "conflicts", mergeResult.Time.Format("2006-01-02-150405"), file.Path)
// 加密笔记本的冲突 .sy 在临时目录里是密文,loadTree 无法从 temp 路径反推 box 解密
if IsEncryptedBox(boxID) {
raw, readErr := os.ReadFile(absPath)
if readErr == nil {
data, decryptErr := decryptRepoDataIfNeeded(raw, file.Path)
if decryptErr != nil {
logging.LogErrorf("decrypt conflicted file [%s] failed: %s", absPath, decryptErr)
continue
}
if writeErr := os.WriteFile(absPath, data, 0644); writeErr != nil {
logging.LogErrorf("decrypt conflicted file [%s] failed: %s", absPath, writeErr)
continue
}
}
// 解密后的冲突文件在函数返回时清理
defer os.Remove(absPath)
}
tree, loadTreeErr := loadTree(absPath, luteEngine)
if nil != loadTreeErr {
logging.LogErrorf("load conflicted file [%s] failed: %s", absPath, loadTreeErr)
continue
}
tree.Box = boxID
tree.Path = strings.TrimPrefix(file.Path, "/"+boxID)
previousPath := tree.Path
resetTree(tree, "Conflicted", true)
createTreeTx(tree)
box := Conf.Box(boxID)
if nil != box {
box.addSort(previousPath, tree.ID)
}
}
needReloadFiletree = 0 < len(conflictCopyFiles)
}
if mergeResult.HasHistory() {
historyDir := filepath.Join(util.HistoryDir, mergeResult.Time.Format("2006-01-02-150405")+"-sync")
indexHistoryDir(filepath.Base(historyDir), luteEngine)
}
}
if !mergeResult.DataChanged() { // 没有数据变更
cleanupSyncedBoxResiduals(nil)
syncSameCount.Add(1)
if 10 < syncSameCount.Load() {
syncSameCount.Store(5)
}
if !byHand {
delay := time.Minute * time.Duration(int(math.Pow(2, float64(syncSameCount.Load()))))
if fixSyncInterval.Minutes() > delay.Minutes() {
delay = time.Minute * 8
}
planSyncAfter(delay)
}
return
}
// 有数据变更,需要重建索引
var upserts, removes []string
changedAttributeViewPaths := map[string]bool{}
var upsertTrees int
// 可能需要重新加载部分功能
var needReloadFlashcard, needReloadInlineStyles, needReloadOcrTexts, needReloadPlugin, needReloadSnippet bool
reloadPluginSet := hashset.New() // 插件代码变更 data/plugins/
dataChangePluginSet := hashset.New() // 插件存储数据变更 data/storage/petal/
needUnindexBoxes, needIndexBoxes := map[string]bool{}, map[string]bool{}
removedBoxConfs, removedBoxCryptoBackups := map[string]bool{}, map[string]bool{}
for _, file := range mergeResult.Upserts {
upserts = append(upserts, file.Path)
if file.Path == "/storage/pinned-docs.json" {
needReloadFiletree = true
}
if strings.HasPrefix(file.Path, "/storage/riff/") {
needReloadFlashcard = true
}
if strings.HasPrefix(file.Path, "/assets/ocr-texts.json") {
needReloadOcrTexts = true
}
if strings.HasSuffix(file.Path, "/.siyuan/conf.json") {
needReloadFiletree = true
boxID := strings.TrimSuffix(strings.TrimPrefix(file.Path, "/"), "/.siyuan/conf.json")
if ast.IsNodeIDPattern(boxID) {
forgetRuntimeNormalBox(boxID)
needUnindexBoxes[boxID] = true
needIndexBoxes[boxID] = true
}
}
if strings.HasSuffix(file.Path, "/.siyuan/boxDoc.json") {
needReloadFiletree = true
boxID := strings.TrimSuffix(strings.TrimPrefix(file.Path, "/"), "/.siyuan/boxDoc.json")
if ast.IsNodeIDPattern(boxID) {
needUnindexBoxes[boxID] = true
needIndexBoxes[boxID] = true
}
}
if strings.HasPrefix(file.Path, "/storage/petal/") {
needReloadPlugin = true
if parts := strings.Split(file.Path, "/"); 3 < len(parts) {
if pluginName := parts[3]; "petals.json" != pluginName {
dataChangePluginSet.Add(pluginName)
}
}
}
if strings.HasPrefix(file.Path, "/plugins/") {
if parts := strings.Split(file.Path, "/"); 2 < len(parts) {
needReloadPlugin = true
reloadPluginSet.Add(parts[2])
}
}
if strings.HasSuffix(file.Path, ".sy") {
upsertTrees++
}
if util.IsMobileContainer() && strings.HasPrefix(file.Path, "/assets/") {
absPath := filepath.Join(util.DataDir, file.Path)
HandleAssetsChangeEvent(absPath)
}
if file.Path == "/snippets/conf.json" {
needReloadSnippet = true
}
if isInlineStylesRepoPath(file.Path) {
needReloadInlineStyles = true
}
if strings.Contains(file.Path, "/storage/av/") && strings.HasSuffix(file.Path, ".json") {
cache.RemoveAVData(strings.TrimSuffix(filepath.Base(file.Path), ".json"))
changedAttributeViewPaths[file.Path] = true
}
}
// 每次同步后都按磁盘实际状态尝试恢复,不能只依赖本轮合并结果是否包含备份文件。
// 备份可能在此前同步中已经落盘,或因冲突等原因未出现在 Upserts 中。
restoreNotebookCryptoConfigFromBackup()
removePluginDirSet, removeWidgetDirSet := hashset.New(), hashset.New()
unloadPluginSet, uninstallPluginSet := hashset.New(), hashset.New()
// 插件目录内有文件被删除时先记为更新,遍历结束后按“目录是否仍含文件”再决定是否卸载
pendingPluginUninstallNames := hashset.New()
for _, file := range mergeResult.Removes {
removes = append(removes, file.Path)
if file.Path == "/storage/pinned-docs.json" {
needReloadFiletree = true
}
if strings.HasPrefix(file.Path, "/storage/riff/") {
needReloadFlashcard = true
}
if strings.HasPrefix(file.Path, "/assets/ocr-texts.json") {
needReloadOcrTexts = true
}
if strings.HasSuffix(file.Path, "/.siyuan/conf.json") {
needReloadFiletree = true
boxID := strings.TrimSuffix(strings.TrimPrefix(file.Path, "/"), "/.siyuan/conf.json")
if ast.IsNodeIDPattern(boxID) {
needUnindexBoxes[boxID] = true
removedBoxConfs[boxID] = true
}
}
if strings.HasSuffix(file.Path, "/.siyuan/"+notebookCryptoBackupFilename) {
boxID := strings.TrimSuffix(strings.TrimPrefix(file.Path, "/"), "/.siyuan/"+notebookCryptoBackupFilename)
if ast.IsNodeIDPattern(boxID) {
removedBoxCryptoBackups[boxID] = true
}
}
if strings.HasSuffix(file.Path, "/.siyuan/boxDoc.json") {
needReloadFiletree = true
boxID := strings.TrimSuffix(strings.TrimPrefix(file.Path, "/"), "/.siyuan/boxDoc.json")
if ast.IsNodeIDPattern(boxID) {
needUnindexBoxes[boxID] = true
needIndexBoxes[boxID] = true
}
}
if strings.HasPrefix(file.Path, "/storage/petal/") {
needReloadPlugin = true
if parts := strings.Split(file.Path, "/"); 3 < len(parts) {
if pluginName := parts[3]; "petals.json" != pluginName {
dataChangePluginSet.Add(pluginName)
}
}
}
if strings.HasPrefix(file.Path, "/plugins/") {
if parts := strings.Split(file.Path, "/"); 2 < len(parts) {
needReloadPlugin = true
// 删除插件目录内的文件:先按更新处理,遍历结束后按目录是否仍含文件决定是否卸载
reloadPluginSet.Add(parts[2])
pendingPluginUninstallNames.Add(parts[2])
}
}
if strings.HasPrefix(file.Path, "/widgets/") {
if parts := strings.Split(file.Path, "/"); 2 < len(parts) {
removeWidgetDirSet.Add(parts[2])
}
}
if util.IsMobileContainer() && strings.HasPrefix(file.Path, "/assets/") {
absPath := filepath.Join(util.DataDir, file.Path)
HandleAssetsRemoveEvent(absPath)
}
if file.Path == "/snippets/conf.json" {
needReloadSnippet = true
}
if isInlineStylesRepoPath(file.Path) {
needReloadInlineStyles = true
}
if strings.Contains(file.Path, "/storage/av/") && strings.HasSuffix(file.Path, ".json") {
cache.RemoveAVData(strings.TrimSuffix(filepath.Base(file.Path), ".json"))
changedAttributeViewPaths[file.Path] = true
}
}
// 插件目录内有文件被删除不代表插件被卸载:仅当目录不再包含实际文件时才视为卸载,
// 否则按更新重载,避免更新过程中的文件改名被误判为卸载
for _, name := range pendingPluginUninstallNames.Values() {
pluginName := name.(string)
pluginDir := filepath.Join(util.DataDir, "plugins", pluginName)
containsFile, statErr := bazaar.PackageDirContainsFile(pluginDir)
if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {
logging.LogWarnf("check plugin dir [%s] failed: %s", pluginDir, statErr)
continue
}
if errors.Is(statErr, os.ErrNotExist) || !containsFile {
// 插件目录已消失或不再包含文件:卸载
uninstallPluginSet.Add(pluginName)
removePluginDirSet.Add(pluginName)
}
}
for _, upsertPetal := range mergeResult.UpsertPetals {
needReloadPlugin = true
reloadPluginSet.Add(upsertPetal)
}
for _, removePetal := range mergeResult.RemovePetals {
needReloadPlugin = true
// Petal 中删除插件:卸载
uninstallPluginSet.Add(removePetal)
}
if needReloadFlashcard {
LoadFlashcards()
}
if needReloadOcrTexts {
util.LoadAssetsTexts()
}
if needReloadPlugin {
PushReloadPlugin(uninstallPluginSet, unloadPluginSet, reloadPluginSet, dataChangePluginSet, "",
DataChangeReasonSync)
}
if needReloadSnippet {
PushReloadSnippet(Conf.Snippet.Load())
}
removeEmptyPackageDirs(filepath.Join(util.DataDir, "plugins"), removePluginDirSet)
removeEmptyPackageDirs(filepath.Join(util.DataDir, "widgets"), removeWidgetDirSet)
syncingFiles = sync.Map{}
if needReloadInlineStyles && !exit {
invalidateWorkspaceAVPaletteCache()
util.BroadcastByType("main", "reloadInlineStyles", 0, "", nil)
util.ReloadPublishServiceSessions()
}
removedEncryptedBox := false
for boxID := range removedBoxConfs {
if IsEncryptedBox(boxID) || removedBoxCryptoBackups[boxID] {
finalizeSyncedEncryptedBoxRemoval(boxID)
unindex(boxID)
removedEncryptedBox = true
needUnindexBoxes[boxID] = true
} else {
forgetRuntimeNormalBox(boxID)
}
delete(needIndexBoxes, boxID)
}
cleanupSyncedBoxResiduals(removedBoxConfs)
if removedEncryptedBox {
sql.FlushQueue()
}
if !exit && ocrModelFilesChanged(upserts, removes) {
notifyOCRModelsChanged()
}
// 需要确认恢复记录时,索引更新必须在当前调用内完成
if needFullReindex(upsertTrees) && (len(immediateIndex) == 0 || !immediateIndex[0]) {
FullReindex(false)
return
}
if exit { // 退出时同步不用推送事件
return
}
for boxID := range needUnindexBoxes {
if len(immediateIndex) != 0 && immediateIndex[0] {
unindex(boxID)
} else {
(&Box{ID: boxID}).Unindex()
}
}
for boxID := range needIndexBoxes {
if box := Conf.GetBox(boxID); nil != box {
if box.Encrypted && box.Closed {
continue
}
if _, err := EnsureBoxDoc(boxID); nil != err {
logging.LogErrorf("ensure box document [%s] after sync failed: %s", boxID, err)
}
if len(immediateIndex) != 0 && immediateIndex[0] {
removeBoxRefs(boxID)
indexBox(boxID)
} else {
box.Index()
}
}
}
needReloadUI := 0 < len(needUnindexBoxes) || 0 < len(needIndexBoxes)
if needReloadUI && len(immediateIndex) != 0 && immediateIndex[0] {
// 应用保护内完成笔记本和引用索引,不将这些任务留到恢复编辑之后。
sql.FlushQueue()
IndexRefs()
sql.FlushQueue()
ResetVirtualBlockRefCache()
}
if needReloadUI {
util.ReloadUI()
}
upsertRootIDs, removeRootIDs := incReindex(upserts, removes)
if len(immediateIndex) != 0 && immediateIndex[0] {
sql.FlushQueue()
}
for changedPath := range changedAttributeViewPaths {
queueExternalAttributeViewRefIndexByRepoPath(changedPath)
}
needReloadFiletree = !needReloadUI && (needReloadFiletree || 0 < len(upsertRootIDs) || 0 < len(removeRootIDs))
if needReloadFiletree {
ReloadFiletree()
}
// 先通知编辑器刷新,再发送任务结束;前端会等待实际渲染完成才恢复输入。
if 0 < len(upsertRootIDs) || 0 < len(removeRootIDs) {
util.BroadcastByType("main", "syncMergeResult", 0, "",
map[string]any{"upsertRootIDs": upsertRootIDs, "removeRootIDs": removeRootIDs})
}
go func() {
util.WaitForUILoaded()
time.Sleep(2 * time.Second)
if mode != "c" {
pushSyncStatusBar(fmt.Sprintf(Conf.Language(149), elapsed.Seconds()))
}
if 0 < mergeResult.ConflictCount() {
syConflict := false
for _, path := range mergeResult.ConflictPaths() {
if strings.HasSuffix(path, ".sy") {
syConflict = true
break
}
}
if syConflict {
// 数据同步发生冲突时在界面上进行提醒 https://github.com/siyuan-note/siyuan/issues/7332
util.PushMsg(Conf.Language(108), 7000)
}
}
}()
}
// removeEmptyPackageDirs 清理同步删除文件后遗留的空集市包目录。
func removeEmptyPackageDirs(basePath string, dirNames *hashset.Set) {
for _, dirName := range dirNames.Values() {
dirPath := filepath.Join(basePath, dirName.(string))
if _, err := removeEmptyDirectoryTree(dirPath); err != nil && !os.IsNotExist(err) {
logging.LogWarnf("remove empty marketplace package directory [%s] failed: %s", dirPath, err)
}
}
}
// cleanupSyncedBoxResiduals 备份并删除同步确认移除的笔记本目录,以及不含正文的历史残留目录。
func cleanupSyncedBoxResiduals(removedBoxIDs map[string]bool) {
boxIDs := map[string]bool{}
for boxID := range removedBoxIDs {
boxIDs[boxID] = true
}
dirs, err := os.ReadDir(util.DataDir)
if err != nil {
logging.LogErrorf("read data dir before cleaning up synced box residuals failed: %s", err)
return
}
for _, dir := range dirs {
boxID := dir.Name()
if !dir.IsDir() || !ast.IsNodeIDPattern(boxID) || boxIDs[boxID] {
continue
}
boxDirPath := filepath.Join(util.DataDir, boxID)
if filelock.IsExist(filepath.Join(boxDirPath, ".siyuan", "conf.json")) {
continue
}
hasDocuments, scanErr := hasLiveBoxDocuments(boxDirPath)
if scanErr != nil {
logging.LogErrorf("scan synced box residual [%s] failed: %s", boxDirPath, scanErr)
continue
}
if hasDocuments || IsEncryptedBox(boxID) {
continue
}
boxIDs[boxID] = true
}
var historyDir string
for boxID := range boxIDs {
if !ast.IsNodeIDPattern(boxID) {
continue
}
boxDirPath := filepath.Join(util.DataDir, boxID)
if !filelock.IsExist(boxDirPath) {
continue
}
if !gulu.File.IsDir(boxDirPath) {
logging.LogWarnf("refuse to clean up synced box residual because it is not a directory [%s]", boxDirPath)
continue
}
if "" == historyDir {
var err error
historyDir, err = getHistoryDir(HistoryOpDelete)
if err != nil {
logging.LogErrorf("create history before cleaning up synced box residuals failed: %s", err)
return
}
}
historyPath := filepath.Join(historyDir, boxID)
if err := filelock.Copy(boxDirPath, historyPath); err != nil {
logging.LogErrorf("backup synced box residual [%s] failed: %s", boxDirPath, err)
continue
}
if err := removeBoxDir(boxDirPath); err != nil {
logging.LogErrorf("remove synced box residual [%s] failed: %s", boxDirPath, err)
continue
}
logging.LogInfof("removed synced box residual [%s]", boxDirPath)
}
}
func appendLANSyncTrafficStat(message string, trafficStat *dejavu.TrafficStat) string {
confSyncReadSnapshot := Conf.LoadSync()
if nil == confSyncReadSnapshot || nil == confSyncReadSnapshot.LAN || !confSyncReadSnapshot.LAN.Enabled {
return message
}
traffic := humanize.BytesCustomCeil(uint64(trafficStat.PeerDownloadBytes), 2)
return message + `<br data-type="lanSyncTraffic"> ` + fmt.Sprintf(Conf.Language(370), traffic)
}
func removeLANSyncTrafficStat(message string) string {
const marker = `<br data-type="lanSyncTraffic">`
if before, _, ok := strings.Cut(message, marker); ok {
return before
}
// 兼容已保存的无标记局域网流量统计。
const separator = "<br> "
index := strings.LastIndex(message, separator)
if 0 > index {
return message
}
lastLine := message[index+len(separator):]
for _, language := range util.Langs {
format := language[370]
prefix := strings.TrimSuffix(format, "%s")
if prefix != format && strings.HasPrefix(lastLine, prefix) {
return message[:index]
}
}
return message
}
func needFullReindex(upsertTrees int) bool {
return 0.2 < float64(upsertTrees)/float64(treenode.CountTrees())
}
var promotedPurgeDataRepo bool
func indexRepoBeforeCloudSync(repo *dejavu.Repo) (beforeIndex, afterIndex *entity.Index, err error) {
if err = checkRepositoryPasswordMigration(); err != nil {
return
}
return indexRepoBeforeCloudSync0(repo)
}
// Download-only recovery keeps the existing local safety index, without publishing it.
func indexRepoBeforeCloudSync0(repo *dejavu.Repo) (beforeIndex, afterIndex *entity.Index, err error) {
start := time.Now()
beforeIndex, _ = repo.Latest()
FlushTxQueue()
if err = processAssetDownloadRecovery(repo, true); err != nil {
return
}
checkChunks := true
if util.IsMobileContainer() {
// 因为移动端私有数据空间不会存在外部操作导致分块损坏的情况,所以不需要检查分块以提升性能 https://github.com/siyuan-note/siyuan/issues/13216
checkChunks = false
}
afterIndex, err = repo.Index("[Sync] Cloud sync", checkChunks, newSyncContext())
if errors.Is(err, dejavu.ErrIndexFileChanged) {
// 索引期间工作空间文件被修改属于瞬时竞态,重试一次以避免同步无谓中止
logging.LogWarnf("index data repo before cloud sync aborted because files changed, retry once: %s", err)
afterIndex, err = repo.Index("[Sync] Cloud sync", checkChunks, newSyncContext())
}
if err != nil {
logging.LogErrorf("index data repo before cloud sync failed: %s", err)
return
}
elapsed := time.Since(start)
if nil == beforeIndex || beforeIndex.ID != afterIndex.ID {
// 对新创建的快照需要更新备注,加入耗时统计
afterIndex.Memo = fmt.Sprintf("[Sync] Cloud sync, completed in %.2fs", elapsed.Seconds())
if err = repo.PutIndex(afterIndex); err != nil {
pushSyncStatusBar("Save data snapshot for cloud sync failed")
logging.LogErrorf("put index into data repo before cloud sync failed: %s", err)
return
}
pushSyncStatusBar(fmt.Sprintf(Conf.Language(147), elapsed.Seconds()))
} else {
pushSyncStatusBar(fmt.Sprintf(Conf.Language(148), elapsed.Seconds()))
}
if Conf.Repo.Load().SyncIndexTiming < elapsed.Milliseconds() {
logging.LogWarnf("index data repo before cloud sync elapsed [%dms]", elapsed.Milliseconds())
if !promotedPurgeDataRepo {
go func() {
util.WaitForUILoaded()
time.Sleep(3 * time.Second)
if indexCount, _ := repo.CountIndexes(); 128 > indexCount {
// 快照数量较少时不推送提示
return
}
util.PushMsg(Conf.language(218), 24000)
promotedPurgeDataRepo = true
}()
}
}
return
}
func newRepository() (ret *dejavu.Repo, err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
return newRepositoryWithAssetSourceLocked()
}
// newCloudRepositoryWithAssetSourceLocked 在访问云端前校验配置,保留未配置云端时的本地快照功能。
func newCloudRepositoryWithAssetSourceLocked() (*dejavu.Repo, error) {
confSyncSnapshot := Conf.LoadSync()
var err error
switch confSyncSnapshot.Provider {
case conf.ProviderS3:
err = validateSyncS3(confSyncSnapshot.S3)
case conf.ProviderWebDAV:
err = validateSyncWebDAV(confSyncSnapshot.WebDAV)
case conf.ProviderLocal:
err = validateSyncLocal(confSyncSnapshot.Local)
}
if err != nil {
return nil, err
}
return newRepositoryWithAssetSourceLocked()
}
// newRepositoryWithAssetSourceLocked 由已持有来源锁的调用方创建仓库,避免读写锁递归等待。
func newRepositoryWithAssetSourceLocked() (ret *dejavu.Repo, err error) {
confRepoSnapshot := Conf.LoadRepo()
confSyncSnapshot := Conf.LoadSync()
confSystemSnapshot := Conf.LoadSystem()
if err = recoverSortConfAppends(); err != nil {
return
}
cloudConf, err := buildCloudConf()
if err != nil {
return
}
var cloudRepo cloud.Cloud
switch confSyncSnapshot.Provider {
case conf.ProviderSiYuan:
cloudRepo = cloud.NewSiYuan(&cloud.BaseCloud{Conf: cloudConf})
case conf.ProviderS3:
// 显式注入 SiYuan UA,覆盖 aws SDK 默认 UA(含架构、Go 版本、SDK 版本等冗余信息),便于 S3 服务端按 SiYuan/ 前缀识别加白名单
s3HTTPClient := httpclient.NewUserAgentClient(httpclient.NewTransport(cloudConf.S3.SkipTlsVerify))
s3HTTPClient.Timeout = time.Duration(cloudConf.S3.Timeout) * time.Second
cloudRepo = cloud.NewS3(&cloud.BaseCloud{Conf: cloudConf}, s3HTTPClient)
case conf.ProviderWebDAV:
webdavClient := gowebdav.NewClient(cloudConf.WebDAV.Endpoint, cloudConf.WebDAV.Username, cloudConf.WebDAV.Password)
logging.LogInfof("WebDAV sync endpoint [%s], directory [%q]", safeWebDAVEndpoint(cloudConf.WebDAV.Endpoint), cloudConf.Dir)
a := cloudConf.WebDAV.Username + ":" + cloudConf.WebDAV.Password
auth := "Basic " + base64.StdEncoding.EncodeToString([]byte(a))
webdavClient.SetHeader("Authorization", auth)
webdavClient.SetHeader("User-Agent", util.UserAgent)
webdavClient.SetTimeout(time.Duration(cloudConf.WebDAV.Timeout) * time.Second)
webdavClient.SetTransport(httpclient.NewTransport(cloudConf.WebDAV.SkipTlsVerify))
cloudRepo = cloud.NewWebDAV(&cloud.BaseCloud{Conf: cloudConf}, webdavClient)
case conf.ProviderLocal:
cloudRepo = cloud.NewLocal(&cloud.BaseCloud{Conf: cloudConf})
default:
err = fmt.Errorf("unknown cloud provider [%d]", confSyncSnapshot.Provider)
return
}
ignoreLines, err := getSyncIgnoreLines()
if err != nil {
return nil, err
}
dataDir := util.DataDir
scope := assetDownloadScope(confSyncSnapshot.Provider, cloudConf, confRepoSnapshot.Key)
ret, err = dejavu.NewRepoWithOptions(dejavu.Options{
DataPath: util.DataDir, RepoPath: util.RepoDir, HistoryPath: util.HistoryDir, TempPath: util.TempDir,
DeviceID: confSystemSnapshot.ID, DeviceName: confSystemSnapshot.Name, DeviceOS: confSystemSnapshot.OS,
AESKey: confRepoSnapshot.Key, IgnoreLines: ignoreLines, Cloud: cloudRepo,
IgnoreRulePath: syncIgnoreRulePath, HiddenDirectoryNames: []string{".siyuan"},
OnCloudIndexList: newSyncCloudIndexListObserver(scope, confSyncSnapshot.Provider),
PathFilter: func(info os.FileInfo, absPath string) (bool, error) {
return syncPathFilter(dataDir, info, absPath)
},
})
if err != nil {
logging.LogErrorf("init data repo failed: %s", err)
return
}
if confSyncSnapshot.Provider == conf.ProviderSiYuan && Conf.GetUser() == nil {
stored, readErr := dejavu.ReadAssetDownloadScope(assetDownloadStatePath(), confRepoSnapshot.Key)
if readErr != nil {
return nil, readErr
}
if stored != "" {
scope = stored
}
}
ret.SetCheckoutHooks(documentAddressingCheckoutHooks(ret))
err = ret.ConfigureAssetDownloads(confSyncSnapshot.AssetDownloadMode == 1, assetDownloadStatePath(), scope)
return
}
func init() {
subscribeRepoEvents()
}
func subscribeRepoEvents() {
eventbus.Subscribe(eventbus.EvtIndexBeforeWalkData, func(context map[string]any, path string) {
msg := fmt.Sprintf(Conf.Language(158), path)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
indexWalkDataCount := 0
eventbus.Subscribe(eventbus.EvtIndexWalkData, func(context map[string]any, path string) {
msg := fmt.Sprintf(Conf.Language(158), filepath.Base(path))
if 0 == indexWalkDataCount%1024 {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
}
indexWalkDataCount++
})
eventbus.Subscribe(eventbus.EvtIndexBeforeGetLatestFiles, func(context map[string]any, total int) {
msg := fmt.Sprintf(Conf.Language(159), 0, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtIndexGetLatestFile, func(context map[string]any, count int, total int) {
msg := fmt.Sprintf(Conf.Language(159), count, total)
if 0 == count%64 {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtIndexUpsertFiles, func(context map[string]any, total int) {
msg := fmt.Sprintf(Conf.Language(160), 0, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtIndexUpsertFile, func(context map[string]any, count int, total int) {
msg := fmt.Sprintf(Conf.Language(160), count, total)
if 0 == count%32 {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtCheckoutBeforeWalkData, func(context map[string]any, path string) {
msg := fmt.Sprintf(Conf.Language(161), path)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCheckoutWalkData, func(context map[string]any, path string) {
msg := fmt.Sprintf(Conf.Language(161), filepath.Base(path))
if repoProgressDue("checkout-walk", false) {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtCheckoutUpsertFiles, func(context map[string]any, total int) {
msg := fmt.Sprintf(Conf.Language(162), 0, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCheckoutUpsertFile, func(context map[string]any, count, total int) {
msg := fmt.Sprintf(Conf.Language(162), count, total)
util.IncBootProgress(repoBootIncrement(count, total), msg)
if repoProgressDue("checkout-upsert", count >= total) {
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtCheckoutRemoveFiles, func(context map[string]any, total int) {
msg := fmt.Sprintf(Conf.Language(163), 0, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCheckoutRemoveFile, func(context map[string]any, count, total int) {
msg := fmt.Sprintf(Conf.Language(163), count, total)
util.IncBootProgress(repoBootIncrement(count, total), msg)
if repoProgressDue("checkout-remove", count >= total) {
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadIndex, func(context map[string]any, id string) {
msg := fmt.Sprintf(Conf.Language(164), id[:7])
util.IncBootProgress(1, msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadFiles, func(context map[string]any, total int) {
msg := fmt.Sprintf(Conf.Language(165), 0, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadFile, func(context map[string]any, count, total int) {
msg := fmt.Sprintf(Conf.Language(165), count, total)
util.IncBootProgress(repoBootIncrement(count, total), msg)
if repoProgressDue("download-files", count >= total) {
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadChunks, func(context map[string]any, total int) {
msg := fmt.Sprintf(Conf.Language(166), 0, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadChunk, func(context map[string]any, count, total int) {
msg := fmt.Sprintf(Conf.Language(166), count, total)
util.IncBootProgress(repoBootIncrement(count, total), msg)
if repoProgressDue("download-chunks", count >= total) {
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtCloudBeforeDownloadRef, func(context map[string]any, ref string) {
msg := fmt.Sprintf(Conf.Language(167), ref)
util.IncBootProgress(1, msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeUploadIndex, func(context map[string]any, id string) {
msg := fmt.Sprintf(Conf.Language(168), id[:7])
util.IncBootProgress(1, msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeUploadFiles, func(context map[string]any, total int) {
msg := fmt.Sprintf(Conf.Language(169), 0, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeUploadFile, func(context map[string]any, count, total int) {
msg := fmt.Sprintf(Conf.Language(169), count, total)
if 0 == count%8 {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtCloudBeforeUploadChunks, func(context map[string]any, total int) {
msg := fmt.Sprintf(Conf.Language(170), 0, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeUploadChunk, func(context map[string]any, count, total int) {
msg := fmt.Sprintf(Conf.Language(170), count, total)
if 0 == count%8 {
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
}
})
eventbus.Subscribe(eventbus.EvtCloudBeforeUploadRef, func(context map[string]any, ref string) {
if uploadingRef, ok := context[syncCloudRefContextKey].(*atomic.Bool); ok && "refs/latest" == ref {
uploadingRef.Store(true)
}
msg := fmt.Sprintf(Conf.Language(171), ref)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudLock, func(context map[string]any) {
msg := Conf.Language(186)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudUnlock, func(context map[string]any) {
msg := Conf.Language(187)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeUploadIndexes, func(context map[string]any) {
msg := Conf.Language(208)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeUploadCheckIndex, func(context map[string]any) {
msg := Conf.Language(209)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudBeforeFixObjects, func(context map[string]any, count, total int) {
msg := fmt.Sprintf(Conf.Language(210), count, total)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudAfterFixObjects, func(context map[string]any) {
msg := Conf.Language(211)
util.SetBootDetails(msg)
util.ContextPushMsg(context, msg)
})
eventbus.Subscribe(eventbus.EvtCloudCorrupted, func() {
util.PushErrMsg(Conf.language(220), 30000)
})
eventbus.Subscribe(eventbus.EvtCloudPurgeListObjects, func(context map[string]any) {
util.ContextPushMsg(context, Conf.language(224))
})
eventbus.Subscribe(eventbus.EvtCloudPurgeListIndexes, func(context map[string]any) {
util.ContextPushMsg(context, Conf.language(225))
})
eventbus.Subscribe(eventbus.EvtCloudPurgeListRefs, func(context map[string]any) {
util.ContextPushMsg(context, Conf.language(226))
})
eventbus.Subscribe(eventbus.EvtCloudPurgeDownloadIndexes, func(context map[string]any) {
util.ContextPushMsg(context, Conf.language(227))
})
eventbus.Subscribe(eventbus.EvtCloudPurgeDownloadFiles, func(context map[string]any) {
util.ContextPushMsg(context, Conf.language(228))
})
eventbus.Subscribe(eventbus.EvtCloudPurgeRemoveIndexes, func(context map[string]any) {
util.ContextPushMsg(context, Conf.language(229))
})
eventbus.Subscribe(eventbus.EvtCloudPurgeRemoveIndexesV2, func(context map[string]any) {
util.ContextPushMsg(context, Conf.language(230))
})
eventbus.Subscribe(eventbus.EvtCloudPurgeRemoveObjects, func(context map[string]any) {
util.ContextPushMsg(context, Conf.language(231))
})
}
func buildCloudConf() (ret *cloud.Conf, err error) {
sync := Conf.Sync.Load()
if conf.ProviderS3 != sync.Provider && !cloud.IsValidCloudDirName(sync.CloudName) {
logging.LogWarnf("invalid cloud repo name, rename it to [main]")
sync = UpdateSyncConf(func(s *conf.Sync) { s.CloudName = "main" })
}
userId, token, availableSize := "0", "", int64(1024*1024*1024*1024*2)
if u := Conf.GetUser(); u != nil && conf.ProviderSiYuan == sync.Provider {
userId = u.UserId
token = u.UserToken
availableSize = u.GetCloudRepoAvailableSize()
}
ret = &cloud.Conf{
Dir: sync.CloudName,
UserID: userId,
Token: token,
AvailableSize: availableSize,
Server: util.GetCloudServer(),
}
switch sync.Provider {
case conf.ProviderSiYuan:
ret.Endpoint = util.GetCloudSyncServer()
case conf.ProviderS3:
ret.S3 = &cloud.ConfS3{
Endpoint: sync.S3.Endpoint,
AccessKey: sync.S3.AccessKey,
SecretKey: sync.S3.SecretKey,
Bucket: sync.S3.Bucket,
Region: sync.S3.Region,
PathStyle: sync.S3.PathStyle,
SkipTlsVerify: sync.S3.SkipTlsVerify,
Timeout: sync.S3.Timeout,
ConcurrentReqs: sync.S3.ConcurrentReqs,
}
case conf.ProviderWebDAV:
ret.WebDAV = &cloud.ConfWebDAV{
Endpoint: sync.WebDAV.Endpoint,
Username: sync.WebDAV.Username,
Password: sync.WebDAV.Password,
SkipTlsVerify: sync.WebDAV.SkipTlsVerify,
Timeout: sync.WebDAV.Timeout,
ConcurrentReqs: sync.WebDAV.ConcurrentReqs,
}
case conf.ProviderLocal:
ret.Local = &cloud.ConfLocal{
Endpoint: sync.Local.Endpoint,
Timeout: sync.Local.Timeout,
ConcurrentReqs: sync.Local.ConcurrentReqs,
}
default:
err = fmt.Errorf("invalid provider [%d]", sync.Provider)
return
}
return
}
type Backup struct {
Size int64 `json:"size"`
HSize string `json:"hSize"`
Updated string `json:"updated"`
SaveDir string `json:"saveDir"` // 本地备份数据存放目录路径
}
type Sync struct {
Size int64 `json:"size"`
HSize string `json:"hSize"`
Updated string `json:"updated"`
CloudName string `json:"cloudName"` // 云端同步数据存放目录名
SaveDir string `json:"saveDir"` // 本地同步数据存放目录路径
}
func GetCloudSpace() (s *Sync, b *Backup, hSize, hAssetSize, hTotalSize, hExchangeSize, hTrafficUploadSize, hTrafficDownloadSize, hTrafficAPIGet, hTrafficAPIPut string, err error) {
stat, err := getCloudSpace()
if err != nil {
err = errors.New(Conf.Language(30) + " " + err.Error())
return
}
syncSize := stat.Sync.Size
syncUpdated := stat.Sync.Updated
s = &Sync{
Size: syncSize,
HSize: "-",
Updated: syncUpdated,
}
backupSize := stat.Backup.Size
backupUpdated := stat.Backup.Updated
b = &Backup{
Size: backupSize,
HSize: "-",
Updated: backupUpdated,
}
assetSize := stat.AssetSize
totalSize := syncSize + backupSize + assetSize
hAssetSize = "-"
hSize = "-"
hTotalSize = "-"
hExchangeSize = "-"
hTrafficUploadSize = "-"
hTrafficDownloadSize = "-"
hTrafficAPIGet = "-"
hTrafficAPIPut = "-"
if conf.ProviderSiYuan == Conf.Sync.Load().Provider {
s.HSize = humanize.BytesCustomCeil(uint64(syncSize), 2)
b.HSize = humanize.BytesCustomCeil(uint64(backupSize), 2)
hAssetSize = humanize.BytesCustomCeil(uint64(assetSize), 2)
hSize = humanize.BytesCustomCeil(uint64(totalSize), 2)
if u := Conf.GetUser(); nil != u {
hTotalSize = humanize.BytesCustomCeil(uint64(u.UserSiYuanRepoSize), 2)
hExchangeSize = humanize.BytesCustomCeil(uint64(u.UserSiYuanPointExchangeRepoSize), 2)
hTrafficUploadSize = humanize.BytesCustomCeil(uint64(u.UserTrafficUpload), 2)
hTrafficDownloadSize = humanize.BytesCustomCeil(uint64(u.UserTrafficDownload), 2)
hTrafficAPIGet = humanize.SIWithDigits(u.UserTrafficAPIGet, 2, "")
hTrafficAPIPut = humanize.SIWithDigits(u.UserTrafficAPIPut, 2, "")
}
}
return
}
func getCloudSpace() (stat *cloud.Stat, err error) {
assetDownloadSourceMu.RLock()
defer assetDownloadSourceMu.RUnlock()
handleCloudError := cloudRepoErrorHandler()
defer func() { handleCloudError(err) }()
repo, err := newCloudRepositoryWithAssetSourceLocked()
if err != nil {
return
}
stat, err = repo.GetCloudRepoStat()
if err != nil {
logging.LogErrorf("get cloud repo stat failed: %s", err)
return
}
return
}