Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

36.6 KB
Raw Blame
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later

package model

import (
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/88250/go-humanize"
	"github.com/88250/gulu"
	"github.com/88250/lute/html"
	"github.com/gorilla/websocket"
	"github.com/siyuan-note/dejavu"
	"github.com/siyuan-note/dejavu/cloud"
	"github.com/siyuan-note/eventbus"
	"github.com/siyuan-note/logging"
	"github.com/siyuan-note/siyuan/kernel/cache"
	"github.com/siyuan-note/siyuan/kernel/conf"
	"github.com/siyuan-note/siyuan/kernel/filesys"
	"github.com/siyuan-note/siyuan/kernel/sql"
	"github.com/siyuan-note/siyuan/kernel/treenode"
	"github.com/siyuan-note/siyuan/kernel/util"
)

func SyncDataDownload() (err error) {
	err = errors.New("sync download did not complete")
	defer logging.Recover()

	if !checkSync(false, false, true) {
		return
	}

	unlock, ok := lockSyncRequest(&syncDownloadRequests)
	if !ok {
		return
	}
	defer unlock()
	err = syncDataDownloadLocked()
	return
}

func syncDataDownloadLocked() (err error) {
	util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)

	now := util.CurrentTimeMillis()
	UpdateSyncConf(func(s *conf.Sync) { s.Synced = now })

	err = syncRepoDownloadWithDNSRetry()
	code := 1
	if err != nil {
		code = 2
	}
	util.BroadcastByType("main", "syncing", code, Conf.Sync.Load().Stat, nil)
	// Download-only completion does not publish local changes, including an empty remote.
	// Keep the pending hint until an upload or full sync actually publishes them.
	if 1 == code {
		consumeShorthands()
	}
	return
}

func getSyncCloudLatestID() (ret string) {
	assetDownloadSourceMu.RLock()
	defer assetDownloadSourceMu.RUnlock()
	// 同步感知消息不包含云端提交 ID,需要先读取最新索引,以便和局域网提交提示使用同一个去重键。
	handleCloudError := cloudRepoErrorHandler()
	repo, err := newSyncRepositoryWithAssetSourceLocked()
	if nil != err {
		logging.LogWarnf("create repo before perceived sync failed: %s", err)
		return
	}
	syncContext := map[string]any{eventbus.CtxPushMsg: eventbus.CtxPushMsgToNone}
	latest, err := repo.GetCloudLatestFast(syncContext)
	handleCloudError(err)
	if nil != err {
		logging.LogWarnf("get cloud latest before perceived sync failed: %s", err)
		return
	}
	if nil != latest {
		ret = latest.ID
	}
	return
}

func completeCurrentSyncRemoteRequest(scope string) {
	// 完整同步可能在合并本地变更后生成新的最新索引,同时记录实际结果可覆盖两类通知到达顺序相反的情况。
	repo, err := newRepository()
	if nil != err {
		logging.LogWarnf("create repo after remote sync failed: %s", err)
		return
	}
	latest, err := repo.Latest()
	if nil != err {
		logging.LogWarnf("get local latest after remote sync failed: %s", err)
		return
	}
	syncRemoteRequests.complete(scope, latest.ID)
}

func SyncDataUpload() (err error) {
	err = errors.New("sync upload did not complete")
	defer logging.Recover()

	if !checkSync(false, false, true) {
		return
	}

	unlock, ok := lockSyncRequest(&syncUploadRequests)
	if !ok {
		return
	}
	defer unlock()
	revision := pendingSync.begin()
	util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)

	now := util.CurrentTimeMillis()
	UpdateSyncConf(func(s *conf.Sync) { s.Synced = now })

	err = syncRepoUploadWithDNSRetry()
	code := 1
	if err != nil {
		code = 2
	}
	util.BroadcastByType("main", "syncing", code, Conf.Sync.Load().Stat, nil)
	pendingSync.finish(revision, err == nil, notifySyncPending)
	return
}

var (
	pendingSync      syncPendingState
	syncSameCount    = atomic.Int32{}
	autoSyncErrCount = atomic.Int32{}
	fixSyncInterval  = 5 * time.Minute

	syncPlanTimeLock = sync.Mutex{}
	syncPlanTime     = time.Now().Add(fixSyncInterval)

	BootSyncSucc atomic.Int32 // -1:未执行,0:执行成功,1:执行失败
)

func init() {
	BootSyncSucc.Store(-1)
}

func SyncDataJob() {
	syncPlanTimeLock.Lock()
	if time.Now().Before(syncPlanTime) {
		syncPlanTimeLock.Unlock()
		return
	}
	syncPlanTimeLock.Unlock()

	SyncData(false)
}

func BootSyncData() {
	defer logging.Recover()
	if scheduleBootSync() {
		return
	}
	bootSyncData()
}

func bootSyncData() {
	refreshLANSyncManager()

	if Conf.Sync.Load().Perception {
		connectSyncWebSocket()
	}

	if !checkSync(true, false, false) {
		return
	}

	lockSync()
	defer unlockSync()
	if !checkSync(true, false, false) {
		return
	}

	afterBoot := util.IsBooted()
	if !afterBoot {
		util.IncBootProgress(3, Conf.Language(307))
	}
	BootSyncSucc.Store(-1)
	if afterBoot {
		logging.LogInfof("sync after boot")
	} else {
		logging.LogInfof("sync before boot")
	}

	now := util.CurrentTimeMillis()
	UpdateSyncConf(func(s *conf.Sync) { s.Synced = now })
	revision := pendingSync.begin()
	if afterBoot {
		// 本地界面已可编辑,保持同步入口可见直到云端合并成功。
		notifySyncPending(true)
	}
	util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
	var cloudPublished bool
	var err error
	if afterBoot {
		cloudPublished, err = syncRepoWithDNSRetry(false, false)
	} else {
		var scheduled bool
		scheduled, err = bootSyncRepoWithDNSRetry(func(err error) {
			finishBootSync(revision, false, err)
		})
		if scheduled {
			return
		}
	}
	finishBootSync(revision, cloudPublished, err)
}

func finishBootSync(revision uint64, cloudPublished bool, err error) {
	code := 1
	if err != nil {
		code = 2
		BootSyncSucc.Store(1)
		if util.IsBooted() && !errors.Is(err, context.Canceled) && !errors.Is(err, errSyncRemoteRequestSkipped) {
			msg := formatSyncRepoErrorMsg(err)
			UpdateSyncConf(func(s *conf.Sync) { s.Stat = msg })
			pushSyncStatusBar(msg)
			util.PushErrMsg(msg, 0)
		}
	} else {
		BootSyncSucc.Store(0)
	}
	util.BroadcastByType("main", "syncing", code, Conf.Sync.Load().Stat, nil)
	pendingSync.finish(revision, err == nil, notifySyncPending)
	if 1 == code {
		// 启动同步成功后消费本地速记临时文件,避免移动端开启云同步时需手动触发同步才能刷新闪念速记
		consumeShorthands()
	}
	if util.IsBooted() {
		completeSyncPerception(cloudPublished, err)
	}
}

func SyncData(byHand bool) error {
	return syncData(false, byHand)
}

// SyncDataBeforeEnableEncryptedNotebook 在启用加密笔记本前执行一次完整同步。
// 未启用数据同步时直接返回;已启用数据同步时,任何同步失败都会阻止继续创建新的密钥体系。
func SyncDataBeforeEnableEncryptedNotebook() error {
	confSyncSnapshot := Conf.LoadSync()

	if !confSyncSnapshot.Enabled {
		return nil
	}
	if conf.ProviderS3 != confSyncSnapshot.Provider && !cloud.IsValidCloudDirName(confSyncSnapshot.CloudName) {
		return errors.New(Conf.Language(123))
	}
	if !checkSync(false, false, true) {
		return errors.New(Conf.Language(53))
	}

	// 不复用请求合并状态,确保调用返回前确实完成了一次由当前启用操作发起的完整同步。
	lockSync()
	defer unlockSync()
	if err := syncDataLocked(false, true); err != nil {
		if confSyncSnapshot.Stat != "" {
			return errors.New(confSyncSnapshot.Stat)
		}
		return err
	}
	return nil
}

func lockSync() {
	syncLock.Lock()
	isSyncing.Store(true)
}

func unlockSync() {
	isSyncing.Store(false)
	syncLock.Unlock()
}

type syncRequestState struct {
	requested atomic.Uint64
	completed atomic.Uint64
}

func lockSyncRequest(state *syncRequestState) (unlock func(), ok bool) {
	request := state.requested.Add(1)
	lockSync()
	if util.IsExiting.Load() {
		unlockSync()
		return nil, false
	}
	if state.completed.Load() >= request {
		unlockSync()
		return nil, false
	}

	runRequests := state.requested.Load()
	unlock = func() {
		state.completed.Store(runRequests)
		unlockSync()
	}
	return unlock, true
}

func syncData(exit, byHand bool) (err error) {
	err = errors.New("sync did not complete")
	defer logging.Recover()

	if !checkSync(false, exit, byHand) {
		if byHand {
			return err
		}
		return nil
	}

	requests := &syncAutoRequests
	if byHand {
		requests = &syncManualRequests
	} else if exit {
		requests = &syncExitRequests
	}
	unlock, ok := lockSyncRequest(requests)
	if !ok {
		if byHand {
			return err
		}
		return nil
	}
	defer unlock()
	return syncDataLocked(exit, byHand)
}

func syncDataLocked(exit, byHand bool) error {
	return syncDataLockedContext(context.Background(), exit, byHand)
}

func syncDataLockedContext(ctx context.Context, exit, byHand bool) (err error) {
	err = errors.New("sync did not complete")
	defer logging.Recover()
	if err := ctx.Err(); err != nil {
		return err
	}
	if err := checkDataImportRecovery(); err != nil {
		return err
	}
	revision := pendingSync.begin()
	util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
	if exit {
		logging.LogInfof("sync before exit")
		msgId := util.PushMsg(Conf.Language(81), 1000*60*15)
		defer func() {
			util.PushClearMsg(msgId)
		}()
	}

	now := util.CurrentTimeMillis()
	UpdateSyncConf(func(s *conf.Sync) { s.Synced = now })

	cloudPublished, err := syncRepoWithDNSRetryContext(ctx, exit, byHand)
	code := 1
	if err != nil {
		code = 2
	} else {
		BootSyncSucc.Store(0)
	}
	util.BroadcastByType("main", "syncing", code, Conf.Sync.Load().Stat, nil)
	pendingSync.finish(revision, err == nil, notifySyncPending)

	if !exit && 1 == code {
		consumeShorthands()
	}

	finishSyncPerception(cloudPublished, err, !exit)
	return err
}

// completeSyncPerception 恢复感知连接,并在自动同步成功发布新版本后通知其他设备。
func completeSyncPerception(cloudPublished bool, err error) {
	finishSyncPerception(cloudPublished, err, true)
}

func finishSyncPerception(cloudPublished bool, err error, reconnect bool) {
	if errors.Is(err, context.Canceled) || util.IsExiting.Load() {
		return
	}
	confSyncSnapshot := Conf.LoadSync()

	if reconnect && nil == webSocketConn && confSyncSnapshot.Perception {
		// 如果 websocket 连接已经断开,则重新连接
		connectSyncWebSocket()
	}

	if nil == err && cloudPublished && 1 == confSyncSnapshot.Mode && nil != webSocketConn && confSyncSnapshot.Perception {
		connection := webSocketConn
		if !reconnect {
			// 退出只使用现有连接通知其他设备,限制写入等待,不重新连接云服务。
			connection.SetWriteDeadline(time.Now().Add(3 * time.Second))
			defer connection.SetWriteDeadline(time.Time{})
		}
		// 自动同步成功发布新云端版本后通知其他设备,纯下载不发送同步信号。
		request := map[string]any{
			"cmd":    "synced",
			"synced": confSyncSnapshot.Synced,
		}
		if writeErr := connection.WriteJSON(request); nil != writeErr {
			logging.LogErrorf("write websocket message failed: %v", writeErr)
		}
	}
}

func checkSync(boot, exit, byHand bool) bool {
	if util.IsExiting.Load() {
		return false
	}
	if !boot && !exit && !byHand && bootSyncScheduled.Load() {
		return false
	}
	confSyncReadSnapshot := Conf.LoadSync()

	if 2 == confSyncReadSnapshot.Mode && !boot && !exit && !byHand { // 手动模式下只有启动和退出进行同步
		return false
	}

	if 3 == confSyncReadSnapshot.Mode && !byHand { // 完全手动模式下只有手动进行同步
		return false
	}

	if !confSyncReadSnapshot.Enabled {
		if byHand {
			util.PushMsg(Conf.Language(124), 5000)
		}
		return false
	}

	if conf.ProviderS3 != confSyncReadSnapshot.Provider && !cloud.IsValidCloudDirName(confSyncReadSnapshot.CloudName) {
		if byHand {
			util.PushMsg(Conf.Language(123), 5000)
		}
		return false
	}
	if nil == Conf.GetUser() {
		return false
	}

	switch confSyncReadSnapshot.Provider {
	case conf.ProviderSiYuan:
		if !IsSubscriber() {
			UpdateSyncConf(func(s *conf.Sync) { s.Enabled = false })
			return false
		}
	case conf.ProviderWebDAV, conf.ProviderS3, conf.ProviderLocal:
		if !IsPaidUser() {
			UpdateSyncConf(func(s *conf.Sync) { s.Enabled = false })
			return false
		}
	}

	if !byHand && !exit && notifyAutoSyncPaused() {
		return false
	}
	return true
}

// incReindex 增量重建索引。
func incReindex(upserts, removes []string) (upsertRootIDs, removeRootIDs []string) {
	upsertRootIDs = []string{}
	removeRootIDs = []string{}

	util.IncBootProgress(3, Conf.Language(308))
	removeRootIDs = removeIndexes(removes) // 先执行 remove,否则移动文档时 upsert 会被忽略,导致未被索引
	upsertRootIDs = upsertIndexes(upserts)

	if 1 > len(removeRootIDs) {
		removeRootIDs = []string{}
	}
	if 1 > len(upsertRootIDs) {
		upsertRootIDs = []string{}
	}
	return
}

func removeIndexes(removeFilePaths []string) (removeRootIDs []string) {
	bootProgressPart := int32(10 / float64(len(removeFilePaths)))
	for _, removeFile := range removeFilePaths {
		if !strings.HasSuffix(removeFile, ".sy") {
			continue
		}

		rootID := util.GetTreeID(removeFile)
		removeRootIDs = append(removeRootIDs, rootID)

		msg := fmt.Sprintf(Conf.Language(39), rootID)
		util.IncBootProgress(bootProgressPart, msg)
		pushSyncStatusBar(msg)

		cache.RemoveTreeData(rootID)
		block := treenode.GetBlockTree(rootID)
		boxID := ""
		if nil != block {
			boxID = block.BoxID
			cache.RemoveDocIAL(block.Path)
		}
		sql.RemoveTreeQueue(boxID, rootID)
		bts := treenode.GetBlockTreesByRootIDInBox(rootID, boxID)
		for _, b := range bts {
			cache.RemoveBlockIAL(b.ID)
		}
		treenode.RemoveBlockTreesByRootID(boxID, rootID)
	}

	if 1 > len(removeRootIDs) {
		removeRootIDs = []string{}
	}
	return
}

func upsertIndexes(upsertFilePaths []string) (upsertRootIDs []string) {
	luteEngine := util.NewLute()
	bootProgressPart := int32(10 / float64(len(upsertFilePaths)))
	for _, upsertFile := range upsertFilePaths {
		rootID, indexed := func() (string, bool) {
			if !strings.HasSuffix(upsertFile, ".sy") {
				return "", false
			}

			upsertFile = filepath.ToSlash(upsertFile)
			upsertFile = strings.TrimPrefix(upsertFile, "/")

			box, _, found := strings.Cut(upsertFile, "/")
			if !found {
				// .sy 直接出现在 data 文件夹下,没有出现在笔记本文件夹下的情况
				return "", false
			}
			if IsEncryptedBox(box) {
				if !isBoxUnlockedForAccess(box) || !isEncryptedBoxMounted(box) {
					return "", false
				}
				if acquireErr := AcquireEncryptedBoxOperation(box); acquireErr != nil {
					return "", false
				}
				defer ReleaseEncryptedBoxOperation(box)
			}

			p := strings.TrimPrefix(upsertFile, box)
			hpathRefresh.Lock()
			defer hpathRefresh.Unlock()
			msg := fmt.Sprintf(Conf.Language(40), util.GetTreeID(p))
			util.IncBootProgress(bootProgressPart, msg)
			pushSyncStatusBar(msg)

			rootID := util.GetTreeID(p)
			cache.RemoveTreeData(rootID)
			tree, err0 := filesys.LoadTree(box, p, luteEngine)
			if nil != err0 {
				return "", false
			}
			oldDoc := treenode.GetBlockTreeInBox(rootID, box)
			refreshPath := oldDoc != nil && oldDoc.BoxID == box && oldDoc.HPath != tree.HPath
			var refreshKey string
			if refreshPath {
				if refreshKey, err0 = queueHPathRefreshLocked(tree); err0 != nil {
					logging.LogErrorf("queue synced document hpaths [%s] failed: %s", rootID, err0)
					return "", false
				}
			}
			treenode.UpsertBlockTree(tree)
			if refreshPath {
				if err0 = treenode.RefreshDocHPaths(tree); err0 != nil {
					logging.LogErrorf("refresh synced document paths [%s] failed: %s", rootID, err0)
					return "", false
				}
				hpathRefresh.tasks[refreshKey].recover = false
			}
			sql.UpsertTreeQueue(tree)

			bts := treenode.GetBlockTreesByRootIDInBox(rootID, tree.Box)
			for _, b := range bts {
				cache.RemoveBlockIAL(b.ID)
			}
			cache.RemoveDocIAL(tree.Path)
			return rootID, true
		}()
		if indexed {
			upsertRootIDs = append(upsertRootIDs, rootID)
		}
	}

	if 1 > len(upsertRootIDs) {
		upsertRootIDs = []string{}
	}
	return
}

func SetCloudSyncDir(name string) error {
	release := lockAssetSourceChange()
	defer release()
	if conf.ProviderS3 == Conf.Sync.Load().Provider {
		return errors.New(Conf.Language(131))
	}
	if !cloud.IsValidCloudDirName(name) {
		return errors.New(Conf.Language(37))
	}

	if Conf.Sync.Load().CloudName == name {
		return nil
	}
	if err := requireCompleteAssetDownloads(); err != nil {
		return err
	}

	UpdateSyncConf(func(s *conf.Sync) { s.CloudName = name })
	refreshLANSyncManager()
	return nil
}

func SetSyncGenerateConflictDoc(b bool) {
	UpdateSyncConf(func(s *conf.Sync) { s.GenerateConflictDoc = b })
}

func SetSyncEnable(b bool) {
	UpdateSyncConf(func(s *conf.Sync) { s.Enabled = b })
	refreshLANSyncManager()
}

func SetSyncInterval(interval int) {
	if 30 > interval {
		interval = 30
	}
	if 43200 < interval {
		interval = 43200
	}

	UpdateSyncConf(func(s *conf.Sync) { s.Interval = interval })
	planSyncAfter(time.Duration(interval) * time.Second)
}

func SetSyncPerception(enabled bool) {
	if util.ContainerDocker == util.Container {
		enabled = false
	}

	UpdateSyncConf(func(s *conf.Sync) { s.Perception = enabled })

	if enabled {
		connectSyncWebSocket()
		return
	}

	closeSyncWebSocket()
}

func SetSyncMode(mode int) {
	UpdateSyncConf(func(s *conf.Sync) { s.Mode = mode })
}

func SetSyncProvider(provider int, completeAssets bool) (err error) {
	release := lockAssetSourceChange()
	defer release()
	if provider != Conf.Sync.Load().Provider {
		if completeAssets {
			util.PushEndlessProgress(Conf.Language(398))
			defer util.ClearPushProgress(100)
			err = ensureCompleteSyncAssets(func() { util.PushEndlessProgress(Conf.Language(399)) })
			if err != nil {
				logging.LogWarnf("complete data before switching sync provider [%d -> %d] failed: %s", Conf.Sync.Load().Provider, provider, err)
				err = fmt.Errorf(Conf.Language(400), err)
			}
		} else {
			err = requireCompleteAssetDownloads()
		}
		if err != nil {
			return
		}
	}
	UpdateSyncConf(func(s *conf.Sync) { s.Provider = provider })
	refreshLANSyncManager()
	return
}

func SetSyncProviderS3(s3 *conf.S3) (err error) {
	release := lockAssetSourceChange()
	defer release()
	if err = validateSyncS3(s3); err != nil {
		return
	}
	s3 = cloneConfSection(s3)
	s3.Endpoint = strings.TrimSpace(s3.Endpoint)
	s3.Endpoint = util.NormalizeEndpoint(s3.Endpoint)
	s3.AccessKey = strings.TrimSpace(s3.AccessKey)
	s3.SecretKey = strings.TrimSpace(s3.SecretKey)
	s3.Bucket = strings.TrimSpace(s3.Bucket)
	s3.Region = strings.TrimSpace(s3.Region)
	s3.Timeout = util.NormalizeTimeout(s3.Timeout)
	s3.ConcurrentReqs = util.NormalizeConcurrentReqs(s3.ConcurrentReqs, conf.ProviderS3)
	if Conf.Sync.Load().Provider == conf.ProviderS3 && (Conf.Sync.Load().S3 == nil ||
		Conf.Sync.Load().S3.Endpoint != s3.Endpoint || Conf.Sync.Load().S3.Bucket != s3.Bucket) {
		if err = requireCompleteAssetDownloads(); err != nil {
			return
		}
	}

	UpdateSyncConf(func(value *conf.Sync) { value.S3 = cloneConfSection(s3) })
	refreshLANSyncManager()
	return
}

func validateSyncS3(s3 *conf.S3) error {
	if s3 == nil {
		return errors.New(Conf.Language(142))
	}
	if conf.MissingSyncProviderField(conf.ProviderS3, map[string]string{
		"endpoint": s3.Endpoint, "accessKey": s3.AccessKey, "secretKey": s3.SecretKey, "bucket": s3.Bucket, "region": s3.Region,
	}) {
		return errors.New(Conf.Language(142))
	}
	return validateSyncHTTPEndpoint(s3.Endpoint)
}

// validateSyncHTTPEndpoint 在归一化前拒绝空地址,保留省略协议和协议占位符的兼容写法。
func validateSyncHTTPEndpoint(value string) error {
	rawEndpoint := strings.TrimSpace(value)
	if rawEndpoint == "" {
		return errors.New(Conf.Language(142))
	}
	rawEndpoint = strings.Replace(rawEndpoint, "http://http(s)://", "https://", 1)
	rawEndpoint = strings.Replace(rawEndpoint, "http(s)://", "https://", 1)
	if strings.Contains(rawEndpoint, "://") && !strings.HasPrefix(rawEndpoint, "http://") && !strings.HasPrefix(rawEndpoint, "https://") {
		return errors.New(Conf.Language(249))
	}
	endpoint, err := url.Parse(util.NormalizeEndpoint(rawEndpoint))
	if err != nil || endpoint.Hostname() == "" || (endpoint.Scheme != "http" && endpoint.Scheme != "https") {
		return errors.New(Conf.Language(249))
	}
	return nil
}

func validateSyncWebDAV(webdav *conf.WebDAV) error {
	if webdav == nil || conf.MissingSyncProviderField(conf.ProviderWebDAV, map[string]string{"endpoint": webdav.Endpoint}) {
		return errors.New(Conf.Language(142))
	}
	return validateSyncHTTPEndpoint(webdav.Endpoint)
}

func validateSyncLocal(local *conf.Local) error {
	if local == nil || conf.MissingSyncProviderField(conf.ProviderLocal, map[string]string{"endpoint": local.Endpoint}) {
		return errors.New(Conf.Language(142))
	}
	return nil
}

func SetSyncProviderWebDAV(webdav *conf.WebDAV) (err error) {
	release := lockAssetSourceChange()
	defer release()
	if err = validateSyncWebDAV(webdav); err != nil {
		return
	}
	webdav = cloneConfSection(webdav)
	webdav.Endpoint = strings.TrimSpace(webdav.Endpoint)
	webdav.Endpoint = util.NormalizeEndpoint(webdav.Endpoint)

	// 不支持配置坚果云 WebDAV 进行同步 https://github.com/siyuan-note/siyuan/issues/7657
	if strings.Contains(strings.ToLower(webdav.Endpoint), "dav.jianguoyun.com") {
		err = errors.New(Conf.Language(194))
		return
	}

	webdav.Username = strings.TrimSpace(webdav.Username)
	webdav.Password = strings.TrimSpace(webdav.Password)
	webdav.Timeout = util.NormalizeTimeout(webdav.Timeout)
	webdav.ConcurrentReqs = util.NormalizeConcurrentReqs(webdav.ConcurrentReqs, conf.ProviderWebDAV)
	if Conf.Sync.Load().Provider == conf.ProviderWebDAV && (Conf.Sync.Load().WebDAV == nil ||
		Conf.Sync.Load().WebDAV.Endpoint != webdav.Endpoint || Conf.Sync.Load().WebDAV.Username != webdav.Username) {
		if err = requireCompleteAssetDownloads(); err != nil {
			return
		}
	}

	UpdateSyncConf(func(s *conf.Sync) { s.WebDAV = webdav })
	refreshLANSyncManager()
	return
}

func SetSyncProviderLocal(local *conf.Local) (err error) {
	release := lockAssetSourceChange()
	defer release()
	if err = validateSyncLocalPath(local); err != nil {
		return
	}
	local = cloneConfSection(local)
	local.Endpoint = strings.TrimSpace(local.Endpoint)
	local.Endpoint = util.NormalizeLocalPath(local.Endpoint)

	local.Timeout = util.NormalizeTimeout(local.Timeout)
	local.ConcurrentReqs = util.NormalizeConcurrentReqs(local.ConcurrentReqs, conf.ProviderLocal)
	if Conf.Sync.Load().Provider == conf.ProviderLocal && (Conf.Sync.Load().Local == nil ||
		filepath.Clean(Conf.Sync.Load().Local.Endpoint) != filepath.Clean(local.Endpoint)) {
		if err = requireCompleteAssetDownloads(); err != nil {
			return
		}
	}

	UpdateSyncConf(func(s *conf.Sync) { s.Local = local })
	refreshLANSyncManager()
	return
}

var (
	syncLock             = sync.Mutex{}
	isSyncing            = atomic.Bool{}
	syncAutoRequests     = syncRequestState{}
	syncManualRequests   = syncRequestState{}
	syncExitRequests     = syncRequestState{}
	syncUploadRequests   = syncRequestState{}
	syncDownloadRequests = syncRequestState{}
)

func CreateCloudSyncDir(name string) (err error) {
	assetDownloadSourceMu.RLock()
	defer assetDownloadSourceMu.RUnlock()
	switch Conf.Sync.Load().Provider {
	case conf.ProviderSiYuan, conf.ProviderLocal:
		break
	default:
		err = errors.New(Conf.Language(131))
		return
	}

	name = util.RemoveInvalid(name)
	if !cloud.IsValidCloudDirName(name) {
		return errors.New(Conf.Language(37))
	}

	handleCloudError := cloudRepoErrorHandler()
	defer func() { handleCloudError(err) }()
	repo, err := newCloudRepositoryWithAssetSourceLocked()
	if err != nil {
		return
	}

	err = repo.CreateCloudRepo(name)
	if err != nil {
		handleCloudError(err)
		err = errors.New(formatRepoErrorMsg(err))
		return
	}
	return
}

func RemoveCloudSyncDir(name string) (err error) {
	release := lockAssetSourceChange()
	defer release()
	if name == Conf.Sync.Load().CloudName {
		if err = requireCompleteAssetDownloads(); err != nil {
			return
		}
	}
	switch Conf.Sync.Load().Provider {
	case conf.ProviderSiYuan, conf.ProviderLocal:
		break
	default:
		err = errors.New(Conf.Language(131))
		return
	}

	msgId := util.PushMsg(Conf.Language(116), 15000)

	handleCloudError := cloudRepoErrorHandler()
	defer func() { handleCloudError(err) }()
	repo, err := newCloudRepositoryWithAssetSourceLocked()
	if err != nil {
		return
	}

	err = repo.RemoveCloudRepo(name)
	if err != nil {
		handleCloudError(err)
		err = errors.New(formatRepoErrorMsg(err))
		return
	}

	util.PushClearMsg(msgId)
	time.Sleep(500 * time.Millisecond)
	if Conf.Sync.Load().CloudName == name {
		UpdateSyncConf(func(s *conf.Sync) { s.CloudName = "main" })
		util.PushMsg(Conf.Language(155), 5000)
	}
	return
}

func ListCloudSyncDir() (syncDirs []*Sync, hSize string, err error) {
	confSyncSnapshot := Conf.LoadSync()

	assetDownloadSourceMu.RLock()
	defer assetDownloadSourceMu.RUnlock()
	syncDirs = []*Sync{}
	var dirs []*cloud.Repo
	var size int64

	handleCloudError := cloudRepoErrorHandler()
	defer func() { handleCloudError(err) }()
	repo, err := newCloudRepositoryWithAssetSourceLocked()
	if err != nil {
		return
	}

	dirs, size, err = repo.GetCloudRepos()
	if err != nil {
		handleCloudError(err)
		err = errors.New(formatRepoErrorMsg(err))
		return
	}
	if 1 > len(dirs) && conf.ProviderS3 != confSyncSnapshot.Provider {
		dirs = append(dirs, &cloud.Repo{
			Name:    "main",
			Size:    0,
			Updated: time.Now().Format("2006-01-02 15:04:05"),
		})
	}

	for _, d := range dirs {
		dirSize := d.Size
		sync := &Sync{
			Size:      dirSize,
			HSize:     "-",
			Updated:   d.Updated,
			CloudName: d.Name,
		}
		if conf.ProviderSiYuan == confSyncSnapshot.Provider {
			sync.HSize = humanize.BytesCustomCeil(uint64(dirSize), 2)
		}
		syncDirs = append(syncDirs, sync)
	}
	hSize = "-"
	if conf.ProviderSiYuan == confSyncSnapshot.Provider {
		hSize = humanize.BytesCustomCeil(uint64(size), 2)
	}
	return
}

func formatRepoErrorMsg(err error) string {
	msg := html.EscapeString(err.Error())
	if errors.Is(err, cloud.ErrCloudAuthFailed) {
		msg = Conf.Language(31)
	} else if errors.Is(err, cloud.ErrCloudObjectNotFound) {
		msg = Conf.Language(129)
	} else if errors.Is(err, dejavu.ErrLockCloudFailed) {
		msg = Conf.Language(188)
		if detail := cloudLockErrorDetail(err, Conf.Language); "" != detail {
			msg += ": " + detail
		}
	} else if errors.Is(err, dejavu.ErrCloudLocked) {
		msg = Conf.Language(189)
	} else if errors.Is(err, dejavu.ErrRepoFatal) {
		msg = Conf.Language(23)
	} else if errors.Is(err, dejavu.ErrIndexFileChanged) {
		// 同步索引期间工作空间文件被修改,明确提示用户稍后重试 https://ld246.com/article/1789052153692
		msg = Conf.Language(384)
	} else if errors.Is(err, cloud.ErrSystemTimeIncorrect) {
		msg = Conf.Language(195)
	} else if errors.Is(err, cloud.ErrDeprecatedVersion) {
		msg = Conf.Language(212)
	} else if errors.Is(err, cloud.ErrCloudCheckFailed) {
		msg = Conf.Language(213)
	} else if errors.Is(err, cloud.ErrCloudServiceUnavailable) {
		msg = Conf.language(219)
	} else if errors.Is(err, cloud.ErrCloudForbidden) {
		msg = Conf.language(249)
	} else if errors.Is(err, cloud.ErrCloudTooManyRequests) {
		msg = Conf.language(250)
	} else if errors.Is(err, cloud.ErrDecryptFailed) {
		msg = Conf.Language(135)
	} else if detail := webDAVErrorDetail(err, Conf.Language); detail != "" {
		msg = detail
	} else {
		logging.LogErrorf("unclassified repository error: %s", msg)
		msgLowerCase := strings.ToLower(msg)
		if strings.Contains(msgLowerCase, "illegal byte sequence") {
			msg = fmt.Sprintf(Conf.Language(397), msg)
		} else if strings.Contains(msgLowerCase, "permission denied") || strings.Contains(msgLowerCase, "access is denied") {
			msg = Conf.Language(33)
		} else if strings.Contains(msgLowerCase, "region was not a valid") {
			msg = Conf.language(254)
		} else if strings.Contains(msgLowerCase, "device or resource busy") || strings.Contains(msg, "is being used by another") {
			msg = fmt.Sprintf(Conf.Language(85), err)
		} else if strings.Contains(msgLowerCase, "cipher: message authentication failed") {
			msg = Conf.Language(135)
		} else if isDNSError(msgLowerCase) {
			msg = Conf.Language(24)
		} else if strings.Contains(msgLowerCase, "net/http: request canceled while waiting for connection") || strings.Contains(msgLowerCase, "exceeded while awaiting") || strings.Contains(msgLowerCase, "context deadline exceeded") || strings.Contains(msgLowerCase, "timeout") || strings.Contains(msgLowerCase, "context cancellation while reading body") {
			msg = Conf.Language(24)
		} else if strings.Contains(msgLowerCase, "connection") || strings.Contains(msgLowerCase, "refused") || strings.Contains(msgLowerCase, "socket") || strings.Contains(msgLowerCase, "eof") || strings.Contains(msgLowerCase, "closed") || strings.Contains(msgLowerCase, "network") {
			msg = Conf.Language(28)
		}
	}
	msg += " (Provider: " + conf.ProviderToStr(Conf.Sync.Load().Provider) + ")"
	return msg
}

// isDNSError 判断错误信息是否属于 DNS 解析类(域名解析失败、主机名无法解析等)。
// dejavu/cloud 层用 fmt.Errorf 原样透传底层网络错误,因此这里用字符串匹配兜底。
func isDNSError(msg string) bool {
	return strings.Contains(msg, "no such host") ||
		strings.Contains(msg, "hostname resolution") ||
		strings.Contains(msg, "no address associated with hostname")
}

// lastDNSFlushTime 及其锁用于 DNS 刷新节流,避免自动同步循环里反复 fork 系统命令。
// 由于同步可能从多个入口并发执行(如启动后台同步与手动同步),这里用互斥锁保护。
var (
	lastDNSFlushTime   time.Time
	lastDNSFlushTimeMu sync.Mutex
)

// flushAndRetryOnDNSError 在确认是 DNS 类错误且距上次刷新超过 5 分钟时,刷新系统 DNS 缓存并返回 true
// 以触发上层重试一次同步;否则返回 false。节流是为了避免高频自动同步反复 fork 系统命令。
func flushAndRetryOnDNSError(err error) bool {
	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
		return false
	}
	if !isDNSError(strings.ToLower(err.Error())) {
		return false
	}

	lastDNSFlushTimeMu.Lock()
	defer lastDNSFlushTimeMu.Unlock()
	if time.Since(lastDNSFlushTime) < 5*time.Minute {
		logging.LogInfof("sync failed with DNS error, but DNS cache was flushed recently, skip retry")
		return false
	}
	lastDNSFlushTime = time.Now()

	logging.LogInfof("sync failed with DNS error [%s], flushing DNS cache and retrying once", err)
	flushDNS()
	return true
}

// syncRepoWithDNSRetry 执行一次同步,若失败且判定为 DNS 类错误,则刷新系统 DNS 缓存后重试一次。
// 统一封装 DNS 重试逻辑,供主同步流程(syncData)与启动后台同步复用。
func syncRepoWithDNSRetry(exit, byHand bool) (cloudPublished bool, err error) {
	return syncRepoWithDNSRetryContext(context.Background(), exit, byHand)
}

func syncRepoWithDNSRetryContext(ctx context.Context, exit, byHand bool) (cloudPublished bool, err error) {
	cloudPublished, err = syncRepoWithContext(ctx, exit, byHand)
	if nil != err && ctx.Err() == nil && flushAndRetryOnDNSError(err) {
		cloudPublished, err = syncRepoWithContext(ctx, exit, byHand)
	}
	return
}

// syncRepoDownloadWithDNSRetry 仅下载同步,DNS 类错误时刷新系统 DNS 缓存后重试一次。
func syncRepoDownloadWithDNSRetry() (err error) {
	err = syncRepoDownload()
	if nil != err && flushAndRetryOnDNSError(err) {
		err = syncRepoDownload()
	}
	return
}

// syncRepoUploadWithDNSRetry 仅上传同步,DNS 类错误时刷新系统 DNS 缓存后重试一次。
func syncRepoUploadWithDNSRetry() (err error) {
	err = syncRepoUpload()
	if nil != err && flushAndRetryOnDNSError(err) {
		err = syncRepoUpload()
	}
	return
}

func bootSyncRepoWithDNSRetry(complete func(error)) (scheduled bool, err error) {
	scheduled, err = bootSyncRepo(complete)
	if nil != err && flushAndRetryOnDNSError(err) {
		scheduled, err = bootSyncRepo(complete)
	}
	return
}

func loadSyncIgnoreLines() (ret []string, err error) {
	// 忽略旧版同步配置,读取用户规则失败时仍需保留此规则。
	defer func() {
		ret = append(ret, "/.siyuan/conf.json")
	}()
	// 同步或快照可能重新带回隔离块,加载规则前完成清理。
	if err = util.MigrateAppearanceSyncIgnore(); err != nil {
		return
	}
	ignore := filepath.Join(util.DataDir, ".siyuan", "syncignore")
	err = os.MkdirAll(filepath.Dir(ignore), 0755)
	if err != nil {
		return
	}
	data, err := os.ReadFile(ignore)
	if os.IsNotExist(err) {
		var file *os.File
		file, err = os.OpenFile(ignore, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
		if err == nil {
			err = file.Close()
		} else if os.IsExist(err) {
			data, err = os.ReadFile(ignore)
		}
	}
	if err != nil {
		logging.LogErrorf("read syncignore [%s] failed: %s", ignore, err)
		return
	}
	dataStr := string(data)
	dataStr = strings.ReplaceAll(dataStr, "\r\n", "\n")
	ret = strings.Split(dataStr, "\n")

	// 忽略用户指南
	for _, id := range userGuideIDs {
		ret = append(ret, id+"/**/*")
	}
	// 视图状态仅在当前设备使用,不参与数据同步。
	ret = append(ret, "/storage/view-state.json")
	ret = append(ret, "/storage/view-state-corrupted-*.json")
	// 忽略用户指南的数据库 JSON 文件
	avNames, err := getAllUserGuideAVJSONFiles()
	if err != nil {
		return nil, err
	}
	for _, avName := range avNames {
		ret = append(ret, "/storage/av/"+avName)
	}

	ret = gulu.Str.RemoveDuplicatedElem(ret)
	return
}

func IncSync() {
	util.MarkIndexFixDirty()
	syncSameCount.Store(0)
	planSyncAfter(time.Duration(Conf.Sync.Load().Interval) * time.Second)
	pendingSync.change(notifySyncPending)
}

func notifySyncPending(pending bool) {
	util.BroadcastByType("main", "syncPending", 0, "", pending)
}

func planSyncAfter(d time.Duration) {
	syncPlanTimeLock.Lock()
	syncPlanTime = time.Now().Add(d)
	syncPlanTimeLock.Unlock()
}

var (
	webSocketConn     *websocket.Conn
	webSocketConnLock = sync.Mutex{}
)

type OnlineKernel struct {
	ID       string `json:"id"`
	Hostname string `json:"hostname"`
	OS       string `json:"os"`
	Ver      string `json:"ver"`
}

var (
	onlineKernels     []*OnlineKernel
	onlineKernelsLock = sync.Mutex{}
)

func GetOnlineKernels() (ret []*OnlineKernel) {
	ret = []*OnlineKernel{}
	onlineKernelsLock.Lock()
	tmp := onlineKernels
	onlineKernelsLock.Unlock()
	for _, kernel := range tmp {
		if kernel.ID == KernelID {
			continue
		}

		ret = append(ret, kernel)
	}
	return
}

var closedSyncWebSocket = atomic.Bool{}

func closeSyncWebSocket() {
	defer logging.Recover()

	webSocketConnLock.Lock()
	defer webSocketConnLock.Unlock()

	if nil != webSocketConn {
		webSocketConn.Close()
		webSocketConn = nil
		closedSyncWebSocket.Store(true)
	}

	logging.LogInfof("sync websocket closed")
}

func connectSyncWebSocket() {
	confSyncReadSnapshot := Conf.LoadSync()

	defer logging.Recover()

	if !confSyncReadSnapshot.Enabled || !IsSubscriber() || conf.ProviderSiYuan != confSyncReadSnapshot.Provider {
		return
	}

	if util.ContainerDocker == util.Container {
		return
	}

	webSocketConnLock.Lock()
	defer webSocketConnLock.Unlock()

	if nil != webSocketConn {
		return
	}

	//logging.LogInfof("connecting sync websocket...")
	var dialErr error
	webSocketConn, dialErr = dialSyncWebSocket()
	if nil != dialErr {
		logging.LogWarnf("connect sync websocket failed: %s", dialErr)
		return
	}
	logging.LogInfof("sync websocket connected")

	webSocketConn.SetCloseHandler(func(code int, text string) error {
		logging.LogWarnf("sync websocket closed: %d, %s", code, text)
		return nil
	})

	go func() {
		defer logging.Recover()

		for {
			result := gulu.Ret.NewResult()
			if readErr := webSocketConn.ReadJSON(&result); nil != readErr {
				time.Sleep(1 * time.Second)
				if closedSyncWebSocket.Load() {
					return
				}

				reconnected := false
				for range 7 {
					time.Sleep(7 * time.Second)
					if nil == Conf.GetUser() {
						return
					}

					//logging.LogInfof("reconnecting sync websocket...")
					webSocketConn, dialErr = dialSyncWebSocket()
					if nil != dialErr {
						logging.LogWarnf("reconnect sync websocket failed: %s", dialErr)
						continue
					}

					logging.LogInfof("sync websocket reconnected")
					reconnected = true
					break
				}
				if !reconnected {
					logging.LogWarnf("reconnect sync websocket failed, do not retry")
					webSocketConn = nil
					return
				}

				continue
			}

			logging.LogInfof("sync websocket message: %v", result)
			data := result.Data.(map[string]any)
			switch data["cmd"].(string) {
			case "synced":
				syncDataFromCloud()
			case "kernels":
				onlineKernelsLock.Lock()

				onlineKernels = []*OnlineKernel{}
				for _, kernel := range data["kernels"].([]any) {
					kernelMap := kernel.(map[string]any)
					onlineKernels = append(onlineKernels, &OnlineKernel{
						ID:       kernelMap["id"].(string),
						Hostname: kernelMap["hostname"].(string),
						OS:       kernelMap["os"].(string),
						Ver:      kernelMap["ver"].(string),
					})
				}

				onlineKernelsLock.Unlock()
			}
		}
	}()
}

var KernelID = gulu.Rand.String(7)

func dialSyncWebSocket() (c *websocket.Conn, err error) {
	endpoint := util.GetCloudWebSocketServer() + "/apis/siyuan/dejavu/ws"
	header := http.Header{
		"User-Agent":        []string{util.UserAgent},
		"x-siyuan-uid":      []string{Conf.GetUser().UserId},
		"x-siyuan-kernel":   []string{KernelID},
		"x-siyuan-ver":      []string{util.Ver},
		"x-siyuan-os":       []string{runtime.GOOS},
		"x-siyuan-hostname": []string{util.GetDeviceName()},
		"x-siyuan-repo":     []string{Conf.Sync.Load().CloudName},
	}
	c, _, err = websocket.DefaultDialer.Dial(endpoint, header)
	if err == nil {
		closedSyncWebSocket.Store(false)
	}
	return
}