Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

19.0 KB
Raw Blame
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later

package model

import (
	"errors"
	"maps"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/88250/go-humanize"
	"github.com/88250/gulu"
	"github.com/88250/lute"
	"github.com/88250/lute/ast"
	"github.com/88250/lute/parse"
	"github.com/88250/lute/render"
	"github.com/emirpasic/gods/sets/hashset"
	"github.com/siyuan-note/logging"
	"github.com/siyuan-note/siyuan/kernel/av"
	"github.com/siyuan-note/siyuan/kernel/conf"
	"github.com/siyuan-note/siyuan/kernel/filesys"
	"github.com/siyuan-note/siyuan/kernel/sql"
	"github.com/siyuan-note/siyuan/kernel/task"
	"github.com/siyuan-note/siyuan/kernel/treenode"
	"github.com/siyuan-note/siyuan/kernel/util"
)

var backgroundRefreshWaitGroup sync.WaitGroup

// runBackgroundRefresh 跟踪延迟刷新任务,便于工作区释放前等待任务结束。
func runBackgroundRefresh(delay time.Duration, refresh func()) {
	backgroundRefreshWaitGroup.Go(func() {
		time.Sleep(delay)
		// 与编辑事务串行刷新,避免异步分组写回覆盖撤销、重做后的数据。
		flushLock.Lock()
		defer flushLock.Unlock()
		refresh()
	})
}

func PushReloadSnippet(snippet *conf.Snpt) {
	util.BroadcastByType("main", "setSnippet", 0, "", snippet)
}

const (
	DataChangeReasonSync      = "sync"
	DataChangeReasonOverwrite = "overwrite"
)

func PushReloadPlugin(uninstallPluginNameSet, unloadPluginNameSet, reloadPluginSet, dataChangePluginSet *hashset.Set,
	excludeApp, dataChangeReason string) {
	// 按优先级从高到低排列,同一插件只保留在优先级最高的集合中
	orderedSets := []*hashset.Set{uninstallPluginNameSet, unloadPluginNameSet, reloadPluginSet, dataChangePluginSet}
	slices := make([][]string, len(orderedSets))
	// 按顺序遍历所有集合
	for i, set := range orderedSets {
		if nil != set {
			// 遍历当前集合的所有插件名称
			for _, n := range set.Values() {
				name := n.(string)
				// 将该插件从所有后续集合中移除
				for _, lowerSet := range orderedSets[i+1:] {
					if nil != lowerSet {
						lowerSet.Remove(name)
					}
				}
			}
		}

		// 将当前集合转换为字符串切片
		if nil == set {
			slices[i] = []string{}
		} else {
			strs := make([]string, 0, set.Size())
			for _, n := range set.Values() {
				strs = append(strs, n.(string))
			}
			slices[i] = strs
		}
	}

	logging.LogInfof("reload plugins, uninstalls=%v, unloads=%v, reloads=%v, dataChanges=%v", slices[0], slices[1], slices[2], slices[3])
	payload := map[string]any{
		"uninstallPlugins":  slices[0],        // 插件卸载
		"unloadPlugins":     slices[1],        // 插件禁用
		"reloadPlugins":     slices[2],        // 插件启用,或插件代码变更
		"dataChangePlugins": slices[3],        // 插件存储数据变更
		"dataChangeReason":  dataChangeReason, // 插件存储数据变更来源
	}
	if 0 < len(slices[0])+len(slices[1])+len(slices[2]) {
		util.ReloadPublishServiceSessions()
	}

	if "" == excludeApp {
		util.BroadcastByType("main", "reloadPlugin", 0, "", payload)
		return
	}
	util.BroadcastByTypeAndExcludeApp(excludeApp, "main", "reloadPlugin", 0, "", payload)
}

// PushPluginStorageDataChanged 通知其他前端实例插件存储数据已变更。
func PushPluginStorageDataChanged(absPath, excludeApp string) {
	pluginName, ok := pluginStorageName(absPath)
	if !ok {
		return
	}
	PushReloadPlugin(nil, nil, nil, hashset.New(pluginName), excludeApp, DataChangeReasonOverwrite)
}

func pluginStorageName(absPath string) (pluginName string, ok bool) {
	relPath, err := filepath.Rel(util.DataDir, absPath)
	if nil != err {
		return "", false
	}
	parts := strings.Split(filepath.ToSlash(relPath), "/")
	if 3 > len(parts) || "storage" != parts[0] || "petal" != parts[1] || "" == parts[2] ||
		"petals.json" == parts[2] {
		return "", false
	}
	return parts[2], true
}

// PushReloadAllEnabledPlugins 向前端推送已启用插件的全局状态,并返回相同的权威状态供请求方应用。
func PushReloadAllEnabledPlugins(enabled, petalDisabled bool, revision uint64, changed bool) map[string]any {
	payload := PluginReloadPayload(enabled, petalDisabled, revision, changed)
	if changed {
		if len(payload["unloadPlugins"].([]string))+len(payload["reloadPlugins"].([]string)) > 0 {
			util.ReloadPublishServiceSessions()
		}
		util.BroadcastByType("main", "reloadPlugin", 0, "", payload)
	}
	return payload
}

// PluginReloadPayload 构造插件生命周期结果,响应与广播复用同一形态。
func PluginReloadPayload(enabled, petalDisabled bool, revision uint64, changed bool) map[string]any {
	pluginNames := []string{}
	for _, petal := range getPetals() {
		if petal.Enabled {
			pluginNames = append(pluginNames, petal.Name)
		}
	}
	sort.Strings(pluginNames)

	unloadPlugins, reloadPlugins := []string{}, []string{}
	if changed {
		if enabled {
			reloadPlugins = pluginNames
		} else {
			unloadPlugins = pluginNames
		}
	}
	payload := map[string]any{
		"uninstallPlugins":    []string{},
		"unloadPlugins":       unloadPlugins,
		"reloadPlugins":       reloadPlugins,
		"dataChangePlugins":   []string{},
		"globalPetalEnabled":  enabled,
		"globalPetalDisabled": petalDisabled,
		"globalPetalRevision": revision,
		"globalPetalChanged":  changed,
	}
	if changed {
		logging.LogInfof("reload plugins for global state, unloads=%v, reloads=%v, revision=%d",
			unloadPlugins, reloadPlugins, revision)
	}
	return payload
}

func refreshDocInfo(tree *parse.Tree) {
	if nil == tree {
		return
	}

	refreshDocInfoWithSize(tree, filesys.TreeSize(tree))
}

func refreshDocInfoWithoutParent(tree *parse.Tree) {
	if nil == tree {
		return
	}

	refreshDocInfo0(tree, filesys.TreeSize(tree))
}

func refreshDocInfoWithSize(tree *parse.Tree, size uint64) {
	if nil == tree {
		return
	}

	refreshDocInfo0(tree, size)
	runBackgroundRefresh(128*time.Millisecond, func() {
		refreshParentDocInfo(tree)
	})
}

func refreshParentDocInfo(tree *parse.Tree) {
	if nil == tree || nil == Conf {
		return
	}

	parentTree := loadParentTree(tree)
	if nil == parentTree {
		return
	}

	luteEngine := lute.New()
	renderer := render.NewJSONRenderer(parentTree, luteEngine.RenderOptions, luteEngine.ParseOptions)
	data := renderer.Render()
	refreshDocInfo0(parentTree, uint64(len(data)))
}

func refreshBoxDocInfo(tree *parse.Tree) {
	if nil == tree || path.Dir(tree.Path) != "/" || IsBoxDoc(tree.Box, tree.ID) {
		return
	}
	refreshBoxDocInfoByBoxID(tree.Box)
}

func refreshBoxDocInfoByBoxID(boxID string) {
	if !IsBoxDocEnabled() {
		return
	}
	box := Conf.Box(boxID)
	if nil == box {
		return
	}
	util.BroadcastByType("filetree", "reloadNotebookInfo", 0, "", boxID)
}

func pushNotebookIconChanged(boxID, icon string) {
	util.BroadcastByType("filetree", "notebookIconChanged", 0, "", map[string]any{
		"boxID": boxID,
		"icon":  icon,
	})
}

func refreshDocInfo0(tree *parse.Tree, size uint64) {
	// 异步刷新可能在测试或退出流程中晚于配置释放,此时放弃刷新而不是解引用空配置
	if nil == tree || nil == tree.Root || nil == Conf {
		return
	}

	cTime, _ := time.ParseInLocation("20060102150405", tree.ID[:14], time.Local)
	mTime := cTime
	if updated := tree.Root.IALAttr("updated"); "" != updated {
		if updatedTime, err := time.ParseInLocation("20060102150405", updated, time.Local); err == nil {
			mTime = updatedTime
		}
	}

	docInfo := map[string]any{
		"box":      tree.Box,
		"rootID":   tree.ID,
		"name":     treenode.ReadDocumentTitle(tree.Root).Text,
		"alias":    tree.Root.IALAttr("alias"),
		"name1":    tree.Root.IALAttr("name"),
		"memo":     tree.Root.IALAttr("memo"),
		"bookmark": tree.Root.IALAttr("bookmark"),
		"size":     size,
		"hSize":    humanize.BytesCustomCeil(size, 2),
		"mtime":    mTime.Unix(),
		"ctime":    cTime.Unix(),
		"hMtime":   mTime.Format("2006-01-02 15:04:05") + ", " + util.HumanizeTime(mTime, Conf.Lang),
		"hCtime":   cTime.Format("2006-01-02 15:04:05") + ", " + util.HumanizeTime(cTime, Conf.Lang),
	}

	boxID, rootID, treePath := tree.Box, tree.ID, tree.Path
	hidden := "true" == tree.Root.IALAttr(DocHiddenAttr)
	task.AppendAsyncTaskWithDelay(task.ReloadProtyle, 500*time.Millisecond, func(docInfo map[string]any) {
		// 发送时统计子文档,避免转换标题等操作期间的中间状态导致文档树图标闪烁。
		subFileCount := 0
		if IsBoxDoc(boxID, rootID) {
			subFileCount = BoxDocSubFileCount(boxID)
		} else if !hidden {
			subDir := filepath.Join(util.DataDir, boxID, strings.TrimSuffix(treePath, ".sy"))
			subFiles, err := os.ReadDir(subDir)
			if err == nil {
				for _, subFile := range subFiles {
					if !strings.HasSuffix(subFile.Name(), ".sy") {
						continue
					}

					subDocIAL, _ := filesys.DocIAL(filepath.Join(subDir, subFile.Name()))
					if "true" == subDocIAL[DocHiddenAttr] {
						continue
					}
					subFileCount++
				}
			}
		}
		docInfo["subFileCount"] = subFileCount
		util.PushReloadDocInfo(docInfo)
	}, docInfo)
}

func ReloadFiletree() {
	task.AppendAsyncTaskWithDelay(task.ReloadFiletree, 200*time.Millisecond, util.PushReloadFiletree)
}

func ReloadTag() {
	task.AppendAsyncTaskWithDelay(task.ReloadTag, 200*time.Millisecond, util.PushReloadTag)
}

func ReloadProtyle(rootID string) {
	// 刷新关联的引用
	defTree, _ := LoadTreeByBlockID(rootID)
	if nil != defTree {
		defIDs := sql.QueryChildDefIDsByRootDefID(rootID)

		var defNodes []*ast.Node
		ast.Walk(defTree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
			if !entering || !n.IsBlock() {
				return ast.WalkContinue
			}

			if gulu.Str.Contains(n.ID, defIDs) {
				defNodes = append(defNodes, n)
			}
			return ast.WalkContinue
		})

		for _, def := range defNodes {
			refreshDynamicRefText(def, defTree)
		}
	}

	// 刷新关联的嵌入块
	refIDs := sql.QueryRefIDsByDefID(rootID, true)
	var rootIDs []string
	bts := treenode.GetBlockTrees(refIDs)
	for _, bt := range bts {
		rootIDs = append(rootIDs, bt.RootID)
	}
	rootIDs = gulu.Str.RemoveDuplicatedElem(rootIDs)
	for _, id := range rootIDs {
		task.AppendAsyncTaskWithDelay(task.ReloadProtyle, 200*time.Millisecond, util.PushReloadProtyle, id)
	}

	task.AppendAsyncTaskWithDelay(task.ReloadProtyle, 200*time.Millisecond, util.PushReloadProtyle, rootID)
}

// refreshRefCount 用于刷新定义块处的引用计数。
func refreshRefCount(blockID string) {
	sql.FlushQueue()

	bt := treenode.GetBlockTree(blockID)
	if nil == bt {
		return
	}

	isDoc := bt.ID == bt.RootID
	var rootRefIDs []string
	var refCount, rootRefCount int
	refIDs := sql.QueryRefIDsByDefIDInBox(bt.ID, isDoc, bt.BoxID)
	if isDoc {
		rootRefIDs = refIDs
	} else {
		rootRefIDs = sql.QueryRefIDsByDefIDInBox(bt.RootID, true, bt.BoxID)
	}
	refCount = len(refIDs)
	rootRefCount = len(rootRefIDs)
	var defIDs []string
	if isDoc {
		defIDs = sql.QueryChildDefIDsByRootDefIDInBox(bt.ID, bt.BoxID)
	} else {
		defIDs = append(defIDs, bt.ID)
	}

	util.PushSetDefRefCount(bt.RootID, blockID, defIDs, refCount, rootRefCount)
}

// refreshDynamicRefText 用于刷新块引用的动态锚文本。
// 该实现依赖了数据库缓存,导致外部调用时可能需要阻塞等待数据库写入后才能获取到 refs
func refreshDynamicRefText(updatedDefNode *ast.Node, updatedTree *parse.Tree) {
	changedDefs := map[string]*ast.Node{updatedDefNode.ID: updatedDefNode}
	changedTrees := map[string]*parse.Tree{updatedTree.ID: updatedTree}
	refreshDynamicRefTexts(changedDefs, changedTrees)
}

// refreshDynamicRefTexts 用于批量刷新块引用的动态锚文本。
// 该实现依赖了数据库缓存,导致外部调用时可能需要阻塞等待数据库写入后才能获取到 refs
func refreshDynamicRefTexts(updatedDefNodes map[string]*ast.Node, updatedTrees map[string]*parse.Tree) (changedRootIDs []string) {
	changedRootIDs, _ = refreshDynamicRefTextsWithWriter(updatedDefNodes, updatedTrees, writeTreeUpsertQueue)
	return
}

func refreshDynamicRefTextsWithWriter(updatedDefNodes map[string]*ast.Node, updatedTrees map[string]*parse.Tree,
	writeTree func(*parse.Tree) error) (changedRootIDs []string, failed bool) {
	for t := range updatedTrees {
		changedRootIDs = append(changedRootIDs, t)
	}

	for range 7 {
		updatedRefNodes, updatedRefTrees, writeFailed := refreshDynamicRefTexts0(updatedDefNodes, updatedTrees, writeTree)
		failed = failed || writeFailed
		if 1 > len(updatedRefNodes) {
			break
		}
		updatedDefNodes, updatedTrees = updatedRefNodes, updatedRefTrees

		for t := range updatedTrees {
			changedRootIDs = append(changedRootIDs, t)
		}
	}

	changedRootIDs = gulu.Str.RemoveDuplicatedElem(changedRootIDs)
	return
}

func refreshDynamicRefTexts0(updatedDefNodes map[string]*ast.Node, updatedTrees map[string]*parse.Tree,
	writeTree func(*parse.Tree) error) (updatedRefNodes map[string]*ast.Node, updatedRefTrees map[string]*parse.Tree, failed bool) {
	updatedRefNodes = map[string]*ast.Node{}
	updatedRefTrees = map[string]*parse.Tree{}

	// 1. 更新引用的动态锚文本
	treeRefNodeIDs := map[string]*hashset.Set{}
	var changedNodes []*ast.Node
	var refs []*sql.Ref
	var attributeViewRefs []*sql.Ref
	for _, updateNode := range updatedDefNodes {
		boxID := updatedNodeBoxID(updateNode, updatedTrees)
		refs, changedNodes = getRefsCacheByDefNode(updateNode, boxID)
		for _, ref := range refs {
			if sql.AttributeViewRefType == ref.Type {
				attributeViewRefs = append(attributeViewRefs, ref)
				continue
			}
			if refIDs, ok := treeRefNodeIDs[ref.RootID]; !ok {
				refIDs = hashset.New()
				refIDs.Add(ref.BlockID)
				treeRefNodeIDs[ref.RootID] = refIDs
			} else {
				refIDs.Add(ref.BlockID)
			}
		}
	}
	for _, n := range changedNodes {
		updatedDefNodes[n.ID] = n
	}
	refreshAttributeViewDynamicRefTexts(updatedDefNodes, attributeViewRefs)

	for refTreeID, refNodeIDs := range treeRefNodeIDs {
		refTree, ok := updatedTrees[refTreeID]
		if !ok {
			var err error
			refTree, err = LoadTreeByBlockID(refTreeID)
			if err != nil {
				if !errors.Is(err, ErrTreeNotFound) {
					failed = true
					queueDynamicRefTextRetries(updatedDefNodes, updatedTrees, err)
				}
				continue
			}
		}

		// 引用文本先在副本中更新,写入失败时保留已提交的树和索引。
		cloned := *refTree
		cloned.Root = cloneRenderNode(refTree.Root)
		refTree = &cloned
		changedRefNodes := map[string]*ast.Node{}
		type refTextUpdate struct {
			blockID string
			defNode *changedDefNode
		}
		var updates []refTextUpdate
		ast.Walk(refTree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
			if !entering {
				return ast.WalkContinue
			}

			if n.IsBlock() && refNodeIDs.Contains(n.ID) {
				changed, changedDefNodes := updateRefText(n, updatedDefNodes)
				if changed {
					changedRefNodes[n.ID] = n
				}

				// 保存成功后再推送动态锚文本节点刷新。
				for _, defNode := range changedDefNodes {
					switch defNode.refType {
					case "ref-d":
						updates = append(updates, refTextUpdate{n.ID, defNode})
					}
				}
				return ast.WalkContinue
			}
			return ast.WalkContinue
		})

		if len(changedRefNodes) == 0 {
			continue
		}
		if err := writeTree(refTree); err != nil {
			failed = true
			queueDynamicRefTextRetries(updatedDefNodes, updatedTrees, err)
			continue
		}
		treenode.UpsertBlockTree(refTree)
		sql.UpdateRefsTreeQueue(refTree)
		updatedRefTrees[refTreeID] = refTree
		if ok {
			updatedTrees[refTreeID] = refTree
		}
		maps.Copy(updatedRefNodes, changedRefNodes)
		for _, update := range updates {
			appendSetRefDynamicTextTask(refTreeID, update.blockID, update.defNode.id, update.defNode.refText, refTree.Box)
		}
	}

	// 2. 更新属性视图主键内容
	updateAttributeViewBlockText(updatedDefNodes)

	return
}

func updatedNodeBoxID(updateNode *ast.Node, updatedTrees map[string]*parse.Tree) string {
	rootID := treenode.TreeRoot(updateNode).ID
	if updatedTree := updatedTrees[rootID]; nil != updatedTree {
		return updatedTree.Box
	}
	return updateNode.Box
}

var (
	setRefDynamicTextTaskLock     sync.Mutex
	setRefDynamicTextTaskSequence uint64
	setRefDynamicTextLatestTasks  = map[string]uint64{}
)

func appendSetRefDynamicTextTask(rootID, blockID, defBlockID, refText, boxID string) {
	key := rootID + "\x00" + blockID + "\x00" + defBlockID
	setRefDynamicTextTaskLock.Lock()
	setRefDynamicTextTaskSequence++
	sequence := setRefDynamicTextTaskSequence
	setRefDynamicTextLatestTasks[key] = sequence
	setRefDynamicTextTaskLock.Unlock()

	task.AppendAsyncTaskWithDelay(task.SetRefDynamicText, 200*time.Millisecond, pushLatestRefDynamicText,
		key, sequence, rootID, blockID, defBlockID, refText, boxID)
}

func pushLatestRefDynamicText(key string, sequence uint64, rootID, blockID, defBlockID, refText, boxID string) {
	setRefDynamicTextTaskLock.Lock()
	latest := setRefDynamicTextLatestTasks[key] == sequence
	if latest {
		delete(setRefDynamicTextLatestTasks, key)
	}
	setRefDynamicTextTaskLock.Unlock()
	if latest {
		util.PushSetRefDynamicText(rootID, blockID, defBlockID, refText, boxID)
	}
}

func updateAttributeViewBlockText(updatedDefNodes map[string]*ast.Node) {
	var parents []*ast.Node
	for _, node := range updatedDefNodes {
		for parent := node.Parent; parent != nil && parent.Type != ast.NodeDocument; parent = parent.Parent {
			parents = append(parents, parent)
		}
	}
	for _, parent := range parents {
		updatedDefNodes[parent.ID] = parent
	}
	// 按数据库汇总主键刷新,同批块更新只解析和保存每个数据库一次。
	byView := map[string]map[string]*ast.Node{}
	for _, node := range updatedDefNodes {
		for id := range strings.SplitSeq(node.IALAttr(av.NodeAttrNameAvs), ",") {
			if id == "" {
				continue
			}
			if byView[id] == nil {
				byView[id] = map[string]*ast.Node{}
			}
			byView[id][node.ID] = node
		}
	}
	for _, id := range sortedAttributeViewFieldKeys(byView) {
		view, err := av.ParseAttributeView(id)
		if err != nil {
			continue
		}
		primary := view.GetBlockKeyValues()
		if primary == nil {
			continue
		}
		changed := false
		for _, value := range primary.Values {
			node := byView[id][value.Block.ID]
			if node == nil {
				continue
			}
			icon, content := getNodeAvBlockText(node, id)
			content = util.UnescapeHTML(content)
			subtype := getNodeAvBlockRefSubtype(node, id)
			if icon != value.Block.Icon || content != value.Block.Content || subtype != value.Block.RefSubtype {
				value.Block.Icon, value.Block.Content, value.Block.RefSubtype = icon, content, subtype
				changed = true
			}
		}
		if changed {
			if err = av.SaveAttributeView(view); err != nil {
				continue
			}
			ReloadAttrView(id)
			refreshRelatedSrcAvs(id, nil)
		}
	}
}

// ReloadAttrView 用于重新加载属性视图。
func ReloadAttrView(avID string) {
	task.AppendAsyncTaskWithDelay(task.ReloadAttributeView, 200*time.Millisecond, pushReloadAttrView, avID)
}

func pushReloadAttrView(avID string) {
	util.BroadcastByType("protyle", "refreshAttributeView", 0, "", map[string]any{"id": avID})
}

func PushCreate(box *Box, p string, arg map[string]any) {
	evt := util.NewCmdResult("create", 0, util.PushModeBroadcast)
	listDocTree := false
	if nil == arg {
		arg = map[string]any{
			"listDocTree": true,
		}
	}

	listDocTreeArg := arg["listDocTree"]
	if nil != listDocTreeArg {
		listDocTree = listDocTreeArg.(bool)
	}

	evt.Data = map[string]any{
		"box":         box,
		"path":        p,
		"listDocTree": listDocTree,
	}
	util.PushEvent(evt)
}