Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

11.4 KB
Raw Blame
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later

package model

import (
	"bytes"
	"regexp"
	"sort"
	"strings"
	"time"
	"unsafe"

	"github.com/88250/gulu"
	"github.com/88250/lute"
	"github.com/88250/lute/ast"
	"github.com/88250/lute/editor"
	"github.com/88250/lute/parse"
	"github.com/ClarkThan/ahocorasick"
	"github.com/dgraph-io/ristretto"
	"github.com/siyuan-note/siyuan/kernel/cache"
	"github.com/siyuan-note/siyuan/kernel/conf"
	"github.com/siyuan-note/siyuan/kernel/search"
	"github.com/siyuan-note/siyuan/kernel/sql"
	"github.com/siyuan-note/siyuan/kernel/task"
	"github.com/siyuan-note/siyuan/kernel/treenode"
	"github.com/siyuan-note/siyuan/kernel/util"
)

// virtualBlockRefCache 用于保存块关联的虚拟引用关键字。
// 改进打开虚拟引用后加载文档的性能 https://github.com/siyuan-note/siyuan/issues/7378
var virtualBlockRefCache, _ = ristretto.NewCache(&ristretto.Config{
	NumCounters: cache.AdmissionCounters,
	MaxCost:     cache.LookupMaxCostBytes,
	BufferItems: 64,
})

// putVirtualRefCache 按关键字切片及字符串的驻留字节数计费,保留调用方指定的有效期。
func putVirtualRefCache(key string, keywords []string, ttl time.Duration) {
	cost := int64(unsafe.Sizeof(keywords)) + int64(cap(keywords))*int64(unsafe.Sizeof("")) + int64(len(key))
	for _, keyword := range keywords {
		cost += int64(len(keyword))
	}
	virtualBlockRefCache.Del(key)
	if cost >= virtualBlockRefCache.MaxCost() {
		return
	}
	virtualBlockRefCache.SetWithTTL(key, keywords, cost, ttl)
}

func getBlockVirtualRefKeywords(root *ast.Node, boxID string) (ret []string) {
	key := boxID + "\x00" + root.ID
	val, ok := virtualBlockRefCache.Get(key)
	if !ok {
		buf := bytes.Buffer{}
		ast.Walk(root, func(n *ast.Node, entering bool) ast.WalkStatus {
			if !entering || !n.IsBlock() {
				return ast.WalkContinue
			}

			content := sql.NodeStaticContent(n, nil, false, false, false)
			content = strings.ReplaceAll(content, editor.Zwsp, "")
			buf.WriteString(content)
			return ast.WalkContinue
		})
		content := buf.String()
		ret = putBlockVirtualRefKeywords(content, root, boxID)
		return
	}
	ret = val.([]string)
	return
}

func putBlockVirtualRefKeywords(blockContent string, root *ast.Node, boxID string) (ret []string) {
	keywords := getVirtualRefKeywords(root, boxID)
	if 1 > len(keywords) {
		return
	}

	contentTmp := blockContent
	var keywordsTmp []string
	if !Conf.Search.Load().CaseSensitive {
		contentTmp = strings.ToLower(blockContent)
		for _, keyword := range keywords {
			keywordsTmp = append(keywordsTmp, strings.ToLower(keyword))
		}
	} else {
		for _, keyword := range keywords {
			keywordsTmp = append(keywordsTmp, keyword)
		}
	}

	m := ahocorasick.NewMatcher()
	m.BuildWithPatterns(keywordsTmp)
	hits := m.Search(contentTmp)
	for _, hit := range hits {
		ret = append(ret, hit)
	}

	if 1 > len(ret) {
		return
	}

	ret = gulu.Str.RemoveDuplicatedElem(ret)
	key := boxID + "\x00" + root.ID
	putVirtualRefCache(key, ret, 10*time.Minute)
	return
}

func CacheVirtualBlockRefJob() {
	if !Conf.Editor.Load().VirtualBlockRef {
		return
	}
	task.AppendTask(task.CacheVirtualBlockRef, ResetVirtualBlockRefCache)
}

func ResetVirtualBlockRefCache() {
	confSearchReadSnapshot := Conf.LoadSearch()

	virtualBlockRefCache.Clear()
	if nil == Conf {
		return
	}
	if !Conf.Editor.Load().VirtualBlockRef {
		return
	}

	searchIgnoreLines := getSearchIgnoreLines()
	refSearchIgnoreLines := getRefSearchIgnoreLines()
	keywords := sql.QueryVirtualRefKeywords(confSearchReadSnapshot.VirtualRefName, confSearchReadSnapshot.VirtualRefAlias, confSearchReadSnapshot.VirtualRefAnchor, confSearchReadSnapshot.VirtualRefDoc, searchIgnoreLines, refSearchIgnoreLines)
	putVirtualRefCache("virtual_ref", keywords, 0)
	for _, boxID := range treenode.GetOpenedEncryptedBoxIDs() {
		boxKeywords := sql.QueryVirtualRefKeywords(confSearchReadSnapshot.VirtualRefName, confSearchReadSnapshot.VirtualRefAlias, confSearchReadSnapshot.VirtualRefAnchor, confSearchReadSnapshot.VirtualRefDoc, searchIgnoreLines, refSearchIgnoreLines, boxID)
		putVirtualRefCache(boxID+"\x00virtual_ref", boxKeywords, 0)
	}
}

// addNewKeywords 将新关键字添加到虚拟引用关键字列表中,如果不存在则追加,保留空白字符
func addNewKeywords(keywordsStr string, newKeywords []string) string {
	keywordsStr = strings.TrimSpace(keywordsStr)
	if 0 == len(newKeywords) {
		return keywordsStr
	}

	var builder strings.Builder
	if "" != keywordsStr {
		if !strings.HasSuffix(keywordsStr, "\\,") {
			keywordsStr = strings.TrimSuffix(keywordsStr, ",")
		}
		builder.WriteString(keywordsStr)
		builder.WriteString(",")
	}

	keywords := gulu.Str.RemoveDuplicatedElem(parseKeywords(keywordsStr))
	newKeywords = gulu.Str.RemoveDuplicatedElem(newKeywords)
	allKeys := make(map[string]bool)

	// 添加新关键字
	for _, keyword := range newKeywords {
		keywordTrimmed := strings.TrimSpace(keyword)
		if "" == keywordTrimmed {
			continue
		}
		if gulu.Str.Contains(keywordTrimmed, keywords) {
			// 剔除已存在的关键字
			continue
		}
		if _, value := allKeys[keywordTrimmed]; value {
			// 剔除重复的关键字
			continue
		}
		allKeys[keywordTrimmed] = true
		builder.WriteString(strings.ReplaceAll(keyword, ",", "\\,")) // 字符串切片转换为字符串,需要转义逗号
		builder.WriteString(",")
	}

	return strings.TrimSuffix(builder.String(), ",")
}

func AddVirtualBlockRefInclude(keyword []string) {
	if 1 > len(keyword) {
		return
	}
	UpdateEditorConf(func(s *conf.Editor) { s.VirtualBlockRefInclude = addNewKeywords(s.VirtualBlockRefInclude, keyword) })
	ResetVirtualBlockRefCache()
}

func AddVirtualBlockRefExclude(keyword []string) {
	if 1 > len(keyword) {
		return
	}
	UpdateEditorConf(func(s *conf.Editor) { s.VirtualBlockRefExclude = addNewKeywords(s.VirtualBlockRefExclude, keyword) })
	ResetVirtualBlockRefCache()
}

func processVirtualRef(n *ast.Node, unlinks *[]*ast.Node, virtualBlockRefKeywords []string, refCount map[string]int, luteEngine *lute.Lute) bool {
	if !Conf.Editor.Load().VirtualBlockRef || 1 > len(virtualBlockRefKeywords) {
		return false
	}

	if ast.NodeText != n.Type {
		return false
	}

	parentBlock := treenode.ParentBlock(n)
	if nil == parentBlock {
		return false
	}

	if 0 < refCount[parentBlock.ID] {
		// 如果块被引用过,则将其自身的文本排除在虚拟引用关键字之外
		// Referenced blocks support rendering virtual references https://github.com/siyuan-note/siyuan/issues/10960
		parentText := getNodeRefText(parentBlock)
		virtualBlockRefKeywords = gulu.Str.RemoveElem(virtualBlockRefKeywords, parentText)
	}

	content := string(n.Tokens)
	tmp := util.RemoveInvalid(content)
	tmp = strings.TrimSpace(tmp)
	if "" == tmp {
		return false
	}

	newContent, matched := markReplaceSpanWithSplit(content, virtualBlockRefKeywords, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd())
	if matched {
		// 虚拟引用排除命中自身块命名和别名的情况 https://github.com/siyuan-note/siyuan/issues/3185
		var blockKeys []string
		if name := parentBlock.IALAttr("name"); "" != name {
			blockKeys = append(blockKeys, name)
		}
		if alias := parentBlock.IALAttr("alias"); "" != alias {
			blockKeys = append(blockKeys, alias)
		}
		if 0 < len(blockKeys) {
			keys := getMarkedTextContents(newContent, search.GetMarkSpanStart(search.VirtualBlockRefDataType), search.GetMarkSpanEnd())
			for _, k := range keys {
				if gulu.Str.Contains(k, blockKeys) {
					return true
				}
			}
		}

		n.Tokens = []byte(newContent)
		linkTree := parse.Inline("", n.Tokens, luteEngine.ParseOptions)
		var children []*ast.Node
		for c := linkTree.Root.FirstChild.FirstChild; nil != c; c = c.Next {
			children = append(children, c)
		}
		for _, c := range children {
			n.InsertBefore(c)
		}
		*unlinks = append(*unlinks, n)
		return true
	}
	return false
}

// parseKeywords 将字符串转换为关键字切片,并剔除前后的空白字符
func parseKeywords(keywordsStr string) (keywords []string) {
	keywords = []string{}
	keywordsStr = strings.TrimSpace(keywordsStr)
	if "" == keywordsStr {
		return
	}
	// 先处理转义的逗号
	keywordsStr = strings.ReplaceAll(keywordsStr, "\\,", "__comma@sep__")
	// 再将连续或单个换行符替换为一个逗号,避免把 `\\\n` 转换为 `\,`
	keywordsStr = util.ReplaceNewline(keywordsStr, ",")
	// 按逗号分隔
	for part := range strings.SplitSeq(keywordsStr, ",") {
		part = strings.TrimSpace(part) // 剔除前后的空白字符
		if "" == part {
			continue
		}
		// 恢复转义的逗号
		part = strings.ReplaceAll(part, "__comma@sep__", ",")
		keywords = append(keywords, part)
	}
	return
}

func getVirtualRefKeywords(root *ast.Node, boxIDs ...string) (ret []string) {
	confEditorSnapshot := Conf.LoadEditor()

	confSearchSnapshot := Conf.LoadSearch()

	if !confEditorSnapshot.VirtualBlockRef {
		return
	}

	key := "virtual_ref"
	if len(boxIDs) > 0 && boxIDs[0] != "" {
		key = boxIDs[0] + "\x00virtual_ref"
	}
	if val, ok := virtualBlockRefCache.Get(key); ok {
		ret = val.([]string)
	} else {
		searchIgnoreLines := getSearchIgnoreLines()
		refSearchIgnoreLines := getRefSearchIgnoreLines()
		ret = sql.QueryVirtualRefKeywords(confSearchSnapshot.VirtualRefName, confSearchSnapshot.VirtualRefAlias,
			confSearchSnapshot.VirtualRefAnchor, confSearchSnapshot.VirtualRefDoc, searchIgnoreLines, refSearchIgnoreLines, boxIDs...)
		putVirtualRefCache(key, ret, 0)
	}

	includes := parseKeywords(confEditorSnapshot.VirtualBlockRefInclude)
	if 0 < len(includes) {
		ret = append(ret, includes...)
		ret = gulu.Str.RemoveDuplicatedElem(ret)
	}

	ret = excludeKeywords(ret, confEditorSnapshot.VirtualBlockRefExclude)

	// 虚拟引用排除当前文档名 https://github.com/siyuan-note/siyuan/issues/4537
	// Virtual references exclude the name and aliases from the current document https://github.com/siyuan-note/siyuan/issues/9204
	title := treenode.ReadDocumentTitle(root).Text
	ret = gulu.Str.ExcludeElem(ret, []string{title})
	if name := root.IALAttr("name"); "" != name {
		ret = gulu.Str.ExcludeElem(ret, []string{name})
	}
	if alias := root.IALAttr("alias"); "" != alias {
		for a := range strings.SplitSeq(alias, ",") {
			ret = gulu.Str.ExcludeElem(ret, []string{a})
		}
	}

	ret = prepareMarkKeywords(ret)
	return
}

func excludeKeywords(keywords []string, excludesStr string) (ret []string) {
	excludes := parseKeywords(excludesStr)
	if 0 < len(excludes) {
		var tmp []string
		var regexps []*regexp.Regexp
		for _, e := range excludes {
			if 2 <= len(e) && strings.HasPrefix(e, "/") && strings.HasSuffix(e, "/") {
				if re, err := regexp.Compile(e[1 : len(e)-1]); nil == err {
					regexps = append(regexps, re)
				}
			} else {
				tmp = append(tmp, e)
			}
		}
		excludes = tmp
		keywords = gulu.Str.ExcludeElem(keywords, excludes)
		if 0 < len(regexps) {
			tmp = nil
			for _, str := range keywords {
				matchExclude := false
				for _, re := range regexps {
					if re.MatchString(str) {
						matchExclude = true
						break
					}
				}
				if !matchExclude {
					tmp = append(tmp, str)
				}
			}
			keywords = tmp
		}
	}
	ret = keywords
	return
}

func prepareMarkKeywords(keywords []string) (ret []string) {
	ret = gulu.Str.RemoveDuplicatedElem(keywords)
	var tmp []string
	for _, k := range ret {
		if "" != k && "*" != k { // 提及和虚引排除 * Ignore `*` back mentions and virtual references https://github.com/siyuan-note/siyuan/issues/10873
			tmp = append(tmp, k)
		}
	}
	ret = tmp

	sort.SliceStable(ret, func(i, j int) bool {
		return len(ret[i]) > len(ret[j])
	})
	return
}