An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later
package model
import (
"bytes"
"context"
"errors"
"fmt"
"io/fs"
"mime"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"sort"
"strings"
"time"
"github.com/88250/go-humanize"
"github.com/88250/gulu"
"github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/88250/lute/editor"
"github.com/88250/lute/html"
"github.com/88250/lute/parse"
"github.com/disintegration/imaging"
"github.com/gabriel-vasile/mimetype"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/httpclient"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/av"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/heif"
"github.com/siyuan-note/siyuan/kernel/search"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
func GetAssetImgSizeInBox(assetPath, boxID string) (width, height int) {
absPath, resolveErr := GetAssetAbsPathInBox(assetPath, boxID)
if resolveErr != nil || !filelock.IsExist(absPath) {
// 计算布局尺寸不触发资源下载,展示图片或完整导出时再补齐内容。
return
}
data, err := readAssetBytesInBox(boxID, assetPath, false)
if err != nil {
logging.LogErrorf("get asset [%s] abs path failed: %s", assetPath, err)
return
}
defer clear(data)
if heif.IsPath(assetPath) {
width, height, err = heif.ImageSize(data)
if err != nil {
logging.LogErrorf("open asset image [%s] failed: %s", assetPath, err)
}
return
}
img, err := imaging.Decode(bytes.NewReader(data))
if err != nil {
logging.LogErrorf("open asset image [%s] failed: %s", assetPath, err)
return
}
width = img.Bounds().Dx()
height = img.Bounds().Dy()
return
}
// ReadAssetBytesInBox 读取指定 box 内的资源文件字节。若 box 为加密笔记本则自动解密返回明文。
// relativePath 形如 "assets/xxx.png",可带 ?box= 查询参数。
func ReadAssetBytesInBox(boxID, relativePath string) ([]byte, error) {
return readAssetBytesInBox(boxID, relativePath, true)
}
func readAssetBytesInBox(boxID, relativePath string, download bool) ([]byte, error) {
absPath, err := GetAssetAbsPathInBox(relativePath, boxID)
if err != nil {
return nil, err
}
if download {
if err = ensureReadableAssetLocal(absPath); err != nil {
return nil, err
}
}
var data []byte
var readErr error
if heif.IsPath(relativePath) {
limit := int64(heif.MaxInputBytes)
if effectiveBoxID := util.BoxIDFromDataPath(absPath); effectiveBoxID != "" && IsEncryptedBox(effectiveBoxID) {
limit += 2 * 1024 * 1024
}
data, readErr = heif.ReadFileLimited(absPath, limit)
} else {
data, readErr = os.ReadFile(absPath)
}
if readErr != nil {
return nil, readErr
}
// 从解析到的绝对路径反推有效 boxID,不轻信传入参数(路径可能通过 ?box= 指定了不同 box)
effectiveBoxID := util.BoxIDFromDataPath(absPath)
if effectiveBoxID != "" && IsEncryptedBox(effectiveBoxID) {
HoldBoxReadLock(effectiveBoxID)
dek, dekErr := GetDEKIfUnlocked(effectiveBoxID)
if dekErr != nil {
ReleaseBoxReadLock(effectiveBoxID)
return nil, dekErr
}
defer clear(dek)
defer ReleaseBoxReadLock(effectiveBoxID)
diskName := filepath.Base(AssetPathWithoutQuery(relativePath))
plain, decErr := DecryptAsset(effectiveBoxID, diskName, dek, data)
if decErr != nil {
return nil, decErr
}
if heif.IsPath(relativePath) && len(plain) > heif.MaxInputBytes {
clear(plain)
return nil, heif.ErrInputTooLarge
}
return plain, nil
}
return data, nil
}
// GetAssetPathByHash 按 hash 查已存在的 asset 路径用于去重。boxID 非空且为加密笔记本时返回空——
// 加密笔记本不参与全局去重(避免复用普通明文 asset);普通 box 走全局 cache/SQL(加密笔记本数据不在全局表)。
func GetAssetPathByHash(hash, boxID string) string {
if boxID != "" && IsEncryptedBox(boxID) {
return "" // 加密 box:跳过全局去重,强制新写(防跨边界复用明文 asset)
}
assetHash := cache.GetAssetHash(hash)
if nil == assetHash {
sqlAsset := sql.QueryAssetByHash(hash)
if nil == sqlAsset {
return ""
}
cache.SetAssetHash(sqlAsset.Hash, sqlAsset.Path)
return sqlAsset.Path
}
return assetHash.Path
}
func HandleAssetsRemoveEvent(assetAbsPath string) {
relativePath, err := filepath.Rel(util.DataDir, assetAbsPath)
if err != nil {
return
}
relativePath = filepath.ToSlash(relativePath)
if _, _, ok := AssetPathFromDataRelativePath(relativePath); !ok {
return
}
info, err := os.Stat(assetAbsPath)
if err != nil && !os.IsNotExist(err) {
return
}
if info != nil && (info.IsDir() || filelock.IsHidden(assetAbsPath)) {
return
}
// 删除后无法读取文件属性,仍按文件名过滤 WPS 临时文件和 .DS_Store 等隐藏文件。
if strings.HasPrefix(filepath.Base(assetAbsPath), ".") {
return
}
if util.IsOfficeTempFile(assetAbsPath) {
return
}
if strings.HasSuffix(assetAbsPath, ".tmp") {
return
}
// 加密笔记本的 asset 是密文,跳过全局索引和 hash 缓存操作
if IsEncryptedAssetPath(assetAbsPath) {
removeAssetThumbnail(assetAbsPath)
return
}
removeIndexAssetContent(assetAbsPath)
removeAssetThumbnail(assetAbsPath)
cache.RemoveAssetHashByPath(relativePath)
}
func HandleAssetsChangeEvent(assetAbsPath string) {
if !filelock.IsExist(assetAbsPath) {
return
}
if gulu.File.IsDir(assetAbsPath) {
return
}
if filelock.IsHidden(assetAbsPath) {
return
}
if util.IsOfficeTempFile(assetAbsPath) {
return
}
if strings.HasSuffix(assetAbsPath, ".tmp") {
return
}
// 加密笔记本的 asset 是密文,跳过全局内容索引和 hash 缓存(避免密文污染搜索索引)
if IsEncryptedAssetPath(assetAbsPath) {
removeAssetThumbnail(assetAbsPath)
return
}
indexAssetContent(assetAbsPath)
removeAssetThumbnail(assetAbsPath)
hash, err := util.GetEtag(assetAbsPath)
if nil != err {
logging.LogErrorf("calc asset [%s] hash failed: %s", assetAbsPath, err)
} else {
p := strings.TrimPrefix(assetAbsPath, util.DataDir)
p = strings.TrimPrefix(filepath.ToSlash(p), "/")
cache.SetAssetHash(hash, p)
}
}
func removeAssetThumbnail(assetAbsPath string) {
relativePath, err := filepath.Rel(util.DataDir, assetAbsPath)
if err != nil {
return
}
// 按数据目录内的资源路径匹配缩略图缓存,保留资源子目录和文件名大小写。
assetPath, _, ok := AssetPathFromDataRelativePath(filepath.ToSlash(relativePath))
if !ok || !util.IsCompressibleAssetImage(assetPath) {
return
}
thumbnailPath := filepath.Join(util.TempDir, "thumbnails", assetPath)
if err = os.Remove(thumbnailPath); err != nil && !os.IsNotExist(err) {
logging.LogErrorf("remove asset thumbnail [%s] failed: %s", thumbnailPath, err)
}
}
func NeedGenerateAssetsThumbnail(sourceImgPath string) bool {
info, err := os.Stat(sourceImgPath)
if err != nil {
return false
}
if info.IsDir() {
return false
}
return info.Size() > 1024*1024
}
func GenerateAssetsThumbnail(sourceImgPath, resizedImgPath string) (err error) {
start := time.Now()
img, err := imaging.Open(sourceImgPath)
if err != nil {
return
}
// 获取原图宽高
originalWidth := img.Bounds().Dx()
originalHeight := img.Bounds().Dy()
// 固定最大宽度为 520,计算缩放比例
maxWidth := 520
scale := float64(maxWidth) / float64(originalWidth)
// 按比例计算新的宽高
newWidth := maxWidth
newHeight := int(float64(originalHeight) * scale)
// 缩放图片
resizedImg := imaging.Resize(img, newWidth, newHeight, imaging.Lanczos)
// 保存缩放后的图片
err = os.MkdirAll(filepath.Dir(resizedImgPath), 0755)
if err != nil {
return
}
err = imaging.Save(resizedImg, resizedImgPath)
if err != nil {
return
}
logging.LogDebugf("generated thumbnail image [%s] to [%s], cost [%d]ms", sourceImgPath, resizedImgPath, time.Since(start).Milliseconds())
return
}
func DocImageAssets(rootID string) (ret []string, err error) {
tree, err := LoadTreeByBlockID(rootID)
if err != nil {
return
}
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeImage == n.Type {
linkDest := n.ChildByType(ast.NodeLinkDest)
dest := linkDest.Tokens
if 1 > len(dest) { // 双击打开图片不对 https://github.com/siyuan-note/siyuan/issues/5876
return ast.WalkContinue
}
ret = append(ret, gulu.Str.FromBytes(dest))
}
return ast.WalkContinue
})
return
}
// AgentMessageImageAssets 返回智能体用户消息中引用的全局资源图片。
func AgentMessageImageAssets(markdown string) []string {
return agentMessageImageAssets(markdown, util.NewLute())
}
func agentMessageImageAssets(markdown string, luteEngine *lute.Lute) (ret []string) {
if strings.TrimSpace(markdown) == "" {
return
}
tree := parse.Parse("", []byte(markdown), luteEngine.ParseOptions)
seen := map[string]bool{}
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || n.Type != ast.NodeImage {
return ast.WalkContinue
}
linkDest := n.ChildByType(ast.NodeLinkDest)
if linkDest == nil {
return ast.WalkContinue
}
dest := strings.TrimSpace(linkDest.TokensStr())
if !strings.HasPrefix(AssetPathWithoutQuery(dest), "assets/") || seen[dest] {
return ast.WalkContinue
}
seen[dest] = true
ret = append(ret, dest)
return ast.WalkContinue
})
return
}
type ImageArtifactRef struct {
Kind string `json:"kind"`
Path string `json:"path"`
MIMEType string `json:"mimeType,omitempty"`
DocumentID string `json:"documentId,omitempty"`
}
type DocumentImageList struct {
DocumentID string `json:"documentId"`
Images []ImageArtifactRef `json:"images"`
}
const (
documentImageMaxBytes = 20 * 1024 * 1024
documentImageMaxPixels = 40 * 1000 * 1000
documentImageMaxEdge = 2048
)
type PreparedDocumentImage struct {
Artifact ImageArtifactRef `json:"artifact"`
Data []byte `json:"-"`
MIMEType string `json:"mimeType"`
Prepared util.PreparedImage `json:"-"`
}
type GenerateImageRequest struct {
Prompt string `json:"prompt"`
Size string `json:"size"`
Quality string `json:"quality"`
OutputFormat string `json:"outputFormat"`
}
type GenerateImageResult struct {
Data []byte `json:"-"`
MIMEType string `json:"mimeType"`
Extension string `json:"extension"`
RevisedPrompt string `json:"revisedPrompt,omitempty"`
}
type GenerateDocumentImageRequest struct {
DocumentID string `json:"documentId"`
Prompt string `json:"prompt"`
Size string `json:"size"`
Quality string `json:"quality"`
OutputFormat string `json:"outputFormat"`
}
type GenerateDocumentImageResult struct {
Artifact ImageArtifactRef `json:"artifact"`
RevisedPrompt string `json:"revisedPrompt,omitempty"`
}
type imageExecutionUnknownError struct {
err error
}
func (err *imageExecutionUnknownError) Error() string {
return err.err.Error()
}
func (err *imageExecutionUnknownError) Unwrap() error {
return err.err
}
// IsImageExecutionUnknown 判断图片提供商调用是否已经开始,调用方不得自动重试这类错误。
func IsImageExecutionUnknown(err error) bool {
var target *imageExecutionUnknownError
return errors.As(err, &target)
}
func markImageExecutionUnknown(err error) error {
if err == nil || IsImageExecutionUnknown(err) {
return err
}
return &imageExecutionUnknownError{err: err}
}
// ListDocumentImages 返回文档引用的本地图片,供智能体工具和编辑器功能复用。
func ListDocumentImages(documentID string) (DocumentImageList, error) {
bt, err := resolveMultimodalDocument(documentID)
if err != nil {
return DocumentImageList{}, err
}
paths, err := DocImageAssets(bt.RootID)
if err != nil {
return DocumentImageList{}, fmt.Errorf("list document images failed: %w", err)
}
refs := make([]ImageArtifactRef, 0, len(paths))
seen := map[string]bool{}
for _, assetPath := range paths {
if !strings.HasPrefix(AssetPathWithoutQuery(assetPath), "assets/") || seen[assetPath] {
continue
}
seen[assetPath] = true
refs = append(refs, ImageArtifactRef{Kind: "image", Path: assetPath, DocumentID: bt.RootID})
}
return DocumentImageList{DocumentID: bt.RootID, Images: refs}, nil
}
// PrepareDocumentImage 校验并读取文档实际引用的本地资源图片,供当前模型直接接收图片输入。
func PrepareDocumentImage(documentID, assetPath string) (PreparedDocumentImage, error) {
assetPath = strings.TrimSpace(assetPath)
if assetPath == "" {
return PreparedDocumentImage{}, errors.New("assetPath is required for analyze")
}
if !strings.HasPrefix(AssetPathWithoutQuery(assetPath), "assets/") {
return PreparedDocumentImage{}, errors.New("only local assets/... images are supported")
}
bt, err := resolveMultimodalDocument(documentID)
if err != nil {
return PreparedDocumentImage{}, err
}
if !documentReferencesImage(bt.RootID, assetPath) {
return PreparedDocumentImage{}, errors.New("assetPath is not an image referenced by the document")
}
data, err := ReadAssetBytesInBox(bt.BoxID, assetPath)
if err != nil {
return PreparedDocumentImage{}, fmt.Errorf("read image failed: %w", err)
}
prepared, err := util.PrepareModelImage(
data, documentImageMaxBytes, documentImageMaxPixels, documentImageMaxEdge,
)
if err != nil {
return PreparedDocumentImage{}, err
}
return PreparedDocumentImage{
Artifact: ImageArtifactRef{Kind: "image", Path: assetPath, DocumentID: bt.RootID},
Data: prepared.Data,
MIMEType: prepared.MIMEType,
Prepared: prepared,
}, nil
}
// PrepareAgentMessageImage 校验并读取智能体用户消息引用的全局资源图片。
func PrepareAgentMessageImage(assetPath string) (PreparedDocumentImage, error) {
assetPath = strings.TrimSpace(assetPath)
cleanPath := AssetPathWithoutQuery(assetPath)
if !strings.HasPrefix(cleanPath, "assets/") {
return PreparedDocumentImage{}, errors.New("only global assets/... images are supported")
}
relativePath, absPath, err := ResolveDataAssetPath(cleanPath)
if err != nil {
return PreparedDocumentImage{}, err
}
if !strings.HasPrefix(relativePath, "assets/") {
return PreparedDocumentImage{}, errors.New("only global assets/... images are supported")
}
if err = EnsureAssetLocal(absPath); err != nil {
return PreparedDocumentImage{}, err
}
data, err := os.ReadFile(absPath)
if err != nil {
return PreparedDocumentImage{}, fmt.Errorf("read image failed: %w", err)
}
prepared, err := util.PrepareModelImage(
data, documentImageMaxBytes, documentImageMaxPixels, documentImageMaxEdge,
)
if err != nil {
return PreparedDocumentImage{}, err
}
return PreparedDocumentImage{
Artifact: ImageArtifactRef{Kind: "image", Path: relativePath},
Data: prepared.Data,
MIMEType: prepared.MIMEType,
Prepared: prepared,
}, nil
}
// GenerateImage 使用全局图片生成配置创建图片字节,可供文档资源、编辑器和其他图片入口复用。
func GenerateImage(ctx context.Context, request GenerateImageRequest) (GenerateImageResult, error) {
confAIReadSnapshot := Conf.LoadAI()
if Conf == nil || confAIReadSnapshot == nil {
return GenerateImageResult{}, errors.New("AI configuration is unavailable")
}
provider, generationModel := confAIReadSnapshot.GetImageGenerationModel()
if err := validateImageModel(provider, generationModel); err != nil {
return GenerateImageResult{}, err
}
prompt := strings.TrimSpace(request.Prompt)
if prompt == "" {
return GenerateImageResult{}, errors.New("prompt is required for image generation")
}
size := multimodalValueOrDefault(request.Size, confAIReadSnapshot.ImageGeneration.Size)
quality := multimodalValueOrDefault(request.Quality, confAIReadSnapshot.ImageGeneration.Quality)
outputFormat := strings.ToLower(multimodalValueOrDefault(request.OutputFormat, confAIReadSnapshot.ImageGeneration.OutputFormat))
if outputFormat != "png" && outputFormat != "jpeg" && outputFormat != "webp" {
return GenerateImageResult{}, errors.New("unsupported image output format")
}
generated, err := util.NewOpenAIImageAdapter(
provider.APIKey, provider.BaseURL, generationModel.Name, confAIReadSnapshot.ImageGeneration.RequestTimeout, ResolveAIProviderHeaders(provider),
).Generate(ctx, util.GenerateImageRequest{
Prompt: prompt, Size: size, Quality: quality, OutputFormat: outputFormat,
})
if err != nil {
return GenerateImageResult{}, markImageExecutionUnknown(fmt.Errorf("generate image failed: %w", err))
}
if ctx.Err() != nil {
return GenerateImageResult{}, markImageExecutionUnknown(errors.New("image generation was cancelled"))
}
return GenerateImageResult{
Data: generated.Data, MIMEType: generated.MIMEType, Extension: generated.Extension, RevisedPrompt: generated.RevisedPrompt,
}, nil
}
// GenerateDocumentImage 使用通用图片生成能力创建文档资源。
func GenerateDocumentImage(ctx context.Context, request GenerateDocumentImageRequest) (GenerateDocumentImageResult, error) {
bt, err := resolveMultimodalDocument(request.DocumentID)
if err != nil {
return GenerateDocumentImageResult{}, err
}
generated, err := GenerateImage(ctx, GenerateImageRequest{
Prompt: request.Prompt, Size: request.Size, Quality: request.Quality, OutputFormat: request.OutputFormat,
})
if err != nil {
return GenerateDocumentImageResult{}, err
}
assetPath, _, err := InsertAssetBytes(bt.RootID, "ai-image"+generated.Extension, generated.Data)
if err != nil {
return GenerateDocumentImageResult{}, markImageExecutionUnknown(fmt.Errorf("save generated image failed: %w", err))
}
return GenerateDocumentImageResult{
Artifact: ImageArtifactRef{
Kind: "image", Path: assetPath, MIMEType: generated.MIMEType, DocumentID: bt.RootID,
},
RevisedPrompt: generated.RevisedPrompt,
}, nil
}
func resolveMultimodalDocument(documentID string) (*treenode.BlockTree, error) {
bt := treenode.GetBlockTree(strings.TrimSpace(documentID))
if bt == nil {
return nil, errors.New("document not found: " + documentID)
}
return bt, nil
}
func validateImageModel(provider *conf.Provider, imageModel *conf.Model) error {
if provider == nil || imageModel == nil {
return errors.New("image model is not configured")
}
if provider.Protocol != "" && provider.Protocol != util.OpenAIProtocolChatCompletions &&
provider.Protocol != util.OpenAIProtocolResponses {
return fmt.Errorf("unsupported multimodal provider protocol: %s", provider.Protocol)
}
return nil
}
func documentReferencesImage(rootID, assetPath string) bool {
paths, err := DocImageAssets(rootID)
if err != nil {
return false
}
wanted := AssetPathWithoutQuery(assetPath)
for _, current := range paths {
if AssetPathWithoutQuery(current) == wanted {
return true
}
}
return false
}
func multimodalValueOrDefault(value, fallback string) string {
value = strings.TrimSpace(value)
if value == "" {
return fallback
}
return value
}
type attributeViewItemFilter func(attrView *av.AttributeView, itemID string) bool
func DocAssets(rootID string, retainQueryStr bool) (ret []string, err error) {
return docAssets(rootID, retainQueryStr, nil)
}
func docAssets(rootID string, retainQueryStr bool, itemFilter attributeViewItemFilter) (ret []string, err error) {
tree, err := LoadTreeByBlockID(rootID)
if err != nil {
return
}
ret = getAssetsLinkDestsWithAttributeViewItemFilter(tree.Root, false, itemFilter)
// 题头图存储在文档属性中,也属于文档引用的附件。
if titleImg := treenode.GetDocTitleImgPath(tree.Root); util.IsAssetLinkDest([]byte(titleImg), false) {
ret = append(ret, titleImg)
}
if !retainQueryStr {
for i, asset := range ret {
if before, _, ok := strings.Cut(asset, "?"); ok {
ret[i] = before
}
}
}
return
}
func NetAssets2LocalAssets(rootID string, onlyImg bool, originalURL string) (err error) {
syncingFiles.Store(rootID, true)
defer syncingFiles.Delete(rootID)
tree, err := LoadTreeByBlockID(rootID)
if err != nil {
return
}
docDirLocalPath := filepath.Join(util.DataDir, tree.Box, path.Dir(tree.Path))
assetsDirPath := getAssetsDir(filepath.Join(util.DataDir, tree.Box), docDirLocalPath)
if !gulu.File.IsExist(assetsDirPath) {
if err = os.MkdirAll(assetsDirPath, 0755); err != nil {
return
}
}
err = netAssets2LocalAssets0(tree, onlyImg, originalURL, assetsDirPath, true)
go func() {
time.Sleep(128 * time.Microsecond)
util.PushReloadProtyle(rootID)
}()
return
}
func netAssets2LocalAssets0(tree *parse.Tree, onlyImg bool, originalURL string, assetsDirPath string, needWriteTree bool) (err error) {
finishInlineTitles := treenode.MaterializeInlineTitles(tree.Root)
defer finishInlineTitles()
var files int
var size int64
msgId := gulu.Rand.String(7)
browserClient := util.NewCustomReqClient() // 自定义了 TLS 指纹,增加下载成功率
forbiddenCount := 0
destNodes := getRemoteAssetsLinkDestsInTree(tree, onlyImg)
assetsMap := map[string]string{}
for _, destNode := range destNodes {
dests := getRemoteAssetsLinkDests(destNode, onlyImg)
if 1 > len(dests) {
continue
}
for _, dest := range dests {
if u := util.FileURLToLocalPath(dest); u != "" { // 处理本地文件链接
if !gulu.File.IsExist(u) {
logging.LogErrorf("local file asset [%s] not exist", u)
continue
}
if gulu.File.IsDir(u) {
logging.LogWarnf("ignore converting directory path [%s] to local asset", u)
continue
}
if util.IsSensitivePath(u) {
logging.LogWarnf("ignore converting sensitive path [%s] to local asset", u)
continue
}
name := assetsMap[u]
if "" != name {
setAssetsLinkDest(destNode, dest, "assets/"+name)
continue
}
name = filepath.Base(u)
name = util.FilterUploadFileName(name)
name = "network-asset-" + name
var writePath string
if IsEncryptedBox(tree.Box) {
// 加密 box:脱敏文件名 + 加密内容(writeAssetFile 内部按路径反查 box 加密)+ 写映射
diskName := encryptedAssetName(util.Ext(name), ast.NewNodeID())
writePath = filepath.Join(assetsDirPath, diskName)
f, openErr := os.Open(u)
if openErr != nil {
logging.LogErrorf("open [%s] failed: %s", u, openErr)
continue
}
if err = writeAssetFile(writePath, f, tree.Box, name); err != nil {
logging.LogErrorf("write encrypted asset [%s] failed: %s", writePath, err)
f.Close()
continue
}
f.Close()
name = diskName
} else {
name = util.AssetName(name, ast.NewNodeID())
writePath = filepath.Join(assetsDirPath, name)
if err = filelock.Copy(u, writePath); err != nil {
logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err)
continue
}
}
assetURL := "assets/" + name
if IsEncryptedBox(tree.Box) {
assetURL += "?box=" + tree.Box
}
setAssetsLinkDest(destNode, dest, assetURL)
assetsMap[u] = name
files++
size += gulu.File.GetFileSize(writePath)
continue
}
if strings.HasPrefix(strings.ToLower(dest), "https://") || strings.HasPrefix(strings.ToLower(dest), "http://") || strings.HasPrefix(dest, "//") {
if strings.HasPrefix(dest, "//") {
// `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598
dest = "https:" + dest
}
u := dest
if strings.Contains(u, "qpic.cn") {
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
u = strings.Replace(u, "http://", "https://", 1)
// 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431
// 下面这部分需要注释掉,否则会导致响应 400
//if strings.HasSuffix(u, "/0") {
// u = strings.Replace(u, "/0", "/640", 1)
//} else if strings.Contains(u, "/0?") {
// u = strings.Replace(u, "/0?", "/640?", 1)
//}
}
name := assetsMap[u]
if "" != name {
setAssetsLinkDest(destNode, dest, "assets/"+name)
continue
}
displayU := u
if 64 < len(displayU) {
displayU = displayU[:64] + "..."
}
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), displayU), 15000)
request := browserClient.R()
request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second)
if "" != originalURL {
request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464
}
resp, reqErr := request.Get(u)
if nil != reqErr {
logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr)
continue
}
if http.StatusForbidden == resp.StatusCode || http.StatusUnauthorized == resp.StatusCode {
forbiddenCount++
}
if strings.Contains(strings.ToLower(resp.GetContentType()), "text/html") {
// 忽略超链接网页 `Convert network assets to local` no longer process webpage https://github.com/siyuan-note/siyuan/issues/9965
continue
}
if 200 != resp.StatusCode {
logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode)
continue
}
if 1024*1024*96 < resp.ContentLength {
logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength)))
continue
}
data, repErr := resp.ToBytes()
if nil != repErr {
logging.LogErrorf("download network asset [%s] failed: %s", u, repErr)
continue
}
if strings.Contains(u, "?") {
name = u[:strings.Index(u, "?")]
name = path.Base(name)
} else {
name = path.Base(u)
}
if strings.Contains(name, "#") {
name = name[:strings.Index(name, "#")]
}
name, _ = url.PathUnescape(name)
name = util.FilterUploadFileName(name)
ext := util.Ext(name)
if !util.IsCommonExt(ext) {
if mtype := mimetype.Detect(data); nil != mtype {
ext = mtype.Extension()
name += ext
}
}
if "" == ext && bytes.HasPrefix(data, []byte("<svg ")) && bytes.HasSuffix(data, []byte("</svg>")) {
ext = ".svg"
name += ext
}
if "" == ext {
contentType := resp.Header.Get("Content-Type")
exts, _ := mime.ExtensionsByType(contentType)
if 0 < len(exts) {
ext = exts[0]
name += ext
}
}
if IsEncryptedBox(tree.Box) {
// 加密 box:脱敏文件名 + 加密内容 + 写映射
name = "network-asset-" + name
diskName := encryptedAssetName(util.Ext(name), ast.NewNodeID())
writePath := filepath.Join(assetsDirPath, diskName)
if err = writeAssetFile(writePath, bytes.NewReader(data), tree.Box, name); err != nil {
logging.LogErrorf("write encrypted network asset [%s] failed: %s", writePath, err)
continue
}
name = diskName
} else {
name = util.AssetName(name, ast.NewNodeID())
name = "network-asset-" + name
writePath := filepath.Join(assetsDirPath, name)
if err = filelock.WriteFile(writePath, data); err != nil {
logging.LogErrorf("write downloaded network asset [%s] to local asset [%s] failed: %s", u, writePath, err)
continue
}
}
assetURL := "assets/" + name
if IsEncryptedBox(tree.Box) {
assetURL += "?box=" + tree.Box
}
setAssetsLinkDest(destNode, dest, assetURL)
assetsMap[u] = name
files++
size += int64(len(data))
continue
}
}
}
util.PushClearMsg(msgId)
finishInlineTitles()
if needWriteTree {
if 0 < files {
msgId = util.PushMsg(Conf.Language(113), 7000)
if err = writeTreeUpsertQueue(tree); err != nil {
return
}
invalidateDocumentHistory(tree.ID)
util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files, humanize.BytesCustomCeil(uint64(size), 2)), 5000)
if 0 < forbiddenCount {
util.PushErrMsg(fmt.Sprintf(Conf.Language(255), forbiddenCount), 5000)
}
} else {
if 0 < forbiddenCount {
util.PushErrMsg(fmt.Sprintf(Conf.Language(255), forbiddenCount), 5000)
} else {
util.PushMsg(Conf.Language(121), 3000)
}
}
}
return
}
// DownloadNetAssets2LocalAssets 将语法树中的网络资源下载到本地并改写链接,
// 不持久化文档树,由调用方负责后续保存与渲染。
func DownloadNetAssets2LocalAssets(tree *parse.Tree, onlyImg bool, originalURL string, assetsDirPath string) {
netAssets2LocalAssets0(tree, onlyImg, originalURL, assetsDirPath, false)
}
type AssetSearchMatch struct {
Field string
Mode string
Value string
}
func SearchAssetsByName(keyword string, exts []string) (ret []*cache.Asset) {
ret, _ = SearchAssetsByNamePage(keyword, exts, nil, 1, Conf.Search.Load().Limit, false)
return
}
// SearchAssetsByNamePage 筛选资源,可合并图片元数据命中,并按稳定顺序返回指定页。
func SearchAssetsByNamePage(keyword string, exts []string, match *AssetSearchMatch, page, pageSize int, includeMetadata bool) (ret []*cache.Asset, err error) {
confSearchSnapshot := Conf.LoadSearch()
ret = []*cache.Asset{}
if page < 1 || pageSize < 1 || pageSize > confSearchSnapshot.Limit {
return nil, errors.New("invalid asset search page")
}
allowedExts := map[string]bool{}
for _, ext := range exts {
ext = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(ext)), ".")
if ext != "" {
allowedExts["."+ext] = true
}
}
var pattern *regexp.Regexp
if match != nil {
if match.Field != "" && match.Field != "name" && match.Field != "path" {
return nil, errors.New("invalid asset match field")
}
if match.Mode != "prefix" && match.Mode != "suffix" && match.Mode != "regex" {
return nil, errors.New("invalid asset match mode")
}
if len(match.Value) > 1024 {
return nil, errors.New("asset match is too long")
}
if match.Mode == "regex" {
pattern, err = regexp.Compile(match.Value)
if err != nil {
return nil, fmt.Errorf("invalid asset regex: %w", err)
}
}
}
var keywords []string
keywords = append(keywords, keyword)
if "" != keyword {
keywords = append(keywords, strings.Split(keyword, " ")...)
}
metadataHits := map[string]int{}
if includeMetadata && strings.TrimSpace(keyword) != "" {
markdowns, queryErr := sql.QueryImageSpanMarkdowns()
if queryErr != nil {
return nil, queryErr
}
metadataHits = imageAssetMetadataHits(markdowns, keywords)
}
pathHitCount := map[string]int{}
filterAsset := func(path string, asset *cache.Asset) bool {
// 扩展名过滤
if len(exts) > 0 {
if !allowedExts[strings.ToLower(filepath.Ext(asset.HName))] {
return false
}
}
if match != nil && match.Value != "" {
value := asset.HName
if match.Field == "path" {
value = asset.Path
}
switch match.Mode {
case "prefix":
if !strings.HasPrefix(strings.ToLower(value), strings.ToLower(match.Value)) {
return false
}
case "suffix":
if !strings.HasSuffix(strings.ToLower(value), strings.ToLower(match.Value)) {
return false
}
case "regex":
if !pattern.MatchString(value) {
return false
}
}
}
// 关键字匹配
lowerHName := strings.ToLower(asset.HName)
lowerPath := strings.ToLower(asset.Path)
var hitNameCount, hitPathCount int
for i, k := range keywords {
lowerKeyword := strings.ToLower(k)
if 0 == i {
// 第一个是完全匹配,权重最高
if strings.Contains(lowerHName, lowerKeyword) {
hitNameCount += 64
}
if strings.Contains(lowerPath, lowerKeyword) {
hitPathCount += 64
}
}
hitNameCount += strings.Count(lowerHName, lowerKeyword)
hitPathCount += strings.Count(lowerPath, lowerKeyword)
if 1 > hitNameCount && 1 > hitPathCount {
continue
}
}
// 只返回有匹配的资源
metadataHitCount := metadataHits[asset.Path]
if includeMetadata && strings.TrimSpace(keyword) != "" {
metadataHitCount += assetMetadataTextHits(util.GetAssetText(asset.Path), keywords)
}
if 1 > hitNameCount+hitPathCount+metadataHitCount {
return false
}
// 记录命中次数用于排序
pathHitCount[asset.Path] = hitNameCount + hitPathCount + metadataHitCount
return true
}
matchedAssets := cache.FilterAssets(filterAsset)
if deferred, err := DeferredSyncAssets(); err == nil {
for _, file := range deferred {
if !strings.HasPrefix(file.Path, "/assets/") {
continue
}
absPath := filepath.Join(util.DataDir, filepath.FromSlash(strings.TrimPrefix(file.Path, "/")))
if IsEncryptedAssetPath(absPath) {
continue
}
assetPath, _, valid := AssetPathFromDataRelativePath(strings.TrimPrefix(file.Path, "/"))
if !valid || matchedAssets[assetPath] != nil {
continue
}
asset := &cache.Asset{HName: util.RemoveID(filepath.Base(assetPath)), Path: assetPath, Updated: file.SecUpdated()}
if filterAsset(assetPath, asset) {
matchedAssets[assetPath] = asset
}
}
}
for _, asset := range matchedAssets {
ret = append(ret, &cache.Asset{
HName: asset.HName,
Path: asset.Path,
Updated: asset.Updated,
})
}
if 0 < len(pathHitCount) {
sort.Slice(ret, func(i, j int) bool {
iHits, jHits := pathHitCount[ret[i].Path], pathHitCount[ret[j].Path]
if iHits != jHits {
return iHits > jHits
}
return ret[i].Path < ret[j].Path
})
} else {
sort.Slice(ret, func(i, j int) bool {
if ret[i].Updated != ret[j].Updated {
return ret[i].Updated > ret[j].Updated
}
return ret[i].Path < ret[j].Path
})
}
// 排序后再分页,保证同一批资源的页边界稳定。
if page-1 > len(ret)/pageSize {
return []*cache.Asset{}, nil
}
start := (page - 1) * pageSize
if start >= len(ret) {
return []*cache.Asset{}, nil
}
end := min(start+pageSize, len(ret))
ret = ret[start:end]
// 仅处理当前页的高亮内容,避免对其他页重复渲染。
highlightKeyword := strings.Join(keywords, search.TermSep)
for _, asset := range ret {
if pathHitCount[asset.Path] > 0 {
_, asset.HName = search.MarkText(asset.HName, highlightKeyword, 64, confSearchSnapshot.CaseSensitive)
}
}
return
}
func GetAssetAbsPath(relativePath string) (string, error) {
return GetAssetAbsPathWithOpt(relativePath, false)
}
// ResolveDataAssetPath 解析 data 相对资源路径,并确保目标位于全局或普通笔记本的资源目录中。
func ResolveDataAssetPath(assetPath string) (relativePath, absPath string, err error) {
if assetPath == "" {
err = errors.New("asset path is required")
return
}
nativePath := filepath.FromSlash(assetPath)
if filepath.IsAbs(nativePath) || filepath.VolumeName(nativePath) != "" ||
(len(nativePath) > 0 && os.IsPathSeparator(nativePath[0])) {
err = fmt.Errorf("asset path must be relative to data directory: %s", assetPath)
return
}
nativePath = filepath.Clean(nativePath)
absPath = filepath.Join(util.DataDir, nativePath)
dataRelativePath, relErr := filepath.Rel(util.DataDir, absPath)
if relErr != nil || dataRelativePath == "." || dataRelativePath == ".." ||
strings.HasPrefix(dataRelativePath, ".."+string(filepath.Separator)) {
err = fmt.Errorf("asset path escapes data directory: %s", assetPath)
return
}
parts := strings.Split(filepath.ToSlash(dataRelativePath), "/")
assetDirIndex := -1
switch {
case len(parts) > 1 && parts[0] == "assets":
assetDirIndex = 0
case len(parts) > 2 && ast.IsNodeIDPattern(parts[0]):
for i := 1; i < len(parts)-1; i++ {
if parts[i] == "assets" {
assetDirIndex = i
break
}
}
if assetDirIndex > 0 {
boxConfPath := filepath.Join(util.DataDir, parts[0], ".siyuan", "conf.json")
if !filelock.IsExist(boxConfPath) {
err = fmt.Errorf("asset path does not belong to a notebook: %s", assetPath)
return
}
if IsEncryptedBox(parts[0]) {
err = fmt.Errorf("accessing assets in encrypted notebook [%s] is not supported", parts[0])
return
}
}
}
if assetDirIndex < 0 {
err = fmt.Errorf("path is not under an assets directory: %s", assetPath)
return
}
assetRootParts := parts[:assetDirIndex+1]
assetRoot := filepath.Join(util.DataDir, filepath.FromSlash(strings.Join(assetRootParts, "/")))
if !gulu.File.IsSubPath(assetRoot, absPath) {
err = fmt.Errorf("path is not a child of assets directory: %s", assetPath)
return
}
resolvedRoot, evalErr := ResolveAssetPathWithMissingLeaf(assetRoot)
if evalErr != nil {
err = fmt.Errorf("resolve assets directory [%s] failed: %w", assetRoot, evalErr)
return
}
if assetDirIndex > 0 {
notebookRoot := filepath.Join(util.DataDir, parts[0])
resolvedDataDir, dataEvalErr := util.ResolveRealPath(util.DataDir)
resolvedNotebookRoot, notebookEvalErr := util.ResolveRealPath(notebookRoot)
if dataEvalErr != nil || notebookEvalErr != nil ||
!gulu.File.IsSubPath(resolvedDataDir, resolvedNotebookRoot) ||
!gulu.File.IsSubPath(resolvedNotebookRoot, resolvedRoot) {
err = fmt.Errorf("notebook asset path resolves outside notebook directory: %s", assetPath)
return
}
}
resolvedPath, evalErr := ResolveAssetPathWithMissingLeaf(absPath)
if evalErr != nil {
err = fmt.Errorf("resolve asset [%s] failed: %w", absPath, evalErr)
return
}
if !gulu.File.IsSubPath(resolvedRoot, resolvedPath) {
err = fmt.Errorf("asset path resolves outside assets directory: %s", assetPath)
return
}
relativePath = filepath.ToSlash(dataRelativePath)
return
}
// ResolveUnusedDataAssetPath 解析可单独删除的资源路径,允许显式关闭 Android 保活通知。
func ResolveUnusedDataAssetPath(assetPath string) (relativePath, absPath string, err error) {
relativePath, absPath, err = ResolveDataAssetPath(assetPath)
if err != nil {
return
}
if relativePath == "assets/android-notification-texts.txt" {
// 保活开关不参与批量清理,但可显式删除普通文件;目录和符号链接不能使用该例外。
info, statErr := os.Lstat(absPath)
if statErr != nil {
err = statErr
return
}
if info.Mode().IsRegular() {
return
}
err = fmt.Errorf("Android notification texts must be a regular file: %s", relativePath)
return
}
unusedAssets, err := UnusedAssets(false)
if err != nil {
return
}
if unusedAssetsContainPath(relativePath, absPath, unusedAssets) {
return
}
err = fmt.Errorf("asset is not unused: %s", relativePath)
return
}
func unusedAssetsContainPath(relativePath, absPath string, items []*UnusedItem) bool {
resolvedPath, _ := ResolveAssetPathWithMissingLeaf(absPath)
for _, item := range items {
if item.AbsPath != "" {
resolvedItemPath, evalErr := ResolveAssetPathWithMissingLeaf(item.AbsPath)
samePath, relErr := filepath.Rel(resolvedPath, resolvedItemPath)
if resolvedPath != "" && evalErr == nil && relErr == nil && samePath == "." {
return true
}
continue
}
itemPath := filepath.ToSlash(filepath.Clean(filepath.FromSlash(item.Item)))
if itemPath == relativePath {
return true
}
}
return false
}
// AssetPathWithoutQuery 返回去掉查询参数后的资源路径,用于复制到导出目录等磁盘路径场景。
func AssetPathWithoutQuery(relativePath string) string {
relativePath = strings.TrimSpace(relativePath)
if idx := strings.Index(relativePath, "?"); idx >= 0 {
relativePath = relativePath[:idx]
}
return filepath.ToSlash(relativePath)
}
// HTMLAssetIFrameSrc 返回用于安全渲染 HTML 资源的 IFrame 地址。
func HTMLAssetIFrameSrc(assetPath string) string {
if IsHTMLAssetIFrameSrc(assetPath) {
return assetPath
}
parsed, err := url.Parse(assetPath)
if err != nil {
return assetPath
}
query := parsed.Query()
query.Set("iframe", "true")
parsed.RawQuery = query.Encode()
return parsed.String()
}
// IsLocalHTMLAssetPath 判断资源地址是否为本地 HTML 文件。
func IsLocalHTMLAssetPath(assetPath string) bool {
parsed, err := url.Parse(assetPath)
if err != nil || parsed.IsAbs() || parsed.Host != "" {
return false
}
assetPath = strings.TrimPrefix(parsed.Path, "/")
assetPath = strings.TrimPrefix(assetPath, "./")
if !strings.HasPrefix(assetPath, "assets/") {
return false
}
ext := strings.ToLower(path.Ext(assetPath))
return ext == ".html" || ext == ".htm"
}
// IsHTMLAssetIFrameSrc 判断资源地址是否为 HTML 文件 IFrame 渲染地址。
func IsHTMLAssetIFrameSrc(assetPath string) bool {
parsed, err := url.Parse(assetPath)
if err != nil || !strings.EqualFold(parsed.Query().Get("iframe"), "true") {
return false
}
return IsLocalHTMLAssetPath(assetPath)
}
func assetPathAndBox(relativePath, defaultBoxID string) (cleanPath, boxID string, err error) {
relativePath = strings.TrimSpace(relativePath)
boxID = defaultBoxID
if idx := strings.Index(relativePath, "?"); idx >= 0 {
query := relativePath[idx+1:]
relativePath = relativePath[:idx]
if values, parseErr := url.ParseQuery(query); parseErr == nil {
if queryBoxID := strings.TrimSpace(values.Get("box")); queryBoxID != "" {
if defaultBoxID != "" && defaultBoxID != queryBoxID {
// 调用方指定了 boxID 但 URL 里是另一个 box:拒绝,防止解析到错误 box
err = fmt.Errorf("box mismatch: caller specified [%s] but URL has [%s]", defaultBoxID, queryBoxID)
return
}
boxID = queryBoxID
}
}
}
cleanPath = filepath.ToSlash(relativePath)
return
}
// GetAssetAbsPathInBox 在指定 box 内解析资源绝对路径,不进行全局遍历。
// relativePath 必须以 assets/ 前缀开头,boxID 为空且路径没有 box 查询参数时只解析普通/全局资源,不遍历加密 box。
// 加密 box 直接从 <boxID>/assets/ 查找,不依赖后缀匹配。
func GetAssetAbsPathInBox(relativePath, boxID string) (string, error) {
var err error
relativePath, boxID, err = assetPathAndBox(relativePath, boxID)
if err != nil {
return "", err
}
relativePath = path.Clean(relativePath)
if relativePath == "." || strings.HasPrefix(relativePath, "../") || relativePath == ".." || path.IsAbs(relativePath) {
return "", fmt.Errorf("[%s] is not an asset path", relativePath)
}
if !strings.HasPrefix(relativePath, "assets/") {
return "", fmt.Errorf("[%s] is not an asset path (must start with assets/)", relativePath)
}
if boxID != "" && !ast.IsNodeIDPattern(boxID) {
return "", fmt.Errorf("[%s] is not a box id", boxID)
}
if boxID == "" {
return GetAssetAbsPathWithOpt(relativePath, false)
}
p := filepath.Join(util.DataDir, boxID, relativePath)
if gulu.File.IsExist(p) {
if !gulu.File.IsSubPath(util.WorkspaceDir, p) {
return "", fmt.Errorf("[%s] is not sub path of workspace", p)
}
// 解析符号链接/目录联接,防止软链接跳出资产根目录
if realP, evalErr := filepath.EvalSymlinks(p); evalErr == nil && realP != p {
if !gulu.File.IsSubPath(util.WorkspaceDir, realP) {
return "", fmt.Errorf("symlink [%s] resolves outside workspace: [%s]", p, realP)
}
// 验证解析后的路径仍在 <boxID>/assets/ 或全局 data/assets/ 下
expectedPrefix := filepath.Join(util.DataDir, "assets")
if boxID != "" {
expectedPrefix = filepath.Join(util.DataDir, boxID, "assets")
}
if !gulu.File.IsSubPath(expectedPrefix, realP) {
return "", fmt.Errorf("symlink [%s] resolves outside assets directory: [%s]", p, realP)
}
}
return p, nil
}
// 非加密 box 的资源可能回退到全局 data/assets(兼容旧笔记本结构)
if deferredPath, deferredErr := deferredAssetPath(relativePath, boxID, true); deferredErr != nil || deferredPath != "" {
return deferredPath, deferredErr
}
if !IsEncryptedBox(boxID) {
return GetAssetAbsPathWithOpt(relativePath, false)
}
return "", fmt.Errorf(Conf.Language(12), relativePath)
}
// GetAssetAbsPathWithOpt 与 GetAssetAbsPath 一致,但可通过 includeEncrypted 控制是否遍历加密 box。
// serveAssets 传 true(下游 serveEncryptedAsset 会按锁定状态 fail-closed),其他调用方传 false(安全跳过)。
func GetAssetAbsPathWithOpt(relativePath string, includeEncrypted bool) (string, error) {
relativePath = strings.TrimSpace(relativePath)
if idx := strings.Index(relativePath, "?"); idx >= 0 {
relativePath = relativePath[:idx]
}
absPath, err := getAssetAbsPath(relativePath, includeEncrypted)
if err == nil && absPath != "" {
return absPath, nil
}
if err != nil {
return "", err
}
return "", fmt.Errorf(Conf.Language(12), relativePath)
}
func getAssetAbsPath(relativePath string, includeEncrypted bool) (absPath string, err error) {
relativePath = filepath.ToSlash(relativePath)
// 在 data 文件夹下搜索,主要是 data/assets 文件夹
p := filepath.Join(util.DataDir, relativePath)
if gulu.File.IsExist(p) {
if !gulu.File.IsSubPath(util.WorkspaceDir, p) {
return "", fmt.Errorf("[%s] is not sub path of workspace", p)
}
// 解析符号链接,验证真实路径仍在 data/assets/ 下
if realP, evalErr := filepath.EvalSymlinks(p); evalErr == nil && realP != p {
assetsRoot := util.GetDataAssetsAbsPath()
realAssetsRoot, rootEvalErr := filepath.EvalSymlinks(assetsRoot)
if rootEvalErr != nil {
return "", fmt.Errorf("resolve assets root [%s] failed: %w", assetsRoot, rootEvalErr)
}
if !gulu.File.IsSubPath(realAssetsRoot, realP) {
return "", fmt.Errorf("symlink [%s] resolves outside data/assets: [%s]", p, realP)
}
// 安全校验使用解析后的路径,返回原路径以便下游与 DataDir 保持同一路径形式
return p, nil
}
return p, nil
}
// 在文档同级 assets 文件夹下搜索
if !strings.HasPrefix(relativePath, "assets/") {
return "", nil
}
notebooks, err := ListNotebooks()
if err != nil {
return "", errors.New(Conf.Language(0))
}
for _, notebook := range notebooks {
if !includeEncrypted && IsEncryptedBox(notebook.ID) {
continue // 加密笔记本的资源不参与全局路径解析(孤岛,资源不跨边界)
}
notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error {
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if p := filepath.ToSlash(path); strings.HasSuffix(p, relativePath) {
if gulu.File.IsExist(path) {
absPath = path
return fs.SkipAll
}
}
return nil
})
if "" != absPath {
if !gulu.File.IsSubPath(util.WorkspaceDir, absPath) {
return "", fmt.Errorf("[%s] is not sub path of workspace", absPath)
}
return absPath, nil
}
}
return deferredAssetPath(relativePath, "", includeEncrypted)
}
// UploadAssets2CloudWithApp 将块资源上传提示限定到发起实例;空标识保留广播。
func UploadAssets2CloudWithApp(id string, ignorePushMsg bool, app string) (count int, err error) {
if !IsSubscriber() {
return
}
tree, err := LoadTreeByBlockID(id)
if err != nil {
return
}
node := treenode.GetNodeInTree(tree, id)
if nil == node {
err = ErrBlockNotFound
return
}
nodes := []*ast.Node{node}
if ast.NodeHeading == node.Type {
nodes = append(nodes, treenode.HeadingChildren(node)...)
}
var assets []string
for _, n := range nodes {
assets = append(assets, getAssetsLinkDests(n, false)...)
assets = append(assets, getQueryEmbedNodesAssetsLinkDests(n)...)
}
assets = gulu.Str.RemoveDuplicatedElem(assets)
count, err = uploadAssets2CloudWithApp(assets, bizTypeUploadAssets, ignorePushMsg, app)
if err != nil {
return
}
return
}
// UploadAssets2CloudByAssetsPathsWithApp 在同一实例显示、更新和清除本次上传的提示。
func UploadAssets2CloudByAssetsPathsWithApp(assetPaths []string, ignorePushMsg bool, app string) (count int, err error) {
if !IsSubscriber() {
return
}
count, err = uploadAssets2CloudWithApp(assetPaths, bizTypeUploadAssets, ignorePushMsg, app)
return
}
const (
bizTypeUploadAssets = "upload-assets"
bizTypeExport2Liandi = "export-liandi"
)
// uploadAssets2Cloud 将资源文件上传到云端图床。
func uploadAssets2Cloud(assetPaths []string, bizType string, ignorePushMsg bool) (count int, err error) {
return uploadAssets2CloudWithApp(assetPaths, bizType, ignorePushMsg, "")
}
func uploadAssets2CloudWithApp(assetPaths []string, bizType string, ignorePushMsg bool, app string) (count int, err error) {
var uploadAbsAssets []string
for _, assetPath := range assetPaths {
var absPath string
absPath, err = GetAssetAbsPath(assetPath)
if err != nil {
logging.LogWarnf("get asset [%s] abs path failed: %s", assetPath, err)
return
}
if "" == absPath {
logging.LogErrorf("not found asset [%s]", assetPath)
continue
}
uploadAbsAssets = append(uploadAbsAssets, absPath)
}
uploadAbsAssets = gulu.Str.RemoveDuplicatedElem(uploadAbsAssets)
if 1 > len(uploadAbsAssets) {
return
}
logging.LogInfof("uploading [%d] assets", len(uploadAbsAssets))
var msgId string
if !ignorePushMsg {
msgId = util.PushMsgWithApp(app, fmt.Sprintf(Conf.Language(27), len(uploadAbsAssets)), 3000)
defer util.PushClearMsgWithApp(app, msgId)
}
if loadErr := LoadUploadToken(); nil != loadErr {
util.PushMsgWithApp(app, loadErr.Error(), 5000)
return
}
limitSize := uint64(3 * 1024 * 1024) // 3MB
if IsSubscriber() {
limitSize = 10 * 1024 * 1024 // 10MB
}
// metaType 为服务端 Filemeta.FILEMETA_TYPE,这里只有两个值:
//
// 5: SiYuan,表示为 SiYuan 上传图床
// 4: Client,表示作为客户端分享发布帖子时上传的文件
var metaType = "5"
if bizTypeUploadAssets == bizType {
metaType = "5"
} else if bizTypeExport2Liandi == bizType {
metaType = "4"
}
pushErrMsgCount := 0
var completedUploadAssets []string
for _, absAsset := range uploadAbsAssets {
if err = ensureReadableAssetLocal(absAsset); err != nil {
return count, err
}
fi, statErr := os.Stat(absAsset)
if nil != statErr {
logging.LogErrorf("stat file [%s] failed: %s", absAsset, statErr)
return count, statErr
}
if limitSize < uint64(fi.Size()) {
logging.LogWarnf("file [%s] larger than limit size [%s], ignore uploading it", absAsset, humanize.IBytes(limitSize))
if 3 > pushErrMsgCount {
msg := fmt.Sprintf(Conf.Language(247), filepath.Base(absAsset), humanize.IBytes(limitSize))
util.PushErrMsgWithApp(app, msg, 30000)
}
pushErrMsgCount++
continue
}
if !ignorePushMsg {
msg := fmt.Sprintf(Conf.Language(27), html.EscapeString(absAsset))
util.PushStatusBar(msg)
util.PushUpdateMsgWithApp(app, msgId, msg, 3000)
}
requestResult := gulu.Ret.NewResult()
request := httpclient.NewCloudFileRequest2m()
resp, reqErr := request.
SetSuccessResult(requestResult).
SetFile("file[]", absAsset).
SetCookies(&http.Cookie{Name: "symphony", Value: uploadToken}).
SetHeader("meta-type", metaType).
SetHeader("biz-type", bizType).
Post(util.GetCloudServer() + "/apis/siyuan/upload?ver=" + util.Ver)
if nil != reqErr {
logging.LogErrorf("upload assets failed: %s", reqErr)
return count, ErrFailedToConnectCloudServer
}
if 401 == resp.StatusCode {
err = errors.New(Conf.Language(31))
return
}
if 0 != requestResult.Code {
logging.LogErrorf("upload assets failed: %s", requestResult.Msg)
err = fmt.Errorf(Conf.Language(94), requestResult.Msg)
return
}
absAsset = filepath.ToSlash(absAsset)
relAsset := absAsset[strings.Index(absAsset, "assets/"):]
completedUploadAssets = append(completedUploadAssets, relAsset)
logging.LogInfof("uploaded asset [%s]", relAsset)
count++
}
if 0 < len(completedUploadAssets) {
logging.LogInfof("uploaded [%d] assets", len(completedUploadAssets))
}
return
}
func RemoveUnusedAssets() (ret []string, err error) {
ret = []string{}
verify, release, err := beginOrdinaryAssetCleanup()
if err != nil {
return
}
defer release()
unusedAssets, err := UnusedAssets(false)
if err != nil {
return
}
var size int64
msgId := util.PushMsg(Conf.Language(100), 30*1000)
defer func() {
if err != nil {
util.PushUpdateMsg(msgId, err.Error(), 7000)
return
}
msg := fmt.Sprintf(Conf.Language(91), len(ret), humanize.BytesCustomCeil(uint64(size), 2))
util.PushUpdateMsg(msgId, msg, 7000)
}()
for _, unusedAsset := range unusedAssets {
if err = EnsureAssetPrefixLocal(filepath.Join(util.DataDir, unusedAsset.Item)); err != nil {
return
}
}
historyDir, err := getHistoryDir(HistoryOpClean)
if err != nil {
logging.LogErrorf("get history dir failed: %s", err)
return
}
var hashes []string
for _, unusedAsset := range unusedAssets {
p := unusedAsset.Item
historyPath := filepath.Join(historyDir, p)
if p = filepath.Join(util.DataDir, p); filelock.IsExist(p) {
if filelock.IsHidden(p) {
continue
}
if err = filelock.Copy(p, historyPath); err != nil {
return
}
hash, _ := util.GetEtag(p)
hashes = append(hashes, hash)
}
}
ret, size, err = removeOrdinaryUnusedAssets(unusedAssets, historyDir, verify, filelock.RemoveWithoutFatal)
if err != nil {
return
}
sql.BatchRemoveAssetsQueue(hashes)
for _, item := range unusedAssets {
cache.RemoveAssetHashByPath(item.Item)
util.RemoveAssetText(item.Item)
if util.IsMobileContainer() {
HandleAssetsRemoveEvent(filepath.Join(util.DataDir, item.Item))
}
}
if 0 < len(ret) {
IncSync()
}
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
cache.LoadAssets()
return
}
func RemoveUnusedAsset(p string) (ret string, err error) {
verify, release, err := beginOrdinaryAssetCleanup()
if err != nil {
return
}
defer release()
relativePath, absPath, err := ResolveUnusedDataAssetPath(p)
if err != nil {
return
}
if err = EnsureAssetPrefixLocal(absPath); err != nil {
return
}
historyDir, err := getHistoryDir(HistoryOpClean)
if err != nil {
logging.LogErrorf("get history dir failed: %s", err)
return
}
historyPath := filepath.Join(historyDir, filepath.FromSlash(relativePath))
if err = filelock.Copy(absPath, historyPath); err != nil {
err = fmt.Errorf("copy unused asset [%s] to history failed: %w", absPath, err)
return
}
hash, _ := util.GetEtag(absPath)
if _, _, err = removeOrdinaryUnusedAssets([]*UnusedItem{{Item: relativePath}}, historyDir, verify, filelock.RemoveWithoutFatal); err != nil {
logging.LogErrorf("remove unused asset [%s] failed: %s", absPath, err)
util.PushErrMsg(fmt.Sprintf("%s", err), 7000)
err = fmt.Errorf("remove unused asset [%s] failed: %w", absPath, err)
return
}
ret = absPath
sql.BatchRemoveAssetsQueue([]string{hash})
cache.RemoveAssetHashByPath(relativePath)
if util.IsMobileContainer() {
HandleAssetsRemoveEvent(absPath)
}
util.RemoveAssetText(relativePath)
IncSync()
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
cache.RemoveAsset(relativePath)
return
}
func RenameAsset(oldPath, newName string) (newPath string, err error) {
util.PushEndlessProgress(Conf.Language(110))
defer util.PushClearProgress()
oldPath = strings.TrimSpace(oldPath)
suffix := ""
if idx := strings.IndexAny(oldPath, "?#"); idx >= 0 {
suffix = oldPath[idx:]
}
defer func() {
if err != nil {
newPath = ""
} else if newPath != "" {
newPath += suffix
}
}()
// 片段不参与磁盘路径解析,查询参数中的 box 仍用于定位笔记本。
if idx := strings.Index(oldPath, "#"); idx >= 0 {
oldPath = oldPath[:idx]
}
oldCleanPath := AssetPathWithoutQuery(oldPath)
oldAbsPath, err := GetAssetAbsPathInBox(oldPath, "")
if err != nil {
logging.LogErrorf("get asset [%s] abs path failed: %s", oldPath, err)
return
}
// 加密笔记本的资源磁盘文件名参与 AAD,重命名需要重新封装密文,当前不支持。
if IsEncryptedAssetPath(oldAbsPath) {
err = errors.New("renaming assets in encrypted notebooks is not supported")
return
}
newName = strings.TrimSpace(newName)
newName = util.FilterUploadFileName(newName)
if path.Base(oldCleanPath) == newName {
return
}
if "" == newName {
return
}
if !gulu.File.IsValidFilename(newName) {
err = errors.New(Conf.Language(151))
return
}
newName = util.AssetName(newName+filepath.Ext(oldCleanPath), ast.NewNodeID())
parentDir := path.Dir(oldCleanPath)
newPath = path.Join(parentDir, newName)
if err = ensureReadableAssetLocal(oldAbsPath); err != nil {
return
}
newAbsPath := filepath.Join(filepath.Dir(oldAbsPath), newName)
// Keep both paths readable until all references have been rewritten. On a
// partial failure or restart, each committed reference still has its file.
if err = copyPromotedAsset(oldAbsPath, newAbsPath); err != nil {
return
}
oldSya := oldAbsPath + ".sya"
hasSya := false
if _, statErr := os.Stat(oldSya); statErr == nil {
// Rename the .sya annotation file when renaming a PDF asset https://github.com/siyuan-note/siyuan/issues/9390
if err = copyPromotedAsset(oldSya, newAbsPath+".sya"); err != nil {
return
}
hasSya = true
} else if !os.IsNotExist(statErr) {
err = statErr
return
}
oldName := path.Base(oldCleanPath)
oldNameBytes := []byte(oldName)
escapedOldNameBytes := []byte((&url.URL{Path: oldName}).EscapedPath())
nameReplacer := newAssetRenameReplacer(oldName, newName)
notebooks, err := ListNotebooks()
if err != nil {
return
}
historyDir, err := getHistoryDir(HistoryOpReplace)
if nil != err {
return
}
luteEngine := util.NewLute()
for _, notebook := range notebooks {
// 加密笔记本的资源重命名已在入口处拦截,这里跳过加密 box
if IsEncryptedBox(notebook.ID) {
continue
}
pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
for _, paths := range pages {
for _, treeAbsPath := range paths {
data, readErr := filelock.ReadFile(treeAbsPath)
if nil != readErr {
logging.LogErrorf("get data [path=%s] failed: %s", treeAbsPath, readErr)
err = readErr
return
}
if !bytes.Contains(data, oldNameBytes) && !bytes.Contains(data, escapedOldNameBytes) {
continue
}
data = []byte(nameReplacer.Replace(string(data)))
if writeErr := filelock.WriteFile(treeAbsPath, data); nil != writeErr {
logging.LogErrorf("write data [path=%s] failed: %s", treeAbsPath, writeErr)
err = writeErr
return
}
cache.RemoveTreeData(util.GetTreeID(treeAbsPath))
p := filepath.ToSlash(strings.TrimPrefix(treeAbsPath, filepath.Join(util.DataDir, notebook.ID)))
tree, parseErr := filesys.LoadTreeByData(data, notebook.ID, p, luteEngine)
if nil != parseErr {
logging.LogWarnf("parse json to tree [%s] failed: %s", treeAbsPath, parseErr)
continue
}
generateTreeHistory(tree, historyDir)
treenode.UpsertBlockTree(tree)
sql.UpsertTreeQueue(tree)
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(treenode.ReadDocumentTitle(tree.Root).Text)))
}
}
}
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
storageAvDir := filepath.Join(util.DataDir, "storage", "av")
if gulu.File.IsDir(storageAvDir) {
entries, readErr := os.ReadDir(storageAvDir)
if nil != readErr {
logging.LogErrorf("read dir [%s] failed: %s", storageAvDir, readErr)
err = readErr
return
}
for _, entry := range entries {
avID := strings.TrimSuffix(entry.Name(), ".json")
if !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(avID) {
continue
}
avJSONPath := filepath.Join(storageAvDir, entry.Name())
if _, replaceErr := replaceAttributeViewAssetPath(avJSONPath, avID, oldCleanPath, newPath); nil != replaceErr {
logging.LogErrorf("replace asset path in attribute view [%s] failed: %s", entry.Name(), replaceErr)
err = replaceErr
return
}
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(entry.Name())))
}
}
if ocrText := util.GetAssetText(oldCleanPath); "" != ocrText {
// 图片重命名后 ocr-texts.json 需要更新 https://github.com/siyuan-note/siyuan/issues/12974
util.SetAssetText(newPath, ocrText)
}
if hasSya {
if err = filelock.RemoveWithoutFatal(oldSya); err != nil {
return
}
}
if err = filelock.RemoveWithoutFatal(oldAbsPath); err != nil {
return
}
IncSync()
return
}
// newAssetRenameReplacer 同时替换字面量和百分号编码形式,保留引用的路径前缀、查询参数和片段。
func newAssetRenameReplacer(oldValue, newValue string) *strings.Replacer {
pairs := []string{oldValue, newValue}
if escapedOld := (&url.URL{Path: oldValue}).EscapedPath(); escapedOld != oldValue {
pairs = append(pairs, escapedOld, (&url.URL{Path: newValue}).EscapedPath())
}
return strings.NewReplacer(pairs...)
}
func replaceAttributeViewAssetPath(avJSONPath, avID, oldPath, newPath string) (updated bool, err error) {
data, err := filelock.ReadFile(avJSONPath)
if nil != err {
return
}
oldPath = strings.SplitN(AssetPathWithoutQuery(oldPath), "#", 2)[0]
newPath = strings.SplitN(AssetPathWithoutQuery(newPath), "#", 2)[0]
if oldPath == "" || oldPath == newPath {
return
}
replacedData := []byte(newAssetRenameReplacer(oldPath, newPath).Replace(string(data)))
if bytes.Equal(data, replacedData) {
return
}
data = replacedData
if err = filelock.WriteFile(avJSONPath, data); nil != err {
return
}
cache.RemoveAVData(avID)
updated = true
return
}
type UnusedItem struct {
Item string `json:"item"`
Name string `json:"name"`
Path string `json:"path,omitempty"`
BlockIDs []string `json:"blockIDs,omitempty"`
ModTime time.Time `json:"-"`
AbsPath string `json:"-"`
}
type missingAssetReference struct {
boxID string
rawDest string
dest string
encrypted bool
}
func lookupAssetPath(assetsPathMap map[string]string, dest string) (resolvedDest, absPath string, found bool) {
if absPath, found = assetsPathMap[dest]; found {
return dest, absPath, true
}
if strings.HasSuffix(dest, "/") {
return "", "", false
}
resolvedDest = dest + "/"
absPath, found = assetsPathMap[resolvedDest]
if found {
return resolvedDest, absPath, true
}
return "", "", false
}
func removeReferencedAssetPaths(assetsPathMap map[string]string, dests map[string]bool) (linkDestFilePaths []string) {
var linkDestFolderPaths []string
for dest := range dests {
if !strings.HasPrefix(dest, "assets/") {
continue
}
if idx := strings.IndexAny(dest, "?#"); 0 <= idx {
// `pdf?page` 资源文件链接会被判定为未引用资源 https://github.com/siyuan-note/siyuan/issues/5649
dest = dest[:idx]
}
// 先移除 URL 后缀,再解码路径;同时保留历史数据中按字面量存储的百分号文件名。
candidates := []string{dest}
if decoded, decodeErr := url.PathUnescape(dest); decodeErr == nil {
if decoded != dest {
candidates = append(candidates, decoded)
}
}
for _, candidate := range candidates {
resolvedDest, _, found := lookupAssetPath(assetsPathMap, candidate)
if !found {
continue
}
if strings.HasSuffix(resolvedDest, "/") {
linkDestFolderPaths = append(linkDestFolderPaths, resolvedDest)
} else {
linkDestFilePaths = append(linkDestFilePaths, resolvedDest)
}
}
}
// 排除文件夹链接
var toRemoves []string
for asset := range assetsPathMap {
for _, linkDestFolder := range linkDestFolderPaths {
if strings.HasPrefix(asset, linkDestFolder) {
toRemoves = append(toRemoves, asset)
}
}
for _, linkDestPath := range linkDestFilePaths {
if strings.HasPrefix(linkDestPath, asset) {
toRemoves = append(toRemoves, asset)
}
}
}
for _, toRemove := range toRemoves {
delete(assetsPathMap, toRemove)
}
return
}
func UnusedAssets(sorted bool) (ret []*UnusedItem, err error) {
defer func() {
if recovered := recover(); recovered != nil {
ret = nil
err = fmt.Errorf("scan asset references failed: %v", recovered)
logging.LogErrorf("%s", err)
}
}()
ret = []*UnusedItem{}
assetsPathMap, err := allAssetAbsPaths()
if err != nil {
return
}
// 排除加密笔记本的资源:加密笔记本锁定时 loadTree 失败会误判引用关系,
// 且加密笔记本是孤岛,资源不参与全局未引用清理
for dest, absPath := range assetsPathMap {
if boxID := util.BoxIDFromDataPath(absPath); boxID != "" && IsEncryptedBox(boxID) {
delete(assetsPathMap, dest)
}
}
linkDestMap := map[string]bool{}
notebooks, err := ListNotebooks()
if err != nil {
return
}
luteEngine := util.NewLute()
for _, notebook := range notebooks {
if IsEncryptedBox(notebook.ID) {
continue // 加密笔记本的资源不参与未引用清理(孤岛,资源不跨边界)
}
dests := map[string]bool{}
// 分页加载,优化清理未引用资源内存占用 https://github.com/siyuan-note/siyuan/issues/5200
pages, walkErr := pagedPathsWithError(filepath.Join(util.DataDir, notebook.ID), 32)
if walkErr != nil {
return nil, walkErr
}
for _, paths := range pages {
var trees []*parse.Tree
for _, localPath := range paths {
tree, loadTreeErr := loadTree(localPath, luteEngine)
if nil != loadTreeErr {
return nil, fmt.Errorf("read asset references [%s] failed: %w", localPath, loadTreeErr)
}
trees = append(trees, tree)
}
for _, tree := range trees {
for _, d := range getAssetsLinkDests(tree.Root, false, normalizeAssetScanLinkDest) {
dests[d] = true
}
if titleImgPath := treenode.GetDocTitleImgPath(tree.Root); "" != titleImgPath {
// 题头图计入
titleImgPath = normalizeAssetScanLinkDest(titleImgPath)
if !util.IsAssetLinkDest([]byte(titleImgPath), false) {
continue
}
dests[titleImgPath] = true
}
}
}
linkDestFilePaths := removeReferencedAssetPaths(assetsPathMap, dests)
for _, dest := range linkDestFilePaths {
linkDestMap[dest] = true
if strings.HasSuffix(dest, ".pdf") {
linkDestMap[dest+".sya"] = true
}
}
}
agentSessionDests, readAgentSessionsErr := agentSessionImageAssetDests(luteEngine)
if readAgentSessionsErr != nil {
logging.LogErrorf("read agent session image assets failed: %s", readAgentSessionsErr)
return nil, readAgentSessionsErr
}
removeReferencedAssetPaths(assetsPathMap, agentSessionDests)
templateDests := map[string]bool{}
templateDir := filepath.Join(util.DataDir, "templates")
if resolved, resolveErr := filepath.EvalSymlinks(templateDir); resolveErr == nil {
templateDir = resolved
} else if !os.IsNotExist(resolveErr) {
return nil, fmt.Errorf("resolve template directory failed: %w", resolveErr)
}
err = filepath.WalkDir(templateDir, func(filename string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
if filename == templateDir && os.IsNotExist(walkErr) {
return nil
}
return walkErr
}
if entry.IsDir() || !strings.HasSuffix(strings.ToLower(entry.Name()), ".md") {
return nil
}
data, readErr := filelock.ReadFile(filename)
if readErr != nil {
return readErr
}
// 只解析模板里的静态引用,不执行模板函数或查询。
tree := parse.Parse("", data, luteEngine.ParseOptions)
for _, dest := range getAssetsLinkDests(tree.Root, false, normalizeAssetScanLinkDest) {
templateDests[dest] = true
}
return nil
})
if err != nil {
return nil, fmt.Errorf("read template asset references failed: %w", err)
}
for _, dest := range removeReferencedAssetPaths(assetsPathMap, templateDests) {
if strings.HasSuffix(dest, ".pdf") {
delete(assetsPathMap, dest+".sya")
}
}
var toRemoves []string
for asset := range assetsPathMap {
if strings.HasSuffix(asset, "ocr-texts.json") {
// 排除 OCR 结果文本
toRemoves = append(toRemoves, asset)
continue
}
if strings.HasSuffix(asset, "android-notification-texts.txt") {
// 排除 Android 通知文本
toRemoves = append(toRemoves, asset)
continue
}
}
// 排除数据库中引用的资源文件。加密笔记本的资源不参与未引用清理(孤岛,资源不跨边界)
storageAvDir := filepath.Join(util.DataDir, "storage", "av")
if gulu.File.IsDir(storageAvDir) {
entries, readErr := os.ReadDir(storageAvDir)
if nil != readErr {
logging.LogErrorf("read dir [%s] failed: %s", storageAvDir, readErr)
err = readErr
return
}
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(entry.Name(), ".json")) {
continue
}
data, readDataErr := filelock.ReadFile(filepath.Join(util.DataDir, "storage", "av", entry.Name()))
if nil != readDataErr {
logging.LogErrorf("read file [%s] failed: %s", entry.Name(), readDataErr)
err = readDataErr
return
}
for asset := range assetsPathMap {
if bytes.Contains(data, []byte(asset)) {
toRemoves = append(toRemoves, asset)
}
}
}
}
for _, toRemove := range toRemoves {
delete(assetsPathMap, toRemove)
}
dataAssetsAbsPath := util.GetDataAssetsAbsPath()
for dest, assetAbsPath := range assetsPathMap {
if _, ok := linkDestMap[dest]; ok {
continue
}
p, ok := unusedAssetRelativePath(dataAssetsAbsPath, assetAbsPath)
if !ok {
logging.LogWarnf("skip unused asset outside data asset roots [%s]", assetAbsPath)
continue
}
name := path.Base(p)
var modTime time.Time
if sorted {
if info, statErr := os.Stat(assetAbsPath); nil == statErr {
modTime = info.ModTime()
}
}
ret = append(ret, &UnusedItem{Item: p, Name: name, Path: dest, ModTime: modTime, AbsPath: assetAbsPath})
}
if sorted {
sort.Slice(ret, func(i, j int) bool {
if !ret[i].ModTime.Equal(ret[j].ModTime) {
return ret[i].ModTime.After(ret[j].ModTime)
}
return ret[i].Item > ret[j].Item
})
}
return
}
type agentAssetSessionFile struct {
Entries []struct {
Type string `json:"type"`
Content string `json:"content"`
} `json:"entries"`
ActiveTurn *struct {
UserContent string `json:"userContent"`
} `json:"activeTurn"`
}
func agentSessionImageAssetDests(luteEngine *lute.Lute) (ret map[string]bool, err error) {
ret = map[string]bool{}
sessionsDir := filepath.Join(util.DataDir, "storage", "ai", "agent", "sessions")
entries, err := os.ReadDir(sessionsDir)
if os.IsNotExist(err) {
err = nil
return
}
if err != nil {
return
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
for _, name := range []string{"session.json", "runtime.json"} {
data, readErr := filelock.ReadFile(filepath.Join(sessionsDir, entry.Name(), name))
if os.IsNotExist(readErr) {
continue
}
if readErr != nil {
err = readErr
return
}
file := &agentAssetSessionFile{}
if unmarshalErr := gulu.JSON.UnmarshalJSON(data, file); unmarshalErr != nil {
err = unmarshalErr
return
}
for _, sessionEntry := range file.Entries {
if sessionEntry.Type != "user" {
continue
}
for _, dest := range agentMessageImageAssets(sessionEntry.Content, luteEngine) {
ret[dest] = true
}
}
if file.ActiveTurn != nil {
for _, dest := range agentMessageImageAssets(file.ActiveTurn.UserContent, luteEngine) {
ret[dest] = true
}
}
}
}
return
}
func unusedAssetRelativePath(dataAssetsAbsPath, assetAbsPath string) (string, bool) {
if gulu.File.IsSubPath(dataAssetsAbsPath, assetAbsPath) {
relativePath, err := filepath.Rel(dataAssetsAbsPath, assetAbsPath)
if err != nil {
return "", false
}
return path.Join("assets", filepath.ToSlash(relativePath)), true
}
if gulu.File.IsSubPath(util.DataDir, assetAbsPath) {
relativePath, err := filepath.Rel(util.DataDir, assetAbsPath)
if err != nil {
return "", false
}
return filepath.ToSlash(relativePath), true
}
return "", false
}
func MissingAssets() (ret []*UnusedItem, err error) {
return missingAssets(allAssetAbsPaths, ListNotebooks)
}
func missingAssets(assetPaths func() (map[string]string, error), notebooks func() ([]*Box, error)) (ret []*UnusedItem, err error) {
defer func() {
if recovered := recover(); recovered != nil {
ret = nil
err = fmt.Errorf("scan missing asset references failed: %v", recovered)
logging.LogErrorf("%s", err)
}
}()
ret = []*UnusedItem{}
assetsPathMap, err := assetPaths()
if err != nil {
return
}
boxes, err := notebooks()
if err != nil {
return
}
luteEngine := util.NewLute()
referenceBlockIDs := map[missingAssetReference]map[string]bool{}
var skippedErr error
for _, notebook := range boxes {
if notebook.Closed {
continue
}
encrypted := IsEncryptedBox(notebook.ID)
pages, walkErr := pagedPathsWithError(filepath.Join(util.DataDir, notebook.ID), 32)
if walkErr != nil {
return nil, fmt.Errorf("scan missing asset references failed: %w", walkErr)
}
for _, paths := range pages {
var trees []*parse.Tree
for _, localPath := range paths {
tree, loadTreeErr := loadTree(localPath, luteEngine)
if nil != loadTreeErr {
// 保留逐篇跳过的行为,但不能将不完整的扫描报告为成功。
skippedErr = errors.Join(skippedErr, fmt.Errorf("read asset references [%s] failed: %w", localPath, loadTreeErr))
continue
}
trees = append(trees, tree)
}
for _, tree := range trees {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
blockID := assetLinkDestBlockID(n)
for _, dest := range getAssetLinkDestsByNode(n, false, normalizeAssetScanLinkDest) {
addAssetLinkDestBlockID(referenceBlockIDs, notebook.ID, encrypted, dest, blockID)
}
return ast.WalkContinue
})
if titleImgPath := treenode.GetDocTitleImgPath(tree.Root); "" != titleImgPath {
// 题头图计入
titleImgPath = normalizeAssetScanLinkDest(titleImgPath)
if !util.IsAssetLinkDest([]byte(titleImgPath), false) {
continue
}
addAssetLinkDestBlockID(referenceBlockIDs, notebook.ID, encrypted, titleImgPath, tree.Root.ID)
}
}
}
}
if skippedErr != nil {
return missingAssetItems(referenceBlockIDs, assetsPathMap), fmt.Errorf("missing asset scan incomplete: %w", skippedErr)
}
return missingAssetItems(referenceBlockIDs, assetsPathMap), nil
}
func missingAssetItems(referenceBlockIDs map[missingAssetReference]map[string]bool, assetsPathMap map[string]string) (ret []*UnusedItem) {
ret = []*UnusedItem{}
missingBlockIDs := map[string]map[string]bool{}
for reference, blockIDSet := range referenceBlockIDs {
if assetReferenceExists(reference, assetsPathMap) {
continue
}
destBlockIDs := missingBlockIDs[reference.dest]
if nil == destBlockIDs {
destBlockIDs = map[string]bool{}
missingBlockIDs[reference.dest] = destBlockIDs
}
for blockID := range blockIDSet {
destBlockIDs[blockID] = true
}
}
for dest, blockIDSet := range missingBlockIDs {
blockIDs := make([]string, 0, len(blockIDSet))
for blockID := range blockIDSet {
blockIDs = append(blockIDs, blockID)
}
sort.Strings(blockIDs)
ret = append(ret, &UnusedItem{Item: dest, Name: path.Base(dest), BlockIDs: blockIDs})
}
sort.Slice(ret, func(i, j int) bool {
return ret[i].Item < ret[j].Item
})
return
}
func assetReferenceExists(reference missingAssetReference, assetsPathMap map[string]string) bool {
if reference.encrypted {
_, err := GetAssetAbsPathInBox(reference.rawDest, reference.boxID)
return nil == err
}
if _, _, err := assetPathAndBox(reference.rawDest, reference.boxID); nil != err {
return false
}
if _, _, found := lookupAssetPath(assetsPathMap, reference.dest); found {
return true
}
literalDest := normalizeAssetScanLinkDest(reference.rawDest)
if idx := strings.IndexAny(literalDest, "?#"); idx >= 0 {
literalDest = literalDest[:idx]
}
if _, _, found := lookupAssetPath(assetsPathMap, literalDest); found {
return true
}
if strings.HasPrefix(reference.dest, "assets/.") {
// Assets starting with `.` should not be considered missing assets https://github.com/siyuan-note/siyuan/issues/8821
return filelock.IsExist(filepath.Join(util.DataDir, reference.dest))
}
return false
}
func addAssetLinkDestBlockID(referenceBlockIDs map[missingAssetReference]map[string]bool, boxID string, encrypted bool, rawDest, blockID string) {
rawDest = strings.TrimSpace(rawDest)
dest := normalizeMissingAssetLinkDest(rawDest)
if "" == dest {
return
}
reference := missingAssetReference{boxID: boxID, rawDest: rawDest, dest: dest, encrypted: encrypted}
blockIDs := referenceBlockIDs[reference]
if nil == blockIDs {
blockIDs = map[string]bool{}
referenceBlockIDs[reference] = blockIDs
}
if "" != blockID {
blockIDs[blockID] = true
}
}
func normalizeScannedAsset(dest string, normalize []func(string) string) string {
if len(normalize) != 0 {
return normalize[0](dest)
}
return dest
}
// normalizeAssetScanLinkDest 按编辑器的根路径解析本地引用,保留 URL 后缀和文件名编码。
func normalizeAssetScanLinkDest(dest string) string {
dest = strings.TrimSpace(dest)
if dest == "" || strings.HasPrefix(dest, "//") || strings.Contains(dest, "\\") {
return ""
}
// 保留历史数据中按字面量存储的百分号文件名,解析仅用于排除协议和主机。
parsed, err := url.Parse(strings.ReplaceAll(dest, "%", "%25"))
if err != nil || parsed.IsAbs() || parsed.Host != "" {
return ""
}
suffix := ""
if idx := strings.IndexAny(dest, "?#"); idx >= 0 {
suffix, dest = dest[idx:], dest[:idx]
}
parts := strings.Split(dest, "/")
for i, part := range parts {
// 浏览器将编码的点路径段视为导航,其他编码留给资源查找处理。
if decoded, err := url.PathUnescape(part); err == nil && (decoded == "." || decoded == "..") {
parts[i] = decoded
}
}
dest = strings.Join(parts, "/")
directory := strings.HasSuffix(dest, "/") || strings.HasSuffix(dest, "/.") || strings.HasSuffix(dest, "/..")
dest = strings.TrimPrefix(path.Clean("/"+dest), "/")
if directory {
dest += "/"
}
if !strings.HasPrefix(dest, "assets/") {
return ""
}
return dest + suffix
}
func normalizeMissingAssetLinkDest(dest string) string {
dest = normalizeAssetScanLinkDest(dest)
if !strings.HasPrefix(dest, "assets/") {
return ""
}
if idx := strings.IndexAny(dest, "?#"); 0 < idx {
dest = dest[:idx]
}
if decoded, err := url.PathUnescape(dest); err == nil {
dest = decoded
}
if strings.HasSuffix(dest, "/") || strings.HasSuffix(dest, ".rtfd") {
return ""
}
if strings.Contains(strings.ToLower(dest), ".pdf/") {
if idx := strings.LastIndex(dest, "/"); -1 < idx && ast.IsNodeIDPattern(dest[idx+1:]) {
// PDF 标注不计入 https://github.com/siyuan-note/siyuan/issues/13891
return ""
}
}
return dest
}
func assetLinkDestBlockID(node *ast.Node) string {
if node.IsBlock() && "" != node.ID {
return node.ID
}
if block := treenode.ParentBlock(node); nil != block {
return block.ID
}
return ""
}
func getAssetLinkDestsByNode(node *ast.Node, includeServePath bool, normalize ...func(string) string) []string {
if !node.IsBlock() && ast.NodeImage != node.Type && ast.NodeLinkDest != node.Type && ast.NodeHTMLBlock != node.Type && ast.NodeInlineHTML != node.Type &&
ast.NodeIFrame != node.Type && ast.NodeWidget != node.Type && ast.NodeAudio != node.Type && ast.NodeVideo != node.Type &&
ast.NodeAttributeView != node.Type && ast.NodeFileAnnotationRefID != node.Type && !node.IsTextMarkType("a") &&
!node.IsTextMarkType("file-annotation-ref") {
return nil
}
// 复用统一的资源链接提取逻辑,但仅处理当前节点,避免重复遍历子树。
nodeCopy := *node
nodeCopy.Parent = nil
nodeCopy.Previous = nil
nodeCopy.Next = nil
nodeCopy.FirstChild = nil
nodeCopy.LastChild = nil
return getAssetsLinkDests(&nodeCopy, includeServePath, normalize...)
}
func emojisInTree(tree *parse.Tree) (ret []string) {
if icon := tree.Root.IALAttr("icon"); "" != icon {
if !strings.Contains(icon, "://") && !strings.HasPrefix(icon, "api/icon/") && !util.IsNativeEmoji(icon) {
ret = append(ret, "/emojis/"+icon)
}
}
treenode.WalkWithInlineTitles(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeEmojiImg == n.Type {
tokens := n.Tokens
_, src, found := bytes.Cut(tokens, []byte("src=\""))
if !found {
return ast.WalkContinue
}
idx := bytes.Index(src, []byte("\""))
if idx == -1 {
return ast.WalkContinue
}
src = src[:idx]
ret = append(ret, string(src))
}
return ast.WalkContinue
})
ret = gulu.Str.RemoveDuplicatedElem(ret)
return
}
func getQueryEmbedNodesAssetsLinkDests(node *ast.Node) (ret []string) {
// The images in the embed blocks are not uploaded to the community hosting https://github.com/siyuan-note/siyuan/issues/10042
ret = []string{}
ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || ast.NodeBlockQueryEmbedScript != n.Type {
return ast.WalkContinue
}
stmt := n.TokensStr()
stmt = html.UnescapeString(stmt)
stmt = strings.ReplaceAll(stmt, editor.IALValEscNewLine, "\n")
sqlBlocks := sql.SelectBlocksRawStmt(stmt, 1, Conf.Search.Load().Limit)
for _, sqlBlock := range sqlBlocks {
subtree, _ := LoadTreeByBlockID(sqlBlock.ID)
if nil == subtree {
continue
}
embedNode := treenode.GetNodeInTree(subtree, sqlBlock.ID)
if nil == embedNode {
continue
}
ret = append(ret, getAssetsLinkDests(embedNode, false)...)
}
return ast.WalkContinue
})
ret = gulu.Str.RemoveDuplicatedElem(ret)
return
}
func getAssetsLinkDests(node *ast.Node, includeServePath bool, normalize ...func(string) string) (ret []string) {
return getAssetsLinkDestsWithAttributeViewItemFilter(node, includeServePath, nil, normalize...)
}
func getAssetsLinkDestsWithAttributeViewItemFilter(
node *ast.Node,
includeServePath bool,
itemFilter attributeViewItemFilter,
normalize ...func(string) string,
) (ret []string) {
return getAssetsLinkDestsWithOptions(node, includeServePath, itemFilter, assetLinkScanOptions{resolveAttributeViews: true}, normalize...)
}
type assetLinkScanOptions struct {
resolveAttributeViews bool
expandAssetLists bool
}
// getAssetsLinkDestsWithOptions 控制数据库加载与资源列表展开;保守展开仅用于已有完整资源清单的扫描。
func getAssetsLinkDestsWithOptions(
node *ast.Node, includeServePath bool, itemFilter attributeViewItemFilter, options assetLinkScanOptions,
normalize ...func(string) string,
) (ret []string) {
ret = []string{}
treenode.WalkWithInlineTitles(node, func(n *ast.Node, entering bool) ast.WalkStatus {
if entering {
if dest := normalizeScannedAsset(treenode.ImageLivePhotoVideo(n), normalize); dest != "" && util.IsAssetLinkDest([]byte(dest), includeServePath) {
ret = append(ret, dest)
}
}
if n.IsBlock() {
// 以 custom-data-assets 开头的块属性值可能是多个资源文件链接,需要计入
// Ignore assets associated with the `custom-data-assets` block attribute when cleaning unreferenced assets https://github.com/siyuan-note/siyuan/issues/12574
for _, kv := range n.KramdownIAL {
k := kv[0]
if strings.HasPrefix(k, "custom-data-assets") {
ret = append(ret, assetListLinkDests(kv[1], includeServePath, options.expandAssetLists, normalize...)...)
}
}
}
// 修改以下代码时需要同时修改 database 构造行级元素实现,增加必要的类型
if !entering || (ast.NodeLinkDest != n.Type && ast.NodeHTMLBlock != n.Type && ast.NodeInlineHTML != n.Type &&
ast.NodeIFrame != n.Type && ast.NodeWidget != n.Type && ast.NodeAudio != n.Type && ast.NodeVideo != n.Type &&
ast.NodeAttributeView != n.Type && ast.NodeFileAnnotationRefID != n.Type && !n.IsTextMarkType("a") &&
!n.IsTextMarkType("file-annotation-ref")) {
return ast.WalkContinue
}
if ast.NodeLinkDest == n.Type {
dest := normalizeScannedAsset(string(n.Tokens), normalize)
if !util.IsAssetLinkDest([]byte(dest), includeServePath) {
return ast.WalkContinue
}
dest = strings.TrimSpace(dest)
ret = append(ret, dest)
} else if n.IsTextMarkType("a") {
dest := normalizeScannedAsset(n.TextMarkAHref, normalize)
if !util.IsAssetLinkDest([]byte(dest), includeServePath) {
return ast.WalkContinue
}
dest = strings.TrimSpace(dest)
ret = append(ret, dest)
} else if n.IsTextMarkType("file-annotation-ref") {
if dest := fileAnnotationAssetLinkDest(n.TextMarkFileAnnotationRefID, includeServePath, normalize...); "" != dest {
ret = append(ret, dest)
}
} else if ast.NodeFileAnnotationRefID == n.Type {
if dest := fileAnnotationAssetLinkDest(n.TokensStr(), includeServePath, normalize...); "" != dest {
ret = append(ret, dest)
}
} else if ast.NodeAttributeView == n.Type {
if !options.resolveAttributeViews {
return ast.WalkContinue
}
attrView, _ := av.ParseAttributeView(n.AttributeViewID)
if nil == attrView {
return ast.WalkContinue
}
ret = append(ret, getAttributeViewAssetsLinkDests(attrView, includeServePath, itemFilter, normalize...)...)
} else {
if ast.NodeWidget == n.Type {
dataAssets := n.IALAttr("custom-data-assets")
if "" == dataAssets {
// 兼容两种属性名 custom-data-assets 和 data-assets https://github.com/siyuan-note/siyuan/issues/4122#issuecomment-1154796568
dataAssets = n.IALAttr("data-assets")
}
ret = append(ret, assetListLinkDests(dataAssets, includeServePath, options.expandAssetLists, normalize...)...)
} else { // HTMLBlock/InlineHTML/IFrame/Audio/Video
ret = append(ret, htmlAssetLinkDestsWithOptions(n.Tokens, includeServePath, options.expandAssetLists, normalize...)...)
}
}
return ast.WalkContinue
})
ret = gulu.Str.RemoveDuplicatedElem(ret)
for i, dest := range ret {
// 对于 macOS 的 rtfd 文件夹格式需要特殊处理,为其加上结尾 /
if strings.HasSuffix(dest, ".rtfd") {
ret[i] = dest + "/"
}
}
return
}
var assetListPattern = regexp.MustCompile(`[^\s,]+`)
// assetListLinkDests 同时保留完整路径与列表项,兼容文件名中的空格、逗号和多值资源属性。
func assetListLinkDests(raw string, includeServePath, expand bool, normalize ...func(string) string) (ret []string) {
values := []string{raw}
if expand {
values = append(values, assetListPattern.FindAllString(raw, -1)...)
}
for _, value := range values {
dest := normalizeScannedAsset(value, normalize)
if dest != "" && util.IsAssetLinkDest([]byte(dest), includeServePath) {
ret = append(ret, dest)
}
}
return
}
func htmlAssetLinkDests(tokens []byte, includeServePath bool, normalize ...func(string) string) (ret []string) {
return htmlAssetLinkDestsWithOptions(tokens, includeServePath, false, normalize...)
}
func htmlAssetLinkDestsWithOptions(tokens []byte, includeServePath, expandAssetLists bool, normalize ...func(string) string) (ret []string) {
// 逐个读取标签属性,兼容内联标签片段并只解码一层 HTML 实体。
tokenizer := html.NewTokenizer(bytes.NewReader(tokens))
for {
switch tokenizer.Next() {
case html.ErrorToken:
return
case html.StartTagToken, html.SelfClosingTagToken:
for _, attr := range tokenizer.Token().Attr {
if expandAssetLists && (attr.Key == "data-assets" || strings.HasPrefix(attr.Key, "custom-data-assets")) {
ret = append(ret, assetListLinkDests(attr.Val, includeServePath, true, normalize...)...)
continue
}
dest := normalizeScannedAsset(strings.TrimSpace(attr.Val), normalize)
if util.IsAssetLinkDest([]byte(dest), includeServePath) {
ret = append(ret, dest)
}
}
}
}
}
func fileAnnotationAssetLinkDest(reference string, includeServePath bool, normalize ...func(string) string) string {
reference = normalizeScannedAsset(reference, normalize)
if !util.IsAssetLinkDest(gulu.Str.ToBytes(reference), includeServePath) {
return ""
}
assetLink, _ := util.SplitFileAnnotationRef(reference)
return assetLink
}
func getAttributeViewAssetsLinkDests(
attrView *av.AttributeView,
includeServePath bool,
itemFilter attributeViewItemFilter,
normalize ...func(string) string,
) (ret []string) {
if nil == attrView {
return
}
collectValue := func(value *av.Value) {
ret = append(ret, getAttributeViewValueAssetsLinkDests(value, includeServePath, normalize...)...)
}
if nil == itemFilter {
attrView.VisitPersistedValues(collectValue)
} else {
for _, keyValues := range attrView.KeyValues {
if nil == keyValues {
continue
}
for _, value := range keyValues.Values {
if nil == value || !itemFilter(attrView, value.BlockID) {
continue
}
visitAttributeViewValue(value, collectValue)
}
}
}
ret = gulu.Str.RemoveDuplicatedElem(ret)
return
}
func visitAttributeViewValue(value *av.Value, visit func(*av.Value)) {
visitAttributeViewValue0(value, visit, map[*av.Value]struct{}{})
}
func visitAttributeViewValue0(value *av.Value, visit func(*av.Value), visited map[*av.Value]struct{}) {
if nil == value {
return
}
if _, ok := visited[value]; ok {
return
}
visited[value] = struct{}{}
visit(value)
if nil != value.Relation {
for _, content := range value.Relation.Contents {
visitAttributeViewValue0(content, visit, visited)
}
}
if nil != value.Rollup {
for _, content := range value.Rollup.Contents {
visitAttributeViewValue0(content, visit, visited)
}
}
}
func getAttributeViewValueAssetsLinkDests(value *av.Value, includeServePath bool, normalize ...func(string) string) (ret []string) {
if nil == value {
return
}
if nil != value.URL && util.IsAssetLinkDest([]byte(normalizeScannedAsset(value.URL.Content, normalize)), includeServePath) {
ret = append(ret, strings.TrimSpace(normalizeScannedAsset(value.URL.Content, normalize)))
}
for _, asset := range value.MAsset {
if nil != asset && util.IsAssetLinkDest([]byte(normalizeScannedAsset(asset.Content, normalize)), includeServePath) {
ret = append(ret, strings.TrimSpace(normalizeScannedAsset(asset.Content, normalize)))
}
}
if nil != value.Text && nil != value.Text.Rich {
if tree, err := av.ParseValueTextRich(value.Text.Rich); nil == err && nil != tree {
ret = append(ret, getAssetsLinkDests(tree.Root, includeServePath, normalize...)...)
}
}
return
}
func getAssetsLinkDestsInTree(tree *parse.Tree, includeServePath bool) (nodes []*ast.Node) {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
dests := getAssetsLinkDests(n, includeServePath)
if 1 > len(dests) {
return ast.WalkContinue
}
nodes = append(nodes, n)
return ast.WalkContinue
})
return
}
func setAssetsLinkDest(node *ast.Node, oldDest, dest string) {
if ast.NodeImage == node.Type {
if video := treenode.ImageLivePhotoVideo(node); video != "" {
treenode.SetImageLivePhotoVideo(node, strings.ReplaceAll(video, oldDest, dest))
}
} else if ast.NodeLinkDest == node.Type {
if bytes.HasPrefix(node.Tokens, []byte("//")) {
node.Tokens = append([]byte("https:"), node.Tokens...)
}
node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest))
} else if node.IsTextMarkType("a") {
if strings.HasPrefix(node.TextMarkAHref, "//") {
node.TextMarkAHref = "https:" + node.TextMarkAHref
}
node.TextMarkAHref = strings.ReplaceAll(node.TextMarkAHref, oldDest, dest)
} else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type {
if strings.HasPrefix(node.TextMarkAHref, "//") {
node.TextMarkAHref = "https:" + node.TextMarkAHref
}
node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest))
} else if ast.NodeAttributeView == node.Type {
needWrite := false
attrView, _ := av.ParseAttributeView(node.AttributeViewID)
if nil == attrView {
return
}
for _, keyValues := range attrView.KeyValues {
if av.KeyTypeMAsset != keyValues.Key.Type {
continue
}
for _, value := range keyValues.Values {
if 1 > len(value.MAsset) {
continue
}
for _, asset := range value.MAsset {
if oldDest == asset.Content && oldDest != dest {
asset.Content = dest
needWrite = true
}
}
}
}
if needWrite {
av.SaveAttributeView(attrView)
}
}
}
func getRemoteAssetsLinkDests(node *ast.Node, onlyImg bool) (ret []string) {
if onlyImg {
if ast.NodeLinkDest == node.Type {
if node.ParentIs(ast.NodeImage) {
if !util.IsAssetLinkDest(node.Tokens, false) {
ret = append(ret, string(node.Tokens))
}
}
} else if ast.NodeAttributeView == node.Type {
attrView, _ := av.ParseAttributeView(node.AttributeViewID)
if nil == attrView {
return
}
for _, keyValues := range attrView.KeyValues {
if av.KeyTypeMAsset != keyValues.Key.Type {
continue
}
for _, value := range keyValues.Values {
if 1 > len(value.MAsset) {
continue
}
for _, asset := range value.MAsset {
if av.AssetTypeImage != asset.Type {
continue
}
dest := asset.Content
if !util.IsAssetLinkDest([]byte(dest), false) {
ret = append(ret, strings.TrimSpace(dest))
}
}
}
}
}
} else {
if ast.NodeLinkDest == node.Type {
if !util.IsAssetLinkDest(node.Tokens, false) {
ret = append(ret, string(node.Tokens))
}
} else if node.IsTextMarkType("a") {
if !util.IsAssetLinkDest([]byte(node.TextMarkAHref), false) {
ret = append(ret, node.TextMarkAHref)
}
} else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type {
src := treenode.GetNodeSrcTokens(node)
if !util.IsAssetLinkDest([]byte(src), false) {
ret = append(ret, src)
}
} else if ast.NodeAttributeView == node.Type {
attrView, _ := av.ParseAttributeView(node.AttributeViewID)
if nil == attrView {
return
}
for _, keyValues := range attrView.KeyValues {
if av.KeyTypeMAsset != keyValues.Key.Type {
continue
}
for _, value := range keyValues.Values {
if 1 > len(value.MAsset) {
continue
}
for _, asset := range value.MAsset {
dest := asset.Content
if !util.IsAssetLinkDest([]byte(dest), false) {
ret = append(ret, strings.TrimSpace(dest))
}
}
}
}
}
}
return
}
func getRemoteAssetsLinkDestsInTree(tree *parse.Tree, onlyImg bool) (nodes []*ast.Node) {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
dests := getRemoteAssetsLinkDests(n, onlyImg)
if 1 > len(dests) {
return ast.WalkContinue
}
nodes = append(nodes, n)
return ast.WalkContinue
})
return
}
// allAssetAbsPaths 返回 asset 相对路径(assets/xxx)到绝对路径(F:\SiYuan\data\assets\xxx)的映射。
func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
notebooks, err := ListNotebooks()
if err != nil {
return
}
assetsAbsPathMap = map[string]string{}
deferred, deferredErr := DeferredSyncAssets()
if deferredErr != nil {
return nil, deferredErr
}
for _, file := range deferred {
absPath := filepath.Join(util.DataDir, filepath.FromSlash(strings.TrimPrefix(file.Path, "/")))
if IsEncryptedAssetPath(absPath) {
continue
}
relativePath, _, resolveErr := ResolveDataAssetPath(strings.TrimPrefix(file.Path, "/"))
if resolveErr != nil {
return nil, resolveErr
}
assetPath, _, valid := AssetPathFromDataRelativePath(relativePath)
if valid {
assetsAbsPathMap[assetPath] = absPath
for dir, absDir := path.Dir(assetPath), filepath.Dir(absPath); dir != "assets" && dir != "."; dir, absDir = path.Dir(dir), filepath.Dir(absDir) {
assetsAbsPathMap[dir+"/"] = absDir
}
}
}
// 笔记本 assets(跳过加密 box,加密资产不参与全局去重/清理)
for _, notebook := range notebooks {
if IsEncryptedBox(notebook.ID) {
continue
}
notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
walkErr := filelock.Walk(notebookAbsPath, func(walkPath string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if notebookAbsPath == walkPath {
return nil
}
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if filelock.IsHidden(walkPath) {
// 清理资源文件时忽略隐藏文件 Ignore hidden files when cleaning unused assets https://github.com/siyuan-note/siyuan/issues/12172
return nil
}
if d.IsDir() && "assets" == d.Name() {
assetsDirPath := walkPath
if nestedWalkErr := filelock.Walk(assetsDirPath, func(assetPath string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if assetsDirPath == assetPath {
return nil
}
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
relPath, relErr := assetPathMapKey(assetsDirPath, assetPath, d.IsDir())
if relErr != nil {
return relErr
}
assetsAbsPathMap[relPath] = assetPath
return nil
}); nestedWalkErr != nil {
return nestedWalkErr
}
return filepath.SkipDir
}
return nil
})
if walkErr != nil {
return nil, fmt.Errorf("walk notebook assets [%s] failed: %w", notebookAbsPath, walkErr)
}
}
// 全局 assets
dataAssetsAbsPath := util.GetDataAssetsAbsPath()
walkErr := filelock.Walk(dataAssetsAbsPath, func(assetPath string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if dataAssetsAbsPath == assetPath {
return nil
}
if isSkipFile(d.Name()) {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if filelock.IsHidden(assetPath) {
// 清理资源文件时忽略隐藏文件 Ignore hidden files when cleaning unused assets https://github.com/siyuan-note/siyuan/issues/12172
return nil
}
relPath, relErr := assetPathMapKey(dataAssetsAbsPath, assetPath, d.IsDir())
if relErr != nil {
return relErr
}
assetsAbsPathMap[relPath] = assetPath
return nil
})
if walkErr != nil {
if os.IsNotExist(walkErr) {
return
}
return nil, fmt.Errorf("walk global assets [%s] failed: %w", dataAssetsAbsPath, walkErr)
}
return
}
func assetPathMapKey(assetsDirPath, assetPath string, isDir bool) (ret string, err error) {
relPath, err := filepath.Rel(assetsDirPath, assetPath)
if err != nil {
return
}
relPath = filepath.ToSlash(relPath)
if relPath == "." || relPath == ".." || strings.HasPrefix(relPath, "../") || path.IsAbs(relPath) {
err = fmt.Errorf("asset path [%s] is outside assets directory [%s]", assetPath, assetsDirPath)
return
}
ret = path.Join("assets", relPath)
if isDir {
ret += "/"
}
return
}
// copyBoxAssetsToDataAssets 将笔记本路径下所有(包括子文档)的 assets 复制一份到 data/assets 中。
func copyBoxAssetsToDataAssets(boxID string) error {
boxLocalPath := filepath.Join(util.DataDir, boxID)
return copyAssetsToDataAssets(boxLocalPath)
}
// copyDocAssetsToDataAssets 将文档路径下所有(包括子文档)的 assets 复制一份到 data/assets 中。
func copyDocAssetsToDataAssets(boxID, parentDocPath string) error {
boxLocalPath := filepath.Join(util.DataDir, boxID)
parentDocDirAbsPath := filepath.Dir(filepath.Join(boxLocalPath, parentDocPath))
return copyAssetsToDataAssets(parentDocDirAbsPath)
}
func copyAssetsToDataAssets(rootPath string) error {
var assetsDirPaths []string
if err := filelock.Walk(rootPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if rootPath == path || nil == d {
return nil
}
isDir, name := d.IsDir(), d.Name()
if isSkipFile(name) {
if isDir {
return filepath.SkipDir
}
return nil
}
if "assets" == name && isDir {
assetsDirPaths = append(assetsDirPaths, path)
}
return nil
}); err != nil {
return err
}
dataAssetsPath := filepath.Join(util.DataDir, "assets")
for _, assetsDirPath := range assetsDirPaths {
if err := promoteAssetDirectory(assetsDirPath, dataAssetsPath); err != nil {
logging.LogErrorf("copy tree assets from [%s] to [%s] failed: %s", assetsDirPaths, dataAssetsPath, err)
return err
}
}
return nil
}