An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later
package model
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
htmlstd "html"
"image"
_ "image/gif"
"image/jpeg"
"image/png"
"io/fs"
"maps"
"os"
"path"
"path/filepath"
"regexp"
"runtime/debug"
"sort"
"strings"
"github.com/88250/gulu"
"github.com/88250/lute"
"github.com/88250/lute/ast"
"github.com/88250/lute/html"
"github.com/88250/lute/html/atom"
"github.com/88250/lute/parse"
"github.com/88250/lute/render"
util2 "github.com/88250/lute/util"
"github.com/siyuan-note/dataparser"
"github.com/siyuan-note/filelock"
"github.com/siyuan-note/logging"
"github.com/siyuan-note/riff"
"github.com/siyuan-note/siyuan/kernel/av"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/task"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
_ "golang.org/x/image/bmp"
_ "golang.org/x/image/webp"
"gopkg.in/yaml.v3"
)
var (
importPrePattern = regexp.MustCompile(`(?sU)<pre [^>]*>(.*)</pre>`)
importSVGPattern = regexp.MustCompile(`(?i)<svg[^>]*>(.*?)</svg>`)
importEncodedNewlinePattern = regexp.MustCompile(`(?i)%0A`)
)
// GetImportAssetsDir 返回导入资源的落盘目录。普通笔记本复用已有本地目录,否则回退到全局目录。
func GetImportAssetsDir(boxID, docDirLocalPath string) string {
globalAssetsDir := filepath.Join(util.DataDir, "assets")
if !ast.IsNodeIDPattern(boxID) {
return globalAssetsDir
}
boxLocalPath := filepath.Join(util.DataDir, boxID)
boxAssetsDir := filepath.Join(boxLocalPath, "assets")
if IsEncryptedBox(boxID) {
return boxAssetsDir
}
if docDirLocalPath != "" && gulu.File.IsSubPath(boxLocalPath, docDirLocalPath) {
docAssetsDir := filepath.Join(docDirLocalPath, "assets")
if gulu.File.IsDir(docAssetsDir) {
return docAssetsDir
}
}
if gulu.File.IsDir(boxAssetsDir) {
return boxAssetsDir
}
return globalAssetsDir
}
type HTML2TreeOptions struct {
SkipBase64Assets bool
SkipInlineSVGAssets bool
}
func HTML2Tree(htmlStr string, luteEngine *lute.Lute, boxID string) (tree *parse.Tree, withMath bool) {
return HTML2TreeWithOptions(htmlStr, luteEngine, boxID, HTML2TreeOptions{})
}
func HTML2TreeWithOptions(htmlStr string, luteEngine *lute.Lute, boxID string,
options HTML2TreeOptions) (tree *parse.Tree, withMath bool) {
htmlStr = gulu.Str.RemovePUA(htmlStr)
assetDirPath := ""
if !options.SkipBase64Assets || !options.SkipInlineSVGAssets {
assetDirPath = GetImportAssetsDir(boxID, "")
_ = os.MkdirAll(assetDirPath, 0755)
}
tree = luteEngine.HTML2Tree(htmlStr)
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
switch n.Type {
case ast.NodeHTMLBlock:
if bytes.HasPrefix(n.Tokens, []byte("<pre ")) && bytes.HasSuffix(n.Tokens, []byte("</pre>")) {
if bytes.Contains(n.Tokens, []byte("data:image/svg+xml;base64")) {
matches := importPrePattern.FindSubmatch(n.Tokens)
if len(matches) >= 2 {
n.Tokens = matches[1]
}
subTree := parse.Inline("", n.Tokens, luteEngine.ParseOptions)
if nil != subTree && nil != subTree.Root && nil != subTree.Root.FirstChild {
n.Type = ast.NodeParagraph
var children []*ast.Node
for c := subTree.Root.FirstChild.FirstChild; nil != c; c = c.Next {
children = append(children, c)
}
for _, c := range children {
n.AppendChild(c)
}
}
} else if !options.SkipInlineSVGAssets && bytes.Contains(n.Tokens, []byte("<svg")) {
processHTMLBlockSvgImg(n, assetDirPath, boxID)
}
}
case ast.NodeText:
if n.ParentIs(ast.NodeTableCell) {
n.Tokens = bytes.ReplaceAll(n.Tokens, []byte("\\|"), []byte("|"))
n.Tokens = bytes.ReplaceAll(n.Tokens, []byte("|"), []byte("\\|"))
n.Tokens = bytes.ReplaceAll(n.Tokens, []byte("\\<br /\\>"), []byte("<br />"))
}
case ast.NodeInlineMath:
withMath = true
case ast.NodeLinkDest:
dest := n.TokensStr()
if !options.SkipBase64Assets && strings.HasPrefix(dest, "data:image") && strings.Contains(dest, ";base64,") {
processBase64Img(n, dest, assetDirPath, boxID)
}
}
return ast.WalkContinue
})
return
}
func ImportSY(zipPath, boxID, toPath string) (err error) {
if isSYNotebookBundle(zipPath) {
return errors.New(Conf.Language(373))
}
_, err = importSY(zipPath, boxID, toPath, false, false)
return
}
func ImportSYNotebook(zipPath string) (boxID string, err error) {
return importSY(zipPath, "", "/", true, false)
}
var ErrSYTargetNotebookRequired = errors.New("target notebook required")
func ImportSYAuto(zipPath, boxID, toPath string) (createdBoxID string, notebook bool, err error) {
createdBoxID, err = importSY(zipPath, boxID, toPath, false, true)
notebook = err == nil && createdBoxID != boxID
return
}
func isSYNotebookExport(hasBoxConf, hasBoxDocMeta bool) bool {
return hasBoxConf || hasBoxDocMeta
}
type importedSYSortDoc struct {
oldID string
newID string
sourcePath string
hidden bool
}
func importedSYRootIDs(importedDocs []*importedSYSortDoc) (ret []string) {
for _, doc := range importedDocs {
if nil == doc || doc.hidden || "/" != path.Dir(doc.sourcePath) {
continue
}
ret = append(ret, doc.newID)
}
sort.Strings(ret)
return
}
func pushDocsImported(boxID, parentPath, format string, rootIDs []string) {
if 1 > len(rootIDs) {
return
}
util.BroadcastByType("main", "docsImported", 0, "", map[string]any{
"notebook": boxID,
"parentPath": parentPath,
"rootIDs": rootIDs,
"format": format,
})
}
func buildImportedSYSortValues(importedDocs []*importedSYSortDoc, sourceSortIDs map[string]int,
existingRootIDs []string, createDocAtTop bool) map[string]int {
groups := map[string][]*importedSYSortDoc{}
for _, doc := range importedDocs {
if nil == doc || doc.hidden {
continue
}
parentPath := path.Dir(doc.sourcePath)
groups[parentPath] = append(groups[parentPath], doc)
}
for _, docs := range groups {
sort.Slice(docs, func(i, j int) bool {
leftSort, rightSort := sourceSortIDs[docs[i].oldID], sourceSortIDs[docs[j].oldID]
if leftSort != rightSort {
return leftSort < rightSort
}
leftCreated, rightCreated := util.TimeFromID(docs[i].oldID), util.TimeFromID(docs[j].oldID)
if leftCreated != rightCreated {
return leftCreated > rightCreated
}
return docs[i].oldID < docs[j].oldID
})
}
sortValues := map[string]int{}
for _, docs := range groups {
for sortValue, doc := range docs {
sortValues[doc.newID] = sortValue + 1
}
}
rootDocs := groups["/"]
rootIDs := make([]string, 0, len(rootDocs))
for _, doc := range rootDocs {
rootIDs = append(rootIDs, doc.newID)
}
orderedRootIDs := make([]string, 0, len(rootIDs)+len(existingRootIDs))
if createDocAtTop {
orderedRootIDs = append(orderedRootIDs, rootIDs...)
orderedRootIDs = append(orderedRootIDs, existingRootIDs...)
} else {
orderedRootIDs = append(orderedRootIDs, existingRootIDs...)
orderedRootIDs = append(orderedRootIDs, rootIDs...)
}
for sortValue, id := range orderedRootIDs {
sortValues[id] = sortValue + 1
}
return sortValues
}
func applyImportedSYSort(boxID, targetPath string, importedDocs []*importedSYSortDoc, sourceSortIDs map[string]int) error {
confFileTreeSnapshot := Conf.LoadFileTree()
if 1 > len(importedDocs) {
return nil
}
fileTreeSortLock.Lock()
defer fileTreeSortLock.Unlock()
confPath := filepath.Join(util.DataDir, boxID, ".siyuan", "sort.json")
fullSortIDs, err := readSortConfMap(confPath)
if nil != err {
return err
}
existingIDs, err := loadSiblingCustomOrder(boxID, targetPath, fullSortIDs)
if nil != err {
return err
}
importedIDs := map[string]struct{}{}
for _, doc := range importedDocs {
if nil != doc {
importedIDs[doc.newID] = struct{}{}
}
}
existingRootIDs := make([]string, 0, len(existingIDs))
for _, id := range existingIDs {
if _, imported := importedIDs[id]; !imported {
existingRootIDs = append(existingRootIDs, id)
}
}
createDocAtTop := nil != confFileTreeSnapshot.CreateDocAtTop && *confFileTreeSnapshot.CreateDocAtTop
sortValues := buildImportedSYSortValues(importedDocs, sourceSortIDs, existingRootIDs, createDocAtTop)
if 1 > len(sortValues) {
return nil
}
maps.Copy(fullSortIDs, sortValues)
return writeSortConfMap(confPath, fullSortIDs)
}
func importSY(zipPath, boxID, toPath string, createNotebook, autoDetect bool) (createdBoxID string, err error) {
return importSY0(zipPath, boxID, toPath, createNotebook, autoDetect, nil, false)
}
// checkEncryptedImportFlashcards 检查待导入文档是否带有闪卡属性。
// 加密笔记本是孤岛,闪卡牌组与调度依赖全局数据库,因此必须拒绝导入;命中时返回具体文档,避免只提示"不支持该操作"。
func checkEncryptedImportFlashcards(tree *parse.Tree, sourcePath string, encryptedTarget bool) error {
if !encryptedTarget || nil == tree || nil == tree.Root {
return nil
}
containsFlashcardAttrs := false
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if entering && n.IsBlock() && n.IALAttr(NodeAttrRiffDecks) != "" {
containsFlashcardAttrs = true
return ast.WalkStop
}
return ast.WalkContinue
})
if !containsFlashcardAttrs {
return nil
}
title := treenode.ReadDocumentTitle(tree.Root).Text
if "" == title {
title = filepath.Base(sourcePath)
}
return fmt.Errorf(Conf.Language(386), htmlstd.EscapeString(title))
}
// importedBlockDocTitles 建立本次导入块 ID 到所属文档标题的映射,供闪卡卡片反查文档。
func importedBlockDocTitles(trees map[string]*parse.Tree) map[string]string {
ret := map[string]string{}
for _, tree := range trees {
if nil == tree || nil == tree.Root {
continue
}
title := treenode.ReadDocumentTitle(tree.Root).Text
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if entering && "" != n.ID {
ret[n.ID] = title
}
return ast.WalkContinue
})
}
return ret
}
// checkEncryptedImportDeck 检查导入包里的闪卡数据。
// 加密笔记本是孤岛,闪卡牌组与调度依赖全局数据库,因此必须拒绝导入;
// 报错会指出包内哪些文档的卡片位于牌组中,便于用户在源笔记本中移除闪卡后重新导出。
func checkEncryptedImportDeck(unzipRootPath string, trees map[string]*parse.Tree, blockIDs map[string]string) error {
confFlashcardSnapshot := Conf.LoadFlashcard()
storageRiffDir := filepath.Join(unzipRootPath, "storage", "riff")
if !gulu.File.IsExist(storageRiffDir) {
return nil
}
deck, loadErr := riff.LoadDeck(storageRiffDir, builtinDeckID, confFlashcardSnapshot.RequestRetention, confFlashcardSnapshot.MaximumInterval, confFlashcardSnapshot.Weights)
if nil != loadErr || nil == deck {
// 牌组无法解析时仍要拒绝导入,此时无法指出具体文档,只报告包内牌组路径
logging.LogErrorf("load imported deck [%s] failed: %s", storageRiffDir, loadErr)
return fmt.Errorf(Conf.Language(385), filepath.ToSlash(filepath.Join("storage", "riff")))
}
var deckDocs []string
blockDocTitles := importedBlockDocTitles(trees)
for _, blockID := range deck.GetBlockIDs() {
// 卡片记录的是导出时的旧块 ID,需按导入时重建的映射换算
newBlockID := blockIDs[blockID]
if "" == newBlockID {
newBlockID = blockID
}
if title := blockDocTitles[newBlockID]; "" != title {
deckDocs = append(deckDocs, title)
} else {
// 卡片所属块不在本次导入范围内,保留块 ID 供用户定位
deckDocs = append(deckDocs, newBlockID)
}
}
if 0 == len(deckDocs) {
// 牌组存在但没有可定位的卡片,退回报告包内牌组路径
return fmt.Errorf(Conf.Language(385), filepath.ToSlash(filepath.Join("storage", "riff")))
}
deckDocs = gulu.Str.RemoveDuplicatedElem(deckDocs)
sort.Strings(deckDocs)
if 5 < len(deckDocs) {
deckDocs = append(deckDocs[:5], "...")
}
return fmt.Errorf(Conf.Language(385), htmlstd.EscapeString(strings.Join(deckDocs, ", ")))
}
func importSY0(zipPath, boxID, toPath string, createNotebook, autoDetect bool, sharedBlockIDs map[string]string,
precreatedBox bool) (createdBoxID string, err error) {
confFlashcardReadSnapshot := Conf.LoadFlashcard()
util.PushEndlessProgress(Conf.Language(73))
defer util.ClearPushProgress(100)
lockSync()
defer unlockSync()
baseName := filepath.Base(zipPath)
ext := filepath.Ext(baseName)
baseName = strings.TrimSuffix(baseName, ext)
unzipPath := filepath.Join(filepath.Dir(zipPath), baseName+"-"+gulu.Rand.String(7))
err = gulu.Zip.Unzip(zipPath, unzipPath)
if err != nil {
return
}
defer os.RemoveAll(unzipPath)
var syPaths []string
filelock.Walk(unzipPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d == nil {
return nil
}
if !d.IsDir() && strings.HasSuffix(d.Name(), ".sy") {
syPaths = append(syPaths, path)
}
return nil
})
entries, err := os.ReadDir(unzipPath)
if err != nil {
logging.LogErrorf("read unzip dir [%s] failed: %s", unzipPath, err)
return
}
if 1 != len(entries) || !entries[0].IsDir() {
logging.LogErrorf("invalid .sy.zip [%v]", entries)
err = errors.New(Conf.Language(199))
return
}
unzipRootPath := filepath.Join(unzipPath, entries[0].Name())
if err = checkImportedDocumentAddressing(unzipRootPath); err != nil {
return
}
// 全部标题源通过检查后才导入表情、创建笔记本或重新分配身份。
for _, syPath := range syPaths {
data, readErr := filelock.ReadFile(syPath)
if readErr != nil {
return "", readErr
}
if err = treenode.CheckSpecJSON(data); err != nil {
return
}
}
name := filepath.Base(unzipRootPath)
if strings.HasPrefix(name, "data-20") && len("data-20230321175442") == len(name) {
logging.LogErrorf("invalid .sy.zip [unzipRootPath=%s, baseName=%s]", unzipRootPath, name)
err = errors.New(Conf.Language(199))
return
}
var importedBoxConf *conf.BoxConf
importedConfPath := filepath.Join(unzipRootPath, ".siyuan", "conf.json")
hasImportedBoxConf := filelock.IsExist(importedConfPath)
var importedMetadataErr error
if hasImportedBoxConf {
confData, readErr := filelock.ReadFile(importedConfPath)
if readErr == nil {
importedBoxConf = conf.NewBoxConf()
if unmarshalErr := gulu.JSON.UnmarshalJSON(confData, importedBoxConf); unmarshalErr != nil {
logging.LogWarnf("parse imported notebook conf failed: %s", unmarshalErr)
importedBoxConf = nil
importedMetadataErr = unmarshalErr
}
} else {
logging.LogWarnf("read imported notebook conf failed: %s", readErr)
importedMetadataErr = readErr
}
if removeErr := filelock.Remove(importedConfPath); removeErr != nil {
err = removeErr
return
}
}
var importedBoxDocID string
importedBoxDocPath := filepath.Join(unzipRootPath, ".siyuan", boxDocMetaName)
hasImportedBoxDocMeta := filelock.IsExist(importedBoxDocPath)
if hasImportedBoxDocMeta {
metaData, readErr := filelock.ReadFile(importedBoxDocPath)
if readErr == nil {
meta := &boxDocMeta{}
if unmarshalErr := gulu.JSON.UnmarshalJSON(metaData, meta); unmarshalErr != nil {
logging.LogWarnf("parse imported notebook document metadata failed: %s", unmarshalErr)
importedMetadataErr = unmarshalErr
} else if meta.Spec != boxDocMetaSpec || !ast.IsNodeIDPattern(meta.BoxDocID) {
logging.LogWarnf("invalid imported notebook document metadata [spec=%d, id=%s]", meta.Spec, meta.BoxDocID)
importedMetadataErr = errors.New("invalid imported notebook document metadata")
} else {
importedBoxDocID = meta.BoxDocID
}
} else {
logging.LogWarnf("read imported notebook document metadata failed: %s", readErr)
importedMetadataErr = readErr
}
if removeErr := filelock.Remove(importedBoxDocPath); removeErr != nil {
err = removeErr
return
}
}
notebookExport := isSYNotebookExport(hasImportedBoxConf, hasImportedBoxDocMeta)
if len(syPaths) < 1 && !notebookExport {
logging.LogErrorf("invalid .sy.zip without documents or notebook metadata [unzipRootPath=%s]", unzipRootPath)
err = errors.New(Conf.Language(199))
return
}
if autoDetect {
if importedMetadataErr != nil {
err = errors.New(Conf.Language(199))
return
}
createNotebook = notebookExport
}
if !createNotebook && len(syPaths) < 1 {
logging.LogErrorf("invalid .sy.zip without documents [unzipRootPath=%s]", unzipRootPath)
err = errors.New(Conf.Language(199))
return
}
if autoDetect && !createNotebook && boxID == "" {
err = ErrSYTargetNotebookRequired
return
}
if !createNotebook && nil == Conf.Box(boxID) {
err = errors.New(Conf.Language(0))
return
}
if _, err = importEmojiFiles(unzipRootPath, true); err != nil {
return
}
if createNotebook {
if importedBoxConf != nil && importedBoxConf.Name != "" {
name = importedBoxConf.Name
}
if !precreatedBox {
boxID, err = CreateBox(util.RemoveInvalid(name))
if err != nil {
return "", err
}
}
createdBoxID = boxID
defer func() {
if err == nil {
return
}
treenode.RemoveBlockTreesByBoxID(boxID)
sql.DeleteBoxQueue(boxID)
if removeErr := filelock.Remove(filepath.Join(util.DataDir, boxID)); removeErr != nil {
logging.LogErrorf("remove notebook [%s] after import failed: %s", boxID, removeErr)
}
}()
if importedBoxConf != nil {
box := &Box{ID: boxID}
boxConf := box.GetConf()
boxConf.Icon = filterBoxIcon(importedBoxConf.Icon)
boxConf.RefCreateSavePath = importedBoxConf.RefCreateSavePath
boxConf.DocCreateSavePath = importedBoxConf.DocCreateSavePath
boxConf.DocCreateTemplatePath = importedBoxConf.DocCreateTemplatePath
boxConf.DailyNoteSavePath = importedBoxConf.DailyNoteSavePath
boxConf.DailyNoteTemplatePath = importedBoxConf.DailyNoteTemplatePath
boxConf.SortMode = importedBoxConf.SortMode
if err = box.SaveConf(boxConf); err != nil {
return createdBoxID, err
}
}
} else {
createdBoxID = boxID
}
encryptedTarget := IsEncryptedBox(boxID)
toPath = normalizeBoxDocTarget(boxID, toPath)
luteEngine := util.NewLute()
blockIDs := sharedBlockIDs
if nil == blockIDs {
blockIDs = map[string]string{}
}
trees := map[string]*parse.Tree{}
var importedSortDocs []*importedSYSortDoc
importedBoxDoc := false
// 重新生成块 ID
for i, syPath := range syPaths {
data, readErr := os.ReadFile(syPath)
if nil != readErr {
logging.LogErrorf("read .sy [%s] failed: %s", syPath, readErr)
err = readErr
return
}
if err = treenode.CheckSpecJSON(data); nil != err {
return
}
tree, _, parseErr := dataparser.ParseJSON(data, luteEngine.ParseOptions)
if nil != parseErr {
logging.LogErrorf("parse .sy [%s] failed: %s", syPath, parseErr)
err = parseErr
return
}
// 加密笔记本不支持闪卡,命中时给出具体文档而不是笼统的"不支持该操作"
if err = checkEncryptedImportFlashcards(tree, syPath, encryptedTarget); nil != err {
return
}
oldRootID := tree.Root.ID
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if "" == n.ID {
return ast.WalkContinue
}
// 新 ID 保留时间部分,仅修改随机值,避免时间变化导致更新时间早于创建时间
// Keep original creation time when importing .sy.zip https://github.com/siyuan-note/siyuan/issues/9923
newNodeID := blockIDs[n.ID]
if "" == newNodeID {
newNodeID = util.TimeFromID(n.ID) + "-" + util.RandString(7)
}
if createNotebook && oldRootID == importedBoxDocID && n.ID == importedBoxDocID {
newNodeID = boxID
}
blockIDs[n.ID] = newNodeID
n.ID = newNodeID
n.SetIALAttr("id", newNodeID)
if icon := n.IALAttr("icon"); "" != icon {
// XSS through emoji name https://github.com/siyuan-note/siyuan/issues/15034
icon = filterBoxIcon(icon)
n.SetIALAttr("icon", icon)
}
return ast.WalkContinue
})
tree.ID = tree.Root.ID
tree.Path = filepath.ToSlash(strings.TrimPrefix(syPath, unzipRootPath))
if createNotebook && oldRootID == importedBoxDocID {
importedBoxDoc = true
tree.Root.SetIALAttr(DocHiddenAttr, "true")
} else if oldRootID == importedBoxDocID {
removeBoxDocHiddenAttr(tree)
}
importedSortDocs = append(importedSortDocs, &importedSYSortDoc{
oldID: oldRootID,
newID: tree.ID,
sourcePath: tree.Path,
hidden: createNotebook && oldRootID == importedBoxDocID,
})
trees[tree.ID] = tree
util.PushEndlessProgress(Conf.language(73) + " " + fmt.Sprintf(Conf.language(70), fmt.Sprintf("%d/%d", i+1, len(syPaths))))
}
if createNotebook && importedBoxConf != nil {
sourceBoxID := importedBoxDocID
if sourceBoxID == "" && ast.IsNodeIDPattern(filepath.Base(unzipRootPath)) {
sourceBoxID = filepath.Base(unzipRootPath)
}
box := &Box{ID: boxID}
boxConf := box.GetConf()
boxConf.DocCreateTarget = remapImportedDocumentTarget(importedBoxConf.DocCreateTarget, sourceBoxID, boxID, blockIDs)
boxConf.RefCreateTarget = remapImportedDocumentTarget(importedBoxConf.RefCreateTarget, sourceBoxID, boxID, blockIDs)
boxConf.DailyNoteTarget = remapImportedDocumentTarget(importedBoxConf.DailyNoteTarget, sourceBoxID, boxID, blockIDs)
boxConf.MetadataVersion = importedBoxConf.MetadataVersion
if err = box.SaveConf(boxConf); err != nil {
return
}
}
// 加密笔记本不支持闪卡:.sy.zip 内的牌组数据与文档闪卡属性都必须拒绝导入。
// 此时块 ID 已完成重映射,可以指出具体是哪些文档的卡片在包内,而不是只提示"不支持该操作"。
if encryptedTarget {
if err = checkEncryptedImportDeck(unzipRootPath, trees, blockIDs); nil != err {
return
}
// 兜底校验:禁止跨加密边界块引。包内文档尚未入库,块树查不到,因此把本次导入的全部块 ID
// 一并放行,既拦住包外引用(普通笔记本或其他加密笔记本的块),又不误伤包内跨文档引用。
importedBlockIDs := make(map[string]bool, len(blockIDs))
for _, newID := range blockIDs {
importedBlockIDs[newID] = true
}
for _, tree := range trees {
degradeCrossBoundaryBlockRefsWithAllowed(tree.Root, tree.Box, nil, importedBlockIDs)
}
}
if importedBoxDoc {
if err = writeBoxDocID(boxID); err != nil {
return
}
}
// 引用和嵌入指向重新生成的块 ID
for _, tree := range trees {
treenode.RemapTabsActiveIDs(tree.Root, blockIDs)
util.PushEndlessProgress(Conf.language(73) + " " + fmt.Sprintf(Conf.language(70), treenode.ReadDocumentTitle(tree.Root).Text))
treenode.WalkWithInlineTitles(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if treenode.IsBlockRef(n) {
defID, _, _ := treenode.GetBlockRef(n)
newDefID := blockIDs[defID]
if "" != newDefID {
n.TextMarkBlockRefID = newDefID
}
} else if ast.NodeTextMark == n.Type && n.IsTextMarkType("a") && strings.HasPrefix(n.TextMarkAHref, "siyuan://blocks/") {
// Block hyperlinks do not point to regenerated block IDs when importing .sy.zip https://github.com/siyuan-note/siyuan/issues/9083
defID := strings.TrimPrefix(n.TextMarkAHref, "siyuan://blocks/")
newDefID := blockIDs[defID]
if "" != newDefID {
n.TextMarkAHref = "siyuan://blocks/" + newDefID
}
} else if ast.NodeBlockQueryEmbedScript == n.Type {
for oldID, newID := range blockIDs {
// 导入 `.sy.zip` 后查询嵌入块失效 https://github.com/siyuan-note/siyuan/issues/5316
n.Tokens = bytes.ReplaceAll(n.Tokens, []byte(oldID), []byte(newID))
}
}
return ast.WalkContinue
})
}
var replacements []string
importedBlockIDs := map[string]bool{}
for _, tree := range trees {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if entering && n.ID != "" {
importedBlockIDs[n.ID] = true
}
return ast.WalkContinue
})
}
for oldID, newID := range blockIDs {
replacements = append(replacements, oldID, newID)
}
blockIDReplacer := strings.NewReplacer(replacements...)
assetPathMap, err := importSYAssets(unzipRootPath, boxID)
if err != nil {
return createdBoxID, err
}
assetRewriteOptions := assetReferenceRewriteOptions{
pathMap: assetPathMap,
targetBoxID: boxID,
bindTargetBox: IsEncryptedBox(boxID),
}
for _, tree := range trees {
rewriteTreeAssetReferences(tree, assetRewriteOptions)
}
// 将关联的数据库文件移动到 data/storage/av/ 下
storage := filepath.Join(unzipRootPath, "storage")
storageAvDir := filepath.Join(storage, "av")
avIDs := map[string]string{}
renameAvPaths := map[string]string{}
if gulu.File.IsExist(storageAvDir) {
// 重新生成数据库数据
filelock.Walk(storageAvDir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d == nil {
return nil
}
if ".json" == d.Name() { // https://github.com/siyuan-note/siyuan/issues/16637
if removeErr := os.RemoveAll(path); nil != removeErr {
logging.LogErrorf("remove empty av file [%s] failed: %s", path, removeErr)
}
return nil
}
if !strings.HasSuffix(path, ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(d.Name(), ".json")) {
return nil
}
// 重命名数据库
newAvID := ast.NewNodeID()
oldAvID := strings.TrimSuffix(d.Name(), ".json")
newPath := filepath.Join(filepath.Dir(path), newAvID+".json")
renameAvPaths[path] = newPath
avIDs[oldAvID] = newAvID
return nil
})
// 重命名数据库文件
for oldPath, newPath := range renameAvPaths {
data, readErr := os.ReadFile(oldPath)
if nil != readErr {
logging.LogErrorf("read av file [%s] failed: %s", oldPath, readErr)
err = readErr
return
}
// 将数据库文件中的 ID 替换为新的 ID
newData := data
for oldAvID, newAvID := range avIDs {
newData = bytes.ReplaceAll(newData, []byte(oldAvID), []byte(newAvID))
}
newData = []byte(blockIDReplacer.Replace(string(newData)))
newData, err = isolateImportedAttributeViewBindings(newData, importedBlockIDs, encryptedTarget)
if err != nil {
return
}
newData, err = rewriteAttributeViewDataAssetReferences(newData, assetRewriteOptions)
if err != nil {
logging.LogErrorf("rewrite imported attribute view assets [%s] failed: %s", oldPath, err)
return
}
if !bytes.Equal(data, newData) {
if writeErr := os.WriteFile(oldPath, newData, 0644); nil != writeErr {
logging.LogErrorf("write av file [%s] failed: %s", oldPath, writeErr)
err = writeErr
return
}
}
if err = os.Rename(oldPath, newPath); err != nil {
logging.LogErrorf("rename av file from [%s] to [%s] failed: %s", oldPath, newPath, err)
return
}
}
// 加密笔记本的 AV 定义不能拷到全局目录(明文泄漏 + 路由冲突),
// 需要先加密写入 <boxID>/storage/av/,后续 mirror/relation 操作才能正确路由
if !IsEncryptedBox(boxID) {
targetStorageAvDir := filepath.Join(util.DataDir, "storage", "av")
if copyErr := filelock.Copy(storageAvDir, targetStorageAvDir); nil != copyErr {
logging.LogErrorf("copy storage av dir from [%s] to [%s] failed: %s", storageAvDir, targetStorageAvDir, copyErr)
err = copyErr
return
}
} else {
// 加密笔记本:先把 AV 定义加密写入笔记本级目录,建立 box 映射后 mirror/relation 才能正确路由
if err = encryptBoxAVFiles(boxID, storageAvDir); err != nil {
return
}
}
// 重新指向数据库属性值
for _, tree := range trees {
util.PushEndlessProgress(Conf.language(73) + " " + fmt.Sprintf(Conf.language(70), treenode.ReadDocumentTitle(tree.Root).Text))
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || "" == n.ID {
return ast.WalkContinue
}
ial := parse.IAL2Map(n.KramdownIAL)
for k, v := range ial {
if strings.HasPrefix(k, av.NodeAttrNameAvs) {
newKey, newVal := k, v
for oldAvID, newAvID := range avIDs {
newKey = strings.ReplaceAll(newKey, oldAvID, newAvID)
newVal = strings.ReplaceAll(newVal, oldAvID, newAvID)
}
n.RemoveIALAttr(k)
n.SetIALAttr(newKey, newVal)
}
}
if ast.NodeAttributeView == n.Type {
n.AttributeViewID = avIDs[n.AttributeViewID]
}
if encryptedTarget {
retainImportedAttributeViewBindings(n, avIDs)
}
return ast.WalkContinue
})
}
// 如果数据库中绑定的块不在导入的文档中,则需要单独更新这些绑定块的属性
var attrViewIDs []string
for _, avID := range avIDs {
attrViewIDs = append(attrViewIDs, avID)
}
avBoxID := ""
if encryptedTarget {
avBoxID = boxID
}
updateBoundBlockAvsAttribute(attrViewIDs, avBoxID)
// 插入关联关系 https://github.com/siyuan-note/siyuan/issues/11628
for _, avID := range avIDs {
attrView, _ := av.ParseAttributeView(avID)
if nil == attrView {
continue
}
for _, keyValues := range attrView.KeyValues {
if nil != keyValues.Key && av.KeyTypeRelation == keyValues.Key.Type && nil != keyValues.Key.Relation && keyValues.Key.Relation.AvID != "" {
av.UpsertAvBackRel(avID, keyValues.Key.Relation.AvID)
}
}
}
}
// 普通笔记本将关联的闪卡数据合并到默认卡包 data/storage/riff/20230218211946-2kw8jgx 中
storageRiffDir := filepath.Join(storage, "riff")
if gulu.File.IsExist(storageRiffDir) {
deckToImport, loadErr := riff.LoadDeck(storageRiffDir, builtinDeckID, confFlashcardReadSnapshot.RequestRetention, confFlashcardReadSnapshot.MaximumInterval, confFlashcardReadSnapshot.Weights)
if nil != loadErr {
logging.LogErrorf("load deck [%s] failed: %s", name, loadErr)
} else {
deck := Decks[builtinDeckID]
if nil == deck {
var createErr error
deck, createErr = createDeck0("Built-in Deck", builtinDeckID)
if nil == createErr {
Decks[deck.ID] = deck
}
}
bIDs := deckToImport.GetBlockIDs()
cards := deckToImport.GetCardsByBlockIDs(bIDs)
for _, card := range cards {
deck.AddCard(ast.NewNodeID(), blockIDs[card.BlockID()])
}
if 0 < len(cards) {
if saveErr := deck.Save(); nil != saveErr {
logging.LogErrorf("save deck [%s] failed: %s", name, saveErr)
}
}
}
}
// storage 文件夹已在上方处理,所以这里删除源 storage 文件夹,避免后面被拷贝到导入目录下 targetDir
// 加密笔记本已在上面完成 notebook 级 AV 拷贝,安全删除
if removeErr := os.RemoveAll(storage); nil != removeErr {
logging.LogErrorf("remove temp storage av dir [%s] failed: %s", storage, removeErr)
}
if 1 > len(avIDs) { // 如果本次没有导入数据库,则清理掉文档中的数据库属性 https://github.com/siyuan-note/siyuan/issues/13011
for _, tree := range trees {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || !n.IsBlock() {
return ast.WalkContinue
}
n.RemoveIALAttr(av.NodeAttrNameAvs)
return ast.WalkContinue
})
}
}
// 写回 .sy
for _, tree := range trees {
util.PushEndlessProgress(Conf.language(73) + " " + fmt.Sprintf(Conf.language(70), treenode.ReadDocumentTitle(tree.Root).Text))
syPath := filepath.Join(unzipRootPath, tree.Path)
// 加密前确定最终文件名:tree.ID + ".sy",确保加密 AAD 用的是最终 box-relative path
finalSyName := tree.ID + ".sy"
finalRelPath := filepath.ToSlash(filepath.Join(filepath.Dir(tree.Path), finalSyName))
treenode.UpgradeSpec(tree)
renderer := render.NewJSONRenderer(tree, luteEngine.RenderOptions, luteEngine.ParseOptions)
data := renderer.Render()
if !util.UseSingleLineSave.Load() {
buf := bytes.Buffer{}
buf.Grow(1024 * 1024 * 2)
if err = json.Indent(&buf, data, "", "\t"); err != nil {
return
}
data = buf.Bytes()
}
newSyPath := filepath.Join(filepath.Dir(syPath), finalSyName)
if err = writeImportedTree(boxID, syPath, newSyPath, finalRelPath, data); err != nil {
logging.LogErrorf("write imported .sy [%s] failed: %s", syPath, err)
// 只有"目标笔记本未解锁导致拒绝写盘"才替换为提示解锁的文案,其余写盘错误原样上抛,避免归因错误。
// 相对路径的父目录名来自导入包,需与文档标题一样转义后再进入错误消息。
if errors.Is(err, errImportedTreeBoxLocked) {
err = fmt.Errorf(Conf.Language(388), htmlstd.EscapeString(finalRelPath))
}
return
}
tree.Path = finalRelPath
}
// 读取包内自定义排序,复制成功后根据目标位置重新编号
sourceSortIDs := map[string]int{}
var sortErr error
sortPath := filepath.Join(unzipRootPath, ".siyuan", "sort.json")
if filelock.IsExist(sortPath) {
sourceSortIDs, sortErr = readSortConfMap(sortPath)
if nil != sortErr {
logging.LogErrorf("read import sort conf failed: %s", sortErr)
sourceSortIDs = map[string]int{}
}
if removeErr := os.RemoveAll(sortPath); nil != removeErr {
logging.LogErrorf("remove temp sort conf failed: %s", removeErr)
}
}
// 重命名文件路径
renamePaths := map[string]string{}
filelock.Walk(unzipRootPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d == nil {
return nil
}
if d.IsDir() && ast.IsNodeIDPattern(d.Name()) {
renamePaths[path] = path
}
return nil
})
for p := range renamePaths {
originalPath := p
p = strings.TrimPrefix(p, unzipRootPath)
p = filepath.ToSlash(p)
parts := strings.Split(p, "/")
buf := bytes.Buffer{}
buf.WriteString("/")
for i, part := range parts {
if "" == part {
continue
}
newNodeID := blockIDs[part]
if "" != newNodeID {
buf.WriteString(newNodeID)
} else {
buf.WriteString(part)
}
if i < len(parts)-1 {
buf.WriteString("/")
}
}
newPath := buf.String()
renamePaths[originalPath] = filepath.Join(unzipRootPath, newPath)
}
var oldPaths []string
for oldPath := range renamePaths {
oldPaths = append(oldPaths, oldPath)
}
sort.Slice(oldPaths, func(i, j int) bool {
return strings.Count(oldPaths[i], string(os.PathSeparator)) < strings.Count(oldPaths[j], string(os.PathSeparator))
})
for i, oldPath := range oldPaths {
newPath := renamePaths[oldPath]
if err = filelock.Rename(oldPath, newPath); err != nil {
logging.LogErrorf("rename path from [%s] to [%s] failed: %s", oldPath, renamePaths[oldPath], err)
err = errors.New("rename path failed")
return
}
delete(renamePaths, oldPath)
var toRemoves []string
newRenamedPaths := map[string]string{}
for oldP, newP := range renamePaths {
if strings.HasPrefix(oldP, oldPath) {
renamedOldP := strings.Replace(oldP, oldPath, newPath, 1)
newRenamedPaths[renamedOldP] = newP
toRemoves = append(toRemoves, oldPath)
}
}
for _, toRemove := range toRemoves {
delete(renamePaths, toRemove)
}
maps.Copy(renamePaths, newRenamedPaths)
for j := i + 1; j < len(oldPaths); j++ {
if strings.HasPrefix(oldPaths[j], oldPath) {
renamedOldP := strings.Replace(oldPaths[j], oldPath, newPath, 1)
oldPaths[j] = renamedOldP
}
}
}
// AV 定义已在 storage 删除前处理(加密笔记本DEK 加密拷到笔记本级,
// 普通 box 拷到全局 storage/av/),这里不再重复处理
// 将包内表情复制到全局目录,冲突时不覆盖已有内容。
emojiDirs, err := importEmojiFiles(unzipRootPath, false)
if err != nil {
return
}
for _, emojis := range emojiDirs {
os.RemoveAll(emojis)
}
var baseTargetPath string
if "/" == toPath {
baseTargetPath = "/"
} else {
block := treenode.GetBlockTreeRootByPath(boxID, toPath)
if nil == block {
logging.LogErrorf("not found block by path [%s]", toPath)
return createdBoxID, nil
}
baseTargetPath = strings.TrimSuffix(block.Path, ".sy")
}
targetDir := filepath.Join(util.DataDir, boxID, baseTargetPath)
if err = os.MkdirAll(targetDir, 0755); err != nil {
return
}
var treePaths []string
filelock.Walk(unzipRootPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d == nil {
return nil
}
if d.IsDir() {
if strings.HasPrefix(d.Name(), ".") {
return filepath.SkipDir
}
return nil
}
if !strings.HasSuffix(d.Name(), ".sy") {
return nil
}
p := strings.TrimPrefix(path, unzipRootPath)
p = filepath.ToSlash(p)
treePaths = append(treePaths, p)
return nil
})
if err = filelock.Copy(unzipRootPath, targetDir); err != nil {
logging.LogErrorf("copy data dir from [%s] to [%s] failed: %s", unzipRootPath, util.DataDir, err)
err = errors.New("copy data failed")
return
}
boxAbsPath := filepath.Join(util.DataDir, boxID)
importedAvIDs := map[string]struct{}{}
for _, importedAvID := range avIDs {
importedAvIDs[importedAvID] = struct{}{}
}
for _, treePath := range treePaths {
absPath := filepath.Join(targetDir, treePath)
p := strings.TrimPrefix(absPath, boxAbsPath)
p = filepath.ToSlash(p)
cache.RemoveTreeDataInBox(util.GetTreeID(p), boxID)
cache.RemoveDocIALInBox(p, boxID)
tree, err := filesys.LoadTree(boxID, p, luteEngine)
if err != nil {
logging.LogErrorf("load tree [%s] failed: %s", treePath, err)
continue
}
treenode.IndexBlockTree(tree)
cache.PutDocIALInBox(tree.Path, tree.Box, parse.IAL2Map(tree.Root.KramdownIAL))
var avNodes []*ast.Node
for _, avNode := range tree.Root.ChildrenByType(ast.NodeAttributeView) {
if _, ok := importedAvIDs[avNode.AttributeViewID]; ok {
avNodes = append(avNodes, avNode)
}
}
av.BatchUpsertBlockRel(avNodes)
sql.IndexTreeQueue(tree)
util.PushEndlessProgress(Conf.language(73) + " " + fmt.Sprintf(Conf.language(70), treenode.ReadDocumentTitle(tree.Root).Text))
}
if sortErr = applyImportedSYSort(boxID, baseTargetPath, importedSortDocs, sourceSortIDs); nil != sortErr {
logging.LogErrorf("apply imported document sort failed: %s", sortErr)
}
IncSync()
if !createNotebook {
pushDocsImported(boxID, baseTargetPath, "sy", importedSYRootIDs(importedSortDocs))
}
task.AppendTask(task.UpdateIDs, util.PushUpdateIDs, blockIDs)
return
}
// importSYAssets 将包内资源写入目标存储,并建立包内路径到目标路径的精确映射。
// 普通笔记本也保留恒等映射,用于清除旧版加密笔记本导出包中残留的 box 查询参数。
func importSYAssets(unzipRootPath, boxID string) (assetPathMap map[string]string, err error) {
assetPathMap = map[string]string{}
var assetsDirs []string
if err = filelock.Walk(unzipRootPath, func(currentPath string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d == nil || unzipRootPath == currentPath {
return nil
}
if d.Name() == "assets" && d.IsDir() {
assetsDirs = append(assetsDirs, currentPath)
return filepath.SkipDir
}
return nil
}); err != nil {
return nil, err
}
if IsEncryptedBox(boxID) {
boxAssetsDir := filepath.Join(util.DataDir, boxID, "assets")
if err = os.MkdirAll(boxAssetsDir, 0755); err != nil {
return nil, err
}
for _, assetsDir := range assetsDirs {
var assetFiles []string
if err = filelock.Walk(assetsDir, func(currentPath string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d != nil && !d.IsDir() && !strings.HasSuffix(strings.ToLower(d.Name()), ".sya") {
assetFiles = append(assetFiles, currentPath)
}
return nil
}); err != nil {
return nil, err
}
sort.Strings(assetFiles)
for _, assetFile := range assetFiles {
relPath, relErr := filepath.Rel(assetsDir, assetFile)
if relErr != nil {
return nil, relErr
}
sourceAssetPath := canonicalAssetReferencePath(path.Join("assets", filepath.ToSlash(relPath)))
data, readErr := filelock.ReadFile(assetFile)
if readErr != nil {
return nil, readErr
}
diskName, storeErr := storeAssetForBox(boxID, boxAssetsDir, filepath.Base(assetFile), data)
if storeErr != nil {
return nil, storeErr
}
assetPathMap[sourceAssetPath] = path.Join("assets", diskName)
annotationPath := assetFile + ".sya"
if filelock.IsExist(annotationPath) {
annotationData, annotationReadErr := filelock.ReadFile(annotationPath)
if annotationReadErr != nil {
return nil, annotationReadErr
}
annotationTargetPath := filepath.Join(boxAssetsDir, diskName+".sya")
if writeErr := writeAssetFile(annotationTargetPath, bytes.NewReader(annotationData), boxID,
filepath.Base(annotationPath)); writeErr != nil {
return nil, writeErr
}
}
}
if removeErr := os.RemoveAll(assetsDir); removeErr != nil {
return nil, removeErr
}
}
return assetPathMap, nil
}
dataAssets := filepath.Join(util.DataDir, "assets")
for _, assetsDir := range assetsDirs {
if err = filelock.Walk(assetsDir, func(currentPath string, d fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d == nil || d.IsDir() || strings.HasSuffix(strings.ToLower(d.Name()), ".sya") {
return nil
}
relPath, relErr := filepath.Rel(assetsDir, currentPath)
if relErr != nil {
return relErr
}
assetPath := canonicalAssetReferencePath(path.Join("assets", filepath.ToSlash(relPath)))
assetPathMap[assetPath] = assetPath
return nil
}); err != nil {
return nil, err
}
if err = filelock.Copy(assetsDir, dataAssets); err != nil {
logging.LogErrorf("copy assets from [%s] to [%s] failed: %s", assetsDir, dataAssets, err)
return nil, err
}
if removeErr := os.RemoveAll(assetsDir); removeErr != nil {
return nil, removeErr
}
}
return assetPathMap, nil
}
// errImportedTreeBoxLocked 表示导入写盘因目标加密笔记本未解锁被拒绝,供调用方映射为"请先解锁"文案。
var errImportedTreeBoxLocked = errors.New("imported tree requires an unlocked encrypted notebook")
func writeImportedTree(boxID, syPath, newSyPath, relPath string, data []byte) error {
if IsEncryptedBox(boxID) {
HoldBoxReadLock(boxID)
defer ReleaseBoxReadLock(boxID)
dek, err := GetDEKIfUnlocked(boxID)
if err != nil {
return errImportedTreeBoxLocked
}
data, err = EncryptFile(boxID, relPath, dek, data)
if err != nil {
return err
}
}
if err := os.WriteFile(syPath, data, 0644); err != nil {
return err
}
return filelock.Rename(syPath, newSyPath)
}
func validateImportedNotebookIdentities(tmpDataPath string) ([]string, error) {
if err := checkDocumentAddressingCopy(tmpDataPath, ""); err != nil {
return nil, err
}
identities, err := inspectDataImportNotebooks(tmpDataPath)
if err != nil {
return nil, err
}
var encryptedBoxIDs []string
for boxID, crypt := range identities {
if _, statErr := os.Lstat(filepath.Join(util.DataDir, boxID)); statErr == nil && IsEncryptedBox(boxID) {
return nil, fmt.Errorf("refuse to overwrite existing encrypted notebook [%s]", boxID)
}
if crypt != nil {
if _, statErr := os.Lstat(filepath.Join(util.DataDir, boxID)); statErr == nil {
return nil, fmt.Errorf(Conf.Language(423), boxID)
}
encryptedBoxIDs = append(encryptedBoxIDs, boxID)
}
}
return encryptedBoxIDs, nil
}
func ImportData(zipPath string) (err error) {
util.PushEndlessProgress(Conf.Language(73))
defer util.ClearPushProgress(100)
lockSync()
defer unlockSync()
logging.LogInfof("import data from [%s]", zipPath)
baseName := filepath.Base(zipPath)
ext := filepath.Ext(baseName)
baseName = strings.TrimSuffix(baseName, ext)
unzipPath := filepath.Join(filepath.Dir(zipPath), baseName)
err = gulu.Zip.Unzip(zipPath, unzipPath)
if err != nil {
return
}
defer os.RemoveAll(unzipPath)
files, err := filepath.Glob(filepath.Join(unzipPath, "*/*.sy"))
if err != nil {
logging.LogErrorf("check data.zip failed: %s", err)
return errors.New("check data.zip failed")
}
if 0 < len(files) {
return errors.New(Conf.Language(198))
}
dirs, err := os.ReadDir(unzipPath)
if err != nil {
logging.LogErrorf("check data.zip failed: %s", err)
return errors.New("check data.zip failed")
}
if 1 != len(dirs) {
return errors.New(Conf.Language(198))
}
tmpDataPath := filepath.Join(unzipPath, dirs[0].Name())
tmpDataEmojisPath := filepath.Join(tmpDataPath, "emojis")
filelock.Walk(tmpDataEmojisPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d == nil {
return nil
}
if !util.IsValidExistingEmojiFileName(d.Name()) {
emojiFullName := path
fullPathFilteredName := filepath.Join(filepath.Dir(path), util.FilterUploadEmojiFileName(d.Name()))
// XSS through emoji name https://github.com/siyuan-note/siyuan/issues/15034
logging.LogWarnf("renaming invalid custom emoji file [%s] to [%s]", d.Name(), fullPathFilteredName)
if removeErr := util.RenameEmojiFile(emojiFullName, fullPathFilteredName); nil != removeErr {
logging.LogErrorf("renaming invalid custom emoji file to [%s] failed: %s", fullPathFilteredName, removeErr)
}
}
return nil
})
importedEncryptedBoxIDs, err := validateImportedNotebookIdentities(tmpDataPath)
if err != nil {
return err
}
if err = filelock.Copy(tmpDataPath, util.DataDir); err != nil {
logging.LogErrorf("copy data dir from [%s] to [%s] failed: %s", tmpDataPath, util.DataDir, err)
err = errors.New("copy data failed")
return
}
for _, boxID := range importedEncryptedBoxIDs {
forgetRuntimeNormalBox(boxID)
}
// 导入的 Data.zip 可能含加密笔记本备份文件:若本机未启用,自动把配置装回 conf.json,
// 让用户输主密码即可解锁导入的加密笔记本(不需主密码,仅装回 salt/verifier 配置)。
restoreNotebookCryptoConfigFromBackup()
logging.LogInfof("import data from [%s] done", zipPath)
IncSync()
FullReindex(false)
return
}
func ImportFromLocalPath(boxID, localPath string, toPath string) (err error) {
return importFromLocalPath(boxID, localPath, toPath, false)
}
// ImportFromLocalPathSkipRoot 导入本地路径,但不将来源根目录转换为文档。
func ImportFromLocalPathSkipRoot(boxID, localPath string, toPath string) (err error) {
return importFromLocalPath(boxID, localPath, toPath, true)
}
// ValidateImportFromLocalPath 校验 Markdown 导入来源和目标,不写入导入数据。
func ValidateImportFromLocalPath(boxID, localPath, toPath string) error {
if _, err := getOpenedBox(boxID); err != nil {
return err
}
if _, err := os.Stat(localPath); err != nil {
return err
}
_, _, err := resolveMarkdownImportTarget(boxID, toPath)
return err
}
func resolveMarkdownImportTarget(boxID, toPath string) (baseHPath, baseTargetPath string, err error) {
toPath = path.Clean("/" + strings.TrimPrefix(toPath, "/"))
if toPath != "/" && !strings.HasSuffix(toPath, ".sy") {
toPath += ".sy"
}
toPath = normalizeBoxDocTarget(boxID, toPath)
if toPath == "/" {
return "/", "/", nil
}
block := treenode.GetBlockTreeRootByPath(boxID, toPath)
if block == nil || block.ID == "" {
return "", "", fmt.Errorf("target document not found in notebook %s: %s", boxID, toPath)
}
return block.HPath, strings.TrimSuffix(block.Path, ".sy"), nil
}
func importFromLocalPath(boxID, localPath string, toPath string, skipRoot bool) (err error) {
box, err := getOpenedBox(boxID)
if nil != err {
return err
}
util.PushEndlessProgress(Conf.Language(73))
defer func() {
util.PushClearProgress()
if e := recover(); nil != e {
stack := debug.Stack()
msg := fmt.Sprintf("PANIC RECOVERED: %v\n\t%s\n", e, stack)
logging.LogErrorf("import from local path failed: %s", msg)
err = errors.New("import from local path failed, please check kernel log for details")
}
}()
lockSync()
defer unlockSync()
FlushTxQueue()
baseHPath, baseTargetPath, err := resolveMarkdownImportTarget(boxID, toPath)
if err != nil {
return err
}
targetDocDirLocalPath := filepath.Join(util.DataDir, boxID,
filepath.FromSlash(strings.TrimPrefix(baseTargetPath, "/")))
assetDirPath := GetImportAssetsDir(boxID, targetDocDirLocalPath)
if err = os.MkdirAll(assetDirPath, 0755); err != nil {
return
}
spool := &markdownImportSpool{searchLinks: map[string]string{}}
if gulu.File.IsDir(localPath) {
spool, err = newMarkdownImportSpool()
if err != nil {
return err
}
}
defer spool.close()
stagedCount, stagedBytes := 0, 0
stageTree := func(tree *parse.Tree, sourcePath string) error {
if err := spool.addSource(tree, sourcePath); err != nil {
return err
}
stagedCount++
stagedBytes += spool.entries[len(spool.entries)-1].rawSize
if stagedCount >= markdownImportBatchDocuments || stagedBytes >= markdownImportBatchBytes {
debug.FreeOSMemory()
stagedCount, stagedBytes = 0, 0
}
return nil
}
hPathsIDs := map[string]string{}
idPaths := map[string]string{}
moveIDs := map[string]string{}
assetsDone := map[string]string{}
if gulu.File.IsDir(localPath) { // 导入文件夹
targetPaths := map[string]string{}
count := 0
// md 转换 sy
if err = filelock.Walk(localPath, func(currentPath string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d == nil {
return nil
}
if strings.HasPrefix(d.Name(), ".") {
if d.IsDir() {
return filepath.SkipDir
}
return nil
}
if !d.IsDir() && (!strings.HasSuffix(currentPath, ".md") && !strings.HasSuffix(currentPath, ".markdown") ||
strings.Contains(filepath.ToSlash(currentPath), "/assets/")) {
// 非 Markdown 文件作为资源文件处理 https://github.com/siyuan-note/siyuan/issues/13817
existName := assetsDone[currentPath]
var name string
if "" == existName {
baseName := filepath.Base(currentPath)
baseName = util.FilterUploadFileName(baseName)
data, readErr := os.ReadFile(currentPath)
if readErr != nil {
logging.LogErrorf("read asset [%s] failed: %s", currentPath, readErr)
return nil
}
name, err = storeAssetForBox(boxID, assetDirPath, baseName, data)
if err != nil {
logging.LogErrorf("store asset [%s] for box [%s] failed: %s", currentPath, boxID, err)
return nil
}
assetsDone[currentPath] = name
}
return nil
}
var tree *parse.Tree
var ext string
title := d.Name()
if !d.IsDir() {
ext = util.Ext(d.Name())
title = strings.TrimSuffix(d.Name(), ext)
}
id := ast.NewNodeID()
curRelPath := filepath.ToSlash(strings.TrimPrefix(currentPath, localPath))
if skipRoot && "" == curRelPath && d.IsDir() {
targetPaths["/"] = baseTargetPath
return nil
}
targetPath := path.Join(baseTargetPath, id)
hPathRoot := filepath.Base(localPath)
if skipRoot {
hPathRoot = ""
}
externalPath := strings.TrimSuffix(path.Join("/", hPathRoot, filepath.ToSlash(strings.TrimPrefix(currentPath, localPath))), ext)
hPath := appendDocumentHPath(baseHPath, strings.TrimPrefix(externalPath, "/"))
if "" == curRelPath {
curRelPath = "/"
hPath = appendDocumentHPath(baseHPath, title)
} else {
dirPath := targetPaths[path.Dir(curRelPath)]
targetPath = path.Join(dirPath, id)
}
targetPath = strings.ReplaceAll(targetPath, ".sy/", "/")
targetPath += ".sy"
if _, ok := targetPaths[curRelPath]; !ok {
targetPaths[curRelPath] = targetPath
} else {
targetPath = targetPaths[curRelPath]
id = util.GetTreeID(targetPath)
}
if d.IsDir() {
if "assets" == d.Name() {
// 如果是 assets 文件夹则跳过,里面的 Markdown 文件算作资源文件 https://github.com/siyuan-note/siyuan/issues/13817
return nil
}
if subMdFiles := util.GetFilePathsByExts(currentPath, []string{".md", ".markdown"}); 1 > len(subMdFiles) {
// 如果该文件夹中不包含 Markdown 文件则不处理 https://github.com/siyuan-note/siyuan/issues/11567
return nil
}
// 如果当前文件夹路径下包含同名的 Markdown 文件,则不创建空文档 https://github.com/siyuan-note/siyuan/issues/13149
for _, extension := range []string{".md", ".markdown"} {
if gulu.File.IsExist(currentPath + extension) {
targetPaths[curRelPath+extension] = targetPath
return nil
}
}
tree = treenode.NewTree(boxID, targetPath, hPath, title)
if err := stageTree(tree, externalPath); err != nil {
return err
}
return nil
}
if !strings.HasSuffix(d.Name(), ".md") && !strings.HasSuffix(d.Name(), ".markdown") {
return nil
}
data, readErr := os.ReadFile(currentPath)
if nil != readErr {
return readErr
}
tree, yfmRootID, yfmTitle, yfmUpdated := parseStdMd(data, true)
if nil == tree {
logging.LogErrorf("parse tree [%s] failed", currentPath)
return nil
}
if "" != yfmRootID {
moveIDs[id] = yfmRootID
id = yfmRootID
}
if "" != yfmTitle {
title = yfmTitle
}
parentDisplay := baseHPath
if curRelPath != "/" {
parentDisplay = appendDocumentHPath(baseHPath, strings.TrimPrefix(path.Dir(externalPath), "/"))
}
hPath = appendDocumentHPath(parentDisplay, title)
updated := yfmUpdated
fname := path.Base(targetPath)
targetPath = strings.ReplaceAll(targetPath, fname, id+".sy")
targetPaths[curRelPath] = targetPath
tree.ID = id
tree.Root.ID = id
tree.Root.SetIALAttr("id", tree.Root.ID)
tree.Root.SetIALAttr("title", title)
tree.Box = boxID
targetPath = path.Join(path.Dir(targetPath), tree.Root.ID+".sy")
tree.Path = targetPath
targetPaths[curRelPath] = targetPath
tree.HPath = hPath
tree.Root.Spec = treenode.BaseSpec
treenode.UpgradeSpec(tree)
currentDir := filepath.Dir(currentPath)
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || (ast.NodeLinkDest != n.Type && !n.IsTextMarkType("a")) {
return ast.WalkContinue
}
var dest string
if ast.NodeLinkDest == n.Type {
dest = n.TokensStr()
} else {
dest = n.TextMarkAHref
}
if strings.HasPrefix(dest, "data:image") && strings.Contains(dest, ";base64,") {
processBase64Img(n, dest, assetDirPath, boxID)
return ast.WalkContinue
}
decodedDest := string(html.DecodeDestination([]byte(dest)))
if decodedDest != dest {
dest = decodedDest
}
absolutePath := filepath.Join(currentDir, dest)
if !gulu.File.IsSubPath(currentDir, absolutePath) {
return ast.WalkContinue
}
if ast.NodeLinkDest == n.Type {
n.Tokens = []byte(dest)
} else {
n.TextMarkAHref = dest
}
if !util.IsRelativePath(dest) {
return ast.WalkContinue
}
dest = filepath.ToSlash(dest)
if "" == dest {
return ast.WalkContinue
}
if !gulu.File.IsExist(absolutePath) {
return ast.WalkContinue
}
if strings.HasSuffix(absolutePath, ".md") || strings.HasSuffix(absolutePath, ".markdown") {
if !strings.Contains(filepath.ToSlash(absolutePath), "/assets/") {
// 链接 .md 文件的情况下只有路径中包含 assets 才算作资源文件,其他情况算作文档链接,后续在 convertMdHyperlinks2WikiLinks 中处理
// Supports converting relative path hyperlinks into document block references after importing Markdown https://github.com/siyuan-note/siyuan/issues/13817
return ast.WalkContinue
}
}
existName := assetsDone[absolutePath]
var name string
if "" == existName {
baseName := filepath.Base(absolutePath)
data, readErr := filelock.ReadFile(absolutePath)
if readErr != nil {
logging.LogErrorf("read asset [%s] failed: %s", absolutePath, readErr)
return ast.WalkContinue
}
name, err = storeAssetForBox(boxID, assetDirPath, util.FilterUploadFileName(baseName), data)
if err != nil {
logging.LogErrorf("store asset [%s] for box [%s] failed: %s", absolutePath, boxID, err)
return ast.WalkContinue
}
assetsDone[absolutePath] = name
} else {
name = existName
}
if ast.NodeLinkDest == n.Type {
assetURL := "assets/" + name
if IsEncryptedBox(boxID) {
assetURL += "?box=" + boxID
}
n.Tokens = []byte(assetURL)
} else {
assetURL := "assets/" + name
if IsEncryptedBox(boxID) {
assetURL += "?box=" + boxID
}
n.TextMarkAHref = assetURL
}
return ast.WalkContinue
})
reassignIDUpdated(tree, id, updated)
if err := stageTree(tree, externalPath); err != nil {
return err
}
hPathsIDs[externalPath] = tree.ID
idPaths[tree.ID] = tree.Path
count++
if 0 == count%4 {
util.PushEndlessProgress(fmt.Sprintf(Conf.language(70), fmt.Sprintf("%s", tree.HPath)))
}
return nil
}); err != nil {
return err
}
} else { // 导入单个文件
fileName := filepath.Base(localPath)
if !strings.HasSuffix(fileName, ".md") && !strings.HasSuffix(fileName, ".markdown") {
return errors.New(Conf.Language(79))
}
title := strings.TrimSuffix(fileName, ".markdown")
title = strings.TrimSuffix(title, ".md")
targetPath := strings.TrimSuffix(toPath, ".sy")
id := ast.NewNodeID()
targetPath = path.Join(targetPath, id+".sy")
var data []byte
data, err = os.ReadFile(localPath)
if err != nil {
return err
}
tree, yfmRootID, yfmTitle, yfmUpdated := parseStdMd(data, true)
if nil == tree {
msg := fmt.Sprintf("parse tree [%s] failed", localPath)
logging.LogError(msg)
return errors.New(msg)
}
if "" != yfmRootID {
id = yfmRootID
}
if "" != yfmTitle {
title = yfmTitle
}
updated := yfmUpdated
fname := path.Base(targetPath)
targetPath = strings.ReplaceAll(targetPath, fname, id+".sy")
tree.ID = id
tree.Root.ID = id
tree.Root.SetIALAttr("id", tree.Root.ID)
tree.Root.SetIALAttr("title", title)
tree.Box = boxID
tree.Path = targetPath
tree.HPath = appendDocumentHPath(baseHPath, title)
tree.Root.Spec = treenode.BaseSpec
treenode.UpgradeSpec(tree)
localPathParentDir := filepath.Dir(localPath)
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || (ast.NodeLinkDest != n.Type && !n.IsTextMarkType("a")) {
return ast.WalkContinue
}
var dest string
if ast.NodeLinkDest == n.Type {
dest = n.TokensStr()
} else {
dest = n.TextMarkAHref
}
if strings.HasPrefix(dest, "data:image") && strings.Contains(dest, ";base64,") {
processBase64Img(n, dest, assetDirPath, boxID)
return ast.WalkContinue
}
decodedDest := string(html.DecodeDestination([]byte(dest)))
if decodedDest != dest {
dest = decodedDest
}
absolutePath := filepath.Join(localPathParentDir, dest)
if !gulu.File.IsSubPath(localPathParentDir, absolutePath) {
return ast.WalkContinue
}
if ast.NodeLinkDest == n.Type {
n.Tokens = []byte(dest)
} else {
n.TextMarkAHref = dest
}
if !util.IsRelativePath(dest) {
return ast.WalkContinue
}
dest = filepath.ToSlash(dest)
if "" == dest {
return ast.WalkContinue
}
if !gulu.File.IsExist(absolutePath) {
return ast.WalkContinue
}
existName := assetsDone[absolutePath]
var name string
if "" == existName {
baseName := filepath.Base(absolutePath)
data, readErr := filelock.ReadFile(absolutePath)
if readErr != nil {
logging.LogErrorf("read asset [%s] failed: %s", absolutePath, readErr)
return ast.WalkContinue
}
name, err = storeAssetForBox(boxID, assetDirPath, util.FilterUploadFileName(baseName), data)
if err != nil {
logging.LogErrorf("store asset [%s] for box [%s] failed: %s", absolutePath, boxID, err)
return ast.WalkContinue
}
assetsDone[absolutePath] = name
} else {
name = existName
}
if ast.NodeLinkDest == n.Type {
assetURL := "assets/" + name
if IsEncryptedBox(boxID) {
assetURL += "?box=" + boxID
}
n.Tokens = []byte(assetURL)
} else {
assetURL := "assets/" + name
if IsEncryptedBox(boxID) {
assetURL += "?box=" + boxID
}
n.TextMarkAHref = assetURL
}
return ast.WalkContinue
})
reassignIDUpdated(tree, id, updated)
// 兜底校验:禁止跨加密边界块引(导入的 Markdown 可能含跨边界引用)
degradeCrossBoundaryBlockRefs(tree.Root, tree.Box)
if err := stageTree(tree, "/"+strings.TrimSuffix(fileName, util.Ext(fileName))); err != nil {
return err
}
}
var importedRootIDs []string
if len(spool.entries) > 0 {
spool.movePaths(moveIDs)
if err = spool.refreshDisplayPaths(baseTargetPath, baseHPath); err != nil {
return err
}
for _, entry := range spool.entries {
if _, ok := idPaths[entry.id]; ok {
idPaths[entry.id] = entry.path
}
}
i := 0
if err = spool.finish(func(tree *parse.Tree) error {
if err := indexWriteTreeIndexQueue(tree); err != nil {
return err
}
if i%4 == 0 {
util.PushEndlessProgress(fmt.Sprintf(Conf.Language(66), fmt.Sprintf("%d/%d ", i, len(spool.entries))+tree.HPath))
}
i++
return nil
}, func() {
sql.FlushQueue()
debug.FreeOSMemory()
}); err != nil {
return err
}
refreshBoxDocInfoByBoxID(boxID)
util.PushClearProgress()
importedRootIDs = spool.rootIDs(baseTargetPath)
// 按照路径排序 Improve sort when importing markdown files https://github.com/siyuan-note/siyuan/issues/11390
var hPaths []string
for hPath := range hPathsIDs {
hPaths = append(hPaths, hPath)
}
sort.Strings(hPaths)
paths := map[string][]string{}
for _, hPath := range hPaths {
p := idPaths[hPathsIDs[hPath]]
parent := path.Dir(p)
for {
if baseTargetPath == parent {
break
}
if ps, ok := paths[parent]; !ok {
paths[parent] = []string{p}
} else {
ps = append(ps, p)
ps = gulu.Str.RemoveDuplicatedElem(ps)
paths[parent] = ps
}
p = parent
parent = path.Dir(parent)
}
}
sortIDVals := map[string]int{}
for _, ps := range paths {
sortVal := 0
for _, p := range ps {
sortIDVals[util.GetTreeID(p)] = sortVal
sortVal++
}
}
box.setImportSort(sortIDVals)
}
IncSync()
pushDocsImported(boxID, baseTargetPath, "markdown", importedRootIDs)
debug.FreeOSMemory()
return
}
func parseStdMd(markdown []byte, footnotes bool) (ret *parse.Tree, yfmRootID, yfmTitle, yfmUpdated string) {
luteEngine := util.NewStdLute()
setImportMathDelimiters(luteEngine)
luteEngine.SetFootnotes(footnotes)
luteEngine.SetYamlFrontMatter(true) // 解析 YAML Front Matter https://github.com/siyuan-note/siyuan/issues/10878
luteEngine.ParseOptions.PreserveTableCellSource = true
ret = parse.Parse("", markdown, luteEngine.ParseOptions)
if nil == ret {
return
}
if first := ret.Root.FirstChild; first != nil && first.Type == ast.NodeYamlFrontMatter {
attrs := map[string]any{}
if content := first.ChildByType(ast.NodeYamlFrontMatterContent); content != nil {
if err := yaml.Unmarshal(content.Tokens, &attrs); err != nil {
// 无效的元数据按普通 Markdown 解析,保留开头分隔线之间的正文。
luteEngine.SetYamlFrontMatter(false)
ret = parse.Parse("", markdown, luteEngine.ParseOptions)
}
}
}
lute.NormalizeMarkdownTableCells(ret)
if footnotes {
convertMarkdownFootnotes(ret)
}
parse.NormalizeInlineHTMLTextStyles(ret)
normalizeImportedHTMLTextStyles(ret)
yfmRootID, yfmTitle, yfmUpdated = normalizeTree(ret)
htmlBlock2Media(ret)
htmlBlock2Inline(ret)
parse.TextMarks2Inlines(ret) // 先将 TextMark 转换为 Inlines https://github.com/siyuan-note/siyuan/issues/13056
parse.NestedInlines2FlattedSpansHybrid(ret, false)
return
}
// htmlBlock2Media 将导入标准 Markdown 时被识别为 HTML 块的 <audio>/<video> 还原为音频/视频块。
// 导入 Markdown 使用的是 NewStdLute,未启用 ProtyleWYSIWYG,故 lute 不会把 <audio>/<video> 解析为
// NodeAudio/NodeVideo,而是兜底为 NodeHTMLBlock,这里在解析后做一次修正。
// Improve Markdown import to parse audio/video tags https://github.com/siyuan-note/siyuan/issues/17985
func htmlBlock2Media(tree *parse.Tree) {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || ast.NodeHTMLBlock != n.Type {
return ast.WalkContinue
}
tokens := bytes.TrimSpace(n.Tokens)
tokens, _ = bytes.CutPrefix(tokens, []byte("<div>"))
tokens, _ = bytes.CutSuffix(tokens, []byte("</div>"))
tokens = bytes.TrimSpace(tokens)
lower := bytes.ToLower(tokens)
if bytes.HasPrefix(lower, []byte("<audio")) && bytes.HasSuffix(tokens, []byte(">")) {
n.Type = ast.NodeAudio
n.Tokens = tokens
} else if bytes.HasPrefix(lower, []byte("<video")) && bytes.HasSuffix(tokens, []byte(">")) {
n.Type = ast.NodeVideo
n.Tokens = tokens
}
return ast.WalkContinue
})
}
func processHTMLBlockSvgImg(n *ast.Node, assetDirPath, boxID string) {
matches := importSVGPattern.FindStringSubmatch(string(n.Tokens))
if 1 >= len(matches) {
return
}
svgContent := matches[0]
name, err := storeAssetForBox(boxID, assetDirPath, "image.svg", []byte(svgContent))
if err != nil {
logging.LogErrorf("store svg asset for box [%s] failed: %s", boxID, err)
return
}
assetURL := "assets/" + name
if boxID != "" && IsEncryptedBox(boxID) {
assetURL += "?box=" + boxID
}
n.Type = ast.NodeParagraph
img := &ast.Node{Type: ast.NodeImage}
img.AppendChild(&ast.Node{Type: ast.NodeBang})
img.AppendChild(&ast.Node{Type: ast.NodeOpenBracket})
img.AppendChild(&ast.Node{Type: ast.NodeLinkText, Tokens: []byte("image")})
img.AppendChild(&ast.Node{Type: ast.NodeCloseBracket})
img.AppendChild(&ast.Node{Type: ast.NodeOpenParen})
img.AppendChild(&ast.Node{Type: ast.NodeLinkDest, Tokens: []byte(assetURL)})
img.AppendChild(&ast.Node{Type: ast.NodeCloseParen})
n.AppendChild(img)
}
func processBase64Img(n *ast.Node, dest string, assetDirPath, boxID string) {
sep := strings.Index(dest, ";base64,")
typ := strings.TrimSpace(dest[5:sep])
if paramSep := strings.IndexByte(typ, ';'); 0 <= paramSep {
typ = typ[:paramSep]
}
typ = strings.ToLower(typ)
str := strings.TrimSpace(dest[sep+8:])
str = importEncodedNewlinePattern.ReplaceAllString(str, "\n")
var decodeErr error
unbased, decodeErr := base64.StdEncoding.DecodeString(str)
if nil != decodeErr {
logging.LogErrorf("decode base64 image failed: declared type [%s], base64 length [%d]: %s", typ, len(str), decodeErr)
return
}
data := unbased
ext := ".svg"
if "image/svg+xml" != typ {
data, ext, decodeErr = normalizeBase64RasterImage(unbased)
if nil != decodeErr {
headerLen := min(len(unbased), 16)
logging.LogErrorf("decode base64 image failed: declared type [%s], data length [%d], header [%x]: %s",
typ, len(unbased), unbased[:headerLen], decodeErr)
return
}
}
name := "image" + ext
alt := n.Parent.ChildByType(ast.NodeLinkText)
if nil != alt {
name = alt.TokensStr() + ext
}
name = util.FilterUploadFileName(name)
diskName, err := storeAssetForBox(boxID, assetDirPath, name, data)
if err != nil {
logging.LogErrorf("store base64 image for box [%s] failed: %s", boxID, err)
return
}
assetURL := "assets/" + diskName
if boxID != "" && IsEncryptedBox(boxID) {
assetURL += "?box=" + boxID
}
n.Tokens = []byte(assetURL)
}
func normalizeBase64RasterImage(data []byte) (normalized []byte, ext string, err error) {
img, format, err := image.Decode(bytes.NewReader(data))
if nil != err {
return nil, "", err
}
var buf bytes.Buffer
switch format {
case "jpeg":
err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: 100})
ext = ".jpg"
default:
err = png.Encode(&buf, img)
ext = ".png"
}
if nil != err {
return nil, "", err
}
return buf.Bytes(), ext, nil
}
// encryptBoxAVFiles 把临时目录中的 AV 定义文件加密写入加密笔记本级目录。
// 写入后通过 SetAVBoxID 建立 AV 归属映射,确保后续 mirror/relation 操作正确路由。
func encryptBoxAVFiles(boxID, storageAvDir string) error {
if !IsEncryptedBox(boxID) || !gulu.File.IsExist(storageAvDir) {
return nil
}
boxAVDir := filepath.Join(util.DataDir, boxID, "storage", "av")
if err := os.MkdirAll(boxAVDir, 0755); err != nil {
return err
}
return filelock.Walk(storageAvDir, func(path string, d fs.DirEntry, walkErr error) error {
if walkErr != nil || d == nil || d.IsDir() {
return walkErr
}
if !strings.HasSuffix(d.Name(), ".json") {
return nil
}
src, readErr := os.ReadFile(path)
if readErr != nil {
return readErr
}
avID := strings.TrimSuffix(d.Name(), ".json")
// 加密写入前建立 box 映射,mirror/relation 操作即可正确路由
av.SetAVBoxID(avID, boxID)
enc, encErr := av.EncryptAVData(boxID, avID, src)
if encErr != nil {
return encErr
}
return os.WriteFile(filepath.Join(boxAVDir, d.Name()), enc, 0644)
})
}
func htmlBlock2Inline(tree *parse.Tree) {
imgHtmlBlocks := map[*ast.Node]*html.Node{}
aHtmlBlocks := map[*ast.Node]*html.Node{}
var unlinks []*ast.Node
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering {
return ast.WalkContinue
}
if ast.NodeHTMLBlock == n.Type || (ast.NodeText == n.Type && bytes.HasPrefix(bytes.ToLower(n.Tokens), []byte("<img "))) {
tokens := bytes.TrimSpace(n.Tokens)
tokens, _ = bytes.CutPrefix(tokens, []byte("<div>"))
tokens, _ = bytes.CutSuffix(tokens, []byte("</div>"))
tokens = bytes.TrimSpace(tokens)
htmlNodes, pErr := html.ParseFragment(bytes.NewReader(tokens), &html.Node{Type: html.ElementNode})
if nil != pErr {
logging.LogErrorf("parse html block [%s] failed: %s", n.Tokens, pErr)
return ast.WalkContinue
}
if 1 > len(htmlNodes) {
return ast.WalkContinue
}
for _, htmlNode := range htmlNodes {
if atom.Img == htmlNode.DataAtom {
imgHtmlBlocks[n] = htmlNode
break
}
}
}
if ast.NodeHTMLBlock == n.Type || (ast.NodeText == n.Type && bytes.HasPrefix(bytes.ToLower(n.Tokens), []byte("<a "))) {
tokens := bytes.TrimSpace(n.Tokens)
tokens, _ = bytes.CutPrefix(tokens, []byte("<div>"))
tokens, _ = bytes.CutSuffix(tokens, []byte("</div>"))
tokens = bytes.TrimSpace(tokens)
if ast.NodeHTMLBlock != n.Type && nil != n.Next && nil != n.Next.Next {
if ast.NodeText == n.Next.Next.Type && bytes.Equal(n.Next.Next.Tokens, []byte("</a>")) {
tokens = append(tokens, n.Next.Tokens...)
tokens = append(tokens, []byte("</a>")...)
unlinks = append(unlinks, n.Next)
unlinks = append(unlinks, n.Next.Next)
}
}
htmlNodes, pErr := html.ParseFragment(bytes.NewReader(tokens), &html.Node{Type: html.ElementNode})
if nil != pErr {
logging.LogErrorf("parse html block [%s] failed: %s", n.Tokens, pErr)
return ast.WalkContinue
}
if 1 > len(htmlNodes) {
return ast.WalkContinue
}
for _, htmlNode := range htmlNodes {
if atom.A == htmlNode.DataAtom {
aHtmlBlocks[n] = htmlNode
break
}
}
}
return ast.WalkContinue
})
for n, htmlImg := range imgHtmlBlocks {
src := domAttrValue(htmlImg, "src")
alt := domAttrValue(htmlImg, "alt")
title := domAttrValue(htmlImg, "title")
p := treenode.NewParagraph(n.ID)
img := &ast.Node{Type: ast.NodeImage}
p.AppendChild(img)
img.AppendChild(&ast.Node{Type: ast.NodeBang})
img.AppendChild(&ast.Node{Type: ast.NodeOpenBracket})
img.AppendChild(&ast.Node{Type: ast.NodeLinkText, Tokens: []byte(alt)})
img.AppendChild(&ast.Node{Type: ast.NodeCloseBracket})
img.AppendChild(&ast.Node{Type: ast.NodeOpenParen})
img.AppendChild(&ast.Node{Type: ast.NodeLinkDest, Tokens: []byte(src)})
if "" != title {
img.AppendChild(&ast.Node{Type: ast.NodeLinkSpace})
img.AppendChild(&ast.Node{Type: ast.NodeLinkTitle})
}
img.AppendChild(&ast.Node{Type: ast.NodeCloseParen})
if width := domAttrValue(htmlImg, "width"); "" != width {
if util2.IsDigit(width) {
width += "px"
}
style := "width: " + width + ";"
ial := &ast.Node{Type: ast.NodeKramdownSpanIAL, Tokens: parse.IAL2Tokens([][]string{{"style", style}})}
img.SetIALAttr("style", style)
img.InsertAfter(ial)
} else if height := domAttrValue(htmlImg, "height"); "" != height {
if util2.IsDigit(height) {
height += "px"
}
style := "height: " + height + ";"
ial := &ast.Node{Type: ast.NodeKramdownSpanIAL, Tokens: parse.IAL2Tokens([][]string{{"style", style}})}
img.SetIALAttr("style", style)
img.InsertAfter(ial)
}
if ast.NodeHTMLBlock == n.Type {
n.InsertBefore(p)
} else if ast.NodeText == n.Type {
if nil != n.Parent {
if n.Parent.IsContainerBlock() {
n.InsertBefore(p)
} else {
n.InsertBefore(img)
}
} else {
n.InsertBefore(p)
}
}
unlinks = append(unlinks, n)
}
for n, htmlA := range aHtmlBlocks {
href := domAttrValue(htmlA, "href")
title := domAttrValue(htmlA, "title")
anchor := strings.TrimSpace(util2.DomText(htmlA))
if "" == anchor {
unlinks = append(unlinks, n)
if nil != n.Next && ast.NodeText == n.Next.Type && "</a>" == n.NextNodeText() {
unlinks = append(unlinks, n.Next)
}
continue
}
p := treenode.NewParagraph(n.ID)
a := &ast.Node{Type: ast.NodeLink}
p.AppendChild(a)
a.AppendChild(&ast.Node{Type: ast.NodeOpenBracket})
a.AppendChild(&ast.Node{Type: ast.NodeLinkText, Tokens: []byte(anchor)})
a.AppendChild(&ast.Node{Type: ast.NodeCloseBracket})
a.AppendChild(&ast.Node{Type: ast.NodeOpenParen})
a.AppendChild(&ast.Node{Type: ast.NodeLinkDest, Tokens: []byte(href)})
if "" != title {
a.AppendChild(&ast.Node{Type: ast.NodeLinkSpace})
a.AppendChild(&ast.Node{Type: ast.NodeLinkTitle, Tokens: []byte(title)})
}
a.AppendChild(&ast.Node{Type: ast.NodeCloseParen})
if ast.NodeHTMLBlock == n.Type || (nil == n.Previous && (nil != n.Next && nil != n.Next.Next && nil == n.Next.Next.Next)) {
n.InsertBefore(p)
} else {
n.InsertBefore(a)
}
unlinks = append(unlinks, n)
}
for _, n := range unlinks {
n.Unlink()
}
}
func reassignIDUpdated(tree *parse.Tree, rootID, updated string) {
blockIDs := map[string]string{}
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || "" == n.ID {
return ast.WalkContinue
}
oldID := n.ID
n.ID = ast.NewNodeID()
if ast.NodeDocument == n.Type && "" != rootID {
n.ID = rootID
}
n.SetIALAttr("id", n.ID)
if "" != updated {
n.SetIALAttr("updated", updated)
if "" == rootID {
n.ID = updated + "-" + gulu.Rand.String(7)
n.SetIALAttr("id", n.ID)
}
} else {
n.SetIALAttr("updated", util.TimeFromID(n.ID))
}
blockIDs[oldID] = n.ID
return ast.WalkContinue
})
// 导入时为块分配新 ID,同时更新文档内脚注等块引用的目标。
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if entering && treenode.IsBlockRef(n) {
if id := blockIDs[n.TextMarkBlockRefID]; id != "" {
n.TextMarkBlockRefID = id
}
}
return ast.WalkContinue
})
tree.ID = tree.Root.ID
tree.Path = path.Join(path.Dir(tree.Path), tree.ID+".sy")
tree.Root.SetIALAttr("id", tree.Root.ID)
}
func domAttrValue(n *html.Node, attrName string) string {
if nil == n {
return ""
}
for _, attr := range n.Attr {
if attr.Key == attrName {
return attr.Val
}
}
return ""
}
func addImportSearchLinks(tree *parse.Tree, searchLinks map[string]string) {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || (ast.NodeDocument != n.Type && ast.NodeHeading != n.Type) {
return ast.WalkContinue
}
nodePath := tree.HPath + "#"
if ast.NodeHeading == n.Type {
nodePath += n.Text()
}
searchLinks[nodePath] = n.ID
return ast.WalkContinue
})
}
func convertMdHyperlinks2WikiLinks(tree *parse.Tree) {
// Supports converting relative path hyperlinks into document block references after importing Markdown https://github.com/siyuan-note/siyuan/issues/13817
var unlinks []*ast.Node
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || ast.NodeTextMark != n.Type {
return ast.WalkContinue
}
if "a" != n.TextMarkType {
return ast.WalkContinue
}
linkText := n.TextMarkTextContent
if "" == linkText {
return ast.WalkContinue
}
linkDest := n.TextMarkAHref
if "" == linkDest {
return ast.WalkContinue
}
if strings.HasPrefix(linkDest, "assets/") {
return ast.WalkContinue
}
linkPath, fragment, hasFragment := strings.Cut(linkDest, "#")
if !strings.HasSuffix(linkPath, ".md") && !strings.HasSuffix(linkPath, ".markdown") {
return ast.WalkContinue
}
linkDest = strings.TrimSuffix(strings.TrimSuffix(linkPath, ".md"), ".markdown")
if hasFragment {
linkDest += "#" + fragment
}
buf := bytes.Buffer{}
buf.WriteString("[[")
buf.WriteString(linkDest)
buf.WriteString("|")
buf.WriteString(linkText)
buf.WriteString("]]")
wikilinkNode := &ast.Node{Type: ast.NodeText, Tokens: buf.Bytes()}
n.InsertBefore(wikilinkNode)
unlinks = append(unlinks, n)
return ast.WalkContinue
})
for _, n := range unlinks {
n.Unlink()
}
}
func convertWikiLinksAndTags(tree *parse.Tree, searchLinks map[string]string) {
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || ast.NodeText != n.Type {
return ast.WalkContinue
}
text := n.TokensStr()
length := len(text)
start, end := 0, length
for {
part := text[start:end]
if idx := strings.Index(part, "]]"); 0 > idx {
break
} else {
end = start + idx
}
if idx := strings.Index(part, "[["); 0 > idx {
break
} else {
start += idx
}
if end <= start {
break
}
link := path.Join(path.Dir(tree.HPath), text[start+2:end]) // 统一转为绝对路径方便后续查找
linkText := path.Base(link)
dynamicAnchorText := true
if linkParts := strings.Split(link, "|"); 1 < len(linkParts) {
link = linkParts[0]
linkText = linkParts[1]
dynamicAnchorText = false
}
link, linkText = strings.TrimSpace(link), strings.TrimSpace(linkText)
if !strings.Contains(link, "#") {
link += "#" // 在结尾统一带上锚点方便后续查找
}
id := searchLinkID(link, searchLinks)
if "" == id {
start, end = end, length
continue
}
linkText = strings.TrimPrefix(linkText, "/")
repl := "((" + id + " '" + linkText + "'))"
if !dynamicAnchorText {
repl = "((" + id + " \"" + linkText + "\"))"
}
end += 2
text = text[:start] + repl + text[end:]
start, end = start+len(repl), len(text)
length = end
}
text = convertTags(text) // 导入标签语法
n.Tokens = gulu.Str.ToBytes(text)
return ast.WalkContinue
})
}
func convertTags(text string) (ret string) {
if !util.MarkdownSettings.Load().InlineTag {
return text
}
pos, i := -1, 0
tokens := []byte(text)
for ; i < len(tokens); i++ {
if '#' == tokens[i] && (0 == i || ' ' == tokens[i-1] || (-1 < pos && '#' == tokens[pos])) {
if i < len(tokens)-1 && '#' == tokens[i+1] {
pos = -1
continue
}
pos = i
continue
}
if -1 < pos && ' ' == tokens[i] {
tokens = append(tokens, 0)
copy(tokens[i+1:], tokens[i:])
tokens[i] = '#'
pos = -1
i++
}
}
if -1 < pos && pos < i {
tokens = append(tokens, '#')
}
return string(tokens)
}
func searchLinkID(link string, searchLinks map[string]string) (id string) {
id = searchLinks[link]
if "" != id {
return
}
baseName := path.Base(link)
for searchLink, searchID := range searchLinks {
if path.Base(searchLink) == baseName {
return searchID
}
}
return
}
func mergeTextAndHandlerNestedInlines(tree *parse.Tree, luteEngine *lute.Lute) {
tree.MergeText()
var unlinkTextNodes []*ast.Node
ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
if !entering || ast.NodeText != n.Type {
return ast.WalkContinue
}
if nil == n.Tokens {
return ast.WalkContinue
}
t := parse.Inline("", n.Tokens, luteEngine.ParseOptions) // 使用行级解析
parse.NestedInlines2FlattedSpans(t, false)
var children []*ast.Node
for c := t.Root.FirstChild.FirstChild; nil != c; c = c.Next {
children = append(children, c)
}
for _, c := range children {
n.InsertBefore(c)
}
unlinkTextNodes = append(unlinkTextNodes, n)
return ast.WalkContinue
})
for _, node := range unlinkTextNodes {
node.Unlink()
}
}