Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

28.6 KB
Raw Blame
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later

package model

import (
	"bytes"
	"fmt"
	"io"
	"io/fs"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"code.sajari.com/docconv"
	"github.com/88250/epub"
	"github.com/88250/go-humanize"
	"github.com/88250/gulu"
	"github.com/88250/lute/ast"
	"github.com/klippa-app/go-pdfium"
	"github.com/klippa-app/go-pdfium/requests"
	"github.com/klippa-app/go-pdfium/webassembly"
	"github.com/siyuan-note/eventbus"
	"github.com/siyuan-note/filelock"
	"github.com/siyuan-note/logging"
	"github.com/siyuan-note/siyuan/kernel/search"
	"github.com/siyuan-note/siyuan/kernel/sql"
	"github.com/siyuan-note/siyuan/kernel/task"
	"github.com/siyuan-note/siyuan/kernel/util"
	"github.com/xuri/excelize/v2"
)

type AssetContent struct {
	ID      string `json:"id"`
	Name    string `json:"name"`
	Ext     string `json:"ext"`
	Path    string `json:"path"`
	Size    int64  `json:"size"`
	HSize   string `json:"hSize"`
	Updated int64  `json:"updated"`
	Content string `json:"content"`
}

func GetAssetContent(id, query string, queryMethod int) (ret *AssetContent) {
	query = strings.TrimSpace(query)
	if !ast.IsNodeIDPattern(id) {
		return
	}

	table := "asset_contents_fts_case_insensitive"
	filter := "id = ?"
	args := []any{id}
	contentProjection := "content"
	if "" != query && (0 == queryMethod || 1 == queryMethod) {
		query = filterQueryInvisibleChars(query)
		if 0 == queryMethod {
			query = stringQuery(query)
		}
		filter += " AND `" + table + "` MATCH ?"
		args = append(args, buildAssetContentColumnFilter()+":("+query+")")
		contentProjection = "highlight(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "') AS content"
	}

	projections := "id, name, ext, path, size, updated, " + contentProjection
	stmt := "SELECT " + projections + " FROM " + table + " WHERE " + filter
	assetContents := sql.SelectAssetContentsRawStmtNoParseArgs(stmt, args, 1)
	if 3 == queryMethod && "" != query && 0 < len(assetContents) {
		// 与 SQL REGEXP 使用相同的转义和匹配语义,在转义 HTML 前标记原文。
		exp := strings.ReplaceAll(filterQueryInvisibleChars(query), "\\\\", "\\")
		if re, err := regexp.Compile(exp); err == nil {
			assetContents[0].Content = re.ReplaceAllStringFunc(assetContents[0].Content, func(match string) string {
				if "" == match {
					return match
				}
				return search.SearchMarkLeft + match + search.SearchMarkRight
			})
		}
	}
	results := fromSQLAssetContents(&assetContents)
	if 1 > len(results) {
		return
	}
	ret = results[0]
	ret.Content = strings.ReplaceAll(ret.Content, "\n", "<br>")
	return
}

// GetAssetContentByPath 按资源文件路径获取已索引的完整内容。
//
// path 为工作空间相对路径,形如 "assets/xxx.pdf"。返回 nil 表示该文件尚未被索引。
// 与 GetAssetContent(按索引记录 id)不同,此处直接返回 content 原文,不做高亮。
func GetAssetContentByPath(path string) (ret *AssetContent) {
	path = strings.TrimSpace(path)
	if "" == path {
		return
	}

	table := "asset_contents_fts_case_insensitive"
	stmt := "SELECT id, name, ext, path, size, updated, content FROM " + table + " WHERE path = ? LIMIT 1"
	assetContents := sql.SelectAssetContentsRawStmtNoParseArgs(stmt, []any{path}, 1)
	results := fromSQLAssetContents(&assetContents)
	if 1 > len(results) {
		return
	}
	ret = results[0]
	return
}

// FullTextSearchAssetContent 搜索资源文件内容。
//
// method:0:关键字,1:查询语法,2:SQL,3:正则表达式
// orderBy: 0:按相关度降序,1:按相关度升序,2:按更新时间升序,3:按更新时间降序
func FullTextSearchAssetContent(query string, types map[string]bool, method, orderBy, page, pageSize int) (ret []*AssetContent, matchedAssetCount, pageCount int, err error) {
	query = strings.TrimSpace(query)
	orderByClause := buildAssetContentOrderBy(orderBy)
	switch method {
	case 1: // 查询语法
		filter, filterArgs := buildAssetContentTypeFilter(types)
		ret, matchedAssetCount = fullTextSearchAssetContentByQuerySyntax(query, filter, filterArgs, orderByClause, page, pageSize)
	case 2: // SQL
		ret, matchedAssetCount, err = searchAssetContentBySQL(query, page, pageSize)
		if err != nil {
			return
		}
	case 3: // 正则表达式
		if 2 != orderBy && 3 != orderBy {
			orderByClause = "ORDER BY path ASC"
		}
		typeFilter, typeArgs := buildAssetContentTypeFilter(types)
		ret, matchedAssetCount = fullTextSearchAssetContentByRegexp(query, typeFilter, typeArgs, orderByClause, page, pageSize)
	default: // 关键字
		filter, filterArgs := buildAssetContentTypeFilter(types)
		ret, matchedAssetCount = fullTextSearchAssetContentByKeyword(query, filter, filterArgs, orderByClause, page, pageSize)
	}
	pageCount = (matchedAssetCount + pageSize - 1) / pageSize

	if 1 > len(ret) {
		ret = []*AssetContent{}
	}
	return
}

func fullTextSearchAssetContentByQuerySyntax(query, typeFilter string, typeArgs []any, orderBy string, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) {
	query = filterQueryInvisibleChars(query)
	return fullTextSearchAssetContentByFTS(query, typeFilter, typeArgs, orderBy, page, pageSize)
}

func fullTextSearchAssetContentByKeyword(query, typeFilter string, typeArgs []any, orderBy string, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) {
	query = filterQueryInvisibleChars(query)
	query = stringQuery(query)
	return fullTextSearchAssetContentByFTS(query, typeFilter, typeArgs, orderBy, page, pageSize)
}

func fullTextSearchAssetContentByRegexp(exp, typeFilter string, typeArgs []any, orderBy string, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) {
	exp = filterQueryInvisibleChars(exp)
	fieldFilter, args := assetContentFieldRegexp(exp)
	args = append(args, typeArgs...)
	stmt := "SELECT * FROM `asset_contents_fts_case_insensitive` WHERE " + fieldFilter + typeFilter
	stmt += " " + orderBy
	stmt += " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize)
	assetContents := sql.SelectAssetContentsRawStmtNoParseArgs(stmt, args, Conf.Search.Load().Limit)
	ret = fromSQLAssetContents(&assetContents)
	if 1 > len(ret) {
		ret = []*AssetContent{}
	}

	matchedAssetCount = fullTextSearchAssetContentCountByRegexp(exp, typeFilter, typeArgs)
	return
}

func assetContentFieldRegexp(exp string) (clause string, args []any) {
	clause = "(name REGEXP ? OR content REGEXP ?)"
	args = []any{exp, exp}
	return
}

func fullTextSearchAssetContentCountByRegexp(exp, typeFilter string, typeArgs []any) (matchedAssetCount int) {
	table := "asset_contents_fts_case_insensitive"
	fieldFilter, args := assetContentFieldRegexp(exp)
	args = append(args, typeArgs...)
	stmt := "SELECT COUNT(path) AS `assets` FROM `" + table + "` WHERE " + fieldFilter + typeFilter
	result, _ := sql.QueryAssetContentNoLimitArgs(stmt, args...)
	if 1 > len(result) {
		return
	}
	matchedAssetCount = int(result[0]["assets"].(int64))
	return
}

func fullTextSearchAssetContentByFTS(query, typeFilter string, typeArgs []any, orderBy string, page, pageSize int) (ret []*AssetContent, matchedAssetCount int) {
	table := "asset_contents_fts_case_insensitive"
	projections := "id, name, ext, path, size, updated, " +
		"snippet(" + table + ", 6, '" + search.SearchMarkLeft + "', '" + search.SearchMarkRight + "', '...', 64) AS content"
	stmt := "SELECT " + projections + " FROM " + table + " WHERE `" + table + "` MATCH ?" + typeFilter
	stmt += " " + orderBy
	stmt += " LIMIT " + strconv.Itoa(pageSize) + " OFFSET " + strconv.Itoa((page-1)*pageSize)
	args := []any{buildAssetContentColumnFilter() + ":(" + query + ")"}
	args = append(args, typeArgs...)
	assetContents := sql.SelectAssetContentsRawStmtNoParseArgs(stmt, args, Conf.Search.Load().Limit)
	ret = fromSQLAssetContents(&assetContents)
	if 1 > len(ret) {
		ret = []*AssetContent{}
	}

	matchedAssetCount = fullTextSearchAssetContentCount(query, typeFilter, typeArgs)
	return
}

func searchAssetContentBySQL(stmt string, page, pageSize int) (ret []*AssetContent, matchedAssetCount int, err error) {
	stmt = filterQueryInvisibleChars(stmt)
	stmt = strings.TrimSpace(stmt)
	if err = sql.CheckSingleStatement(stmt); err != nil {
		return
	}
	if err = sql.CheckAssetContentReadonlyStatement(stmt); err != nil {
		return
	}
	assetContents := sql.SelectAssetContentsRawStmt(stmt, page, pageSize)
	ret = fromSQLAssetContents(&assetContents)
	if 1 > len(ret) {
		ret = []*AssetContent{}
		return
	}

	stmt = strings.ToLower(stmt)
	stmt = strings.ReplaceAll(stmt, "select * ", "select COUNT(path) AS `assets` ")
	stmt = removeLimitClause(stmt)
	result, _ := sql.QueryAssetContentNoLimit(stmt)
	if 1 > len(result) {
		return
	}

	if assets, ok := result[0]["assets"].(int64); ok {
		matchedAssetCount = int(assets)
	}
	return
}

func fullTextSearchAssetContentCount(query, typeFilter string, typeArgs []any) (matchedAssetCount int) {
	query = filterQueryInvisibleChars(query)

	table := "asset_contents_fts_case_insensitive"
	stmt := "SELECT COUNT(path) AS `assets` FROM `" + table + "` WHERE `" + table + "` MATCH ?" + typeFilter
	args := []any{buildAssetContentColumnFilter() + ":(" + query + ")"}
	args = append(args, typeArgs...)
	result, _ := sql.QueryAssetContentNoLimitArgs(stmt, args...)
	if 1 > len(result) {
		return
	}
	matchedAssetCount = int(result[0]["assets"].(int64))
	return
}

func fromSQLAssetContents(assetContents *[]*sql.AssetContent) (ret []*AssetContent) {
	ret = []*AssetContent{}
	for _, assetContent := range *assetContents {
		ret = append(ret, fromSQLAssetContent(assetContent))
	}
	return
}

func fromSQLAssetContent(assetContent *sql.AssetContent) *AssetContent {
	content := util.EscapeHTML(assetContent.Content)
	if strings.Contains(content, search.SearchMarkLeft) {
		content = strings.ReplaceAll(content, search.SearchMarkLeft, "<mark>")
		content = strings.ReplaceAll(content, search.SearchMarkRight, "</mark>")
	}

	return &AssetContent{
		ID:      assetContent.ID,
		Name:    assetContent.Name,
		Ext:     assetContent.Ext,
		Path:    assetContent.Path,
		Size:    assetContent.Size,
		HSize:   humanize.BytesCustomCeil(uint64(assetContent.Size), 2),
		Updated: assetContent.Updated,
		Content: content,
	}
}

func buildAssetContentColumnFilter() string {
	return "{name content}"
}

func buildAssetContentTypeFilter(types map[string]bool) (clause string, args []any) {
	if 0 == len(types) {
		return
	}

	var enabledTypes []string
	for k, enabled := range types {
		if enabled {
			enabledTypes = append(enabledTypes, k)
		}
	}
	if 0 == len(enabledTypes) {
		clause = " AND 1 = 0"
		return
	}

	sort.Strings(enabledTypes)
	placeholders := strings.TrimSuffix(strings.Repeat("?, ", len(enabledTypes)), ", ")
	clause = " AND ext IN (" + placeholders + ")"
	for _, assetType := range enabledTypes {
		args = append(args, assetType)
	}
	return
}

func buildAssetContentOrderBy(orderBy int) string {
	switch orderBy {
	case 0:
		return "ORDER BY rank ASC"
	case 1:
		return "ORDER BY rank DESC"
	case 2:
		return "ORDER BY updated ASC"
	case 3:
		return "ORDER BY updated DESC"
	default:
		return "ORDER BY rank ASC"
	}
}

var assetContentSearcher = NewAssetsSearcher()

func removeIndexAssetContent(absPath string) {
	defer logging.Recover()

	assetsDir := util.GetDataAssetsAbsPath()
	p := "assets" + filepath.ToSlash(strings.TrimPrefix(absPath, assetsDir))
	sql.DeleteAssetContentsByPathQueue(p)
}

func indexAssetContent(absPath string) {
	defer logging.Recover()

	ext := filepath.Ext(absPath)
	parser := assetContentSearcher.GetParser(ext)
	if nil == parser {
		return
	}
	indexed := false
	defer func() {
		if !indexed {
			removeIndexAssetContent(absPath)
		}
	}()

	result := parser.Parse(absPath)
	if nil == result {
		return
	}

	info, err := os.Stat(absPath)
	if err != nil {
		logging.LogErrorf("stat [%s] failed: %s", absPath, err)
		return
	}

	assetsDir := util.GetDataAssetsAbsPath()
	p := "assets" + filepath.ToSlash(strings.TrimPrefix(absPath, assetsDir))

	assetContents := []*sql.AssetContent{
		{
			ID:      ast.NewNodeID(),
			Name:    util.RemoveID(filepath.Base(p)),
			Ext:     ext,
			Path:    p,
			Size:    info.Size(),
			Updated: info.ModTime().Unix(),
			Content: result.Content,
		},
	}

	sql.DeleteAssetContentsByPathQueue(p)
	sql.IndexAssetContentsQueue(assetContents)
	indexed = true
}

func ReindexAssetContent() {
	task.AppendTask(task.AssetContentDatabaseIndexFull, fullReindexAssetContent)
}

func fullReindexAssetContent() {
	defer func() {
		if recovered := recover(); recovered != nil {
			logging.LogErrorf("asset content reindex failed: %v", recovered)
			util.PushErrMsg(Conf.Language(443), 7000)
		}
	}()
	util.PushMsg(Conf.Language(216), 7*1000)
	err := sql.InitAssetContentDatabase(true)
	if err == nil {
		err = assetContentSearcher.FullIndex()
	}
	if err != nil {
		logging.LogErrorf("asset content reindex failed: %s", err)
		util.PushErrMsg(Conf.Language(443), 7000)
		return
	}
	util.PushMsg(Conf.Language(442), 5000)
	util.BroadcastByType("main", "assetContentIndexCompleted", 0, "", nil)
}

func init() {
	subscribeSQLAssetContentEvents()
}

func subscribeSQLAssetContentEvents() {
	eventbus.Subscribe(util.EvtSQLAssetContentRebuild, func() {
		ReindexAssetContent()
	})
}

type AssetsSearcher struct {
	parsers map[string]AssetParser
	lock    *sync.Mutex
}

func (searcher *AssetsSearcher) GetParser(ext string) AssetParser {
	searcher.lock.Lock()
	defer searcher.lock.Unlock()

	return searcher.parsers[strings.ToLower(ext)]
}

func (searcher *AssetsSearcher) FullIndex() error {
	return searcher.fullIndex(func(batch []*sql.AssetContent) error {
		sql.IndexAssetContentsQueue(batch)
		// 等待本批写入完成,避免解析结果继续在索引队列中累积。
		return sql.FlushAssetContentQueue()
	})
}

func (searcher *AssetsSearcher) fullIndex(writeBatch func([]*sql.AssetContent) error) (err error) {
	defer func() {
		if recovered := recover(); recovered != nil {
			err = fmt.Errorf("asset content indexing failed: %v", recovered)
		}
	}()

	assetsDir := util.GetDataAssetsAbsPath()
	if info, statErr := os.Stat(assetsDir); os.IsNotExist(statErr) {
		return nil
	} else if statErr != nil {
		return statErr
	} else if !info.IsDir() {
		return fmt.Errorf("asset directory is not a directory")
	}
	started := time.Now()
	scanned, parsed := 0, 0
	logging.LogInfof("start asset content full indexing")

	const maxBatchCount, maxBatchBytes = 128, 4 * 1024 * 1024
	var batch []*sql.AssetContent
	batchBytes := 0
	flush := func() error {
		if len(batch) == 0 {
			return nil
		}
		if err := writeBatch(batch); err != nil {
			return err
		}
		batch, batchBytes = nil, 0
		return nil
	}
	walkErr := filelock.Walk(assetsDir, func(absPath string, d fs.DirEntry, err error) error {
		if err != nil {
			logging.LogErrorf("walk dir [%s] failed: %s", absPath, err)
			return err
		}

		if d.IsDir() {
			return nil
		}
		scanned++

		// 加密笔记本的 asset 是密文,跳过全量内容索引(避免密文污染搜索索引、泄漏文件名集合)
		if IsEncryptedAssetPath(absPath) {
			return nil
		}

		ext := filepath.Ext(absPath)
		parser := searcher.GetParser(ext)
		if nil == parser {
			return nil
		}

		logging.LogDebugf("parsing asset content [%s]", absPath)

		result := parser.Parse(absPath)
		if nil == result {
			return nil
		}

		info, err := d.Info()
		if err != nil {
			logging.LogErrorf("stat file [%s] failed: %s", absPath, err)
			return err
		}

		result.Path = "assets" + filepath.ToSlash(strings.TrimPrefix(absPath, assetsDir))
		result.Size = info.Size()
		result.Updated = info.ModTime().Unix()
		parsed++
		if batchBytes+len(result.Content) > maxBatchBytes {
			if err := flush(); err != nil {
				return err
			}
		}
		batch = append(batch, &sql.AssetContent{
			ID:      ast.NewNodeID(),
			Name:    util.RemoveID(filepath.Base(result.Path)),
			Ext:     strings.ToLower(filepath.Ext(result.Path)),
			Path:    result.Path,
			Size:    result.Size,
			Updated: result.Updated,
			Content: result.Content,
		})
		batchBytes += len(result.Content)
		if len(batch) >= maxBatchCount || batchBytes >= maxBatchBytes {
			return flush()
		}
		return nil
	})
	if walkErr == nil {
		walkErr = flush()
	}
	logging.LogInfof("asset content full indexing finished: files=%d, parsed=%d, skipped=%d, elapsed=%s, error=%v",
		scanned, parsed, scanned-parsed, time.Since(started), walkErr)
	return walkErr
}

func NewAssetsSearcher() *AssetsSearcher {
	txtAssetParser := &TxtAssetParser{}
	return &AssetsSearcher{
		parsers: map[string]AssetParser{
			".txt":      txtAssetParser,
			".md":       txtAssetParser,
			".markdown": txtAssetParser,
			".json":     txtAssetParser,
			".log":      txtAssetParser,
			".sql":      txtAssetParser,
			".html":     txtAssetParser,
			".xml":      txtAssetParser,
			".java":     txtAssetParser,
			".h":        txtAssetParser,
			".c":        txtAssetParser,
			".cpp":      txtAssetParser,
			".go":       txtAssetParser,
			".rs":       txtAssetParser,
			".swift":    txtAssetParser,
			".kt":       txtAssetParser,
			".py":       txtAssetParser,
			".php":      txtAssetParser,
			".js":       txtAssetParser,
			".css":      txtAssetParser,
			".ts":       txtAssetParser,
			".sh":       txtAssetParser,
			".bat":      txtAssetParser,
			".cmd":      txtAssetParser,
			".ini":      txtAssetParser,
			".yaml":     txtAssetParser,
			".rst":      txtAssetParser,
			".adoc":     txtAssetParser,
			".textile":  txtAssetParser,
			".opml":     txtAssetParser,
			".org":      txtAssetParser,
			".wiki":     txtAssetParser,
			".cs":       txtAssetParser,
			".docx":     &DocxAssetParser{},
			".pptx":     &PptxAssetParser{},
			".xlsx":     &XlsxAssetParser{},
			".pdf":      &PdfAssetParser{},
			".epub":     &EpubAssetParser{},
		},

		lock: &sync.Mutex{},
	}
}

const (
	TxtAssetContentMaxSize = 1024 * 1024 * 4
	PDFAssetContentMaxPage = 1024
)

var (
	PDFAssetContentMaxSize uint64 = 1024 * 1024 * 128
)

type AssetParseResult struct {
	Path    string
	Size    int64
	Updated int64
	Content string
}

type AssetParser interface {
	Parse(absPath string) *AssetParseResult
}

type TxtAssetParser struct {
}

func (parser *TxtAssetParser) Parse(absPath string) (ret *AssetParseResult) {
	info, err := os.Stat(absPath)
	if err != nil {
		logging.LogErrorf("stat file [%s] failed: %s", absPath, err)
		return
	}

	if TxtAssetContentMaxSize < info.Size() {
		logging.LogWarnf("text asset [%s] is too large [%s]", absPath, humanize.BytesCustomCeil(uint64(info.Size()), 2))
		return
	}

	tmp := copyTempAsset(absPath)
	if "" == tmp {
		return
	}
	defer os.RemoveAll(tmp)

	data, err := os.ReadFile(tmp)
	if err != nil {
		logging.LogErrorf("read file [%s] failed: %s", absPath, err)
		return
	}

	if !utf8.Valid(data) {
		// Non-UTF-8 encoded text files are not included in asset file content searching https://github.com/siyuan-note/siyuan/issues/9052
		logging.LogWarnf("text asset [%s] is not UTF-8 encoded", absPath)
		return
	}

	content := string(data)
	ret = &AssetParseResult{
		Content: content,
	}
	return
}

func normalizeNonTxtAssetContent(content string) (ret string) {
	ret = strings.Join(strings.Fields(content), " ")
	return
}

func copyTempAsset(absPath string) (ret string) {
	dir := filepath.Join(util.TempDir, "convert", "asset_content")
	if err := os.MkdirAll(dir, 0755); err != nil {
		logging.LogErrorf("mkdir [%s] failed: [%s]", dir, err)
		return
	}

	baseName := filepath.Base(absPath)
	if strings.HasPrefix(baseName, "~") {
		return
	}

	filelock.Lock(absPath)
	defer filelock.Unlock(absPath)

	ext := filepath.Ext(absPath)
	ret = filepath.Join(dir, gulu.Rand.String(7)+ext)
	if err := gulu.File.Copy(absPath, ret); err != nil {
		logging.LogErrorf("copy [src=%s, dest=%s] failed: %s", absPath, ret, err)
		return
	}
	return
}

type DocxAssetParser struct {
}

func (parser *DocxAssetParser) Parse(absPath string) (ret *AssetParseResult) {
	if !strings.HasSuffix(strings.ToLower(absPath), ".docx") {
		return
	}

	if !gulu.File.IsExist(absPath) {
		return
	}

	tmp := copyTempAsset(absPath)
	if "" == tmp {
		return
	}
	defer os.RemoveAll(tmp)

	f, err := os.Open(tmp)
	if err != nil {
		logging.LogErrorf("open [%s] failed: [%s]", tmp, err)
		return
	}
	defer f.Close()

	data, _, err := docconv.ConvertDocx(f)
	if err != nil {
		logging.LogErrorf("convert [%s] failed: [%s]", tmp, err)
		return
	}

	var content = normalizeNonTxtAssetContent(data)
	ret = &AssetParseResult{
		Content: content,
	}
	return
}

type PptxAssetParser struct {
}

func (parser *PptxAssetParser) Parse(absPath string) (ret *AssetParseResult) {
	if !strings.HasSuffix(strings.ToLower(absPath), ".pptx") {
		return
	}

	if !gulu.File.IsExist(absPath) {
		return
	}

	tmp := copyTempAsset(absPath)
	if "" == tmp {
		return
	}
	defer os.RemoveAll(tmp)

	f, err := os.Open(tmp)
	if err != nil {
		logging.LogErrorf("open [%s] failed: [%s]", tmp, err)
		return
	}
	defer f.Close()

	data, _, err := docconv.ConvertPptx(f)
	if err != nil {
		logging.LogErrorf("convert [%s] failed: [%s]", tmp, err)
		return
	}

	var content = normalizeNonTxtAssetContent(data)
	ret = &AssetParseResult{
		Content: content,
	}
	return
}

type XlsxAssetParser struct {
}

func (parser *XlsxAssetParser) Parse(absPath string) (ret *AssetParseResult) {
	if !strings.HasSuffix(strings.ToLower(absPath), ".xlsx") {
		return
	}

	if !gulu.File.IsExist(absPath) {
		return
	}

	tmp := copyTempAsset(absPath)
	if "" == tmp {
		return
	}
	defer os.RemoveAll(tmp)

	x, err := excelize.OpenFile(tmp)
	if err != nil {
		logging.LogErrorf("open [%s] failed: [%s]", tmp, err)
		return
	}
	defer x.Close()

	buf := bytes.Buffer{}
	sheetMap := x.GetSheetMap()
	for _, sheetName := range sheetMap {
		rows, getErr := x.GetRows(sheetName)
		if nil != getErr {
			logging.LogErrorf("get rows from sheet [%s] failed: [%s]", sheetName, getErr)
			return
		}
		for _, row := range rows {
			for _, colCell := range row {
				buf.WriteString(colCell)
				buf.WriteString(" ")
			}
		}
	}

	var content = normalizeNonTxtAssetContent(buf.String())
	ret = &AssetParseResult{
		Content: content,
	}
	return
}

// PdfAssetParser parser factory product
type PdfAssetParser struct {
}

// pdfPage struct defines a worker job for text extraction
type pdfPage struct {
	pageNo int     // page number for text extraction
	data   *[]byte // pointer to PDF document data
}

// pdfTextResult struct defines the extracted PDF text result
type pdfTextResult struct {
	pageNo int    // page number of PDF document
	text   string // text of converted page
	err    error  // processing error
}

// getTextPageWorker will extract the text from a given PDF page and return its result
func (parser *PdfAssetParser) getTextPageWorker(instance pdfium.Pdfium, page <-chan *pdfPage, result chan<- *pdfTextResult) {
	defer instance.Close()
	for pd := range page {
		doc, err := instance.OpenDocument(&requests.OpenDocument{
			File: pd.data,
		})
		if err != nil {
			instance.FPDF_CloseDocument(&requests.FPDF_CloseDocument{
				Document: doc.Document,
			})
			result <- &pdfTextResult{
				pageNo: pd.pageNo,
				err:    err,
			}
			continue
		}

		req := &requests.GetPageText{
			Page: requests.Page{
				ByIndex: &requests.PageByIndex{
					Document: doc.Document,
					Index:    pd.pageNo,
				},
			},
		}
		res, err := instance.GetPageText(req)
		if err != nil {
			instance.FPDF_CloseDocument(&requests.FPDF_CloseDocument{
				Document: doc.Document,
			})
			result <- &pdfTextResult{
				pageNo: pd.pageNo,
				err:    err,
			}
			continue
		}
		instance.FPDF_CloseDocument(&requests.FPDF_CloseDocument{
			Document: doc.Document,
		})
		result <- &pdfTextResult{
			pageNo: pd.pageNo,
			text:   res.Text,
			err:    nil,
		}
	}
}

// Parse will parse a PDF document using PDFium webassembly module using a worker pool
func (parser *PdfAssetParser) Parse(absPath string) (ret *AssetParseResult) {
	if util.IsMobileContainer() {
		// PDF asset content searching is not supported on mobile platforms
		return
	}

	now := time.Now()
	if !strings.HasSuffix(strings.ToLower(absPath), ".pdf") {
		return
	}

	info, err := os.Stat(absPath)
	if err != nil || !info.Mode().IsRegular() {
		return
	}
	if maxSizeVal := os.Getenv("SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE"); "" != maxSizeVal {
		if maxSize, parseErr := strconv.ParseUint(maxSizeVal, 10, 64); nil == parseErr {
			if maxSize != PDFAssetContentMaxSize {
				PDFAssetContentMaxSize = maxSize
				logging.LogInfof("set PDF asset content index max size to [%s]", humanize.BytesCustomCeil(maxSize, 2))
			}
		} else {
			logging.LogWarnf("invalid env [SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE]: [%s], parsing failed: %s", maxSizeVal, parseErr)
		}
	}
	if PDFAssetContentMaxSize < uint64(info.Size()) {
		logging.LogWarnf("ignore large PDF asset [%s] with [%s]", absPath, humanize.BytesCustomCeil(uint64(info.Size()), 2))
		return
	}

	tmp := copyTempAsset(absPath)
	if "" == tmp {
		return
	}
	defer os.RemoveAll(tmp)

	// PDF blob will be processed in-memory making sharing of PDF document data across worker goroutines possible
	pdfFile, err := os.Open(tmp)
	if err != nil {
		logging.LogErrorf("open [%s] failed: [%s]", tmp, err)
		return
	}
	// 复制期间源文件可能增长,读取上限仍需独立约束。
	pdfData, err := io.ReadAll(io.LimitReader(pdfFile, int64(min(PDFAssetContentMaxSize, uint64(math.MaxInt64-1)))+1))
	pdfFile.Close()
	if err != nil {
		logging.LogErrorf("open [%s] failed: [%s]", tmp, err)
		return
	}
	if PDFAssetContentMaxSize < uint64(len(pdfData)) {
		logging.LogWarnf("ignore large PDF asset [%s] with [%s]", absPath, humanize.BytesCustomCeil(uint64(len(pdfData)), 2))
		return
	}

	cores := min(runtime.NumCPU(), 4) // Limit memory usage

	pool, err := webassembly.Init(webassembly.Config{
		MinIdle:  cores,
		MaxIdle:  cores,
		MaxTotal: cores,
	})
	if err != nil {
		logging.LogErrorf("convert [%s] failed: [%s]", tmp, err)
		return
	}
	defer pool.Close()

	// first get the number of PDF pages to convert into text
	instance, err := pool.GetInstance(time.Second * 30)
	if err != nil {
		logging.LogErrorf("convert [%s] failed: [%s]", tmp, err)
		return
	}
	doc, err := instance.OpenDocument(&requests.OpenDocument{
		File: &pdfData,
	})
	if err != nil {
		instance.Close()
		logging.LogErrorf("convert [%s] failed: [%s]", tmp, err)
		return
	}
	pc, err := instance.FPDF_GetPageCount(&requests.FPDF_GetPageCount{Document: doc.Document})
	if err != nil {
		instance.FPDF_CloseDocument(&requests.FPDF_CloseDocument{
			Document: doc.Document,
		})
		instance.Close()
		logging.LogErrorf("convert [%s] failed: [%s]", tmp, err)
		return
	}
	instance.Close()

	if PDFAssetContentMaxPage < pc.PageCount {
		// PDF files longer than 1024 pages are not included in asset file content searching https://github.com/siyuan-note/siyuan/issues/9053
		logging.LogWarnf("ignore large PDF asset [%s] with [%d] pages", absPath, pc.PageCount)
		return
	}

	// next setup worker pool for processing PDF pages
	pages := make(chan *pdfPage, pc.PageCount)
	results := make(chan *pdfTextResult, pc.PageCount)
	for range cores {
		inst, err := pool.GetInstance(time.Second * 30)
		if err != nil {
			close(pages)
			close(results)
			logging.LogErrorf("convert [%s] failed: [%s]", tmp, err)
			return
		}
		go parser.getTextPageWorker(inst, pages, results)
	}

	// now split pages and let them process by worker pool
	for p := 0; p < pc.PageCount; p++ {
		pages <- &pdfPage{
			pageNo: p,
			data:   &pdfData,
		}
	}
	close(pages)

	// finally fetch the PDF page text results
	// Note: some workers will process pages faster than other workers depending on the page contents
	// the order of returned PDF text pages is random and must be sorted using the pageNo index
	pageText := make([]string, pc.PageCount)
	for p := 0; p < pc.PageCount; p++ {
		res := <-results
		pageText[res.pageNo] = res.text
		if nil != res.err {
			logging.LogErrorf("convert [%s] of page %d failed: [%s]", tmp, res.pageNo, res.err)
		}
	}
	close(results)

	if 128 < pc.PageCount {
		logging.LogInfof("convert [%s] PDF with [%d] pages using [%d] workers took [%s]", absPath, pc.PageCount, cores, time.Since(now))
	}

	// loop through ordered PDF text pages and join content for asset parse DB result
	contentBuilder := bytes.Buffer{}
	for _, pt := range pageText {
		contentBuilder.WriteString(" ")
		contentBuilder.WriteString(normalizeNonTxtAssetContent(pt))
	}
	ret = &AssetParseResult{
		Content: contentBuilder.String(),
	}
	return
}

type EpubAssetParser struct {
}

func (parser *EpubAssetParser) Parse(absPath string) (ret *AssetParseResult) {
	if !strings.HasSuffix(strings.ToLower(absPath), ".epub") {
		return
	}

	if !gulu.File.IsExist(absPath) {
		return
	}

	tmp := copyTempAsset(absPath)
	if "" == tmp {
		return
	}
	defer os.RemoveAll(tmp)

	f, err := os.Open(tmp)
	if err != nil {
		logging.LogErrorf("open [%s] failed: [%s]", tmp, err)
		return
	}
	defer f.Close()

	buf := bytes.Buffer{}
	if err = epub.ToTxt(tmp, &buf); err != nil {
		logging.LogErrorf("convert [%s] failed: [%s]", tmp, err)
		return
	}

	content := normalizeNonTxtAssetContent(buf.String())
	ret = &AssetParseResult{
		Content: content,
	}
	return
}