An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:sparkles: Support image metadata matching in asset search https://github.com/siyuan-note/siyuan/issues/19976
Danielde3de000af65
diff --git a/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20200924100744-br924ar.sy b/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20200924100744-br924ar.syindex 12225d5289..57c5141013 100644--- a/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20200924100744-br924ar.sy+++ b/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20200924100744-br924ar.sy@@ -6,7 +6,7 @@"id": "20200924100744-br924ar","title": "Assets","type": "doc",- "updated": "20260820133158"+ "updated": "20260930085927"},"Children": [{@@ -343,6 +343,20 @@}]},+ {+ "ID": "20260930085927-fd25q1l",+ "Type": "NodeParagraph",+ "Properties": {+ "id": "20260930085927-fd25q1l",+ "updated": "20260930085927"+ },+ "Children": [+ {+ "Type": "NodeText",+ "Data": "When inserting an existing asset, keywords also match image titles, tooltip text, and existing OCR text. The same image may have different titles or tooltip text in different documents; any matching reference can locate the asset. Encrypted notebook assets are excluded from this global search."+ }+ ]+ },{"ID": "20210115224203-q8bj2gd","Type": "NodeHeading",@@ -386,7 +400,7 @@{"Type": "NodeTextMark","TextMarkType": "code",- "TextMarkTextContent": "\u0026lt;img\u0026gt;"+ "TextMarkTextContent": "<img>"},{"Type": "NodeText",@@ -395,7 +409,7 @@{"Type": "NodeTextMark","TextMarkType": "code",- "TextMarkTextContent": "\u0026lt;iframe\u0026gt;"+ "TextMarkTextContent": "<iframe>"},{"Type": "NodeText",diff --git a/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20200915214115-42b8zma.sy b/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20200915214115-42b8zma.syindex a20d33e98d..76a2d44923 100644--- a/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20200915214115-42b8zma.sy+++ b/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20200915214115-42b8zma.sy@@ -6,7 +6,7 @@"id": "20200915214115-42b8zma","title": "资源文件","type": "doc",- "updated": "20260820133158"+ "updated": "20260930085927"},"Children": [{@@ -343,6 +343,20 @@}]},+ {+ "ID": "20260930085927-dnnymxe",+ "Type": "NodeParagraph",+ "Properties": {+ "id": "20260930085927-dnnymxe",+ "updated": "20260930085927"+ },+ "Children": [+ {+ "Type": "NodeText",+ "Data": "插入已有资源时,关键词也会匹配图片标题、提示文本和已有 OCR 文本。同一图片在不同文档中的标题或提示文本可以不同,任一引用匹配即可找到该资源。此全局搜索不包含加密笔记本的资源。"+ }+ ]+ },{"ID": "20210115222011-bzsftun","Type": "NodeHeading",@@ -377,7 +391,7 @@{"Type": "NodeTextMark","TextMarkType": "code",- "TextMarkTextContent": "\u0026lt;img\u0026gt;"+ "TextMarkTextContent": "<img>"},{"Type": "NodeText",@@ -386,7 +400,7 @@{"Type": "NodeTextMark","TextMarkType": "code",- "TextMarkTextContent": "\u0026lt;iframe\u0026gt;"+ "TextMarkTextContent": "<iframe>"},{"Type": "NodeText",diff --git a/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20211226123038-4umgpxy.sy b/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20211226123038-4umgpxy.syindex 73dea8abcf..f2031f8c50 100644--- a/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20211226123038-4umgpxy.sy+++ b/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20211226123038-4umgpxy.sy@@ -6,7 +6,7 @@"id": "20211226123038-4umgpxy","title": "資料文件","type": "doc",- "updated": "20260820133158"+ "updated": "20260930085927"},"Children": [{@@ -339,6 +339,20 @@}]},+ {+ "ID": "20260930085927-6il5688",+ "Type": "NodeParagraph",+ "Properties": {+ "id": "20260930085927-6il5688",+ "updated": "20260930085927"+ },+ "Children": [+ {+ "Type": "NodeText",+ "Data": "插入已有資源時,關鍵字也會比對圖片標題、提示文字及已有的 OCR 文字。同一圖片在不同文件中的標題或提示文字可以不同,任一引用符合即可找到該資源。此全域搜尋不包含加密筆記本的資源。"+ }+ ]+ },{"ID": "20211226123051-5250xg7","Type": "NodeHeading",@@ -369,7 +383,7 @@{"Type": "NodeTextMark","TextMarkType": "code",- "TextMarkTextContent": "\u0026lt;img\u0026gt;"+ "TextMarkTextContent": "<img>"},{"Type": "NodeText",@@ -378,7 +392,7 @@{"Type": "NodeTextMark","TextMarkType": "code",- "TextMarkTextContent": "\u0026lt;iframe\u0026gt;"+ "TextMarkTextContent": "<iframe>"},{"Type": "NodeText",diff --git a/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-1m5un7l.sy b/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-1m5un7l.syindex eec06c376d..6dc65a5523 100644--- a/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-1m5un7l.sy+++ b/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-1m5un7l.sy@@ -7,7 +7,7 @@"id": "20240530101000-1m5un7l","title": "アセット","type": "doc",- "updated": "20260820133158"+ "updated": "20260930085927"},"Children": [{@@ -350,6 +350,20 @@}]},+ {+ "ID": "20260930085927-1t89t26",+ "Type": "NodeParagraph",+ "Properties": {+ "id": "20260930085927-1t89t26",+ "updated": "20260930085927"+ },+ "Children": [+ {+ "Type": "NodeText",+ "Data": "既存のアセットを挿入するときは、画像のタイトル、ツールチップのテキスト、既存の OCR テキストでも検索できます。同じ画像にドキュメントごとに異なるタイトルや説明が付いている場合、いずれかの参照が一致すればアセットを検索できます。このグローバル検索には暗号化ノートブックのアセットは含まれません。"+ }+ ]+ },{"ID": "20240530101000-m2u22gy","Type": "NodeHeading",@@ -391,7 +405,7 @@{"Type": "NodeTextMark","TextMarkType": "code",- "TextMarkTextContent": "\u0026lt;img\u0026gt;"+ "TextMarkTextContent": "<img>"},{"Type": "NodeText",@@ -400,7 +414,7 @@{"Type": "NodeTextMark","TextMarkType": "code",- "TextMarkTextContent": "\u0026lt;iframe\u0026gt;"+ "TextMarkTextContent": "<iframe>"},{"Type": "NodeText",diff --git a/app/src/asset/picker.ts b/app/src/asset/picker.tsindex 72648dc61d..4dd3d4b61b 100644--- a/app/src/asset/picker.ts+++ b/app/src/asset/picker.ts@@ -105,6 +105,7 @@ export const openAssetPicker = (options: AssetPickerOptions = {}): Promise<Assettry {const response = await fetchSyncPost("/api/search/searchAsset", {k: keyword,+ includeMetadata: true,exts: options.exts || [],match: options.match,page,diff --git a/app/src/menus/protyle.ts b/app/src/menus/protyle.tsindex 23eed6ef97..a53317c881 100644--- a/app/src/menus/protyle.ts+++ b/app/src/menus/protyle.ts@@ -91,7 +91,8 @@ import {const renderAssetList = (element: Element, k: string, position: IPosition, exts: string[] = []) => {fetchPost("/api/search/searchAsset", {k,- exts+ exts,+ includeMetadata: true}, (response) => {let searchHTML = "";response.data.forEach((item: { path: string, hName: string }, index: number) => {diff --git a/app/src/types/api/index.d.ts b/app/src/types/api/index.d.tsindex da884cff18..ad8d2e26f3 100644--- a/app/src/types/api/index.d.ts+++ b/app/src/types/api/index.d.ts@@ -1684,7 +1684,7 @@ export type SearchAssetContentRequestInput = { "method"?: number | null; "orderBexport type SearchAssetMatchInput = { "field"?: "name" | "path"; "mode": "prefix" | "suffix" | "regex"; "value": string; };-export type SearchAssetRequestInput = { "exts"?: Array<string> | null; "k": string; "match"?: SearchAssetMatchInput | null; "page"?: number | null; "pageSize"?: number | null; };+export type SearchAssetRequestInput = { "exts"?: Array<string> | null; "includeMetadata"?: boolean; "k": string; "match"?: SearchAssetMatchInput | null; "page"?: number | null; "pageSize"?: number | null; };export type SearchAttributeViewRelationKeyRequestInput = { "avID": string; "keyword": string; };diff --git a/kernel/api/contract_search_test.go b/kernel/api/contract_search_test.goindex 2d0c621821..9cc86b0e9b 100644--- a/kernel/api/contract_search_test.go+++ b/kernel/api/contract_search_test.go@@ -9,11 +9,13 @@ import ("strings""testing"+ "github.com/88250/lute/parse""github.com/gin-gonic/gin""github.com/siyuan-note/siyuan/kernel/apicontract""github.com/siyuan-note/siyuan/kernel/cache""github.com/siyuan-note/siyuan/kernel/conf""github.com/siyuan-note/siyuan/kernel/model"+ "github.com/siyuan-note/siyuan/kernel/sql""github.com/siyuan-note/siyuan/kernel/util")@@ -183,6 +185,7 @@ func TestAPIContractSearchPathAndSubtypeCompatibility(t *testing.T) {// 搜索查询在契约测试的独立进程内访问临时数据库。func testSearchQueryContracts(t *testing.T, docID string) {t.Helper()+ testSearchAssetMetadataContract(t)for _, entry := range []struct {path, body stringhandler gin.HandlerFunc@@ -215,3 +218,63 @@ func testSearchQueryContracts(t *testing.T, docID string) {}}}++// testSearchAssetMetadataContract 在独立数据库中验证图片元数据、文件筛选和分页。+func testSearchAssetMetadataContract(t *testing.T) {+ t.Helper()+ assetsDir := filepath.Join(util.DataDir, "assets")+ if err := os.MkdirAll(assetsDir, 0755); err != nil {+ t.Fatal(err)+ }+ for _, name := range []string{"metadata-a.png", "metadata-b.webp", "metadata-c.png", "metadata-orphan.png"} {+ if err := os.WriteFile(filepath.Join(assetsDir, name), []byte("image"), 0644); err != nil {+ t.Fatal(err)+ }+ }+ cache.LoadAssets()+ lute := util.NewLute()+ tree := parse.Parse("", []byte(` `), lute.ParseOptions)+ tree.ID, tree.Box, tree.Path, tree.HPath = tree.Root.ID, "20260912000000-boxapi1", "/"+tree.Root.ID+".sy", "/Images"+ sql.UpsertTreeQueue(tree)+ sql.FlushQueue()+ util.SetAssetText("assets/metadata-c.png", "metadata recognized text")+ util.SetAssetText("assets/metadata-orphan.png", "metadata orphan text")+ t.Cleanup(func() {+ util.SetAssetText("assets/metadata-c.png", "")+ util.SetAssetText("assets/metadata-orphan.png", "")+ })+ engine := gin.New()+ engine.POST("/api/search/searchAsset", searchAsset)+ for _, tc := range []struct {+ body string+ want []string+ }{+ {`{"k":"caption"}`, nil},+ {`{"k":"caption","includeMetadata":false}`, nil},+ {`{"k":"caption","includeMetadata":true}`, []string{"assets/metadata-a.png", "assets/metadata-b.webp"}},+ {`{"k":"other","includeMetadata":true}`, []string{"assets/metadata-b.webp"}},+ {`{"k":"recognized","includeMetadata":true}`, []string{"assets/metadata-c.png"}},+ {`{"k":"orphan text","includeMetadata":true,"exts":["webp"]}`, nil},+ {`{"k":"caption","includeMetadata":true,"exts":["png"],"match":{"mode":"suffix","value":"-a.png"}}`, []string{"assets/metadata-a.png"}},+ {`{"k":"recognized","includeMetadata":true,"pageSize":1,"page":2}`, nil},+ {`{"k":"%_'; --","includeMetadata":true}`, nil},+ } {+ recorder := httptest.NewRecorder()+ engine.ServeHTTP(recorder, httptest.NewRequest("POST", "/api/search/searchAsset", strings.NewReader(tc.body)))+ requireAPIContract(t, "POST", "/api/search/searchAsset", recorder)+ var response struct {+ Code int `json:"code"`+ Data []*apicontract.SearchAsset `json:"data"`+ }+ if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil || response.Code != 0 {+ t.Fatalf("metadata search failed: %s, %v", recorder.Body.String(), err)+ }+ var paths []string+ for _, asset := range response.Data {+ paths = append(paths, asset.Path)+ }+ if !reflect.DeepEqual(paths, tc.want) {+ t.Fatalf("metadata search %s: got %v, want %v", tc.body, paths, tc.want)+ }+ }+}diff --git a/kernel/api/search.go b/kernel/api/search.goindex 3ff19e55d0..baa206d839 100644--- a/kernel/api/search.go+++ b/kernel/api/search.go@@ -152,7 +152,7 @@ var searchAsset = contractHandler(apicontract.SearchAssetByName, func(c *gin.Conif request.Match != nil {match = &model.AssetSearchMatch{Field: request.Match.Field, Mode: request.Match.Mode, Value: request.Match.Value}}- assets, err := model.SearchAssetsByNamePage(request.K, request.Exts, match, page, pageSize)+ assets, err := model.SearchAssetsByNamePage(request.K, request.Exts, match, page, pageSize, request.IncludeMetadata)if err != nil {return apicontract.Failure[[]*apicontract.SearchAsset](-1, err.Error())}diff --git a/kernel/apicontract/schema.json b/kernel/apicontract/schema.jsonindex 6bf83ad307..c421de4ade 100644--- a/kernel/apicontract/schema.json+++ b/kernel/apicontract/schema.json@@ -36300,6 +36300,9 @@}]},+ "includeMetadata": {+ "type": "boolean"+ },"k": {"type": "string"},diff --git a/kernel/apicontract/search.go b/kernel/apicontract/search.goindex 5721473ee1..21235a452a 100644--- a/kernel/apicontract/search.go+++ b/kernel/apicontract/search.go@@ -39,6 +39,9 @@ type SearchPathRequest struct {type SearchAssetRequest struct {// K 保留原有文件名和路径的关键词搜索与排序语义。K string `json:"k"`+ // IncludeMetadata 为 true 时额外匹配普通笔记本中图片的标题、提示文本和已有 OCR 文本。+ // 省略或 false 保留文件名和路径搜索;不查询加密笔记本,筛选、排序及分页仍在合并命中后执行。+ IncludeMetadata bool `json:"includeMetadata" api:"optional"`// Exts 可省略或留空以搜索全部类型,扩展名接受 png 和 .png 两种写法,匹配时不区分大小写。Exts []string `json:"exts" api:"optional,nullable"`// Match 在关键词和扩展名筛选后、分页前作用于原始文件名或 assets/ 相对路径。diff --git a/kernel/model/asset_search.go b/kernel/model/asset_search.gonew file mode 100644index 0000000000..34454e1cfd--- /dev/null+++ b/kernel/model/asset_search.go@@ -0,0 +1,61 @@+package model++import (+ "net/url"+ "strings"++ "github.com/88250/lute"+ "github.com/88250/lute/ast"+ "github.com/88250/lute/parse"+ "github.com/siyuan-note/siyuan/kernel/util"+)++// imageAssetMetadataHits 按图片引用归并标题和提示文本命中,同一资源的多个引用取最高权重。+func imageAssetMetadataHits(markdowns, keywords []string) map[string]int {+ ret := map[string]int{}+ luteEngine := lute.New()+ for _, markdown := range markdowns {+ tree := parse.Parse("", []byte(markdown), luteEngine.ParseOptions)+ ast.Walk(tree.Root, func(node *ast.Node, entering bool) ast.WalkStatus {+ if !entering || node.Type != ast.NodeImage {+ return ast.WalkContinue+ }+ dest := node.ChildByType(ast.NodeLinkDest)+ if dest == nil {+ return ast.WalkSkipChildren+ }+ assetPath := strings.SplitN(util.UnescapeHTML(dest.TokensStr()), "?", 2)[0]+ if decoded, err := url.PathUnescape(assetPath); err == nil {+ assetPath = decoded+ }+ if !strings.HasPrefix(assetPath, "assets/") {+ return ast.WalkSkipChildren+ }+ hits := assetMetadataTextHits(util.UnescapeHTML(node.Content()), keywords)+ if title := node.ChildByType(ast.NodeLinkTitle); title != nil {+ hits += assetMetadataTextHits(util.UnescapeHTML(title.TokensStr()), keywords)+ }+ ret[assetPath] = max(ret[assetPath], hits)+ return ast.WalkSkipChildren+ })+ }+ return ret+}++func assetMetadataTextHits(text string, keywords []string) int {+ text = strings.ToLower(text)+ hits := 0+ for i, keyword := range keywords {+ if strings.TrimSpace(keyword) == "" {+ continue+ }+ keyword = strings.ToLower(keyword)+ if strings.Contains(text, keyword) {+ if i == 0 {+ hits += 32+ }+ hits += strings.Count(text, keyword)+ }+ }+ return hits+}diff --git a/kernel/model/asset_search_test.go b/kernel/model/asset_search_test.gonew file mode 100644index 0000000000..197e44980a--- /dev/null+++ b/kernel/model/asset_search_test.go@@ -0,0 +1,31 @@+package model++import "testing"++func TestImageAssetMetadataHits(t *testing.T) {+ markdowns := []string{+ ` `,+ ``,+ ``,+ ``,+ }+ for _, tc := range []struct {+ keyword, path string+ }{+ {"TRAINING", "assets/first.png"},+ {"second hint", "assets/second.png"},+ {"alternate", "assets/first.png"},+ {"100%", "assets/a b.png"},+ {"space & percent", "assets/a b.png"},+ } {+ hits := imageAssetMetadataHits(markdowns, []string{tc.keyword})+ for path, count := range hits {+ if (count > 0) != (path == tc.path) {+ t.Fatalf("metadata search %q: %s=%d, expected %s", tc.keyword, path, count, tc.path)+ }+ }+ if hits[tc.path] == 0 {+ t.Fatalf("metadata search %q did not match %s", tc.keyword, tc.path)+ }+ }+}diff --git a/kernel/model/assets.go b/kernel/model/assets.goindex 91d98b7586..8a6c5dd339 100644--- a/kernel/model/assets.go+++ b/kernel/model/assets.go@@ -918,12 +918,12 @@ type AssetSearchMatch struct {}func SearchAssetsByName(keyword string, exts []string) (ret []*cache.Asset) {- ret, _ = SearchAssetsByNamePage(keyword, exts, nil, 1, Conf.Search.Limit)+ ret, _ = SearchAssetsByNamePage(keyword, exts, nil, 1, Conf.Search.Limit, false)return}-// SearchAssetsByNamePage 根据文件名、路径和扩展名筛选资源,并按稳定顺序返回指定页。-func SearchAssetsByNamePage(keyword string, exts []string, match *AssetSearchMatch, page, pageSize int) (ret []*cache.Asset, err error) {+// SearchAssetsByNamePage 筛选资源,可合并图片元数据命中,并按稳定顺序返回指定页。+func SearchAssetsByNamePage(keyword string, exts []string, match *AssetSearchMatch, page, pageSize int, includeMetadata bool) (ret []*cache.Asset, err error) {ret = []*cache.Asset{}if page < 1 || pageSize < 1 || pageSize > Conf.Search.Limit {return nil, errors.New("invalid asset search page")@@ -958,6 +958,14 @@ func SearchAssetsByNamePage(keyword string, exts []string, match *AssetSearchMatif "" != keyword {keywords = append(keywords, strings.Split(keyword, " ")...)}+ metadataHits := map[string]int{}+ if includeMetadata && strings.TrimSpace(keyword) != "" {+ markdowns, queryErr := sql.QueryImageSpanMarkdowns()+ if queryErr != nil {+ return nil, queryErr+ }+ metadataHits = imageAssetMetadataHits(markdowns, keywords)+ }pathHitCount := map[string]int{}filterAsset := func(path string, asset *cache.Asset) bool {@@ -1012,12 +1020,16 @@ func SearchAssetsByNamePage(keyword string, exts []string, match *AssetSearchMat}// 只返回有匹配的资源- if 1 > hitNameCount+hitPathCount {+ metadataHitCount := metadataHits[asset.Path]+ if includeMetadata && strings.TrimSpace(keyword) != "" {+ metadataHitCount += assetMetadataTextHits(util.GetAssetText(asset.Path), keywords)+ }+ if 1 > hitNameCount+hitPathCount+metadataHitCount {return false}// 记录命中次数用于排序- pathHitCount[asset.Path] = hitNameCount + hitPathCount+ pathHitCount[asset.Path] = hitNameCount + hitPathCount + metadataHitCountreturn true}matchedAssets := cache.FilterAssets(filterAsset)diff --git a/kernel/sql/span.go b/kernel/sql/span.goindex ae245dd2f0..c5ae5cff59 100644--- a/kernel/sql/span.go+++ b/kernel/sql/span.go@@ -49,6 +49,23 @@ type Span struct {IAL string}+// QueryImageSpanMarkdowns 读取全局索引中的图片引用,保留同一段落内各图片的元数据归属。+func QueryImageSpanMarkdowns() (ret []string, err error) {+ rows, err := query("SELECT DISTINCT markdown FROM spans WHERE type = 'img'")+ if err != nil {+ return nil, err+ }+ defer rows.Close()+ for rows.Next() {+ var markdown string+ if err = rows.Scan(&markdown); err != nil {+ return nil, err+ }+ ret = append(ret, markdown)+ }+ return ret, rows.Err()+}+func SelectSpansRawStmt(stmt string, limit int) (ret []*Span) {return selectSpansRawStmt(stmt, limit, func(stmt string) (*sql.Rows, error) {return query(stmt)