An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:bug: Reject oversized PDF assets before copying and parsing https://github.com/siyuan-note/siyuan/issues/20357
Daniel114b11fc5c01
diff --git a/kernel/model/asset_content.go b/kernel/model/asset_content.goindex 431eebde8f..43b55bbf07 100644--- a/kernel/model/asset_content.go+++ b/kernel/model/asset_content.go@@ -18,7 +18,9 @@ package modelimport ("bytes"+ "io""io/fs"+ "math""os""path/filepath""runtime"@@ -810,7 +812,22 @@ func (parser *PdfAssetParser) Parse(absPath string) (ret *AssetParseResult) {return}- if !gulu.File.IsExist(absPath) {+ info, err := os.Stat(absPath)+ if err != nil || !info.Mode().IsRegular() {+ return+ }+ if maxSizeVal := os.Getenv("SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE"); "" != maxSizeVal {+ if maxSize, parseErr := strconv.ParseUint(maxSizeVal, 10, 64); nil == parseErr {+ if maxSize != PDFAssetContentMaxSize {+ PDFAssetContentMaxSize = maxSize+ logging.LogInfof("set PDF asset content index max size to [%s]", humanize.BytesCustomCeil(maxSize, 2))+ }+ } else {+ logging.LogWarnf("invalid env [SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE]: [%s], parsing failed: %s", maxSizeVal, parseErr)+ }+ }+ if PDFAssetContentMaxSize < uint64(info.Size()) {+ logging.LogWarnf("ignore large PDF asset [%s] with [%s]", absPath, humanize.BytesCustomCeil(uint64(info.Size()), 2))return}@@ -821,11 +838,22 @@ func (parser *PdfAssetParser) Parse(absPath string) (ret *AssetParseResult) {defer os.RemoveAll(tmp)// PDF blob will be processed in-memory making sharing of PDF document data across worker goroutines possible- pdfData, err := os.ReadFile(tmp)+ pdfFile, err := os.Open(tmp)if err != nil {logging.LogErrorf("open [%s] failed: [%s]", tmp, err)return}+ // 复制期间源文件可能增长,读取上限仍需独立约束。+ pdfData, err := io.ReadAll(io.LimitReader(pdfFile, int64(min(PDFAssetContentMaxSize, uint64(math.MaxInt64-1)))+1))+ pdfFile.Close()+ if err != nil {+ logging.LogErrorf("open [%s] failed: [%s]", tmp, err)+ return+ }+ if PDFAssetContentMaxSize < uint64(len(pdfData)) {+ logging.LogWarnf("ignore large PDF asset [%s] with [%s]", absPath, humanize.BytesCustomCeil(uint64(len(pdfData)), 2))+ return+ }cores := min(runtime.NumCPU(), 4) // Limit memory usage@@ -871,23 +899,6 @@ func (parser *PdfAssetParser) Parse(absPath string) (ret *AssetParseResult) {return}- if maxSizeVal := os.Getenv("SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE"); "" != maxSizeVal {- if maxSize, parseErr := strconv.ParseUint(maxSizeVal, 10, 64); nil == parseErr {- if maxSize != PDFAssetContentMaxSize {- PDFAssetContentMaxSize = maxSize- logging.LogInfof("set PDF asset content index max size to [%s]", humanize.BytesCustomCeil(maxSize, 2))- }- } else {- logging.LogWarnf("invalid env [SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE]: [%s], parsing failed: %s", maxSizeVal, parseErr)- }- }-- if PDFAssetContentMaxSize < uint64(len(pdfData)) {- // PDF files larger than 128MB are not included in asset file content searching https://github.com/siyuan-note/siyuan/issues/9500- logging.LogWarnf("ignore large PDF asset [%s] with [%s]", absPath, humanize.BytesCustomCeil(uint64(len(pdfData)), 2))- return- }-// next setup worker pool for processing PDF pagespages := make(chan *pdfPage, pc.PageCount)results := make(chan *pdfTextResult, pc.PageCount)diff --git a/kernel/model/asset_content_test.go b/kernel/model/asset_content_test.goindex 41794f6bc2..3e248afc34 100644--- a/kernel/model/asset_content_test.go+++ b/kernel/model/asset_content_test.go@@ -17,9 +17,13 @@package modelimport (+ "os"+ "path/filepath""slices""strings""testing"++ "github.com/siyuan-note/siyuan/kernel/util")func TestAssetContentFieldRegexpUsesArguments(t *testing.T) {@@ -73,3 +77,23 @@ func TestPDFParser(t *testing.T) {t.Fatalf("empty or nil PDF content result")}}++func TestPDFParserRejectsOversizeBeforeCopy(t *testing.T) {+ oldLimit, oldTemp := PDFAssetContentMaxSize, util.TempDir+ t.Cleanup(func() { PDFAssetContentMaxSize, util.TempDir = oldLimit, oldTemp })+ t.Setenv("SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE", "16")+ util.TempDir = filepath.Join(t.TempDir(), "unused-temp")+ pdf := filepath.Join(t.TempDir(), "oversize.pdf")+ if err := os.WriteFile(pdf, []byte(strings.Repeat("x", 17)), 0644); err != nil {+ t.Fatal(err)+ }+ if result := (&PdfAssetParser{}).Parse(pdf); result != nil {+ t.Fatal("oversize PDF must be skipped")+ }+ if PDFAssetContentMaxSize != 16 {+ t.Fatal("environment limit was not applied before the size check")+ }+ if _, err := os.Stat(util.TempDir); !os.IsNotExist(err) {+ t.Fatalf("rejected PDF must not create temporary data: %v", err)+ }+}