Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Reject oversized PDF assets before copying and parsing https://github.com/siyuan-note/siyuan/issues/20357

Daniel114b11fc5c01
diff --git a/kernel/model/asset_content.go b/kernel/model/asset_content.go
index 431eebde8f..43b55bbf07 100644
--- a/kernel/model/asset_content.go
+++ b/kernel/model/asset_content.go
@@ -18,7 +18,9 @@ package model
import (
"bytes"
+ "io"
"io/fs"
+ "math"
"os"
"path/filepath"
"runtime"
@@ -810,7 +812,22 @@ func (parser *PdfAssetParser) Parse(absPath string) (ret *AssetParseResult) {
return
}
- if !gulu.File.IsExist(absPath) {
+ info, err := os.Stat(absPath)
+ if err != nil || !info.Mode().IsRegular() {
+ return
+ }
+ if maxSizeVal := os.Getenv("SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE"); "" != maxSizeVal {
+ if maxSize, parseErr := strconv.ParseUint(maxSizeVal, 10, 64); nil == parseErr {
+ if maxSize != PDFAssetContentMaxSize {
+ PDFAssetContentMaxSize = maxSize
+ logging.LogInfof("set PDF asset content index max size to [%s]", humanize.BytesCustomCeil(maxSize, 2))
+ }
+ } else {
+ logging.LogWarnf("invalid env [SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE]: [%s], parsing failed: %s", maxSizeVal, parseErr)
+ }
+ }
+ if PDFAssetContentMaxSize < uint64(info.Size()) {
+ logging.LogWarnf("ignore large PDF asset [%s] with [%s]", absPath, humanize.BytesCustomCeil(uint64(info.Size()), 2))
return
}
@@ -821,11 +838,22 @@ func (parser *PdfAssetParser) Parse(absPath string) (ret *AssetParseResult) {
defer os.RemoveAll(tmp)
// PDF blob will be processed in-memory making sharing of PDF document data across worker goroutines possible
- pdfData, err := os.ReadFile(tmp)
+ pdfFile, err := os.Open(tmp)
if err != nil {
logging.LogErrorf("open [%s] failed: [%s]", tmp, err)
return
}
+ // 复制期间源文件可能增长,读取上限仍需独立约束。
+ pdfData, err := io.ReadAll(io.LimitReader(pdfFile, int64(min(PDFAssetContentMaxSize, uint64(math.MaxInt64-1)))+1))
+ pdfFile.Close()
+ if err != nil {
+ logging.LogErrorf("open [%s] failed: [%s]", tmp, err)
+ return
+ }
+ if PDFAssetContentMaxSize < uint64(len(pdfData)) {
+ logging.LogWarnf("ignore large PDF asset [%s] with [%s]", absPath, humanize.BytesCustomCeil(uint64(len(pdfData)), 2))
+ return
+ }
cores := min(runtime.NumCPU(), 4) // Limit memory usage
@@ -871,23 +899,6 @@ func (parser *PdfAssetParser) Parse(absPath string) (ret *AssetParseResult) {
return
}
- if maxSizeVal := os.Getenv("SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE"); "" != maxSizeVal {
- if maxSize, parseErr := strconv.ParseUint(maxSizeVal, 10, 64); nil == parseErr {
- if maxSize != PDFAssetContentMaxSize {
- PDFAssetContentMaxSize = maxSize
- logging.LogInfof("set PDF asset content index max size to [%s]", humanize.BytesCustomCeil(maxSize, 2))
- }
- } else {
- logging.LogWarnf("invalid env [SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE]: [%s], parsing failed: %s", maxSizeVal, parseErr)
- }
- }
-
- if PDFAssetContentMaxSize < uint64(len(pdfData)) {
- // PDF files larger than 128MB are not included in asset file content searching https://github.com/siyuan-note/siyuan/issues/9500
- logging.LogWarnf("ignore large PDF asset [%s] with [%s]", absPath, humanize.BytesCustomCeil(uint64(len(pdfData)), 2))
- return
- }
-
// next setup worker pool for processing PDF pages
pages := make(chan *pdfPage, pc.PageCount)
results := make(chan *pdfTextResult, pc.PageCount)
diff --git a/kernel/model/asset_content_test.go b/kernel/model/asset_content_test.go
index 41794f6bc2..3e248afc34 100644
--- a/kernel/model/asset_content_test.go
+++ b/kernel/model/asset_content_test.go
@@ -17,9 +17,13 @@
package model
import (
+ "os"
+ "path/filepath"
"slices"
"strings"
"testing"
+
+ "github.com/siyuan-note/siyuan/kernel/util"
)
func TestAssetContentFieldRegexpUsesArguments(t *testing.T) {
@@ -73,3 +77,23 @@ func TestPDFParser(t *testing.T) {
t.Fatalf("empty or nil PDF content result")
}
}
+
+func TestPDFParserRejectsOversizeBeforeCopy(t *testing.T) {
+ oldLimit, oldTemp := PDFAssetContentMaxSize, util.TempDir
+ t.Cleanup(func() { PDFAssetContentMaxSize, util.TempDir = oldLimit, oldTemp })
+ t.Setenv("SIYUAN_PDF_ASSET_CONTENT_INDEX_MAX_SIZE", "16")
+ util.TempDir = filepath.Join(t.TempDir(), "unused-temp")
+ pdf := filepath.Join(t.TempDir(), "oversize.pdf")
+ if err := os.WriteFile(pdf, []byte(strings.Repeat("x", 17)), 0644); err != nil {
+ t.Fatal(err)
+ }
+ if result := (&PdfAssetParser{}).Parse(pdf); result != nil {
+ t.Fatal("oversize PDF must be skipped")
+ }
+ if PDFAssetContentMaxSize != 16 {
+ t.Fatal("environment limit was not applied before the size check")
+ }
+ if _, err := os.Stat(util.TempDir); !os.IsNotExist(err) {
+ t.Fatalf("rejected PDF must not create temporary data: %v", err)
+ }
+}