Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
feat(docparser): upgrade AnyDoc to 0.2.4 and parse assets once
wizardchen75fc0ad15907
diff --git a/internal/infrastructure/docparser/anydoc/anydoc.go b/internal/infrastructure/docparser/anydoc/anydoc.goindex 5965f26e6..9cd383139 100644--- a/internal/infrastructure/docparser/anydoc/anydoc.go+++ b/internal/infrastructure/docparser/anydoc/anydoc.go@@ -166,6 +166,10 @@ func PDFNeedsOCR(err error) bool {if err == nil {return false}+ var ocrError interface{ NeedsOCR() bool }+ if errors.As(err, &ocrError) && ocrError.NeedsOCR() {+ return true+ }msg := strings.ToLower(err.Error())return strings.Contains(msg, "ocr is required") ||strings.Contains(msg, "no extractable text")diff --git a/internal/infrastructure/docparser/anydoc/backend_cgo.go b/internal/infrastructure/docparser/anydoc/backend_cgo.goindex 4a1040222..625c2ebd6 100644--- a/internal/infrastructure/docparser/anydoc/backend_cgo.go+++ b/internal/infrastructure/docparser/anydoc/backend_cgo.go@@ -35,8 +35,9 @@ func backendConvert(data []byte, opts Options) (*Result, error) {}// Official GFM serializer after rewriting Asset images to External URLs- // (`images/image-N.ext`). ToDocument still supplies the image bytes.- document, err := upstream.ToDocument(data, format)+ // (`images/image-N.ext`). One Rust parse supplies both the original model+ // (for image bytes and placements) and the rendered Markdown.+ document, markdown, err := upstream.ToDocumentWithAssetLinks(data, format)if err != nil {markdown, mdErr := upstream.ToMarkdownBytes(data, format)if mdErr != nil {@@ -47,10 +48,6 @@ func backendConvert(data []byte, opts Options) (*Result, error) {AssetsError: fmt.Errorf("anydoc: image extraction failed: %w", err),}, nil}- markdown, err := upstream.ToMarkdownWithAssetLinks(data, format)- if err != nil {- return nil, fmt.Errorf("anydoc: markdown conversion failed: %w", err)- }return &Result{Markdown: markdown, Assets: collectAssets(document)}, nil}@@ -169,10 +166,16 @@ func inlineText(inlines []upstream.Inline) string {var text strings.Builderfor _, inline := range inlines {switch inline.Kind {- case "text":+ case "text", "math":if inline.Text != nil {text.WriteString(*inline.Text)}+ case "checkbox":+ if inline.Checked != nil && *inline.Checked {+ text.WriteString("[x]")+ } else {+ text.WriteString("[ ]")+ }case "link":text.WriteString(inlineText(inline.Content))}diff --git a/internal/infrastructure/docparser/anydoc/benchmark_linked_test.go b/internal/infrastructure/docparser/anydoc/benchmark_linked_test.gonew file mode 100644index 000000000..bada81348--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/benchmark_linked_test.go@@ -0,0 +1,151 @@+//go:build anydoc && cgo++package anydoc++import (+ "archive/zip"+ "bytes"+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++var benchmarkDocuments = []string{+ "doc/text.doc", "docx/text.docx", "ppt/pres.ppt", "pptx/pres.pptx",+ "odt/text.odt", "ods/sheet.ods", "odp/pres.odp", "rtf/text.rtf",+ "epub/book.epub", "xls/sheet.xls", "xlsx/sheet.xlsx",+ "xlsb/handmade-sheet.xlsb", "csv/sheet.csv", "pdf/text.pdf",+}++// Synthetic size sweeps cover the Go/Rust document-copy path and the bounded+// hostile-PDF detector. They are separate from the upstream fixture comparison.+func BenchmarkConvertScaling(b *testing.B) {+ for _, rows := range []int{100, 10000} {+ b.Run(fmt.Sprintf("csv_%d_rows", rows), func(b *testing.B) {+ data := []byte("name,quantity,description\n" ++ strings.Repeat("widget,42,Quarterly shipping report\n", rows))+ b.SetBytes(int64(len(data)))+ b.ReportAllocs()+ b.ResetTimer()+ for b.Loop() {+ result, err := Convert(data, Options{Format: "csv", WithAssets: true})+ if err != nil || result.AssetsError != nil {+ b.Fatalf("conversion: %v", err)+ }+ }+ })+ }+ for _, rows := range []int{100, 10000} {+ b.Run(fmt.Sprintf("xlsx_%d_rows", rows), func(b *testing.B) {+ data := largeWorkbook(b, rows)+ result, err := Convert(data, Options{Format: "xlsx", WithAssets: true})+ if err != nil || !strings.Contains(result.Markdown, fmt.Sprintf("row-%d", rows)) {+ b.Fatalf("preflight: last row was lost, err=%v", err)+ }+ b.SetBytes(int64(len(data)))+ b.ReportAllocs()+ b.ResetTimer()+ for b.Loop() {+ result, err := Convert(data, Options{Format: "xlsx", WithAssets: true})+ if err != nil || result.AssetsError != nil {+ b.Fatalf("conversion: %v", err)+ }+ }+ })+ }+ for _, operators := range []int{40000, 200000} {+ b.Run(fmt.Sprintf("pdf_%d_operators", operators), func(b *testing.B) {+ data := unmatchedShowTextPDF(operators)+ b.SetBytes(int64(len(data)))+ b.ReportAllocs()+ b.ResetTimer()+ for b.Loop() {+ if _, err := Convert(data, Options{Format: "pdf"}); err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}++func largeWorkbook(t testing.TB, rows int) []byte {+ t.Helper()+ data := readFixture(t, "xlsx/sheet.xlsx")+ r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))+ if err != nil {+ t.Fatal(err)+ }+ var out bytes.Buffer+ w := zip.NewWriter(&out)+ for _, file := range r.File {+ if file.Name == "xl/worksheets/sheet1.xml" {+ continue+ }+ if err := w.Copy(file); err != nil {+ t.Fatal(err)+ }+ }+ var sheet strings.Builder+ sheet.WriteString(`<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>`)+ for row := 1; row <= rows; row++ {+ fmt.Fprintf(&sheet, `<row r="%d"><c r="A%d" t="inlineStr"><is><t>row-%d</t></is></c>`++ `<c r="B%d"><v>42</v></c></row>`,+ row, row, row, row)+ }+ sheet.WriteString(`</sheetData></worksheet>`)+ part, err := w.Create("xl/worksheets/sheet1.xml")+ if err != nil {+ t.Fatal(err)+ }+ if _, err := part.Write([]byte(sheet.String())); err != nil {+ t.Fatal(err)+ }+ if err := w.Close(); err != nil {+ t.Fatal(err)+ }+ return out.Bytes()+}++func readFixture(t testing.TB, name string) []byte {+ t.Helper()+ data, err := os.ReadFile(filepath.Join("testdata", "upstream", name))+ if err != nil {+ t.Fatal(err)+ }+ return data+}++// WithAssets exercises the default reader path, including Rust parsing,+// the document ABI, Go decoding, image collection, and Markdown serialization.+// Input I/O is excluded. B/op measures Go allocations, not the Rust heap.+func BenchmarkConvert(b *testing.B) {+ for _, name := range benchmarkDocuments {+ b.Run(strings.Split(name, "/")[0], func(b *testing.B) {+ data := readFixture(b, name)+ format := strings.TrimPrefix(filepath.Ext(name), ".")+ if format == "xls" || format == "xlsb" {+ format = "xlsx"+ }+ opts := Options{Format: format, WithAssets: true}+ if result, err := Convert(data, opts); err != nil ||+ result.Markdown == "" ||+ result.AssetsError != nil {+ b.Fatalf("preflight conversion: result=%+v err=%v", result, err)+ }+ b.ReportAllocs()+ b.SetBytes(int64(len(data)))+ b.ResetTimer()+ b.RunParallel(func(pb *testing.PB) {+ for pb.Next() {+ result, err := Convert(data, opts)+ if err != nil || result.Markdown == "" || result.AssetsError != nil {+ b.Errorf("conversion failed: result=%+v err=%v", result, err)+ return+ }+ }+ })+ })+ }+}diff --git a/internal/infrastructure/docparser/anydoc/convert_linked_test.go b/internal/infrastructure/docparser/anydoc/convert_linked_test.goindex daf8512fb..556a32519 100644--- a/internal/infrastructure/docparser/anydoc/convert_linked_test.go+++ b/internal/infrastructure/docparser/anydoc/convert_linked_test.go@@ -116,9 +116,9 @@ func TestErrorDetailSurvivesConcurrency(t *testing.T) {defer wg.Done()// docx and xlsx failures carry distinctive messages, so a detail// naming the other format proves it came from the wrong thread.- format, want, other := "docx", "not a readable zip archive", "unreadable workbook"+ format, want, other := "docx", "not a readable zip archive", "not a readable workbook container"if i%2 == 1 {- format, want, other = "xlsx", "unreadable workbook", "not a readable zip archive"+ format, want, other = "xlsx", "not a readable workbook container", "not a readable zip archive"}_, err := Convert([]byte(fmt.Sprintf("garbage %d", i)), Options{Format: format})if err == nil {diff --git a/internal/infrastructure/docparser/anydoc/corpus_linked_test.go b/internal/infrastructure/docparser/anydoc/corpus_linked_test.gonew file mode 100644index 000000000..f128d8678--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/corpus_linked_test.go@@ -0,0 +1,89 @@+//go:build anydoc && cgo++package anydoc++import (+ "errors"+ "os"+ "path/filepath"+ "reflect"+ "strings"+ "testing"++ upstream "github.com/firecrawl/anydoc/go"+)++// ANYDOC_UPSTREAM_DIR points to the checked-out release tag. This opt-in+// upgrade check runs the full upstream corpus through our actual Go ABI and+// compares Markdown/error details to the upstream Rust snapshots.+func TestUpstreamCorpus(t *testing.T) {+ root := os.Getenv("ANYDOC_UPSTREAM_DIR")+ if root == "" {+ t.Skip("set ANYDOC_UPSTREAM_DIR to an anydoc release checkout")+ }+ snapshots, err := filepath.Glob(filepath.Join(root, "tests/snapshots/snapshots__*.snap"))+ if err != nil || len(snapshots) == 0 {+ t.Fatalf("no snapshots: %v", err)+ }+ for _, path := range snapshots {+ name := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(path), "snapshots__"), ".snap")+ parts := strings.Split(name, "__")+ if len(parts) != 2 {+ continue+ }+ t.Run(name, func(t *testing.T) {+ data, err := os.ReadFile(filepath.Join(root, "tests/fixtures", parts[0], parts[1]))+ if err != nil {+ t.Fatal(err)+ }+ snapshot, err := os.ReadFile(path)+ if err != nil {+ t.Fatal(err)+ }+ _, expected, ok := strings.Cut(string(snapshot[4:]), "---\n")+ if !ok {+ t.Fatal("invalid snapshot header")+ }+ format, ok := upstream.FormatFromExtension(strings.TrimPrefix(filepath.Ext(parts[1]), "."))+ if !ok {+ t.Fatal("unrecognized fixture extension")+ }+ // The Go binding rejects empty slices before entering C, retaining+ // its established empty-input error instead of the ZIP diagnostic.+ if len(data) == 0 {+ if !strings.HasPrefix(expected, "ERROR:") {+ t.Fatal("empty fixture should fail")+ }+ expected = "ERROR: empty input"+ }+ actual, err := upstream.ToMarkdownBytes(data, &format)+ if err != nil {+ var typed *upstream.ConvertError+ if !errors.As(err, &typed) {+ t.Fatal(err)+ }+ actual = "ERROR: " + typed.Detail+ }+ if strings.TrimSpace(actual) != strings.TrimSpace(expected) {+ t.Fatalf("snapshot mismatch\nwant: %s\ngot: %s", expected, actual)+ }+ if err == nil && format != upstream.FormatPdf {+ document, err := upstream.ToDocument(data, &format)+ if err != nil {+ t.Fatalf("document ABI: %v", err)+ }+ markdown, err := upstream.ToMarkdownWithAssetLinks(data, &format)+ if err != nil {+ t.Fatalf("asset renderer: %v", err)+ }+ combined, combinedMarkdown, err := upstream.ToDocumentWithAssetLinks(data, &format)+ if err != nil {+ t.Fatalf("combined ABI: %v", err)+ }+ if !reflect.DeepEqual(combined, document) || combinedMarkdown != markdown {+ t.Fatal("single-parse output differs from independent document/Markdown APIs")+ }+ }+ })+ }+}diff --git a/internal/infrastructure/docparser/anydoc/stress_linked_test.go b/internal/infrastructure/docparser/anydoc/stress_linked_test.gonew file mode 100644index 000000000..127bb8649--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/stress_linked_test.go@@ -0,0 +1,94 @@+//go:build anydoc && cgo++package anydoc++import (+ "bytes"+ "os"+ "runtime"+ "sort"+ "strings"+ "sync"+ "testing"+ "time"+)++// Opt-in sustained mixed-format load through the production WithAssets path.+// Use /usr/bin/time -l (macOS) or -v (Linux) to measure native peak RSS too:+// Go's memory statistics do not include Rust allocations.+func TestConcurrentConversionStress(t *testing.T) {+ setting := os.Getenv("ANYDOC_STRESS_DURATION")+ if setting == "" {+ t.Skip("set ANYDOC_STRESS_DURATION, e.g. 30s")+ }+ duration, err := time.ParseDuration(setting)+ if err != nil || duration <= 0 {+ t.Fatalf("invalid stress duration %q", setting)+ }+ type input struct {+ data []byte+ opts Options+ want *Result+ }+ inputs := make([]input, 0, len(benchmarkDocuments))+ for _, name := range benchmarkDocuments {+ format := strings.Split(name, "/")[0]+ if format == "xls" || format == "xlsb" {+ format = "xlsx"+ }+ i := input{data: readFixture(t, name), opts: Options{Format: format, WithAssets: true}}+ i.want, err = Convert(i.data, i.opts)+ if err != nil || i.want.AssetsError != nil {+ t.Fatalf("preflight %s: %v", name, err)+ }+ inputs = append(inputs, i)+ }+ const workers = 8+ samples := make([][]time.Duration, workers)+ var wg sync.WaitGroup+ start := time.Now()+ deadline := start.Add(duration)+ for worker := 0; worker < workers; worker++ {+ wg.Add(1)+ go func(worker int) {+ defer wg.Done()+ for n := worker; time.Now().Before(deadline); n++ {+ i := inputs[n%len(inputs)]+ began := time.Now()+ got, err := Convert(i.data, i.opts)+ samples[worker] = append(samples[worker], time.Since(began))+ if err != nil ||+ got.AssetsError != nil ||+ got.Markdown != i.want.Markdown ||+ len(got.Assets) != len(i.want.Assets) {+ t.Errorf("worker %d: inconsistent conversion, err=%v", worker, err)+ return+ }+ for j, asset := range got.Assets {+ if !bytes.Equal(asset.Data, i.want.Assets[j].Data) ||+ asset.Name != i.want.Assets[j].Name {+ t.Errorf("worker %d: inconsistent asset %d", worker, j)+ return+ }+ }+ }+ }(worker)+ }+ wg.Wait()+ elapsed := time.Since(start)+ var latencies []time.Duration+ for _, sample := range samples {+ latencies = append(latencies, sample...)+ }+ if len(latencies) == 0 {+ t.Fatal("no conversions completed")+ }+ sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] })+ var memory runtime.MemStats+ runtime.ReadMemStats(&memory)+ t.Logf("workers=%d conversions=%d elapsed=%s throughput=%.1f docs/s "++ "p50=%s p95=%s p99=%s GoHeapAlloc=%d GoTotalAlloc=%d",+ workers, len(latencies), elapsed, float64(len(latencies))/elapsed.Seconds(),+ latencies[len(latencies)*50/100], latencies[len(latencies)*95/100], latencies[len(latencies)*99/100],+ memory.HeapAlloc, memory.TotalAlloc)+}diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/.gitattributes b/internal/infrastructure/docparser/anydoc/testdata/upstream/.gitattributesnew file mode 100644index 000000000..20915656e--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/.gitattributes@@ -0,0 +1,2 @@+# Preserve PDF byte offsets and fixed-width xref records from upstream fixtures.+*.pdf binarydiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/LICENSE b/internal/infrastructure/docparser/anydoc/testdata/upstream/LICENSEnew file mode 100644index 000000000..7ad9f5abd--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/LICENSE@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2026 Sideguide Technologies Inc.++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/README.md b/internal/infrastructure/docparser/anydoc/testdata/upstream/README.mdnew file mode 100644index 000000000..19cdac28e--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/README.md@@ -0,0 +1 @@+Fixtures from [firecrawl/anydoc v0.2.4](https://github.com/firecrawl/anydoc/tree/42bf1c5ecdde9eb0d96d6bd75a9e6698cf93b14c/tests/fixtures), under the included MIT license. Used for Go ABI regression tests and repeatable conversion benchmarks. Files are copied unchanged.diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/csv/sheet.csv b/internal/infrastructure/docparser/anydoc/testdata/upstream/csv/sheet.csvnew file mode 100644index 000000000..ab50c8756--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/csv/sheet.csv@@ -0,0 +1,8 @@+Kind,Value,Note+Percent,15.5%,fifteen and a half+Currency,1234.5,dollars+Thousands,9876543,grouped+Date,2026-03-15,ides of March+Duration,26:30:15,over a day+Tiny,0.0000004,four ten-millionths+Boolean,TRUE,yesdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/doc/text.doc b/internal/infrastructure/docparser/anydoc/testdata/upstream/doc/text.docnew file mode 100644index 000000000..7de1e813fBinary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/doc/text.doc differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/handmade-math.docx b/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/handmade-math.docxnew file mode 100644index 000000000..736ed3f7fBinary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/handmade-math.docx differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/text.docx b/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/text.docxnew file mode 100644index 000000000..0d556258dBinary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/text.docx differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/book.epub b/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/book.epubnew file mode 100644index 000000000..430296407Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/book.epub differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/handmade-math.epub b/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/handmade-math.epubnew file mode 100644index 000000000..9df629c07Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/handmade-math.epub differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/odp/pres.odp b/internal/infrastructure/docparser/anydoc/testdata/upstream/odp/pres.odpnew file mode 100644index 000000000..45d72f1a4Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/odp/pres.odp differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/ods/sheet.ods b/internal/infrastructure/docparser/anydoc/testdata/upstream/ods/sheet.odsnew file mode 100644index 000000000..5c1558cc1Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/ods/sheet.ods differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/handmade-math.odt b/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/handmade-math.odtnew file mode 100644index 000000000..98f34c180Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/handmade-math.odt differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/text.odt b/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/text.odtnew file mode 100644index 000000000..a9fdba542Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/text.odt differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-mixed.pdf b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-mixed.pdfnew file mode 100644index 000000000..7b7164dad--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-mixed.pdf@@ -0,0 +1,50 @@+%PDF-1.4+1 0 obj+<< /Type /Catalog /Pages 2 0 R >>+endobj+2 0 obj+<< /Type /Pages /Kids [5 0 R 7 0 R] /Count 2 >>+endobj+3 0 obj+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>+endobj+4 0 obj+<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>+stream+�+endstream+endobj+5 0 obj+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 6 0 R >>+endobj+6 0 obj+<< /Length 53 >>+stream+BT /F1 24 Tf 72 700 Td (Text on the first page) Tj ET+endstream+endobj+7 0 obj+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 8 0 R >>+endobj+8 0 obj+<< /Length 32 >>+stream+q 468 0 0 648 72 72 cm /Im1 Do Q+endstream+endobj+xref+0 9+0000000000 65535 f+0000000009 00000 n+0000000058 00000 n+0000000121 00000 n+0000000191 00000 n+0000000335 00000 n+0000000487 00000 n+0000000590 00000 n+0000000742 00000 n+trailer+<< /Size 9 /Root 1 0 R >>+startxref+824+%%EOFdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-scanned.pdf b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-scanned.pdfnew file mode 100644index 000000000..071281ba3--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-scanned.pdf@@ -0,0 +1,50 @@+%PDF-1.4+1 0 obj+<< /Type /Catalog /Pages 2 0 R >>+endobj+2 0 obj+<< /Type /Pages /Kids [5 0 R 7 0 R] /Count 2 >>+endobj+3 0 obj+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>+endobj+4 0 obj+<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>+stream+�+endstream+endobj+5 0 obj+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 6 0 R >>+endobj+6 0 obj+<< /Length 32 >>+stream+q 468 0 0 648 72 72 cm /Im1 Do Q+endstream+endobj+7 0 obj+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 8 0 R >>+endobj+8 0 obj+<< /Length 32 >>+stream+q 468 0 0 648 72 72 cm /Im1 Do Q+endstream+endobj+xref+0 9+0000000000 65535 f+0000000009 00000 n+0000000058 00000 n+0000000121 00000 n+0000000191 00000 n+0000000335 00000 n+0000000487 00000 n+0000000569 00000 n+0000000721 00000 n+trailer+<< /Size 9 /Root 1 0 R >>+startxref+803+%%EOFdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/text.pdf b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/text.pdfnew file mode 100644index 000000000..557e1d2d7Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/text.pdf differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/ppt/pres.ppt b/internal/infrastructure/docparser/anydoc/testdata/upstream/ppt/pres.pptnew file mode 100644index 000000000..1e0b433e9Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/ppt/pres.ppt differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/handmade-math.pptx b/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/handmade-math.pptxnew file mode 100644index 000000000..277c97aa0Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/handmade-math.pptx differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/pres.pptx b/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/pres.pptxnew file mode 100644index 000000000..de93ada99Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/pres.pptx differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/handmade-math.rtf b/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/handmade-math.rtfnew file mode 100644index 000000000..6ed5fe9bc--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/handmade-math.rtf@@ -0,0 +1,5 @@+{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0 Cambria Math;}}+\pard The roots are {\mmath{\*\moMath{\rtlch\fcs1 \af0 \ltrch\fcs0 \f0\insrsid1 }{\mr\mscr0\msty2 x}{\mr\mscr0\msty2 =}{\mf{\mfPr{\mctrlPr\f0 }}{\mnum{\mr\mscr0\msty2 -b\'b1}{\mrad{\mradPr{\mdegHide on}{\mctrlPr\f0 }}{\mdeg}{\me{\msSup{\me{\mr\mscr0\msty2 b}}{\msup{\mr\mscr0\msty2 2}}}{\mr\mscr0\msty2 -4ac}}}}{\mden{\mr\mscr0\msty2 2a}}}}{\mmathPict{\*\mmathPr}{\pict\pngblip\picw1\pich1 89504e47}}} for any a.\par+\pard {\mmath{\*\moMathPara{\moMathParaPr{\mjc centerGroup}}{\*\moMath{\mnary{\mnaryPr{\mchr \u8721 ?}{\mlimLoc undOvr}{\mctrlPr\f0 }}{\msub{\mr\mscr0\msty2 i=1}}{\msup{\mr\mscr0\msty2 n}}{\me{\msSub{\me{\mr\mscr0\msty2 x}}{\msub{\mr\mscr0\msty2 i}}}}}{\mr\mscr0\msty2 =}{\mfunc{\mfName{\mr\mscr0\msty0 sin}}{\me{\md{\mdPr{\mctrlPr\f0 }}{\me{\mr\mscr0\msty2 \u952 ?}}}}}}{\*\moMath{\mr\mnor where }{\mr\mscr0\msty2 \u8722 ?1<x}}}}\par+\pard A price of $5 or $6 is not math.\par+}\ No newline at end of filediff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/text.rtf b/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/text.rtfnew file mode 100644index 000000000..24fc6caa9--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/text.rtf@@ -0,0 +1,177 @@+{\rtf1\ansi\deff3\adeflang1025+{\fonttbl{\f0\froman\fprq2\fcharset0 Times New Roman;}{\f1\froman\fprq2\fcharset2 Symbol;}{\f2\fswiss\fprq2\fcharset0 Arial;}{\f3\froman\fprq2\fcharset0 Liberation Serif{\*\falt Times New Roman};}{\f4\fswiss\fprq2\fcharset0 Liberation Sans{\*\falt Arial};}{\f5\fnil\fprq2\fcharset0 Microsoft YaHei;}{\f6\fnil\fprq2\fcharset0 Arial;}{\f7\fswiss\fprq0\fcharset128 Arial;}}+{\colortbl;\red0\green0\blue0;\red0\green0\blue255;\red0\green255\blue255;\red0\green255\blue0;\red255\green0\blue255;\red255\green0\blue0;\red255\green255\blue0;\red255\green255\blue255;\red0\green0\blue128;\red0\green128\blue128;\red0\green128\blue0;\red128\green0\blue128;\red128\green0\blue0;\red128\green128\blue0;\red128\green128\blue128;\red192\green192\blue192;}+{\stylesheet{\s0\snext0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052 Normal;}+{\s1\snext25\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl0\outlinelevel0\nowidctlpar\hyphpar0\ltrpar\fs36\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052 heading 1;}+{\s2\snext25\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052 heading 2;}+{\s3\snext25\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl2\outlinelevel2\nowidctlpar\hyphpar0\ltrpar\fs26\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052 heading 3;}+{\*\cs15\snext15\loch\b0 NotBold;}+{\*\cs16\snext16\loch\b Bold;}+{\*\cs17\snext17\loch\i Italic;}+{\*\cs18\snext18\loch\strike Struck;}+{\*\cs19\snext19\loch\cf9\ul\ulc0 Hyperlink;}+{\*\cs20\snext20 Footnote Characters;}+{\*\cs21\snext21\loch\super footnote reference;}+{\*\cs22\snext22 Endnote Characters;}+{\*\cs23\snext23\loch\super endnote reference;}+{\s24\sbasedon0\snext25\rtlch\af6\afs28 \ltrch\hich\af4\loch\sb240\sa120\keepn\f4\fs28\dbch\af5 Heading;}+{\s25\sbasedon0\snext25\loch\sl276\slmult1\sb0\sa140 Body Text;}+{\s26\sbasedon25\snext26\rtlch\af7 \ltrch List;}+{\s27\sbasedon0\snext27\rtlch\af7\afs24\ai \ltrch\loch\sb120\sa120\noline\fs24\i caption;}+{\s28\sbasedon0\snext28\rtlch\af7 \ltrch\loch\noline Index;}+{\s29\snext29\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\b\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052 StrongPara;}+{\s30\sbasedon0\snext30\loch\nowidctlpar\noline Table Contents;}+{\s31\sbasedon0\snext31\rtlch\afs20 \ltrch\loch\fi-340\li340\lin340\noline\fs20 footnote text;}+{\s32\sbasedon0\snext32\rtlch\afs20 \ltrch\loch\fi-340\li340\lin340\noline\fs20 endnote text;}+{\s33\sbasedon0\snext33 Frame Contents;}+}{\*\listtable{\list\listtemplateid1+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}\listid1}+{\list\listtemplateid2+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow2{\leveltext \'02\'00.;}{\levelnumbers\'01;}\fi0\li0}+{\listlevel\levelnfc4\leveljc0\levelstartat1\levelfollow2{\leveltext \'02\'01);}{\levelnumbers\'01;}\fi0\li0}+{\listlevel\levelnfc2\leveljc0\levelstartat1\levelfollow2{\leveltext \'02\'02.;}{\levelnumbers\'01;}\fi0\li0}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'03.;}{\levelnumbers\'01;}\fi-360\li1800}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'04.;}{\levelnumbers\'01;}\fi-360\li2160}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'05.;}{\levelnumbers\'01;}\fi-360\li2520}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'06.;}{\levelnumbers\'01;}\fi-360\li2880}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'07.;}{\levelnumbers\'01;}\fi-360\li3240}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'08.;}{\levelnumbers\'01;}\fi-360\li3600}\listid2}+{\list\listtemplateid3+{\listlevel\levelnfc23\leveljc0\levelstartat1\levelfollow2{\leveltext \'01\u8226 ?;}{\levelnumbers;}\f9\fi0\li0}+{\listlevel\levelnfc23\leveljc0\levelstartat1\levelfollow2{\leveltext \'01\u9702 ?;}{\levelnumbers;}\f9\fi0\li0}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'02.;}{\levelnumbers\'01;}\fi-360\li1440}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'03.;}{\levelnumbers\'01;}\fi-360\li1800}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'04.;}{\levelnumbers\'01;}\fi-360\li2160}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'05.;}{\levelnumbers\'01;}\fi-360\li2520}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'06.;}{\levelnumbers\'01;}\fi-360\li2880}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'07.;}{\levelnumbers\'01;}\fi-360\li3240}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'08.;}{\levelnumbers\'01;}\fi-360\li3600}\listid3}+{\list\listtemplateid4+{\listlevel\levelnfc1\leveljc0\levelstartat1\levelfollow2{\leveltext \'02\'00.;}{\levelnumbers\'01;}\fi0\li0}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'01.;}{\levelnumbers\'01;}\fi-360\li1080}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'02.;}{\levelnumbers\'01;}\fi-360\li1440}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'03.;}{\levelnumbers\'01;}\fi-360\li1800}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'04.;}{\levelnumbers\'01;}\fi-360\li2160}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'05.;}{\levelnumbers\'01;}\fi-360\li2520}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'06.;}{\levelnumbers\'01;}\fi-360\li2880}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'07.;}{\levelnumbers\'01;}\fi-360\li3240}+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'08.;}{\levelnumbers\'01;}\fi-360\li3600}\listid4}+}{\listoverridetable{\listoverride\listid1\listoverridecount0\ls1}{\listoverride\listid2\listoverridecount0\ls2}{\listoverride\listid3\listoverridecount0\ls3}{\listoverride\listid4\listoverridecount0\ls4}}{\*\generator LibreOffice/25.2.7.2$Windows_X86_64 LibreOffice_project/5cbfd1ab6520636bb5f7b99185aa69bd7456825d}{\info{\creatim\yr0\mo0\dy0\hr0\min0}{\revtim\yr0\mo0\dy0\hr0\min0}{\printim\yr0\mo0\dy0\hr0\min0}}{\*\userprops}\deftab709\deftab709\deftab709\deftab709\deftab709+\hyphauto1\viewscale100\formshade\nobrkwrptbl\paperh16838\paperw11906\margl1134\margr1134\margt1134\margb1134\sectd\sbknone\sftnnar\saftnnrlc\sectunlocked1\pgwsxn11906\pghsxn16838\marglsxn1134\margrsxn1134\margtsxn1134\margbsxn1134\ftnbj\ftnstart1\ftnrstcont\ftnnar\fet2\aftnrstcont\aftnstart1\aftnnrlc+{\*\ftnsep\chftnsep}\pgndec\pard\plain \s1\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl0\outlinelevel0\nowidctlpar\hyphpar0\ltrpar\fs36\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch+Fixture Document}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Plain paragraph with }{\loch\cs16\loch\b\loch+bold}{\loch+, }{\loch\cs17\loch\i\loch+italic}{\loch+, and }{\loch\cs18\loch\strike\loch+struck}{\loch+ runs.}+\par \pard\plain \s29\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\b\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Style-bold paragraph with a }{\loch\cs15\loch\b0\loch+NotBold-styled span}{\loch+ inside.}+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch+Lists}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain 1.\tab}\ilvl0\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+First numbered}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain 2.\tab}\ilvl0\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Second numbered}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain a)\tab}\ilvl1\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Alpha sub one}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain b)\tab}\ilvl1\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Alpha sub two}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain i.\tab}\ilvl2\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Roman sub sub}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain 3.\tab}\ilvl0\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Third numbered}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Interrupting paragraph between lists.}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain 4.\tab}\ilvl0\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Fourth, continuing the count}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain IV.\tab}\ilvl0\ls4 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Roman starting at four}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain V.\tab}\ilvl0\ls4 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Roman five}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \u8226\'95\tab}\ilvl0\ls3 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Bullet one}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \u8226\'95\tab}\ilvl0\ls3 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Bullet two}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \u9702\'3f\tab}\ilvl1\ls3 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch+Nested bullet}+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch+Table}+\par \trowd\trql\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\cellx6425\cellx9638\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch+Wide head}\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch+End}\cell\row\pard \trowd\trql\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\clvmgf\cellx3213\cellx6425\cellx9638\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch+Tall}\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch+B2}\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch+C2}\cell\row\pard \trowd\trql\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\clvmrg\cellx3213\cellx6425\cellx9638\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar\loch+\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch+B3}\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch+C3}\cell\row\pard \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch+Notes and special text}+\par \ql\fi-340\li340\lin340\noline\ltrpar\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Music clef \uc0 \u55348\uc1 \u56606\'3f appears before this footnote}{{\super \loch \chftn{\*\footnote \chftn\pard\plain \s31\rtlch\afs20 \ltrch\loch\fi-340\li340\lin340\noline\fs20{\loch+\tab Footnote after an astral character.}+}}+}{\loch+ reference.}+\par \ql\fi-340\li340\lin340\noline\ltrpar\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+An endnote follows here}{{\super \loch \chftn{\*\footnote\ftnalt \chftn\pard\plain \s32\rtlch\afs20 \ltrch\loch\fi-340\li340\lin340\noline\fs20{\loch+\tab Endnote body text.}+}}+}{\loch+.}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Persian with ZWNJ: }{+\u1605\'3f\u1740\'3f\u8204\'3f\u1582\'3f\u1608\'3f\u1575\'3f\u1607\'3f\u1605\'3f}{\loch+. Family emoji: \uc0 \u55357\uc1 \u56424\'3f\u8205\'3f\uc0 \u55357\uc1 \u56425\'3f\u8205\'3f\uc0 \u55357\uc1 \u56423\'3f.}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Markdown specials: *stars* _under_ [bracket] `tick` #hash 1. dotted | pipe.}+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch+Links and anchors}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+External link to }{{\field{\*\fldinst HYPERLINK "https://example.com/page" }{\fldrslt {\loch\loch\cf9\ul\ulc0\loch+example}{}}}\loch+.}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Relative link to }{{\field{\*\fldinst HYPERLINK "file:///anydoc/tests/fixture-src/sibling.odt" }{\fldrslt {\loch\loch\cf9\ul\ulc0\loch+a sibling file}{}}}\loch+.}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+{\*\bkmkstart plainmark}{\*\bkmkend plainmark}This plain paragraph carries a bookmark.}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Jump to }{{\field{\*\fldinst HYPERLINK "#plainmark" }{\fldrslt {\loch\loch\cf9\ul\ulc0\loch+the bookmarked paragraph}{}}}\loch+.}+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch+Objects}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Inline image: }{\loch+{\pict{\*\picprop{\sp{\sn wzDescription}{\sv tiny red dot}}{\sp{\sn wzName}{\sv }}}\picscalex1700\picscaley1700\piccropl0\piccropr0\piccropt0\piccropb0\picw1\pich1\picwgoal10\pichgoal10\pngblip+89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d4944415478da63fccfc0500f000485018084a98c21000000004945+4e44ae426082}+}{\loch+ done.}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Text box: }{+{\shp{\*\shpinst\shpwr2\shpwrk0\shpbypara\shpbyignore\shptop-1362\shpbottom-228\shpbxcolumn\shpbxignore\shpleft0\shpright2835\shpz1{\sp{\sn shapeType}{\sv 202}}{\sp{\sn dxWrapDistLeft}{\sv 72390}}{\sp{\sn dxWrapDistRight}{\sv 72390}}{\sp{\sn dyWrapDistTop}{\sv 72390}}{\sp{\sn dyWrapDistBottom}{\sv 72390}}{\sp{\sn posrelv}{\sv 2}}{\sp{\sn posv}{\sv 1}}{\sp{\sn posrelh}{\sv 2}}{\sp{\sn posh}{\sv 2}}{\sp{\sn dxTextLeft}{\sv 53975}}{\sp{\sn dyTextTop}{\sv 53975}}{\sp{\sn dxTextRight}{\sv 53975}}{\sp{\sn dyTextBottom}{\sv 53975}}{\sp{\sn lineColor}{\sv 0}}{\sp{\sn lineWidth}{\sv 635}}{\shptxt\s33\ql{\loch+Inside the text box.}+\par \pard}}}+}{\loch+ after the box.}+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch+Quote and code}+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch+Value below one millionth: 0.0000004 should survive.}+\par }\ No newline at end of filediff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/xls/sheet.xls b/internal/infrastructure/docparser/anydoc/testdata/upstream/xls/sheet.xlsnew file mode 100644index 000000000..6d830f15aBinary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/xls/sheet.xls differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsb/handmade-sheet.xlsb b/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsb/handmade-sheet.xlsbnew file mode 100644index 000000000..e69c29451Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsb/handmade-sheet.xlsb differdiff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsx/sheet.xlsx b/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsx/sheet.xlsxnew file mode 100644index 000000000..44dba2b51Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsx/sheet.xlsx differdiff --git a/internal/infrastructure/docparser/anydoc/upgrade_linked_test.go b/internal/infrastructure/docparser/anydoc/upgrade_linked_test.gonew file mode 100644index 000000000..d5eaa2a0e--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/upgrade_linked_test.go@@ -0,0 +1,225 @@+//go:build anydoc && cgo++package anydoc++import (+ "archive/zip"+ "bytes"+ "errors"+ "fmt"+ "reflect"+ "strings"+ "testing"++ upstream "github.com/firecrawl/anydoc/go"+)++func TestConvertEquations(t *testing.T) {+ for _, format := range []string{"docx", "pptx", "odt", "epub", "rtf"} {+ t.Run(format, func(t *testing.T) {+ data := readFixture(t, format+"/handmade-math."+format)+ for _, assets := range []bool{false, true} {+ result, err := Convert(data, Options{Format: format, WithAssets: assets})+ if err != nil {+ t.Fatal(err)+ }+ if result.AssetsError != nil {+ t.Fatalf("document ABI failed: %v", result.AssetsError)+ }+ if !strings.Contains(result.Markdown, `\frac`) ||+ !strings.Contains(result.Markdown, "$") {+ t.Fatalf("equation lost (assets=%v): %s", assets, result.Markdown)+ }+ }+ })+ }+ // Check the actual decoded fields, not only the independent Markdown path.+ doc, err := upstream.ToDocument(readFixture(t, "docx/handmade-math.docx"), nil)+ if err != nil {+ t.Fatal(err)+ }+ var inline, block bool+ for _, b := range doc.Blocks {+ block = block || b.Kind == "math" && b.Text != nil && strings.Contains(*b.Text, `\sum`)+ for _, in := range b.Content {+ inline = inline || in.Kind == "math" && in.Text != nil && strings.Contains(*in.Text, `\frac`)+ }+ }+ if !inline || !block {+ t.Fatalf("decoded equation nodes missing: inline=%v block=%v", inline, block)+ }+}++func TestConvertPDFReportsScannedPages(t *testing.T) {+ for _, tc := range []struct{ file, detail string }{+ {"handmade-mixed.pdf", "page 2 of 2 needs OCR"},+ {"handmade-scanned.pdf", "all 2 pages need OCR"},+ } {+ t.Run(tc.file, func(t *testing.T) {+ result, err := Convert(readFixture(t, "pdf/"+tc.file), Options{Format: "pdf", WithAssets: true})+ var typed *upstream.ConvertError+ if result != nil ||+ !errors.As(err, &typed) ||+ typed.Kind != "needs_ocr" ||+ !strings.Contains(typed.Detail, tc.detail) {+ t.Fatalf("want scanned-page error, got result=%+v err=%v", result, err)+ }+ if !PDFNeedsOCR(fmt.Errorf("reader: %w", err)) {+ t.Fatal("wrapped typed error did not trigger OCR fallback")+ }+ })+ }+ if !PDFNeedsOCR(&upstream.ConvertError{Kind: "needs_ocr", Detail: "localized detail"}) {+ t.Fatal("OCR detection still depends on English wording")+ }+ if PDFNeedsOCR(&upstream.ConvertError{Kind: "malformed", Detail: "invalid PDF"}) {+ t.Fatal("malformed PDF incorrectly triggers OCR")+ }+}++func TestConvertSpreadsheetCheckboxes(t *testing.T) {+ // Add VML form controls to an upstream workbook. Both boolean states must+ // survive Rust encoding and Go decoding, including subsequent table cells.+ data := readFixture(t, "xlsx/sheet.xlsx")+ reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))+ if err != nil {+ t.Fatal(err)+ }+ var out bytes.Buffer+ w := zip.NewWriter(&out)+ for _, f := range reader.File {+ if f.Name == "xl/worksheets/_rels/sheet1.xml.rels" {+ continue+ }+ if err := w.Copy(f); err != nil {+ t.Fatal(err)+ }+ }+ parts := map[string]string{+ "xl/worksheets/_rels/sheet1.xml.rels": `<Relationships ` ++ `xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><` ++ `Relationship Id="checkboxes" ` ++ `Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/vmlDrawing"` ++ ` Target="../drawings/checkboxes.vml"/></Relationships>`,+ "xl/drawings/checkboxes.vml": `<xml xmlns:v="urn:schemas-microsoft-com:vml" ` ++ `xmlns:x="urn:schemas-microsoft-com:office:excel"><v:shape><v:textbox>Approved</v:text` ++ `box><x:ClientData ObjectType="Checkbox"><x:Anchor>0,0,0,0,1,0,1,0</x:Anchor><x:Checke` ++ `d>1</x:Checked></x:ClientData></v:shape><v:shape><v:textbox>Pending</v:textbox><x:Cli` ++ `entData ObjectType="Checkbox"><x:Anchor>1,0,0,0,2,0,1,0</x:Anchor></x:ClientData></v:` ++ `shape></xml>`,+ }+ for name, content := range parts {+ part, err := w.Create(name)+ if err != nil {+ t.Fatal(err)+ }+ if _, err := part.Write([]byte(content)); err != nil {+ t.Fatal(err)+ }+ }+ if err := w.Close(); err != nil {+ t.Fatal(err)+ }+ result, err := Convert(out.Bytes(), Options{Format: "xlsx", WithAssets: true})+ if err != nil {+ t.Fatal(err)+ }+ if result.AssetsError != nil ||+ !strings.Contains(result.Markdown, "[x] Approved") ||+ !strings.Contains(result.Markdown, "[ ] Pending") {+ t.Fatalf("checkbox conversion: %+v", result)+ }+ doc, err := upstream.ToDocument(out.Bytes(), nil)+ if err != nil {+ t.Fatal(err)+ }+ states := map[bool]bool{}+ for _, block := range doc.Blocks {+ if block.Table == nil {+ continue+ }+ for _, row := range block.Table.Grid {+ for _, slot := range row {+ if slot.Cell == nil {+ continue+ }+ for _, b := range slot.Cell.Blocks {+ for _, in := range b.Content {+ if in.Kind == "checkbox" && in.Checked != nil {+ states[*in.Checked] = true+ }+ }+ }+ }+ }+ }+ if !states[false] || !states[true] {+ t.Fatalf("decoded checkbox states: %v", states)+ }+}++func TestConvertAllBenchmarkFormats(t *testing.T) {+ for _, name := range benchmarkDocuments {+ t.Run(name, func(t *testing.T) {+ format := strings.Split(name, "/")[0]+ if format == "xls" || format == "xlsb" {+ format = "xlsx"+ }+ result, err := Convert(readFixture(t, name), Options{Format: format, WithAssets: true})+ if err != nil || result.Markdown == "" || result.AssetsError != nil {+ t.Fatalf("conversion: result=%+v err=%v", result, err)+ }+ })+ }+}++func TestCombinedConversionPreservesModelAndMarkdown(t *testing.T) {+ // Exercise auto-detection as well as all normal formats, without requiring+ // an external upstream checkout for this regression in CI.+ for _, name := range benchmarkDocuments {+ if strings.HasPrefix(name, "pdf/") {+ continue+ }+ t.Run(name, func(t *testing.T) {+ data := readFixture(t, name)+ var format *upstream.Format+ if strings.HasPrefix(name, "csv/") {+ f := upstream.FormatCsv+ format = &f+ }+ want, err := upstream.ToDocument(data, format)+ if err != nil {+ t.Fatal(err)+ }+ wantMarkdown, err := upstream.ToMarkdownWithAssetLinks(data, format)+ if err != nil {+ t.Fatal(err)+ }+ got, markdown, err := upstream.ToDocumentWithAssetLinks(data, format)+ if err != nil {+ t.Fatal(err)+ }+ if !reflect.DeepEqual(got, want) || markdown != wantMarkdown {+ t.Fatal("single parse changed model or Markdown")+ }+ })+ }+ for _, tc := range []struct {+ name string+ data []byte+ format *upstream.Format+ kind string+ }{+ {"empty", nil, nil, "unsupported"},+ {"garbage", []byte("garbage"), nil, "unsupported"},+ {"pdf", minimalPDF(), nil, "pdf_no_model"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ doc, md, err := upstream.ToDocumentWithAssetLinks(tc.data, tc.format)+ var typed *upstream.ConvertError+ if doc != nil || md != "" || !errors.As(err, &typed) || typed.Kind != tc.kind {+ t.Fatalf("error contract: doc=%+v md=%q err=%v", doc, md, err)+ }+ })+ }+}diff --git a/internal/infrastructure/docparser/anydoc_reader.go b/internal/infrastructure/docparser/anydoc_reader.goindex 10a38e472..8ae7bf01a 100644--- a/internal/infrastructure/docparser/anydoc_reader.go+++ b/internal/infrastructure/docparser/anydoc_reader.go@@ -89,7 +89,8 @@ func (r *AnydocReader) readScannedPDF(ctx context.Context, req *types.ReadRequesreturn nil, fmt.Errorf("anydoc conversion failed for %q: PDF has no extractable text; OCR is required", req.FileName)}- logger.Infof(ctx, "[anydoc] %q has no text layer, falling back to builtin for scanned-PDF OCR", req.FileName)+ logger.Infof(ctx, "[anydoc] %q contains pages requiring OCR, falling back to builtin for scanned-PDF OCR",+ req.FileName)fallbackReq := *reqfallbackReq.ParserEngine = BuiltinEngineNameresult, err := r.fallback.Read(ctx, &fallbackReq)diff --git a/internal/infrastructure/docparser/anydoc_reader_linked_test.go b/internal/infrastructure/docparser/anydoc_reader_linked_test.gonew file mode 100644index 000000000..ad3c6efb2--- /dev/null+++ b/internal/infrastructure/docparser/anydoc_reader_linked_test.go@@ -0,0 +1,35 @@+//go:build anydoc && cgo++package docparser++import (+ "bytes"+ "context"+ "os"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+)++func TestAnydocReaderMixedPDFFallsBackThroughRead(t *testing.T) {+ data, err := os.ReadFile("anydoc/testdata/upstream/pdf/handmade-mixed.pdf")+ if err != nil {+ t.Fatal(err)+ }+ fallback := &stubDocReader{result: &types.ReadResult{MarkdownContent: "Both pages including OCR"}}+ req := &types.ReadRequest{FileName: "mixed.pdf", FileType: "pdf", FileContent: data}+ result, err := NewAnydocReader(nil, fallback).Read(context.Background(), req)+ if err != nil {+ t.Fatal(err)+ }+ if fallback.got == nil ||+ !bytes.Equal(fallback.got.FileContent, req.FileContent) ||+ fallback.got.ParserEngine != BuiltinEngineName ||+ result.MarkdownContent != "Both pages including OCR" ||+ result.Metadata["anydoc_fallback"] != "scanned_pdf" {+ t.Fatalf("mixed PDF did not reach fallback intact: %+v", result)+ }+ if result, err := NewAnydocReader(nil, nil).Read(context.Background(), req); err == nil || result != nil {+ t.Fatalf("mixed PDF without OCR returned partial success: result=%+v err=%v", result, err)+ }+}diff --git a/third_party/anydoc-go/Cargo.lock b/third_party/anydoc-go/Cargo.lockindex b33122f56..1dd6acad7 100644--- a/third_party/anydoc-go/Cargo.lock+++ b/third_party/anydoc-go/Cargo.lock@@ -89,9 +89,8 @@ dependencies = [[[package]]name = "anydoc"-version = "0.1.9"+version = "0.2.4"dependencies = [- "calamine","cfb","csv","encoding_rs",@@ -104,22 +103,12 @@ dependencies = [[[package]]name = "anydoc-go"-version = "0.1.9"+version = "0.2.4"dependencies = ["anydoc","cbindgen",]-[[package]]-name = "atoi_simd"-version = "0.18.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44"-dependencies = [- "debug_unsafe",- "rustversion",-]-[[package]]name = "autocfg"version = "1.5.1"@@ -162,30 +151,6 @@ version = "3.20.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"-[[package]]-name = "byteorder"-version = "1.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"--[[package]]-name = "calamine"-version = "0.36.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3"-dependencies = [- "atoi_simd",- "byteorder",- "chrono",- "codepage",- "encoding_rs",- "fast-float2",- "log",- "quick-xml",- "serde",- "zip",-]-[[package]]name = "cbc"version = "0.1.2"@@ -243,9 +208,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"[[package]]name = "chacha20"-version = "0.10.1"+version = "0.10.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"+checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"dependencies = ["cfg-if","cpufeatures 0.3.0",@@ -260,7 +225,6 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"dependencies = ["iana-time-zone","num-traits",- "serde","windows-link",]@@ -301,15 +265,6 @@ version = "1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"-[[package]]-name = "codepage"-version = "0.1.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4"-dependencies = [- "encoding_rs",-]-[[package]]name = "colorchoice"version = "1.0.5"@@ -405,12 +360,6 @@ dependencies = ["memchr",]-[[package]]-name = "debug_unsafe"-version = "0.1.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2"-[[package]]name = "defmt"version = "1.1.1"@@ -521,12 +470,6 @@ dependencies = ["windows-sys",]-[[package]]-name = "fast-float2"-version = "0.2.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"-[[package]]name = "fastrand"version = "2.5.0"@@ -926,7 +869,6 @@ version = "0.41.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"dependencies = [- "encoding_rs","memchr",]diff --git a/third_party/anydoc-go/Cargo.toml b/third_party/anydoc-go/Cargo.tomlindex 945ef5ff8..a3e996023 100644--- a/third_party/anydoc-go/Cargo.toml+++ b/third_party/anydoc-go/Cargo.toml@@ -5,7 +5,7 @@# symbols from, not a loadable shared object.[package]name = "anydoc-go"-version = "0.1.9"+version = "0.2.4"edition = "2024"description = "Go bindings (C ABI) for anydoc"license = "MIT"@@ -19,7 +19,7 @@ crate-type = ["staticlib"]# workspace, so it takes the published crate rather than a path dependency.# Pinned exactly: the C ABI mirrors this version's error and format enums.[dependencies]-anydoc = "=0.1.9"+anydoc = "=0.2.4"[build-dependencies]cbindgen = "0.29"diff --git a/third_party/anydoc-go/README.md b/third_party/anydoc-go/README.mdindex 7448e0254..953d3c64c 100644--- a/third_party/anydoc-go/README.md+++ b/third_party/anydoc-go/README.md@@ -18,9 +18,11 @@ not exist yet. So the binding source lives here and `go.mod` points at it:replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go```-When the upstream module is published, delete this directory, drop the-`replace`, and require the published version. Nothing else in WeKnora changes:-only `internal/infrastructure/docparser/anydoc/backend_cgo.go` imports it.+When an upstream module is published, first verify that it includes the local+safety fixes, the 0.2.x model/error additions, and the asset-link rendering API+listed below. Only then replace this directory and the `replace` directive.+WeKnora's backend calls our additional `ToDocumentWithAssetLinks` API, so the+original PR cannot replace this fork unchanged.## Provenance@@ -29,7 +31,9 @@ only `internal/infrastructure/docparser/anydoc/backend_cgo.go` imports it.| Upstream PR | firecrawl/anydoc#30 ("feat: add Go bindings") || PR head | `1a7a6c0` || Rebased onto | `4e3089b` (`chore: release v0.1.8`) |-| anydoc crate | `0.1.9` (from crates.io, pinned with `=`) |+| anydoc crate | `0.2.4` (from crates.io, pinned with `=`) |+| Upstream release commit | `42bf1c5ecdde9eb0d96d6bd75a9e6698cf93b14c` |+| Last upstream check | 2026-09-29: PR #30 is open, unmerged, and conflicting; head unchanged || License | MIT (see LICENSE) |The PR branched before anydoc 0.1.7, so it was rebased onto v0.1.8 before@@ -48,7 +52,27 @@ unchanged, and none of the local modifications below had to be re-applied. What1.14.2 (the jump in the number is that project unifying its Rust, Python, andNode versions, not fourteen major releases). See "Why 0.1.9 matters here" below.-## Why 0.1.9 matters here+## Current upgrade: 0.2.4++The [0.1.9 to 0.2.4 source diff](https://github.com/firecrawl/anydoc/compare/v0.1.9...v0.2.4)+changes the document model and parsing behavior:++- 0.2.0 replaces calamine with native XLS/XLSX/XLSB readers and improves number+ formatting, merged cells, DOCX whitespace, EPUB repeated spine parts, and+ Markdown escaping.+- 0.2.1/0.2.2 preserve equations as inline/block LaTeX and refine dollar escaping.+- 0.2.3 adds inline checkboxes and spreadsheet form controls. Task-list state+ moves from `ListItem.checked` to `Inline::Checkbox`.+- 0.2.4 returns `NeedsOcr` for PDFs containing scanned pages, including mixed+ PDFs that previously returned incomplete text. The Go ABI now has a distinct+ `needs_ocr` error; page numbers remain in `ConvertError.Detail`. `NeedsOCR()`+ lets WeKnora recognize wrapped errors without matching English messages.++Node/Python/CLI hosted OCR is outside the Rust crate. WeKnora continues using+its existing builtin docreader fallback and does not send documents to Firecrawl.+The PDF default engine remains builtin. See [upgrade verification and benchmarks](UPGRADE-0.2.4.md).++## Previous upgrade: why 0.1.9 matters hereanydoc's changelog lists 0.1.9 as a single dependency bump, which reads likehousekeeping. It is nine `pdf-inspector` fixes, and seven of them are@@ -106,7 +130,7 @@ required" and still fall back to the docreader.Keep this list current: it is the diff a future upgrade has to re-apply. Items2–4 are bugs in the upstream PR and are worth sending back to it.-1. `Cargo.toml` — depends on the published `anydoc = "=0.1.9"` crate instead of+1. `Cargo.toml` — depends on the published `anydoc = "=0.2.4"` crate instead ofthe workspace path dependency, declares its own empty `[workspace]`, andrepeats the upstream release profile (`lto`, `strip`), which it wouldotherwise inherit from the anydoc workspace.@@ -124,7 +148,7 @@ Keep this list current: it is the diff a future upgrade has to re-apply. Itemsstraight from the buffer is only trustworthy while the Rust encoder and thisdecoder agree; on a skew, `make([]Block, 0, n)` would exhaust memory beforethe first bounds check, turning a version mismatch into a dead process.-4. `src/lib.rs` — the three conversion entry points run inside `guarded()`,+4. `src/lib.rs` — conversion entry points run inside `guarded()`,which catches a panic and reports it as a malformed document. A panicescaping an `extern "C"` function aborts the process, and WeKnora parsesuntrusted uploads in the same process that serves the API. Note the limit:@@ -136,11 +160,26 @@ Keep this list current: it is the diff a future upgrade has to re-apply. Items6. `scripts/build-anydoc-lib.sh` copies the pinned anydoc release to`patched-anydoc/` (gitignored) and re-exports `document_to_markdown`, whichis crate-private in the published crate. It reads the version from the- `anydoc = "=X.Y.Z"` pin above and checks it against `version.go`, so a bump- is one line and cannot half-land.+ `anydoc = "=X.Y.Z"` pin above and checks it against `version.go`, so a+ dependency bump cannot silently leave the reported version behind.`src/asset_links.rs` then rewrites `ImageSource::Asset`to `External("images/image-N.ext")` so the official serializer emits in-placeimage links. `anydoc_to_markdown_with_asset_links` is the ABI for that path.+7. `src/model.rs`, `model.go`, and the generated header carry block/inline math+ and inline checkbox nodes. Existing wire tags are unchanged. The legacy+ `ListItem.Checked` slot is retained as nil for compatibility; callers read+ `Inline.Checked`. Asset traversal accepts math blocks. Heading text retains+ math and checkbox content when assigning image sections.+8. `src/lib.rs` and `errors.go` add `ERR_NEEDS_OCR` / `needs_ocr` and `NeedsOCR()`.+ Page numbers are preserved in the detail string; structured `pages` and+ `page_count` fields are not exposed by this Go ABI.+9. `anydoc_to_document_with_asset_links` / `ToDocumentWithAssetLinks` parse once,+ encode the original document (preserving asset IDs), then rewrite image+ sources and append the official Markdown to the same owned buffer. The+ backend uses this combined API to avoid parsing each uploaded document twice.+ The standalone APIs remain available. Corpus tests compare both paths for+ identical document trees and Markdown; all allocations use the existing+ `anydoc_buffer_free` ownership contract.## Dependency pinning and audit@@ -158,10 +197,17 @@ and the same input comes back as an ordinary error;Moving the pin to 0.1.9 added no crate and removed none: `lopdf` stays at 0.42and the whole tree below `pdf-inspector` is unchanged, so the lockfile diff is-three version lines and one checksum. The audit result is therefore the same-one described below.+three version lines and one checksum. That was the 0.1.9 dependency-only update;+the 0.2.4 changes are described below.++The 0.2.4 lockfile removes calamine and five exclusive dependencies, retains+`pdf-inspector` 1.14.2 and `lopdf` 0.42.0, and updates `chacha20` from the yanked+0.10.1 to 0.10.2. The latter fixes an SSE4.1 intrinsic used by the SSE2 backend+([upstream fix](https://github.com/RustCrypto/stream-ciphers/pull/580)); other+transitive versions are retained.-`cargo audit` currently reports one allowed warning: `ttf-parser` 0.25.1 is+`cargo audit` on 2026-09-29 reports no vulnerabilities and one allowed warning:+`ttf-parser` 0.25.1 isunmaintained (`RUSTSEC-2026-0192`), pulled in transitively by the PDF stack. Itis not a vulnerability and nothing here can fix it, so warnings report withoutfailing the job.@@ -174,8 +220,8 @@ renderer (`document_to_markdown`) is also crate-private. WeKnora keeps thatserializer: `scripts/build-anydoc-lib.sh` re-exports the one function, then`anydoc_to_markdown_with_asset_links` rewrites `Asset` images to`ImageSource::External("images/image-N.ext")` so the official GFM output-places them in reading order. PDF has no document model; scanned pages fall-back to the builtin docreader so they can be rasterized for OCR.+places them in reading order. PDF has no document model; PDFs containing scanned+pages fall back to the builtin docreader so they can be rasterized for OCR.## Building the archivediff --git a/third_party/anydoc-go/UPGRADE-0.2.4.md b/third_party/anydoc-go/UPGRADE-0.2.4.mdnew file mode 100644index 000000000..e27da4865--- /dev/null+++ b/third_party/anydoc-go/UPGRADE-0.2.4.md@@ -0,0 +1,155 @@+# AnyDoc 0.2.4 升级核查与压测++核查日期:2026-09-29。项目基线:`9114e4e4f905be71976a77c684d3f92731e6f9a6`。++## 结论++从 0.1.9 跟进到 0.2.4,并继续维护 vendored Go binding。升级重点是文档完整性:公式、复选框及混合 PDF 的扫描页不再被静默丢弃。++- [Go binding PR #30](https://github.com/firecrawl/anydoc/pull/30) 仍为 open,`merged=false`、`mergeable=false`,head 为 `1a7a6c04ff8f2689bf8ebbd1e0d9d61d2d109164`,最后更新时间为 2026-08-08。没有 `go/v*` 发布 tag。+- 最新 release tag 为 [v0.2.4](https://github.com/firecrawl/anydoc/tree/v0.2.4),2026-08-27 发布,commit `42bf1c5ecdde9eb0d96d6bd75a9e6698cf93b14c`。main 在该 tag 后的三次提交只改 README。+- 已逐文件确认下载的 crate 源码与 v0.2.4 tag 一致,仅保留原有 renderer re-export 补丁。+- 检查了 [v0.1.9…v0.2.4 完整源码差异](https://github.com/firecrawl/anydoc/compare/v0.1.9...v0.2.4),包括模型、序列化、PDF 路径、Excel 解析器和 bindings;不是仅根据版本说明判断。++## 核心变化与本项目处理++| 上游变化 | 影响与处理 |+| --- | --- |+| 0.2.0:移除 calamine,自行解析 XLS/XLSX/XLSB;改进数值显示、合并单元格与隐藏内容处理 | 保留已有格式路由,回归各容器。上游 XLSB 样例在旧版失败,新版成功;这不代表应用新增了 XLSB 上传入口。 |+| 0.2.0:DOCX run 边缘空格、重复 EPUB spine、Markdown 分隔符及表格代码中的竖线修复 | 使用上游 66 个快照,经实际 Go ABI 比较 Markdown 与错误详情。 |+| 0.2.1–0.2.2:OMML/MathML/RTF 公式转 LaTeX,美元符号转义修正 | 新增 block/inline math ABI tag、编码及解码;图片路径接受公式节点。回归 DOCX/PPTX/ODT/EPUB/RTF。 |+| 0.2.3:复选框成为 inline 节点,移除 Rust `ListItem.checked` | 新增 checkbox tag 与 bool 字段。旧 wire slot 保留为 nil;Go 的旧字段标为 deprecated。实测有标题的选中、未选中控件。 |+| 0.2.4:混合或扫描 PDF 返回 `NeedsOcr`,错误标出页码 | 新增独立错误码及 `NeedsOCR()`,通过 `errors.As` 识别包装错误,实际 Reader 测试验证回退与无 OCR 时的失败行为。页码保留在 Detail,未新增结构化页码 API。 |+| Node/Python/CLI 可选择 Firecrawl hosted OCR | Rust 不含该功能;本项目继续使用原有 builtin/docreader 回退。 |++本地线程绑定、panic 捕获、解码分配上限和图片链接 API 均保留。+本次还消除了图片提取路径的重复解析:新的 `ToDocumentWithAssetLinks` 一次解析返回原始文档树和图片 Markdown,复用单一 buffer 的分配/释放协议。先编码原始 asset ID,再重写图片链接,避免丢失图片位置与章节信息。上游语料和内置 13 种非 PDF 样例均验证合并接口与两个独立接口的输出一致。不能在 PR 合入后直接删除本地版本:还需确认上游补齐这些能力。详细差异见 [README](README.md)。++依赖锁保持 `pdf-inspector=1.14.2`、`lopdf=0.42.0`,移除 calamine 及五个专属依赖。审计发现原有 `chacha20=0.10.1` 已撤回,按[上游修复 #580](https://github.com/RustCrypto/stream-ciphers/pull/580) 更新到 0.10.2,修复 SSE2 后端误用 SSE4.1 指令的问题;其他传递依赖版本未更新。++## 已完成验证++- `scripts/build-anydoc-lib.sh`:release archive 构建成功,使用 `--locked`;C header 已同步生成。+- `ANYDOC_UPSTREAM_DIR=... go test -tags anydoc -count=1 ./internal/infrastructure/docparser/...`:通过,包括 66 个上游快照、正常文档的 Go 模型解码及图片 Markdown 路径。空输入保留 Go binding 原有的前置错误,单独断言。+- `go test -race -tags anydoc -count=1 ./internal/infrastructure/docparser/...`:通过。Go race 检测不覆盖 Rust 内存访问。+- `go test -count=1 ./internal/infrastructure/docparser/...`:未链接引擎路径通过。+- `go vet -tags anydoc ./internal/infrastructure/docparser/...`:通过。+- `go build -tags anydoc -o /tmp/weknora-anydoc-server ./cmd/server`:完整服务编译通过。+- 既有 4,000 次并发错误详情、恶意 PDF 深层嵌套和 TJ lookback 回归:通过。+- `cargo audit --file third_party/anydoc-go/Cargo.lock`:无已知漏洞;保留 `ttf-parser 0.25.1` 的停止维护警告 `RUSTSEC-2026-0192`。++## 性能测量方法++Apple M4 Pro,64 GiB RAM,macOS arm64;Go 1.26.3、Rust 1.97.1,Rust release + thin LTO。+使用相同的 Go benchmark 和上游固定样例分别链接 0.1.9 与 0.2.4,排除文件读取和编译时间。+`WithAssets=true` 覆盖生产 Reader 默认路径:Rust 解析、Go 文档解码、图片收集与官方 Markdown 输出。旧版/直接升级版解析两次,最终版解析一次。PDF 仍走文本路径。+每项 500 ms × 3 次,`GOMAXPROCS=1,4`;表中给出三次中位数。四并发的 ns/op 是总耗时除以总次数,表示吞吐能力,不是单个请求延迟。`B/op` 只统计 Go 堆,不能代表 Rust 内存。+编译和其他测试结束后顺序测量;这是本机小样本比较,不声称统计显著,也不等价于生产端到端性能。++### 同机对比:单并发++单位为 µs/op,越低越好;变化率相对 0.1.9。++| 格式 | 0.1.9 | 0.2.4 直接升级 | 0.2.4 最终版 | 最终变化 |+| --- | ---: | ---: | ---: | ---: |+| doc | 228.7 | 237.7 | 147.2 | -35.6% |+| docx | 1040.3 | 1082.4 | 559.8 | -46.2% |+| ppt | 60.0 | 62.7 | 37.0 | -38.4% |+| pptx | 887.2 | 895.4 | 450.3 | -49.2% |+| odt | 719.5 | 727.5 | 390.7 | -45.7% |+| ods | 478.2 | 483.2 | 259.3 | -45.8% |+| odp | 1157.2 | 1167.6 | 588.2 | -49.2% |+| rtf | 742.8 | 757.4 | 407.2 | -45.2% |+| epub | 313.4 | 314.5 | 174.9 | -44.2% |+| xls | 69.8 | 127.9 | 76.1 | +9.0% |+| xlsx | 217.7 | 500.3 | 261.2 | +20.0% |+| xlsb | 转换失败 | 84.3 | 48.7 | 新增可用样例 |+| csv | 155.9 | 161.2 | 90.1 | -42.2% |+| pdf | 1941.6 | 3605.4 | 3631.9 | +87.1% |++### 四并发吞吐++单位为 µs/op(每次转换分摊的墙钟时间,非请求延迟)。++| 格式 | 0.1.9 | 最终版 | 变化 |+| --- | ---: | ---: | ---: |+| doc | 53.8 | 36.6 | -32.0% |+| docx | 268.3 | 142.5 | -46.9% |+| ppt | 15.9 | 9.8 | -38.6% |+| pptx | 226.5 | 115.0 | -49.2% |+| odt | 179.6 | 99.8 | -44.4% |+| ods | 120.5 | 64.1 | -46.8% |+| odp | 297.9 | 150.2 | -49.6% |+| rtf | 187.9 | 104.3 | -44.5% |+| epub | 78.7 | 43.6 | -44.6% |+| xls | 19.1 | 19.8 | +3.5% |+| xlsx | 52.3 | 65.6 | +25.4% |+| xlsb | 转换失败 | 12.5 | 新增可用样例 |+| csv | 38.3 | 22.6 | -41.0% |+| pdf | 577.5 | 1060.4 | +83.6% |++### 性能判断++- 消除重复解析后,DOC/DOCX/PPT/PPTX/ODT/ODS/ODP/RTF/EPUB 的单并发样例耗时降低约 36%–49%,CSV 降低约 42%。+- XLS 从 69.8 µs 到 76.1 µs(约 +9%),XLSX 从 217.7 µs 到 261.2 µs(约 +20%)。新解析器的数值/布局/控件处理仍有额外代价;不能宣称所有格式加速。+- PDF 从 1.94 ms 到 3.63 ms(约 +87%)。源码新增对被标记扫描页的二次提取确认,以避免丢页和检测误报。这个路径不使用文档模型,合并解析优化不作用于它。保留完整性检查;应用默认 PDF 引擎仍是 builtin,只有选择 AnyDoc 的 PDF 路径受影响。+- Go 模型新增一个 checkbox 指针字段,Go 分配字节略有增加;原始输出保留了 B/op 和 allocs/op。++### 持续混合负载++8 workers、14 种格式轮换、30 秒,逐次检查 Markdown 与图片字节/名称一致。++| 指标 | 直接升级 | 最终版 |+| --- | ---: | ---: |+| 完成转换 | 231,291 | 306,252 |+| 吞吐 | 7,708.5 docs/s | 10,207.1 docs/s |+| p50 | 0.644 ms | 0.359 ms |+| p95 | 5.320 ms | 5.491 ms |+| p99 | 6.684 ms | 6.843 ms |+| 进程峰值 RSS(含 Rust) | 61.1 MiB | 63.3 MiB |++两轮均零错误、无输出或图片不一致。最终吞吐比直接升级提高约 32%;尾延迟没有改善,PDF 占据较长执行时间。RSS 包含测试保存的延迟样本,因此不能简单归因于解析器。该测试均匀轮换小文档,不能外推为大文档生产容量。++### 输入规模测试++| 输入 | 中位耗时 | Go B/op |+| --- | ---: | ---: |+| csv_100_rows | 0.398 ms | 129,360 |+| csv_10000_rows | 32.133 ms | 12,581,816 |+| pdf_40000_operators | 1.074 ms | 152 |+| pdf_200000_operators | 4.346 ms | 152 |+| xlsx_100_rows | 0.592 ms | 82,624 |+| xlsx_10000_rows | 46.186 ms | 7,946,442 |++CSV 为三列,XLSX 为两列;每种格式内按相同行结构扩展,生成及压缩输入不计入转换耗时。万行 XLSX 约 46.2 ms,校验最后一行未丢失。PDF 的裸 TJ token 从 4 万增到 20 万(5 倍输入),耗时约增至 4 倍,没有重现历史平方级退化。Go 分配不包括 Rust 原生堆。XLSX 原始结果见 [规模压测](benchmarks/2026-09-29/xlsx-scaling.txt)。++原始数据:[0.1.9](benchmarks/2026-09-29/baseline.txt)、[直接升级](benchmarks/2026-09-29/upgrade.txt)、[最终版](benchmarks/2026-09-29/optimized.txt)、[直接升级压力](benchmarks/2026-09-29/upgrade-stress.txt)、[最终版压力](benchmarks/2026-09-29/optimized-stress.txt)、[规模测试](benchmarks/2026-09-29/scaling.txt)、[审计](benchmarks/2026-09-29/audit.txt)。++## 复现++在仓库根目录执行功能核查:++```bash+scripts/build-anydoc-lib.sh+git clone --branch v0.2.4 --depth 1 https://github.com/firecrawl/anydoc.git /tmp/anydoc-v0.2.4+ANYDOC_UPSTREAM_DIR=/tmp/anydoc-v0.2.4 go test -tags anydoc -count=1 ./internal/infrastructure/docparser/...+go test -race -tags anydoc -count=1 ./internal/infrastructure/docparser/...+go vet -tags anydoc ./internal/infrastructure/docparser/...+cargo audit --file third_party/anydoc-go/Cargo.lock+```++性能与混合负载:++```bash+go test -tags anydoc -run '^$' -bench '^BenchmarkConvert$' -benchmem -benchtime=500ms -count=3 -cpu=1,4 ./internal/infrastructure/docparser/anydoc+go test -tags anydoc -run '^$' -bench '^BenchmarkConvertScaling$' -benchmem -benchtime=500ms -count=3 -cpu=1 ./internal/infrastructure/docparser/anydoc+go test -tags anydoc -c -o /tmp/anydoc.test ./internal/infrastructure/docparser/anydoc+cd internal/infrastructure/docparser/anydoc+ANYDOC_STRESS_DURATION=30s /usr/bin/time -l /tmp/anydoc.test -test.run '^TestConcurrentConversionStress$' -test.v+```++Linux 用 `/usr/bin/time -v` 记录 RSS。测试二进制需从包目录运行以读取样例。基线比较需要在旧版本单独编译同一 benchmark 和样例,再顺序运行两个二进制;旧版 XLSB 样例转换失败,基线应排除该项,不能把失败计入吞吐。++## 验证边界++本次运行平台为 macOS arm64,Linux glibc/musl、Windows 交叉构建未在本机执行。混合 PDF 测试使用 stub 验证回退请求和结果;未调用外部 OCR 服务、存储或检索索引。压测覆盖解析层,不含上传排队、网络与 OCR 耗时。峰值 RSS 和短时压力不能证明长期不存在原生内存泄漏。diff --git a/third_party/anydoc-go/anydoc.go b/third_party/anydoc-go/anydoc.goindex 5830e7583..646ed6a97 100644--- a/third_party/anydoc-go/anydoc.go+++ b/third_party/anydoc-go/anydoc.go@@ -165,6 +165,46 @@ func ToDocument(data []byte, format *Format) (*Document, error) {return decodeDocument(raw)}+// ToDocumentWithAssetLinks parses once and returns both the original model+// (including asset IDs and bytes) and Markdown with embedded image links.+// PDF has no document model; use ToMarkdownBytes for it.+func ToDocumentWithAssetLinks(data []byte, format *Format) (*Document, string, error) {+ if len(data) == 0 {+ return nil, "", &ConvertError{Kind: "unsupported", Detail: "empty input"}+ }+ tag := C.int(C.ANYDOC_FORMAT_NONE)+ if format != nil {+ tag = formatToTag(*format)+ if tag == C.int(C.ANYDOC_FORMAT_NONE) {+ return nil, "", &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)}+ }+ }+ var buf *C.uint8_t+ var bufLen C.uintptr_t+ if err := call(func() C.int {+ return C.anydoc_to_document_with_asset_links(+ (*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &buf, &bufLen,+ )+ }); err != nil {+ return nil, "", err+ }+ defer C.anydoc_buffer_free(buf, bufLen)+ raw, err := cGoBytes(unsafe.Pointer(buf), bufLen)+ if err != nil {+ return nil, "", err+ }+ d := &decoder{buf: raw}+ document, err := d.document()+ if err != nil {+ return nil, "", err+ }+ markdown, err := d.str()+ if err != nil {+ return nil, "", err+ }+ return document, markdown, nil+}+// cStringN reads a length-prefixed C buffer as a Go string. The bytes are// copied, so the caller may free the C buffer immediately after.func cStringN(s *C.char, n C.uintptr_t) (string, error) {diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/audit.txt b/third_party/anydoc-go/benchmarks/2026-09-29/audit.txtnew file mode 100644index 000000000..5cabc4a62--- /dev/null+++ b/third_party/anydoc-go/benchmarks/2026-09-29/audit.txt@@ -0,0 +1,13 @@+ Fetching advisory database from `https://github.com/RustSec/advisory-db.git`+ Loaded 1273 security advisories (from /Users/wizard/.cargo/advisory-db)+ Updating crates.io index+ Scanning third_party/anydoc-go/Cargo.lock for vulnerabilities (162 crate dependencies)+Crate: ttf-parser+Version: 0.25.1+Warning: unmaintained+Title: `ttf-parser` is unmaintained+Date: 2026-06-28+ID: RUSTSEC-2026-0192+URL: https://rustsec.org/advisories/RUSTSEC-2026-0192++warning: 1 allowed warning founddiff --git a/third_party/anydoc-go/benchmarks/2026-09-29/baseline.txt b/third_party/anydoc-go/benchmarks/2026-09-29/baseline.txtnew file mode 100644index 000000000..4b43068a1--- /dev/null+++ b/third_party/anydoc-go/benchmarks/2026-09-29/baseline.txt@@ -0,0 +1,83 @@+goos: darwin+goarch: arm64+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc+cpu: Apple M4 Pro+BenchmarkConvert/doc 2556 228702 ns/op 89.55 MB/s 22793 B/op 325 allocs/op+BenchmarkConvert/doc 2606 227909 ns/op 89.86 MB/s 22793 B/op 325 allocs/op+BenchmarkConvert/doc 2588 229236 ns/op 89.34 MB/s 22793 B/op 325 allocs/op+BenchmarkConvert/doc-4 9982 53926 ns/op 379.78 MB/s 22798 B/op 325 allocs/op+BenchmarkConvert/doc-4 10000 53771 ns/op 380.87 MB/s 22796 B/op 325 allocs/op+BenchmarkConvert/doc-4 10000 53729 ns/op 381.18 MB/s 22796 B/op 325 allocs/op+BenchmarkConvert/docx 572 1044082 ns/op 9.73 MB/s 23945 B/op 342 allocs/op+BenchmarkConvert/docx 574 1040160 ns/op 9.77 MB/s 23945 B/op 342 allocs/op+BenchmarkConvert/docx 578 1040338 ns/op 9.77 MB/s 23945 B/op 342 allocs/op+BenchmarkConvert/docx-4 2168 268537 ns/op 37.83 MB/s 23950 B/op 342 allocs/op+BenchmarkConvert/docx-4 2217 267350 ns/op 38.00 MB/s 23949 B/op 342 allocs/op+BenchmarkConvert/docx-4 2229 268327 ns/op 37.86 MB/s 23950 B/op 342 allocs/op+BenchmarkConvert/ppt 9592 60023 ns/op 7736.71 MB/s 5312 B/op 73 allocs/op+BenchmarkConvert/ppt 10000 59602 ns/op 7791.48 MB/s 5312 B/op 73 allocs/op+BenchmarkConvert/ppt 9822 60202 ns/op 7713.73 MB/s 5312 B/op 73 allocs/op+BenchmarkConvert/ppt-4 37984 15919 ns/op 29171.29 MB/s 5312 B/op 73 allocs/op+BenchmarkConvert/ppt-4 37519 16036 ns/op 28958.44 MB/s 5312 B/op 73 allocs/op+BenchmarkConvert/ppt-4 37756 15898 ns/op 29210.70 MB/s 5312 B/op 73 allocs/op+BenchmarkConvert/pptx 661 885527 ns/op 17.77 MB/s 6016 B/op 85 allocs/op+BenchmarkConvert/pptx 674 887205 ns/op 17.73 MB/s 6016 B/op 85 allocs/op+BenchmarkConvert/pptx 670 887945 ns/op 17.72 MB/s 6016 B/op 85 allocs/op+BenchmarkConvert/pptx-4 2632 226007 ns/op 69.62 MB/s 6016 B/op 85 allocs/op+BenchmarkConvert/pptx-4 2641 226665 ns/op 69.42 MB/s 6016 B/op 85 allocs/op+BenchmarkConvert/pptx-4 2631 226480 ns/op 69.47 MB/s 6016 B/op 85 allocs/op+BenchmarkConvert/odt 822 719914 ns/op 35.41 MB/s 23761 B/op 348 allocs/op+BenchmarkConvert/odt 806 719450 ns/op 35.43 MB/s 23761 B/op 348 allocs/op+BenchmarkConvert/odt 837 709504 ns/op 35.93 MB/s 23761 B/op 348 allocs/op+BenchmarkConvert/odt-4 3228 180369 ns/op 141.32 MB/s 23765 B/op 348 allocs/op+BenchmarkConvert/odt-4 3318 179551 ns/op 141.96 MB/s 23767 B/op 348 allocs/op+BenchmarkConvert/odt-4 3246 179267 ns/op 142.18 MB/s 23765 B/op 348 allocs/op+BenchmarkConvert/ods 1246 478194 ns/op 33.76 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/ods 1254 475562 ns/op 33.94 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/ods 1250 478694 ns/op 33.72 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/ods-4 4875 120484 ns/op 133.98 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/ods-4 4864 120548 ns/op 133.90 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/ods-4 4912 120833 ns/op 133.59 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/odp 512 1157174 ns/op 16.48 MB/s 12184 B/op 101 allocs/op+BenchmarkConvert/odp 518 1157307 ns/op 16.47 MB/s 12184 B/op 101 allocs/op+BenchmarkConvert/odp 524 1149194 ns/op 16.59 MB/s 12184 B/op 101 allocs/op+BenchmarkConvert/odp-4 2001 297899 ns/op 64.00 MB/s 12186 B/op 101 allocs/op+BenchmarkConvert/odp-4 2006 297995 ns/op 63.98 MB/s 12187 B/op 101 allocs/op+BenchmarkConvert/odp-4 2017 296884 ns/op 64.22 MB/s 12186 B/op 101 allocs/op+BenchmarkConvert/rtf 798 742790 ns/op 27.11 MB/s 25865 B/op 383 allocs/op+BenchmarkConvert/rtf 807 737679 ns/op 27.30 MB/s 25865 B/op 383 allocs/op+BenchmarkConvert/rtf 811 751647 ns/op 26.79 MB/s 25865 B/op 383 allocs/op+BenchmarkConvert/rtf-4 3104 187634 ns/op 107.32 MB/s 25871 B/op 383 allocs/op+BenchmarkConvert/rtf-4 3144 188102 ns/op 107.05 MB/s 25870 B/op 383 allocs/op+BenchmarkConvert/rtf-4 3145 187886 ns/op 107.17 MB/s 25872 B/op 383 allocs/op+BenchmarkConvert/epub 1939 314142 ns/op 19.63 MB/s 15392 B/op 216 allocs/op+BenchmarkConvert/epub 1911 312957 ns/op 19.71 MB/s 15392 B/op 216 allocs/op+BenchmarkConvert/epub 1920 313373 ns/op 19.68 MB/s 15392 B/op 216 allocs/op+BenchmarkConvert/epub-4 6810 79097 ns/op 77.97 MB/s 15395 B/op 216 allocs/op+BenchmarkConvert/epub-4 7555 78542 ns/op 78.52 MB/s 15395 B/op 216 allocs/op+BenchmarkConvert/epub-4 7537 78666 ns/op 78.40 MB/s 15395 B/op 216 allocs/op+BenchmarkConvert/xls 8110 69795 ns/op 102.70 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xls 8925 69836 ns/op 102.64 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xls 8622 69393 ns/op 103.30 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xls-4 31742 19081 ns/op 375.65 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xls-4 31390 19092 ns/op 375.44 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xls-4 31503 19032 ns/op 376.63 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xlsx 2749 216875 ns/op 32.77 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xlsx 2732 217725 ns/op 32.64 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xlsx 2757 218701 ns/op 32.49 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xlsx-4 10000 52325 ns/op 135.80 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xlsx-4 10000 52190 ns/op 136.16 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/xlsx-4 10000 52334 ns/op 135.78 MB/s 12720 B/op 208 allocs/op+BenchmarkConvert/csv 4044 154284 ns/op 1.36 MB/s 9856 B/op 166 allocs/op+BenchmarkConvert/csv 3782 155902 ns/op 1.35 MB/s 9856 B/op 166 allocs/op+BenchmarkConvert/csv 3861 156793 ns/op 1.34 MB/s 9856 B/op 166 allocs/op+BenchmarkConvert/csv-4 15680 38338 ns/op 5.48 MB/s 9856 B/op 166 allocs/op+BenchmarkConvert/csv-4 15589 38179 ns/op 5.50 MB/s 9856 B/op 166 allocs/op+BenchmarkConvert/csv-4 15614 38383 ns/op 5.47 MB/s 9856 B/op 166 allocs/op+BenchmarkConvert/pdf 297 1945666 ns/op 46.29 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf 308 1939296 ns/op 46.44 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf 307 1941629 ns/op 46.38 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf-4 951 579561 ns/op 155.39 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf-4 1015 577545 ns/op 155.94 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf-4 1051 574035 ns/op 156.89 MB/s 2408 B/op 7 allocs/op+PASSdiff --git a/third_party/anydoc-go/benchmarks/2026-09-29/optimized-stress.txt b/third_party/anydoc-go/benchmarks/2026-09-29/optimized-stress.txtnew file mode 100644index 000000000..d753d9d7b--- /dev/null+++ b/third_party/anydoc-go/benchmarks/2026-09-29/optimized-stress.txt@@ -0,0 +1,22 @@+=== RUN TestConcurrentConversionStress+ stress_linked_test.go:85: workers=8 conversions=306252 elapsed=30.003928708s throughput=10207.1 docs/s p50=358.792µs p95=5.490542ms p99=6.842708ms GoHeapAlloc=9018048 GoTotalAlloc=4384020160+--- PASS: TestConcurrentConversionStress (30.04s)+PASS+ 30.04 real 255.23 user 34.26 sys+ 66420736 maximum resident set size+ 0 average shared memory size+ 0 average unshared data size+ 0 average unshared stack size+ 4413 page reclaims+ 32 page faults+ 0 swaps+ 0 block input operations+ 0 block output operations+ 0 messages sent+ 0 messages received+ 11824 signals received+ 1459 voluntary context switches+ 4500950 involuntary context switches+ 3742707465228 instructions retired+ 1062875391967 cycles elapsed+ 57263016 peak memory footprintdiff --git a/third_party/anydoc-go/benchmarks/2026-09-29/optimized.txt b/third_party/anydoc-go/benchmarks/2026-09-29/optimized.txtnew file mode 100644index 000000000..07c1d6ac5--- /dev/null+++ b/third_party/anydoc-go/benchmarks/2026-09-29/optimized.txt@@ -0,0 +1,89 @@+goos: darwin+goarch: arm64+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc+cpu: Apple M4 Pro+BenchmarkConvert/doc 4023 147242 ns/op 139.09 MB/s 23409 B/op 322 allocs/op+BenchmarkConvert/doc 3994 147396 ns/op 138.95 MB/s 23409 B/op 322 allocs/op+BenchmarkConvert/doc 4172 147069 ns/op 139.25 MB/s 23409 B/op 322 allocs/op+BenchmarkConvert/doc-4 16426 36492 ns/op 561.22 MB/s 23413 B/op 322 allocs/op+BenchmarkConvert/doc-4 16423 36598 ns/op 559.60 MB/s 23413 B/op 322 allocs/op+BenchmarkConvert/doc-4 16438 36571 ns/op 560.00 MB/s 23412 B/op 322 allocs/op+BenchmarkConvert/docx 1050 559418 ns/op 18.16 MB/s 25233 B/op 339 allocs/op+BenchmarkConvert/docx 1066 559967 ns/op 18.14 MB/s 25233 B/op 339 allocs/op+BenchmarkConvert/docx 1063 559776 ns/op 18.15 MB/s 25233 B/op 339 allocs/op+BenchmarkConvert/docx-4 4047 142463 ns/op 71.31 MB/s 25238 B/op 339 allocs/op+BenchmarkConvert/docx-4 4219 142316 ns/op 71.38 MB/s 25238 B/op 339 allocs/op+BenchmarkConvert/docx-4 4200 142903 ns/op 71.09 MB/s 25238 B/op 339 allocs/op+BenchmarkConvert/ppt 16210 36971 ns/op 12560.88 MB/s 5496 B/op 70 allocs/op+BenchmarkConvert/ppt 16308 37312 ns/op 12446.12 MB/s 5496 B/op 70 allocs/op+BenchmarkConvert/ppt 16060 36782 ns/op 12625.21 MB/s 5496 B/op 70 allocs/op+BenchmarkConvert/ppt-4 61270 9777 ns/op 47497.14 MB/s 5496 B/op 70 allocs/op+BenchmarkConvert/ppt-4 60988 9764 ns/op 47558.72 MB/s 5496 B/op 70 allocs/op+BenchmarkConvert/ppt-4 61044 9798 ns/op 47396.27 MB/s 5496 B/op 70 allocs/op+BenchmarkConvert/pptx 1304 450272 ns/op 34.94 MB/s 6232 B/op 82 allocs/op+BenchmarkConvert/pptx 1338 448129 ns/op 35.11 MB/s 6232 B/op 82 allocs/op+BenchmarkConvert/pptx 1318 457588 ns/op 34.38 MB/s 6232 B/op 82 allocs/op+BenchmarkConvert/pptx-4 5043 115265 ns/op 136.50 MB/s 6232 B/op 82 allocs/op+BenchmarkConvert/pptx-4 5125 114971 ns/op 136.85 MB/s 6232 B/op 82 allocs/op+BenchmarkConvert/pptx-4 5152 114589 ns/op 137.31 MB/s 6232 B/op 82 allocs/op+BenchmarkConvert/odt 1538 390421 ns/op 65.29 MB/s 25033 B/op 345 allocs/op+BenchmarkConvert/odt 1534 390736 ns/op 65.23 MB/s 25033 B/op 345 allocs/op+BenchmarkConvert/odt 1522 393443 ns/op 64.78 MB/s 25033 B/op 345 allocs/op+BenchmarkConvert/odt-4 5874 99768 ns/op 255.48 MB/s 25037 B/op 345 allocs/op+BenchmarkConvert/odt-4 5952 99740 ns/op 255.55 MB/s 25038 B/op 345 allocs/op+BenchmarkConvert/odt-4 6004 99918 ns/op 255.10 MB/s 25038 B/op 345 allocs/op+BenchmarkConvert/ods 2328 255942 ns/op 63.07 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/ods 2349 259334 ns/op 62.24 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/ods 2276 259975 ns/op 62.09 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/ods-4 8428 64285 ns/op 251.10 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/ods-4 9387 63957 ns/op 252.39 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/ods-4 9322 64114 ns/op 251.77 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/odp 1016 588178 ns/op 32.41 MB/s 12320 B/op 98 allocs/op+BenchmarkConvert/odp 1026 592461 ns/op 32.18 MB/s 12320 B/op 98 allocs/op+BenchmarkConvert/odp 1008 586609 ns/op 32.50 MB/s 12320 B/op 98 allocs/op+BenchmarkConvert/odp-4 3958 149873 ns/op 127.21 MB/s 12322 B/op 98 allocs/op+BenchmarkConvert/odp-4 3969 150168 ns/op 126.96 MB/s 12323 B/op 98 allocs/op+BenchmarkConvert/odp-4 3978 150448 ns/op 126.72 MB/s 12323 B/op 98 allocs/op+BenchmarkConvert/rtf 1465 408854 ns/op 49.25 MB/s 27057 B/op 380 allocs/op+BenchmarkConvert/rtf 1471 406746 ns/op 49.51 MB/s 27057 B/op 380 allocs/op+BenchmarkConvert/rtf 1478 407178 ns/op 49.45 MB/s 27057 B/op 380 allocs/op+BenchmarkConvert/rtf-4 5679 103736 ns/op 194.11 MB/s 27064 B/op 380 allocs/op+BenchmarkConvert/rtf-4 5692 104277 ns/op 193.10 MB/s 27064 B/op 380 allocs/op+BenchmarkConvert/rtf-4 5516 104262 ns/op 193.13 MB/s 27064 B/op 380 allocs/op+BenchmarkConvert/epub 3432 177030 ns/op 34.84 MB/s 15784 B/op 213 allocs/op+BenchmarkConvert/epub 3459 174859 ns/op 35.27 MB/s 15784 B/op 213 allocs/op+BenchmarkConvert/epub 3442 174040 ns/op 35.43 MB/s 15784 B/op 213 allocs/op+BenchmarkConvert/epub-4 13758 43527 ns/op 141.68 MB/s 15788 B/op 213 allocs/op+BenchmarkConvert/epub-4 13788 43570 ns/op 141.54 MB/s 15788 B/op 213 allocs/op+BenchmarkConvert/epub-4 13702 43761 ns/op 140.92 MB/s 15788 B/op 213 allocs/op+BenchmarkConvert/xls 7941 75715 ns/op 94.67 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xls 7932 76079 ns/op 94.22 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xls 7746 76241 ns/op 94.02 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xls-4 30616 19507 ns/op 367.45 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xls-4 30681 19899 ns/op 360.23 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xls-4 30157 19757 ns/op 362.81 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xlsx 2239 263510 ns/op 26.97 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xlsx 2320 261242 ns/op 27.20 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xlsx 2313 259344 ns/op 27.40 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xlsx-4 8738 65600 ns/op 108.32 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xlsx-4 9079 65609 ns/op 108.31 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xlsx-4 9020 65581 ns/op 108.35 MB/s 13288 B/op 205 allocs/op+BenchmarkConvert/xlsb 12316 48681 ns/op 32.91 MB/s 5768 B/op 81 allocs/op+BenchmarkConvert/xlsb 12358 48634 ns/op 32.94 MB/s 5768 B/op 81 allocs/op+BenchmarkConvert/xlsb 12310 48954 ns/op 32.72 MB/s 5768 B/op 81 allocs/op+BenchmarkConvert/xlsb-4 48037 12567 ns/op 127.48 MB/s 5768 B/op 81 allocs/op+BenchmarkConvert/xlsb-4 47362 12528 ns/op 127.88 MB/s 5768 B/op 81 allocs/op+BenchmarkConvert/xlsb-4 48376 12470 ns/op 128.47 MB/s 5768 B/op 81 allocs/op+BenchmarkConvert/csv 6783 90056 ns/op 2.33 MB/s 10168 B/op 163 allocs/op+BenchmarkConvert/csv 6709 90130 ns/op 2.33 MB/s 10168 B/op 163 allocs/op+BenchmarkConvert/csv 6710 89996 ns/op 2.33 MB/s 10168 B/op 163 allocs/op+BenchmarkConvert/csv-4 26893 22533 ns/op 9.32 MB/s 10168 B/op 163 allocs/op+BenchmarkConvert/csv-4 26581 22763 ns/op 9.23 MB/s 10168 B/op 163 allocs/op+BenchmarkConvert/csv-4 26870 22610 ns/op 9.29 MB/s 10168 B/op 163 allocs/op+BenchmarkConvert/pdf 163 3827329 ns/op 23.53 MB/s 2409 B/op 7 allocs/op+BenchmarkConvert/pdf 160 3622717 ns/op 24.86 MB/s 2409 B/op 7 allocs/op+BenchmarkConvert/pdf 163 3631949 ns/op 24.80 MB/s 2409 B/op 7 allocs/op+BenchmarkConvert/pdf-4 576 1061518 ns/op 84.84 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf-4 562 1060361 ns/op 84.93 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf-4 556 1046297 ns/op 86.07 MB/s 2408 B/op 7 allocs/op+PASSdiff --git a/third_party/anydoc-go/benchmarks/2026-09-29/scaling.txt b/third_party/anydoc-go/benchmarks/2026-09-29/scaling.txtnew file mode 100644index 000000000..2d556da0c--- /dev/null+++ b/third_party/anydoc-go/benchmarks/2026-09-29/scaling.txt@@ -0,0 +1,17 @@+goos: darwin+goarch: arm64+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc+cpu: Apple M4 Pro+BenchmarkConvertScaling/csv_100_rows 1497 397701 ns/op 9.12 MB/s 129360 B/op 1930 allocs/op+BenchmarkConvertScaling/csv_100_rows 1510 398194 ns/op 9.11 MB/s 129360 B/op 1930 allocs/op+BenchmarkConvertScaling/csv_100_rows 1504 395770 ns/op 9.16 MB/s 129360 B/op 1930 allocs/op+BenchmarkConvertScaling/csv_10000_rows 18 31730509 ns/op 11.35 MB/s 12581816 B/op 190030 allocs/op+BenchmarkConvertScaling/csv_10000_rows 16 32328646 ns/op 11.14 MB/s 12581816 B/op 190030 allocs/op+BenchmarkConvertScaling/csv_10000_rows 16 32133409 ns/op 11.20 MB/s 12581816 B/op 190030 allocs/op+BenchmarkConvertScaling/pdf_40000_operators 547 1076646 ns/op 186.32 MB/s 152 B/op 7 allocs/op+BenchmarkConvertScaling/pdf_40000_operators 559 1074017 ns/op 186.77 MB/s 152 B/op 7 allocs/op+BenchmarkConvertScaling/pdf_40000_operators 558 1064584 ns/op 188.43 MB/s 152 B/op 7 allocs/op+BenchmarkConvertScaling/pdf_200000_operators 138 4345923 ns/op 230.24 MB/s 152 B/op 7 allocs/op+BenchmarkConvertScaling/pdf_200000_operators 136 4356177 ns/op 229.70 MB/s 152 B/op 7 allocs/op+BenchmarkConvertScaling/pdf_200000_operators 138 4319291 ns/op 231.66 MB/s 152 B/op 7 allocs/op+PASSdiff --git a/third_party/anydoc-go/benchmarks/2026-09-29/upgrade-stress.txt b/third_party/anydoc-go/benchmarks/2026-09-29/upgrade-stress.txtnew file mode 100644index 000000000..fd8640fea--- /dev/null+++ b/third_party/anydoc-go/benchmarks/2026-09-29/upgrade-stress.txt@@ -0,0 +1,22 @@+=== RUN TestConcurrentConversionStress+ stress_linked_test.go:85: workers=8 conversions=231291 elapsed=30.004531458s throughput=7708.5 docs/s p50=644.084µs p95=5.320084ms p99=6.684167ms GoHeapAlloc=2712016 GoTotalAlloc=3303175600+--- PASS: TestConcurrentConversionStress (30.04s)+PASS+ 30.04 real 253.87 user 28.25 sys+ 64110592 maximum resident set size+ 0 average shared memory size+ 0 average unshared data size+ 0 average unshared stack size+ 4274 page reclaims+ 32 page faults+ 0 swaps+ 0 block input operations+ 0 block output operations+ 0 messages sent+ 0 messages received+ 8607 signals received+ 879 voluntary context switches+ 3964104 involuntary context switches+ 3957497347565 instructions retired+ 1043824362343 cycles elapsed+ 54395768 peak memory footprintdiff --git a/third_party/anydoc-go/benchmarks/2026-09-29/upgrade.txt b/third_party/anydoc-go/benchmarks/2026-09-29/upgrade.txtnew file mode 100644index 000000000..e4621b9b0--- /dev/null+++ b/third_party/anydoc-go/benchmarks/2026-09-29/upgrade.txt@@ -0,0 +1,89 @@+goos: darwin+goarch: arm64+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc+cpu: Apple M4 Pro+BenchmarkConvert/doc 2497 237727 ns/op 86.15 MB/s 23673 B/op 325 allocs/op+BenchmarkConvert/doc 2479 237260 ns/op 86.32 MB/s 23673 B/op 325 allocs/op+BenchmarkConvert/doc 2466 237862 ns/op 86.10 MB/s 23673 B/op 325 allocs/op+BenchmarkConvert/doc-4 9816 58760 ns/op 348.54 MB/s 23678 B/op 325 allocs/op+BenchmarkConvert/doc-4 9968 58307 ns/op 351.25 MB/s 23677 B/op 325 allocs/op+BenchmarkConvert/doc-4 10000 58467 ns/op 350.28 MB/s 23677 B/op 325 allocs/op+BenchmarkConvert/docx 565 1054643 ns/op 9.63 MB/s 24857 B/op 342 allocs/op+BenchmarkConvert/docx 556 1082361 ns/op 9.39 MB/s 24857 B/op 342 allocs/op+BenchmarkConvert/docx 541 1100535 ns/op 9.23 MB/s 24857 B/op 342 allocs/op+BenchmarkConvert/docx-4 2130 271626 ns/op 37.40 MB/s 24862 B/op 342 allocs/op+BenchmarkConvert/docx-4 2180 269896 ns/op 37.64 MB/s 24861 B/op 342 allocs/op+BenchmarkConvert/docx-4 2109 270594 ns/op 37.54 MB/s 24861 B/op 342 allocs/op+BenchmarkConvert/ppt 9344 62726 ns/op 7403.33 MB/s 5520 B/op 73 allocs/op+BenchmarkConvert/ppt 9786 62456 ns/op 7435.34 MB/s 5520 B/op 73 allocs/op+BenchmarkConvert/ppt 9612 63643 ns/op 7296.73 MB/s 5520 B/op 73 allocs/op+BenchmarkConvert/ppt-4 36694 16371 ns/op 28366.29 MB/s 5520 B/op 73 allocs/op+BenchmarkConvert/ppt-4 36626 16350 ns/op 28402.93 MB/s 5520 B/op 73 allocs/op+BenchmarkConvert/ppt-4 36825 16367 ns/op 28373.05 MB/s 5520 B/op 73 allocs/op+BenchmarkConvert/pptx 657 898570 ns/op 17.51 MB/s 6224 B/op 85 allocs/op+BenchmarkConvert/pptx 674 895435 ns/op 17.57 MB/s 6224 B/op 85 allocs/op+BenchmarkConvert/pptx 678 893230 ns/op 17.61 MB/s 6224 B/op 85 allocs/op+BenchmarkConvert/pptx-4 2608 227670 ns/op 69.11 MB/s 6224 B/op 85 allocs/op+BenchmarkConvert/pptx-4 2575 227581 ns/op 69.14 MB/s 6224 B/op 85 allocs/op+BenchmarkConvert/pptx-4 2533 227921 ns/op 69.03 MB/s 6224 B/op 85 allocs/op+BenchmarkConvert/odt 817 730569 ns/op 34.89 MB/s 24673 B/op 348 allocs/op+BenchmarkConvert/odt 822 727546 ns/op 35.03 MB/s 24673 B/op 348 allocs/op+BenchmarkConvert/odt 826 726398 ns/op 35.09 MB/s 24673 B/op 348 allocs/op+BenchmarkConvert/odt-4 3181 183722 ns/op 138.74 MB/s 24677 B/op 348 allocs/op+BenchmarkConvert/odt-4 3240 183166 ns/op 139.16 MB/s 24677 B/op 348 allocs/op+BenchmarkConvert/odt-4 3290 182945 ns/op 139.33 MB/s 24677 B/op 348 allocs/op+BenchmarkConvert/ods 1221 483640 ns/op 33.38 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/ods 1249 483216 ns/op 33.41 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/ods 1252 482840 ns/op 33.43 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/ods-4 4846 121986 ns/op 132.33 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/ods-4 4741 122610 ns/op 131.65 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/ods-4 4682 122870 ns/op 131.37 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/odp 508 1167552 ns/op 16.33 MB/s 12408 B/op 101 allocs/op+BenchmarkConvert/odp 512 1163936 ns/op 16.38 MB/s 12408 B/op 101 allocs/op+BenchmarkConvert/odp 516 1172509 ns/op 16.26 MB/s 12408 B/op 101 allocs/op+BenchmarkConvert/odp-4 2010 296137 ns/op 64.38 MB/s 12411 B/op 101 allocs/op+BenchmarkConvert/odp-4 1977 296402 ns/op 64.32 MB/s 12411 B/op 101 allocs/op+BenchmarkConvert/odp-4 2013 297303 ns/op 64.13 MB/s 12410 B/op 101 allocs/op+BenchmarkConvert/rtf 781 760683 ns/op 26.47 MB/s 26809 B/op 383 allocs/op+BenchmarkConvert/rtf 796 757376 ns/op 26.59 MB/s 26809 B/op 383 allocs/op+BenchmarkConvert/rtf 792 757098 ns/op 26.60 MB/s 26809 B/op 383 allocs/op+BenchmarkConvert/rtf-4 3044 192481 ns/op 104.61 MB/s 26815 B/op 383 allocs/op+BenchmarkConvert/rtf-4 3078 192209 ns/op 104.76 MB/s 26816 B/op 383 allocs/op+BenchmarkConvert/rtf-4 3087 192072 ns/op 104.84 MB/s 26816 B/op 383 allocs/op+BenchmarkConvert/epub 1946 314520 ns/op 19.61 MB/s 16064 B/op 216 allocs/op+BenchmarkConvert/epub 1914 313805 ns/op 19.65 MB/s 16064 B/op 216 allocs/op+BenchmarkConvert/epub 1910 318963 ns/op 19.33 MB/s 16064 B/op 216 allocs/op+BenchmarkConvert/epub-4 7438 79126 ns/op 77.94 MB/s 16068 B/op 216 allocs/op+BenchmarkConvert/epub-4 7432 78533 ns/op 78.53 MB/s 16068 B/op 216 allocs/op+BenchmarkConvert/epub-4 7393 78452 ns/op 78.61 MB/s 16068 B/op 216 allocs/op+BenchmarkConvert/xls 4747 126316 ns/op 56.75 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xls 4825 127954 ns/op 56.02 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xls 4706 127874 ns/op 56.06 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xls-4 18931 31484 ns/op 227.67 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xls-4 18969 31668 ns/op 226.35 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xls-4 18921 31662 ns/op 226.39 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xlsx 1207 500980 ns/op 14.18 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xlsx 1207 498708 ns/op 14.25 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xlsx 1209 500326 ns/op 14.20 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xlsx-4 4579 126024 ns/op 56.39 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xlsx-4 4734 125090 ns/op 56.81 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xlsx-4 4790 124966 ns/op 56.86 MB/s 13216 B/op 208 allocs/op+BenchmarkConvert/xlsb 6988 84341 ns/op 18.99 MB/s 5792 B/op 84 allocs/op+BenchmarkConvert/xlsb 7204 83794 ns/op 19.12 MB/s 5792 B/op 84 allocs/op+BenchmarkConvert/xlsb 6828 84787 ns/op 18.89 MB/s 5792 B/op 84 allocs/op+BenchmarkConvert/xlsb-4 27576 21564 ns/op 74.29 MB/s 5792 B/op 84 allocs/op+BenchmarkConvert/xlsb-4 27964 21786 ns/op 73.53 MB/s 5792 B/op 84 allocs/op+BenchmarkConvert/xlsb-4 27409 21783 ns/op 73.55 MB/s 5792 B/op 84 allocs/op+BenchmarkConvert/csv 3846 161235 ns/op 1.30 MB/s 10240 B/op 166 allocs/op+BenchmarkConvert/csv 3718 161213 ns/op 1.30 MB/s 10240 B/op 166 allocs/op+BenchmarkConvert/csv 3720 161295 ns/op 1.30 MB/s 10240 B/op 166 allocs/op+BenchmarkConvert/csv-4 15117 39604 ns/op 5.30 MB/s 10240 B/op 166 allocs/op+BenchmarkConvert/csv-4 15140 39760 ns/op 5.28 MB/s 10240 B/op 166 allocs/op+BenchmarkConvert/csv-4 15150 39692 ns/op 5.29 MB/s 10240 B/op 166 allocs/op+BenchmarkConvert/pdf 162 3627058 ns/op 24.83 MB/s 2409 B/op 7 allocs/op+BenchmarkConvert/pdf 165 3605448 ns/op 24.98 MB/s 2409 B/op 7 allocs/op+BenchmarkConvert/pdf 165 3559522 ns/op 25.30 MB/s 2409 B/op 7 allocs/op+BenchmarkConvert/pdf-4 560 1048032 ns/op 85.93 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf-4 554 1052198 ns/op 85.59 MB/s 2408 B/op 7 allocs/op+BenchmarkConvert/pdf-4 576 1039713 ns/op 86.62 MB/s 2408 B/op 7 allocs/op+PASSdiff --git a/third_party/anydoc-go/benchmarks/2026-09-29/xlsx-scaling.txt b/third_party/anydoc-go/benchmarks/2026-09-29/xlsx-scaling.txtnew file mode 100644index 000000000..0660627e1--- /dev/null+++ b/third_party/anydoc-go/benchmarks/2026-09-29/xlsx-scaling.txt@@ -0,0 +1,12 @@+goos: darwin+goarch: arm64+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc+cpu: Apple M4 Pro+BenchmarkConvertScaling/xlsx_100_rows 982 606582 ns/op 11.68 MB/s 82624 B/op 1353 allocs/op+BenchmarkConvertScaling/xlsx_100_rows 1010 591741 ns/op 11.97 MB/s 82624 B/op 1353 allocs/op+BenchmarkConvertScaling/xlsx_100_rows 1010 590599 ns/op 11.99 MB/s 82624 B/op 1353 allocs/op+BenchmarkConvertScaling/xlsx_10000_rows 12 46185531 ns/op 2.51 MB/s 7946440 B/op 130053 allocs/op+BenchmarkConvertScaling/xlsx_10000_rows 12 46568236 ns/op 2.49 MB/s 7946442 B/op 130053 allocs/op+BenchmarkConvertScaling/xlsx_10000_rows 13 45783734 ns/op 2.53 MB/s 7946442 B/op 130053 allocs/op+PASS+ok github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc 4.609sdiff --git a/third_party/anydoc-go/errors.go b/third_party/anydoc-go/errors.goindex 9000e8378..1493fee16 100644--- a/third_party/anydoc-go/errors.go+++ b/third_party/anydoc-go/errors.go@@ -10,6 +10,7 @@ import "fmt"// Error codes mirroring the C ABI (ERR_*). Stable across versions.const (+ errNeedsOCR = C.ERR_NEEDS_OCRerrOK = C.ERR_OKerrUnsupported = C.ERR_UNSUPPORTEDerrMalformed = C.ERR_MALFORMED@@ -27,15 +28,19 @@ const (// ("unsupported", "malformed", "encrypted", ...) plus the crate's// human-readable detail.type ConvertError struct {- // Kind is the lowercase variant name, matching the Node and Python- // bindings: "unsupported", "malformed", "encrypted", "resource_limit",- // "missing_part", "io", "pdf_no_model". Go also reports+ // Kind is the snake-case variant name: "unsupported", "malformed", "encrypted", "resource_limit",+ // "missing_part", "io", "pdf_no_model", "needs_ocr". Go also reports// "unknown_format" for an invalid explicit Format.Kind string// Detail is the crate's Display output for the error.Detail string}+// NeedsOCR reports the typed scanned-page failure without matching error text.+func (e *ConvertError) NeedsOCR() bool {+ return e != nil && e.Kind == "needs_ocr"+}+func (e *ConvertError) Error() string {if e == nil {return "<nil>"@@ -59,6 +64,8 @@ func convertError(code C.int) error {func errorKind(code C.int) string {switch code {+ case errNeedsOCR:+ return "needs_ocr"case errUnsupported:return "unsupported"case errMalformed:diff --git a/third_party/anydoc-go/include/anydoc.h b/third_party/anydoc-go/include/anydoc.hindex 179ff5ce4..c3fbb1ecb 100644--- a/third_party/anydoc-go/include/anydoc.h+++ b/third_party/anydoc-go/include/anydoc.h@@ -58,6 +58,11 @@*/#define ERR_UNKNOWN_FORMAT 9+/**+ * `ConvertError::NeedsOcr`; page numbers are retained in the error detail.+ */+#define ERR_NEEDS_OCR 10+/*** C-side format tag. Stable; mirrors the Node/Python lowercase string names* via `format_name`. `ANYDOC_FORMAT_NONE` is the `Option::None` sentinel.@@ -106,6 +111,8 @@#define BLOCK_RULE 6+#define BLOCK_MATH 7+#define INLINE_TEXT 0#define INLINE_LINK 1@@ -118,6 +125,10 @@#define INLINE_LINEBREAK 5+#define INLINE_MATH 6++#define INLINE_CHECKBOX 7+#define LINK_EXTERNAL 0#define LINK_RELATIVE 1@@ -229,6 +240,17 @@ int anydoc_to_document(const uint8_t *bytes,uint8_t **out_buf,uintptr_t *out_len);+/**+ * Parse once and return the original document model followed by asset-linked+ * Markdown in a single flat buffer. Free it with `anydoc_buffer_free`.+ * PDF has no document model and returns `ERR_PDF_NO_MODEL`.+ */+int anydoc_to_document_with_asset_links(const uint8_t *bytes,+ uintptr_t len,+ int format_tag,+ uint8_t **out_buf,+ uintptr_t *out_len);+/*** Free a string returned by `anydoc_to_markdown*` or `anydoc_last_error`.* Null is a no-op (the empty-string sentinel).diff --git a/third_party/anydoc-go/model.go b/third_party/anydoc-go/model.goindex c0656950c..8ddf49095 100644--- a/third_party/anydoc-go/model.go+++ b/third_party/anydoc-go/model.go@@ -24,7 +24,7 @@ type Document struct {// which of the optional fields is populated; the rest are nil.type Block struct {// "heading", "paragraph", "list", "table", "block_quote", "code_block",- // or "rule".+ // "rule", or "math".Kind stringLevel *uint8 // heading: 1-based outline depthAnchor *string // heading: stable anchor id when the document targets it@@ -33,15 +33,15 @@ type Block struct {Table *Table // tableBlocks []Block // block_quoteLang *string // code_block- Text *string // code_block+ Text *string // code_block, math}// Inline is one span of inline content. The Kind field selects which optional// fields are populated.type Inline struct {- // "text", "link", "image", "anchor", "note_ref", or "line_break".+ // "text", "link", "image", "anchor", "note_ref", "line_break", "math", or "checkbox".Kind string- Text *string // text+ Text *string // text, mathStyle *Style // textContent []Inline // linkTarget *LinkTarget // link@@ -49,6 +49,7 @@ type Inline struct {Source *ImageSource // imageAnchor *string // anchor: the anchor idNoteID *string // note_ref: the id of the note in Document.Notes+ Checked *bool // checkbox}// Style is a fully resolved character style.@@ -92,7 +93,7 @@ type List struct {// further lists.type ListItem struct {Blocks []Block- Checked *bool // task-list state, when the item carries a checkbox+ Checked *bool // Deprecated: always nil since 0.2.3; read Inline.Checked.MarkerLabel *string // literal marker text overriding the level marker}@@ -156,6 +157,10 @@ type decoder struct {func decodeDocument(raw []byte) (*Document, error) {d := &decoder{buf: raw}+ return d.document()+}++func (d *decoder) document() (*Document, error) {doc := &Document{}var err errorif doc.Blocks, err = d.blocks(); err != nil {@@ -356,6 +361,12 @@ func (d *decoder) block() (Block, error) {return Block{}, err}return Block{Kind: "code_block", Lang: lang, Text: &text}, nil+ case C.BLOCK_MATH:+ text, err := d.str()+ if err != nil {+ return Block{}, err+ }+ return Block{Kind: "math", Text: &text}, nilcase C.BLOCK_RULE:return Block{Kind: "rule"}, nildefault:@@ -427,6 +438,18 @@ func (d *decoder) inline() (Inline, error) {return Inline{}, err}return Inline{Kind: "note_ref", NoteID: &id}, nil+ case C.INLINE_MATH:+ text, err := d.str()+ if err != nil {+ return Inline{}, err+ }+ return Inline{Kind: "math", Text: &text}, nil+ case C.INLINE_CHECKBOX:+ checked, err := d.bool()+ if err != nil {+ return Inline{}, err+ }+ return Inline{Kind: "checkbox", Checked: &checked}, nilcase C.INLINE_LINEBREAK:return Inline{Kind: "line_break"}, nildefault:diff --git a/third_party/anydoc-go/src/asset_links.rs b/third_party/anydoc-go/src/asset_links.rsindex b5247df77..629da7151 100644--- a/third_party/anydoc-go/src/asset_links.rs+++ b/third_party/anydoc-go/src/asset_links.rs@@ -48,7 +48,7 @@ fn rewrite_blocks(blocks: &mut [Block], urls: &[Option<String>]) {}}Block::BlockQuote(inner) => rewrite_blocks(inner, urls),- Block::CodeBlock { .. } | Block::Rule => {}+ Block::CodeBlock { .. } | Block::Rule | Block::Math(_) => {}}}}diff --git a/third_party/anydoc-go/src/lib.rs b/third_party/anydoc-go/src/lib.rsindex 18b787f79..252066d33 100644--- a/third_party/anydoc-go/src/lib.rs+++ b/third_party/anydoc-go/src/lib.rs@@ -47,6 +47,8 @@ pub const ERR_PDF_NO_MODEL: c_int = 7;pub const ERR_INVALID_ARG: c_int = 8;/// Unknown format name passed to the ABI.pub const ERR_UNKNOWN_FORMAT: c_int = 9;+/// `ConvertError::NeedsOcr`; page numbers are retained in the error detail.+pub const ERR_NEEDS_OCR: c_int = 10;thread_local! {static LAST_ERROR: std::cell::RefCell<Option<CString>> = const { std::cell::RefCell::new(None) };@@ -60,6 +62,7 @@ fn set_last_error(msg: &str) {fn error_code(err: &ConvertError) -> c_int {match err {ConvertError::Unsupported(_) => ERR_UNSUPPORTED,+ ConvertError::NeedsOcr { .. } => ERR_NEEDS_OCR,ConvertError::Malformed { .. } => ERR_MALFORMED,ConvertError::Encrypted => ERR_ENCRYPTED,ConvertError::ResourceLimit { .. } => ERR_RESOURCE_LIMIT,@@ -438,6 +441,31 @@ pub unsafe extern "C" fn anydoc_to_document(format_tag: c_int,out_buf: *mut *mut u8,out_len: *mut usize,+) -> c_int {+ unsafe { document_output(bytes, len, format_tag, out_buf, out_len, false) }+}++/// Parse once and return the original document model followed by asset-linked+/// Markdown in a single flat buffer. Free it with `anydoc_buffer_free`.+/// PDF has no document model and returns `ERR_PDF_NO_MODEL`.+#[unsafe(no_mangle)]+pub unsafe extern "C" fn anydoc_to_document_with_asset_links(+ bytes: *const u8,+ len: usize,+ format_tag: c_int,+ out_buf: *mut *mut u8,+ out_len: *mut usize,+) -> c_int {+ unsafe { document_output(bytes, len, format_tag, out_buf, out_len, true) }+}++unsafe fn document_output(+ bytes: *const u8,+ len: usize,+ format_tag: c_int,+ out_buf: *mut *mut u8,+ out_len: *mut usize,+ with_markdown: bool,) -> c_int {if bytes.is_null() || out_buf.is_null() {set_last_error("anydoc: null pointer passed to anydoc_to_document");@@ -475,13 +503,21 @@ pub unsafe extern "C" fn anydoc_to_document();return ERR_PDF_NO_MODEL;}- let document = match guarded("anydoc_to_document", || anydoc::to_document(slice, resolved)) {- Ok(d) => d,+ let body = match guarded("anydoc_to_document", || {+ let mut document = anydoc::to_document(slice, resolved)?;+ let mut encoder = model::Encoder::new();+ // Encode before rewriting: Go needs the original asset IDs for image+ // placement and section metadata, while Markdown needs external URLs.+ model::write_document(&mut encoder, &document);+ if with_markdown {+ asset_links::rewrite_asset_images(&mut document);+ encoder.str(&document_to_markdown(&document));+ }+ Ok(encoder.into_vec())+ }) {+ Ok(body) => body,Err(e) => return fail(e),};- let mut encoder = model::Encoder::new();- model::write_document(&mut encoder, &document);- let body = encoder.into_vec();let len = body.len();// Layout: [usize len][usize cap][u8 data..len]. The caller receives a// pointer to the data area and the length; `anydoc_buffer_free` walks backdiff --git a/third_party/anydoc-go/src/model.rs b/third_party/anydoc-go/src/model.rsindex 5a97b0448..0dae5d88b 100644--- a/third_party/anydoc-go/src/model.rs+++ b/third_party/anydoc-go/src/model.rs@@ -27,6 +27,7 @@ pub const BLOCK_TABLE: c_int = 3;pub const BLOCK_QUOTE: c_int = 4;pub const BLOCK_CODE: c_int = 5;pub const BLOCK_RULE: c_int = 6;+pub const BLOCK_MATH: c_int = 7;pub const INLINE_TEXT: c_int = 0;pub const INLINE_LINK: c_int = 1;@@ -34,6 +35,8 @@ pub const INLINE_IMAGE: c_int = 2;pub const INLINE_ANCHOR: c_int = 3;pub const INLINE_NOTEREF: c_int = 4;pub const INLINE_LINEBREAK: c_int = 5;+pub const INLINE_MATH: c_int = 6;+pub const INLINE_CHECKBOX: c_int = 7;pub const LINK_EXTERNAL: c_int = 0;pub const LINK_RELATIVE: c_int = 1;@@ -161,6 +164,10 @@ fn write_block(e: &mut Encoder, block: &model::Block) {e.opt_str(lang);e.str(text);}+ model::Block::Math(text) => {+ e.i32(BLOCK_MATH);+ e.str(text);+ }model::Block::Rule => {e.i32(BLOCK_RULE);}@@ -199,6 +206,14 @@ fn write_inline(e: &mut Encoder, inline: &model::Inline) {e.i32(INLINE_NOTEREF);e.str(id);}+ model::Inline::Math(text) => {+ e.i32(INLINE_MATH);+ e.str(text);+ }+ model::Inline::Checkbox(checked) => {+ e.i32(INLINE_CHECKBOX);+ e.bool(*checked);+ }model::Inline::LineBreak => {e.i32(INLINE_LINEBREAK);}@@ -256,11 +271,9 @@ fn write_list(e: &mut Encoder, list: &model::List) {fn write_list_item(e: &mut Encoder, item: &model::ListItem) {write_blocks(e, &item.blocks);- match item.checked {- None => e.i32(-1),- Some(false) => e.i32(0),- Some(true) => e.i32(1),- }+ // Preserve the legacy optional field in the wire format. Upstream 0.2.3+ // moved task state into Inline::Checkbox; new callers read those nodes.+ e.i32(-1);e.opt_str(&item.marker_label);}diff --git a/third_party/anydoc-go/version.go b/third_party/anydoc-go/version.goindex e493d82fd..566c6b9cc 100644--- a/third_party/anydoc-go/version.go+++ b/third_party/anydoc-go/version.go@@ -3,4 +3,4 @@ package anydoc// Version is the version of the anydoc-go module, bumped in lockstep with the// Rust crate version (go/Cargo.toml) and the GitHub release tag. The// packaged static libraries.-const Version = "0.1.9"+const Version = "0.2.4"