Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

feat(docparser): upgrade AnyDoc to 0.2.4 and parse assets once

wizardchen75fc0ad15907
diff --git a/internal/infrastructure/docparser/anydoc/anydoc.go b/internal/infrastructure/docparser/anydoc/anydoc.go
index 5965f26e6..9cd383139 100644
--- a/internal/infrastructure/docparser/anydoc/anydoc.go
+++ b/internal/infrastructure/docparser/anydoc/anydoc.go
@@ -166,6 +166,10 @@ func PDFNeedsOCR(err error) bool {
if err == nil {
return false
}
+ var ocrError interface{ NeedsOCR() bool }
+ if errors.As(err, &ocrError) && ocrError.NeedsOCR() {
+ return true
+ }
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "ocr is required") ||
strings.Contains(msg, "no extractable text")
diff --git a/internal/infrastructure/docparser/anydoc/backend_cgo.go b/internal/infrastructure/docparser/anydoc/backend_cgo.go
index 4a1040222..625c2ebd6 100644
--- a/internal/infrastructure/docparser/anydoc/backend_cgo.go
+++ b/internal/infrastructure/docparser/anydoc/backend_cgo.go
@@ -35,8 +35,9 @@ func backendConvert(data []byte, opts Options) (*Result, error) {
}
// Official GFM serializer after rewriting Asset images to External URLs
- // (`images/image-N.ext`). ToDocument still supplies the image bytes.
- document, err := upstream.ToDocument(data, format)
+ // (`images/image-N.ext`). One Rust parse supplies both the original model
+ // (for image bytes and placements) and the rendered Markdown.
+ document, markdown, err := upstream.ToDocumentWithAssetLinks(data, format)
if err != nil {
markdown, mdErr := upstream.ToMarkdownBytes(data, format)
if mdErr != nil {
@@ -47,10 +48,6 @@ func backendConvert(data []byte, opts Options) (*Result, error) {
AssetsError: fmt.Errorf("anydoc: image extraction failed: %w", err),
}, nil
}
- markdown, err := upstream.ToMarkdownWithAssetLinks(data, format)
- if err != nil {
- return nil, fmt.Errorf("anydoc: markdown conversion failed: %w", err)
- }
return &Result{Markdown: markdown, Assets: collectAssets(document)}, nil
}
@@ -169,10 +166,16 @@ func inlineText(inlines []upstream.Inline) string {
var text strings.Builder
for _, inline := range inlines {
switch inline.Kind {
- case "text":
+ case "text", "math":
if inline.Text != nil {
text.WriteString(*inline.Text)
}
+ case "checkbox":
+ if inline.Checked != nil && *inline.Checked {
+ text.WriteString("[x]")
+ } else {
+ text.WriteString("[ ]")
+ }
case "link":
text.WriteString(inlineText(inline.Content))
}
diff --git a/internal/infrastructure/docparser/anydoc/benchmark_linked_test.go b/internal/infrastructure/docparser/anydoc/benchmark_linked_test.go
new file mode 100644
index 000000000..bada81348
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/benchmark_linked_test.go
@@ -0,0 +1,151 @@
+//go:build anydoc && cgo
+
+package anydoc
+
+import (
+ "archive/zip"
+ "bytes"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+var benchmarkDocuments = []string{
+ "doc/text.doc", "docx/text.docx", "ppt/pres.ppt", "pptx/pres.pptx",
+ "odt/text.odt", "ods/sheet.ods", "odp/pres.odp", "rtf/text.rtf",
+ "epub/book.epub", "xls/sheet.xls", "xlsx/sheet.xlsx",
+ "xlsb/handmade-sheet.xlsb", "csv/sheet.csv", "pdf/text.pdf",
+}
+
+// Synthetic size sweeps cover the Go/Rust document-copy path and the bounded
+// hostile-PDF detector. They are separate from the upstream fixture comparison.
+func BenchmarkConvertScaling(b *testing.B) {
+ for _, rows := range []int{100, 10000} {
+ b.Run(fmt.Sprintf("csv_%d_rows", rows), func(b *testing.B) {
+ data := []byte("name,quantity,description\n" +
+ strings.Repeat("widget,42,Quarterly shipping report\n", rows))
+ b.SetBytes(int64(len(data)))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ result, err := Convert(data, Options{Format: "csv", WithAssets: true})
+ if err != nil || result.AssetsError != nil {
+ b.Fatalf("conversion: %v", err)
+ }
+ }
+ })
+ }
+ for _, rows := range []int{100, 10000} {
+ b.Run(fmt.Sprintf("xlsx_%d_rows", rows), func(b *testing.B) {
+ data := largeWorkbook(b, rows)
+ result, err := Convert(data, Options{Format: "xlsx", WithAssets: true})
+ if err != nil || !strings.Contains(result.Markdown, fmt.Sprintf("row-%d", rows)) {
+ b.Fatalf("preflight: last row was lost, err=%v", err)
+ }
+ b.SetBytes(int64(len(data)))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ result, err := Convert(data, Options{Format: "xlsx", WithAssets: true})
+ if err != nil || result.AssetsError != nil {
+ b.Fatalf("conversion: %v", err)
+ }
+ }
+ })
+ }
+ for _, operators := range []int{40000, 200000} {
+ b.Run(fmt.Sprintf("pdf_%d_operators", operators), func(b *testing.B) {
+ data := unmatchedShowTextPDF(operators)
+ b.SetBytes(int64(len(data)))
+ b.ReportAllocs()
+ b.ResetTimer()
+ for b.Loop() {
+ if _, err := Convert(data, Options{Format: "pdf"}); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}
+
+func largeWorkbook(t testing.TB, rows int) []byte {
+ t.Helper()
+ data := readFixture(t, "xlsx/sheet.xlsx")
+ r, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var out bytes.Buffer
+ w := zip.NewWriter(&out)
+ for _, file := range r.File {
+ if file.Name == "xl/worksheets/sheet1.xml" {
+ continue
+ }
+ if err := w.Copy(file); err != nil {
+ t.Fatal(err)
+ }
+ }
+ var sheet strings.Builder
+ sheet.WriteString(`<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>`)
+ for row := 1; row <= rows; row++ {
+ fmt.Fprintf(&sheet, `<row r="%d"><c r="A%d" t="inlineStr"><is><t>row-%d</t></is></c>`+
+ `<c r="B%d"><v>42</v></c></row>`,
+ row, row, row, row)
+ }
+ sheet.WriteString(`</sheetData></worksheet>`)
+ part, err := w.Create("xl/worksheets/sheet1.xml")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := part.Write([]byte(sheet.String())); err != nil {
+ t.Fatal(err)
+ }
+ if err := w.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return out.Bytes()
+}
+
+func readFixture(t testing.TB, name string) []byte {
+ t.Helper()
+ data, err := os.ReadFile(filepath.Join("testdata", "upstream", name))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return data
+}
+
+// WithAssets exercises the default reader path, including Rust parsing,
+// the document ABI, Go decoding, image collection, and Markdown serialization.
+// Input I/O is excluded. B/op measures Go allocations, not the Rust heap.
+func BenchmarkConvert(b *testing.B) {
+ for _, name := range benchmarkDocuments {
+ b.Run(strings.Split(name, "/")[0], func(b *testing.B) {
+ data := readFixture(b, name)
+ format := strings.TrimPrefix(filepath.Ext(name), ".")
+ if format == "xls" || format == "xlsb" {
+ format = "xlsx"
+ }
+ opts := Options{Format: format, WithAssets: true}
+ if result, err := Convert(data, opts); err != nil ||
+ result.Markdown == "" ||
+ result.AssetsError != nil {
+ b.Fatalf("preflight conversion: result=%+v err=%v", result, err)
+ }
+ b.ReportAllocs()
+ b.SetBytes(int64(len(data)))
+ b.ResetTimer()
+ b.RunParallel(func(pb *testing.PB) {
+ for pb.Next() {
+ result, err := Convert(data, opts)
+ if err != nil || result.Markdown == "" || result.AssetsError != nil {
+ b.Errorf("conversion failed: result=%+v err=%v", result, err)
+ return
+ }
+ }
+ })
+ })
+ }
+}
diff --git a/internal/infrastructure/docparser/anydoc/convert_linked_test.go b/internal/infrastructure/docparser/anydoc/convert_linked_test.go
index daf8512fb..556a32519 100644
--- a/internal/infrastructure/docparser/anydoc/convert_linked_test.go
+++ b/internal/infrastructure/docparser/anydoc/convert_linked_test.go
@@ -116,9 +116,9 @@ func TestErrorDetailSurvivesConcurrency(t *testing.T) {
defer wg.Done()
// docx and xlsx failures carry distinctive messages, so a detail
// naming the other format proves it came from the wrong thread.
- format, want, other := "docx", "not a readable zip archive", "unreadable workbook"
+ format, want, other := "docx", "not a readable zip archive", "not a readable workbook container"
if i%2 == 1 {
- format, want, other = "xlsx", "unreadable workbook", "not a readable zip archive"
+ format, want, other = "xlsx", "not a readable workbook container", "not a readable zip archive"
}
_, err := Convert([]byte(fmt.Sprintf("garbage %d", i)), Options{Format: format})
if err == nil {
diff --git a/internal/infrastructure/docparser/anydoc/corpus_linked_test.go b/internal/infrastructure/docparser/anydoc/corpus_linked_test.go
new file mode 100644
index 000000000..f128d8678
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/corpus_linked_test.go
@@ -0,0 +1,89 @@
+//go:build anydoc && cgo
+
+package anydoc
+
+import (
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+
+ upstream "github.com/firecrawl/anydoc/go"
+)
+
+// ANYDOC_UPSTREAM_DIR points to the checked-out release tag. This opt-in
+// upgrade check runs the full upstream corpus through our actual Go ABI and
+// compares Markdown/error details to the upstream Rust snapshots.
+func TestUpstreamCorpus(t *testing.T) {
+ root := os.Getenv("ANYDOC_UPSTREAM_DIR")
+ if root == "" {
+ t.Skip("set ANYDOC_UPSTREAM_DIR to an anydoc release checkout")
+ }
+ snapshots, err := filepath.Glob(filepath.Join(root, "tests/snapshots/snapshots__*.snap"))
+ if err != nil || len(snapshots) == 0 {
+ t.Fatalf("no snapshots: %v", err)
+ }
+ for _, path := range snapshots {
+ name := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(path), "snapshots__"), ".snap")
+ parts := strings.Split(name, "__")
+ if len(parts) != 2 {
+ continue
+ }
+ t.Run(name, func(t *testing.T) {
+ data, err := os.ReadFile(filepath.Join(root, "tests/fixtures", parts[0], parts[1]))
+ if err != nil {
+ t.Fatal(err)
+ }
+ snapshot, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, expected, ok := strings.Cut(string(snapshot[4:]), "---\n")
+ if !ok {
+ t.Fatal("invalid snapshot header")
+ }
+ format, ok := upstream.FormatFromExtension(strings.TrimPrefix(filepath.Ext(parts[1]), "."))
+ if !ok {
+ t.Fatal("unrecognized fixture extension")
+ }
+ // The Go binding rejects empty slices before entering C, retaining
+ // its established empty-input error instead of the ZIP diagnostic.
+ if len(data) == 0 {
+ if !strings.HasPrefix(expected, "ERROR:") {
+ t.Fatal("empty fixture should fail")
+ }
+ expected = "ERROR: empty input"
+ }
+ actual, err := upstream.ToMarkdownBytes(data, &format)
+ if err != nil {
+ var typed *upstream.ConvertError
+ if !errors.As(err, &typed) {
+ t.Fatal(err)
+ }
+ actual = "ERROR: " + typed.Detail
+ }
+ if strings.TrimSpace(actual) != strings.TrimSpace(expected) {
+ t.Fatalf("snapshot mismatch\nwant: %s\ngot: %s", expected, actual)
+ }
+ if err == nil && format != upstream.FormatPdf {
+ document, err := upstream.ToDocument(data, &format)
+ if err != nil {
+ t.Fatalf("document ABI: %v", err)
+ }
+ markdown, err := upstream.ToMarkdownWithAssetLinks(data, &format)
+ if err != nil {
+ t.Fatalf("asset renderer: %v", err)
+ }
+ combined, combinedMarkdown, err := upstream.ToDocumentWithAssetLinks(data, &format)
+ if err != nil {
+ t.Fatalf("combined ABI: %v", err)
+ }
+ if !reflect.DeepEqual(combined, document) || combinedMarkdown != markdown {
+ t.Fatal("single-parse output differs from independent document/Markdown APIs")
+ }
+ }
+ })
+ }
+}
diff --git a/internal/infrastructure/docparser/anydoc/stress_linked_test.go b/internal/infrastructure/docparser/anydoc/stress_linked_test.go
new file mode 100644
index 000000000..127bb8649
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/stress_linked_test.go
@@ -0,0 +1,94 @@
+//go:build anydoc && cgo
+
+package anydoc
+
+import (
+ "bytes"
+ "os"
+ "runtime"
+ "sort"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+// Opt-in sustained mixed-format load through the production WithAssets path.
+// Use /usr/bin/time -l (macOS) or -v (Linux) to measure native peak RSS too:
+// Go's memory statistics do not include Rust allocations.
+func TestConcurrentConversionStress(t *testing.T) {
+ setting := os.Getenv("ANYDOC_STRESS_DURATION")
+ if setting == "" {
+ t.Skip("set ANYDOC_STRESS_DURATION, e.g. 30s")
+ }
+ duration, err := time.ParseDuration(setting)
+ if err != nil || duration <= 0 {
+ t.Fatalf("invalid stress duration %q", setting)
+ }
+ type input struct {
+ data []byte
+ opts Options
+ want *Result
+ }
+ inputs := make([]input, 0, len(benchmarkDocuments))
+ for _, name := range benchmarkDocuments {
+ format := strings.Split(name, "/")[0]
+ if format == "xls" || format == "xlsb" {
+ format = "xlsx"
+ }
+ i := input{data: readFixture(t, name), opts: Options{Format: format, WithAssets: true}}
+ i.want, err = Convert(i.data, i.opts)
+ if err != nil || i.want.AssetsError != nil {
+ t.Fatalf("preflight %s: %v", name, err)
+ }
+ inputs = append(inputs, i)
+ }
+ const workers = 8
+ samples := make([][]time.Duration, workers)
+ var wg sync.WaitGroup
+ start := time.Now()
+ deadline := start.Add(duration)
+ for worker := 0; worker < workers; worker++ {
+ wg.Add(1)
+ go func(worker int) {
+ defer wg.Done()
+ for n := worker; time.Now().Before(deadline); n++ {
+ i := inputs[n%len(inputs)]
+ began := time.Now()
+ got, err := Convert(i.data, i.opts)
+ samples[worker] = append(samples[worker], time.Since(began))
+ if err != nil ||
+ got.AssetsError != nil ||
+ got.Markdown != i.want.Markdown ||
+ len(got.Assets) != len(i.want.Assets) {
+ t.Errorf("worker %d: inconsistent conversion, err=%v", worker, err)
+ return
+ }
+ for j, asset := range got.Assets {
+ if !bytes.Equal(asset.Data, i.want.Assets[j].Data) ||
+ asset.Name != i.want.Assets[j].Name {
+ t.Errorf("worker %d: inconsistent asset %d", worker, j)
+ return
+ }
+ }
+ }
+ }(worker)
+ }
+ wg.Wait()
+ elapsed := time.Since(start)
+ var latencies []time.Duration
+ for _, sample := range samples {
+ latencies = append(latencies, sample...)
+ }
+ if len(latencies) == 0 {
+ t.Fatal("no conversions completed")
+ }
+ sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] })
+ var memory runtime.MemStats
+ runtime.ReadMemStats(&memory)
+ t.Logf("workers=%d conversions=%d elapsed=%s throughput=%.1f docs/s "+
+ "p50=%s p95=%s p99=%s GoHeapAlloc=%d GoTotalAlloc=%d",
+ workers, len(latencies), elapsed, float64(len(latencies))/elapsed.Seconds(),
+ latencies[len(latencies)*50/100], latencies[len(latencies)*95/100], latencies[len(latencies)*99/100],
+ memory.HeapAlloc, memory.TotalAlloc)
+}
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/.gitattributes b/internal/infrastructure/docparser/anydoc/testdata/upstream/.gitattributes
new file mode 100644
index 000000000..20915656e
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/.gitattributes
@@ -0,0 +1,2 @@
+# Preserve PDF byte offsets and fixed-width xref records from upstream fixtures.
+*.pdf binary
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/LICENSE b/internal/infrastructure/docparser/anydoc/testdata/upstream/LICENSE
new file mode 100644
index 000000000..7ad9f5abd
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sideguide Technologies Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/README.md b/internal/infrastructure/docparser/anydoc/testdata/upstream/README.md
new file mode 100644
index 000000000..19cdac28e
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/README.md
@@ -0,0 +1 @@
+Fixtures from [firecrawl/anydoc v0.2.4](https://github.com/firecrawl/anydoc/tree/42bf1c5ecdde9eb0d96d6bd75a9e6698cf93b14c/tests/fixtures), under the included MIT license. Used for Go ABI regression tests and repeatable conversion benchmarks. Files are copied unchanged.
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/csv/sheet.csv b/internal/infrastructure/docparser/anydoc/testdata/upstream/csv/sheet.csv
new file mode 100644
index 000000000..ab50c8756
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/csv/sheet.csv
@@ -0,0 +1,8 @@
+Kind,Value,Note
+Percent,15.5%,fifteen and a half
+Currency,1234.5,dollars
+Thousands,9876543,grouped
+Date,2026-03-15,ides of March
+Duration,26:30:15,over a day
+Tiny,0.0000004,four ten-millionths
+Boolean,TRUE,yes
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/doc/text.doc b/internal/infrastructure/docparser/anydoc/testdata/upstream/doc/text.doc
new file mode 100644
index 000000000..7de1e813f
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/doc/text.doc differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/handmade-math.docx b/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/handmade-math.docx
new file mode 100644
index 000000000..736ed3f7f
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/handmade-math.docx differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/text.docx b/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/text.docx
new file mode 100644
index 000000000..0d556258d
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/docx/text.docx differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/book.epub b/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/book.epub
new file mode 100644
index 000000000..430296407
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/book.epub differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/handmade-math.epub b/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/handmade-math.epub
new file mode 100644
index 000000000..9df629c07
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/epub/handmade-math.epub differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/odp/pres.odp b/internal/infrastructure/docparser/anydoc/testdata/upstream/odp/pres.odp
new file mode 100644
index 000000000..45d72f1a4
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/odp/pres.odp differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/ods/sheet.ods b/internal/infrastructure/docparser/anydoc/testdata/upstream/ods/sheet.ods
new file mode 100644
index 000000000..5c1558cc1
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/ods/sheet.ods differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/handmade-math.odt b/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/handmade-math.odt
new file mode 100644
index 000000000..98f34c180
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/handmade-math.odt differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/text.odt b/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/text.odt
new file mode 100644
index 000000000..a9fdba542
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/odt/text.odt differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-mixed.pdf b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-mixed.pdf
new file mode 100644
index 000000000..7b7164dad
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-mixed.pdf
@@ -0,0 +1,50 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [5 0 R 7 0 R] /Count 2 >>
+endobj
+3 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+4 0 obj
+<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>
+stream
+�
+endstream
+endobj
+5 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 6 0 R >>
+endobj
+6 0 obj
+<< /Length 53 >>
+stream
+BT /F1 24 Tf 72 700 Td (Text on the first page) Tj ET
+endstream
+endobj
+7 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 8 0 R >>
+endobj
+8 0 obj
+<< /Length 32 >>
+stream
+q 468 0 0 648 72 72 cm /Im1 Do Q
+endstream
+endobj
+xref
+0 9
+0000000000 65535 f
+0000000009 00000 n
+0000000058 00000 n
+0000000121 00000 n
+0000000191 00000 n
+0000000335 00000 n
+0000000487 00000 n
+0000000590 00000 n
+0000000742 00000 n
+trailer
+<< /Size 9 /Root 1 0 R >>
+startxref
+824
+%%EOF
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-scanned.pdf b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-scanned.pdf
new file mode 100644
index 000000000..071281ba3
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/handmade-scanned.pdf
@@ -0,0 +1,50 @@
+%PDF-1.4
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [5 0 R 7 0 R] /Count 2 >>
+endobj
+3 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+4 0 obj
+<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>
+stream
+�
+endstream
+endobj
+5 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 6 0 R >>
+endobj
+6 0 obj
+<< /Length 32 >>
+stream
+q 468 0 0 648 72 72 cm /Im1 Do Q
+endstream
+endobj
+7 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents 8 0 R >>
+endobj
+8 0 obj
+<< /Length 32 >>
+stream
+q 468 0 0 648 72 72 cm /Im1 Do Q
+endstream
+endobj
+xref
+0 9
+0000000000 65535 f
+0000000009 00000 n
+0000000058 00000 n
+0000000121 00000 n
+0000000191 00000 n
+0000000335 00000 n
+0000000487 00000 n
+0000000569 00000 n
+0000000721 00000 n
+trailer
+<< /Size 9 /Root 1 0 R >>
+startxref
+803
+%%EOF
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/text.pdf b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/text.pdf
new file mode 100644
index 000000000..557e1d2d7
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/pdf/text.pdf differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/ppt/pres.ppt b/internal/infrastructure/docparser/anydoc/testdata/upstream/ppt/pres.ppt
new file mode 100644
index 000000000..1e0b433e9
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/ppt/pres.ppt differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/handmade-math.pptx b/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/handmade-math.pptx
new file mode 100644
index 000000000..277c97aa0
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/handmade-math.pptx differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/pres.pptx b/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/pres.pptx
new file mode 100644
index 000000000..de93ada99
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/pptx/pres.pptx differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/handmade-math.rtf b/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/handmade-math.rtf
new file mode 100644
index 000000000..6ed5fe9bc
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/handmade-math.rtf
@@ -0,0 +1,5 @@
+{\rtf1\ansi\ansicpg1252\deff0{\fonttbl{\f0 Cambria Math;}}
+\pard The roots are {\mmath{\*\moMath{\rtlch\fcs1 \af0 \ltrch\fcs0 \f0\insrsid1 }{\mr\mscr0\msty2 x}{\mr\mscr0\msty2 =}{\mf{\mfPr{\mctrlPr\f0 }}{\mnum{\mr\mscr0\msty2 -b\'b1}{\mrad{\mradPr{\mdegHide on}{\mctrlPr\f0 }}{\mdeg}{\me{\msSup{\me{\mr\mscr0\msty2 b}}{\msup{\mr\mscr0\msty2 2}}}{\mr\mscr0\msty2 -4ac}}}}{\mden{\mr\mscr0\msty2 2a}}}}{\mmathPict{\*\mmathPr}{\pict\pngblip\picw1\pich1 89504e47}}} for any a.\par
+\pard {\mmath{\*\moMathPara{\moMathParaPr{\mjc centerGroup}}{\*\moMath{\mnary{\mnaryPr{\mchr \u8721 ?}{\mlimLoc undOvr}{\mctrlPr\f0 }}{\msub{\mr\mscr0\msty2 i=1}}{\msup{\mr\mscr0\msty2 n}}{\me{\msSub{\me{\mr\mscr0\msty2 x}}{\msub{\mr\mscr0\msty2 i}}}}}{\mr\mscr0\msty2 =}{\mfunc{\mfName{\mr\mscr0\msty0 sin}}{\me{\md{\mdPr{\mctrlPr\f0 }}{\me{\mr\mscr0\msty2 \u952 ?}}}}}}{\*\moMath{\mr\mnor where }{\mr\mscr0\msty2 \u8722 ?1<x}}}}\par
+\pard A price of $5 or $6 is not math.\par
+}
\ No newline at end of file
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/text.rtf b/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/text.rtf
new file mode 100644
index 000000000..24fc6caa9
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/testdata/upstream/rtf/text.rtf
@@ -0,0 +1,177 @@
+{\rtf1\ansi\deff3\adeflang1025
+{\fonttbl{\f0\froman\fprq2\fcharset0 Times New Roman;}{\f1\froman\fprq2\fcharset2 Symbol;}{\f2\fswiss\fprq2\fcharset0 Arial;}{\f3\froman\fprq2\fcharset0 Liberation Serif{\*\falt Times New Roman};}{\f4\fswiss\fprq2\fcharset0 Liberation Sans{\*\falt Arial};}{\f5\fnil\fprq2\fcharset0 Microsoft YaHei;}{\f6\fnil\fprq2\fcharset0 Arial;}{\f7\fswiss\fprq0\fcharset128 Arial;}}
+{\colortbl;\red0\green0\blue0;\red0\green0\blue255;\red0\green255\blue255;\red0\green255\blue0;\red255\green0\blue255;\red255\green0\blue0;\red255\green255\blue0;\red255\green255\blue255;\red0\green0\blue128;\red0\green128\blue128;\red0\green128\blue0;\red128\green0\blue128;\red128\green0\blue0;\red128\green128\blue0;\red128\green128\blue128;\red192\green192\blue192;}
+{\stylesheet{\s0\snext0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052 Normal;}
+{\s1\snext25\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl0\outlinelevel0\nowidctlpar\hyphpar0\ltrpar\fs36\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052 heading 1;}
+{\s2\snext25\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052 heading 2;}
+{\s3\snext25\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl2\outlinelevel2\nowidctlpar\hyphpar0\ltrpar\fs26\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052 heading 3;}
+{\*\cs15\snext15\loch\b0 NotBold;}
+{\*\cs16\snext16\loch\b Bold;}
+{\*\cs17\snext17\loch\i Italic;}
+{\*\cs18\snext18\loch\strike Struck;}
+{\*\cs19\snext19\loch\cf9\ul\ulc0 Hyperlink;}
+{\*\cs20\snext20 Footnote Characters;}
+{\*\cs21\snext21\loch\super footnote reference;}
+{\*\cs22\snext22 Endnote Characters;}
+{\*\cs23\snext23\loch\super endnote reference;}
+{\s24\sbasedon0\snext25\rtlch\af6\afs28 \ltrch\hich\af4\loch\sb240\sa120\keepn\f4\fs28\dbch\af5 Heading;}
+{\s25\sbasedon0\snext25\loch\sl276\slmult1\sb0\sa140 Body Text;}
+{\s26\sbasedon25\snext26\rtlch\af7 \ltrch List;}
+{\s27\sbasedon0\snext27\rtlch\af7\afs24\ai \ltrch\loch\sb120\sa120\noline\fs24\i caption;}
+{\s28\sbasedon0\snext28\rtlch\af7 \ltrch\loch\noline Index;}
+{\s29\snext29\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\b\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052 StrongPara;}
+{\s30\sbasedon0\snext30\loch\nowidctlpar\noline Table Contents;}
+{\s31\sbasedon0\snext31\rtlch\afs20 \ltrch\loch\fi-340\li340\lin340\noline\fs20 footnote text;}
+{\s32\sbasedon0\snext32\rtlch\afs20 \ltrch\loch\fi-340\li340\lin340\noline\fs20 endnote text;}
+{\s33\sbasedon0\snext33 Frame Contents;}
+}{\*\listtable{\list\listtemplateid1
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}
+{\listlevel\levelnfc255\leveljc0\levelstartat1\levelfollow2{\leveltext \'00;}{\levelnumbers;}\fi0\li0}\listid1}
+{\list\listtemplateid2
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow2{\leveltext \'02\'00.;}{\levelnumbers\'01;}\fi0\li0}
+{\listlevel\levelnfc4\leveljc0\levelstartat1\levelfollow2{\leveltext \'02\'01);}{\levelnumbers\'01;}\fi0\li0}
+{\listlevel\levelnfc2\leveljc0\levelstartat1\levelfollow2{\leveltext \'02\'02.;}{\levelnumbers\'01;}\fi0\li0}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'03.;}{\levelnumbers\'01;}\fi-360\li1800}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'04.;}{\levelnumbers\'01;}\fi-360\li2160}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'05.;}{\levelnumbers\'01;}\fi-360\li2520}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'06.;}{\levelnumbers\'01;}\fi-360\li2880}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'07.;}{\levelnumbers\'01;}\fi-360\li3240}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'08.;}{\levelnumbers\'01;}\fi-360\li3600}\listid2}
+{\list\listtemplateid3
+{\listlevel\levelnfc23\leveljc0\levelstartat1\levelfollow2{\leveltext \'01\u8226 ?;}{\levelnumbers;}\f9\fi0\li0}
+{\listlevel\levelnfc23\leveljc0\levelstartat1\levelfollow2{\leveltext \'01\u9702 ?;}{\levelnumbers;}\f9\fi0\li0}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'02.;}{\levelnumbers\'01;}\fi-360\li1440}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'03.;}{\levelnumbers\'01;}\fi-360\li1800}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'04.;}{\levelnumbers\'01;}\fi-360\li2160}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'05.;}{\levelnumbers\'01;}\fi-360\li2520}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'06.;}{\levelnumbers\'01;}\fi-360\li2880}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'07.;}{\levelnumbers\'01;}\fi-360\li3240}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'08.;}{\levelnumbers\'01;}\fi-360\li3600}\listid3}
+{\list\listtemplateid4
+{\listlevel\levelnfc1\leveljc0\levelstartat1\levelfollow2{\leveltext \'02\'00.;}{\levelnumbers\'01;}\fi0\li0}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'01.;}{\levelnumbers\'01;}\fi-360\li1080}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'02.;}{\levelnumbers\'01;}\fi-360\li1440}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'03.;}{\levelnumbers\'01;}\fi-360\li1800}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'04.;}{\levelnumbers\'01;}\fi-360\li2160}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'05.;}{\levelnumbers\'01;}\fi-360\li2520}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'06.;}{\levelnumbers\'01;}\fi-360\li2880}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'07.;}{\levelnumbers\'01;}\fi-360\li3240}
+{\listlevel\levelnfc0\leveljc0\levelstartat1\levelfollow0{\leveltext \'02\'08.;}{\levelnumbers\'01;}\fi-360\li3600}\listid4}
+}{\listoverridetable{\listoverride\listid1\listoverridecount0\ls1}{\listoverride\listid2\listoverridecount0\ls2}{\listoverride\listid3\listoverridecount0\ls3}{\listoverride\listid4\listoverridecount0\ls4}}{\*\generator LibreOffice/25.2.7.2$Windows_X86_64 LibreOffice_project/5cbfd1ab6520636bb5f7b99185aa69bd7456825d}{\info{\creatim\yr0\mo0\dy0\hr0\min0}{\revtim\yr0\mo0\dy0\hr0\min0}{\printim\yr0\mo0\dy0\hr0\min0}}{\*\userprops}\deftab709\deftab709\deftab709\deftab709\deftab709
+\hyphauto1\viewscale100\formshade\nobrkwrptbl\paperh16838\paperw11906\margl1134\margr1134\margt1134\margb1134\sectd\sbknone\sftnnar\saftnnrlc\sectunlocked1\pgwsxn11906\pghsxn16838\marglsxn1134\margrsxn1134\margtsxn1134\margbsxn1134\ftnbj\ftnstart1\ftnrstcont\ftnnar\fet2\aftnrstcont\aftnstart1\aftnnrlc
+{\*\ftnsep\chftnsep}\pgndec\pard\plain \s1\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl0\outlinelevel0\nowidctlpar\hyphpar0\ltrpar\fs36\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch
+Fixture Document}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Plain paragraph with }{\loch\cs16\loch\b\loch
+bold}{\loch
+, }{\loch\cs17\loch\i\loch
+italic}{\loch
+, and }{\loch\cs18\loch\strike\loch
+struck}{\loch
+ runs.}
+\par \pard\plain \s29\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\b\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Style-bold paragraph with a }{\loch\cs15\loch\b0\loch
+NotBold-styled span}{\loch
+ inside.}
+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch
+Lists}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain 1.\tab}\ilvl0\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+First numbered}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain 2.\tab}\ilvl0\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Second numbered}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain a)\tab}\ilvl1\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Alpha sub one}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain b)\tab}\ilvl1\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Alpha sub two}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain i.\tab}\ilvl2\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Roman sub sub}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain 3.\tab}\ilvl0\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Third numbered}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Interrupting paragraph between lists.}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain 4.\tab}\ilvl0\ls2 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Fourth, continuing the count}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain IV.\tab}\ilvl0\ls4 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Roman starting at four}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain V.\tab}\ilvl0\ls4 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Roman five}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \u8226\'95\tab}\ilvl0\ls3 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Bullet one}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \u8226\'95\tab}\ilvl0\ls3 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Bullet two}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \u9702\'3f\tab}\ilvl1\ls3 \fi0\li0\lin0\ql\tx0\fi0\li0\lin0\ltrpar{\loch
+Nested bullet}
+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch
+Table}
+\par \trowd\trql\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\cellx6425\cellx9638\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch
+Wide head}\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch
+End}\cell\row\pard \trowd\trql\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\clvmgf\cellx3213\cellx6425\cellx9638\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch
+Tall}\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch
+B2}\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch
+C2}\cell\row\pard \trowd\trql\ltrrow\trpaddft3\trpaddt0\trpaddfl3\trpaddl0\trpaddfb3\trpaddb0\trpaddfr3\trpaddr0\clvmrg\cellx3213\cellx6425\cellx9638\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar\loch
+\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch
+B3}\cell\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\intbl\ql\ltrpar{\loch
+C3}\cell\row\pard \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch
+Notes and special text}
+\par \ql\fi-340\li340\lin340\noline\ltrpar\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Music clef \uc0 \u55348\uc1 \u56606\'3f appears before this footnote}{{\super \loch \chftn{\*\footnote \chftn\pard\plain \s31\rtlch\afs20 \ltrch\loch\fi-340\li340\lin340\noline\fs20{\loch
+\tab Footnote after an astral character.}
+}}
+}{\loch
+ reference.}
+\par \ql\fi-340\li340\lin340\noline\ltrpar\pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+An endnote follows here}{{\super \loch \chftn{\*\footnote\ftnalt \chftn\pard\plain \s32\rtlch\afs20 \ltrch\loch\fi-340\li340\lin340\noline\fs20{\loch
+\tab Endnote body text.}
+}}
+}{\loch
+.}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Persian with ZWNJ: }{
+\u1605\'3f\u1740\'3f\u8204\'3f\u1582\'3f\u1608\'3f\u1575\'3f\u1607\'3f\u1605\'3f}{\loch
+. Family emoji: \uc0 \u55357\uc1 \u56424\'3f\u8205\'3f\uc0 \u55357\uc1 \u56425\'3f\u8205\'3f\uc0 \u55357\uc1 \u56423\'3f.}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Markdown specials: *stars* _under_ [bracket] `tick` #hash 1. dotted | pipe.}
+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch
+Links and anchors}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+External link to }{{\field{\*\fldinst HYPERLINK "https://example.com/page" }{\fldrslt {\loch\loch\cf9\ul\ulc0\loch
+example}{}}}\loch
+.}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Relative link to }{{\field{\*\fldinst HYPERLINK "file:///anydoc/tests/fixture-src/sibling.odt" }{\fldrslt {\loch\loch\cf9\ul\ulc0\loch
+a sibling file}{}}}\loch
+.}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+{\*\bkmkstart plainmark}{\*\bkmkend plainmark}This plain paragraph carries a bookmark.}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Jump to }{{\field{\*\fldinst HYPERLINK "#plainmark" }{\fldrslt {\loch\loch\cf9\ul\ulc0\loch
+the bookmarked paragraph}{}}}\loch
+.}
+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch
+Objects}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Inline image: }{\loch
+{\pict{\*\picprop{\sp{\sn wzDescription}{\sv tiny red dot}}{\sp{\sn wzName}{\sv }}}\picscalex1700\picscaley1700\piccropl0\piccropr0\piccropt0\piccropb0\picw1\pich1\picwgoal10\pichgoal10\pngblip
+89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c4890000000d4944415478da63fccfc0500f000485018084a98c21000000004945
+4e44ae426082}
+}{\loch
+ done.}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Text box: }{
+{\shp{\*\shpinst\shpwr2\shpwrk0\shpbypara\shpbyignore\shptop-1362\shpbottom-228\shpbxcolumn\shpbxignore\shpleft0\shpright2835\shpz1{\sp{\sn shapeType}{\sv 202}}{\sp{\sn dxWrapDistLeft}{\sv 72390}}{\sp{\sn dxWrapDistRight}{\sv 72390}}{\sp{\sn dyWrapDistTop}{\sv 72390}}{\sp{\sn dyWrapDistBottom}{\sv 72390}}{\sp{\sn posrelv}{\sv 2}}{\sp{\sn posv}{\sv 1}}{\sp{\sn posrelh}{\sv 2}}{\sp{\sn posh}{\sv 2}}{\sp{\sn dxTextLeft}{\sv 53975}}{\sp{\sn dyTextTop}{\sv 53975}}{\sp{\sn dxTextRight}{\sv 53975}}{\sp{\sn dyTextBottom}{\sv 53975}}{\sp{\sn lineColor}{\sv 0}}{\sp{\sn lineWidth}{\sv 635}}{\shptxt\s33\ql{\loch
+Inside the text box.}
+\par \pard}}}
+}{\loch
+ after the box.}
+\par \pard\plain \s2\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\ilvl1\outlinelevel1\nowidctlpar\hyphpar0\ltrpar\fs30\b\cf0\f3\lang2057\kerning1\dbch\af8\langfe2052{\listtext\pard\plain \tab}\ls1 \fi0\li0\lin0\ql\fi0\li0\lin0\ltrpar{\loch
+Quote and code}
+\par \pard\plain \s0\rtlch\af6\afs24\alang1081 \ltrch\lang2057\langfe2052\hich\af3\loch\nowidctlpar\hyphpar0\ltrpar\cf0\f3\fs24\lang2057\kerning1\dbch\af8\langfe2052\ql\ltrpar{\loch
+Value below one millionth: 0.0000004 should survive.}
+\par }
\ No newline at end of file
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/xls/sheet.xls b/internal/infrastructure/docparser/anydoc/testdata/upstream/xls/sheet.xls
new file mode 100644
index 000000000..6d830f15a
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/xls/sheet.xls differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsb/handmade-sheet.xlsb b/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsb/handmade-sheet.xlsb
new file mode 100644
index 000000000..e69c29451
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsb/handmade-sheet.xlsb differ
diff --git a/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsx/sheet.xlsx b/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsx/sheet.xlsx
new file mode 100644
index 000000000..44dba2b51
Binary files /dev/null and b/internal/infrastructure/docparser/anydoc/testdata/upstream/xlsx/sheet.xlsx differ
diff --git a/internal/infrastructure/docparser/anydoc/upgrade_linked_test.go b/internal/infrastructure/docparser/anydoc/upgrade_linked_test.go
new file mode 100644
index 000000000..d5eaa2a0e
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/upgrade_linked_test.go
@@ -0,0 +1,225 @@
+//go:build anydoc && cgo
+
+package anydoc
+
+import (
+ "archive/zip"
+ "bytes"
+ "errors"
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+
+ upstream "github.com/firecrawl/anydoc/go"
+)
+
+func TestConvertEquations(t *testing.T) {
+ for _, format := range []string{"docx", "pptx", "odt", "epub", "rtf"} {
+ t.Run(format, func(t *testing.T) {
+ data := readFixture(t, format+"/handmade-math."+format)
+ for _, assets := range []bool{false, true} {
+ result, err := Convert(data, Options{Format: format, WithAssets: assets})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.AssetsError != nil {
+ t.Fatalf("document ABI failed: %v", result.AssetsError)
+ }
+ if !strings.Contains(result.Markdown, `\frac`) ||
+ !strings.Contains(result.Markdown, "$") {
+ t.Fatalf("equation lost (assets=%v): %s", assets, result.Markdown)
+ }
+ }
+ })
+ }
+ // Check the actual decoded fields, not only the independent Markdown path.
+ doc, err := upstream.ToDocument(readFixture(t, "docx/handmade-math.docx"), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var inline, block bool
+ for _, b := range doc.Blocks {
+ block = block || b.Kind == "math" && b.Text != nil && strings.Contains(*b.Text, `\sum`)
+ for _, in := range b.Content {
+ inline = inline || in.Kind == "math" && in.Text != nil && strings.Contains(*in.Text, `\frac`)
+ }
+ }
+ if !inline || !block {
+ t.Fatalf("decoded equation nodes missing: inline=%v block=%v", inline, block)
+ }
+}
+
+func TestConvertPDFReportsScannedPages(t *testing.T) {
+ for _, tc := range []struct{ file, detail string }{
+ {"handmade-mixed.pdf", "page 2 of 2 needs OCR"},
+ {"handmade-scanned.pdf", "all 2 pages need OCR"},
+ } {
+ t.Run(tc.file, func(t *testing.T) {
+ result, err := Convert(readFixture(t, "pdf/"+tc.file), Options{Format: "pdf", WithAssets: true})
+ var typed *upstream.ConvertError
+ if result != nil ||
+ !errors.As(err, &typed) ||
+ typed.Kind != "needs_ocr" ||
+ !strings.Contains(typed.Detail, tc.detail) {
+ t.Fatalf("want scanned-page error, got result=%+v err=%v", result, err)
+ }
+ if !PDFNeedsOCR(fmt.Errorf("reader: %w", err)) {
+ t.Fatal("wrapped typed error did not trigger OCR fallback")
+ }
+ })
+ }
+ if !PDFNeedsOCR(&upstream.ConvertError{Kind: "needs_ocr", Detail: "localized detail"}) {
+ t.Fatal("OCR detection still depends on English wording")
+ }
+ if PDFNeedsOCR(&upstream.ConvertError{Kind: "malformed", Detail: "invalid PDF"}) {
+ t.Fatal("malformed PDF incorrectly triggers OCR")
+ }
+}
+
+func TestConvertSpreadsheetCheckboxes(t *testing.T) {
+ // Add VML form controls to an upstream workbook. Both boolean states must
+ // survive Rust encoding and Go decoding, including subsequent table cells.
+ data := readFixture(t, "xlsx/sheet.xlsx")
+ reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var out bytes.Buffer
+ w := zip.NewWriter(&out)
+ for _, f := range reader.File {
+ if f.Name == "xl/worksheets/_rels/sheet1.xml.rels" {
+ continue
+ }
+ if err := w.Copy(f); err != nil {
+ t.Fatal(err)
+ }
+ }
+ parts := map[string]string{
+ "xl/worksheets/_rels/sheet1.xml.rels": `<Relationships ` +
+ `xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><` +
+ `Relationship Id="checkboxes" ` +
+ `Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/vmlDrawing"` +
+ ` Target="../drawings/checkboxes.vml"/></Relationships>`,
+ "xl/drawings/checkboxes.vml": `<xml xmlns:v="urn:schemas-microsoft-com:vml" ` +
+ `xmlns:x="urn:schemas-microsoft-com:office:excel"><v:shape><v:textbox>Approved</v:text` +
+ `box><x:ClientData ObjectType="Checkbox"><x:Anchor>0,0,0,0,1,0,1,0</x:Anchor><x:Checke` +
+ `d>1</x:Checked></x:ClientData></v:shape><v:shape><v:textbox>Pending</v:textbox><x:Cli` +
+ `entData ObjectType="Checkbox"><x:Anchor>1,0,0,0,2,0,1,0</x:Anchor></x:ClientData></v:` +
+ `shape></xml>`,
+ }
+ for name, content := range parts {
+ part, err := w.Create(name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := part.Write([]byte(content)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := w.Close(); err != nil {
+ t.Fatal(err)
+ }
+ result, err := Convert(out.Bytes(), Options{Format: "xlsx", WithAssets: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.AssetsError != nil ||
+ !strings.Contains(result.Markdown, "[x] Approved") ||
+ !strings.Contains(result.Markdown, "[ ] Pending") {
+ t.Fatalf("checkbox conversion: %+v", result)
+ }
+ doc, err := upstream.ToDocument(out.Bytes(), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ states := map[bool]bool{}
+ for _, block := range doc.Blocks {
+ if block.Table == nil {
+ continue
+ }
+ for _, row := range block.Table.Grid {
+ for _, slot := range row {
+ if slot.Cell == nil {
+ continue
+ }
+ for _, b := range slot.Cell.Blocks {
+ for _, in := range b.Content {
+ if in.Kind == "checkbox" && in.Checked != nil {
+ states[*in.Checked] = true
+ }
+ }
+ }
+ }
+ }
+ }
+ if !states[false] || !states[true] {
+ t.Fatalf("decoded checkbox states: %v", states)
+ }
+}
+
+func TestConvertAllBenchmarkFormats(t *testing.T) {
+ for _, name := range benchmarkDocuments {
+ t.Run(name, func(t *testing.T) {
+ format := strings.Split(name, "/")[0]
+ if format == "xls" || format == "xlsb" {
+ format = "xlsx"
+ }
+ result, err := Convert(readFixture(t, name), Options{Format: format, WithAssets: true})
+ if err != nil || result.Markdown == "" || result.AssetsError != nil {
+ t.Fatalf("conversion: result=%+v err=%v", result, err)
+ }
+ })
+ }
+}
+
+func TestCombinedConversionPreservesModelAndMarkdown(t *testing.T) {
+ // Exercise auto-detection as well as all normal formats, without requiring
+ // an external upstream checkout for this regression in CI.
+ for _, name := range benchmarkDocuments {
+ if strings.HasPrefix(name, "pdf/") {
+ continue
+ }
+ t.Run(name, func(t *testing.T) {
+ data := readFixture(t, name)
+ var format *upstream.Format
+ if strings.HasPrefix(name, "csv/") {
+ f := upstream.FormatCsv
+ format = &f
+ }
+ want, err := upstream.ToDocument(data, format)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wantMarkdown, err := upstream.ToMarkdownWithAssetLinks(data, format)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, markdown, err := upstream.ToDocumentWithAssetLinks(data, format)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(got, want) || markdown != wantMarkdown {
+ t.Fatal("single parse changed model or Markdown")
+ }
+ })
+ }
+ for _, tc := range []struct {
+ name string
+ data []byte
+ format *upstream.Format
+ kind string
+ }{
+ {"empty", nil, nil, "unsupported"},
+ {"garbage", []byte("garbage"), nil, "unsupported"},
+ {"pdf", minimalPDF(), nil, "pdf_no_model"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ doc, md, err := upstream.ToDocumentWithAssetLinks(tc.data, tc.format)
+ var typed *upstream.ConvertError
+ if doc != nil || md != "" || !errors.As(err, &typed) || typed.Kind != tc.kind {
+ t.Fatalf("error contract: doc=%+v md=%q err=%v", doc, md, err)
+ }
+ })
+ }
+}
diff --git a/internal/infrastructure/docparser/anydoc_reader.go b/internal/infrastructure/docparser/anydoc_reader.go
index 10a38e472..8ae7bf01a 100644
--- a/internal/infrastructure/docparser/anydoc_reader.go
+++ b/internal/infrastructure/docparser/anydoc_reader.go
@@ -89,7 +89,8 @@ func (r *AnydocReader) readScannedPDF(ctx context.Context, req *types.ReadReques
return nil, fmt.Errorf("anydoc conversion failed for %q: PDF has no extractable text; OCR is required", req.FileName)
}
- logger.Infof(ctx, "[anydoc] %q has no text layer, falling back to builtin for scanned-PDF OCR", req.FileName)
+ logger.Infof(ctx, "[anydoc] %q contains pages requiring OCR, falling back to builtin for scanned-PDF OCR",
+ req.FileName)
fallbackReq := *req
fallbackReq.ParserEngine = BuiltinEngineName
result, err := r.fallback.Read(ctx, &fallbackReq)
diff --git a/internal/infrastructure/docparser/anydoc_reader_linked_test.go b/internal/infrastructure/docparser/anydoc_reader_linked_test.go
new file mode 100644
index 000000000..ad3c6efb2
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc_reader_linked_test.go
@@ -0,0 +1,35 @@
+//go:build anydoc && cgo
+
+package docparser
+
+import (
+ "bytes"
+ "context"
+ "os"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+func TestAnydocReaderMixedPDFFallsBackThroughRead(t *testing.T) {
+ data, err := os.ReadFile("anydoc/testdata/upstream/pdf/handmade-mixed.pdf")
+ if err != nil {
+ t.Fatal(err)
+ }
+ fallback := &stubDocReader{result: &types.ReadResult{MarkdownContent: "Both pages including OCR"}}
+ req := &types.ReadRequest{FileName: "mixed.pdf", FileType: "pdf", FileContent: data}
+ result, err := NewAnydocReader(nil, fallback).Read(context.Background(), req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fallback.got == nil ||
+ !bytes.Equal(fallback.got.FileContent, req.FileContent) ||
+ fallback.got.ParserEngine != BuiltinEngineName ||
+ result.MarkdownContent != "Both pages including OCR" ||
+ result.Metadata["anydoc_fallback"] != "scanned_pdf" {
+ t.Fatalf("mixed PDF did not reach fallback intact: %+v", result)
+ }
+ if result, err := NewAnydocReader(nil, nil).Read(context.Background(), req); err == nil || result != nil {
+ t.Fatalf("mixed PDF without OCR returned partial success: result=%+v err=%v", result, err)
+ }
+}
diff --git a/third_party/anydoc-go/Cargo.lock b/third_party/anydoc-go/Cargo.lock
index b33122f56..1dd6acad7 100644
--- a/third_party/anydoc-go/Cargo.lock
+++ b/third_party/anydoc-go/Cargo.lock
@@ -89,9 +89,8 @@ dependencies = [
[[package]]
name = "anydoc"
-version = "0.1.9"
+version = "0.2.4"
dependencies = [
- "calamine",
"cfb",
"csv",
"encoding_rs",
@@ -104,22 +103,12 @@ dependencies = [
[[package]]
name = "anydoc-go"
-version = "0.1.9"
+version = "0.2.4"
dependencies = [
"anydoc",
"cbindgen",
]
-[[package]]
-name = "atoi_simd"
-version = "0.18.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44"
-dependencies = [
- "debug_unsafe",
- "rustversion",
-]
-
[[package]]
name = "autocfg"
version = "1.5.1"
@@ -162,30 +151,6 @@ version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
-[[package]]
-name = "calamine"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3"
-dependencies = [
- "atoi_simd",
- "byteorder",
- "chrono",
- "codepage",
- "encoding_rs",
- "fast-float2",
- "log",
- "quick-xml",
- "serde",
- "zip",
-]
-
[[package]]
name = "cbc"
version = "0.1.2"
@@ -243,9 +208,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "chacha20"
-version = "0.10.1"
+version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
+checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
@@ -260,7 +225,6 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327"
dependencies = [
"iana-time-zone",
"num-traits",
- "serde",
"windows-link",
]
@@ -301,15 +265,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
-[[package]]
-name = "codepage"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4"
-dependencies = [
- "encoding_rs",
-]
-
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -405,12 +360,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "debug_unsafe"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2"
-
[[package]]
name = "defmt"
version = "1.1.1"
@@ -521,12 +470,6 @@ dependencies = [
"windows-sys",
]
-[[package]]
-name = "fast-float2"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
-
[[package]]
name = "fastrand"
version = "2.5.0"
@@ -926,7 +869,6 @@ version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
- "encoding_rs",
"memchr",
]
diff --git a/third_party/anydoc-go/Cargo.toml b/third_party/anydoc-go/Cargo.toml
index 945ef5ff8..a3e996023 100644
--- a/third_party/anydoc-go/Cargo.toml
+++ b/third_party/anydoc-go/Cargo.toml
@@ -5,7 +5,7 @@
# symbols from, not a loadable shared object.
[package]
name = "anydoc-go"
-version = "0.1.9"
+version = "0.2.4"
edition = "2024"
description = "Go bindings (C ABI) for anydoc"
license = "MIT"
@@ -19,7 +19,7 @@ crate-type = ["staticlib"]
# workspace, so it takes the published crate rather than a path dependency.
# Pinned exactly: the C ABI mirrors this version's error and format enums.
[dependencies]
-anydoc = "=0.1.9"
+anydoc = "=0.2.4"
[build-dependencies]
cbindgen = "0.29"
diff --git a/third_party/anydoc-go/README.md b/third_party/anydoc-go/README.md
index 7448e0254..953d3c64c 100644
--- a/third_party/anydoc-go/README.md
+++ b/third_party/anydoc-go/README.md
@@ -18,9 +18,11 @@ not exist yet. So the binding source lives here and `go.mod` points at it:
replace github.com/firecrawl/anydoc/go => ./third_party/anydoc-go
```
-When the upstream module is published, delete this directory, drop the
-`replace`, and require the published version. Nothing else in WeKnora changes:
-only `internal/infrastructure/docparser/anydoc/backend_cgo.go` imports it.
+When an upstream module is published, first verify that it includes the local
+safety fixes, the 0.2.x model/error additions, and the asset-link rendering API
+listed below. Only then replace this directory and the `replace` directive.
+WeKnora's backend calls our additional `ToDocumentWithAssetLinks` API, so the
+original PR cannot replace this fork unchanged.
## Provenance
@@ -29,7 +31,9 @@ only `internal/infrastructure/docparser/anydoc/backend_cgo.go` imports it.
| Upstream PR | firecrawl/anydoc#30 ("feat: add Go bindings") |
| PR head | `1a7a6c0` |
| Rebased onto | `4e3089b` (`chore: release v0.1.8`) |
-| anydoc crate | `0.1.9` (from crates.io, pinned with `=`) |
+| anydoc crate | `0.2.4` (from crates.io, pinned with `=`) |
+| Upstream release commit | `42bf1c5ecdde9eb0d96d6bd75a9e6698cf93b14c` |
+| Last upstream check | 2026-09-29: PR #30 is open, unmerged, and conflicting; head unchanged |
| License | MIT (see LICENSE) |
The PR branched before anydoc 0.1.7, so it was rebased onto v0.1.8 before
@@ -48,7 +52,27 @@ unchanged, and none of the local modifications below had to be re-applied. What
1.14.2 (the jump in the number is that project unifying its Rust, Python, and
Node versions, not fourteen major releases). See "Why 0.1.9 matters here" below.
-## Why 0.1.9 matters here
+## Current upgrade: 0.2.4
+
+The [0.1.9 to 0.2.4 source diff](https://github.com/firecrawl/anydoc/compare/v0.1.9...v0.2.4)
+changes the document model and parsing behavior:
+
+- 0.2.0 replaces calamine with native XLS/XLSX/XLSB readers and improves number
+ formatting, merged cells, DOCX whitespace, EPUB repeated spine parts, and
+ Markdown escaping.
+- 0.2.1/0.2.2 preserve equations as inline/block LaTeX and refine dollar escaping.
+- 0.2.3 adds inline checkboxes and spreadsheet form controls. Task-list state
+ moves from `ListItem.checked` to `Inline::Checkbox`.
+- 0.2.4 returns `NeedsOcr` for PDFs containing scanned pages, including mixed
+ PDFs that previously returned incomplete text. The Go ABI now has a distinct
+ `needs_ocr` error; page numbers remain in `ConvertError.Detail`. `NeedsOCR()`
+ lets WeKnora recognize wrapped errors without matching English messages.
+
+Node/Python/CLI hosted OCR is outside the Rust crate. WeKnora continues using
+its existing builtin docreader fallback and does not send documents to Firecrawl.
+The PDF default engine remains builtin. See [upgrade verification and benchmarks](UPGRADE-0.2.4.md).
+
+## Previous upgrade: why 0.1.9 matters here
anydoc's changelog lists 0.1.9 as a single dependency bump, which reads like
housekeeping. It is nine `pdf-inspector` fixes, and seven of them are
@@ -106,7 +130,7 @@ required" and still fall back to the docreader.
Keep this list current: it is the diff a future upgrade has to re-apply. Items
2–4 are bugs in the upstream PR and are worth sending back to it.
-1. `Cargo.toml` — depends on the published `anydoc = "=0.1.9"` crate instead of
+1. `Cargo.toml` — depends on the published `anydoc = "=0.2.4"` crate instead of
the workspace path dependency, declares its own empty `[workspace]`, and
repeats the upstream release profile (`lto`, `strip`), which it would
otherwise inherit from the anydoc workspace.
@@ -124,7 +148,7 @@ Keep this list current: it is the diff a future upgrade has to re-apply. Items
straight from the buffer is only trustworthy while the Rust encoder and this
decoder agree; on a skew, `make([]Block, 0, n)` would exhaust memory before
the first bounds check, turning a version mismatch into a dead process.
-4. `src/lib.rs` — the three conversion entry points run inside `guarded()`,
+4. `src/lib.rs` — conversion entry points run inside `guarded()`,
which catches a panic and reports it as a malformed document. A panic
escaping an `extern "C"` function aborts the process, and WeKnora parses
untrusted uploads in the same process that serves the API. Note the limit:
@@ -136,11 +160,26 @@ Keep this list current: it is the diff a future upgrade has to re-apply. Items
6. `scripts/build-anydoc-lib.sh` copies the pinned anydoc release to
`patched-anydoc/` (gitignored) and re-exports `document_to_markdown`, which
is crate-private in the published crate. It reads the version from the
- `anydoc = "=X.Y.Z"` pin above and checks it against `version.go`, so a bump
- is one line and cannot half-land.
+ `anydoc = "=X.Y.Z"` pin above and checks it against `version.go`, so a
+ dependency bump cannot silently leave the reported version behind.
`src/asset_links.rs` then rewrites `ImageSource::Asset`
to `External("images/image-N.ext")` so the official serializer emits in-place
image links. `anydoc_to_markdown_with_asset_links` is the ABI for that path.
+7. `src/model.rs`, `model.go`, and the generated header carry block/inline math
+ and inline checkbox nodes. Existing wire tags are unchanged. The legacy
+ `ListItem.Checked` slot is retained as nil for compatibility; callers read
+ `Inline.Checked`. Asset traversal accepts math blocks. Heading text retains
+ math and checkbox content when assigning image sections.
+8. `src/lib.rs` and `errors.go` add `ERR_NEEDS_OCR` / `needs_ocr` and `NeedsOCR()`.
+ Page numbers are preserved in the detail string; structured `pages` and
+ `page_count` fields are not exposed by this Go ABI.
+9. `anydoc_to_document_with_asset_links` / `ToDocumentWithAssetLinks` parse once,
+ encode the original document (preserving asset IDs), then rewrite image
+ sources and append the official Markdown to the same owned buffer. The
+ backend uses this combined API to avoid parsing each uploaded document twice.
+ The standalone APIs remain available. Corpus tests compare both paths for
+ identical document trees and Markdown; all allocations use the existing
+ `anydoc_buffer_free` ownership contract.
## Dependency pinning and audit
@@ -158,10 +197,17 @@ and the same input comes back as an ordinary error;
Moving the pin to 0.1.9 added no crate and removed none: `lopdf` stays at 0.42
and the whole tree below `pdf-inspector` is unchanged, so the lockfile diff is
-three version lines and one checksum. The audit result is therefore the same
-one described below.
+three version lines and one checksum. That was the 0.1.9 dependency-only update;
+the 0.2.4 changes are described below.
+
+The 0.2.4 lockfile removes calamine and five exclusive dependencies, retains
+`pdf-inspector` 1.14.2 and `lopdf` 0.42.0, and updates `chacha20` from the yanked
+0.10.1 to 0.10.2. The latter fixes an SSE4.1 intrinsic used by the SSE2 backend
+([upstream fix](https://github.com/RustCrypto/stream-ciphers/pull/580)); other
+transitive versions are retained.
-`cargo audit` currently reports one allowed warning: `ttf-parser` 0.25.1 is
+`cargo audit` on 2026-09-29 reports no vulnerabilities and one allowed warning:
+`ttf-parser` 0.25.1 is
unmaintained (`RUSTSEC-2026-0192`), pulled in transitively by the PDF stack. It
is not a vulnerability and nothing here can fix it, so warnings report without
failing the job.
@@ -174,8 +220,8 @@ renderer (`document_to_markdown`) is also crate-private. WeKnora keeps that
serializer: `scripts/build-anydoc-lib.sh` re-exports the one function, then
`anydoc_to_markdown_with_asset_links` rewrites `Asset` images to
`ImageSource::External("images/image-N.ext")` so the official GFM output
-places them in reading order. PDF has no document model; scanned pages fall
-back to the builtin docreader so they can be rasterized for OCR.
+places them in reading order. PDF has no document model; PDFs containing scanned
+pages fall back to the builtin docreader so they can be rasterized for OCR.
## Building the archive
diff --git a/third_party/anydoc-go/UPGRADE-0.2.4.md b/third_party/anydoc-go/UPGRADE-0.2.4.md
new file mode 100644
index 000000000..e27da4865
--- /dev/null
+++ b/third_party/anydoc-go/UPGRADE-0.2.4.md
@@ -0,0 +1,155 @@
+# AnyDoc 0.2.4 升级核查与压测
+
+核查日期:2026-09-29。项目基线:`9114e4e4f905be71976a77c684d3f92731e6f9a6`。
+
+## 结论
+
+从 0.1.9 跟进到 0.2.4,并继续维护 vendored Go binding。升级重点是文档完整性:公式、复选框及混合 PDF 的扫描页不再被静默丢弃。
+
+- [Go binding PR #30](https://github.com/firecrawl/anydoc/pull/30) 仍为 open,`merged=false`、`mergeable=false`,head 为 `1a7a6c04ff8f2689bf8ebbd1e0d9d61d2d109164`,最后更新时间为 2026-08-08。没有 `go/v*` 发布 tag。
+- 最新 release tag 为 [v0.2.4](https://github.com/firecrawl/anydoc/tree/v0.2.4),2026-08-27 发布,commit `42bf1c5ecdde9eb0d96d6bd75a9e6698cf93b14c`。main 在该 tag 后的三次提交只改 README。
+- 已逐文件确认下载的 crate 源码与 v0.2.4 tag 一致,仅保留原有 renderer re-export 补丁。
+- 检查了 [v0.1.9…v0.2.4 完整源码差异](https://github.com/firecrawl/anydoc/compare/v0.1.9...v0.2.4),包括模型、序列化、PDF 路径、Excel 解析器和 bindings;不是仅根据版本说明判断。
+
+## 核心变化与本项目处理
+
+| 上游变化 | 影响与处理 |
+| --- | --- |
+| 0.2.0:移除 calamine,自行解析 XLS/XLSX/XLSB;改进数值显示、合并单元格与隐藏内容处理 | 保留已有格式路由,回归各容器。上游 XLSB 样例在旧版失败,新版成功;这不代表应用新增了 XLSB 上传入口。 |
+| 0.2.0:DOCX run 边缘空格、重复 EPUB spine、Markdown 分隔符及表格代码中的竖线修复 | 使用上游 66 个快照,经实际 Go ABI 比较 Markdown 与错误详情。 |
+| 0.2.1–0.2.2:OMML/MathML/RTF 公式转 LaTeX,美元符号转义修正 | 新增 block/inline math ABI tag、编码及解码;图片路径接受公式节点。回归 DOCX/PPTX/ODT/EPUB/RTF。 |
+| 0.2.3:复选框成为 inline 节点,移除 Rust `ListItem.checked` | 新增 checkbox tag 与 bool 字段。旧 wire slot 保留为 nil;Go 的旧字段标为 deprecated。实测有标题的选中、未选中控件。 |
+| 0.2.4:混合或扫描 PDF 返回 `NeedsOcr`,错误标出页码 | 新增独立错误码及 `NeedsOCR()`,通过 `errors.As` 识别包装错误,实际 Reader 测试验证回退与无 OCR 时的失败行为。页码保留在 Detail,未新增结构化页码 API。 |
+| Node/Python/CLI 可选择 Firecrawl hosted OCR | Rust 不含该功能;本项目继续使用原有 builtin/docreader 回退。 |
+
+本地线程绑定、panic 捕获、解码分配上限和图片链接 API 均保留。
+本次还消除了图片提取路径的重复解析:新的 `ToDocumentWithAssetLinks` 一次解析返回原始文档树和图片 Markdown,复用单一 buffer 的分配/释放协议。先编码原始 asset ID,再重写图片链接,避免丢失图片位置与章节信息。上游语料和内置 13 种非 PDF 样例均验证合并接口与两个独立接口的输出一致。不能在 PR 合入后直接删除本地版本:还需确认上游补齐这些能力。详细差异见 [README](README.md)。
+
+依赖锁保持 `pdf-inspector=1.14.2`、`lopdf=0.42.0`,移除 calamine 及五个专属依赖。审计发现原有 `chacha20=0.10.1` 已撤回,按[上游修复 #580](https://github.com/RustCrypto/stream-ciphers/pull/580) 更新到 0.10.2,修复 SSE2 后端误用 SSE4.1 指令的问题;其他传递依赖版本未更新。
+
+## 已完成验证
+
+- `scripts/build-anydoc-lib.sh`:release archive 构建成功,使用 `--locked`;C header 已同步生成。
+- `ANYDOC_UPSTREAM_DIR=... go test -tags anydoc -count=1 ./internal/infrastructure/docparser/...`:通过,包括 66 个上游快照、正常文档的 Go 模型解码及图片 Markdown 路径。空输入保留 Go binding 原有的前置错误,单独断言。
+- `go test -race -tags anydoc -count=1 ./internal/infrastructure/docparser/...`:通过。Go race 检测不覆盖 Rust 内存访问。
+- `go test -count=1 ./internal/infrastructure/docparser/...`:未链接引擎路径通过。
+- `go vet -tags anydoc ./internal/infrastructure/docparser/...`:通过。
+- `go build -tags anydoc -o /tmp/weknora-anydoc-server ./cmd/server`:完整服务编译通过。
+- 既有 4,000 次并发错误详情、恶意 PDF 深层嵌套和 TJ lookback 回归:通过。
+- `cargo audit --file third_party/anydoc-go/Cargo.lock`:无已知漏洞;保留 `ttf-parser 0.25.1` 的停止维护警告 `RUSTSEC-2026-0192`。
+
+## 性能测量方法
+
+Apple M4 Pro,64 GiB RAM,macOS arm64;Go 1.26.3、Rust 1.97.1,Rust release + thin LTO。
+使用相同的 Go benchmark 和上游固定样例分别链接 0.1.9 与 0.2.4,排除文件读取和编译时间。
+`WithAssets=true` 覆盖生产 Reader 默认路径:Rust 解析、Go 文档解码、图片收集与官方 Markdown 输出。旧版/直接升级版解析两次,最终版解析一次。PDF 仍走文本路径。
+每项 500 ms × 3 次,`GOMAXPROCS=1,4`;表中给出三次中位数。四并发的 ns/op 是总耗时除以总次数,表示吞吐能力,不是单个请求延迟。`B/op` 只统计 Go 堆,不能代表 Rust 内存。
+编译和其他测试结束后顺序测量;这是本机小样本比较,不声称统计显著,也不等价于生产端到端性能。
+
+### 同机对比:单并发
+
+单位为 µs/op,越低越好;变化率相对 0.1.9。
+
+| 格式 | 0.1.9 | 0.2.4 直接升级 | 0.2.4 最终版 | 最终变化 |
+| --- | ---: | ---: | ---: | ---: |
+| doc | 228.7 | 237.7 | 147.2 | -35.6% |
+| docx | 1040.3 | 1082.4 | 559.8 | -46.2% |
+| ppt | 60.0 | 62.7 | 37.0 | -38.4% |
+| pptx | 887.2 | 895.4 | 450.3 | -49.2% |
+| odt | 719.5 | 727.5 | 390.7 | -45.7% |
+| ods | 478.2 | 483.2 | 259.3 | -45.8% |
+| odp | 1157.2 | 1167.6 | 588.2 | -49.2% |
+| rtf | 742.8 | 757.4 | 407.2 | -45.2% |
+| epub | 313.4 | 314.5 | 174.9 | -44.2% |
+| xls | 69.8 | 127.9 | 76.1 | +9.0% |
+| xlsx | 217.7 | 500.3 | 261.2 | +20.0% |
+| xlsb | 转换失败 | 84.3 | 48.7 | 新增可用样例 |
+| csv | 155.9 | 161.2 | 90.1 | -42.2% |
+| pdf | 1941.6 | 3605.4 | 3631.9 | +87.1% |
+
+### 四并发吞吐
+
+单位为 µs/op(每次转换分摊的墙钟时间,非请求延迟)。
+
+| 格式 | 0.1.9 | 最终版 | 变化 |
+| --- | ---: | ---: | ---: |
+| doc | 53.8 | 36.6 | -32.0% |
+| docx | 268.3 | 142.5 | -46.9% |
+| ppt | 15.9 | 9.8 | -38.6% |
+| pptx | 226.5 | 115.0 | -49.2% |
+| odt | 179.6 | 99.8 | -44.4% |
+| ods | 120.5 | 64.1 | -46.8% |
+| odp | 297.9 | 150.2 | -49.6% |
+| rtf | 187.9 | 104.3 | -44.5% |
+| epub | 78.7 | 43.6 | -44.6% |
+| xls | 19.1 | 19.8 | +3.5% |
+| xlsx | 52.3 | 65.6 | +25.4% |
+| xlsb | 转换失败 | 12.5 | 新增可用样例 |
+| csv | 38.3 | 22.6 | -41.0% |
+| pdf | 577.5 | 1060.4 | +83.6% |
+
+### 性能判断
+
+- 消除重复解析后,DOC/DOCX/PPT/PPTX/ODT/ODS/ODP/RTF/EPUB 的单并发样例耗时降低约 36%–49%,CSV 降低约 42%。
+- XLS 从 69.8 µs 到 76.1 µs(约 +9%),XLSX 从 217.7 µs 到 261.2 µs(约 +20%)。新解析器的数值/布局/控件处理仍有额外代价;不能宣称所有格式加速。
+- PDF 从 1.94 ms 到 3.63 ms(约 +87%)。源码新增对被标记扫描页的二次提取确认,以避免丢页和检测误报。这个路径不使用文档模型,合并解析优化不作用于它。保留完整性检查;应用默认 PDF 引擎仍是 builtin,只有选择 AnyDoc 的 PDF 路径受影响。
+- Go 模型新增一个 checkbox 指针字段,Go 分配字节略有增加;原始输出保留了 B/op 和 allocs/op。
+
+### 持续混合负载
+
+8 workers、14 种格式轮换、30 秒,逐次检查 Markdown 与图片字节/名称一致。
+
+| 指标 | 直接升级 | 最终版 |
+| --- | ---: | ---: |
+| 完成转换 | 231,291 | 306,252 |
+| 吞吐 | 7,708.5 docs/s | 10,207.1 docs/s |
+| p50 | 0.644 ms | 0.359 ms |
+| p95 | 5.320 ms | 5.491 ms |
+| p99 | 6.684 ms | 6.843 ms |
+| 进程峰值 RSS(含 Rust) | 61.1 MiB | 63.3 MiB |
+
+两轮均零错误、无输出或图片不一致。最终吞吐比直接升级提高约 32%;尾延迟没有改善,PDF 占据较长执行时间。RSS 包含测试保存的延迟样本,因此不能简单归因于解析器。该测试均匀轮换小文档,不能外推为大文档生产容量。
+
+### 输入规模测试
+
+| 输入 | 中位耗时 | Go B/op |
+| --- | ---: | ---: |
+| csv_100_rows | 0.398 ms | 129,360 |
+| csv_10000_rows | 32.133 ms | 12,581,816 |
+| pdf_40000_operators | 1.074 ms | 152 |
+| pdf_200000_operators | 4.346 ms | 152 |
+| xlsx_100_rows | 0.592 ms | 82,624 |
+| xlsx_10000_rows | 46.186 ms | 7,946,442 |
+
+CSV 为三列,XLSX 为两列;每种格式内按相同行结构扩展,生成及压缩输入不计入转换耗时。万行 XLSX 约 46.2 ms,校验最后一行未丢失。PDF 的裸 TJ token 从 4 万增到 20 万(5 倍输入),耗时约增至 4 倍,没有重现历史平方级退化。Go 分配不包括 Rust 原生堆。XLSX 原始结果见 [规模压测](benchmarks/2026-09-29/xlsx-scaling.txt)。
+
+原始数据:[0.1.9](benchmarks/2026-09-29/baseline.txt)、[直接升级](benchmarks/2026-09-29/upgrade.txt)、[最终版](benchmarks/2026-09-29/optimized.txt)、[直接升级压力](benchmarks/2026-09-29/upgrade-stress.txt)、[最终版压力](benchmarks/2026-09-29/optimized-stress.txt)、[规模测试](benchmarks/2026-09-29/scaling.txt)、[审计](benchmarks/2026-09-29/audit.txt)。
+
+## 复现
+
+在仓库根目录执行功能核查:
+
+```bash
+scripts/build-anydoc-lib.sh
+git clone --branch v0.2.4 --depth 1 https://github.com/firecrawl/anydoc.git /tmp/anydoc-v0.2.4
+ANYDOC_UPSTREAM_DIR=/tmp/anydoc-v0.2.4 go test -tags anydoc -count=1 ./internal/infrastructure/docparser/...
+go test -race -tags anydoc -count=1 ./internal/infrastructure/docparser/...
+go vet -tags anydoc ./internal/infrastructure/docparser/...
+cargo audit --file third_party/anydoc-go/Cargo.lock
+```
+
+性能与混合负载:
+
+```bash
+go test -tags anydoc -run '^$' -bench '^BenchmarkConvert$' -benchmem -benchtime=500ms -count=3 -cpu=1,4 ./internal/infrastructure/docparser/anydoc
+go test -tags anydoc -run '^$' -bench '^BenchmarkConvertScaling$' -benchmem -benchtime=500ms -count=3 -cpu=1 ./internal/infrastructure/docparser/anydoc
+go test -tags anydoc -c -o /tmp/anydoc.test ./internal/infrastructure/docparser/anydoc
+cd internal/infrastructure/docparser/anydoc
+ANYDOC_STRESS_DURATION=30s /usr/bin/time -l /tmp/anydoc.test -test.run '^TestConcurrentConversionStress$' -test.v
+```
+
+Linux 用 `/usr/bin/time -v` 记录 RSS。测试二进制需从包目录运行以读取样例。基线比较需要在旧版本单独编译同一 benchmark 和样例,再顺序运行两个二进制;旧版 XLSB 样例转换失败,基线应排除该项,不能把失败计入吞吐。
+
+## 验证边界
+
+本次运行平台为 macOS arm64,Linux glibc/musl、Windows 交叉构建未在本机执行。混合 PDF 测试使用 stub 验证回退请求和结果;未调用外部 OCR 服务、存储或检索索引。压测覆盖解析层,不含上传排队、网络与 OCR 耗时。峰值 RSS 和短时压力不能证明长期不存在原生内存泄漏。
diff --git a/third_party/anydoc-go/anydoc.go b/third_party/anydoc-go/anydoc.go
index 5830e7583..646ed6a97 100644
--- a/third_party/anydoc-go/anydoc.go
+++ b/third_party/anydoc-go/anydoc.go
@@ -165,6 +165,46 @@ func ToDocument(data []byte, format *Format) (*Document, error) {
return decodeDocument(raw)
}
+// ToDocumentWithAssetLinks parses once and returns both the original model
+// (including asset IDs and bytes) and Markdown with embedded image links.
+// PDF has no document model; use ToMarkdownBytes for it.
+func ToDocumentWithAssetLinks(data []byte, format *Format) (*Document, string, error) {
+ if len(data) == 0 {
+ return nil, "", &ConvertError{Kind: "unsupported", Detail: "empty input"}
+ }
+ tag := C.int(C.ANYDOC_FORMAT_NONE)
+ if format != nil {
+ tag = formatToTag(*format)
+ if tag == C.int(C.ANYDOC_FORMAT_NONE) {
+ return nil, "", &ConvertError{Kind: "unknown_format", Detail: "unknown format: " + string(*format)}
+ }
+ }
+ var buf *C.uint8_t
+ var bufLen C.uintptr_t
+ if err := call(func() C.int {
+ return C.anydoc_to_document_with_asset_links(
+ (*C.uint8_t)(unsafe.Pointer(&data[0])), C.uintptr_t(len(data)), tag, &buf, &bufLen,
+ )
+ }); err != nil {
+ return nil, "", err
+ }
+ defer C.anydoc_buffer_free(buf, bufLen)
+ raw, err := cGoBytes(unsafe.Pointer(buf), bufLen)
+ if err != nil {
+ return nil, "", err
+ }
+ d := &decoder{buf: raw}
+ document, err := d.document()
+ if err != nil {
+ return nil, "", err
+ }
+ markdown, err := d.str()
+ if err != nil {
+ return nil, "", err
+ }
+ return document, markdown, nil
+}
+
// cStringN reads a length-prefixed C buffer as a Go string. The bytes are
// copied, so the caller may free the C buffer immediately after.
func cStringN(s *C.char, n C.uintptr_t) (string, error) {
diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/audit.txt b/third_party/anydoc-go/benchmarks/2026-09-29/audit.txt
new file mode 100644
index 000000000..5cabc4a62
--- /dev/null
+++ b/third_party/anydoc-go/benchmarks/2026-09-29/audit.txt
@@ -0,0 +1,13 @@
+ Fetching advisory database from `https://github.com/RustSec/advisory-db.git`
+ Loaded 1273 security advisories (from /Users/wizard/.cargo/advisory-db)
+ Updating crates.io index
+ Scanning third_party/anydoc-go/Cargo.lock for vulnerabilities (162 crate dependencies)
+Crate: ttf-parser
+Version: 0.25.1
+Warning: unmaintained
+Title: `ttf-parser` is unmaintained
+Date: 2026-06-28
+ID: RUSTSEC-2026-0192
+URL: https://rustsec.org/advisories/RUSTSEC-2026-0192
+
+warning: 1 allowed warning found
diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/baseline.txt b/third_party/anydoc-go/benchmarks/2026-09-29/baseline.txt
new file mode 100644
index 000000000..4b43068a1
--- /dev/null
+++ b/third_party/anydoc-go/benchmarks/2026-09-29/baseline.txt
@@ -0,0 +1,83 @@
+goos: darwin
+goarch: arm64
+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc
+cpu: Apple M4 Pro
+BenchmarkConvert/doc 2556 228702 ns/op 89.55 MB/s 22793 B/op 325 allocs/op
+BenchmarkConvert/doc 2606 227909 ns/op 89.86 MB/s 22793 B/op 325 allocs/op
+BenchmarkConvert/doc 2588 229236 ns/op 89.34 MB/s 22793 B/op 325 allocs/op
+BenchmarkConvert/doc-4 9982 53926 ns/op 379.78 MB/s 22798 B/op 325 allocs/op
+BenchmarkConvert/doc-4 10000 53771 ns/op 380.87 MB/s 22796 B/op 325 allocs/op
+BenchmarkConvert/doc-4 10000 53729 ns/op 381.18 MB/s 22796 B/op 325 allocs/op
+BenchmarkConvert/docx 572 1044082 ns/op 9.73 MB/s 23945 B/op 342 allocs/op
+BenchmarkConvert/docx 574 1040160 ns/op 9.77 MB/s 23945 B/op 342 allocs/op
+BenchmarkConvert/docx 578 1040338 ns/op 9.77 MB/s 23945 B/op 342 allocs/op
+BenchmarkConvert/docx-4 2168 268537 ns/op 37.83 MB/s 23950 B/op 342 allocs/op
+BenchmarkConvert/docx-4 2217 267350 ns/op 38.00 MB/s 23949 B/op 342 allocs/op
+BenchmarkConvert/docx-4 2229 268327 ns/op 37.86 MB/s 23950 B/op 342 allocs/op
+BenchmarkConvert/ppt 9592 60023 ns/op 7736.71 MB/s 5312 B/op 73 allocs/op
+BenchmarkConvert/ppt 10000 59602 ns/op 7791.48 MB/s 5312 B/op 73 allocs/op
+BenchmarkConvert/ppt 9822 60202 ns/op 7713.73 MB/s 5312 B/op 73 allocs/op
+BenchmarkConvert/ppt-4 37984 15919 ns/op 29171.29 MB/s 5312 B/op 73 allocs/op
+BenchmarkConvert/ppt-4 37519 16036 ns/op 28958.44 MB/s 5312 B/op 73 allocs/op
+BenchmarkConvert/ppt-4 37756 15898 ns/op 29210.70 MB/s 5312 B/op 73 allocs/op
+BenchmarkConvert/pptx 661 885527 ns/op 17.77 MB/s 6016 B/op 85 allocs/op
+BenchmarkConvert/pptx 674 887205 ns/op 17.73 MB/s 6016 B/op 85 allocs/op
+BenchmarkConvert/pptx 670 887945 ns/op 17.72 MB/s 6016 B/op 85 allocs/op
+BenchmarkConvert/pptx-4 2632 226007 ns/op 69.62 MB/s 6016 B/op 85 allocs/op
+BenchmarkConvert/pptx-4 2641 226665 ns/op 69.42 MB/s 6016 B/op 85 allocs/op
+BenchmarkConvert/pptx-4 2631 226480 ns/op 69.47 MB/s 6016 B/op 85 allocs/op
+BenchmarkConvert/odt 822 719914 ns/op 35.41 MB/s 23761 B/op 348 allocs/op
+BenchmarkConvert/odt 806 719450 ns/op 35.43 MB/s 23761 B/op 348 allocs/op
+BenchmarkConvert/odt 837 709504 ns/op 35.93 MB/s 23761 B/op 348 allocs/op
+BenchmarkConvert/odt-4 3228 180369 ns/op 141.32 MB/s 23765 B/op 348 allocs/op
+BenchmarkConvert/odt-4 3318 179551 ns/op 141.96 MB/s 23767 B/op 348 allocs/op
+BenchmarkConvert/odt-4 3246 179267 ns/op 142.18 MB/s 23765 B/op 348 allocs/op
+BenchmarkConvert/ods 1246 478194 ns/op 33.76 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/ods 1254 475562 ns/op 33.94 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/ods 1250 478694 ns/op 33.72 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/ods-4 4875 120484 ns/op 133.98 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/ods-4 4864 120548 ns/op 133.90 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/ods-4 4912 120833 ns/op 133.59 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/odp 512 1157174 ns/op 16.48 MB/s 12184 B/op 101 allocs/op
+BenchmarkConvert/odp 518 1157307 ns/op 16.47 MB/s 12184 B/op 101 allocs/op
+BenchmarkConvert/odp 524 1149194 ns/op 16.59 MB/s 12184 B/op 101 allocs/op
+BenchmarkConvert/odp-4 2001 297899 ns/op 64.00 MB/s 12186 B/op 101 allocs/op
+BenchmarkConvert/odp-4 2006 297995 ns/op 63.98 MB/s 12187 B/op 101 allocs/op
+BenchmarkConvert/odp-4 2017 296884 ns/op 64.22 MB/s 12186 B/op 101 allocs/op
+BenchmarkConvert/rtf 798 742790 ns/op 27.11 MB/s 25865 B/op 383 allocs/op
+BenchmarkConvert/rtf 807 737679 ns/op 27.30 MB/s 25865 B/op 383 allocs/op
+BenchmarkConvert/rtf 811 751647 ns/op 26.79 MB/s 25865 B/op 383 allocs/op
+BenchmarkConvert/rtf-4 3104 187634 ns/op 107.32 MB/s 25871 B/op 383 allocs/op
+BenchmarkConvert/rtf-4 3144 188102 ns/op 107.05 MB/s 25870 B/op 383 allocs/op
+BenchmarkConvert/rtf-4 3145 187886 ns/op 107.17 MB/s 25872 B/op 383 allocs/op
+BenchmarkConvert/epub 1939 314142 ns/op 19.63 MB/s 15392 B/op 216 allocs/op
+BenchmarkConvert/epub 1911 312957 ns/op 19.71 MB/s 15392 B/op 216 allocs/op
+BenchmarkConvert/epub 1920 313373 ns/op 19.68 MB/s 15392 B/op 216 allocs/op
+BenchmarkConvert/epub-4 6810 79097 ns/op 77.97 MB/s 15395 B/op 216 allocs/op
+BenchmarkConvert/epub-4 7555 78542 ns/op 78.52 MB/s 15395 B/op 216 allocs/op
+BenchmarkConvert/epub-4 7537 78666 ns/op 78.40 MB/s 15395 B/op 216 allocs/op
+BenchmarkConvert/xls 8110 69795 ns/op 102.70 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xls 8925 69836 ns/op 102.64 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xls 8622 69393 ns/op 103.30 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xls-4 31742 19081 ns/op 375.65 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xls-4 31390 19092 ns/op 375.44 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xls-4 31503 19032 ns/op 376.63 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xlsx 2749 216875 ns/op 32.77 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xlsx 2732 217725 ns/op 32.64 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xlsx 2757 218701 ns/op 32.49 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xlsx-4 10000 52325 ns/op 135.80 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xlsx-4 10000 52190 ns/op 136.16 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/xlsx-4 10000 52334 ns/op 135.78 MB/s 12720 B/op 208 allocs/op
+BenchmarkConvert/csv 4044 154284 ns/op 1.36 MB/s 9856 B/op 166 allocs/op
+BenchmarkConvert/csv 3782 155902 ns/op 1.35 MB/s 9856 B/op 166 allocs/op
+BenchmarkConvert/csv 3861 156793 ns/op 1.34 MB/s 9856 B/op 166 allocs/op
+BenchmarkConvert/csv-4 15680 38338 ns/op 5.48 MB/s 9856 B/op 166 allocs/op
+BenchmarkConvert/csv-4 15589 38179 ns/op 5.50 MB/s 9856 B/op 166 allocs/op
+BenchmarkConvert/csv-4 15614 38383 ns/op 5.47 MB/s 9856 B/op 166 allocs/op
+BenchmarkConvert/pdf 297 1945666 ns/op 46.29 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf 308 1939296 ns/op 46.44 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf 307 1941629 ns/op 46.38 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 951 579561 ns/op 155.39 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 1015 577545 ns/op 155.94 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 1051 574035 ns/op 156.89 MB/s 2408 B/op 7 allocs/op
+PASS
diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/optimized-stress.txt b/third_party/anydoc-go/benchmarks/2026-09-29/optimized-stress.txt
new file mode 100644
index 000000000..d753d9d7b
--- /dev/null
+++ b/third_party/anydoc-go/benchmarks/2026-09-29/optimized-stress.txt
@@ -0,0 +1,22 @@
+=== RUN TestConcurrentConversionStress
+ stress_linked_test.go:85: workers=8 conversions=306252 elapsed=30.003928708s throughput=10207.1 docs/s p50=358.792µs p95=5.490542ms p99=6.842708ms GoHeapAlloc=9018048 GoTotalAlloc=4384020160
+--- PASS: TestConcurrentConversionStress (30.04s)
+PASS
+ 30.04 real 255.23 user 34.26 sys
+ 66420736 maximum resident set size
+ 0 average shared memory size
+ 0 average unshared data size
+ 0 average unshared stack size
+ 4413 page reclaims
+ 32 page faults
+ 0 swaps
+ 0 block input operations
+ 0 block output operations
+ 0 messages sent
+ 0 messages received
+ 11824 signals received
+ 1459 voluntary context switches
+ 4500950 involuntary context switches
+ 3742707465228 instructions retired
+ 1062875391967 cycles elapsed
+ 57263016 peak memory footprint
diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/optimized.txt b/third_party/anydoc-go/benchmarks/2026-09-29/optimized.txt
new file mode 100644
index 000000000..07c1d6ac5
--- /dev/null
+++ b/third_party/anydoc-go/benchmarks/2026-09-29/optimized.txt
@@ -0,0 +1,89 @@
+goos: darwin
+goarch: arm64
+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc
+cpu: Apple M4 Pro
+BenchmarkConvert/doc 4023 147242 ns/op 139.09 MB/s 23409 B/op 322 allocs/op
+BenchmarkConvert/doc 3994 147396 ns/op 138.95 MB/s 23409 B/op 322 allocs/op
+BenchmarkConvert/doc 4172 147069 ns/op 139.25 MB/s 23409 B/op 322 allocs/op
+BenchmarkConvert/doc-4 16426 36492 ns/op 561.22 MB/s 23413 B/op 322 allocs/op
+BenchmarkConvert/doc-4 16423 36598 ns/op 559.60 MB/s 23413 B/op 322 allocs/op
+BenchmarkConvert/doc-4 16438 36571 ns/op 560.00 MB/s 23412 B/op 322 allocs/op
+BenchmarkConvert/docx 1050 559418 ns/op 18.16 MB/s 25233 B/op 339 allocs/op
+BenchmarkConvert/docx 1066 559967 ns/op 18.14 MB/s 25233 B/op 339 allocs/op
+BenchmarkConvert/docx 1063 559776 ns/op 18.15 MB/s 25233 B/op 339 allocs/op
+BenchmarkConvert/docx-4 4047 142463 ns/op 71.31 MB/s 25238 B/op 339 allocs/op
+BenchmarkConvert/docx-4 4219 142316 ns/op 71.38 MB/s 25238 B/op 339 allocs/op
+BenchmarkConvert/docx-4 4200 142903 ns/op 71.09 MB/s 25238 B/op 339 allocs/op
+BenchmarkConvert/ppt 16210 36971 ns/op 12560.88 MB/s 5496 B/op 70 allocs/op
+BenchmarkConvert/ppt 16308 37312 ns/op 12446.12 MB/s 5496 B/op 70 allocs/op
+BenchmarkConvert/ppt 16060 36782 ns/op 12625.21 MB/s 5496 B/op 70 allocs/op
+BenchmarkConvert/ppt-4 61270 9777 ns/op 47497.14 MB/s 5496 B/op 70 allocs/op
+BenchmarkConvert/ppt-4 60988 9764 ns/op 47558.72 MB/s 5496 B/op 70 allocs/op
+BenchmarkConvert/ppt-4 61044 9798 ns/op 47396.27 MB/s 5496 B/op 70 allocs/op
+BenchmarkConvert/pptx 1304 450272 ns/op 34.94 MB/s 6232 B/op 82 allocs/op
+BenchmarkConvert/pptx 1338 448129 ns/op 35.11 MB/s 6232 B/op 82 allocs/op
+BenchmarkConvert/pptx 1318 457588 ns/op 34.38 MB/s 6232 B/op 82 allocs/op
+BenchmarkConvert/pptx-4 5043 115265 ns/op 136.50 MB/s 6232 B/op 82 allocs/op
+BenchmarkConvert/pptx-4 5125 114971 ns/op 136.85 MB/s 6232 B/op 82 allocs/op
+BenchmarkConvert/pptx-4 5152 114589 ns/op 137.31 MB/s 6232 B/op 82 allocs/op
+BenchmarkConvert/odt 1538 390421 ns/op 65.29 MB/s 25033 B/op 345 allocs/op
+BenchmarkConvert/odt 1534 390736 ns/op 65.23 MB/s 25033 B/op 345 allocs/op
+BenchmarkConvert/odt 1522 393443 ns/op 64.78 MB/s 25033 B/op 345 allocs/op
+BenchmarkConvert/odt-4 5874 99768 ns/op 255.48 MB/s 25037 B/op 345 allocs/op
+BenchmarkConvert/odt-4 5952 99740 ns/op 255.55 MB/s 25038 B/op 345 allocs/op
+BenchmarkConvert/odt-4 6004 99918 ns/op 255.10 MB/s 25038 B/op 345 allocs/op
+BenchmarkConvert/ods 2328 255942 ns/op 63.07 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/ods 2349 259334 ns/op 62.24 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/ods 2276 259975 ns/op 62.09 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/ods-4 8428 64285 ns/op 251.10 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/ods-4 9387 63957 ns/op 252.39 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/ods-4 9322 64114 ns/op 251.77 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/odp 1016 588178 ns/op 32.41 MB/s 12320 B/op 98 allocs/op
+BenchmarkConvert/odp 1026 592461 ns/op 32.18 MB/s 12320 B/op 98 allocs/op
+BenchmarkConvert/odp 1008 586609 ns/op 32.50 MB/s 12320 B/op 98 allocs/op
+BenchmarkConvert/odp-4 3958 149873 ns/op 127.21 MB/s 12322 B/op 98 allocs/op
+BenchmarkConvert/odp-4 3969 150168 ns/op 126.96 MB/s 12323 B/op 98 allocs/op
+BenchmarkConvert/odp-4 3978 150448 ns/op 126.72 MB/s 12323 B/op 98 allocs/op
+BenchmarkConvert/rtf 1465 408854 ns/op 49.25 MB/s 27057 B/op 380 allocs/op
+BenchmarkConvert/rtf 1471 406746 ns/op 49.51 MB/s 27057 B/op 380 allocs/op
+BenchmarkConvert/rtf 1478 407178 ns/op 49.45 MB/s 27057 B/op 380 allocs/op
+BenchmarkConvert/rtf-4 5679 103736 ns/op 194.11 MB/s 27064 B/op 380 allocs/op
+BenchmarkConvert/rtf-4 5692 104277 ns/op 193.10 MB/s 27064 B/op 380 allocs/op
+BenchmarkConvert/rtf-4 5516 104262 ns/op 193.13 MB/s 27064 B/op 380 allocs/op
+BenchmarkConvert/epub 3432 177030 ns/op 34.84 MB/s 15784 B/op 213 allocs/op
+BenchmarkConvert/epub 3459 174859 ns/op 35.27 MB/s 15784 B/op 213 allocs/op
+BenchmarkConvert/epub 3442 174040 ns/op 35.43 MB/s 15784 B/op 213 allocs/op
+BenchmarkConvert/epub-4 13758 43527 ns/op 141.68 MB/s 15788 B/op 213 allocs/op
+BenchmarkConvert/epub-4 13788 43570 ns/op 141.54 MB/s 15788 B/op 213 allocs/op
+BenchmarkConvert/epub-4 13702 43761 ns/op 140.92 MB/s 15788 B/op 213 allocs/op
+BenchmarkConvert/xls 7941 75715 ns/op 94.67 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xls 7932 76079 ns/op 94.22 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xls 7746 76241 ns/op 94.02 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xls-4 30616 19507 ns/op 367.45 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xls-4 30681 19899 ns/op 360.23 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xls-4 30157 19757 ns/op 362.81 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xlsx 2239 263510 ns/op 26.97 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xlsx 2320 261242 ns/op 27.20 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xlsx 2313 259344 ns/op 27.40 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xlsx-4 8738 65600 ns/op 108.32 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xlsx-4 9079 65609 ns/op 108.31 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xlsx-4 9020 65581 ns/op 108.35 MB/s 13288 B/op 205 allocs/op
+BenchmarkConvert/xlsb 12316 48681 ns/op 32.91 MB/s 5768 B/op 81 allocs/op
+BenchmarkConvert/xlsb 12358 48634 ns/op 32.94 MB/s 5768 B/op 81 allocs/op
+BenchmarkConvert/xlsb 12310 48954 ns/op 32.72 MB/s 5768 B/op 81 allocs/op
+BenchmarkConvert/xlsb-4 48037 12567 ns/op 127.48 MB/s 5768 B/op 81 allocs/op
+BenchmarkConvert/xlsb-4 47362 12528 ns/op 127.88 MB/s 5768 B/op 81 allocs/op
+BenchmarkConvert/xlsb-4 48376 12470 ns/op 128.47 MB/s 5768 B/op 81 allocs/op
+BenchmarkConvert/csv 6783 90056 ns/op 2.33 MB/s 10168 B/op 163 allocs/op
+BenchmarkConvert/csv 6709 90130 ns/op 2.33 MB/s 10168 B/op 163 allocs/op
+BenchmarkConvert/csv 6710 89996 ns/op 2.33 MB/s 10168 B/op 163 allocs/op
+BenchmarkConvert/csv-4 26893 22533 ns/op 9.32 MB/s 10168 B/op 163 allocs/op
+BenchmarkConvert/csv-4 26581 22763 ns/op 9.23 MB/s 10168 B/op 163 allocs/op
+BenchmarkConvert/csv-4 26870 22610 ns/op 9.29 MB/s 10168 B/op 163 allocs/op
+BenchmarkConvert/pdf 163 3827329 ns/op 23.53 MB/s 2409 B/op 7 allocs/op
+BenchmarkConvert/pdf 160 3622717 ns/op 24.86 MB/s 2409 B/op 7 allocs/op
+BenchmarkConvert/pdf 163 3631949 ns/op 24.80 MB/s 2409 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 576 1061518 ns/op 84.84 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 562 1060361 ns/op 84.93 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 556 1046297 ns/op 86.07 MB/s 2408 B/op 7 allocs/op
+PASS
diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/scaling.txt b/third_party/anydoc-go/benchmarks/2026-09-29/scaling.txt
new file mode 100644
index 000000000..2d556da0c
--- /dev/null
+++ b/third_party/anydoc-go/benchmarks/2026-09-29/scaling.txt
@@ -0,0 +1,17 @@
+goos: darwin
+goarch: arm64
+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc
+cpu: Apple M4 Pro
+BenchmarkConvertScaling/csv_100_rows 1497 397701 ns/op 9.12 MB/s 129360 B/op 1930 allocs/op
+BenchmarkConvertScaling/csv_100_rows 1510 398194 ns/op 9.11 MB/s 129360 B/op 1930 allocs/op
+BenchmarkConvertScaling/csv_100_rows 1504 395770 ns/op 9.16 MB/s 129360 B/op 1930 allocs/op
+BenchmarkConvertScaling/csv_10000_rows 18 31730509 ns/op 11.35 MB/s 12581816 B/op 190030 allocs/op
+BenchmarkConvertScaling/csv_10000_rows 16 32328646 ns/op 11.14 MB/s 12581816 B/op 190030 allocs/op
+BenchmarkConvertScaling/csv_10000_rows 16 32133409 ns/op 11.20 MB/s 12581816 B/op 190030 allocs/op
+BenchmarkConvertScaling/pdf_40000_operators 547 1076646 ns/op 186.32 MB/s 152 B/op 7 allocs/op
+BenchmarkConvertScaling/pdf_40000_operators 559 1074017 ns/op 186.77 MB/s 152 B/op 7 allocs/op
+BenchmarkConvertScaling/pdf_40000_operators 558 1064584 ns/op 188.43 MB/s 152 B/op 7 allocs/op
+BenchmarkConvertScaling/pdf_200000_operators 138 4345923 ns/op 230.24 MB/s 152 B/op 7 allocs/op
+BenchmarkConvertScaling/pdf_200000_operators 136 4356177 ns/op 229.70 MB/s 152 B/op 7 allocs/op
+BenchmarkConvertScaling/pdf_200000_operators 138 4319291 ns/op 231.66 MB/s 152 B/op 7 allocs/op
+PASS
diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/upgrade-stress.txt b/third_party/anydoc-go/benchmarks/2026-09-29/upgrade-stress.txt
new file mode 100644
index 000000000..fd8640fea
--- /dev/null
+++ b/third_party/anydoc-go/benchmarks/2026-09-29/upgrade-stress.txt
@@ -0,0 +1,22 @@
+=== RUN TestConcurrentConversionStress
+ stress_linked_test.go:85: workers=8 conversions=231291 elapsed=30.004531458s throughput=7708.5 docs/s p50=644.084µs p95=5.320084ms p99=6.684167ms GoHeapAlloc=2712016 GoTotalAlloc=3303175600
+--- PASS: TestConcurrentConversionStress (30.04s)
+PASS
+ 30.04 real 253.87 user 28.25 sys
+ 64110592 maximum resident set size
+ 0 average shared memory size
+ 0 average unshared data size
+ 0 average unshared stack size
+ 4274 page reclaims
+ 32 page faults
+ 0 swaps
+ 0 block input operations
+ 0 block output operations
+ 0 messages sent
+ 0 messages received
+ 8607 signals received
+ 879 voluntary context switches
+ 3964104 involuntary context switches
+ 3957497347565 instructions retired
+ 1043824362343 cycles elapsed
+ 54395768 peak memory footprint
diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/upgrade.txt b/third_party/anydoc-go/benchmarks/2026-09-29/upgrade.txt
new file mode 100644
index 000000000..e4621b9b0
--- /dev/null
+++ b/third_party/anydoc-go/benchmarks/2026-09-29/upgrade.txt
@@ -0,0 +1,89 @@
+goos: darwin
+goarch: arm64
+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc
+cpu: Apple M4 Pro
+BenchmarkConvert/doc 2497 237727 ns/op 86.15 MB/s 23673 B/op 325 allocs/op
+BenchmarkConvert/doc 2479 237260 ns/op 86.32 MB/s 23673 B/op 325 allocs/op
+BenchmarkConvert/doc 2466 237862 ns/op 86.10 MB/s 23673 B/op 325 allocs/op
+BenchmarkConvert/doc-4 9816 58760 ns/op 348.54 MB/s 23678 B/op 325 allocs/op
+BenchmarkConvert/doc-4 9968 58307 ns/op 351.25 MB/s 23677 B/op 325 allocs/op
+BenchmarkConvert/doc-4 10000 58467 ns/op 350.28 MB/s 23677 B/op 325 allocs/op
+BenchmarkConvert/docx 565 1054643 ns/op 9.63 MB/s 24857 B/op 342 allocs/op
+BenchmarkConvert/docx 556 1082361 ns/op 9.39 MB/s 24857 B/op 342 allocs/op
+BenchmarkConvert/docx 541 1100535 ns/op 9.23 MB/s 24857 B/op 342 allocs/op
+BenchmarkConvert/docx-4 2130 271626 ns/op 37.40 MB/s 24862 B/op 342 allocs/op
+BenchmarkConvert/docx-4 2180 269896 ns/op 37.64 MB/s 24861 B/op 342 allocs/op
+BenchmarkConvert/docx-4 2109 270594 ns/op 37.54 MB/s 24861 B/op 342 allocs/op
+BenchmarkConvert/ppt 9344 62726 ns/op 7403.33 MB/s 5520 B/op 73 allocs/op
+BenchmarkConvert/ppt 9786 62456 ns/op 7435.34 MB/s 5520 B/op 73 allocs/op
+BenchmarkConvert/ppt 9612 63643 ns/op 7296.73 MB/s 5520 B/op 73 allocs/op
+BenchmarkConvert/ppt-4 36694 16371 ns/op 28366.29 MB/s 5520 B/op 73 allocs/op
+BenchmarkConvert/ppt-4 36626 16350 ns/op 28402.93 MB/s 5520 B/op 73 allocs/op
+BenchmarkConvert/ppt-4 36825 16367 ns/op 28373.05 MB/s 5520 B/op 73 allocs/op
+BenchmarkConvert/pptx 657 898570 ns/op 17.51 MB/s 6224 B/op 85 allocs/op
+BenchmarkConvert/pptx 674 895435 ns/op 17.57 MB/s 6224 B/op 85 allocs/op
+BenchmarkConvert/pptx 678 893230 ns/op 17.61 MB/s 6224 B/op 85 allocs/op
+BenchmarkConvert/pptx-4 2608 227670 ns/op 69.11 MB/s 6224 B/op 85 allocs/op
+BenchmarkConvert/pptx-4 2575 227581 ns/op 69.14 MB/s 6224 B/op 85 allocs/op
+BenchmarkConvert/pptx-4 2533 227921 ns/op 69.03 MB/s 6224 B/op 85 allocs/op
+BenchmarkConvert/odt 817 730569 ns/op 34.89 MB/s 24673 B/op 348 allocs/op
+BenchmarkConvert/odt 822 727546 ns/op 35.03 MB/s 24673 B/op 348 allocs/op
+BenchmarkConvert/odt 826 726398 ns/op 35.09 MB/s 24673 B/op 348 allocs/op
+BenchmarkConvert/odt-4 3181 183722 ns/op 138.74 MB/s 24677 B/op 348 allocs/op
+BenchmarkConvert/odt-4 3240 183166 ns/op 139.16 MB/s 24677 B/op 348 allocs/op
+BenchmarkConvert/odt-4 3290 182945 ns/op 139.33 MB/s 24677 B/op 348 allocs/op
+BenchmarkConvert/ods 1221 483640 ns/op 33.38 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/ods 1249 483216 ns/op 33.41 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/ods 1252 482840 ns/op 33.43 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/ods-4 4846 121986 ns/op 132.33 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/ods-4 4741 122610 ns/op 131.65 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/ods-4 4682 122870 ns/op 131.37 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/odp 508 1167552 ns/op 16.33 MB/s 12408 B/op 101 allocs/op
+BenchmarkConvert/odp 512 1163936 ns/op 16.38 MB/s 12408 B/op 101 allocs/op
+BenchmarkConvert/odp 516 1172509 ns/op 16.26 MB/s 12408 B/op 101 allocs/op
+BenchmarkConvert/odp-4 2010 296137 ns/op 64.38 MB/s 12411 B/op 101 allocs/op
+BenchmarkConvert/odp-4 1977 296402 ns/op 64.32 MB/s 12411 B/op 101 allocs/op
+BenchmarkConvert/odp-4 2013 297303 ns/op 64.13 MB/s 12410 B/op 101 allocs/op
+BenchmarkConvert/rtf 781 760683 ns/op 26.47 MB/s 26809 B/op 383 allocs/op
+BenchmarkConvert/rtf 796 757376 ns/op 26.59 MB/s 26809 B/op 383 allocs/op
+BenchmarkConvert/rtf 792 757098 ns/op 26.60 MB/s 26809 B/op 383 allocs/op
+BenchmarkConvert/rtf-4 3044 192481 ns/op 104.61 MB/s 26815 B/op 383 allocs/op
+BenchmarkConvert/rtf-4 3078 192209 ns/op 104.76 MB/s 26816 B/op 383 allocs/op
+BenchmarkConvert/rtf-4 3087 192072 ns/op 104.84 MB/s 26816 B/op 383 allocs/op
+BenchmarkConvert/epub 1946 314520 ns/op 19.61 MB/s 16064 B/op 216 allocs/op
+BenchmarkConvert/epub 1914 313805 ns/op 19.65 MB/s 16064 B/op 216 allocs/op
+BenchmarkConvert/epub 1910 318963 ns/op 19.33 MB/s 16064 B/op 216 allocs/op
+BenchmarkConvert/epub-4 7438 79126 ns/op 77.94 MB/s 16068 B/op 216 allocs/op
+BenchmarkConvert/epub-4 7432 78533 ns/op 78.53 MB/s 16068 B/op 216 allocs/op
+BenchmarkConvert/epub-4 7393 78452 ns/op 78.61 MB/s 16068 B/op 216 allocs/op
+BenchmarkConvert/xls 4747 126316 ns/op 56.75 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xls 4825 127954 ns/op 56.02 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xls 4706 127874 ns/op 56.06 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xls-4 18931 31484 ns/op 227.67 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xls-4 18969 31668 ns/op 226.35 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xls-4 18921 31662 ns/op 226.39 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xlsx 1207 500980 ns/op 14.18 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xlsx 1207 498708 ns/op 14.25 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xlsx 1209 500326 ns/op 14.20 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xlsx-4 4579 126024 ns/op 56.39 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xlsx-4 4734 125090 ns/op 56.81 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xlsx-4 4790 124966 ns/op 56.86 MB/s 13216 B/op 208 allocs/op
+BenchmarkConvert/xlsb 6988 84341 ns/op 18.99 MB/s 5792 B/op 84 allocs/op
+BenchmarkConvert/xlsb 7204 83794 ns/op 19.12 MB/s 5792 B/op 84 allocs/op
+BenchmarkConvert/xlsb 6828 84787 ns/op 18.89 MB/s 5792 B/op 84 allocs/op
+BenchmarkConvert/xlsb-4 27576 21564 ns/op 74.29 MB/s 5792 B/op 84 allocs/op
+BenchmarkConvert/xlsb-4 27964 21786 ns/op 73.53 MB/s 5792 B/op 84 allocs/op
+BenchmarkConvert/xlsb-4 27409 21783 ns/op 73.55 MB/s 5792 B/op 84 allocs/op
+BenchmarkConvert/csv 3846 161235 ns/op 1.30 MB/s 10240 B/op 166 allocs/op
+BenchmarkConvert/csv 3718 161213 ns/op 1.30 MB/s 10240 B/op 166 allocs/op
+BenchmarkConvert/csv 3720 161295 ns/op 1.30 MB/s 10240 B/op 166 allocs/op
+BenchmarkConvert/csv-4 15117 39604 ns/op 5.30 MB/s 10240 B/op 166 allocs/op
+BenchmarkConvert/csv-4 15140 39760 ns/op 5.28 MB/s 10240 B/op 166 allocs/op
+BenchmarkConvert/csv-4 15150 39692 ns/op 5.29 MB/s 10240 B/op 166 allocs/op
+BenchmarkConvert/pdf 162 3627058 ns/op 24.83 MB/s 2409 B/op 7 allocs/op
+BenchmarkConvert/pdf 165 3605448 ns/op 24.98 MB/s 2409 B/op 7 allocs/op
+BenchmarkConvert/pdf 165 3559522 ns/op 25.30 MB/s 2409 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 560 1048032 ns/op 85.93 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 554 1052198 ns/op 85.59 MB/s 2408 B/op 7 allocs/op
+BenchmarkConvert/pdf-4 576 1039713 ns/op 86.62 MB/s 2408 B/op 7 allocs/op
+PASS
diff --git a/third_party/anydoc-go/benchmarks/2026-09-29/xlsx-scaling.txt b/third_party/anydoc-go/benchmarks/2026-09-29/xlsx-scaling.txt
new file mode 100644
index 000000000..0660627e1
--- /dev/null
+++ b/third_party/anydoc-go/benchmarks/2026-09-29/xlsx-scaling.txt
@@ -0,0 +1,12 @@
+goos: darwin
+goarch: arm64
+pkg: github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc
+cpu: Apple M4 Pro
+BenchmarkConvertScaling/xlsx_100_rows 982 606582 ns/op 11.68 MB/s 82624 B/op 1353 allocs/op
+BenchmarkConvertScaling/xlsx_100_rows 1010 591741 ns/op 11.97 MB/s 82624 B/op 1353 allocs/op
+BenchmarkConvertScaling/xlsx_100_rows 1010 590599 ns/op 11.99 MB/s 82624 B/op 1353 allocs/op
+BenchmarkConvertScaling/xlsx_10000_rows 12 46185531 ns/op 2.51 MB/s 7946440 B/op 130053 allocs/op
+BenchmarkConvertScaling/xlsx_10000_rows 12 46568236 ns/op 2.49 MB/s 7946442 B/op 130053 allocs/op
+BenchmarkConvertScaling/xlsx_10000_rows 13 45783734 ns/op 2.53 MB/s 7946442 B/op 130053 allocs/op
+PASS
+ok github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc 4.609s
diff --git a/third_party/anydoc-go/errors.go b/third_party/anydoc-go/errors.go
index 9000e8378..1493fee16 100644
--- a/third_party/anydoc-go/errors.go
+++ b/third_party/anydoc-go/errors.go
@@ -10,6 +10,7 @@ import "fmt"
// Error codes mirroring the C ABI (ERR_*). Stable across versions.
const (
+ errNeedsOCR = C.ERR_NEEDS_OCR
errOK = C.ERR_OK
errUnsupported = C.ERR_UNSUPPORTED
errMalformed = C.ERR_MALFORMED
@@ -27,15 +28,19 @@ const (
// ("unsupported", "malformed", "encrypted", ...) plus the crate's
// human-readable detail.
type ConvertError struct {
- // Kind is the lowercase variant name, matching the Node and Python
- // bindings: "unsupported", "malformed", "encrypted", "resource_limit",
- // "missing_part", "io", "pdf_no_model". Go also reports
+ // Kind is the snake-case variant name: "unsupported", "malformed", "encrypted", "resource_limit",
+ // "missing_part", "io", "pdf_no_model", "needs_ocr". Go also reports
// "unknown_format" for an invalid explicit Format.
Kind string
// Detail is the crate's Display output for the error.
Detail string
}
+// NeedsOCR reports the typed scanned-page failure without matching error text.
+func (e *ConvertError) NeedsOCR() bool {
+ return e != nil && e.Kind == "needs_ocr"
+}
+
func (e *ConvertError) Error() string {
if e == nil {
return "<nil>"
@@ -59,6 +64,8 @@ func convertError(code C.int) error {
func errorKind(code C.int) string {
switch code {
+ case errNeedsOCR:
+ return "needs_ocr"
case errUnsupported:
return "unsupported"
case errMalformed:
diff --git a/third_party/anydoc-go/include/anydoc.h b/third_party/anydoc-go/include/anydoc.h
index 179ff5ce4..c3fbb1ecb 100644
--- a/third_party/anydoc-go/include/anydoc.h
+++ b/third_party/anydoc-go/include/anydoc.h
@@ -58,6 +58,11 @@
*/
#define ERR_UNKNOWN_FORMAT 9
+/**
+ * `ConvertError::NeedsOcr`; page numbers are retained in the error detail.
+ */
+#define ERR_NEEDS_OCR 10
+
/**
* C-side format tag. Stable; mirrors the Node/Python lowercase string names
* via `format_name`. `ANYDOC_FORMAT_NONE` is the `Option::None` sentinel.
@@ -106,6 +111,8 @@
#define BLOCK_RULE 6
+#define BLOCK_MATH 7
+
#define INLINE_TEXT 0
#define INLINE_LINK 1
@@ -118,6 +125,10 @@
#define INLINE_LINEBREAK 5
+#define INLINE_MATH 6
+
+#define INLINE_CHECKBOX 7
+
#define LINK_EXTERNAL 0
#define LINK_RELATIVE 1
@@ -229,6 +240,17 @@ int anydoc_to_document(const uint8_t *bytes,
uint8_t **out_buf,
uintptr_t *out_len);
+/**
+ * Parse once and return the original document model followed by asset-linked
+ * Markdown in a single flat buffer. Free it with `anydoc_buffer_free`.
+ * PDF has no document model and returns `ERR_PDF_NO_MODEL`.
+ */
+int anydoc_to_document_with_asset_links(const uint8_t *bytes,
+ uintptr_t len,
+ int format_tag,
+ uint8_t **out_buf,
+ uintptr_t *out_len);
+
/**
* Free a string returned by `anydoc_to_markdown*` or `anydoc_last_error`.
* Null is a no-op (the empty-string sentinel).
diff --git a/third_party/anydoc-go/model.go b/third_party/anydoc-go/model.go
index c0656950c..8ddf49095 100644
--- a/third_party/anydoc-go/model.go
+++ b/third_party/anydoc-go/model.go
@@ -24,7 +24,7 @@ type Document struct {
// which of the optional fields is populated; the rest are nil.
type Block struct {
// "heading", "paragraph", "list", "table", "block_quote", "code_block",
- // or "rule".
+ // "rule", or "math".
Kind string
Level *uint8 // heading: 1-based outline depth
Anchor *string // heading: stable anchor id when the document targets it
@@ -33,15 +33,15 @@ type Block struct {
Table *Table // table
Blocks []Block // block_quote
Lang *string // code_block
- Text *string // code_block
+ Text *string // code_block, math
}
// Inline is one span of inline content. The Kind field selects which optional
// fields are populated.
type Inline struct {
- // "text", "link", "image", "anchor", "note_ref", or "line_break".
+ // "text", "link", "image", "anchor", "note_ref", "line_break", "math", or "checkbox".
Kind string
- Text *string // text
+ Text *string // text, math
Style *Style // text
Content []Inline // link
Target *LinkTarget // link
@@ -49,6 +49,7 @@ type Inline struct {
Source *ImageSource // image
Anchor *string // anchor: the anchor id
NoteID *string // note_ref: the id of the note in Document.Notes
+ Checked *bool // checkbox
}
// Style is a fully resolved character style.
@@ -92,7 +93,7 @@ type List struct {
// further lists.
type ListItem struct {
Blocks []Block
- Checked *bool // task-list state, when the item carries a checkbox
+ Checked *bool // Deprecated: always nil since 0.2.3; read Inline.Checked.
MarkerLabel *string // literal marker text overriding the level marker
}
@@ -156,6 +157,10 @@ type decoder struct {
func decodeDocument(raw []byte) (*Document, error) {
d := &decoder{buf: raw}
+ return d.document()
+}
+
+func (d *decoder) document() (*Document, error) {
doc := &Document{}
var err error
if doc.Blocks, err = d.blocks(); err != nil {
@@ -356,6 +361,12 @@ func (d *decoder) block() (Block, error) {
return Block{}, err
}
return Block{Kind: "code_block", Lang: lang, Text: &text}, nil
+ case C.BLOCK_MATH:
+ text, err := d.str()
+ if err != nil {
+ return Block{}, err
+ }
+ return Block{Kind: "math", Text: &text}, nil
case C.BLOCK_RULE:
return Block{Kind: "rule"}, nil
default:
@@ -427,6 +438,18 @@ func (d *decoder) inline() (Inline, error) {
return Inline{}, err
}
return Inline{Kind: "note_ref", NoteID: &id}, nil
+ case C.INLINE_MATH:
+ text, err := d.str()
+ if err != nil {
+ return Inline{}, err
+ }
+ return Inline{Kind: "math", Text: &text}, nil
+ case C.INLINE_CHECKBOX:
+ checked, err := d.bool()
+ if err != nil {
+ return Inline{}, err
+ }
+ return Inline{Kind: "checkbox", Checked: &checked}, nil
case C.INLINE_LINEBREAK:
return Inline{Kind: "line_break"}, nil
default:
diff --git a/third_party/anydoc-go/src/asset_links.rs b/third_party/anydoc-go/src/asset_links.rs
index b5247df77..629da7151 100644
--- a/third_party/anydoc-go/src/asset_links.rs
+++ b/third_party/anydoc-go/src/asset_links.rs
@@ -48,7 +48,7 @@ fn rewrite_blocks(blocks: &mut [Block], urls: &[Option<String>]) {
}
}
Block::BlockQuote(inner) => rewrite_blocks(inner, urls),
- Block::CodeBlock { .. } | Block::Rule => {}
+ Block::CodeBlock { .. } | Block::Rule | Block::Math(_) => {}
}
}
}
diff --git a/third_party/anydoc-go/src/lib.rs b/third_party/anydoc-go/src/lib.rs
index 18b787f79..252066d33 100644
--- a/third_party/anydoc-go/src/lib.rs
+++ b/third_party/anydoc-go/src/lib.rs
@@ -47,6 +47,8 @@ pub const ERR_PDF_NO_MODEL: c_int = 7;
pub const ERR_INVALID_ARG: c_int = 8;
/// Unknown format name passed to the ABI.
pub const ERR_UNKNOWN_FORMAT: c_int = 9;
+/// `ConvertError::NeedsOcr`; page numbers are retained in the error detail.
+pub const ERR_NEEDS_OCR: c_int = 10;
thread_local! {
static LAST_ERROR: std::cell::RefCell<Option<CString>> = const { std::cell::RefCell::new(None) };
@@ -60,6 +62,7 @@ fn set_last_error(msg: &str) {
fn error_code(err: &ConvertError) -> c_int {
match err {
ConvertError::Unsupported(_) => ERR_UNSUPPORTED,
+ ConvertError::NeedsOcr { .. } => ERR_NEEDS_OCR,
ConvertError::Malformed { .. } => ERR_MALFORMED,
ConvertError::Encrypted => ERR_ENCRYPTED,
ConvertError::ResourceLimit { .. } => ERR_RESOURCE_LIMIT,
@@ -438,6 +441,31 @@ pub unsafe extern "C" fn anydoc_to_document(
format_tag: c_int,
out_buf: *mut *mut u8,
out_len: *mut usize,
+) -> c_int {
+ unsafe { document_output(bytes, len, format_tag, out_buf, out_len, false) }
+}
+
+/// Parse once and return the original document model followed by asset-linked
+/// Markdown in a single flat buffer. Free it with `anydoc_buffer_free`.
+/// PDF has no document model and returns `ERR_PDF_NO_MODEL`.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn anydoc_to_document_with_asset_links(
+ bytes: *const u8,
+ len: usize,
+ format_tag: c_int,
+ out_buf: *mut *mut u8,
+ out_len: *mut usize,
+) -> c_int {
+ unsafe { document_output(bytes, len, format_tag, out_buf, out_len, true) }
+}
+
+unsafe fn document_output(
+ bytes: *const u8,
+ len: usize,
+ format_tag: c_int,
+ out_buf: *mut *mut u8,
+ out_len: *mut usize,
+ with_markdown: bool,
) -> c_int {
if bytes.is_null() || out_buf.is_null() {
set_last_error("anydoc: null pointer passed to anydoc_to_document");
@@ -475,13 +503,21 @@ pub unsafe extern "C" fn anydoc_to_document(
);
return ERR_PDF_NO_MODEL;
}
- let document = match guarded("anydoc_to_document", || anydoc::to_document(slice, resolved)) {
- Ok(d) => d,
+ let body = match guarded("anydoc_to_document", || {
+ let mut document = anydoc::to_document(slice, resolved)?;
+ let mut encoder = model::Encoder::new();
+ // Encode before rewriting: Go needs the original asset IDs for image
+ // placement and section metadata, while Markdown needs external URLs.
+ model::write_document(&mut encoder, &document);
+ if with_markdown {
+ asset_links::rewrite_asset_images(&mut document);
+ encoder.str(&document_to_markdown(&document));
+ }
+ Ok(encoder.into_vec())
+ }) {
+ Ok(body) => body,
Err(e) => return fail(e),
};
- let mut encoder = model::Encoder::new();
- model::write_document(&mut encoder, &document);
- let body = encoder.into_vec();
let len = body.len();
// Layout: [usize len][usize cap][u8 data..len]. The caller receives a
// pointer to the data area and the length; `anydoc_buffer_free` walks back
diff --git a/third_party/anydoc-go/src/model.rs b/third_party/anydoc-go/src/model.rs
index 5a97b0448..0dae5d88b 100644
--- a/third_party/anydoc-go/src/model.rs
+++ b/third_party/anydoc-go/src/model.rs
@@ -27,6 +27,7 @@ pub const BLOCK_TABLE: c_int = 3;
pub const BLOCK_QUOTE: c_int = 4;
pub const BLOCK_CODE: c_int = 5;
pub const BLOCK_RULE: c_int = 6;
+pub const BLOCK_MATH: c_int = 7;
pub const INLINE_TEXT: c_int = 0;
pub const INLINE_LINK: c_int = 1;
@@ -34,6 +35,8 @@ pub const INLINE_IMAGE: c_int = 2;
pub const INLINE_ANCHOR: c_int = 3;
pub const INLINE_NOTEREF: c_int = 4;
pub const INLINE_LINEBREAK: c_int = 5;
+pub const INLINE_MATH: c_int = 6;
+pub const INLINE_CHECKBOX: c_int = 7;
pub const LINK_EXTERNAL: c_int = 0;
pub const LINK_RELATIVE: c_int = 1;
@@ -161,6 +164,10 @@ fn write_block(e: &mut Encoder, block: &model::Block) {
e.opt_str(lang);
e.str(text);
}
+ model::Block::Math(text) => {
+ e.i32(BLOCK_MATH);
+ e.str(text);
+ }
model::Block::Rule => {
e.i32(BLOCK_RULE);
}
@@ -199,6 +206,14 @@ fn write_inline(e: &mut Encoder, inline: &model::Inline) {
e.i32(INLINE_NOTEREF);
e.str(id);
}
+ model::Inline::Math(text) => {
+ e.i32(INLINE_MATH);
+ e.str(text);
+ }
+ model::Inline::Checkbox(checked) => {
+ e.i32(INLINE_CHECKBOX);
+ e.bool(*checked);
+ }
model::Inline::LineBreak => {
e.i32(INLINE_LINEBREAK);
}
@@ -256,11 +271,9 @@ fn write_list(e: &mut Encoder, list: &model::List) {
fn write_list_item(e: &mut Encoder, item: &model::ListItem) {
write_blocks(e, &item.blocks);
- match item.checked {
- None => e.i32(-1),
- Some(false) => e.i32(0),
- Some(true) => e.i32(1),
- }
+ // Preserve the legacy optional field in the wire format. Upstream 0.2.3
+ // moved task state into Inline::Checkbox; new callers read those nodes.
+ e.i32(-1);
e.opt_str(&item.marker_label);
}
diff --git a/third_party/anydoc-go/version.go b/third_party/anydoc-go/version.go
index e493d82fd..566c6b9cc 100644
--- a/third_party/anydoc-go/version.go
+++ b/third_party/anydoc-go/version.go
@@ -3,4 +3,4 @@ package anydoc
// Version is the version of the anydoc-go module, bumped in lockstep with the
// Rust crate version (go/Cargo.toml) and the GitHub release tag. The
// packaged static libraries.
-const Version = "0.1.9"
+const Version = "0.2.4"