Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docparser): treat anydoc resource limits as permanent parse failures (#4573)

jackfb2c66ec2d4a
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index c0e697714..da664464e 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -18,6 +18,7 @@ import (
werrors "github.com/Tencent/WeKnora/internal/errors"
"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/asr"
"github.com/Tencent/WeKnora/internal/models/chat"
@@ -4265,6 +4266,10 @@ func (s *knowledgeService) convert(
}
result, err := s.callDocReaderWithTimeout(ctx, reader, req)
+ // Parser resource limits are permanent document rejections, not transient reader failures.
+ if errors.Is(err, anydoc.ErrResourceLimit) {
+ result, err = &types.ReadResult{Error: err.Error()}, nil
+ }
if err != nil {
code, message := docReaderFailure(err)
logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err)
diff --git a/internal/application/service/knowledge_process_span_retry_test.go b/internal/application/service/knowledge_process_span_retry_test.go
index 2eca9adc4..b0a0be307 100644
--- a/internal/application/service/knowledge_process_span_retry_test.go
+++ b/internal/application/service/knowledge_process_span_retry_test.go
@@ -3,12 +3,14 @@ package service
import (
"context"
"errors"
+ "fmt"
"io"
"strings"
"testing"
"testing/iotest"
werrors "github.com/Tencent/WeKnora/internal/errors"
+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
"github.com/stretchr/testify/require"
@@ -111,3 +113,55 @@ func TestConvertFailureSpanMatchesTaskOutcome(t *testing.T) {
}
}
}
+
+// An anydoc resource_limit is a property of the file. Retries burn memory and
+// still fail, and the knowledge row must say which parser limit was hit
+// instead of sending the user to DocReader logs (anydoc runs in-process).
+func TestConvertAnydocResourceLimitIsPermanent(t *testing.T) {
+ const limitDetail = "resource limit exceeded (max_xml_nodes): part exceeds 2000000 xml nodes"
+ rpcErr := fmt.Errorf("anydoc conversion failed for %q: %w: %s",
+ "xlsx50k.xlsx", anydoc.ErrResourceLimit, limitDetail)
+
+ tracker, db := setupSpanTrackerTest(t)
+ // Retries are still available. A permanent rejection must not use them.
+ ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)
+ knowledge := &types.Knowledge{
+ ID: "doc", TenantID: 1, KnowledgeBaseID: "kb", FilePath: "xlsx50k.xlsx",
+ ParseStatus: types.ParseStatusProcessing,
+ }
+ _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")
+ require.NoError(t, err)
+ ctx = withAttempt(ctx, attempt)
+ svc := &knowledgeService{
+ repo: &embedFailureKnowledgeRepo{knowledge: knowledge},
+ spanTracker: tracker,
+ tenantService: &stubTenantServiceForModelDelete{},
+ fileSvc: spanRetryFileService{},
+ documentReader: &spanRetryDocReader{err: rpcErr},
+ }
+ result, err := svc.convert(ctx, types.DocumentProcessPayload{
+ FilePath: "xlsx50k.xlsx", FileName: "xlsx50k.xlsx", FileType: "xlsx",
+ }, &types.KnowledgeBase{ID: "kb"}, knowledge, types.EffectiveProcessConfig{
+ ChunkingConfig: types.ChunkingConfig{
+ ParserEngineRules: []types.ParserEngineRule{{
+ FileTypes: []string{"xlsx"},
+ Engine: "builtin",
+ }},
+ },
+ }, false)
+
+ // nil error is what ProcessDocument returns to asynq, so the task is not
+ // retried. ReadResult.Error is the existing permanent-failure path.
+ require.Nil(t, result)
+ require.NoError(t, err)
+ require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)
+ require.NotContains(t, knowledge.ErrorMessage, "DocReader logs")
+ require.Contains(t, knowledge.ErrorMessage, "too large")
+ require.Contains(t, knowledge.ErrorMessage, "max_xml_nodes")
+ require.Contains(t, knowledge.ErrorMessage, "2000000")
+
+ status, code, message := rootSpan(t, db, knowledge.ID, attempt)
+ require.Equal(t, types.SpanStatusFailed, status)
+ require.Equal(t, werrors.ErrCodeDocReaderParseFailed, code)
+ require.Equal(t, knowledge.ErrorMessage, message)
+}
diff --git a/internal/infrastructure/docparser/anydoc/anydoc.go b/internal/infrastructure/docparser/anydoc/anydoc.go
index 9cd383139..97b416e01 100644
--- a/internal/infrastructure/docparser/anydoc/anydoc.go
+++ b/internal/infrastructure/docparser/anydoc/anydoc.go
@@ -79,6 +79,53 @@ type Options struct {
// with errors.Is.
var ErrUnavailable = errors.New("anydoc: binding not built into this binary")
+// ErrResourceLimit marks a permanent anydoc rejection: the document exceeds a
+// parser budget (for example max_xml_nodes) and retrying the same bytes cannot
+// succeed. Callers detect it with errors.Is. The wrapped error's text names
+// the limit the converter reported.
+var ErrResourceLimit = errors.New("document is too large for the parser")
+
+// resourceLimiter is the method set anydoc's *ConvertError uses to report a
+// parser budget failure. The cgo binding implements it; the default build does
+// not import that package, so classification goes through this interface.
+type resourceLimiter interface {
+ ResourceLimit() bool
+ LimitDetail() string
+}
+
+// permanentResourceLimit is the errors.Is(ErrResourceLimit) wrapper. Error
+// keeps the user-facing reason and the converter's limit detail, without the
+// transport prefixes that sit further down the chain.
+type permanentResourceLimit struct {
+ detail string
+ cause error
+}
+
+func (e *permanentResourceLimit) Error() string {
+ if e.detail == "" {
+ return ErrResourceLimit.Error()
+ }
+ return ErrResourceLimit.Error() + ": " + e.detail
+}
+
+func (e *permanentResourceLimit) Unwrap() error { return e.cause }
+
+func (e *permanentResourceLimit) Is(target error) bool { return target == ErrResourceLimit }
+
+// markResourceLimit wraps a converter error whose kind is resource_limit.
+// Other errors, including untyped text that merely mentions the kind, are
+// returned unchanged so a transient failure can still be retried.
+func markResourceLimit(err error) error {
+ if err == nil {
+ return nil
+ }
+ var limiter resourceLimiter
+ if !errors.As(err, &limiter) || !limiter.ResourceLimit() {
+ return err
+ }
+ return &permanentResourceLimit{detail: strings.TrimSpace(limiter.LimitDetail()), cause: err}
+}
+
// supportedExtensions are the file types anydoc converts, mapped to the format
// name its parser selector uses. The list is static so that engine metadata
// (file types shown in the UI) is identical whether or not the binding is
@@ -156,7 +203,11 @@ func Convert(data []byte, opts Options) (*Result, error) {
if opts.Format == "pdf" {
opts.WithAssets = false
}
- return backendConvert(data, opts)
+ result, err := backendConvert(data, opts)
+ if err != nil {
+ err = markResourceLimit(err)
+ }
+ return result, err
}
// PDFNeedsOCR reports whether a Convert error means the PDF has no usable
diff --git a/internal/infrastructure/docparser/anydoc/backend_cgo.go b/internal/infrastructure/docparser/anydoc/backend_cgo.go
index c8e10e887..775f6f42e 100644
--- a/internal/infrastructure/docparser/anydoc/backend_cgo.go
+++ b/internal/infrastructure/docparser/anydoc/backend_cgo.go
@@ -13,6 +13,10 @@ import (
// (vendored under third_party/anydoc-go until they are published as a module).
// Conversion runs in-process: no subprocess, no HTTP hop, no Python.
+// *ConvertError must keep implementing resourceLimiter so Convert can classify
+// a parser budget failure without this file's build tag.
+var _ resourceLimiter = (*upstream.ConvertError)(nil)
+
func backendAvailable() bool { return true }
func backendUnavailableReason() string { return "" }
diff --git a/internal/infrastructure/docparser/anydoc/resource_limit_test.go b/internal/infrastructure/docparser/anydoc/resource_limit_test.go
new file mode 100644
index 000000000..176c35728
--- /dev/null
+++ b/internal/infrastructure/docparser/anydoc/resource_limit_test.go
@@ -0,0 +1,85 @@
+package anydoc
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+)
+
+// fakeConvertError stands in for the cgo binding's *ConvertError. The
+// production type exposes the same two methods; this fake lets the default
+// build classify a resource_limit failure without linking the parser.
+type fakeConvertError struct {
+ kind string
+ detail string
+}
+
+func (e *fakeConvertError) Error() string {
+ if e.detail == "" {
+ return "anydoc: " + e.kind
+ }
+ return "anydoc: " + e.kind + ": " + e.detail
+}
+
+func (e *fakeConvertError) ResourceLimit() bool { return e.kind == "resource_limit" }
+
+func (e *fakeConvertError) LimitDetail() string { return e.detail }
+
+func TestMarkResourceLimitClassifiesPermanentBudgetFailure(t *testing.T) {
+ const detail = "resource limit exceeded (max_xml_nodes): part exceeds 2000000 xml nodes"
+ raw := &fakeConvertError{kind: "resource_limit", detail: detail}
+ // Same wrap the cgo backend applies before the reader names the file.
+ backendErr := fmt.Errorf("anydoc: markdown conversion failed: %w", raw)
+
+ got := markResourceLimit(backendErr)
+ if !errors.Is(got, ErrResourceLimit) {
+ t.Fatalf("markResourceLimit() = %v, want errors.Is ErrResourceLimit", got)
+ }
+ // The reader keeps the chain with %w, which is what convert inspects.
+ readerErr := fmt.Errorf("anydoc conversion failed for %q: %w", "xlsx50k.xlsx", got)
+ if !errors.Is(readerErr, ErrResourceLimit) {
+ t.Fatalf("reader wrap = %v, want errors.Is ErrResourceLimit", readerErr)
+ }
+ message := readerErr.Error()
+ for _, want := range []string{"too large", "max_xml_nodes", "2000000"} {
+ if !strings.Contains(message, want) {
+ t.Errorf("message %q does not mention %q", message, want)
+ }
+ }
+ if strings.Contains(message, "DocReader") {
+ t.Errorf("message points at DocReader, which did not parse the file: %q", message)
+ }
+ if !errors.Is(got, raw) {
+ t.Fatal("resource-limit wrap dropped the converter error")
+ }
+}
+
+func TestMarkResourceLimitLeavesOtherFailuresRetryable(t *testing.T) {
+ malformed := &fakeConvertError{kind: "malformed", detail: "not a readable zip archive"}
+ in := fmt.Errorf("anydoc: markdown conversion failed: %w", malformed)
+ if got := markResourceLimit(in); got != in || errors.Is(got, ErrResourceLimit) {
+ t.Fatalf("malformed error = %v, want it unchanged and retryable", got)
+ }
+
+ // The kind has to be typed. A matching sentence from somewhere else is
+ // not a permanent parser rejection.
+ untyped := errors.New("anydoc: resource_limit: resource limit exceeded (max_xml_nodes): " +
+ "part exceeds 2000000 xml nodes")
+ if got := markResourceLimit(untyped); errors.Is(got, ErrResourceLimit) {
+ t.Fatalf("untyped error was classified as a resource limit: %v", got)
+ }
+
+ if got := markResourceLimit(nil); got != nil {
+ t.Fatalf("markResourceLimit(nil) = %v, want nil", got)
+ }
+
+ bare := &fakeConvertError{kind: "resource_limit"}
+ got := markResourceLimit(bare)
+ if !errors.Is(got, ErrResourceLimit) {
+ t.Fatalf("limit without detail = %v, want ErrResourceLimit", got)
+ }
+ if !strings.Contains(got.Error(), "too large") {
+ t.Fatalf("limit without detail = %q, want the permanent-failure reason", got.Error())
+ }
+}
diff --git a/third_party/anydoc-go/errors.go b/third_party/anydoc-go/errors.go
index 1493fee16..d05eee617 100644
--- a/third_party/anydoc-go/errors.go
+++ b/third_party/anydoc-go/errors.go
@@ -41,6 +41,21 @@ func (e *ConvertError) NeedsOCR() bool {
return e != nil && e.Kind == "needs_ocr"
}
+// ResourceLimit reports a permanent parser budget rejection. Retrying the
+// same document cannot succeed.
+func (e *ConvertError) ResourceLimit() bool {
+ return e != nil && e.Kind == "resource_limit"
+}
+
+// LimitDetail is the converter's description of the exceeded budget, for
+// example "resource limit exceeded (max_xml_nodes): part exceeds 2000000 xml nodes".
+func (e *ConvertError) LimitDetail() string {
+ if e == nil {
+ return ""
+ }
+ return e.Detail
+}
+
func (e *ConvertError) Error() string {
if e == nil {
return "<nil>"