Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docparser): treat anydoc resource limits as permanent parse failures (#4573)
jackfb2c66ec2d4a
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.goindex c0e697714..da664464e 100644--- a/internal/application/service/knowledge_process.go+++ b/internal/application/service/knowledge_process.go@@ -18,6 +18,7 @@ import (werrors "github.com/Tencent/WeKnora/internal/errors""github.com/Tencent/WeKnora/internal/infrastructure/chunker""github.com/Tencent/WeKnora/internal/infrastructure/docparser"+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc""github.com/Tencent/WeKnora/internal/logger""github.com/Tencent/WeKnora/internal/models/asr""github.com/Tencent/WeKnora/internal/models/chat"@@ -4265,6 +4266,10 @@ func (s *knowledgeService) convert(}result, err := s.callDocReaderWithTimeout(ctx, reader, req)+ // Parser resource limits are permanent document rejections, not transient reader failures.+ if errors.Is(err, anydoc.ErrResourceLimit) {+ result, err = &types.ReadResult{Error: err.Error()}, nil+ }if err != nil {code, message := docReaderFailure(err)logger.Errorf(ctx, "[convert] DocReader failed knowledge=%s code=%s: %v", knowledge.ID, code, err)diff --git a/internal/application/service/knowledge_process_span_retry_test.go b/internal/application/service/knowledge_process_span_retry_test.goindex 2eca9adc4..b0a0be307 100644--- a/internal/application/service/knowledge_process_span_retry_test.go+++ b/internal/application/service/knowledge_process_span_retry_test.go@@ -3,12 +3,14 @@ package serviceimport ("context""errors"+ "fmt""io""strings""testing""testing/iotest"werrors "github.com/Tencent/WeKnora/internal/errors"+ "github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc""github.com/Tencent/WeKnora/internal/types""github.com/Tencent/WeKnora/internal/types/interfaces""github.com/stretchr/testify/require"@@ -111,3 +113,55 @@ func TestConvertFailureSpanMatchesTaskOutcome(t *testing.T) {}}}++// An anydoc resource_limit is a property of the file. Retries burn memory and+// still fail, and the knowledge row must say which parser limit was hit+// instead of sending the user to DocReader logs (anydoc runs in-process).+func TestConvertAnydocResourceLimitIsPermanent(t *testing.T) {+ const limitDetail = "resource limit exceeded (max_xml_nodes): part exceeds 2000000 xml nodes"+ rpcErr := fmt.Errorf("anydoc conversion failed for %q: %w: %s",+ "xlsx50k.xlsx", anydoc.ErrResourceLimit, limitDetail)++ tracker, db := setupSpanTrackerTest(t)+ // Retries are still available. A permanent rejection must not use them.+ ctx := types.WithTaskRetryMetadata(context.Background(), 0, 3)+ knowledge := &types.Knowledge{+ ID: "doc", TenantID: 1, KnowledgeBaseID: "kb", FilePath: "xlsx50k.xlsx",+ ParseStatus: types.ParseStatusProcessing,+ }+ _, attempt, err := tracker.OpenAttempt(ctx, knowledge.ID, "")+ require.NoError(t, err)+ ctx = withAttempt(ctx, attempt)+ svc := &knowledgeService{+ repo: &embedFailureKnowledgeRepo{knowledge: knowledge},+ spanTracker: tracker,+ tenantService: &stubTenantServiceForModelDelete{},+ fileSvc: spanRetryFileService{},+ documentReader: &spanRetryDocReader{err: rpcErr},+ }+ result, err := svc.convert(ctx, types.DocumentProcessPayload{+ FilePath: "xlsx50k.xlsx", FileName: "xlsx50k.xlsx", FileType: "xlsx",+ }, &types.KnowledgeBase{ID: "kb"}, knowledge, types.EffectiveProcessConfig{+ ChunkingConfig: types.ChunkingConfig{+ ParserEngineRules: []types.ParserEngineRule{{+ FileTypes: []string{"xlsx"},+ Engine: "builtin",+ }},+ },+ }, false)++ // nil error is what ProcessDocument returns to asynq, so the task is not+ // retried. ReadResult.Error is the existing permanent-failure path.+ require.Nil(t, result)+ require.NoError(t, err)+ require.Equal(t, types.ParseStatusFailed, knowledge.ParseStatus)+ require.NotContains(t, knowledge.ErrorMessage, "DocReader logs")+ require.Contains(t, knowledge.ErrorMessage, "too large")+ require.Contains(t, knowledge.ErrorMessage, "max_xml_nodes")+ require.Contains(t, knowledge.ErrorMessage, "2000000")++ status, code, message := rootSpan(t, db, knowledge.ID, attempt)+ require.Equal(t, types.SpanStatusFailed, status)+ require.Equal(t, werrors.ErrCodeDocReaderParseFailed, code)+ require.Equal(t, knowledge.ErrorMessage, message)+}diff --git a/internal/infrastructure/docparser/anydoc/anydoc.go b/internal/infrastructure/docparser/anydoc/anydoc.goindex 9cd383139..97b416e01 100644--- a/internal/infrastructure/docparser/anydoc/anydoc.go+++ b/internal/infrastructure/docparser/anydoc/anydoc.go@@ -79,6 +79,53 @@ type Options struct {// with errors.Is.var ErrUnavailable = errors.New("anydoc: binding not built into this binary")+// ErrResourceLimit marks a permanent anydoc rejection: the document exceeds a+// parser budget (for example max_xml_nodes) and retrying the same bytes cannot+// succeed. Callers detect it with errors.Is. The wrapped error's text names+// the limit the converter reported.+var ErrResourceLimit = errors.New("document is too large for the parser")++// resourceLimiter is the method set anydoc's *ConvertError uses to report a+// parser budget failure. The cgo binding implements it; the default build does+// not import that package, so classification goes through this interface.+type resourceLimiter interface {+ ResourceLimit() bool+ LimitDetail() string+}++// permanentResourceLimit is the errors.Is(ErrResourceLimit) wrapper. Error+// keeps the user-facing reason and the converter's limit detail, without the+// transport prefixes that sit further down the chain.+type permanentResourceLimit struct {+ detail string+ cause error+}++func (e *permanentResourceLimit) Error() string {+ if e.detail == "" {+ return ErrResourceLimit.Error()+ }+ return ErrResourceLimit.Error() + ": " + e.detail+}++func (e *permanentResourceLimit) Unwrap() error { return e.cause }++func (e *permanentResourceLimit) Is(target error) bool { return target == ErrResourceLimit }++// markResourceLimit wraps a converter error whose kind is resource_limit.+// Other errors, including untyped text that merely mentions the kind, are+// returned unchanged so a transient failure can still be retried.+func markResourceLimit(err error) error {+ if err == nil {+ return nil+ }+ var limiter resourceLimiter+ if !errors.As(err, &limiter) || !limiter.ResourceLimit() {+ return err+ }+ return &permanentResourceLimit{detail: strings.TrimSpace(limiter.LimitDetail()), cause: err}+}+// supportedExtensions are the file types anydoc converts, mapped to the format// name its parser selector uses. The list is static so that engine metadata// (file types shown in the UI) is identical whether or not the binding is@@ -156,7 +203,11 @@ func Convert(data []byte, opts Options) (*Result, error) {if opts.Format == "pdf" {opts.WithAssets = false}- return backendConvert(data, opts)+ result, err := backendConvert(data, opts)+ if err != nil {+ err = markResourceLimit(err)+ }+ return result, err}// PDFNeedsOCR reports whether a Convert error means the PDF has no usablediff --git a/internal/infrastructure/docparser/anydoc/backend_cgo.go b/internal/infrastructure/docparser/anydoc/backend_cgo.goindex c8e10e887..775f6f42e 100644--- a/internal/infrastructure/docparser/anydoc/backend_cgo.go+++ b/internal/infrastructure/docparser/anydoc/backend_cgo.go@@ -13,6 +13,10 @@ import (// (vendored under third_party/anydoc-go until they are published as a module).// Conversion runs in-process: no subprocess, no HTTP hop, no Python.+// *ConvertError must keep implementing resourceLimiter so Convert can classify+// a parser budget failure without this file's build tag.+var _ resourceLimiter = (*upstream.ConvertError)(nil)+func backendAvailable() bool { return true }func backendUnavailableReason() string { return "" }diff --git a/internal/infrastructure/docparser/anydoc/resource_limit_test.go b/internal/infrastructure/docparser/anydoc/resource_limit_test.gonew file mode 100644index 000000000..176c35728--- /dev/null+++ b/internal/infrastructure/docparser/anydoc/resource_limit_test.go@@ -0,0 +1,85 @@+package anydoc++import (+ "errors"+ "fmt"+ "strings"+ "testing"+)++// fakeConvertError stands in for the cgo binding's *ConvertError. The+// production type exposes the same two methods; this fake lets the default+// build classify a resource_limit failure without linking the parser.+type fakeConvertError struct {+ kind string+ detail string+}++func (e *fakeConvertError) Error() string {+ if e.detail == "" {+ return "anydoc: " + e.kind+ }+ return "anydoc: " + e.kind + ": " + e.detail+}++func (e *fakeConvertError) ResourceLimit() bool { return e.kind == "resource_limit" }++func (e *fakeConvertError) LimitDetail() string { return e.detail }++func TestMarkResourceLimitClassifiesPermanentBudgetFailure(t *testing.T) {+ const detail = "resource limit exceeded (max_xml_nodes): part exceeds 2000000 xml nodes"+ raw := &fakeConvertError{kind: "resource_limit", detail: detail}+ // Same wrap the cgo backend applies before the reader names the file.+ backendErr := fmt.Errorf("anydoc: markdown conversion failed: %w", raw)++ got := markResourceLimit(backendErr)+ if !errors.Is(got, ErrResourceLimit) {+ t.Fatalf("markResourceLimit() = %v, want errors.Is ErrResourceLimit", got)+ }+ // The reader keeps the chain with %w, which is what convert inspects.+ readerErr := fmt.Errorf("anydoc conversion failed for %q: %w", "xlsx50k.xlsx", got)+ if !errors.Is(readerErr, ErrResourceLimit) {+ t.Fatalf("reader wrap = %v, want errors.Is ErrResourceLimit", readerErr)+ }+ message := readerErr.Error()+ for _, want := range []string{"too large", "max_xml_nodes", "2000000"} {+ if !strings.Contains(message, want) {+ t.Errorf("message %q does not mention %q", message, want)+ }+ }+ if strings.Contains(message, "DocReader") {+ t.Errorf("message points at DocReader, which did not parse the file: %q", message)+ }+ if !errors.Is(got, raw) {+ t.Fatal("resource-limit wrap dropped the converter error")+ }+}++func TestMarkResourceLimitLeavesOtherFailuresRetryable(t *testing.T) {+ malformed := &fakeConvertError{kind: "malformed", detail: "not a readable zip archive"}+ in := fmt.Errorf("anydoc: markdown conversion failed: %w", malformed)+ if got := markResourceLimit(in); got != in || errors.Is(got, ErrResourceLimit) {+ t.Fatalf("malformed error = %v, want it unchanged and retryable", got)+ }++ // The kind has to be typed. A matching sentence from somewhere else is+ // not a permanent parser rejection.+ untyped := errors.New("anydoc: resource_limit: resource limit exceeded (max_xml_nodes): " ++ "part exceeds 2000000 xml nodes")+ if got := markResourceLimit(untyped); errors.Is(got, ErrResourceLimit) {+ t.Fatalf("untyped error was classified as a resource limit: %v", got)+ }++ if got := markResourceLimit(nil); got != nil {+ t.Fatalf("markResourceLimit(nil) = %v, want nil", got)+ }++ bare := &fakeConvertError{kind: "resource_limit"}+ got := markResourceLimit(bare)+ if !errors.Is(got, ErrResourceLimit) {+ t.Fatalf("limit without detail = %v, want ErrResourceLimit", got)+ }+ if !strings.Contains(got.Error(), "too large") {+ t.Fatalf("limit without detail = %q, want the permanent-failure reason", got.Error())+ }+}diff --git a/third_party/anydoc-go/errors.go b/third_party/anydoc-go/errors.goindex 1493fee16..d05eee617 100644--- a/third_party/anydoc-go/errors.go+++ b/third_party/anydoc-go/errors.go@@ -41,6 +41,21 @@ func (e *ConvertError) NeedsOCR() bool {return e != nil && e.Kind == "needs_ocr"}+// ResourceLimit reports a permanent parser budget rejection. Retrying the+// same document cannot succeed.+func (e *ConvertError) ResourceLimit() bool {+ return e != nil && e.Kind == "resource_limit"+}++// LimitDetail is the converter's description of the exceeded budget, for+// example "resource limit exceeded (max_xml_nodes): part exceeds 2000000 xml nodes".+func (e *ConvertError) LimitDetail() string {+ if e == nil {+ return ""+ }+ return e.Detail+}+func (e *ConvertError) Error() string {if e == nil {return "<nil>"