Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Perf(deepdoc): free ONNX in/out tensors per Run instead of pinning them per session (#19843)

xugangqiangacabc97c1306
diff --git a/internal/deepdoc/native/det_core.go b/internal/deepdoc/native/det_core.go
index 192272f093..b930c9a402 100644
--- a/internal/deepdoc/native/det_core.go
+++ b/internal/deepdoc/native/det_core.go
@@ -49,10 +49,12 @@ type DetResult struct {
// at a VARIABLE input size: each page is aspect-preserved-rescaled to a round32
// size bounded by detLimitSideLen, so the pool is keyed by the resized
// (height, width) and distinct page sizes get distinct sessions. Sessions are
-// pooled per instance, never shared across concurrent Run calls, because
-// session.Run mutates the session's fixed-shape input/output tensors; the
+// pooled per instance, never shared across concurrent Run calls, because a
+// session's underlying ONNX handle is not safe for concurrent use; the
// native det branch runs concurrently across the page worker pool, so a
-// naively shared single session would race.
+// naively shared single session would race. (Each Run allocates its own
+// input/output tensors and frees them before returning, so the constraint is
+// about the session handle, not any pinned buffer.)
//
// The set of distinct shapes is BOUNDED (detMaxShapePools). A long-running
// server ingesting many differently-sized pages would otherwise pin a pool
@@ -86,8 +88,7 @@ func getDetSession(ctx context.Context, modelPath string, rh, rw int64) (*sessio
key := detSessKey{modelPath, rh, rw}
return detSessions.Get(ctx, key, func() (*session, error) {
return NewSession(modelPath, "x",
- []int64{1, 3, rh, rw}, "sigmoid_0.tmp_0",
- []int64{1, 1, rh, rw})
+ []int64{1, 3, rh, rw}, "sigmoid_0.tmp_0")
})
}
@@ -106,6 +107,9 @@ func RunDet(ctx context.Context, modelDir string, img *Image) (DetResult, error)
if e != nil {
return DetResult{}, e
}
+ if e := checkOutputLength("det", len(out), rh*rw); e != nil {
+ return DetResult{}, e
+ }
// out is [1,1,rh,rw]; flatten to [rh,rw].
p := make([]float32, rh*rw)
copy(p, out)
diff --git a/internal/deepdoc/native/dla.go b/internal/deepdoc/native/dla.go
index 356f189bb1..63c5c06705 100644
--- a/internal/deepdoc/native/dla.go
+++ b/internal/deepdoc/native/dla.go
@@ -58,8 +58,7 @@ var (
func RunDLA(ctx context.Context, modelDir string, img *Image) (DLAResult, error) {
blob, sf := dlaPreprocess(img)
sess, release, err := getModelSession(ctx, filepath.Join(modelDir, "layout.ort"), "images",
- []int64{1, 3, dlaInputSize, dlaInputSize}, "output0",
- []int64{1, dlaMaxBoxes, 6})
+ []int64{1, 3, dlaInputSize, dlaInputSize}, "output0")
if err != nil {
return DLAResult{}, err
}
@@ -69,6 +68,9 @@ func RunDLA(ctx context.Context, modelDir string, img *Image) (DLAResult, error)
if err != nil {
return DLAResult{}, err
}
+ if err := checkOutputLength("dla", len(out), dlaMaxBoxes*6); err != nil {
+ return DLAResult{}, err
+ }
res := dlaPostprocess(out, sf)
res.W, res.H = img.W, img.H
return res, nil
diff --git a/internal/deepdoc/native/ocr_rec.go b/internal/deepdoc/native/ocr_rec.go
index 16e604b9b9..ab87a4fa3d 100644
--- a/internal/deepdoc/native/ocr_rec.go
+++ b/internal/deepdoc/native/ocr_rec.go
@@ -290,61 +290,42 @@ func (r OCRRecResult) Wire() string {
// pre-sized per width and even a width-matched session would still emit a
// varying seq length. Instead we use a DynamicAdvancedSession and pass a nil
// output on every Run: onnxruntime allocates the correctly-shaped output
-// tensor, which we copy out before destroying it. The input tensor is
-// fixed-shape per (model, width), so one recSession is reused per width.
+// tensor, which we copy out before destroying it. The input tensor is NOT
+// cached: each Run allocates a fresh input tensor and frees it after (see
+// recSession.Run), so a pooled rec session holds only its weights in steady
+// state.
type recSession struct {
inName string
outName string
+ inShape []int64
sess *ort.DynamicAdvancedSession
- in *ort.Tensor[float32]
poisoned bool
}
func newRecSession(modelPath, inName string, inShape []int64, outName string) (*recSession, error) {
- in := make([]float32, prod(inShape))
- inT, err := ort.NewTensor(ort.NewShape(inShape...), in)
+ opts, err := newSessionOptions()
if err != nil {
return nil, err
}
- opts, err := ort.NewSessionOptions()
- if err != nil {
- inT.Destroy()
- return nil, err
- }
- // One intra-op thread per session, matching NewSession: see the
- // intraOpThreads constant in session.go.
- if err := opts.SetIntraOpNumThreads(intraOpThreads); err != nil {
- opts.Destroy()
- inT.Destroy()
- return nil, err
- }
- // Disable the BFC memory arena for this session. See the matching comment in
- // NewSession (session.go): the arena never shrinks, so pooled rec sessions
- // (up to recMaxShapePools*recShapePoolCap = 128 idle) each hoard a native
- // block, which drove most of the ~14 GB of native memory at the ~20 GB OOM
- // peak. With the arena off, idle rec sessions keep only their weights.
- if err := opts.SetCpuMemArena(false); err != nil {
- opts.Destroy()
- inT.Destroy()
- return nil, err
- }
+ // See NewSession: the C session does not take ownership of opts, so release
+ // it once the session is built. newSessionOptions centralizes the
+ // intra-op-thread and arena settings shared with NewSession.
+ defer opts.Destroy()
sess, err := ort.NewDynamicAdvancedSession(modelPath,
[]string{inName}, []string{outName}, opts)
if err != nil {
- opts.Destroy()
- inT.Destroy()
return nil, err
}
- return &recSession{inName: inName, outName: outName, sess: sess, in: inT}, nil
+ return &recSession{inName: inName, outName: outName, inShape: inShape, sess: sess}, nil
}
-// Run copies input into the input tensor, executes with an auto-allocated
-// (dynamic) output, and returns the output data. The allocated output tensor
-// is destroyed before returning; out is a fresh copy the caller owns.
+// Run allocates a fresh input tensor, executes with an auto-allocated
+// (dynamic) output, and returns the output data. Both tensors are destroyed
+// before returning; out is a fresh copy the caller owns.
func (s *recSession) Run(ctx context.Context, input []float32) ([]float32, error) {
- if len(input) != len(s.in.GetData()) {
- return nil, fmt.Errorf("recSession %s: input len %d != tensor len %d",
- s.outName, len(input), len(s.in.GetData()))
+ if len(input) != int(prod(s.inShape)) {
+ return nil, fmt.Errorf("recSession %s: input len %d != expected %d",
+ s.outName, len(input), int(prod(s.inShape)))
}
opts, err := ort.NewRunOptions()
if err != nil {
@@ -363,10 +344,15 @@ func (s *recSession) Run(ctx context.Context, input []float32) ([]float32, error
}
}()
- copy(s.in.GetData(), input)
- // nil output → onnxruntime allocates the actual-shaped tensor.
+ // Fresh input tensor per Run; freed right after the call. nil output →
+ // onnxruntime allocates the actual-shaped tensor.
+ inT, err := ort.NewTensor(ort.NewShape(s.inShape...), input)
+ if err != nil {
+ return nil, err
+ }
+ defer inT.Destroy()
outputs := []ort.Value{nil}
- if err := s.sess.RunWithOptions([]ort.Value{s.in}, outputs, opts); err != nil {
+ if err := s.sess.RunWithOptions([]ort.Value{inT}, outputs, opts); err != nil {
if ctx.Err() != nil {
s.poisoned = true
}
@@ -387,14 +373,12 @@ func (s *recSession) Run(ctx context.Context, input []float32) ([]float32, error
return out, nil
}
-// Destroy releases the dynamic session and input tensor.
+// Destroy releases the dynamic session. Input/output tensors are no longer
+// owned by the session (allocated per Run and freed there).
func (s *recSession) Destroy() {
if s.sess != nil {
s.sess.Destroy()
}
- if s.in != nil {
- s.in.Destroy()
- }
}
func (s *recSession) isPoisoned() bool { return s.poisoned }
diff --git a/internal/deepdoc/native/session.go b/internal/deepdoc/native/session.go
index 666cb54ec0..8ddd3fd687 100644
--- a/internal/deepdoc/native/session.go
+++ b/internal/deepdoc/native/session.go
@@ -9,10 +9,11 @@ package native
// DeepDoc ONNX we port fits this shape). CPU-only by design.
//
// The ONNX Runtime environment is process-global: InitORT sets the shared
-// library and initializes it exactly once. Sessions only own their tensors and
-// the advanced-session handle, so running several tasks in one process (or one
-// task per CLI invocation) never double-initializes or prematurely tears down
-// the shared environment.
+// library and initializes it exactly once. A session owns only its
+// DynamicAdvancedSession handle (input/output tensors are allocated per Run and
+// freed afterwards), so running several tasks in one process (or one task per
+// CLI invocation) never double-initializes or prematurely tears down the
+// shared environment.
import (
"context"
@@ -76,41 +77,60 @@ func InitORT() error {
func Initialized() bool { return ortReady }
// session loads one ONNX model and runs single-input/single-output inference.
+// It is a DynamicAdvancedSession: it owns no input/output tensors. Every Run
+// allocates fresh input/output tensors and frees them afterwards (see Run), so
+// a pooled session's steady-state native memory is just its weights plus ORT's
+// plan cache — never the pinned in/out buffers that previously dominated the
+// ~14 GB of native memory across the rec/det/DLA/TSR pools.
type session struct {
inName string
outName string
- outSize int64
- sess *ort.AdvancedSession
- in *ort.Tensor[float32]
- out *ort.Tensor[float32]
+ inShape []int64
+ sess *ort.DynamicAdvancedSession
// poisoned is set when a Run is cancelled/terminated via context. ONNX
// Runtime does not guarantee a session is reusable after a forced
// termination, so the pool must Destroy rather than re-Put it.
poisoned bool
}
-// NewSession opens modelPath. inShape/outShape describe the fixed tensor
-// dimensions; outSize is the total element count of the output tensor. The
-// session runs intraOpThreads intra-op threads (see the constant): one thread
-// per Run, with the process-wide ceiling owned by the inference budget the
-// process owner registers (see inference_limit.go). InitORT must have been
-// called first.
-func NewSession(modelPath, inName string, inShape []int64, outName string, outShape []int64) (*session, error) {
- in := make([]float32, prod(inShape))
- out := make([]float32, prod(outShape))
- inT, err := ort.NewTensor(ort.NewShape(inShape...), in)
+// NewSession opens modelPath. inShape describes the fixed input tensor
+// dimensions; output tensors are allocated per Run (their shape is
+// model-determined, so no outShape argument is needed). The session runs
+// intraOpThreads intra-op threads (see the constant): one thread per Run, with
+// the process-wide ceiling owned by the inference budget the process owner
+// registers (see inference_limit.go). Input/output tensors are allocated and
+// freed on every Run (see session.Run), so a pooled session holds only its
+// weights in steady state. InitORT must have been called first.
+func NewSession(modelPath, inName string, inShape []int64, outName string) (*session, error) {
+ opts, err := newSessionOptions()
if err != nil {
return nil, err
}
- outT, err := ort.NewTensor(ort.NewShape(outShape...), out)
+ // newSessionOptions builds opts; the C session does not take ownership of it
+ // (see newSessionOptions / onnxruntime_go), so release it once the session is
+ // built. Releasing after a successful build is the documented pattern and
+ // avoids leaking the native SessionOptions handle on every session.
+ defer opts.Destroy()
+ sess, err := ort.NewDynamicAdvancedSession(modelPath,
+ []string{inName}, []string{outName}, opts)
if err != nil {
- inT.Destroy()
return nil, err
}
+ return &session{
+ inName: inName, outName: outName,
+ inShape: inShape,
+ sess: sess,
+ }, nil
+}
+
+// newSessionOptions builds the SessionOptions shared by every running session:
+// one intra-op thread (so each Run costs exactly one CPU thread — see the
+// intraOpThreads constant and inference_limit.go) and the BFC arena disabled
+// (idle sessions then keep only their weights; activation tensors are allocated
+// per Run and freed after).
+func newSessionOptions() (*ort.SessionOptions, error) {
opts, err := ort.NewSessionOptions()
if err != nil {
- inT.Destroy()
- outT.Destroy()
return nil, err
}
// One intra-op thread per session: the session's Runs then cost one thread
@@ -118,8 +138,6 @@ func NewSession(modelPath, inName string, inShape []int64, outName string, outSh
// concurrent Runs the caller admits (see the intraOpThreads constant).
if err := opts.SetIntraOpNumThreads(intraOpThreads); err != nil {
opts.Destroy()
- inT.Destroy()
- outT.Destroy()
return nil, err
}
// Disable the BFC memory arena for this session. The arena pre-reserves a
@@ -131,35 +149,36 @@ func NewSession(modelPath, inName string, inShape []int64, outName string, outSh
// after each Run.
if err := opts.SetCpuMemArena(false); err != nil {
opts.Destroy()
- inT.Destroy()
- outT.Destroy()
return nil, err
}
- sess, err := ort.NewAdvancedSession(modelPath,
- []string{inName}, []string{outName},
- []ort.Value{inT}, []ort.Value{outT}, opts)
- if err != nil {
- opts.Destroy()
- inT.Destroy()
- outT.Destroy()
- return nil, err
+ return opts, nil
+}
+
+// checkOutputLength fails fast when a model emits an unexpectedly sized output
+// tensor. The previous pinned-output design got this check for free: ORT errored
+// when the bound output tensor's shape mismatched the model. The dynamic design
+// allocates the output per Run (nil output, ORT allocates), so ORT no longer
+// validates the shape; without this guard, postprocessing (dlaPostprocess
+// indexes 300*6, tsrPostprocess indexes 11*8400, RunDet fills rh*rw) panics or
+// silently misreads a truncated output instead of returning a clean error.
+func checkOutputLength(model string, got, want int) error {
+ if got != want {
+ return fmt.Errorf("%s model output length %d, expected %d", model, got, want)
}
- return &session{
- inName: inName, outName: outName,
- outSize: prod(outShape),
- sess: sess, in: inT, out: outT,
- }, nil
+ return nil
}
-// Run copies input into the input tensor, executes, and returns the output
-// tensor contents. ctx bounds the inference: if it is cancelled while Run is
-// in flight, the underlying ONNX Runtime call is terminated via RunOptions.
-// A terminated session is left in an indeterminate state, so it is marked
-// poisoned and the pool destroys it instead of reusing it.
+// Run allocates a fresh input tensor, executes with an auto-allocated (dynamic)
+// output, and returns the output data. Both tensors are destroyed before
+// returning; out is a fresh copy the caller owns. ctx bounds the inference: if
+// it is cancelled while Run is in flight, the underlying ONNX Runtime call is
+// terminated via RunOptions. A terminated session is left in an indeterminate
+// state, so it is marked poisoned and the pool destroys it instead of reusing
+// it.
func (s *session) Run(ctx context.Context, input []float32) ([]float32, error) {
- if len(input) != len(s.in.GetData()) {
- return nil, fmt.Errorf("session %s: input len %d != tensor len %d",
- s.outName, len(input), len(s.in.GetData()))
+ if len(input) != int(prod(s.inShape)) {
+ return nil, fmt.Errorf("session %s: input len %d != expected %d",
+ s.outName, len(input), int(prod(s.inShape)))
}
opts, err := ort.NewRunOptions()
if err != nil {
@@ -178,30 +197,47 @@ func (s *session) Run(ctx context.Context, input []float32) ([]float32, error) {
}
}()
- copy(s.in.GetData(), input)
- if err := s.sess.RunWithOptions(opts); err != nil {
+ // Allocate a fresh input tensor for this Run and free it right after the
+ // call. The output is passed as nil so ORT allocates it; we copy the data
+ // out and free the returned Value. This keeps each pooled session's
+ // steady-state native memory to just its weights (plus ORT's plan cache)
+ // instead of pinning fixed in/out buffers that previously dominated the
+ // ~14 GB of native memory seen across the rec/det/DLA/TSR pools.
+ inT, err := ort.NewTensor(ort.NewShape(s.inShape...), input)
+ if err != nil {
+ return nil, err
+ }
+ defer inT.Destroy()
+ outputs := []ort.Value{nil}
+ if err := s.sess.RunWithOptions([]ort.Value{inT}, outputs, opts); err != nil {
if ctx.Err() != nil {
s.poisoned = true
}
return nil, err
}
- out := make([]float32, s.outSize)
- copy(out, s.out.GetData())
+ outVal := outputs[0]
+ if outVal == nil {
+ return nil, fmt.Errorf("session %s: nil output tensor", s.outName)
+ }
+ defer outVal.Destroy()
+ outT, ok := outVal.(*ort.Tensor[float32])
+ if !ok {
+ return nil, fmt.Errorf("session %s: unexpected output value type %T",
+ s.outName, outVal)
+ }
+ out := make([]float32, len(outT.GetData()))
+ copy(out, outT.GetData())
return out, nil
}
-// Destroy releases the tensors and advanced-session handle. It does NOT touch
-// the process-global environment.
+// Destroy releases the dynamic advanced-session handle. Input/output tensors
+// are no longer owned by the session (they are allocated per Run and freed
+// there), so there is nothing else to release here. It does NOT touch the
+// process-global environment.
func (s *session) Destroy() {
if s.sess != nil {
s.sess.Destroy()
}
- if s.in != nil {
- s.in.Destroy()
- }
- if s.out != nil {
- s.out.Destroy()
- }
}
func (s *session) isPoisoned() bool { return s.poisoned }
diff --git a/internal/deepdoc/native/session_output_test.go b/internal/deepdoc/native/session_output_test.go
new file mode 100644
index 0000000000..7f556ad886
--- /dev/null
+++ b/internal/deepdoc/native/session_output_test.go
@@ -0,0 +1,38 @@
+//go:build cgo
+
+package native
+
+import "testing"
+
+// TestCheckOutputLength guards the output-length validation that the dynamic
+// (per-Run allocated) session relies on. The previous pinned-output design got
+// this check for free: ORT errored when the bound output tensor's shape
+// mismatched the model. After switching to a nil output that ORT allocates, the
+// shape is no longer checked by ORT, so postprocessing (dlaPostprocess indexes
+// 300*6, tsrPostprocess indexes 11*8400, RunDet fills rh*rw) would panic or
+// silently misread a truncated output. This test locks the expected lengths.
+func TestCheckOutputLength(t *testing.T) {
+ cases := []struct {
+ model string
+ got int
+ want int
+ ok bool
+ }{
+ {"dla", dlaMaxBoxes * 6, dlaMaxBoxes * 6, true},
+ {"dla", dlaMaxBoxes*6 - 1, dlaMaxBoxes * 6, false},
+ {"dla", 0, dlaMaxBoxes * 6, false},
+ {"tsr", 11 * tsrCandidates, 11 * tsrCandidates, true},
+ {"tsr", 11*tsrCandidates - 7, 11 * tsrCandidates, false},
+ {"det", 100, 100, true},
+ {"det", 99, 100, false},
+ }
+ for _, c := range cases {
+ err := checkOutputLength(c.model, c.got, c.want)
+ if c.ok && err != nil {
+ t.Errorf("checkOutputLength(%q,%d,%d) = %v, want nil", c.model, c.got, c.want, err)
+ }
+ if !c.ok && err == nil {
+ t.Errorf("checkOutputLength(%q,%d,%d) = nil, want error", c.model, c.got, c.want)
+ }
+ }
+}
diff --git a/internal/deepdoc/native/session_pool.go b/internal/deepdoc/native/session_pool.go
index 5d3e23869f..37233af748 100644
--- a/internal/deepdoc/native/session_pool.go
+++ b/internal/deepdoc/native/session_pool.go
@@ -10,9 +10,11 @@ package native
// model. This pool caches one session per (model signature) tuple and hands it
// back between calls.
//
-// Sessions are pooled, not shared concurrently: session.Run copies the caller's
-// input into the session's fixed-shape input tensor and then executes, so a
-// single session must never be touched by two goroutines at once. Get returns a
+// Sessions are pooled, not shared concurrently: a session's underlying ONNX
+// handle is not safe for concurrent use, so a single session must never be
+// touched by two goroutines at once. (Each Run allocates its own input/output
+// tensors and frees them before returning; the constraint is about the handle,
+// not any pinned buffer.) Get returns a
// session owned by the caller until release is called; release returns it to
// the pool for reuse. This keeps the Get/Run/Release window single-owner, which
// is what makes reuse safe under the page/region worker pools.
@@ -201,16 +203,15 @@ func (p *sessionPool[K, V]) evictLRU() {
// per modelDir in practice.
type sessKey struct {
modelPath, inName, outName string
- inShape, outShape string
+ inShape string
}
-func sessKeyOf(modelPath, inName string, inShape []int64, outName string, outShape []int64) sessKey {
+func sessKeyOf(modelPath, inName string, inShape []int64, outName string) sessKey {
return sessKey{
modelPath: modelPath,
inName: inName,
outName: outName,
inShape: shapeKey(inShape),
- outShape: shapeKey(outShape),
}
}
@@ -228,10 +229,10 @@ var modelSessions = newSessionPool[sessKey, *session](0, 0)
// getModelSession returns a reusable session for the given model signature plus
// a release func. The caller must call release exactly once.
-func getModelSession(ctx context.Context, modelPath, inName string, inShape []int64, outName string, outShape []int64) (*session, func(), error) {
- key := sessKeyOf(modelPath, inName, inShape, outName, outShape)
+func getModelSession(ctx context.Context, modelPath, inName string, inShape []int64, outName string) (*session, func(), error) {
+ key := sessKeyOf(modelPath, inName, inShape, outName)
return modelSessions.Get(ctx, key, func() (*session, error) {
- return NewSession(modelPath, inName, inShape, outName, outShape)
+ return NewSession(modelPath, inName, inShape, outName)
})
}
diff --git a/internal/deepdoc/native/tsr.go b/internal/deepdoc/native/tsr.go
index 49fdee7a8d..d65af2e620 100644
--- a/internal/deepdoc/native/tsr.go
+++ b/internal/deepdoc/native/tsr.go
@@ -45,8 +45,7 @@ type TSRResult struct {
func RunTSR(ctx context.Context, modelDir string, img *Image) (TSRResult, error) {
blob, sf := tsrPreprocess(img)
sess, release, err := getModelSession(ctx, filepath.Join(modelDir, "tsr.ort"), "images",
- []int64{1, 3, tsrInputSize, tsrInputSize}, "output0",
- []int64{1, 11, tsrCandidates})
+ []int64{1, 3, tsrInputSize, tsrInputSize}, "output0")
if err != nil {
return TSRResult{}, err
}
@@ -56,6 +55,9 @@ func RunTSR(ctx context.Context, modelDir string, img *Image) (TSRResult, error)
if err != nil {
return TSRResult{}, err
}
+ if err := checkOutputLength("tsr", len(out), 11*tsrCandidates); err != nil {
+ return TSRResult{}, err
+ }
res := tsrPostprocess(out, sf)
res.W, res.H = img.W, img.H
return res, nil