RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Perf(deepdoc): free ONNX in/out tensors per Run instead of pinning them per session (#19843)
xugangqiangacabc97c1306
diff --git a/internal/deepdoc/native/det_core.go b/internal/deepdoc/native/det_core.goindex 192272f093..b930c9a402 100644--- a/internal/deepdoc/native/det_core.go+++ b/internal/deepdoc/native/det_core.go@@ -49,10 +49,12 @@ type DetResult struct {// at a VARIABLE input size: each page is aspect-preserved-rescaled to a round32// size bounded by detLimitSideLen, so the pool is keyed by the resized// (height, width) and distinct page sizes get distinct sessions. Sessions are-// pooled per instance, never shared across concurrent Run calls, because-// session.Run mutates the session's fixed-shape input/output tensors; the+// pooled per instance, never shared across concurrent Run calls, because a+// session's underlying ONNX handle is not safe for concurrent use; the// native det branch runs concurrently across the page worker pool, so a-// naively shared single session would race.+// naively shared single session would race. (Each Run allocates its own+// input/output tensors and frees them before returning, so the constraint is+// about the session handle, not any pinned buffer.)//// The set of distinct shapes is BOUNDED (detMaxShapePools). A long-running// server ingesting many differently-sized pages would otherwise pin a pool@@ -86,8 +88,7 @@ func getDetSession(ctx context.Context, modelPath string, rh, rw int64) (*sessiokey := detSessKey{modelPath, rh, rw}return detSessions.Get(ctx, key, func() (*session, error) {return NewSession(modelPath, "x",- []int64{1, 3, rh, rw}, "sigmoid_0.tmp_0",- []int64{1, 1, rh, rw})+ []int64{1, 3, rh, rw}, "sigmoid_0.tmp_0")})}@@ -106,6 +107,9 @@ func RunDet(ctx context.Context, modelDir string, img *Image) (DetResult, error)if e != nil {return DetResult{}, e}+ if e := checkOutputLength("det", len(out), rh*rw); e != nil {+ return DetResult{}, e+ }// out is [1,1,rh,rw]; flatten to [rh,rw].p := make([]float32, rh*rw)copy(p, out)diff --git a/internal/deepdoc/native/dla.go b/internal/deepdoc/native/dla.goindex 356f189bb1..63c5c06705 100644--- a/internal/deepdoc/native/dla.go+++ b/internal/deepdoc/native/dla.go@@ -58,8 +58,7 @@ var (func RunDLA(ctx context.Context, modelDir string, img *Image) (DLAResult, error) {blob, sf := dlaPreprocess(img)sess, release, err := getModelSession(ctx, filepath.Join(modelDir, "layout.ort"), "images",- []int64{1, 3, dlaInputSize, dlaInputSize}, "output0",- []int64{1, dlaMaxBoxes, 6})+ []int64{1, 3, dlaInputSize, dlaInputSize}, "output0")if err != nil {return DLAResult{}, err}@@ -69,6 +68,9 @@ func RunDLA(ctx context.Context, modelDir string, img *Image) (DLAResult, error)if err != nil {return DLAResult{}, err}+ if err := checkOutputLength("dla", len(out), dlaMaxBoxes*6); err != nil {+ return DLAResult{}, err+ }res := dlaPostprocess(out, sf)res.W, res.H = img.W, img.Hreturn res, nildiff --git a/internal/deepdoc/native/ocr_rec.go b/internal/deepdoc/native/ocr_rec.goindex 16e604b9b9..ab87a4fa3d 100644--- a/internal/deepdoc/native/ocr_rec.go+++ b/internal/deepdoc/native/ocr_rec.go@@ -290,61 +290,42 @@ func (r OCRRecResult) Wire() string {// pre-sized per width and even a width-matched session would still emit a// varying seq length. Instead we use a DynamicAdvancedSession and pass a nil// output on every Run: onnxruntime allocates the correctly-shaped output-// tensor, which we copy out before destroying it. The input tensor is-// fixed-shape per (model, width), so one recSession is reused per width.+// tensor, which we copy out before destroying it. The input tensor is NOT+// cached: each Run allocates a fresh input tensor and frees it after (see+// recSession.Run), so a pooled rec session holds only its weights in steady+// state.type recSession struct {inName stringoutName string+ inShape []int64sess *ort.DynamicAdvancedSession- in *ort.Tensor[float32]poisoned bool}func newRecSession(modelPath, inName string, inShape []int64, outName string) (*recSession, error) {- in := make([]float32, prod(inShape))- inT, err := ort.NewTensor(ort.NewShape(inShape...), in)+ opts, err := newSessionOptions()if err != nil {return nil, err}- opts, err := ort.NewSessionOptions()- if err != nil {- inT.Destroy()- return nil, err- }- // One intra-op thread per session, matching NewSession: see the- // intraOpThreads constant in session.go.- if err := opts.SetIntraOpNumThreads(intraOpThreads); err != nil {- opts.Destroy()- inT.Destroy()- return nil, err- }- // Disable the BFC memory arena for this session. See the matching comment in- // NewSession (session.go): the arena never shrinks, so pooled rec sessions- // (up to recMaxShapePools*recShapePoolCap = 128 idle) each hoard a native- // block, which drove most of the ~14 GB of native memory at the ~20 GB OOM- // peak. With the arena off, idle rec sessions keep only their weights.- if err := opts.SetCpuMemArena(false); err != nil {- opts.Destroy()- inT.Destroy()- return nil, err- }+ // See NewSession: the C session does not take ownership of opts, so release+ // it once the session is built. newSessionOptions centralizes the+ // intra-op-thread and arena settings shared with NewSession.+ defer opts.Destroy()sess, err := ort.NewDynamicAdvancedSession(modelPath,[]string{inName}, []string{outName}, opts)if err != nil {- opts.Destroy()- inT.Destroy()return nil, err}- return &recSession{inName: inName, outName: outName, sess: sess, in: inT}, nil+ return &recSession{inName: inName, outName: outName, inShape: inShape, sess: sess}, nil}-// Run copies input into the input tensor, executes with an auto-allocated-// (dynamic) output, and returns the output data. The allocated output tensor-// is destroyed before returning; out is a fresh copy the caller owns.+// Run allocates a fresh input tensor, executes with an auto-allocated+// (dynamic) output, and returns the output data. Both tensors are destroyed+// before returning; out is a fresh copy the caller owns.func (s *recSession) Run(ctx context.Context, input []float32) ([]float32, error) {- if len(input) != len(s.in.GetData()) {- return nil, fmt.Errorf("recSession %s: input len %d != tensor len %d",- s.outName, len(input), len(s.in.GetData()))+ if len(input) != int(prod(s.inShape)) {+ return nil, fmt.Errorf("recSession %s: input len %d != expected %d",+ s.outName, len(input), int(prod(s.inShape)))}opts, err := ort.NewRunOptions()if err != nil {@@ -363,10 +344,15 @@ func (s *recSession) Run(ctx context.Context, input []float32) ([]float32, error}}()- copy(s.in.GetData(), input)- // nil output → onnxruntime allocates the actual-shaped tensor.+ // Fresh input tensor per Run; freed right after the call. nil output →+ // onnxruntime allocates the actual-shaped tensor.+ inT, err := ort.NewTensor(ort.NewShape(s.inShape...), input)+ if err != nil {+ return nil, err+ }+ defer inT.Destroy()outputs := []ort.Value{nil}- if err := s.sess.RunWithOptions([]ort.Value{s.in}, outputs, opts); err != nil {+ if err := s.sess.RunWithOptions([]ort.Value{inT}, outputs, opts); err != nil {if ctx.Err() != nil {s.poisoned = true}@@ -387,14 +373,12 @@ func (s *recSession) Run(ctx context.Context, input []float32) ([]float32, errorreturn out, nil}-// Destroy releases the dynamic session and input tensor.+// Destroy releases the dynamic session. Input/output tensors are no longer+// owned by the session (allocated per Run and freed there).func (s *recSession) Destroy() {if s.sess != nil {s.sess.Destroy()}- if s.in != nil {- s.in.Destroy()- }}func (s *recSession) isPoisoned() bool { return s.poisoned }diff --git a/internal/deepdoc/native/session.go b/internal/deepdoc/native/session.goindex 666cb54ec0..8ddd3fd687 100644--- a/internal/deepdoc/native/session.go+++ b/internal/deepdoc/native/session.go@@ -9,10 +9,11 @@ package native// DeepDoc ONNX we port fits this shape). CPU-only by design.//// The ONNX Runtime environment is process-global: InitORT sets the shared-// library and initializes it exactly once. Sessions only own their tensors and-// the advanced-session handle, so running several tasks in one process (or one-// task per CLI invocation) never double-initializes or prematurely tears down-// the shared environment.+// library and initializes it exactly once. A session owns only its+// DynamicAdvancedSession handle (input/output tensors are allocated per Run and+// freed afterwards), so running several tasks in one process (or one task per+// CLI invocation) never double-initializes or prematurely tears down the+// shared environment.import ("context"@@ -76,41 +77,60 @@ func InitORT() error {func Initialized() bool { return ortReady }// session loads one ONNX model and runs single-input/single-output inference.+// It is a DynamicAdvancedSession: it owns no input/output tensors. Every Run+// allocates fresh input/output tensors and frees them afterwards (see Run), so+// a pooled session's steady-state native memory is just its weights plus ORT's+// plan cache — never the pinned in/out buffers that previously dominated the+// ~14 GB of native memory across the rec/det/DLA/TSR pools.type session struct {inName stringoutName string- outSize int64- sess *ort.AdvancedSession- in *ort.Tensor[float32]- out *ort.Tensor[float32]+ inShape []int64+ sess *ort.DynamicAdvancedSession// poisoned is set when a Run is cancelled/terminated via context. ONNX// Runtime does not guarantee a session is reusable after a forced// termination, so the pool must Destroy rather than re-Put it.poisoned bool}-// NewSession opens modelPath. inShape/outShape describe the fixed tensor-// dimensions; outSize is the total element count of the output tensor. The-// session runs intraOpThreads intra-op threads (see the constant): one thread-// per Run, with the process-wide ceiling owned by the inference budget the-// process owner registers (see inference_limit.go). InitORT must have been-// called first.-func NewSession(modelPath, inName string, inShape []int64, outName string, outShape []int64) (*session, error) {- in := make([]float32, prod(inShape))- out := make([]float32, prod(outShape))- inT, err := ort.NewTensor(ort.NewShape(inShape...), in)+// NewSession opens modelPath. inShape describes the fixed input tensor+// dimensions; output tensors are allocated per Run (their shape is+// model-determined, so no outShape argument is needed). The session runs+// intraOpThreads intra-op threads (see the constant): one thread per Run, with+// the process-wide ceiling owned by the inference budget the process owner+// registers (see inference_limit.go). Input/output tensors are allocated and+// freed on every Run (see session.Run), so a pooled session holds only its+// weights in steady state. InitORT must have been called first.+func NewSession(modelPath, inName string, inShape []int64, outName string) (*session, error) {+ opts, err := newSessionOptions()if err != nil {return nil, err}- outT, err := ort.NewTensor(ort.NewShape(outShape...), out)+ // newSessionOptions builds opts; the C session does not take ownership of it+ // (see newSessionOptions / onnxruntime_go), so release it once the session is+ // built. Releasing after a successful build is the documented pattern and+ // avoids leaking the native SessionOptions handle on every session.+ defer opts.Destroy()+ sess, err := ort.NewDynamicAdvancedSession(modelPath,+ []string{inName}, []string{outName}, opts)if err != nil {- inT.Destroy()return nil, err}+ return &session{+ inName: inName, outName: outName,+ inShape: inShape,+ sess: sess,+ }, nil+}++// newSessionOptions builds the SessionOptions shared by every running session:+// one intra-op thread (so each Run costs exactly one CPU thread — see the+// intraOpThreads constant and inference_limit.go) and the BFC arena disabled+// (idle sessions then keep only their weights; activation tensors are allocated+// per Run and freed after).+func newSessionOptions() (*ort.SessionOptions, error) {opts, err := ort.NewSessionOptions()if err != nil {- inT.Destroy()- outT.Destroy()return nil, err}// One intra-op thread per session: the session's Runs then cost one thread@@ -118,8 +138,6 @@ func NewSession(modelPath, inName string, inShape []int64, outName string, outSh// concurrent Runs the caller admits (see the intraOpThreads constant).if err := opts.SetIntraOpNumThreads(intraOpThreads); err != nil {opts.Destroy()- inT.Destroy()- outT.Destroy()return nil, err}// Disable the BFC memory arena for this session. The arena pre-reserves a@@ -131,35 +149,36 @@ func NewSession(modelPath, inName string, inShape []int64, outName string, outSh// after each Run.if err := opts.SetCpuMemArena(false); err != nil {opts.Destroy()- inT.Destroy()- outT.Destroy()return nil, err}- sess, err := ort.NewAdvancedSession(modelPath,- []string{inName}, []string{outName},- []ort.Value{inT}, []ort.Value{outT}, opts)- if err != nil {- opts.Destroy()- inT.Destroy()- outT.Destroy()- return nil, err+ return opts, nil+}++// checkOutputLength fails fast when a model emits an unexpectedly sized output+// tensor. The previous pinned-output design got this check for free: ORT errored+// when the bound output tensor's shape mismatched the model. The dynamic design+// allocates the output per Run (nil output, ORT allocates), so ORT no longer+// validates the shape; without this guard, postprocessing (dlaPostprocess+// indexes 300*6, tsrPostprocess indexes 11*8400, RunDet fills rh*rw) panics or+// silently misreads a truncated output instead of returning a clean error.+func checkOutputLength(model string, got, want int) error {+ if got != want {+ return fmt.Errorf("%s model output length %d, expected %d", model, got, want)}- return &session{- inName: inName, outName: outName,- outSize: prod(outShape),- sess: sess, in: inT, out: outT,- }, nil+ return nil}-// Run copies input into the input tensor, executes, and returns the output-// tensor contents. ctx bounds the inference: if it is cancelled while Run is-// in flight, the underlying ONNX Runtime call is terminated via RunOptions.-// A terminated session is left in an indeterminate state, so it is marked-// poisoned and the pool destroys it instead of reusing it.+// Run allocates a fresh input tensor, executes with an auto-allocated (dynamic)+// output, and returns the output data. Both tensors are destroyed before+// returning; out is a fresh copy the caller owns. ctx bounds the inference: if+// it is cancelled while Run is in flight, the underlying ONNX Runtime call is+// terminated via RunOptions. A terminated session is left in an indeterminate+// state, so it is marked poisoned and the pool destroys it instead of reusing+// it.func (s *session) Run(ctx context.Context, input []float32) ([]float32, error) {- if len(input) != len(s.in.GetData()) {- return nil, fmt.Errorf("session %s: input len %d != tensor len %d",- s.outName, len(input), len(s.in.GetData()))+ if len(input) != int(prod(s.inShape)) {+ return nil, fmt.Errorf("session %s: input len %d != expected %d",+ s.outName, len(input), int(prod(s.inShape)))}opts, err := ort.NewRunOptions()if err != nil {@@ -178,30 +197,47 @@ func (s *session) Run(ctx context.Context, input []float32) ([]float32, error) {}}()- copy(s.in.GetData(), input)- if err := s.sess.RunWithOptions(opts); err != nil {+ // Allocate a fresh input tensor for this Run and free it right after the+ // call. The output is passed as nil so ORT allocates it; we copy the data+ // out and free the returned Value. This keeps each pooled session's+ // steady-state native memory to just its weights (plus ORT's plan cache)+ // instead of pinning fixed in/out buffers that previously dominated the+ // ~14 GB of native memory seen across the rec/det/DLA/TSR pools.+ inT, err := ort.NewTensor(ort.NewShape(s.inShape...), input)+ if err != nil {+ return nil, err+ }+ defer inT.Destroy()+ outputs := []ort.Value{nil}+ if err := s.sess.RunWithOptions([]ort.Value{inT}, outputs, opts); err != nil {if ctx.Err() != nil {s.poisoned = true}return nil, err}- out := make([]float32, s.outSize)- copy(out, s.out.GetData())+ outVal := outputs[0]+ if outVal == nil {+ return nil, fmt.Errorf("session %s: nil output tensor", s.outName)+ }+ defer outVal.Destroy()+ outT, ok := outVal.(*ort.Tensor[float32])+ if !ok {+ return nil, fmt.Errorf("session %s: unexpected output value type %T",+ s.outName, outVal)+ }+ out := make([]float32, len(outT.GetData()))+ copy(out, outT.GetData())return out, nil}-// Destroy releases the tensors and advanced-session handle. It does NOT touch-// the process-global environment.+// Destroy releases the dynamic advanced-session handle. Input/output tensors+// are no longer owned by the session (they are allocated per Run and freed+// there), so there is nothing else to release here. It does NOT touch the+// process-global environment.func (s *session) Destroy() {if s.sess != nil {s.sess.Destroy()}- if s.in != nil {- s.in.Destroy()- }- if s.out != nil {- s.out.Destroy()- }}func (s *session) isPoisoned() bool { return s.poisoned }diff --git a/internal/deepdoc/native/session_output_test.go b/internal/deepdoc/native/session_output_test.gonew file mode 100644index 0000000000..7f556ad886--- /dev/null+++ b/internal/deepdoc/native/session_output_test.go@@ -0,0 +1,38 @@+//go:build cgo++package native++import "testing"++// TestCheckOutputLength guards the output-length validation that the dynamic+// (per-Run allocated) session relies on. The previous pinned-output design got+// this check for free: ORT errored when the bound output tensor's shape+// mismatched the model. After switching to a nil output that ORT allocates, the+// shape is no longer checked by ORT, so postprocessing (dlaPostprocess indexes+// 300*6, tsrPostprocess indexes 11*8400, RunDet fills rh*rw) would panic or+// silently misread a truncated output. This test locks the expected lengths.+func TestCheckOutputLength(t *testing.T) {+ cases := []struct {+ model string+ got int+ want int+ ok bool+ }{+ {"dla", dlaMaxBoxes * 6, dlaMaxBoxes * 6, true},+ {"dla", dlaMaxBoxes*6 - 1, dlaMaxBoxes * 6, false},+ {"dla", 0, dlaMaxBoxes * 6, false},+ {"tsr", 11 * tsrCandidates, 11 * tsrCandidates, true},+ {"tsr", 11*tsrCandidates - 7, 11 * tsrCandidates, false},+ {"det", 100, 100, true},+ {"det", 99, 100, false},+ }+ for _, c := range cases {+ err := checkOutputLength(c.model, c.got, c.want)+ if c.ok && err != nil {+ t.Errorf("checkOutputLength(%q,%d,%d) = %v, want nil", c.model, c.got, c.want, err)+ }+ if !c.ok && err == nil {+ t.Errorf("checkOutputLength(%q,%d,%d) = nil, want error", c.model, c.got, c.want)+ }+ }+}diff --git a/internal/deepdoc/native/session_pool.go b/internal/deepdoc/native/session_pool.goindex 5d3e23869f..37233af748 100644--- a/internal/deepdoc/native/session_pool.go+++ b/internal/deepdoc/native/session_pool.go@@ -10,9 +10,11 @@ package native// model. This pool caches one session per (model signature) tuple and hands it// back between calls.//-// Sessions are pooled, not shared concurrently: session.Run copies the caller's-// input into the session's fixed-shape input tensor and then executes, so a-// single session must never be touched by two goroutines at once. Get returns a+// Sessions are pooled, not shared concurrently: a session's underlying ONNX+// handle is not safe for concurrent use, so a single session must never be+// touched by two goroutines at once. (Each Run allocates its own input/output+// tensors and frees them before returning; the constraint is about the handle,+// not any pinned buffer.) Get returns a// session owned by the caller until release is called; release returns it to// the pool for reuse. This keeps the Get/Run/Release window single-owner, which// is what makes reuse safe under the page/region worker pools.@@ -201,16 +203,15 @@ func (p *sessionPool[K, V]) evictLRU() {// per modelDir in practice.type sessKey struct {modelPath, inName, outName string- inShape, outShape string+ inShape string}-func sessKeyOf(modelPath, inName string, inShape []int64, outName string, outShape []int64) sessKey {+func sessKeyOf(modelPath, inName string, inShape []int64, outName string) sessKey {return sessKey{modelPath: modelPath,inName: inName,outName: outName,inShape: shapeKey(inShape),- outShape: shapeKey(outShape),}}@@ -228,10 +229,10 @@ var modelSessions = newSessionPool[sessKey, *session](0, 0)// getModelSession returns a reusable session for the given model signature plus// a release func. The caller must call release exactly once.-func getModelSession(ctx context.Context, modelPath, inName string, inShape []int64, outName string, outShape []int64) (*session, func(), error) {- key := sessKeyOf(modelPath, inName, inShape, outName, outShape)+func getModelSession(ctx context.Context, modelPath, inName string, inShape []int64, outName string) (*session, func(), error) {+ key := sessKeyOf(modelPath, inName, inShape, outName)return modelSessions.Get(ctx, key, func() (*session, error) {- return NewSession(modelPath, inName, inShape, outName, outShape)+ return NewSession(modelPath, inName, inShape, outName)})}diff --git a/internal/deepdoc/native/tsr.go b/internal/deepdoc/native/tsr.goindex 49fdee7a8d..d65af2e620 100644--- a/internal/deepdoc/native/tsr.go+++ b/internal/deepdoc/native/tsr.go@@ -45,8 +45,7 @@ type TSRResult struct {func RunTSR(ctx context.Context, modelDir string, img *Image) (TSRResult, error) {blob, sf := tsrPreprocess(img)sess, release, err := getModelSession(ctx, filepath.Join(modelDir, "tsr.ort"), "images",- []int64{1, 3, tsrInputSize, tsrInputSize}, "output0",- []int64{1, 11, tsrCandidates})+ []int64{1, 3, tsrInputSize, tsrInputSize}, "output0")if err != nil {return TSRResult{}, err}@@ -56,6 +55,9 @@ func RunTSR(ctx context.Context, modelDir string, img *Image) (TSRResult, error)if err != nil {return TSRResult{}, err}+ if err := checkOutputLength("tsr", len(out), 11*tsrCandidates); err != nil {+ return TSRResult{}, err+ }res := tsrPostprocess(out, sf)res.W, res.H = img.W, img.Hreturn res, nil