RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Add token usage for agentic rag (#21134)
Yingfenga7ecd12a4e41
diff --git a/internal/agentic_rag/agent.go b/internal/agentic_rag/agent.goindex 991b956e00..1ada439985 100644--- a/internal/agentic_rag/agent.go+++ b/internal/agentic_rag/agent.go@@ -36,6 +36,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/entity/models"+ "ragflow/internal/tokenizer")// Input carries everything Run needs to spin up a one-shot ReAct conversation@@ -467,7 +468,11 @@ func Run(ctx context.Context, in Input) (string, error) {seed = seed[:len(seed)-1]}runMessages := sess.explorer.turnMessages(seed, "", question)- iter := sess.explorer.runner(ctx, explorerAgent, in.Stream).Run(ctx, runMessages)+ // Everything the ReAct loop calls the model for is the exploration phase:+ // the loop is where a turn's token budget is actually spent, so it must be+ // attributable on its own and not merge into the final synthesis.+ exploreCtx := tokenizer.WithUsagePhase(ctx, "explore")+ iter := sess.explorer.runner(exploreCtx, explorerAgent, in.Stream).Run(exploreCtx, runMessages)// Runner assigns IDs to input messages before persisting them. Exclude// these from recovery so a failed turn cannot reuse a historical answer.inputMessageIDs := make(map[string]bool, len(runMessages))@@ -510,6 +515,7 @@ func Run(ctx context.Context, in Input) (string, error) {// by the bounded budget attached here.synthCtx, synthCancel := context.WithTimeout(context.WithoutCancel(ctx), finalizeTimeout)defer synthCancel()+ synthCtx = tokenizer.WithUsagePhase(synthCtx, "synth")synthModel := in.Modelif in.SynthModel != nil {synthModel = in.SynthModeldiff --git a/internal/agentic_rag/agentic_rag_test.go b/internal/agentic_rag/agentic_rag_test.goindex 763986471e..3fa9087040 100644--- a/internal/agentic_rag/agentic_rag_test.go+++ b/internal/agentic_rag/agentic_rag_test.go@@ -30,6 +30,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/entity/models"+ "ragflow/internal/tokenizer")// TestRun_NilModel: Run must reject a nil model up front.@@ -223,3 +224,49 @@ func TestRunDispatchesModelSelectedRetrievalTools(t *testing.T) {}}}++// TestRunRecordsTokenUsageByPhase pins the accounting contract: every model call+// the ReAct loop makes lands in the run sink tagged with the exploration phase,+// with the provider-reported split. Without this the turn's cost exists nowhere+// -- chat completions returns no usage, so the sink is the only record.+func TestRunRecordsTokenUsageByPhase(t *testing.T) {+ setupTestConfig(t)+ runtime.SetRetrievalService(newCaptureService())+ t.Cleanup(func() { runtime.SetRetrievalService(nil) })++ driver := &retrievalToolDriver{respond: func([]models.Message, *models.ChatConfig) *models.ChatResponse {+ answer := "Answer based on the available evidence."+ return &models.ChatResponse{+ Answer: &answer,+ Usage: &models.TokenUsage{PromptTokens: 900, CompletionTokens: 100, TotalTokens: 1000},+ }+ }}+ name := driver.Name()+ ctx := tokenizer.WithRunUsage(t.Context())+ answer, err := Run(ctx, Input{+ Model: models.NewEinoChatModel(models.NewChatModel(driver, &name, &models.APIConfig{}), nil),+ TemplateID: "smart-reasoning", TenantID: "tenant", Tools: []tool.BaseTool{NewThinkTool()},+ Messages: []*schema.Message{schema.UserMessage("Research a topic.")},+ MaxIterations: 2, ToolCallCounts: map[string]int{}, OnDelta: func(string, string) {},+ })+ if err != nil || answer == "" {+ t.Fatalf("Run: answer=%q err=%v", answer, err)+ }++ sink := tokenizer.GetRunUsage(ctx)+ if sink == nil {+ t.Fatal("no run sink installed")+ }+ _, _, total, calls := sink.Snapshot()+ if calls == 0 || total != 1000*calls {+ t.Fatalf("aggregate calls=%d total=%d, want %d calls of 1000 tokens", calls, total, calls)+ }+ for _, call := range sink.CallSnapshot() {+ if call.Phase != "explore" {+ t.Fatalf("call %d phase = %q, want explore", call.Seq, call.Phase)+ }+ if call.PromptTokens != 900 || call.CompletionTokens != 100 || call.TotalTokens != 1000 {+ t.Fatalf("call %d split = %d/%d/%d", call.Seq, call.PromptTokens, call.CompletionTokens, call.TotalTokens)+ }+ }+}diff --git a/internal/service/agentic_pipeline.go b/internal/service/agentic_pipeline.goindex 3d74a9c5fa..9b7c17c561 100644--- a/internal/service/agentic_pipeline.go+++ b/internal/service/agentic_pipeline.go@@ -30,6 +30,7 @@ import ("ragflow/internal/common""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/tokenizer")// agenticRunTimeout is the wall-clock budget of one agentic turn. The ReAct@@ -96,6 +97,14 @@ func (s *ChatPipelineService) agenticRag(// those handles against the same numbering.registry := agentic_rag.NewEvidenceRegistry()runCtx = agentic_rag.WithEvidenceRegistry(runCtx, registry)+ // Per-turn token accounting. Every LLM call inside the run records its+ // usage into this sink (models.EinoChatModel does it for both the+ // non-streaming and streaming paths), which is the only place the cost of+ // an agentic turn exists: chat completions carries no usage field, so a+ // caller cannot recover it from the answer.+ runCtx = tokenizer.WithRunUsage(runCtx)+ runCtx = tokenizer.WithRunAttrs(runCtx, &tokenizer.RunAttrs{UserID: userID})+ turnStarted := time.Now()// The retrieval tools authorize their datasets as the CALLER of the run,// not as the chat's tenant: NLPRetrievalAdapter.Search reads that identity// from the run's runtime state (state.Sys["user_id"]) and treats a missing@@ -228,16 +237,31 @@ func (s *ChatPipelineService) agenticRag(if thinking {emitResult(AsyncChatResult{Final: false, EndToThink: true})}+ usageFields := agenticUsageFields(runCtx, tokenizer.GetRunUsage(runCtx))if runErr != nil {common.ErrorCtx(runCtx, "agentic_rag: run", runErr)+ // A failed turn still spent tokens; log them under their own event so+ // the cost of timeouts and tool failures is visible instead of+ // disappearing with the error.+ common.InfoCtx(ctx, "agentic turn failed", append([]zap.Field{+ zap.String("mode", mode),+ zap.Bool("stream", stream),+ zap.Int64("elapsed_ms", time.Since(turnStarted).Milliseconds()),+ zap.Any("tool_calls", toolCounts),+ zap.Any("tool_errors", toolErrors),+ zap.String("error", runErr.Error()),+ }, usageFields...)...)emitResult(AsyncChatResult{Answer: fmt.Sprintf("**ERROR**: %s", runErr.Error()), Final: true})return}- common.InfoCtx(ctx, "agentic turn done",+ common.InfoCtx(ctx, "agentic turn done", append([]zap.Field{zap.String("mode", mode),+ zap.Bool("stream", stream),+ zap.Int64("elapsed_ms", time.Since(turnStarted).Milliseconds()),zap.Any("tool_calls", toolCounts),zap.Any("tool_errors", toolErrors),- zap.Int("final_bytes", len(final)))+ zap.Int("final_bytes", len(final)),+ }, usageFields...)...)// Citations off means neither the [ID:N] markers nor the reference// payload, matching what the regular pipeline ships when quote is@@ -265,6 +289,73 @@ func (s *ChatPipelineService) agenticRag(return out, nil}+// agenticUsageFields renders a turn's LLM cost as structured log fields.+//+// The chat-completions response carries no usage, and conversation storage has+// no token column, so this log line is the only per-question cost record: it is+// what a benchmark run aggregates and what a support investigation greps. The+// per-call breakdown goes to the debug log to keep the INFO line one line per+// turn.+//+// usage_missing_calls counts calls whose provider reported no usable split.+// They are counted rather than silently folded into the totals, so "cheap"+// turns that actually lost usage are distinguishable from genuinely cheap ones.+func agenticUsageFields(ctx context.Context, usage *tokenizer.RunUsage) []zap.Field {+ if usage == nil {+ return []zap.Field{zap.Bool("usage_tracked", false)}+ }+ promptTokens, completionTokens, totalTokens, calls := usage.Snapshot()+ records := usage.CallSnapshot()+ missing := 0+ byModel := map[string]int{}+ byPhase := map[string]int{}+ topModel, topTokens := "", 0+ for _, call := range records {+ if call.TotalTokens <= 0 {+ missing+++ }+ if call.Model != "" {+ byModel[call.Model] += call.TotalTokens+ if byModel[call.Model] > topTokens {+ topModel, topTokens = call.Model, byModel[call.Model]+ }+ }+ phase := call.Phase+ if phase == "" {+ phase = "unspecified"+ }+ byPhase[phase] += call.TotalTokens+ common.DebugCtx(ctx, "agentic_rag: llm call",+ zap.Int("seq", call.Seq),+ zap.Float64("at_seconds", call.AtSeconds),+ zap.String("model", call.Model),+ zap.String("phase", call.Phase),+ zap.Int("prompt_tokens", call.PromptTokens),+ zap.Int("completion_tokens", call.CompletionTokens),+ zap.Int("total_tokens", call.TotalTokens))+ }++ fields := []zap.Field{+ zap.Int("prompt_tokens", promptTokens),+ zap.Int("completion_tokens", completionTokens),+ zap.Int("total_tokens", totalTokens),+ zap.Int("llm_calls", calls),+ zap.Int("usage_missing_calls", missing),+ zap.Any("tokens_by_phase", byPhase),+ zap.String("top_model", topModel),+ }+ if calls > 0 {+ fields = append(fields, zap.Int("tokens_per_call", totalTokens/calls))+ }+ if len(byModel) > 1 {+ fields = append(fields, zap.Any("tokens_by_model", byModel))+ }+ if missing > 0 {+ fields = append(fields, zap.Bool("usage_incomplete", true))+ }+ return fields+}+// agenticModelChain resolves only the model explicitly selected by the dialog// (or the tenant default when no model is selected), plus any failover members// the dialog configures. Agentic RAG must not silently broaden the dialog'sdiff --git a/internal/service/agentic_usage_test.go b/internal/service/agentic_usage_test.gonew file mode 100644index 0000000000..e9e28189dd--- /dev/null+++ b/internal/service/agentic_usage_test.go@@ -0,0 +1,102 @@+package service++import (+ "bytes"+ "context"+ "encoding/json"+ "testing"++ "go.uber.org/zap"+ "go.uber.org/zap/zapcore"++ "ragflow/internal/tokenizer"+)++// TestAgenticUsageFields pins the per-turn cost record that stands in for the+// token usage the chat-completions response never carries: totals, call count,+// per-phase split, and the count of calls whose provider reported nothing.+func TestAgenticUsageFields(t *testing.T) {+ base := tokenizer.WithRunUsage(context.Background())+ explore := tokenizer.WithUsagePhase(base, "explore")+ tokenizer.RecordRunTokenUsageFor(explore, "MiniMax-M3", 1000, 100, 1100)+ tokenizer.RecordRunTokenUsageFor(explore, "MiniMax-M3", 2000, 200, 2200)+ synth := tokenizer.WithUsagePhase(base, "synth")+ tokenizer.RecordRunTokenUsageFor(synth, "MiniMax-M3", 100, 300, 400)+ // A provider that reports no usage must be counted, not silently folded in+ // as a free call.+ tokenizer.RecordRunTokenUsageFor(synth, "MiniMax-M3", 0, 0, 0)++ fields := agenticLogFields(t, agenticUsageFields(base, tokenizer.GetRunUsage(base)))+ assertLogField(t, fields, "prompt_tokens", 3100)+ assertLogField(t, fields, "completion_tokens", 600)+ assertLogField(t, fields, "total_tokens", 3700)+ assertLogField(t, fields, "llm_calls", 4)+ assertLogField(t, fields, "usage_missing_calls", 1)+ assertLogField(t, fields, "tokens_per_call", 925)+ assertLogField(t, fields, "top_model", "MiniMax-M3")+ assertLogField(t, fields, "usage_incomplete", true)++ phases, ok := fields["tokens_by_phase"].(map[string]interface{})+ if !ok {+ t.Fatalf("tokens_by_phase has type %T", fields["tokens_by_phase"])+ }+ if phases["explore"] != float64(3300) || phases["synth"] != float64(400) {+ t.Fatalf("tokens_by_phase = %v", phases)+ }+}++// TestAgenticUsageFieldsWithoutSink covers the path where accounting was never+// installed: the log line must say so instead of reporting zeros as a real+// measurement.+func TestAgenticUsageFieldsWithoutSink(t *testing.T) {+ fields := agenticLogFields(t, agenticUsageFields(context.Background(), nil))+ assertLogField(t, fields, "usage_tracked", false)+ if _, ok := fields["total_tokens"]; ok {+ t.Fatalf("untracked turn must not report totals: %v", fields)+ }+}++// agenticLogFields renders zap fields through a real JSON encoder so the test+// observes exactly what the log sink would emit.+func agenticLogFields(t *testing.T, fields []zap.Field) map[string]interface{} {+ t.Helper()+ var buf bytes.Buffer+ logger := zap.New(zapcore.NewCore(+ zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()),+ zapcore.AddSync(&buf),+ zapcore.InfoLevel,+ ))+ logger.Info("agentic turn done", fields...)+ var decoded map[string]interface{}+ if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil {+ t.Fatalf("decode log line %q: %v", buf.String(), err)+ }+ return decoded+}++func assertLogField(t *testing.T, fields map[string]interface{}, key string, want interface{}) {+ t.Helper()+ got, ok := fields[key]+ if !ok {+ t.Fatalf("missing log field %q in %v", key, fields)+ }+ switch expected := want.(type) {+ case int:+ number, ok := got.(float64)+ if !ok || number != float64(expected) {+ t.Fatalf("%s = %v, want %d", key, got, expected)+ }+ case bool:+ flag, ok := got.(bool)+ if !ok || flag != expected {+ t.Fatalf("%s = %v, want %v", key, got, expected)+ }+ case string:+ text, ok := got.(string)+ if !ok || text != expected {+ t.Fatalf("%s = %v, want %q", key, got, expected)+ }+ default:+ t.Fatalf("unsupported assertion type %T for %s", want, key)+ }+}diff --git a/internal/tokenizer/usage.go b/internal/tokenizer/usage.goindex cfd89632cf..c5012a5303 100644--- a/internal/tokenizer/usage.go+++ b/internal/tokenizer/usage.go@@ -50,6 +50,11 @@ type CallUsage struct {// Model is the model the call went to, when the caller knows it (the// explorer, the auditor and the synthesis fallback can use different ones).Model string `json:"model,omitempty"`+ // Phase names the run section that issued the call ("explore", "synth"),+ // set by WithUsagePhase on the section's context. Empty when the caller did+ // not scope the call, which keeps per-phase attribution honest instead of+ // guessing from the call order.+ Phase string `json:"phase,omitempty"`// PromptTokens / CompletionTokens are the call's input and output split.PromptTokens int `json:"prompt_tokens"`CompletionTokens int `json:"completion_tokens"`@@ -59,6 +64,7 @@ type CallUsage struct {// Context key types — unexported to prevent direct external access.type runUsageKeyType struct{}type runAttrsKeyType struct{}+type usagePhaseKeyType struct{}// RunUsage is the mutable per-run token usage accumulator installed on// the context by the service layer at the start of a canvas turn.@@ -87,6 +93,13 @@ func (u *RunUsage) Add(prompt, completion, total int) {// token split. Callers that know the model (the chat-model wrapper does) get a// per-turn breakdown for free; the rest still get a record with an empty model.func (u *RunUsage) AddFor(model string, prompt, completion, total int) {+ u.AddForPhase(model, "", prompt, completion, total)+}++// AddForPhase is AddFor plus the run section that issued the call. The phase is+// what makes "the exploration burned 38k prompt tokens, the final synthesis 3k"+// answerable; a call that carries no phase is recorded as such, never guessed.+func (u *RunUsage) AddForPhase(model, phase string, prompt, completion, total int) {if u == nil {return}@@ -110,6 +123,7 @@ func (u *RunUsage) AddFor(model string, prompt, completion, total int) {Seq: u.Calls,AtSeconds: float64(int(at*1000)) / 1000,Model: model,+ Phase: phase,PromptTokens: prompt,CompletionTokens: completion,TotalTokens: total,@@ -192,13 +206,38 @@ func RecordRunTokenUsage(ctx context.Context, promptTokens, completionTokens, to// RecordRunTokenUsageFor is RecordRunTokenUsage plus the model that served the// call, so the per-call breakdown can attribute tokens to the explorer, the-// auditor or the synthesis fallback.+// auditor or the synthesis fallback. The run section is read from the context+// (see WithUsagePhase) so call sites stay unaware of the accounting.func RecordRunTokenUsageFor(ctx context.Context, model string, promptTokens, completionTokens, totalTokens int) {sink := GetRunUsage(ctx)if sink == nil {return}- sink.AddFor(model, promptTokens, completionTokens, totalTokens)+ sink.AddForPhase(model, UsagePhase(ctx), promptTokens, completionTokens, totalTokens)+}++// WithUsagePhase tags ctx with the run section that is about to issue LLM+// calls. Nested sections inherit it, so a section that does not set its own+// phase is attributed to its caller — which is why every top-level section of a+// run sets one explicitly.+func WithUsagePhase(ctx context.Context, phase string) context.Context {+ if phase == "" {+ return ctx+ }+ return context.WithValue(ctx, usagePhaseKeyType{}, phase)+}++// UsagePhase returns the run section installed by WithUsagePhase, if any.+func UsagePhase(ctx context.Context) string {+ if ctx == nil {+ return ""+ }+ if v := ctx.Value(usagePhaseKeyType{}); v != nil {+ if phase, ok := v.(string); ok {+ return phase+ }+ }+ return ""}// UsageFromMap extracts a token usage split from a raw API response map.diff --git a/internal/tokenizer/usage_test.go b/internal/tokenizer/usage_test.goindex cc9e6d2414..468892494c 100644--- a/internal/tokenizer/usage_test.go+++ b/internal/tokenizer/usage_test.go@@ -52,3 +52,36 @@ func TestRunUsageCallSnapshot(t *testing.T) {// Without a sink installed, recording is a no-op (not a panic).RecordRunTokenUsageFor(context.Background(), "m", 1, 1, 2)}++// TestRunUsagePhases pins phase attribution: a call issued inside a section+// that installed a phase is tagged with it, a section that installs nothing+// inherits its caller's phase, and the aggregate is untouched by attribution.+func TestRunUsagePhases(t *testing.T) {+ base := WithRunUsage(context.Background())+ explore := WithUsagePhase(base, "explore")+ RecordRunTokenUsageFor(explore, "MiniMax-M3", 1000, 100, 1100)+ // An empty phase means "inherit", not "clear".+ RecordRunTokenUsageFor(WithUsagePhase(explore, ""), "MiniMax-M3", 2000, 200, 2200)+ synth := WithUsagePhase(base, "synth")+ RecordRunTokenUsageFor(synth, "MiniMax-M3", 100, 300, 400)++ if got := UsagePhase(base); got != "" {+ t.Fatalf("base context phase = %q, want empty", got)+ }+ if got := UsagePhase(explore); got != "explore" {+ t.Fatalf("explore phase = %q", got)+ }++ sink := GetRunUsage(base)+ pt, ct, tt, calls := sink.Snapshot()+ if pt != 3100 || ct != 600 || tt != 3700 || calls != 3 {+ t.Fatalf("aggregate = %d/%d/%d calls=%d", pt, ct, tt, calls)+ }+ turns := sink.CallSnapshot()+ want := []string{"explore", "explore", "synth"}+ for i, phase := range want {+ if turns[i].Phase != phase {+ t.Fatalf("call %d phase = %q, want %q", i+1, turns[i].Phase, phase)+ }+ }+}