Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Add token usage for agentic rag (#21134)

Yingfenga7ecd12a4e41
diff --git a/internal/agentic_rag/agent.go b/internal/agentic_rag/agent.go
index 991b956e00..1ada439985 100644
--- a/internal/agentic_rag/agent.go
+++ b/internal/agentic_rag/agent.go
@@ -36,6 +36,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/entity/models"
+ "ragflow/internal/tokenizer"
)
// Input carries everything Run needs to spin up a one-shot ReAct conversation
@@ -467,7 +468,11 @@ func Run(ctx context.Context, in Input) (string, error) {
seed = seed[:len(seed)-1]
}
runMessages := sess.explorer.turnMessages(seed, "", question)
- iter := sess.explorer.runner(ctx, explorerAgent, in.Stream).Run(ctx, runMessages)
+ // Everything the ReAct loop calls the model for is the exploration phase:
+ // the loop is where a turn's token budget is actually spent, so it must be
+ // attributable on its own and not merge into the final synthesis.
+ exploreCtx := tokenizer.WithUsagePhase(ctx, "explore")
+ iter := sess.explorer.runner(exploreCtx, explorerAgent, in.Stream).Run(exploreCtx, runMessages)
// Runner assigns IDs to input messages before persisting them. Exclude
// these from recovery so a failed turn cannot reuse a historical answer.
inputMessageIDs := make(map[string]bool, len(runMessages))
@@ -510,6 +515,7 @@ func Run(ctx context.Context, in Input) (string, error) {
// by the bounded budget attached here.
synthCtx, synthCancel := context.WithTimeout(context.WithoutCancel(ctx), finalizeTimeout)
defer synthCancel()
+ synthCtx = tokenizer.WithUsagePhase(synthCtx, "synth")
synthModel := in.Model
if in.SynthModel != nil {
synthModel = in.SynthModel
diff --git a/internal/agentic_rag/agentic_rag_test.go b/internal/agentic_rag/agentic_rag_test.go
index 763986471e..3fa9087040 100644
--- a/internal/agentic_rag/agentic_rag_test.go
+++ b/internal/agentic_rag/agentic_rag_test.go
@@ -30,6 +30,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/entity/models"
+ "ragflow/internal/tokenizer"
)
// TestRun_NilModel: Run must reject a nil model up front.
@@ -223,3 +224,49 @@ func TestRunDispatchesModelSelectedRetrievalTools(t *testing.T) {
}
}
}
+
+// TestRunRecordsTokenUsageByPhase pins the accounting contract: every model call
+// the ReAct loop makes lands in the run sink tagged with the exploration phase,
+// with the provider-reported split. Without this the turn's cost exists nowhere
+// -- chat completions returns no usage, so the sink is the only record.
+func TestRunRecordsTokenUsageByPhase(t *testing.T) {
+ setupTestConfig(t)
+ runtime.SetRetrievalService(newCaptureService())
+ t.Cleanup(func() { runtime.SetRetrievalService(nil) })
+
+ driver := &retrievalToolDriver{respond: func([]models.Message, *models.ChatConfig) *models.ChatResponse {
+ answer := "Answer based on the available evidence."
+ return &models.ChatResponse{
+ Answer: &answer,
+ Usage: &models.TokenUsage{PromptTokens: 900, CompletionTokens: 100, TotalTokens: 1000},
+ }
+ }}
+ name := driver.Name()
+ ctx := tokenizer.WithRunUsage(t.Context())
+ answer, err := Run(ctx, Input{
+ Model: models.NewEinoChatModel(models.NewChatModel(driver, &name, &models.APIConfig{}), nil),
+ TemplateID: "smart-reasoning", TenantID: "tenant", Tools: []tool.BaseTool{NewThinkTool()},
+ Messages: []*schema.Message{schema.UserMessage("Research a topic.")},
+ MaxIterations: 2, ToolCallCounts: map[string]int{}, OnDelta: func(string, string) {},
+ })
+ if err != nil || answer == "" {
+ t.Fatalf("Run: answer=%q err=%v", answer, err)
+ }
+
+ sink := tokenizer.GetRunUsage(ctx)
+ if sink == nil {
+ t.Fatal("no run sink installed")
+ }
+ _, _, total, calls := sink.Snapshot()
+ if calls == 0 || total != 1000*calls {
+ t.Fatalf("aggregate calls=%d total=%d, want %d calls of 1000 tokens", calls, total, calls)
+ }
+ for _, call := range sink.CallSnapshot() {
+ if call.Phase != "explore" {
+ t.Fatalf("call %d phase = %q, want explore", call.Seq, call.Phase)
+ }
+ if call.PromptTokens != 900 || call.CompletionTokens != 100 || call.TotalTokens != 1000 {
+ t.Fatalf("call %d split = %d/%d/%d", call.Seq, call.PromptTokens, call.CompletionTokens, call.TotalTokens)
+ }
+ }
+}
diff --git a/internal/service/agentic_pipeline.go b/internal/service/agentic_pipeline.go
index 3d74a9c5fa..9b7c17c561 100644
--- a/internal/service/agentic_pipeline.go
+++ b/internal/service/agentic_pipeline.go
@@ -30,6 +30,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/tokenizer"
)
// agenticRunTimeout is the wall-clock budget of one agentic turn. The ReAct
@@ -96,6 +97,14 @@ func (s *ChatPipelineService) agenticRag(
// those handles against the same numbering.
registry := agentic_rag.NewEvidenceRegistry()
runCtx = agentic_rag.WithEvidenceRegistry(runCtx, registry)
+ // Per-turn token accounting. Every LLM call inside the run records its
+ // usage into this sink (models.EinoChatModel does it for both the
+ // non-streaming and streaming paths), which is the only place the cost of
+ // an agentic turn exists: chat completions carries no usage field, so a
+ // caller cannot recover it from the answer.
+ runCtx = tokenizer.WithRunUsage(runCtx)
+ runCtx = tokenizer.WithRunAttrs(runCtx, &tokenizer.RunAttrs{UserID: userID})
+ turnStarted := time.Now()
// The retrieval tools authorize their datasets as the CALLER of the run,
// not as the chat's tenant: NLPRetrievalAdapter.Search reads that identity
// from the run's runtime state (state.Sys["user_id"]) and treats a missing
@@ -228,16 +237,31 @@ func (s *ChatPipelineService) agenticRag(
if thinking {
emitResult(AsyncChatResult{Final: false, EndToThink: true})
}
+ usageFields := agenticUsageFields(runCtx, tokenizer.GetRunUsage(runCtx))
if runErr != nil {
common.ErrorCtx(runCtx, "agentic_rag: run", runErr)
+ // A failed turn still spent tokens; log them under their own event so
+ // the cost of timeouts and tool failures is visible instead of
+ // disappearing with the error.
+ common.InfoCtx(ctx, "agentic turn failed", append([]zap.Field{
+ zap.String("mode", mode),
+ zap.Bool("stream", stream),
+ zap.Int64("elapsed_ms", time.Since(turnStarted).Milliseconds()),
+ zap.Any("tool_calls", toolCounts),
+ zap.Any("tool_errors", toolErrors),
+ zap.String("error", runErr.Error()),
+ }, usageFields...)...)
emitResult(AsyncChatResult{Answer: fmt.Sprintf("**ERROR**: %s", runErr.Error()), Final: true})
return
}
- common.InfoCtx(ctx, "agentic turn done",
+ common.InfoCtx(ctx, "agentic turn done", append([]zap.Field{
zap.String("mode", mode),
+ zap.Bool("stream", stream),
+ zap.Int64("elapsed_ms", time.Since(turnStarted).Milliseconds()),
zap.Any("tool_calls", toolCounts),
zap.Any("tool_errors", toolErrors),
- zap.Int("final_bytes", len(final)))
+ zap.Int("final_bytes", len(final)),
+ }, usageFields...)...)
// Citations off means neither the [ID:N] markers nor the reference
// payload, matching what the regular pipeline ships when quote is
@@ -265,6 +289,73 @@ func (s *ChatPipelineService) agenticRag(
return out, nil
}
+// agenticUsageFields renders a turn's LLM cost as structured log fields.
+//
+// The chat-completions response carries no usage, and conversation storage has
+// no token column, so this log line is the only per-question cost record: it is
+// what a benchmark run aggregates and what a support investigation greps. The
+// per-call breakdown goes to the debug log to keep the INFO line one line per
+// turn.
+//
+// usage_missing_calls counts calls whose provider reported no usable split.
+// They are counted rather than silently folded into the totals, so "cheap"
+// turns that actually lost usage are distinguishable from genuinely cheap ones.
+func agenticUsageFields(ctx context.Context, usage *tokenizer.RunUsage) []zap.Field {
+ if usage == nil {
+ return []zap.Field{zap.Bool("usage_tracked", false)}
+ }
+ promptTokens, completionTokens, totalTokens, calls := usage.Snapshot()
+ records := usage.CallSnapshot()
+ missing := 0
+ byModel := map[string]int{}
+ byPhase := map[string]int{}
+ topModel, topTokens := "", 0
+ for _, call := range records {
+ if call.TotalTokens <= 0 {
+ missing++
+ }
+ if call.Model != "" {
+ byModel[call.Model] += call.TotalTokens
+ if byModel[call.Model] > topTokens {
+ topModel, topTokens = call.Model, byModel[call.Model]
+ }
+ }
+ phase := call.Phase
+ if phase == "" {
+ phase = "unspecified"
+ }
+ byPhase[phase] += call.TotalTokens
+ common.DebugCtx(ctx, "agentic_rag: llm call",
+ zap.Int("seq", call.Seq),
+ zap.Float64("at_seconds", call.AtSeconds),
+ zap.String("model", call.Model),
+ zap.String("phase", call.Phase),
+ zap.Int("prompt_tokens", call.PromptTokens),
+ zap.Int("completion_tokens", call.CompletionTokens),
+ zap.Int("total_tokens", call.TotalTokens))
+ }
+
+ fields := []zap.Field{
+ zap.Int("prompt_tokens", promptTokens),
+ zap.Int("completion_tokens", completionTokens),
+ zap.Int("total_tokens", totalTokens),
+ zap.Int("llm_calls", calls),
+ zap.Int("usage_missing_calls", missing),
+ zap.Any("tokens_by_phase", byPhase),
+ zap.String("top_model", topModel),
+ }
+ if calls > 0 {
+ fields = append(fields, zap.Int("tokens_per_call", totalTokens/calls))
+ }
+ if len(byModel) > 1 {
+ fields = append(fields, zap.Any("tokens_by_model", byModel))
+ }
+ if missing > 0 {
+ fields = append(fields, zap.Bool("usage_incomplete", true))
+ }
+ return fields
+}
+
// agenticModelChain resolves only the model explicitly selected by the dialog
// (or the tenant default when no model is selected), plus any failover members
// the dialog configures. Agentic RAG must not silently broaden the dialog's
diff --git a/internal/service/agentic_usage_test.go b/internal/service/agentic_usage_test.go
new file mode 100644
index 0000000000..e9e28189dd
--- /dev/null
+++ b/internal/service/agentic_usage_test.go
@@ -0,0 +1,102 @@
+package service
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "testing"
+
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+
+ "ragflow/internal/tokenizer"
+)
+
+// TestAgenticUsageFields pins the per-turn cost record that stands in for the
+// token usage the chat-completions response never carries: totals, call count,
+// per-phase split, and the count of calls whose provider reported nothing.
+func TestAgenticUsageFields(t *testing.T) {
+ base := tokenizer.WithRunUsage(context.Background())
+ explore := tokenizer.WithUsagePhase(base, "explore")
+ tokenizer.RecordRunTokenUsageFor(explore, "MiniMax-M3", 1000, 100, 1100)
+ tokenizer.RecordRunTokenUsageFor(explore, "MiniMax-M3", 2000, 200, 2200)
+ synth := tokenizer.WithUsagePhase(base, "synth")
+ tokenizer.RecordRunTokenUsageFor(synth, "MiniMax-M3", 100, 300, 400)
+ // A provider that reports no usage must be counted, not silently folded in
+ // as a free call.
+ tokenizer.RecordRunTokenUsageFor(synth, "MiniMax-M3", 0, 0, 0)
+
+ fields := agenticLogFields(t, agenticUsageFields(base, tokenizer.GetRunUsage(base)))
+ assertLogField(t, fields, "prompt_tokens", 3100)
+ assertLogField(t, fields, "completion_tokens", 600)
+ assertLogField(t, fields, "total_tokens", 3700)
+ assertLogField(t, fields, "llm_calls", 4)
+ assertLogField(t, fields, "usage_missing_calls", 1)
+ assertLogField(t, fields, "tokens_per_call", 925)
+ assertLogField(t, fields, "top_model", "MiniMax-M3")
+ assertLogField(t, fields, "usage_incomplete", true)
+
+ phases, ok := fields["tokens_by_phase"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("tokens_by_phase has type %T", fields["tokens_by_phase"])
+ }
+ if phases["explore"] != float64(3300) || phases["synth"] != float64(400) {
+ t.Fatalf("tokens_by_phase = %v", phases)
+ }
+}
+
+// TestAgenticUsageFieldsWithoutSink covers the path where accounting was never
+// installed: the log line must say so instead of reporting zeros as a real
+// measurement.
+func TestAgenticUsageFieldsWithoutSink(t *testing.T) {
+ fields := agenticLogFields(t, agenticUsageFields(context.Background(), nil))
+ assertLogField(t, fields, "usage_tracked", false)
+ if _, ok := fields["total_tokens"]; ok {
+ t.Fatalf("untracked turn must not report totals: %v", fields)
+ }
+}
+
+// agenticLogFields renders zap fields through a real JSON encoder so the test
+// observes exactly what the log sink would emit.
+func agenticLogFields(t *testing.T, fields []zap.Field) map[string]interface{} {
+ t.Helper()
+ var buf bytes.Buffer
+ logger := zap.New(zapcore.NewCore(
+ zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()),
+ zapcore.AddSync(&buf),
+ zapcore.InfoLevel,
+ ))
+ logger.Info("agentic turn done", fields...)
+ var decoded map[string]interface{}
+ if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil {
+ t.Fatalf("decode log line %q: %v", buf.String(), err)
+ }
+ return decoded
+}
+
+func assertLogField(t *testing.T, fields map[string]interface{}, key string, want interface{}) {
+ t.Helper()
+ got, ok := fields[key]
+ if !ok {
+ t.Fatalf("missing log field %q in %v", key, fields)
+ }
+ switch expected := want.(type) {
+ case int:
+ number, ok := got.(float64)
+ if !ok || number != float64(expected) {
+ t.Fatalf("%s = %v, want %d", key, got, expected)
+ }
+ case bool:
+ flag, ok := got.(bool)
+ if !ok || flag != expected {
+ t.Fatalf("%s = %v, want %v", key, got, expected)
+ }
+ case string:
+ text, ok := got.(string)
+ if !ok || text != expected {
+ t.Fatalf("%s = %v, want %q", key, got, expected)
+ }
+ default:
+ t.Fatalf("unsupported assertion type %T for %s", want, key)
+ }
+}
diff --git a/internal/tokenizer/usage.go b/internal/tokenizer/usage.go
index cfd89632cf..c5012a5303 100644
--- a/internal/tokenizer/usage.go
+++ b/internal/tokenizer/usage.go
@@ -50,6 +50,11 @@ type CallUsage struct {
// Model is the model the call went to, when the caller knows it (the
// explorer, the auditor and the synthesis fallback can use different ones).
Model string `json:"model,omitempty"`
+ // Phase names the run section that issued the call ("explore", "synth"),
+ // set by WithUsagePhase on the section's context. Empty when the caller did
+ // not scope the call, which keeps per-phase attribution honest instead of
+ // guessing from the call order.
+ Phase string `json:"phase,omitempty"`
// PromptTokens / CompletionTokens are the call's input and output split.
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
@@ -59,6 +64,7 @@ type CallUsage struct {
// Context key types — unexported to prevent direct external access.
type runUsageKeyType struct{}
type runAttrsKeyType struct{}
+type usagePhaseKeyType struct{}
// RunUsage is the mutable per-run token usage accumulator installed on
// the context by the service layer at the start of a canvas turn.
@@ -87,6 +93,13 @@ func (u *RunUsage) Add(prompt, completion, total int) {
// token split. Callers that know the model (the chat-model wrapper does) get a
// per-turn breakdown for free; the rest still get a record with an empty model.
func (u *RunUsage) AddFor(model string, prompt, completion, total int) {
+ u.AddForPhase(model, "", prompt, completion, total)
+}
+
+// AddForPhase is AddFor plus the run section that issued the call. The phase is
+// what makes "the exploration burned 38k prompt tokens, the final synthesis 3k"
+// answerable; a call that carries no phase is recorded as such, never guessed.
+func (u *RunUsage) AddForPhase(model, phase string, prompt, completion, total int) {
if u == nil {
return
}
@@ -110,6 +123,7 @@ func (u *RunUsage) AddFor(model string, prompt, completion, total int) {
Seq: u.Calls,
AtSeconds: float64(int(at*1000)) / 1000,
Model: model,
+ Phase: phase,
PromptTokens: prompt,
CompletionTokens: completion,
TotalTokens: total,
@@ -192,13 +206,38 @@ func RecordRunTokenUsage(ctx context.Context, promptTokens, completionTokens, to
// RecordRunTokenUsageFor is RecordRunTokenUsage plus the model that served the
// call, so the per-call breakdown can attribute tokens to the explorer, the
-// auditor or the synthesis fallback.
+// auditor or the synthesis fallback. The run section is read from the context
+// (see WithUsagePhase) so call sites stay unaware of the accounting.
func RecordRunTokenUsageFor(ctx context.Context, model string, promptTokens, completionTokens, totalTokens int) {
sink := GetRunUsage(ctx)
if sink == nil {
return
}
- sink.AddFor(model, promptTokens, completionTokens, totalTokens)
+ sink.AddForPhase(model, UsagePhase(ctx), promptTokens, completionTokens, totalTokens)
+}
+
+// WithUsagePhase tags ctx with the run section that is about to issue LLM
+// calls. Nested sections inherit it, so a section that does not set its own
+// phase is attributed to its caller — which is why every top-level section of a
+// run sets one explicitly.
+func WithUsagePhase(ctx context.Context, phase string) context.Context {
+ if phase == "" {
+ return ctx
+ }
+ return context.WithValue(ctx, usagePhaseKeyType{}, phase)
+}
+
+// UsagePhase returns the run section installed by WithUsagePhase, if any.
+func UsagePhase(ctx context.Context) string {
+ if ctx == nil {
+ return ""
+ }
+ if v := ctx.Value(usagePhaseKeyType{}); v != nil {
+ if phase, ok := v.(string); ok {
+ return phase
+ }
+ }
+ return ""
}
// UsageFromMap extracts a token usage split from a raw API response map.
diff --git a/internal/tokenizer/usage_test.go b/internal/tokenizer/usage_test.go
index cc9e6d2414..468892494c 100644
--- a/internal/tokenizer/usage_test.go
+++ b/internal/tokenizer/usage_test.go
@@ -52,3 +52,36 @@ func TestRunUsageCallSnapshot(t *testing.T) {
// Without a sink installed, recording is a no-op (not a panic).
RecordRunTokenUsageFor(context.Background(), "m", 1, 1, 2)
}
+
+// TestRunUsagePhases pins phase attribution: a call issued inside a section
+// that installed a phase is tagged with it, a section that installs nothing
+// inherits its caller's phase, and the aggregate is untouched by attribution.
+func TestRunUsagePhases(t *testing.T) {
+ base := WithRunUsage(context.Background())
+ explore := WithUsagePhase(base, "explore")
+ RecordRunTokenUsageFor(explore, "MiniMax-M3", 1000, 100, 1100)
+ // An empty phase means "inherit", not "clear".
+ RecordRunTokenUsageFor(WithUsagePhase(explore, ""), "MiniMax-M3", 2000, 200, 2200)
+ synth := WithUsagePhase(base, "synth")
+ RecordRunTokenUsageFor(synth, "MiniMax-M3", 100, 300, 400)
+
+ if got := UsagePhase(base); got != "" {
+ t.Fatalf("base context phase = %q, want empty", got)
+ }
+ if got := UsagePhase(explore); got != "explore" {
+ t.Fatalf("explore phase = %q", got)
+ }
+
+ sink := GetRunUsage(base)
+ pt, ct, tt, calls := sink.Snapshot()
+ if pt != 3100 || ct != 600 || tt != 3700 || calls != 3 {
+ t.Fatalf("aggregate = %d/%d/%d calls=%d", pt, ct, tt, calls)
+ }
+ turns := sink.CallSnapshot()
+ want := []string{"explore", "explore", "synth"}
+ for i, phase := range want {
+ if turns[i].Phase != phase {
+ t.Fatalf("call %d phase = %q, want %q", i+1, turns[i].Phase, phase)
+ }
+ }
+}