RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: num token usage (#19495)
Lynn97029b08f126
diff --git a/common/token_utils.py b/common/token_utils.pyindex 391f0f3b89..a87d329911 100644--- a/common/token_utils.py+++ b/common/token_utils.py@@ -104,8 +104,10 @@ def record_run_token_usage(prompt_tokens: int = 0, completion_tokens: int = 0, tdef usage_from_response(resp) -> dict:"""Extract a {prompt_tokens, completion_tokens, total_tokens} split from an LLM response.- Handles OpenAI/OpenRouter-style ``resp.usage`` objects and dict variants. Missing- fields default to 0; ``total_tokens`` falls back to prompt+completion when absent.+ Handles OpenAI/OpenRouter-style ``resp.usage`` objects and dict variants, including+ the camelCase spelling (``promptTokens``/``completionTokens``/``totalTokens``) that+ some providers such as Moark return. Missing fields default to 0; ``total_tokens``+ falls back to prompt+completion when absent."""out = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}if resp is None:@@ -131,9 +133,9 @@ def usage_from_response(resp) -> dict:return int(v)return 0- out["prompt_tokens"] = _get(usage, "prompt_tokens", "input_tokens")- out["completion_tokens"] = _get(usage, "completion_tokens", "output_tokens")- out["total_tokens"] = _get(usage, "total_tokens")+ out["prompt_tokens"] = _get(usage, "prompt_tokens", "input_tokens", "promptTokens")+ out["completion_tokens"] = _get(usage, "completion_tokens", "output_tokens", "completionTokens")+ out["total_tokens"] = _get(usage, "total_tokens", "totalTokens")if not out["total_tokens"]:out["total_tokens"] = out["prompt_tokens"] + out["completion_tokens"]return outdiff --git a/internal/tokenizer/usage.go b/internal/tokenizer/usage.goindex 891c7fea0c..ec41d973b7 100644--- a/internal/tokenizer/usage.go+++ b/internal/tokenizer/usage.go@@ -137,9 +137,10 @@ func RecordRunTokenUsage(ctx context.Context, promptTokens, completionTokens, to}// UsageFromMap extracts a token usage split from a raw API response map.-// Handles OpenAI/OpenRouter-style resp["usage"] dicts. Missing fields-// default to 0; total_tokens falls back to prompt+completion when absent.-// Returns nil when no usage found.+// Handles OpenAI/OpenRouter-style resp["usage"] dicts, including the camelCase+// spelling (promptTokens/completionTokens/totalTokens) that some providers+// return. Missing fields default to 0; total_tokens falls back to+// prompt+completion when absent. Returns zeros when no usage found.// Mirrors Python's common.token_utils.usage_from_response().func UsageFromMap(raw map[string]interface{}) (promptTokens, completionTokens, totalTokens int) {if raw == nil {@@ -153,9 +154,9 @@ func UsageFromMap(raw map[string]interface{}) (promptTokens, completionTokens, tif !ok {return 0, 0, 0}- pt := getInt(usage, "prompt_tokens", "input_tokens")- ct := getInt(usage, "completion_tokens", "output_tokens")- tt := getInt(usage, "total_tokens")+ pt := getInt(usage, "prompt_tokens", "input_tokens", "promptTokens")+ ct := getInt(usage, "completion_tokens", "output_tokens", "completionTokens")+ tt := getInt(usage, "total_tokens", "totalTokens")if tt == 0 {tt = pt + ct}diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.pyindex f474182005..dc868a2d70 100644--- a/rag/llm/rerank_model.py+++ b/rag/llm/rerank_model.py@@ -27,7 +27,7 @@ import numpy as npimport requestsfrom common.log_utils import log_exception-from common.token_utils import num_tokens_from_string, truncate, total_token_count_from_response+from common.token_utils import num_tokens_from_string, truncate, total_token_count_from_response, usage_from_responsefrom rag.llm.mws_utils import mws_api_url, require_mws_tokenfrom rag.utils.url_utils import append_api_path, ensure_v1@@ -37,6 +37,23 @@ if RERANK_TOKEN_LIMIT_MODE not in {"truncate", "passthrough", "raise_error"}:raise ValueError(f"Invalid RERANK_TOKEN_LIMIT_MODE {RERANK_TOKEN_LIMIT_MODE!r}; expected 'truncate', 'passthrough', or 'raise_error'")+def _describe_rerank_usage(res) -> str:+ """Describe where a rerank response keeps its token usage, for diagnostics.++ Providers disagree on the usage layout (``usage.total_tokens``,+ ``usage.prompt_tokens``, ``meta.tokens``, ``meta.billed_units``, ...), and an+ unmapped shape silently degrades ``token_count`` to 0. Only structural+ fields are rendered here so the query and documents are never logged.+ """+ if not isinstance(res, dict):+ return f"response_type={type(res).__name__} (expected dict)"+ usage = res.get("usage")+ meta = res.get("meta")+ usage_keys = sorted(usage) if isinstance(usage, dict) else usage+ meta_keys = sorted(meta) if isinstance(meta, dict) else meta+ return f"top_level_keys={sorted(res)}, usage={usage_keys}, meta={meta_keys}"++class Base(ABC):def __init__(self, key, model_name, max_token=MAX_RERANK_TOKEN, **kwargs):self.max_token = max_token or MAX_RERANK_TOKEN@@ -57,6 +74,7 @@ class Base(ABC):and must not normalize themselves."""if not query or not texts:+ logging.debug("Rerank %s short-circuits on empty input: query=%s texts=%d, returning 0 tokens", self.__class__.__name__, bool(query), len(texts) if texts else 0)return np.zeros(len(texts) if texts else 0, dtype=float), 0if RERANK_TOKEN_LIMIT_MODE != "passthrough":query_tokens = num_tokens_from_string(query)@@ -131,7 +149,30 @@ class JinaRerank(Base):rank[d["index"]] = d["relevance_score"]except Exception as _e:log_exception(_e, res)- return rank, total_token_count_from_response(res)+ usage = usage_from_response(res)+ token_count = usage["total_tokens"]+ if token_count:+ logging.debug(+ "Rerank %s token usage from response: total_tokens=%d prompt_tokens=%d completion_tokens=%d",+ self.__class__.__name__,+ token_count,+ usage["prompt_tokens"],+ usage["completion_tokens"],+ )+ else:+ # Moark/GiteeAI report usage in camelCase and can still send zeros, so+ # count the query and documents locally instead of reporting a+ # zero-token rerank.+ token_count = num_tokens_from_string(query) + sum(num_tokens_from_string(text) for text in texts)+ logging.debug(+ "Rerank %s response carried no usable token usage; fell back to local count=%d (model=%s endpoint=%s); response shape: %s",+ self.__class__.__name__,+ token_count,+ self.model_name,+ self.base_url,+ _describe_rerank_usage(res),+ )+ return rank, token_countclass GreenPTRerank(JinaRerank):diff --git a/test/unit_test/rag/llm/test_rerank_token_count.py b/test/unit_test/rag/llm/test_rerank_token_count.pynew file mode 100644index 0000000000..f2396cd4f9--- /dev/null+++ b/test/unit_test/rag/llm/test_rerank_token_count.py@@ -0,0 +1,165 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+"""Regression tests for reranker token accounting.++``JinaRerank`` and every provider that inherits it (NovitaAI, GiteeAI,+Jiekou.AI, GreenPT) report ``token_count`` from the HTTP response. The+providers disagree on the usage field names: Jina emits+``usage.total_tokens`` while GiteeAI/Moark emits OpenAI-style+``usage.prompt_tokens``/``completion_tokens`` in camelCase+(``promptTokens``/``completionTokens``/``totalTokens``). Reading the response+through the legacy ``total_token_count_from_response`` helper silently returned+0 for those shapes, so ``_compute_rank`` now goes through the shared+``usage_from_response`` helper and falls back to a local count when the+provider reports nothing usable. These tests pin every supported shape.+"""++import logging+from unittest.mock import MagicMock, patch++import numpy as np+import pytest++from rag.llm.rerank_model import (+ GiteeRerank,+ GreenPTRerank,+ JiekouAIRerank,+ JinaRerank,+ NovitaRerank,+)++pytestmark = pytest.mark.p1+++def _mock_post(payload):+ """Patch ``requests.post`` so ``response.json()`` returns ``payload``."""+ response = MagicMock()+ response.raise_for_status.return_value = None+ response.json.return_value = payload+ return patch("rag.llm.rerank_model.requests.post", return_value=response)+++def _results(*scores):+ return {"results": [{"index": index, "relevance_score": score} for index, score in enumerate(scores)]}+++def _similarity(reranker, payload):+ """Run ``similarity`` against a mocked response, bypassing real tokenization."""+ with _mock_post(payload), patch("rag.llm.rerank_model.num_tokens_from_string", return_value=1):+ return reranker.similarity("query", ["first", "second"])+++def _jina(**kwargs):+ return JinaRerank("key", base_url="http://example.test/rerank", **kwargs)+++# --- Supported usage shapes -------------------------------------------------+++def test_jina_native_total_tokens_is_used():+ rank, tokens = _similarity(_jina(), {**_results(0.9, 0.1), "usage": {"total_tokens": 42}})+ assert tokens == 42+ assert np.allclose(rank, [0.9, 0.1])+++def test_openai_style_prompt_and_completion_tokens_are_summed():+ """GiteeAI/Moark omits ``total_tokens``; the count must still be reported."""+ payload = {**_results(0.9, 0.1), "usage": {"prompt_tokens": 7, "completion_tokens": 3}}+ _, tokens = _similarity(_jina(), payload)+ assert tokens == 10+++def test_openai_style_total_tokens_takes_precedence():+ payload = {**_results(0.5), "usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 12}}+ _, tokens = _similarity(_jina(), payload)+ assert tokens == 12+++def test_anthropic_style_input_and_output_tokens_are_summed():+ payload = {**_results(0.5), "usage": {"input_tokens": 5, "output_tokens": 2}}+ _, tokens = _similarity(_jina(), payload)+ assert tokens == 7+++def test_camel_case_prompt_and_completion_tokens_are_summed():+ """Moark/GiteeAI spell the OpenAI fields in camelCase."""+ payload = {**_results(0.5), "usage": {"promptTokens": 7, "completionTokens": 3}}+ _, tokens = _similarity(_jina(), payload)+ assert tokens == 10+++def test_camel_case_total_tokens_takes_precedence():+ payload = {**_results(0.5), "usage": {"promptTokens": 7, "completionTokens": 3, "totalTokens": 12}}+ _, tokens = _similarity(_jina(), payload)+ assert tokens == 12+++# --- Local fallback when the provider reports nothing usable ----------------+#+# ``_similarity`` patches ``num_tokens_from_string`` to return 1, so the local+# fallback counts the query plus the two documents as 3.+++def test_missing_usage_falls_back_to_local_count():+ _, tokens = _similarity(_jina(), _results(0.5))+ assert tokens == 3+++def test_zero_valued_usage_falls_back_to_local_count():+ """The exact Moark shape: camelCase keys that still carry zeros."""+ payload = {**_results(0.5), "usage": {"totalTokens": 0, "promptTokens": 0}}+ _, tokens = _similarity(_jina(), payload)+ assert tokens == 3+++# --- Diagnostics ------------------------------------------------------------+++def test_unmapped_usage_falls_back_and_logs_response_shape(caplog):+ """An unrecognized usage layout must be traceable without dumping content."""+ payload = {**_results(0.5), "meta": {"billed_units": {"search_units": 1}}}+ with caplog.at_level(logging.DEBUG):+ _, tokens = _similarity(_jina(), payload)+ assert tokens == 3+ assert any("fell back to local count=3" in record.getMessage() for record in caplog.records)+ assert any("meta=['billed_units']" in record.getMessage() for record in caplog.records)+++def test_mapped_usage_logs_resolved_split(caplog):+ payload = {**_results(0.5), "usage": {"prompt_tokens": 7, "completion_tokens": 3}}+ with caplog.at_level(logging.DEBUG):+ _, tokens = _similarity(_jina(), payload)+ assert tokens == 10+ assert any("total_tokens=10 prompt_tokens=7 completion_tokens=3" in record.getMessage() for record in caplog.records)+++# --- Every Jina-family provider inherits the fix ----------------------------+++ ("factory", "kwargs"),+ [+ (JinaRerank, {"base_url": "http://example.test/rerank"}),+ (NovitaRerank, {}),+ (GiteeRerank, {}),+ (JiekouAIRerank, {}),+ (GreenPTRerank, {}),+ ],+)+def test_jina_family_providers_count_openai_style_usage(factory, kwargs):+ payload = {**_results(0.4, 0.6), "usage": {"prompt_tokens": 11, "completion_tokens": 4}}+ _, tokens = _similarity(factory("key", "test-model", **kwargs), payload)+ assert tokens == 15