Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: num token usage (#19495)

Lynn97029b08f126
diff --git a/common/token_utils.py b/common/token_utils.py
index 391f0f3b89..a87d329911 100644
--- a/common/token_utils.py
+++ b/common/token_utils.py
@@ -104,8 +104,10 @@ def record_run_token_usage(prompt_tokens: int = 0, completion_tokens: int = 0, t
def usage_from_response(resp) -> dict:
"""Extract a {prompt_tokens, completion_tokens, total_tokens} split from an LLM response.
- Handles OpenAI/OpenRouter-style ``resp.usage`` objects and dict variants. Missing
- fields default to 0; ``total_tokens`` falls back to prompt+completion when absent.
+ Handles OpenAI/OpenRouter-style ``resp.usage`` objects and dict variants, including
+ the camelCase spelling (``promptTokens``/``completionTokens``/``totalTokens``) that
+ some providers such as Moark return. Missing fields default to 0; ``total_tokens``
+ falls back to prompt+completion when absent.
"""
out = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
if resp is None:
@@ -131,9 +133,9 @@ def usage_from_response(resp) -> dict:
return int(v)
return 0
- out["prompt_tokens"] = _get(usage, "prompt_tokens", "input_tokens")
- out["completion_tokens"] = _get(usage, "completion_tokens", "output_tokens")
- out["total_tokens"] = _get(usage, "total_tokens")
+ out["prompt_tokens"] = _get(usage, "prompt_tokens", "input_tokens", "promptTokens")
+ out["completion_tokens"] = _get(usage, "completion_tokens", "output_tokens", "completionTokens")
+ out["total_tokens"] = _get(usage, "total_tokens", "totalTokens")
if not out["total_tokens"]:
out["total_tokens"] = out["prompt_tokens"] + out["completion_tokens"]
return out
diff --git a/internal/tokenizer/usage.go b/internal/tokenizer/usage.go
index 891c7fea0c..ec41d973b7 100644
--- a/internal/tokenizer/usage.go
+++ b/internal/tokenizer/usage.go
@@ -137,9 +137,10 @@ func RecordRunTokenUsage(ctx context.Context, promptTokens, completionTokens, to
}
// UsageFromMap extracts a token usage split from a raw API response map.
-// Handles OpenAI/OpenRouter-style resp["usage"] dicts. Missing fields
-// default to 0; total_tokens falls back to prompt+completion when absent.
-// Returns nil when no usage found.
+// Handles OpenAI/OpenRouter-style resp["usage"] dicts, including the camelCase
+// spelling (promptTokens/completionTokens/totalTokens) that some providers
+// return. Missing fields default to 0; total_tokens falls back to
+// prompt+completion when absent. Returns zeros when no usage found.
// Mirrors Python's common.token_utils.usage_from_response().
func UsageFromMap(raw map[string]interface{}) (promptTokens, completionTokens, totalTokens int) {
if raw == nil {
@@ -153,9 +154,9 @@ func UsageFromMap(raw map[string]interface{}) (promptTokens, completionTokens, t
if !ok {
return 0, 0, 0
}
- pt := getInt(usage, "prompt_tokens", "input_tokens")
- ct := getInt(usage, "completion_tokens", "output_tokens")
- tt := getInt(usage, "total_tokens")
+ pt := getInt(usage, "prompt_tokens", "input_tokens", "promptTokens")
+ ct := getInt(usage, "completion_tokens", "output_tokens", "completionTokens")
+ tt := getInt(usage, "total_tokens", "totalTokens")
if tt == 0 {
tt = pt + ct
}
diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py
index f474182005..dc868a2d70 100644
--- a/rag/llm/rerank_model.py
+++ b/rag/llm/rerank_model.py
@@ -27,7 +27,7 @@ import numpy as np
import requests
from common.log_utils import log_exception
-from common.token_utils import num_tokens_from_string, truncate, total_token_count_from_response
+from common.token_utils import num_tokens_from_string, truncate, total_token_count_from_response, usage_from_response
from rag.llm.mws_utils import mws_api_url, require_mws_token
from rag.utils.url_utils import append_api_path, ensure_v1
@@ -37,6 +37,23 @@ if RERANK_TOKEN_LIMIT_MODE not in {"truncate", "passthrough", "raise_error"}:
raise ValueError(f"Invalid RERANK_TOKEN_LIMIT_MODE {RERANK_TOKEN_LIMIT_MODE!r}; expected 'truncate', 'passthrough', or 'raise_error'")
+def _describe_rerank_usage(res) -> str:
+ """Describe where a rerank response keeps its token usage, for diagnostics.
+
+ Providers disagree on the usage layout (``usage.total_tokens``,
+ ``usage.prompt_tokens``, ``meta.tokens``, ``meta.billed_units``, ...), and an
+ unmapped shape silently degrades ``token_count`` to 0. Only structural
+ fields are rendered here so the query and documents are never logged.
+ """
+ if not isinstance(res, dict):
+ return f"response_type={type(res).__name__} (expected dict)"
+ usage = res.get("usage")
+ meta = res.get("meta")
+ usage_keys = sorted(usage) if isinstance(usage, dict) else usage
+ meta_keys = sorted(meta) if isinstance(meta, dict) else meta
+ return f"top_level_keys={sorted(res)}, usage={usage_keys}, meta={meta_keys}"
+
+
class Base(ABC):
def __init__(self, key, model_name, max_token=MAX_RERANK_TOKEN, **kwargs):
self.max_token = max_token or MAX_RERANK_TOKEN
@@ -57,6 +74,7 @@ class Base(ABC):
and must not normalize themselves.
"""
if not query or not texts:
+ logging.debug("Rerank %s short-circuits on empty input: query=%s texts=%d, returning 0 tokens", self.__class__.__name__, bool(query), len(texts) if texts else 0)
return np.zeros(len(texts) if texts else 0, dtype=float), 0
if RERANK_TOKEN_LIMIT_MODE != "passthrough":
query_tokens = num_tokens_from_string(query)
@@ -131,7 +149,30 @@ class JinaRerank(Base):
rank[d["index"]] = d["relevance_score"]
except Exception as _e:
log_exception(_e, res)
- return rank, total_token_count_from_response(res)
+ usage = usage_from_response(res)
+ token_count = usage["total_tokens"]
+ if token_count:
+ logging.debug(
+ "Rerank %s token usage from response: total_tokens=%d prompt_tokens=%d completion_tokens=%d",
+ self.__class__.__name__,
+ token_count,
+ usage["prompt_tokens"],
+ usage["completion_tokens"],
+ )
+ else:
+ # Moark/GiteeAI report usage in camelCase and can still send zeros, so
+ # count the query and documents locally instead of reporting a
+ # zero-token rerank.
+ token_count = num_tokens_from_string(query) + sum(num_tokens_from_string(text) for text in texts)
+ logging.debug(
+ "Rerank %s response carried no usable token usage; fell back to local count=%d (model=%s endpoint=%s); response shape: %s",
+ self.__class__.__name__,
+ token_count,
+ self.model_name,
+ self.base_url,
+ _describe_rerank_usage(res),
+ )
+ return rank, token_count
class GreenPTRerank(JinaRerank):
diff --git a/test/unit_test/rag/llm/test_rerank_token_count.py b/test/unit_test/rag/llm/test_rerank_token_count.py
new file mode 100644
index 0000000000..f2396cd4f9
--- /dev/null
+++ b/test/unit_test/rag/llm/test_rerank_token_count.py
@@ -0,0 +1,165 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Regression tests for reranker token accounting.
+
+``JinaRerank`` and every provider that inherits it (NovitaAI, GiteeAI,
+Jiekou.AI, GreenPT) report ``token_count`` from the HTTP response. The
+providers disagree on the usage field names: Jina emits
+``usage.total_tokens`` while GiteeAI/Moark emits OpenAI-style
+``usage.prompt_tokens``/``completion_tokens`` in camelCase
+(``promptTokens``/``completionTokens``/``totalTokens``). Reading the response
+through the legacy ``total_token_count_from_response`` helper silently returned
+0 for those shapes, so ``_compute_rank`` now goes through the shared
+``usage_from_response`` helper and falls back to a local count when the
+provider reports nothing usable. These tests pin every supported shape.
+"""
+
+import logging
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pytest
+
+from rag.llm.rerank_model import (
+ GiteeRerank,
+ GreenPTRerank,
+ JiekouAIRerank,
+ JinaRerank,
+ NovitaRerank,
+)
+
+pytestmark = pytest.mark.p1
+
+
+def _mock_post(payload):
+ """Patch ``requests.post`` so ``response.json()`` returns ``payload``."""
+ response = MagicMock()
+ response.raise_for_status.return_value = None
+ response.json.return_value = payload
+ return patch("rag.llm.rerank_model.requests.post", return_value=response)
+
+
+def _results(*scores):
+ return {"results": [{"index": index, "relevance_score": score} for index, score in enumerate(scores)]}
+
+
+def _similarity(reranker, payload):
+ """Run ``similarity`` against a mocked response, bypassing real tokenization."""
+ with _mock_post(payload), patch("rag.llm.rerank_model.num_tokens_from_string", return_value=1):
+ return reranker.similarity("query", ["first", "second"])
+
+
+def _jina(**kwargs):
+ return JinaRerank("key", base_url="http://example.test/rerank", **kwargs)
+
+
+# --- Supported usage shapes -------------------------------------------------
+
+
+def test_jina_native_total_tokens_is_used():
+ rank, tokens = _similarity(_jina(), {**_results(0.9, 0.1), "usage": {"total_tokens": 42}})
+ assert tokens == 42
+ assert np.allclose(rank, [0.9, 0.1])
+
+
+def test_openai_style_prompt_and_completion_tokens_are_summed():
+ """GiteeAI/Moark omits ``total_tokens``; the count must still be reported."""
+ payload = {**_results(0.9, 0.1), "usage": {"prompt_tokens": 7, "completion_tokens": 3}}
+ _, tokens = _similarity(_jina(), payload)
+ assert tokens == 10
+
+
+def test_openai_style_total_tokens_takes_precedence():
+ payload = {**_results(0.5), "usage": {"prompt_tokens": 7, "completion_tokens": 3, "total_tokens": 12}}
+ _, tokens = _similarity(_jina(), payload)
+ assert tokens == 12
+
+
+def test_anthropic_style_input_and_output_tokens_are_summed():
+ payload = {**_results(0.5), "usage": {"input_tokens": 5, "output_tokens": 2}}
+ _, tokens = _similarity(_jina(), payload)
+ assert tokens == 7
+
+
+def test_camel_case_prompt_and_completion_tokens_are_summed():
+ """Moark/GiteeAI spell the OpenAI fields in camelCase."""
+ payload = {**_results(0.5), "usage": {"promptTokens": 7, "completionTokens": 3}}
+ _, tokens = _similarity(_jina(), payload)
+ assert tokens == 10
+
+
+def test_camel_case_total_tokens_takes_precedence():
+ payload = {**_results(0.5), "usage": {"promptTokens": 7, "completionTokens": 3, "totalTokens": 12}}
+ _, tokens = _similarity(_jina(), payload)
+ assert tokens == 12
+
+
+# --- Local fallback when the provider reports nothing usable ----------------
+#
+# ``_similarity`` patches ``num_tokens_from_string`` to return 1, so the local
+# fallback counts the query plus the two documents as 3.
+
+
+def test_missing_usage_falls_back_to_local_count():
+ _, tokens = _similarity(_jina(), _results(0.5))
+ assert tokens == 3
+
+
+def test_zero_valued_usage_falls_back_to_local_count():
+ """The exact Moark shape: camelCase keys that still carry zeros."""
+ payload = {**_results(0.5), "usage": {"totalTokens": 0, "promptTokens": 0}}
+ _, tokens = _similarity(_jina(), payload)
+ assert tokens == 3
+
+
+# --- Diagnostics ------------------------------------------------------------
+
+
+def test_unmapped_usage_falls_back_and_logs_response_shape(caplog):
+ """An unrecognized usage layout must be traceable without dumping content."""
+ payload = {**_results(0.5), "meta": {"billed_units": {"search_units": 1}}}
+ with caplog.at_level(logging.DEBUG):
+ _, tokens = _similarity(_jina(), payload)
+ assert tokens == 3
+ assert any("fell back to local count=3" in record.getMessage() for record in caplog.records)
+ assert any("meta=['billed_units']" in record.getMessage() for record in caplog.records)
+
+
+def test_mapped_usage_logs_resolved_split(caplog):
+ payload = {**_results(0.5), "usage": {"prompt_tokens": 7, "completion_tokens": 3}}
+ with caplog.at_level(logging.DEBUG):
+ _, tokens = _similarity(_jina(), payload)
+ assert tokens == 10
+ assert any("total_tokens=10 prompt_tokens=7 completion_tokens=3" in record.getMessage() for record in caplog.records)
+
+
+# --- Every Jina-family provider inherits the fix ----------------------------
+
+
+ ("factory", "kwargs"),
+ [
+ (JinaRerank, {"base_url": "http://example.test/rerank"}),
+ (NovitaRerank, {}),
+ (GiteeRerank, {}),
+ (JiekouAIRerank, {}),
+ (GreenPTRerank, {}),
+ ],
+)
+def test_jina_family_providers_count_openai_style_usage(factory, kwargs):
+ payload = {**_results(0.4, 0.6), "usage": {"prompt_tokens": 11, "completion_tokens": 4}}
+ _, tokens = _similarity(factory("key", "test-model", **kwargs), payload)
+ assert tokens == 15