Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(metrics): merge "unknown"/empty model & engine buckets in star-reviews (#4790)

LearningCircuit9ed0c6da86ee
diff --git a/changelog.d/4790.bugfix.md b/changelog.d/4790.bugfix.md
new file mode 100644
index 0000000000..07d1ad7953
--- /dev/null
+++ b/changelog.d/4790.bugfix.md
@@ -0,0 +1 @@
+**Star Reviews merges the "Unknown" buckets.** The LLM and search-engine breakdowns no longer split unidentified entries across separate bars — an empty string, the lowercase `unknown` sentinel written for an unidentified model, and the `Unknown` shown for ratings with no recorded model/engine now collapse into a single "Unknown" bucket (real model names keep their exact casing).
diff --git a/src/local_deep_research/web/routes/metrics_routes.py b/src/local_deep_research/web/routes/metrics_routes.py
index e2302d2540..7bd54cc0d0 100644
--- a/src/local_deep_research/web/routes/metrics_routes.py
+++ b/src/local_deep_research/web/routes/metrics_routes.py
@@ -1441,14 +1441,31 @@ def api_star_reviews():
# LLM call), so joining TokenUsage directly fans out and multiplies the
# counts/averages. Collapse to distinct (research_id, model) pairs first
# so each rating is counted once per model it actually used.
+ # Normalize the "missing model" sentinels — empty/whitespace and the
+ # lowercase "unknown" fallback the token counter writes — to a single
+ # "Unknown" *inside* the subquery, so DISTINCT dedups on the normalized
+ # value. (Normalizing only at group_by time lets "" and "unknown"
+ # survive DISTINCT as separate rows and double-count one rating into the
+ # merged bucket.) Real model names keep their original casing via else_.
+ normalized_model = case(
+ (
+ func.lower(func.trim(TokenUsage.model_name)).in_(
+ ["", "unknown"]
+ ),
+ "Unknown",
+ ),
+ else_=TokenUsage.model_name,
+ )
research_models = (
session.query(
TokenUsage.research_id.label("research_id"),
- TokenUsage.model_name.label("model_name"),
+ normalized_model.label("model_name"),
)
.distinct()
.subquery()
)
+ # A rating with no token rows has no subquery match -> NULL via the
+ # outerjoin -> "Unknown".
model_label = func.coalesce(research_models.c.model_name, "Unknown")
llm_ratings_query = (
@@ -1484,16 +1501,22 @@ def api_star_reviews():
# Ratings by search engine. Same one-to-many fan-out concern as the LLM
# query — collapse to distinct (research_id, search_engine) pairs first.
- # search_engine_selected is NULL on non-search LLM-call rows; exclude
- # those so a research that used a real engine isn't ALSO attributed to
- # the "Unknown" bucket (a research with no engine still falls through
- # the outerjoin to "Unknown" exactly once).
+ # search_engine_selected is NULL on non-search LLM-call rows (and is
+ # occasionally an empty/whitespace string); exclude both so a research
+ # that used a real engine isn't ALSO attributed to the "Unknown" bucket.
+ # A research with no recorded engine still falls through the outerjoin to
+ # "Unknown" exactly once.
research_engines = (
session.query(
TokenUsage.research_id.label("research_id"),
TokenUsage.search_engine_selected.label("search_engine"),
)
- .filter(TokenUsage.search_engine_selected.isnot(None))
+ .filter(
+ func.trim(
+ func.coalesce(TokenUsage.search_engine_selected, "")
+ )
+ != ""
+ )
.distinct()
.subquery()
)
diff --git a/tests/web/routes/test_metrics_routes.py b/tests/web/routes/test_metrics_routes.py
index 03de9b029c..4bdc550d9f 100644
--- a/tests/web/routes/test_metrics_routes.py
+++ b/tests/web/routes/test_metrics_routes.py
@@ -1722,6 +1722,79 @@ class TestApiStarReviewsFanOut:
# the NULL-engine row must not create an 'Unknown' bucket here
assert "Unknown" not in buckets
+ def test_llm_unknown_buckets_merge(self, authenticated_client):
+ """All "missing model" sentinels collapse into a single 'Unknown' bucket
+ — empty string, lowercase 'unknown', uppercase 'UNKNOWN', whitespace, and
+ no-token (NULL) — counted once each, while real model names keep casing.
+
+ r1 carries BOTH '' and 'unknown' token rows: normalization must happen
+ inside the DISTINCT subquery so its single rating is counted ONCE, not
+ once per sentinel variant (the merged bucket must total 4, not 5).
+ """
+ now = datetime.now(UTC)
+ rows = [
+ # r1: two distinct sentinels on ONE research -> must count once
+ ResearchRating(research_id="r1", rating=5, created_at=now),
+ _token_usage("r1", model_name="", ts=now),
+ _token_usage("r1", model_name="unknown", ts=now),
+ # r2: uppercase sentinel (exercises lower())
+ ResearchRating(research_id="r2", rating=4, created_at=now),
+ _token_usage("r2", model_name="UNKNOWN", ts=now),
+ # r3: whitespace-only (exercises trim())
+ ResearchRating(research_id="r3", rating=3, created_at=now),
+ _token_usage("r3", model_name=" ", ts=now),
+ # r4: no token rows -> NULL model via the outerjoin
+ ResearchRating(research_id="r4", rating=2, created_at=now),
+ # r5: a real model -> its own bucket, casing preserved
+ ResearchRating(research_id="r5", rating=5, created_at=now),
+ _token_usage("r5", model_name="GPT-4", ts=now),
+ ]
+ with _seeded_metrics_db(*rows):
+ response = authenticated_client.get(
+ f"{METRICS_PREFIX}/api/star-reviews?period=all"
+ )
+ assert response.status_code == 200, response.get_json()
+ buckets = {r["model"]: r for r in response.get_json()["llm_ratings"]}
+ # no sentinel variant survives as its own bucket
+ for sentinel in ("", "unknown", "UNKNOWN", " "):
+ assert sentinel not in buckets
+ # r1..r4 merge into Unknown, each counted once (r1's two sentinels = 1)
+ assert buckets["Unknown"]["rating_count"] == 4
+ assert buckets["Unknown"]["avg_rating"] == 3.5 # (5+4+3+2)/4
+ assert buckets["Unknown"]["positive_ratings"] == 2 # ratings 5 and 4
+ # real model keeps its original casing and stays separate
+ assert "GPT-4" in buckets
+ assert buckets["GPT-4"]["rating_count"] == 1
+
+ def test_search_engine_empty_string_merges_into_unknown(
+ self, authenticated_client
+ ):
+ """An empty-string search_engine_selected is treated as 'no engine' and
+ merges into 'Unknown' rather than forming a separate '' bucket."""
+ now = datetime.now(UTC)
+ rows = [
+ ResearchRating(research_id="r1", rating=4, created_at=now),
+ _token_usage("r1", ts=now, search_engine=""),
+ ResearchRating(research_id="r2", rating=2, created_at=now),
+ _token_usage("r2", ts=now, search_engine=None),
+ # whitespace-only engine (exercises trim()) -> also Unknown
+ ResearchRating(research_id="r3", rating=3, created_at=now),
+ _token_usage("r3", ts=now, search_engine=" "),
+ ]
+ with _seeded_metrics_db(*rows):
+ response = authenticated_client.get(
+ f"{METRICS_PREFIX}/api/star-reviews?period=all"
+ )
+ assert response.status_code == 200, response.get_json()
+ buckets = {
+ r["search_engine"]: r
+ for r in response.get_json()["search_engine_ratings"]
+ }
+ assert "" not in buckets
+ assert " " not in buckets
+ # r1 (''), r2 (NULL), r3 (whitespace) all land in 'Unknown'
+ assert buckets["Unknown"]["rating_count"] == 3
+
class TestApiStarReviewsQualityDimensions:
"""Quality radar must not be suppressed when only some dimensions are set."""