~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(metrics): merge "unknown"/empty model & engine buckets in star-reviews (#4790)
LearningCircuit9ed0c6da86ee
diff --git a/changelog.d/4790.bugfix.md b/changelog.d/4790.bugfix.mdnew file mode 100644index 0000000000..07d1ad7953--- /dev/null+++ b/changelog.d/4790.bugfix.md@@ -0,0 +1 @@+**Star Reviews merges the "Unknown" buckets.** The LLM and search-engine breakdowns no longer split unidentified entries across separate bars — an empty string, the lowercase `unknown` sentinel written for an unidentified model, and the `Unknown` shown for ratings with no recorded model/engine now collapse into a single "Unknown" bucket (real model names keep their exact casing).diff --git a/src/local_deep_research/web/routes/metrics_routes.py b/src/local_deep_research/web/routes/metrics_routes.pyindex e2302d2540..7bd54cc0d0 100644--- a/src/local_deep_research/web/routes/metrics_routes.py+++ b/src/local_deep_research/web/routes/metrics_routes.py@@ -1441,14 +1441,31 @@ def api_star_reviews():# LLM call), so joining TokenUsage directly fans out and multiplies the# counts/averages. Collapse to distinct (research_id, model) pairs first# so each rating is counted once per model it actually used.+ # Normalize the "missing model" sentinels — empty/whitespace and the+ # lowercase "unknown" fallback the token counter writes — to a single+ # "Unknown" *inside* the subquery, so DISTINCT dedups on the normalized+ # value. (Normalizing only at group_by time lets "" and "unknown"+ # survive DISTINCT as separate rows and double-count one rating into the+ # merged bucket.) Real model names keep their original casing via else_.+ normalized_model = case(+ (+ func.lower(func.trim(TokenUsage.model_name)).in_(+ ["", "unknown"]+ ),+ "Unknown",+ ),+ else_=TokenUsage.model_name,+ )research_models = (session.query(TokenUsage.research_id.label("research_id"),- TokenUsage.model_name.label("model_name"),+ normalized_model.label("model_name"),).distinct().subquery())+ # A rating with no token rows has no subquery match -> NULL via the+ # outerjoin -> "Unknown".model_label = func.coalesce(research_models.c.model_name, "Unknown")llm_ratings_query = (@@ -1484,16 +1501,22 @@ def api_star_reviews():# Ratings by search engine. Same one-to-many fan-out concern as the LLM# query — collapse to distinct (research_id, search_engine) pairs first.- # search_engine_selected is NULL on non-search LLM-call rows; exclude- # those so a research that used a real engine isn't ALSO attributed to- # the "Unknown" bucket (a research with no engine still falls through- # the outerjoin to "Unknown" exactly once).+ # search_engine_selected is NULL on non-search LLM-call rows (and is+ # occasionally an empty/whitespace string); exclude both so a research+ # that used a real engine isn't ALSO attributed to the "Unknown" bucket.+ # A research with no recorded engine still falls through the outerjoin to+ # "Unknown" exactly once.research_engines = (session.query(TokenUsage.research_id.label("research_id"),TokenUsage.search_engine_selected.label("search_engine"),)- .filter(TokenUsage.search_engine_selected.isnot(None))+ .filter(+ func.trim(+ func.coalesce(TokenUsage.search_engine_selected, "")+ )+ != ""+ ).distinct().subquery())diff --git a/tests/web/routes/test_metrics_routes.py b/tests/web/routes/test_metrics_routes.pyindex 03de9b029c..4bdc550d9f 100644--- a/tests/web/routes/test_metrics_routes.py+++ b/tests/web/routes/test_metrics_routes.py@@ -1722,6 +1722,79 @@ class TestApiStarReviewsFanOut:# the NULL-engine row must not create an 'Unknown' bucket hereassert "Unknown" not in buckets+ def test_llm_unknown_buckets_merge(self, authenticated_client):+ """All "missing model" sentinels collapse into a single 'Unknown' bucket+ — empty string, lowercase 'unknown', uppercase 'UNKNOWN', whitespace, and+ no-token (NULL) — counted once each, while real model names keep casing.++ r1 carries BOTH '' and 'unknown' token rows: normalization must happen+ inside the DISTINCT subquery so its single rating is counted ONCE, not+ once per sentinel variant (the merged bucket must total 4, not 5).+ """+ now = datetime.now(UTC)+ rows = [+ # r1: two distinct sentinels on ONE research -> must count once+ ResearchRating(research_id="r1", rating=5, created_at=now),+ _token_usage("r1", model_name="", ts=now),+ _token_usage("r1", model_name="unknown", ts=now),+ # r2: uppercase sentinel (exercises lower())+ ResearchRating(research_id="r2", rating=4, created_at=now),+ _token_usage("r2", model_name="UNKNOWN", ts=now),+ # r3: whitespace-only (exercises trim())+ ResearchRating(research_id="r3", rating=3, created_at=now),+ _token_usage("r3", model_name=" ", ts=now),+ # r4: no token rows -> NULL model via the outerjoin+ ResearchRating(research_id="r4", rating=2, created_at=now),+ # r5: a real model -> its own bucket, casing preserved+ ResearchRating(research_id="r5", rating=5, created_at=now),+ _token_usage("r5", model_name="GPT-4", ts=now),+ ]+ with _seeded_metrics_db(*rows):+ response = authenticated_client.get(+ f"{METRICS_PREFIX}/api/star-reviews?period=all"+ )+ assert response.status_code == 200, response.get_json()+ buckets = {r["model"]: r for r in response.get_json()["llm_ratings"]}+ # no sentinel variant survives as its own bucket+ for sentinel in ("", "unknown", "UNKNOWN", " "):+ assert sentinel not in buckets+ # r1..r4 merge into Unknown, each counted once (r1's two sentinels = 1)+ assert buckets["Unknown"]["rating_count"] == 4+ assert buckets["Unknown"]["avg_rating"] == 3.5 # (5+4+3+2)/4+ assert buckets["Unknown"]["positive_ratings"] == 2 # ratings 5 and 4+ # real model keeps its original casing and stays separate+ assert "GPT-4" in buckets+ assert buckets["GPT-4"]["rating_count"] == 1++ def test_search_engine_empty_string_merges_into_unknown(+ self, authenticated_client+ ):+ """An empty-string search_engine_selected is treated as 'no engine' and+ merges into 'Unknown' rather than forming a separate '' bucket."""+ now = datetime.now(UTC)+ rows = [+ ResearchRating(research_id="r1", rating=4, created_at=now),+ _token_usage("r1", ts=now, search_engine=""),+ ResearchRating(research_id="r2", rating=2, created_at=now),+ _token_usage("r2", ts=now, search_engine=None),+ # whitespace-only engine (exercises trim()) -> also Unknown+ ResearchRating(research_id="r3", rating=3, created_at=now),+ _token_usage("r3", ts=now, search_engine=" "),+ ]+ with _seeded_metrics_db(*rows):+ response = authenticated_client.get(+ f"{METRICS_PREFIX}/api/star-reviews?period=all"+ )+ assert response.status_code == 200, response.get_json()+ buckets = {+ r["search_engine"]: r+ for r in response.get_json()["search_engine_ratings"]+ }+ assert "" not in buckets+ assert " " not in buckets+ # r1 (''), r2 (NULL), r3 (whitespace) all land in 'Unknown'+ assert buckets["Unknown"]["rating_count"] == 3+class TestApiStarReviewsQualityDimensions:"""Quality radar must not be suppressed when only some dimensions are set."""