Skip to content

Personal AI, On Personal Devices

Python

feat: query complexity analyzer for local/cloud routing (#102)

Jana Bergant555e982f5196
diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx
index a0a0cb2e..c395e71c 100644
--- a/frontend/src/components/Chat/InputArea.tsx
+++ b/frontend/src/components/Chat/InputArea.tsx
@@ -18,6 +18,8 @@ export function InputArea() {
const streamState = useAppStore((s) => s.streamState);
const messages = useAppStore((s) => s.messages);
const speechEnabled = useAppStore((s) => s.settings.speechEnabled);
+ const maxTokens = useAppStore((s) => s.settings.maxTokens);
+ const temperature = useAppStore((s) => s.settings.temperature);
const createConversation = useAppStore((s) => s.createConversation);
const addMessage = useAppStore((s) => s.addMessage);
const updateLastAssistant = useAppStore((s) => s.updateLastAssistant);
@@ -127,6 +129,7 @@ export function InputArea() {
let accumulatedContent = '';
let usage: TokenUsage | undefined;
+ let complexity: { score: number; tier: string; suggested_max_tokens: number } | undefined;
const toolCalls: ToolCallInfo[] = [];
let lastFlush = 0;
let ttftMs: number | undefined;
@@ -147,7 +150,7 @@ export function InputArea() {
try {
for await (const sseEvent of streamChat(
- { model: selectedModel, messages: apiMessages, stream: true },
+ { model: selectedModel, messages: apiMessages, stream: true, temperature, max_tokens: maxTokens },
controller.signal,
)) {
const eventName = sseEvent.event;
@@ -202,6 +205,7 @@ export function InputArea() {
const data = JSON.parse(sseEvent.data);
const delta = data.choices?.[0]?.delta;
if (data.usage) usage = data.usage;
+ if (data.complexity) complexity = data.complexity;
if (delta?.content) {
if (!ttftMs) ttftMs = Date.now() - startTime;
accumulatedContent += delta.content;
@@ -248,6 +252,9 @@ export function InputArea() {
tokens_per_sec: usage?.completion_tokens
? usage.completion_tokens / (totalMs / 1000)
: undefined,
+ complexity_score: complexity?.score,
+ complexity_tier: complexity?.tier,
+ suggested_max_tokens: complexity?.suggested_max_tokens,
};
updateLastAssistant(
convId,
diff --git a/frontend/src/components/Chat/XRayFooter.tsx b/frontend/src/components/Chat/XRayFooter.tsx
index 4aad1e58..b13126dc 100644
--- a/frontend/src/components/Chat/XRayFooter.tsx
+++ b/frontend/src/components/Chat/XRayFooter.tsx
@@ -18,6 +18,7 @@ export function XRayFooter({ usage, telemetry }: Props) {
const parts: string[] = [];
if (telemetry?.engine) parts.push(telemetry.engine);
if (telemetry?.model_id) parts.push(telemetry.model_id);
+ if (telemetry?.complexity_tier) parts.push(telemetry.complexity_tier);
if (telemetry?.total_ms) parts.push(formatMs(telemetry.total_ms));
if (usage && (usage.prompt_tokens || usage.completion_tokens)) {
parts.push(`${usage.prompt_tokens} input tokens`);
@@ -50,6 +51,15 @@ export function XRayFooter({ usage, telemetry }: Props) {
}
rows.push({ label: 'Tokens', value: tokenParts.join(' \u00B7 ') });
}
+ if (telemetry?.complexity_tier) {
+ rows.push({
+ label: 'Complexity',
+ value: `${telemetry.complexity_tier} (${telemetry.complexity_score?.toFixed(2)})`,
+ });
+ }
+ if (telemetry?.suggested_max_tokens) {
+ rows.push({ label: 'Token budget', value: `${telemetry.suggested_max_tokens}` });
+ }
if (telemetry?.tokens_per_sec) {
rows.push({ label: 'Speed', value: `${Math.round(telemetry.tokens_per_sec)} tok/s` });
}
diff --git a/frontend/src/lib/sse.ts b/frontend/src/lib/sse.ts
index addd10b9..d00fae92 100644
--- a/frontend/src/lib/sse.ts
+++ b/frontend/src/lib/sse.ts
@@ -5,6 +5,8 @@ export interface ChatRequest {
model: string;
messages: Array<{ role: string; content: string }>;
stream: true;
+ temperature?: number;
+ max_tokens?: number;
}
export async function* streamChat(
diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts
index fa186d32..a01205b3 100644
--- a/frontend/src/types/index.ts
+++ b/frontend/src/types/index.ts
@@ -56,6 +56,9 @@ export interface MessageTelemetry {
tokens_per_sec?: number;
ttft_ms?: number;
total_ms?: number;
+ complexity_score?: number;
+ complexity_tier?: string;
+ suggested_max_tokens?: number;
}
export interface ChatMessage {
diff --git a/src/openjarvis/cli/ask.py b/src/openjarvis/cli/ask.py
index 85455506..43014909 100644
--- a/src/openjarvis/cli/ask.py
+++ b/src/openjarvis/cli/ask.py
@@ -165,6 +165,7 @@ def _print_profile(
engine_name: str,
model_name: str,
console: Console,
+ complexity_result=None,
) -> None:
"""Print an inference telemetry profile table from EventBus history."""
# Collect all INFERENCE_END events (agents may fire multiple)
@@ -227,6 +228,12 @@ def _print_profile(
def _row(label: str, val: str) -> None:
table.add_row(label, val)
+ if complexity_result is not None:
+ _row("Complexity score", f"{complexity_result.score:.3f}")
+ _row("Complexity tier", complexity_result.tier)
+ _row("Suggested max tokens", str(complexity_result.suggested_max_tokens))
+ _row("", "") # separator
+
_row("Wall time", f"{wall_seconds:.3f} s")
_row("Inference calls", str(total_calls))
_row("Total latency", f"{total_latency:.3f} s")
@@ -324,12 +331,30 @@ def ask(
# Load config
config = load_config()
+ # Track whether the user explicitly set --max-tokens
+ user_set_max_tokens = max_tokens is not None
+
# Fall back to config values for generation params
if temperature is None:
temperature = config.intelligence.temperature
if max_tokens is None:
max_tokens = config.intelligence.max_tokens
+ # Run complexity analysis on the query
+ from openjarvis.learning.routing.complexity import (
+ ComplexityResult,
+ adjust_tokens_for_model,
+ score_complexity,
+ )
+
+ complexity_result: ComplexityResult = score_complexity(query_text)
+ logger.debug(
+ "Complexity analysis: score=%.3f tier=%s suggested_max_tokens=%d",
+ complexity_result.score,
+ complexity_result.tier,
+ complexity_result.suggested_max_tokens,
+ )
+
# Set up telemetry
bus = EventBus(record_history=True)
telem_store: TelemetryStore | None = None
@@ -397,6 +422,19 @@ def ask(
console.print("[red]No model available on engine.[/red]")
sys.exit(1)
+ # Apply complexity-suggested token budget when user didn't override.
+ # Use at least the config default so we never reduce tokens below what
+ # the user would have gotten without the analyzer.
+ if not user_set_max_tokens:
+ suggested = adjust_tokens_for_model(
+ complexity_result.suggested_max_tokens, model_name,
+ )
+ max_tokens = max(suggested, config.intelligence.max_tokens)
+ logger.debug(
+ "Using complexity-suggested max_tokens=%d (model=%s)",
+ max_tokens, model_name,
+ )
+
# Agent mode
if agent_name is not None:
parsed_tools = resolve_tool_names(
@@ -435,6 +473,7 @@ def ask(
_print_profile(
bus, time.monotonic() - wall_start,
engine_name, model_name, console,
+ complexity_result=complexity_result,
)
if telem_store is not None:
@@ -494,6 +533,7 @@ def ask(
_print_profile(
bus, time.monotonic() - wall_start,
engine_name, model_name, console,
+ complexity_result=complexity_result,
)
# Cleanup
diff --git a/src/openjarvis/core/types.py b/src/openjarvis/core/types.py
index 1d354ccf..a8629752 100644
--- a/src/openjarvis/core/types.py
+++ b/src/openjarvis/core/types.py
@@ -235,8 +235,11 @@ class RoutingContext:
query_length: int = 0
has_code: bool = False
has_math: bool = False
+ has_reasoning: bool = False
language: str = "en"
urgency: float = 0.5
+ complexity_score: float = 0.0 # 0.0 (trivial) to 1.0 (very complex)
+ suggested_max_tokens: int = 1024
metadata: Dict[str, Any] = field(default_factory=dict)
diff --git a/src/openjarvis/learning/__init__.py b/src/openjarvis/learning/__init__.py
index ec1c8315..9623c340 100644
--- a/src/openjarvis/learning/__init__.py
+++ b/src/openjarvis/learning/__init__.py
@@ -13,6 +13,10 @@ from openjarvis.learning.learning_orchestrator import LearningOrchestrator
from openjarvis.learning.optimize.llm_optimizer import LLMOptimizer
from openjarvis.learning.optimize.optimizer import OptimizationEngine
from openjarvis.learning.optimize.store import OptimizationStore
+from openjarvis.learning.routing.complexity import (
+ ComplexityQueryAnalyzer,
+ score_complexity,
+)
from openjarvis.learning.routing.heuristic_reward import HeuristicRewardFunction
from openjarvis.learning.routing.router import (
HeuristicRouter,
@@ -59,6 +63,7 @@ def ensure_registered() -> None:
__all__ = [
"AgentConfigEvolver",
+ "ComplexityQueryAnalyzer",
"HAS_TORCH",
"HeuristicRewardFunction",
"HeuristicRouter",
@@ -75,4 +80,5 @@ __all__ = [
"TrainingDataMiner",
"build_routing_context",
"ensure_registered",
+ "score_complexity",
]
diff --git a/src/openjarvis/learning/routing/complexity.py b/src/openjarvis/learning/routing/complexity.py
new file mode 100644
index 00000000..7ce77137
--- /dev/null
+++ b/src/openjarvis/learning/routing/complexity.py
@@ -0,0 +1,261 @@
+"""Query complexity analyzer — scores queries and suggests token budgets.
+
+Produces a numeric complexity score (0.0–1.0) and a suggested
+``max_tokens`` budget based on query characteristics such as length,
+domain signals (code, math, multi-step reasoning), and whether the
+target model is a thinking model that needs extra headroom.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Optional
+
+from openjarvis.core.types import RoutingContext
+from openjarvis.learning._stubs import QueryAnalyzer
+
+# ---------------------------------------------------------------------------
+# Signal patterns
+# ---------------------------------------------------------------------------
+
+_CODE_PATTERNS = re.compile(
+ r"```|`[^`]+`|\bdef\s|\bclass\s|\bimport\s|\bfunction\s|\bconst\s|\bvar\s|\blet\s|"
+ r"\bif\s*\(|->|=>|\{\s*\}|\bfor\s+\w+\s+in\s|#include|System\.out",
+ re.IGNORECASE,
+)
+_MATH_PATTERNS = re.compile(
+ r"\bsolve\b|\bintegral\b|\bequation\b|\bproof\b|\bderivative\b|\bmatrix\b|"
+ r"\btheorem\b|\bcalculate\b|\bcompute\b|\bsigma\b|\bsum\b|\blimit\b|\bprobability\b",
+ re.IGNORECASE,
+)
+_REASONING_PATTERNS = re.compile(
+ r"\bexplain\b|\banalyze\b|\bcompare\b|\bwhy\b"
+ r"|\bstep[- ]by[- ]step\b|\breason\b|\bthink\b"
+ r"|\bpros\s+and\s+cons\b|\btrade-?\s*offs?\b|\bevaluate\b",
+ re.IGNORECASE,
+)
+_MULTI_STEP_PATTERNS = re.compile(
+ r"\bthen\b.*\bthen\b|\bfirst\b.*\bnext\b|\bstep\s*\d"
+ r"|\b(?:and\s+also|additionally|furthermore)\b"
+ r"|\b\d+\.\s",
+ re.IGNORECASE | re.DOTALL,
+)
+_CREATIVE_PATTERNS = re.compile(
+ r"\bwrite\b.*\b(?:essay|story|article|report|poem)\b"
+ r"|\bgenerate\b.*\b(?:code|script|program)\b"
+ r"|\bcreate\b|\bdesign\b|\bdraft\b|\bcompose\b",
+ re.IGNORECASE,
+)
+
+# Models known to use internal chain-of-thought that consumes output tokens.
+_THINKING_MODEL_PATTERNS = re.compile(
+ r"qwen3\.5|qwq|deepseek-r1|o1-|o3-|o4-", re.IGNORECASE
+)
+
+# ---------------------------------------------------------------------------
+# Token budget tiers
+# ---------------------------------------------------------------------------
+
+_TOKEN_TIERS = {
+ "trivial": 1024, # greetings, yes/no, factoid lookups
+ "simple": 2048, # short answers, definitions
+ "moderate": 4096, # explanations, summaries
+ "complex": 8192, # analysis, code generation, multi-step
+ "very_complex": 16384, # long-form, multi-part reasoning
+}
+
+# Thinking models need extra headroom for internal chain-of-thought.
+_THINKING_TOKEN_MULTIPLIER = 2
+
+
+# ---------------------------------------------------------------------------
+# Complexity scoring
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class ComplexityResult:
+ """Output of the complexity analysis."""
+
+ score: float # 0.0–1.0
+ tier: str
+ suggested_max_tokens: int
+ signals: dict
+
+
+def _count_questions(query: str) -> int:
+ """Count the number of question marks in the query."""
+ return query.count("?")
+
+
+def _count_sub_tasks(query: str) -> int:
+ """Estimate the number of sub-tasks or enumerated items."""
+ numbered = len(re.findall(r"^\s*\d+[.)]\s", query, re.MULTILINE))
+ bulleted = len(re.findall(r"^\s*[-*]\s", query, re.MULTILINE))
+ return numbered + bulleted
+
+
+def score_complexity(query: str) -> ComplexityResult:
+ """Score a query's complexity from 0.0 (trivial) to 1.0 (very complex).
+
+ The score is a weighted combination of independent signals, each
+ contributing a fraction between 0 and 1. Weights reflect how much
+ each signal correlates with the amount of reasoning and output
+ tokens a model will need.
+ """
+ signals: dict = {}
+ score = 0.0
+
+ # --- Length signal (0–0.20) ---
+ length = len(query)
+ if length < 20:
+ length_score = 0.0
+ elif length < 100:
+ length_score = 0.3
+ elif length < 300:
+ length_score = 0.6
+ elif length < 800:
+ length_score = 0.8
+ else:
+ length_score = 1.0
+ signals["length"] = length_score
+ score += 0.20 * length_score
+
+ # --- Domain signals (0–0.25) ---
+ has_code = bool(_CODE_PATTERNS.search(query))
+ has_math = bool(_MATH_PATTERNS.search(query))
+ domain_score = 0.0
+ if has_code:
+ domain_score = max(domain_score, 0.7)
+ if has_math:
+ domain_score = max(domain_score, 0.8)
+ if has_code and has_math:
+ domain_score = 1.0
+ signals["domain"] = domain_score
+ signals["has_code"] = has_code
+ signals["has_math"] = has_math
+ score += 0.25 * domain_score
+
+ # --- Reasoning signal (0–0.25) ---
+ has_reasoning = bool(_REASONING_PATTERNS.search(query))
+ has_multi_step = bool(_MULTI_STEP_PATTERNS.search(query))
+ reasoning_score = 0.0
+ if has_reasoning:
+ reasoning_score = 0.6
+ if has_multi_step:
+ reasoning_score = max(reasoning_score, 0.8)
+ if has_reasoning and has_multi_step:
+ reasoning_score = 1.0
+ signals["reasoning"] = reasoning_score
+ signals["has_reasoning"] = has_reasoning
+ signals["has_multi_step"] = has_multi_step
+ score += 0.25 * reasoning_score
+
+ # --- Question / sub-task count (0–0.15) ---
+ n_questions = _count_questions(query)
+ n_subtasks = _count_sub_tasks(query)
+ multi_part = n_questions + n_subtasks
+ if multi_part <= 1:
+ multi_score = 0.0
+ elif multi_part <= 3:
+ multi_score = 0.5
+ else:
+ multi_score = 1.0
+ signals["multi_part"] = multi_score
+ signals["n_questions"] = n_questions
+ signals["n_subtasks"] = n_subtasks
+ score += 0.15 * multi_score
+
+ # --- Creative / generative signal (0–0.15) ---
+ has_creative = bool(_CREATIVE_PATTERNS.search(query))
+ creative_score = 0.7 if has_creative else 0.0
+ signals["creative"] = creative_score
+ signals["has_creative"] = has_creative
+ score += 0.15 * creative_score
+
+ # Clamp
+ score = max(0.0, min(1.0, score))
+
+ # Map to tier
+ if score < 0.15:
+ tier = "trivial"
+ elif score < 0.30:
+ tier = "simple"
+ elif score < 0.55:
+ tier = "moderate"
+ elif score < 0.80:
+ tier = "complex"
+ else:
+ tier = "very_complex"
+
+ suggested_max_tokens = _TOKEN_TIERS[tier]
+
+ return ComplexityResult(
+ score=round(score, 3),
+ tier=tier,
+ suggested_max_tokens=suggested_max_tokens,
+ signals=signals,
+ )
+
+
+def is_thinking_model(model_name: str) -> bool:
+ """Return True if the model is known to use internal chain-of-thought."""
+ return bool(_THINKING_MODEL_PATTERNS.search(model_name))
+
+
+def adjust_tokens_for_model(
+ suggested: int, model_name: Optional[str] = None
+) -> int:
+ """Multiply the token budget when the target is a thinking model."""
+ if model_name and is_thinking_model(model_name):
+ return suggested * _THINKING_TOKEN_MULTIPLIER
+ return suggested
+
+
+# ---------------------------------------------------------------------------
+# QueryAnalyzer implementation
+# ---------------------------------------------------------------------------
+
+
+class ComplexityQueryAnalyzer(QueryAnalyzer):
+ """Query analyzer that produces a complexity-aware RoutingContext.
+
+ Drop-in replacement for ``DefaultQueryAnalyzer`` — adds
+ ``complexity_score``, ``has_reasoning``, and ``suggested_max_tokens``
+ to the returned ``RoutingContext``.
+ """
+
+ def analyze(
+ self, query: str, **kwargs: object
+ ) -> RoutingContext:
+ urgency = kwargs.get("urgency", 0.5)
+ if not isinstance(urgency, (int, float)):
+ urgency = 0.5
+ model_name = kwargs.get("model")
+ if not isinstance(model_name, str):
+ model_name = None
+
+ result = score_complexity(query)
+ tokens = adjust_tokens_for_model(result.suggested_max_tokens, model_name)
+
+ return RoutingContext(
+ query=query,
+ query_length=len(query),
+ has_code=result.signals.get("has_code", False),
+ has_math=result.signals.get("has_math", False),
+ has_reasoning=result.signals.get("has_reasoning", False),
+ urgency=float(urgency),
+ complexity_score=result.score,
+ suggested_max_tokens=tokens,
+ metadata={"complexity_tier": result.tier, "signals": result.signals},
+ )
+
+
+__all__ = [
+ "ComplexityQueryAnalyzer",
+ "ComplexityResult",
+ "adjust_tokens_for_model",
+ "is_thinking_model",
+ "score_complexity",
+]
diff --git a/src/openjarvis/learning/routing/router.py b/src/openjarvis/learning/routing/router.py
index 3bc4744b..4862c01f 100644
--- a/src/openjarvis/learning/routing/router.py
+++ b/src/openjarvis/learning/routing/router.py
@@ -3,7 +3,6 @@
from __future__ import annotations
import logging
-import re
from typing import List, Optional
from openjarvis.core.registry import ModelRegistry
@@ -12,32 +11,33 @@ from openjarvis.learning._stubs import QueryAnalyzer, RouterPolicy
logger = logging.getLogger(__name__)
-# Detection patterns
-_CODE_PATTERNS = re.compile(
- r"```|`[^`]+`|\bdef\s|\bclass\s|\bimport\s|\bfunction\s|\bconst\s|\bvar\s|\blet\s|"
- r"\bif\s*\(|->|=>|\{\s*\}|\bfor\s+\w+\s+in\s|#include|System\.out",
- re.IGNORECASE,
-)
-_MATH_PATTERNS = re.compile(
- r"\bsolve\b|\bintegral\b|\bequation\b|\bproof\b|\bderivative\b|\bmatrix\b|"
- r"\btheorem\b|\bcalculate\b|\bcompute\b|\bsigma\b|\bsum\b|\blimit\b|\bprobability\b",
- re.IGNORECASE,
-)
-_REASONING_KEYWORDS = re.compile(
- r"\bexplain\b|\banalyze\b|\bcompare\b|\bwhy\b"
- r"|\bstep[- ]by[- ]step\b|\breason\b|\bthink\b",
- re.IGNORECASE,
-)
-
-
-def build_routing_context(query: str, *, urgency: float = 0.5) -> RoutingContext:
- """Populate a ``RoutingContext`` from a raw query string."""
+
+def build_routing_context(
+ query: str, *, urgency: float = 0.5, model: str | None = None
+) -> RoutingContext:
+ """Populate a ``RoutingContext`` from a raw query string.
+
+ When *model* is provided, the suggested token budget is adjusted
+ for thinking models that need extra headroom.
+ """
+ from openjarvis.learning.routing.complexity import (
+ adjust_tokens_for_model,
+ score_complexity,
+ )
+
+ result = score_complexity(query)
+ tokens = adjust_tokens_for_model(result.suggested_max_tokens, model)
+
return RoutingContext(
query=query,
query_length=len(query),
- has_code=bool(_CODE_PATTERNS.search(query)),
- has_math=bool(_MATH_PATTERNS.search(query)),
+ has_code=result.signals.get("has_code", False),
+ has_math=result.signals.get("has_math", False),
+ has_reasoning=result.signals.get("has_reasoning", False),
urgency=urgency,
+ complexity_score=result.score,
+ suggested_max_tokens=tokens,
+ metadata={"complexity_tier": result.tier, "signals": result.signals},
)
@@ -94,8 +94,8 @@ class HeuristicRouter(RouterPolicy):
Rules (applied in order):
1. Code detected → prefer model with "code"/"coder" in name
2. Math detected → prefer larger model
- 3. Short query (<50 chars, no code/math) → prefer smaller/faster model
- 4. Long/complex query (>500 chars OR reasoning keywords) → prefer larger model
+ 3. Low complexity (score < 0.20) → prefer smaller/faster model
+ 4. High complexity (score >= 0.55 OR reasoning keywords) → prefer larger model
5. High urgency (>0.8) → override to smaller model
6. Default fallback → default_model → fallback_model → first available
"""
@@ -139,12 +139,12 @@ class HeuristicRouter(RouterPolicy):
if context.has_math:
return _largest_model(available) or available[0]
- # Rule 3: Short simple query → prefer smaller model
- if context.query_length < 50:
+ # Rule 3: Low complexity → prefer smaller model
+ if context.complexity_score < 0.20:
return _smallest_model(available) or available[0]
- # Rule 4: Long/complex query → prefer larger model
- if context.query_length > 500 or _REASONING_KEYWORDS.search(context.query):
+ # Rule 4: High complexity or reasoning → prefer larger model
+ if context.complexity_score >= 0.55 or context.has_reasoning:
return _largest_model(available) or available[0]
# Rule 6: Default fallback
@@ -162,7 +162,14 @@ class DefaultQueryAnalyzer(QueryAnalyzer):
urgency = kwargs.get("urgency", 0.5)
if not isinstance(urgency, (int, float)):
urgency = 0.5
- return build_routing_context(query, urgency=urgency)
+ model = kwargs.get("model")
+ if not isinstance(model, str):
+ model = None
+ return build_routing_context(query, urgency=urgency, model=model)
-__all__ = ["DefaultQueryAnalyzer", "HeuristicRouter", "build_routing_context"]
+__all__ = [
+ "DefaultQueryAnalyzer",
+ "HeuristicRouter",
+ "build_routing_context",
+]
diff --git a/src/openjarvis/server/models.py b/src/openjarvis/server/models.py
index 45140767..8086f078 100644
--- a/src/openjarvis/server/models.py
+++ b/src/openjarvis/server/models.py
@@ -53,6 +53,12 @@ class Choice(BaseModel):
finish_reason: str = "stop"
+class ComplexityInfo(BaseModel):
+ score: float
+ tier: str
+ suggested_max_tokens: int
+
+
class ChatCompletionResponse(BaseModel):
id: str = Field(default_factory=lambda: f"chatcmpl-{uuid.uuid4().hex[:12]}")
object: str = "chat.completion"
@@ -60,6 +66,7 @@ class ChatCompletionResponse(BaseModel):
model: str = ""
choices: List[Choice] = Field(default_factory=list)
usage: UsageInfo = Field(default_factory=UsageInfo)
+ complexity: Optional[ComplexityInfo] = None
# ---------------------------------------------------------------------------
@@ -110,6 +117,7 @@ __all__ = [
"ChatMessage",
"Choice",
"ChoiceMessage",
+ "ComplexityInfo",
"DeltaMessage",
"ModelListResponse",
"ModelObject",
diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py
index c6625cb4..bf935bef 100644
--- a/src/openjarvis/server/routes.py
+++ b/src/openjarvis/server/routes.py
@@ -16,6 +16,7 @@ from openjarvis.server.models import (
ChatCompletionResponse,
Choice,
ChoiceMessage,
+ ComplexityInfo,
DeltaMessage,
ModelListResponse,
ModelObject,
@@ -94,6 +95,38 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request
"Memory context injection failed", exc_info=True,
)
+ # Run complexity analysis on the last user message
+ complexity_info = None
+ query_text_for_complexity = ""
+ for m in reversed(request_body.messages):
+ if m.role == "user" and m.content:
+ query_text_for_complexity = m.content
+ break
+ if query_text_for_complexity:
+ try:
+ from openjarvis.learning.routing.complexity import (
+ adjust_tokens_for_model,
+ score_complexity,
+ )
+
+ cr = score_complexity(query_text_for_complexity)
+ suggested = adjust_tokens_for_model(
+ cr.suggested_max_tokens, model,
+ )
+ complexity_info = ComplexityInfo(
+ score=cr.score,
+ tier=cr.tier,
+ suggested_max_tokens=suggested,
+ )
+ # Bump max_tokens when complexity suggests more than what
+ # the client requested — never reduce below the request value.
+ if suggested > request_body.max_tokens:
+ request_body.max_tokens = suggested
+ except Exception:
+ logging.getLogger("openjarvis.server").debug(
+ "Complexity analysis failed", exc_info=True,
+ )
+
if request_body.stream:
bus = getattr(request.app.state, "bus", None)
# Use the agent stream bridge only when tools are present (the
@@ -102,18 +135,22 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request
# directly from the engine for true token-by-token output.
if agent is not None and bus is not None and request_body.tools:
return await _handle_agent_stream(agent, bus, model, request_body)
- return await _handle_stream(engine, model, request_body)
+ return await _handle_stream(engine, model, request_body, complexity_info)
# Non-streaming: use agent if available, otherwise direct engine call
if agent is not None:
- return _handle_agent(agent, model, request_body)
+ return _handle_agent(agent, model, request_body, complexity_info)
bus = getattr(request.app.state, "bus", None)
- return _handle_direct(engine, model, request_body, bus=bus)
+ return _handle_direct(
+ engine, model, request_body,
+ bus=bus, complexity_info=complexity_info,
+ )
def _handle_direct(
engine, model: str, req: ChatCompletionRequest, bus=None,
+ complexity_info=None,
) -> ChatCompletionResponse:
"""Direct engine call without agent."""
messages = _to_messages(req.messages)
@@ -166,11 +203,13 @@ def _handle_direct(
completion_tokens=usage.get("completion_tokens", 0),
total_tokens=usage.get("total_tokens", 0),
),
+ complexity=complexity_info,
)
def _handle_agent(
agent, model: str, req: ChatCompletionRequest,
+ complexity_info=None,
) -> ChatCompletionResponse:
"""Run through agent."""
from openjarvis.agents._stubs import AgentContext
@@ -207,6 +246,7 @@ def _handle_agent(
finish_reason="stop",
)],
usage=usage,
+ complexity=complexity_info,
)
@@ -217,7 +257,10 @@ async def _handle_agent_stream(agent, bus, model, req):
return await create_agent_stream(agent, bus, model, req)
-async def _handle_stream(engine, model: str, req: ChatCompletionRequest):
+async def _handle_stream(
+ engine, model: str, req: ChatCompletionRequest,
+ complexity_info=None,
+):
"""Stream response using SSE format."""
messages = _to_messages(req.messages)
chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"
@@ -294,6 +337,9 @@ async def _handle_stream(engine, model: str, req: ChatCompletionRequest):
if isinstance(stream_usage, dict) and stream_usage.get("total_tokens", 0) > 0:
finish_dict["usage"] = stream_usage
+ if complexity_info is not None:
+ finish_dict["complexity"] = complexity_info.model_dump()
+
yield f"data: {_json.dumps(finish_dict)}\n\n"
yield "data: [DONE]\n\n"
diff --git a/tests/cli/test_ask_router.py b/tests/cli/test_ask_router.py
index 39758979..a4f5fbbe 100644
--- a/tests/cli/test_ask_router.py
+++ b/tests/cli/test_ask_router.py
@@ -93,6 +93,8 @@ class TestAskModelResolution:
cfg.telemetry.enabled = False
cfg.intelligence.default_model = ""
cfg.intelligence.fallback_model = ""
+ cfg.intelligence.temperature = 0.7
+ cfg.intelligence.max_tokens = 1024
cfg.agent.context_from_memory = False
result = CliRunner().invoke(cli, ["ask", "Hello"])
assert result.exit_code == 0
@@ -122,6 +124,8 @@ class TestAskModelResolution:
cfg.telemetry.enabled = False
cfg.intelligence.default_model = ""
cfg.intelligence.fallback_model = "fallback-model"
+ cfg.intelligence.temperature = 0.7
+ cfg.intelligence.max_tokens = 1024
cfg.agent.context_from_memory = False
result = CliRunner().invoke(cli, ["ask", "Hello"])
assert result.exit_code == 0
diff --git a/tests/learning/routing/test_complexity.py b/tests/learning/routing/test_complexity.py
new file mode 100644
index 00000000..999eaf1e
--- /dev/null
+++ b/tests/learning/routing/test_complexity.py
@@ -0,0 +1,183 @@
+"""Tests for query complexity analyzer."""
+
+from __future__ import annotations
+
+from openjarvis.learning.routing.complexity import (
+ ComplexityQueryAnalyzer,
+ ComplexityResult,
+ adjust_tokens_for_model,
+ is_thinking_model,
+ score_complexity,
+)
+
+
+class TestScoreComplexity:
+ def test_trivial_query(self) -> None:
+ result = score_complexity("Hi")
+ assert result.tier == "trivial"
+ assert result.score < 0.15
+
+ def test_simple_query(self) -> None:
+ result = score_complexity("What is the capital of France?")
+ assert result.tier in ("trivial", "simple")
+ assert result.score < 0.30
+
+ def test_code_signal(self) -> None:
+ result = score_complexity("def hello():\n return 'world'")
+ assert result.signals["has_code"] is True
+ assert result.score > 0.0
+
+ def test_math_signal(self) -> None:
+ result = score_complexity("Solve the integral of x^2 dx")
+ assert result.signals["has_math"] is True
+ assert result.score > 0.0
+
+ def test_code_and_math_gives_high_domain(self) -> None:
+ query = "```python\nimport numpy\n```\nSolve the integral of x^2"
+ result = score_complexity(query)
+ assert result.signals["has_code"] is True
+ assert result.signals["has_math"] is True
+ assert result.signals["domain"] == 1.0
+
+ def test_reasoning_signal(self) -> None:
+ result = score_complexity("Explain why the sky is blue step by step")
+ assert result.signals["has_reasoning"] is True
+
+ def test_multi_step_signal(self) -> None:
+ result = score_complexity("First do X, then do Y, then do Z")
+ assert result.signals["has_multi_step"] is True
+
+ def test_reasoning_and_multi_step_combined(self) -> None:
+ result = score_complexity(
+ "Explain why X works, then analyze Y, then compare them"
+ )
+ assert result.signals["has_reasoning"] is True
+ assert result.signals["has_multi_step"] is True
+ assert result.signals["reasoning"] == 1.0
+
+ def test_multi_part_questions(self) -> None:
+ query = "What is X? What is Y? What is Z? What is W?"
+ result = score_complexity(query)
+ assert result.signals["n_questions"] == 4
+ assert result.signals["multi_part"] == 1.0
+
+ def test_creative_signal(self) -> None:
+ result = score_complexity("Write an essay about climate change")
+ assert result.signals["has_creative"] is True
+
+ def test_very_complex_query(self) -> None:
+ query = (
+ "Explain step by step how to solve the integral of x^2, "
+ "then write Python code to compute it numerically. "
+ "1. Derive the analytical solution\n"
+ "2. Implement numerical integration\n"
+ "3. Compare the results and analyze the error"
+ )
+ result = score_complexity(query)
+ assert result.tier in ("complex", "very_complex")
+ assert result.score >= 0.55
+
+ def test_score_clamped_to_unit_interval(self) -> None:
+ result = score_complexity("x" * 2000)
+ assert 0.0 <= result.score <= 1.0
+
+ def test_result_type(self) -> None:
+ result = score_complexity("hello")
+ assert isinstance(result, ComplexityResult)
+ assert isinstance(result.score, float)
+ assert isinstance(result.tier, str)
+ assert isinstance(result.suggested_max_tokens, int)
+ assert isinstance(result.signals, dict)
+
+ def test_token_tiers_increase_with_complexity(self) -> None:
+ trivial = score_complexity("Hi")
+ complex_ = score_complexity(
+ "Explain step by step how to solve the integral of x^2 "
+ "and write code to compute the derivative of the matrix equation"
+ )
+ assert complex_.suggested_max_tokens >= trivial.suggested_max_tokens
+
+ def test_subtask_counting(self) -> None:
+ query = "1. First task\n2. Second task\n- Bullet item"
+ result = score_complexity(query)
+ assert result.signals["n_subtasks"] == 3
+
+
+class TestIsThinkingModel:
+ def test_deepseek_r1(self) -> None:
+ assert is_thinking_model("deepseek-r1-32b") is True
+
+ def test_o1_model(self) -> None:
+ assert is_thinking_model("o1-preview") is True
+
+ def test_o3_model(self) -> None:
+ assert is_thinking_model("o3-mini") is True
+
+ def test_qwq(self) -> None:
+ assert is_thinking_model("qwq-32b") is True
+
+ def test_regular_model(self) -> None:
+ assert is_thinking_model("llama-3.1-8b") is False
+
+ def test_gpt4(self) -> None:
+ assert is_thinking_model("gpt-4o") is False
+
+
+class TestAdjustTokensForModel:
+ def test_thinking_model_doubles(self) -> None:
+ assert adjust_tokens_for_model(1024, "deepseek-r1-32b") == 2048
+
+ def test_regular_model_unchanged(self) -> None:
+ assert adjust_tokens_for_model(1024, "llama-3.1-8b") == 1024
+
+ def test_no_model_unchanged(self) -> None:
+ assert adjust_tokens_for_model(1024) == 1024
+
+ def test_none_model_unchanged(self) -> None:
+ assert adjust_tokens_for_model(1024, None) == 1024
+
+
+class TestComplexityQueryAnalyzer:
+ def test_returns_routing_context(self) -> None:
+ analyzer = ComplexityQueryAnalyzer()
+ ctx = analyzer.analyze("Hello world")
+ assert ctx.query == "Hello world"
+ assert ctx.query_length == len("Hello world")
+
+ def test_complexity_score_populated(self) -> None:
+ analyzer = ComplexityQueryAnalyzer()
+ ctx = analyzer.analyze("Explain step by step how photosynthesis works")
+ assert ctx.complexity_score > 0.0
+ assert ctx.has_reasoning is True
+
+ def test_code_detection(self) -> None:
+ analyzer = ComplexityQueryAnalyzer()
+ ctx = analyzer.analyze("def foo(): pass")
+ assert ctx.has_code is True
+
+ def test_math_detection(self) -> None:
+ analyzer = ComplexityQueryAnalyzer()
+ ctx = analyzer.analyze("solve the equation x^2 = 4")
+ assert ctx.has_math is True
+
+ def test_urgency_passthrough(self) -> None:
+ analyzer = ComplexityQueryAnalyzer()
+ ctx = analyzer.analyze("test", urgency=0.9)
+ assert ctx.urgency == 0.9
+
+ def test_invalid_urgency_defaults(self) -> None:
+ analyzer = ComplexityQueryAnalyzer()
+ ctx = analyzer.analyze("test", urgency="invalid")
+ assert ctx.urgency == 0.5
+
+ def test_thinking_model_adjusts_tokens(self) -> None:
+ analyzer = ComplexityQueryAnalyzer()
+ ctx_normal = analyzer.analyze("Hello", model="llama-3.1-8b")
+ ctx_thinking = analyzer.analyze("Hello", model="deepseek-r1-32b")
+ assert ctx_thinking.suggested_max_tokens == ctx_normal.suggested_max_tokens * 2
+
+ def test_metadata_contains_tier(self) -> None:
+ analyzer = ComplexityQueryAnalyzer()
+ ctx = analyzer.analyze("Hi")
+ assert "complexity_tier" in ctx.metadata
+ assert "signals" in ctx.metadata
diff --git a/tests/learning/test_router.py b/tests/learning/test_router.py
index bf80b4ea..736a7f8c 100644
--- a/tests/learning/test_router.py
+++ b/tests/learning/test_router.py
@@ -95,12 +95,12 @@ class TestHeuristicRouter:
)
assert router.select_model(ctx) == "large"
- def test_long_query_prefers_large(self) -> None:
+ def test_high_complexity_prefers_large(self) -> None:
_register_models()
router = HeuristicRouter(
available_models=["small", "large", "coder"],
)
- ctx = RoutingContext(query="x" * 501, query_length=501)
+ ctx = RoutingContext(query="x" * 501, query_length=501, complexity_score=0.7)
assert router.select_model(ctx) == "large"
def test_high_urgency_overrides_to_small(self) -> None:
@@ -122,10 +122,11 @@ class TestHeuristicRouter:
default_model="large",
fallback_model="small",
)
- # Medium-length, no code/math, no reasoning → falls to default
+ # Medium complexity, no code/math, no reasoning → falls to default
ctx = RoutingContext(
query="Tell me about cats",
query_length=60,
+ complexity_score=0.35,
)
assert router.select_model(ctx) == "large"
diff --git a/tests/learning/test_routing_models.py b/tests/learning/test_routing_models.py
index febbd22c..ebe4917a 100644
--- a/tests/learning/test_routing_models.py
+++ b/tests/learning/test_routing_models.py
@@ -85,12 +85,12 @@ class TestRouterWithNewModels:
selected = router.select_model(ctx)
assert selected == "gpt-oss:120b"
- def test_long_context_routes_to_largest(self) -> None:
+ def test_high_complexity_routes_to_largest(self) -> None:
_setup_models()
router = HeuristicRouter(
available_models=NEW_LOCAL_MODELS,
)
- ctx = RoutingContext(query="x" * 501, query_length=501)
+ ctx = RoutingContext(query="x" * 501, query_length=501, complexity_score=0.7)
selected = router.select_model(ctx)
assert selected == "gpt-oss:120b"
@@ -128,6 +128,7 @@ class TestRouterWithNewModels:
ctx = RoutingContext(
query="Tell me about the weather today",
query_length=60,
+ complexity_score=0.35,
)
selected = router.select_model(ctx)
assert selected == "glm-4.7-flash"
@@ -172,7 +173,11 @@ class TestRouterParameterized:
("hi", False),
("solve the integral of sin(x)", True),
("def foo(): pass", True),
- ("x" * 501, True),
+ (
+ "Explain step by step how to solve the integral of x^2 "
+ "and then write code to compute the derivative",
+ True,
+ ),
],
)
def test_query_type_selects_expected_size(