Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

feat(providers): add LOCAL_REASONING_EFFORT for thinking-only models (#274)

Rufused06025aabf3
diff --git a/.env.example b/.env.example
index 5d0fca38..10c4dc5c 100644
--- a/.env.example
+++ b/.env.example
@@ -124,6 +124,11 @@ LOCAL_TIMEOUT_S=300
# gateways do, and then this restores real USD cost tracking on the
# /audit cache_event rows for those calls.
LOCAL_INCLUDE_USAGE_ACCOUNTING=false
+# Optional reasoning_effort sent on every local request. Thinking-only
+# models (e.g. GLM on Fireworks) can otherwise spend the whole max_tokens
+# budget reasoning and return no answer/tool call. Unset = not sent.
+# One of none, minimal, low, medium, high (anything else fails at startup).
+# LOCAL_REASONING_EFFORT=low
#
# To run with NO Anthropic key, configure the three lines above AND point the
# model settings at local slugs:
diff --git a/README.md b/README.md
index 70e26bbe..a4e4b46f 100644
--- a/README.md
+++ b/README.md
@@ -375,6 +375,7 @@ the app refuses to start.
| `LOCAL_API_KEY` | No | — | Optional bearer token (vLLM / gateways); Ollama & LM Studio need none |
| `LOCAL_MODELS` | No | — | Comma-separated local model slugs to surface in the Council UI and route locally, e.g. `llama3.3,qwen2.5` |
| `LOCAL_TIMEOUT_S` | No | `300` | Per-call timeout for local generation, in seconds |
+| `LOCAL_REASONING_EFFORT` | No | — | `reasoning_effort` sent on every local request: `none`, `minimal`, `low`, `medium` or `high`. Set it for thinking-only models (GLM on Fireworks) that otherwise spend the whole token budget reasoning |
| `HONCHO_ENABLED` | No | `false` | Per-person memory layer ([honcho.dev](https://honcho.dev)) — a peer card shared across all channels |
| `HONCHO_API_KEY` | No | — | Required when `HONCHO_ENABLED=true` |
| `HONCHO_BASE_URL` | No | — | Self-hosted Honcho endpoint |
diff --git a/packages/core/openexecutive/architecture/architecture-facts.yaml b/packages/core/openexecutive/architecture/architecture-facts.yaml
index 8fb15162..7ec9b23c 100644
--- a/packages/core/openexecutive/architecture/architecture-facts.yaml
+++ b/packages/core/openexecutive/architecture/architecture-facts.yaml
@@ -62,6 +62,17 @@ system:
Settings validator just requires that at least one provider is
reachable. Requesting a Claude model with no key and OpenRouter off
returns an actionable 400.
+ LOCAL_REASONING_EFFORT (optional, unset by default; one of none,
+ minimal, low, medium, high, validated at startup) sends a top-level
+ ``reasoning_effort`` on every local-backend request, streamed or not,
+ unconditionally: local slugs get a no-thinking spec, so the Council
+ deep-reasoning toggle never adds a per-call ``reasoning`` object on
+ that path. It is added in the provider's ``_extend_body`` hook and
+ logged once per model. Thinking-only
+ hosted models (e.g. GLM on Fireworks, which rejects "none") otherwise
+ spend the whole max_tokens budget reasoning and return no tool call —
+ the onboarding interview then fails with "did not return a usable
+ response".
Slugs listed in LOCAL_MODELS route to the local backend before any
other rule is consulted; remaining non-Claude slugs route through
OpenRouter; selecting an
diff --git a/packages/core/openexecutive/config.py b/packages/core/openexecutive/config.py
index be06348a..b2c6afef 100644
--- a/packages/core/openexecutive/config.py
+++ b/packages/core/openexecutive/config.py
@@ -292,12 +292,27 @@ class Settings(BaseSettings):
# Ollama / LM Studio / vLLM. Off, a PDF for a local model is OCR'd on
# this server instead (knowledge/pdf_reader.py).
local_pdf_input: bool = Field(False, alias="LOCAL_PDF_INPUT")
+ # Optional `reasoning_effort` sent on every local request (e.g. "low").
+ # Thinking-only models (GLM on Fireworks) otherwise spend the whole
+ # max_tokens budget reasoning and return no tool call. Unset = not sent.
+ # A Literal so a typo fails at startup instead of 400ing every local call.
+ local_reasoning_effort: (
+ Literal["none", "minimal", "low", "medium", "high"] | None
+ ) = Field(None, alias="LOCAL_REASONING_EFFORT")
@field_validator("local_models", mode="before")
@classmethod
def _parse_local_models(cls, v: Any) -> list[str]:
return _parse_csv_list(v)
+ @field_validator("local_reasoning_effort", mode="before")
+ @classmethod
+ def _parse_local_reasoning_effort(cls, v: Any) -> Any:
+ # `LOCAL_REASONING_EFFORT=` (or a bare `# comment`) means unset.
+ if _blank_or_comment(v):
+ return None
+ return v.strip().lower() if isinstance(v, str) else v
+
@model_validator(mode="after")
def _validate_local_models(self) -> "Settings":
if self.local_models_enabled and not self.local_base_url:
diff --git a/packages/core/openexecutive/providers/openai_compatible.py b/packages/core/openexecutive/providers/openai_compatible.py
index 807f4f78..a457b469 100644
--- a/packages/core/openexecutive/providers/openai_compatible.py
+++ b/packages/core/openexecutive/providers/openai_compatible.py
@@ -64,6 +64,23 @@ def _announce_reasoning(slug: str, body: dict[str, Any]) -> None:
)
+# Slugs for which we've already logged the LOCAL_REASONING_EFFORT being
+# sent. A missing or wrong effort is what makes a thinking-only model burn
+# its whole max_tokens budget, so the log shows which value is in play.
+_effort_announced: set[str] = set()
+
+
+def _announce_effort(slug: str, effort: str) -> None:
+ if slug not in _effort_announced:
+ _effort_announced.add(slug)
+ logger.info(
+ "reasoning_effort=%s sent to %s via OpenAI-compatible backend "
+ "(LOCAL_REASONING_EFFORT)",
+ effort,
+ slug,
+ )
+
+
class OpenAICompatibleProvider:
"""LLMProvider implementation backed by any OpenAI-compatible endpoint.
@@ -82,6 +99,7 @@ class OpenAICompatibleProvider:
spec_lookup: dict[str, FeatureSpec] | None = None,
model_resolver: Callable[[str], tuple[str, FeatureSpec] | None] | None = None,
include_usage_accounting: bool = False,
+ reasoning_effort: str | None = None,
) -> None:
self._api_key = api_key
self._base_url = base_url.rstrip("/")
@@ -106,6 +124,9 @@ class OpenAICompatibleProvider:
# behind a LiteLLM gateway), which rejects an unrecognized top-level
# `usage` field outright rather than ignoring it.
self._include_usage_accounting = include_usage_accounting
+ # Top-level `reasoning_effort` for thinking-only backends
+ # (LOCAL_REASONING_EFFORT). None = not sent.
+ self._reasoning_effort = reasoning_effort
# ------------------------------------------------------------------
# internal helpers
@@ -145,8 +166,17 @@ class OpenAICompatibleProvider:
def _extend_body(self, slug: str, body: dict[str, Any]) -> None:
"""Backend-specific request fields, added to the translated body in
- place. None for a plain OpenAI-compatible server: an unknown
- top-level field can 400 there (see include_usage_accounting)."""
+ place. A plain OpenAI-compatible server gets only what the operator
+ opted into: an unknown top-level field can 400 there (see
+ include_usage_accounting).
+
+ ``reasoning_effort`` is sent unconditionally when configured. Local
+ slugs never carry a per-call ``reasoning`` object (their spec has
+ ``supports_thinking=False``), so there is nothing for it to clash
+ with."""
+ if self._reasoning_effort:
+ body["reasoning_effort"] = self._reasoning_effort
+ _announce_effort(slug, self._reasoning_effort)
def messages_create(self, **kwargs: Any) -> Awaitable[Any]:
return self._messages_create(kwargs)
diff --git a/packages/core/openexecutive/providers/registry.py b/packages/core/openexecutive/providers/registry.py
index e43d546a..46f97d83 100644
--- a/packages/core/openexecutive/providers/registry.py
+++ b/packages/core/openexecutive/providers/registry.py
@@ -413,6 +413,7 @@ def _local() -> OpenAICompatibleProvider:
include_usage_accounting=getattr(
settings, "local_include_usage_accounting", False
),
+ reasoning_effort=getattr(settings, "local_reasoning_effort", None),
)
return _local_provider
diff --git a/packages/core/tests/unit/test_config_local_models.py b/packages/core/tests/unit/test_config_local_models.py
index 0960f750..aa673931 100644
--- a/packages/core/tests/unit/test_config_local_models.py
+++ b/packages/core/tests/unit/test_config_local_models.py
@@ -18,6 +18,7 @@ _PROVIDER_VARS = (
"LOCAL_BASE_URL",
"LOCAL_MODELS",
"LOCAL_API_KEY",
+ "LOCAL_REASONING_EFFORT",
)
@@ -75,3 +76,31 @@ def test_no_provider_configured_is_rejected(monkeypatch: pytest.MonkeyPatch) ->
def test_anthropic_key_alone_still_boots(monkeypatch: pytest.MonkeyPatch) -> None:
s = _build(monkeypatch, ANTHROPIC_API_KEY="sk-test")
assert s.anthropic_api_key == "sk-test"
+
+
+def test_local_reasoning_effort_unset_by_default(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ s = _build(monkeypatch, ANTHROPIC_API_KEY="sk-test")
+ assert s.local_reasoning_effort is None
+
+
[email protected]("raw", ["", " ", "# e.g. low"])
+def test_local_reasoning_effort_blank_means_unset(
+ monkeypatch: pytest.MonkeyPatch, raw: str
+) -> None:
+ s = _build(monkeypatch, ANTHROPIC_API_KEY="sk-test", LOCAL_REASONING_EFFORT=raw)
+ assert s.local_reasoning_effort is None
+
+
+def test_local_reasoning_effort_normalized(monkeypatch: pytest.MonkeyPatch) -> None:
+ s = _build(monkeypatch, ANTHROPIC_API_KEY="sk-test", LOCAL_REASONING_EFFORT=" Low ")
+ assert s.local_reasoning_effort == "low"
+
+
+def test_local_reasoning_effort_typo_fails_at_startup(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A typo would otherwise 400 every local call at request time."""
+ with pytest.raises(ValueError, match="LOCAL_REASONING_EFFORT"):
+ _build(monkeypatch, ANTHROPIC_API_KEY="sk-test", LOCAL_REASONING_EFFORT="lwo")
diff --git a/packages/core/tests/unit/test_openai_compatible_provider.py b/packages/core/tests/unit/test_openai_compatible_provider.py
index 65b37fe4..ede885e2 100644
--- a/packages/core/tests/unit/test_openai_compatible_provider.py
+++ b/packages/core/tests/unit/test_openai_compatible_provider.py
@@ -12,6 +12,9 @@ import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock
+import pytest
+
+from openexecutive.providers import openai_compatible
from openexecutive.providers.feature_gate import FeatureSpec
from openexecutive.providers.openai_compatible import OpenAICompatibleProvider
@@ -105,3 +108,49 @@ def test_anthropic_only_fields_stripped_for_local_model() -> None:
assert "output_config" not in body
# system flattened into a plain string message — no cache_control survives.
assert isinstance(body["messages"][0]["content"], str)
+
+
+def _effort_provider(effort: str | None) -> OpenAICompatibleProvider:
+ return OpenAICompatibleProvider(
+ base_url="https://api.fireworks.ai/inference/v1",
+ spec_lookup={"llama3.3": _LOCAL_SPEC},
+ reasoning_effort=effort,
+ )
+
+
+def test_reasoning_effort_omitted_by_default() -> None:
+ """Most OpenAI-compatible servers don't know the field; unset = not sent."""
+ captured = _run_create(_local_provider())
+ assert "reasoning_effort" not in captured["json"]
+
+
+def test_reasoning_effort_sent_when_configured() -> None:
+ """Thinking-only models (GLM on Fireworks) otherwise burn the whole
+ max_tokens budget reasoning and return no tool call."""
+ captured = _run_create(_effort_provider("low"))
+ assert captured["json"]["reasoning_effort"] == "low"
+
+
+def test_reasoning_effort_sent_on_stream() -> None:
+ provider = _effort_provider("low")
+ stream = provider.messages_stream(
+ model="llama3.3",
+ max_tokens=8,
+ messages=[{"role": "user", "content": "hi"}],
+ )
+ assert stream._body["reasoning_effort"] == "low" # type: ignore[attr-defined]
+
+
+def test_reasoning_effort_logged_once_per_slug(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The effort in play is what decides whether a thinking-only model
+ answers at all, so it's logged, but once per model, not per call."""
+ monkeypatch.setattr(openai_compatible, "_effort_announced", set())
+ fake_logger = MagicMock()
+ monkeypatch.setattr(openai_compatible, "logger", fake_logger)
+ provider = _effort_provider("low")
+ _run_create(provider)
+ _run_create(provider)
+ assert fake_logger.info.call_count == 1
+ assert fake_logger.info.call_args.args[1:] == ("low", "llama3.3")
diff --git a/packages/core/tests/unit/test_provider_registry.py b/packages/core/tests/unit/test_provider_registry.py
index 8870ae01..f6aa9305 100644
--- a/packages/core/tests/unit/test_provider_registry.py
+++ b/packages/core/tests/unit/test_provider_registry.py
@@ -167,6 +167,7 @@ def _settings_stub(
local_api_key: str | None = None,
local_timeout_s: float = 300.0,
local_include_usage_accounting: bool = False,
+ local_reasoning_effort: str | None = None,
) -> Any:
return SimpleNamespace(
anthropic_api_key=anthropic_key,
@@ -182,6 +183,7 @@ def _settings_stub(
local_api_key=local_api_key,
local_timeout_s=local_timeout_s,
local_include_usage_accounting=local_include_usage_accounting,
+ local_reasoning_effort=local_reasoning_effort,
)
@@ -346,6 +348,23 @@ def test_local_provider_respects_usage_accounting_opt_in(
assert provider._include_usage_accounting is True # type: ignore[attr-defined]
+def test_local_provider_reasoning_effort_unset_by_default(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _local_stub(monkeypatch, enabled=False)
+ provider = get_provider("llama3.3")
+ assert provider._reasoning_effort is None # type: ignore[attr-defined]
+
+
+def test_local_provider_passes_configured_reasoning_effort(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """LOCAL_REASONING_EFFORT reaches the local provider."""
+ _local_stub(monkeypatch, enabled=False, local_reasoning_effort="low")
+ provider = get_provider("llama3.3")
+ assert provider._reasoning_effort == "low" # type: ignore[attr-defined]
+
+
def test_local_provider_distinct_from_openrouter_singleton(
monkeypatch: pytest.MonkeyPatch,
) -> None: