Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(brain): scheduled turns run on the Tool Model chain, not the chat provider plus one retry (BUG-212)

rubenluetke10-beepcebb5518c5af
diff --git a/docs/BUGS.md b/docs/BUGS.md
index 443ad03c92..48f66adb4a 100644
--- a/docs/BUGS.md
+++ b/docs/BUGS.md
@@ -14907,7 +14907,13 @@ crash). Recurring tasks re-arm on their wall-clock grid
(`next_every_due_ns`). `BrainPromptStep` grew `tools` and `model_tier`, and
the workflow runner prefers `brain.run_task(...)` — an isolated turn with an
empty history and the step's allowlist — over the callable chat path.
-`run_task` now passes `turn_context`. The seed Morning Briefing is v2: a
+`run_task` now passes `turn_context`, and its provider order is the Tool
+Model chain (`_task_provider_chain`): the provider the user set under API
+Keys for tool calls leads, then every credential-ready, tool-capable
+provider of another family, capped at four — it used to anchor on the
+chat's active provider with exactly one retry, so OpenRouter 402 + Gemini
+429 killed the run while a ready Vertex Tool Model sat unused. The seed
+Morning Briefing is v2: a
read-only allowlist (`google_calendar`, `gmail`, `wiki-recall`,
`search_web`) and a prompt that grounds every sentence in tool output, greets
by the actual time of day and skips a disconnected area in one clause;
diff --git a/jarvis/brain/manager.py b/jarvis/brain/manager.py
index e668749aaa..633859ddd1 100644
--- a/jarvis/brain/manager.py
+++ b/jarvis/brain/manager.py
@@ -13149,30 +13149,6 @@ class BrainManager:
cache[name] = tool
return tool
- def _task_fallback_provider(self) -> str | None:
- """The one provider a scheduled task retries on when the active brain
- rejects the turn with a credential/credit/rate error.
-
- A different credential FAMILY than the active provider (AP-22: a
- same-family fallback is a brick), registered, not dead-listed this
- session, with a fast model and a portable credential on this box.
- The persistent active provider is never touched (user-only lock).
- """
- active = self._active_name
- active_family = self._tool_model_family(active)
- available = set(self._registry.available())
- for name in _TASK_FALLBACK_ORDER:
- if name == active or name not in available or name in self._dead_providers:
- continue
- if self._tool_model_family(name) == active_family:
- continue
- if not self._fast_model(name):
- continue
- if not self._tool_model_credential_ready(name):
- continue
- return name
- return None
-
async def run_task(
self,
*,
@@ -13191,13 +13167,13 @@ class BrainManager:
actions still hit the approval gate (which, with no human present,
means they block until the unattended-approval wave wires Option B).
- Provider fallback (2026-08-24): when the active provider rejects the
- turn with a credential / credit / rate-limit error (401/402/403/429
- — the same classes the chat path dead-lists or cools down), the SAME
- turn is retried exactly once on the next provider of a different
- credential family (:meth:`_task_fallback_provider`). Any other error,
- or a second failure, propagates so the runner records it in
- ``last_error``. The persistent active provider is never switched.
+ Provider order (BUG-212): the Tool Model leads, then every other
+ credential-ready, tool-capable provider of a different family — see
+ :meth:`_task_provider_chain`. A credential / credit / rate-limit
+ error (401/402/403/429 — the same classes the chat path dead-lists
+ or cools down) moves the SAME turn on to the next candidate; any
+ other error propagates so the runner records it in ``last_error``.
+ The persistent active provider is never switched.
Returns the final assistant text.
"""
@@ -13213,16 +13189,10 @@ class BrainManager:
except Exception: # noqa: BLE001 — the context block is a nicety
log.debug("run_task: turn context skipped", exc_info=True)
turn_context = ""
- attempts: list[str] = [self._active_name]
+ attempts = self._task_provider_chain(intent)
failures: list[str] = []
- while attempts:
- name = attempts.pop(0)
- if intent == "deep":
- model = self._deep_model(name) or self._fast_model(name)
- else:
- # "fast" and "auto" both resolve to the fast model — the
- # cheapest correct default for an unattended background turn.
- model = self._fast_model(name)
+ original: Exception | None = None
+ for index, (name, model) in enumerate(attempts):
try:
brain = self._get_brain(name, model)
dispatcher = self._build_dispatcher(
@@ -13241,23 +13211,60 @@ class BrainManager:
kind = _classify_provider_error(str(exc), default="call_fail")
if kind not in _TASK_FALLBACK_KINDS:
raise
+ original = original or exc
failures.append(f"{name}: {_short_provider_error(exc)}")
- if failures and len(failures) == 1:
- fallback = self._task_fallback_provider()
- if fallback is not None:
- log.warning(
- "run_task: %s rejected the turn (%s) — retrying once on %s",
- name, kind, fallback,
- )
- attempts.append(fallback)
- continue
+ remaining = attempts[index + 1:]
+ if remaining:
log.warning(
- "run_task: %s rejected the turn (%s) and no cross-family "
- "fallback provider is usable", name, kind,
+ "run_task: %s rejected the turn (%s) — trying %s next",
+ name, kind, remaining[0][0],
)
- raise
+ continue
+ log.warning(
+ "run_task: %s rejected the turn (%s) and no further "
+ "credential-ready tool-capable provider is usable", name, kind,
+ )
+ if len(failures) == 1 and original is not None:
+ raise original
raise RuntimeError("all brain providers failed: " + "; ".join(failures))
+ def _task_provider_chain(self, intent: str) -> list[tuple[str, str | None]]:
+ """The providers a scheduled turn tries, in order.
+
+ The Tool Model leads (the provider and key the user set for tool
+ calls — ``[brain.tool_model]``, the "API Keys" section), then every
+ other credential-ready, tool-capable provider of a different family
+ (:meth:`_hoist_tool_model` over the automatic chain). The chat's
+ active provider is just one candidate in that chain, never the
+ anchor: it used to be the only one, and with OpenRouter at 0 credits
+ every automation died at its action step while a ready Vertex Tool
+ Model sat unused (BUG-212). Without any ready candidate the active
+ provider is tried alone so the caller sees its real error.
+
+ ``deep`` turns swap in the provider's deep model; ``fast``/``auto``
+ run on the Tool Model's own model — the cheapest correct default for
+ an unattended background turn. Capped so an unattended run cannot
+ walk a long chain for minutes.
+ """
+ try:
+ ready = self._hoist_tool_model(self._tool_model_base_chain())
+ except Exception: # noqa: BLE001 — a broken probe must not kill the task
+ log.debug("run_task: tool-model chain unavailable", exc_info=True)
+ ready = []
+ chain: list[tuple[str, str | None]] = []
+ for name, model in ready[:_TASK_MAX_ATTEMPTS]:
+ if intent == "deep":
+ model = self._deep_model(name) or model
+ chain.append((name, model))
+ if chain:
+ return chain
+ name = self._active_name
+ model = (
+ (self._deep_model(name) or self._fast_model(name))
+ if intent == "deep" else self._fast_model(name)
+ )
+ return [(name, model)]
+
def snapshot(self) -> dict[str, Any]:
return {
"active_provider": self._active_name,
@@ -13277,18 +13284,14 @@ class BrainManager:
#: spawn tool (AP-5/AP-14).
_TASK_ONLY_TOOLS: frozenset[str] = frozenset({"remember"})
-#: Error classes that make a scheduled task retry once on another provider
+#: Error classes that make a scheduled task move on to the next provider
#: family: the dead-list kinds (missing/bad key, 402 credits) plus a rate limit.
_TASK_FALLBACK_KINDS: frozenset[str] = frozenset(
{"missing_key", "account_blocked", "bad_key", "rate_limit"}
)
-#: Cross-provider order for the scheduled-task fallback (mirrors the chat
-#: path's ``cross_order``; family-filtered at runtime).
-_TASK_FALLBACK_ORDER: tuple[str, ...] = (
- "gemini", "claude-api", "openai", "openrouter", "grok", "nvidia",
-)
-
+#: Most providers one scheduled turn walks before it gives up (BUG-212).
+_TASK_MAX_ATTEMPTS: int = 4
def _scheduled_turn_context(turn_context: str, tools: dict[str, Any]) -> str:
"""The per-turn context of an unattended scheduled turn.
diff --git a/tests/unit/brain/test_run_task_fallback.py b/tests/unit/brain/test_run_task_fallback.py
index c39b284253..e673ae8245 100644
--- a/tests/unit/brain/test_run_task_fallback.py
+++ b/tests/unit/brain/test_run_task_fallback.py
@@ -1,11 +1,14 @@
-"""BrainManager.run_task — cross-family provider fallback, task-only tools and
-written delivery for scheduled tasks (Automations, 2026-08-24).
-
-Live defect: the active provider (OpenRouter, 0 credits) answered every
-scheduled turn with ``APIStatusError 402`` in 0.4 s and NOTHING retried, so
-every automation failed at its action step. The chat path walks a fallback
-chain; the task path must at least retry once on another credential family
-(AP-22) — and never switch the persistent active provider (user-only lock).
+"""BrainManager.run_task — the Tool Model leads, then the cross-family chain;
+task-only tools and written delivery for scheduled tasks.
+
+Live defect 2026-08-24: the active provider (OpenRouter, 0 credits) answered
+every scheduled turn with ``APIStatusError 402`` in 0.4 s and NOTHING retried.
+Live defect 2026-09-02 (BUG-212): one retry on one other family was not
+enough either — OpenRouter 402, Gemini 429, and the Vertex Tool Model the
+user had configured under API Keys (state "ready") was never asked. A
+scheduled turn now walks :meth:`BrainManager._task_provider_chain`: the Tool
+Model first, then every credential-ready, tool-capable provider of another
+family, capped — and never switches the persistent active provider.
"""
from __future__ import annotations
@@ -47,8 +50,21 @@ class _Credit402(Exception):
pass
-def _manager(monkeypatch: pytest.MonkeyPatch, *, active: str = "openrouter",
- providers: list[str] | None = None) -> BrainManager:
+_DEFAULT_CHAIN: list[tuple[str, str | None]] = [
+ ("openrouter", "openrouter-tool"),
+ ("gemini", "gemini-tool"),
+ ("openai", "openai-tool"),
+]
+
+
+def _manager(
+ monkeypatch: pytest.MonkeyPatch,
+ *,
+ active: str = "openrouter",
+ providers: list[str] | None = None,
+ ready_chain: list[tuple[str, str | None]] | None = None,
+) -> BrainManager:
+ """A manager whose hoisted Tool Model chain is ``ready_chain``."""
mgr = BrainManager(
config=JarvisConfig(),
bus=EventBus(),
@@ -63,6 +79,8 @@ def _manager(monkeypatch: pytest.MonkeyPatch, *, active: str = "openrouter",
monkeypatch.setattr(mgr, "_get_brain", lambda name, model=None: SimpleNamespace(
name=name, model=model,
))
+ chain = list(_DEFAULT_CHAIN if ready_chain is None else ready_chain)
+ monkeypatch.setattr(mgr, "_hoist_tool_model", lambda base: list(chain))
return mgr
@@ -89,7 +107,30 @@ def _install_dispatch(
return calls
-async def test_402_retries_once_on_a_different_family(monkeypatch: pytest.MonkeyPatch) -> None:
+# ----------------------------------------------------------------------
+# Provider order
+# ----------------------------------------------------------------------
+
+async def test_the_configured_tool_model_leads_not_the_chat_provider(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The user set Vertex under API Keys as the Tool Model; the chat runs on
+ OpenRouter. A scheduled turn asks Vertex first."""
+ mgr = _manager(monkeypatch, active="openrouter", ready_chain=[
+ ("vertex", "gemini-3.7-flash"), ("openrouter", "openrouter-tool"),
+ ])
+ calls = _install_dispatch(monkeypatch, mgr, {
+ "vertex": "brief from vertex", "openrouter": "never reached",
+ })
+
+ out = await mgr.run_task(prompt="p", allowed_tools=("gmail",), model_tier="fast")
+
+ assert out == "brief from vertex"
+ assert [(c[0], c[1]) for c in calls] == [("vertex", "gemini-3.7-flash")]
+ assert mgr._active_name == "openrouter", "the persistent provider is never switched"
+
+
+async def test_402_moves_on_to_the_next_family(monkeypatch: pytest.MonkeyPatch) -> None:
mgr = _manager(monkeypatch)
calls = _install_dispatch(monkeypatch, mgr, {
"openrouter": _Credit402("Error code: 402 - Insufficient credits"),
@@ -99,33 +140,66 @@ async def test_402_retries_once_on_a_different_family(monkeypatch: pytest.Monkey
out = await mgr.run_task(prompt="p", allowed_tools=("gmail",), model_tier="fast")
assert out == "digest from gemini"
- assert [(c[0], c[1]) for c in calls] == [
- ("openrouter", "openrouter-fast"), ("gemini", "gemini-fast"),
- ]
- assert mgr._active_name == "openrouter", "the persistent provider is never switched"
+ assert [c[0] for c in calls] == ["openrouter", "gemini"]
-async def test_fallback_skips_same_family_and_dead_providers(
+async def test_the_whole_chain_is_walked_not_just_one_fallback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
- # codex shares the "openai" family with openai-api; gemini is dead-listed.
+ """OpenRouter 402 AND Gemini 429 used to end the turn; the third ready
+ provider must still get its chance."""
+ mgr = _manager(monkeypatch)
+ calls = _install_dispatch(monkeypatch, mgr, {
+ "openrouter": _Credit402("Error code: 402 - Insufficient credits"),
+ "gemini": RuntimeError("Error code: 429 - rate limit exceeded"),
+ "openai": "brief from openai",
+ })
+
+ out = await mgr.run_task(prompt="p", model_tier="fast")
+
+ assert out == "brief from openai"
+ assert [c[0] for c in calls] == ["openrouter", "gemini", "openai"]
+
+
+def test_the_chain_is_capped(monkeypatch: pytest.MonkeyPatch) -> None:
+ ready = [(f"p{i}", f"p{i}-tool") for i in range(8)]
+ mgr = _manager(monkeypatch, ready_chain=ready)
+ chain = mgr._task_provider_chain("fast")
+ assert len(chain) == manager_mod._TASK_MAX_ATTEMPTS
+ assert chain == ready[: manager_mod._TASK_MAX_ATTEMPTS]
+
+
+def test_deep_tier_swaps_in_the_deep_model(monkeypatch: pytest.MonkeyPatch) -> None:
+ mgr = _manager(monkeypatch)
+ assert mgr._task_provider_chain("fast")[0] == ("openrouter", "openrouter-tool")
+ assert mgr._task_provider_chain("deep")[0] == ("openrouter", "openrouter-deep")
+
+
+def test_the_real_hoist_skips_same_family_and_dead_providers(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Through the real Tool Model hoist: codex shares the openai family with
+ openai-api; a dead-listed gemini is skipped; grok stays."""
mgr = _manager(monkeypatch, active="openai-api",
providers=["openai-api", "codex", "gemini", "grok"])
+ monkeypatch.delattr(mgr, "_hoist_tool_model")
+ monkeypatch.setattr(mgr, "_tool_model_provider", lambda: "")
+ monkeypatch.setattr(mgr, "_tool_model_base_chain", lambda: [
+ ("openai-api", None), ("codex", None), ("gemini", None), ("grok", None),
+ ])
mgr._dead_providers.add("gemini")
- assert mgr._task_fallback_provider() == "grok"
+ chain = mgr._task_provider_chain("fast")
-async def test_fallback_requires_a_usable_credential(monkeypatch: pytest.MonkeyPatch) -> None:
- mgr = _manager(monkeypatch)
- monkeypatch.setattr(mgr, "_tool_model_credential_ready", lambda name: name == "openai")
- assert mgr._task_fallback_provider() == "openai"
+ assert [name for name, _ in chain] == ["openai-api", "grok"]
-async def test_both_failing_reports_both(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_all_failing_reports_every_provider(monkeypatch: pytest.MonkeyPatch) -> None:
mgr = _manager(monkeypatch)
_install_dispatch(monkeypatch, mgr, {
"openrouter": _Credit402("Error code: 402 - Insufficient credits"),
"gemini": RuntimeError("Error code: 429 - rate limit exceeded"),
+ "openai": RuntimeError("Error code: 401 - invalid api key"),
})
with pytest.raises(RuntimeError) as info:
await mgr.run_task(prompt="p", model_tier="fast")
@@ -133,6 +207,7 @@ async def test_both_failing_reports_both(monkeypatch: pytest.MonkeyPatch) -> Non
assert msg.startswith("all brain providers failed: ")
assert "openrouter: _Credit402: Error code: 402" in msg
assert "gemini: RuntimeError: Error code: 429" in msg
+ assert "openai: RuntimeError: Error code: 401" in msg
async def test_non_credential_error_propagates_without_retry(
@@ -148,20 +223,33 @@ async def test_non_credential_error_propagates_without_retry(
assert [c[0] for c in calls] == ["openrouter"]
-async def test_no_usable_fallback_reraises_original(monkeypatch: pytest.MonkeyPatch) -> None:
- mgr = _manager(monkeypatch, providers=["openrouter"])
- _install_dispatch(monkeypatch, mgr, {
+async def test_no_ready_candidate_tries_the_active_provider_and_reraises(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Nothing credential-ready: the active provider runs alone so the
+ caller sees its real error, not a generic 'all failed'."""
+ mgr = _manager(monkeypatch, providers=["openrouter"], ready_chain=[])
+ calls = _install_dispatch(monkeypatch, mgr, {
"openrouter": _Credit402("Error code: 402 - Insufficient credits"),
})
with pytest.raises(_Credit402):
await mgr.run_task(prompt="p", model_tier="fast")
+ assert [(c[0], c[1]) for c in calls] == [("openrouter", "openrouter-fast")]
+
+# ----------------------------------------------------------------------
+# Turn shape
+# ----------------------------------------------------------------------
-async def test_task_turn_declares_written_delivery(monkeypatch: pytest.MonkeyPatch) -> None:
+async def test_task_turn_declares_written_unattended_delivery(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
mgr = _manager(monkeypatch)
calls = _install_dispatch(monkeypatch, mgr, {"openrouter": "ok"})
await mgr.run_task(prompt="p", model_tier="deep")
- assert calls == [("openrouter", "openrouter-deep", {"delivery": "written"})]
+ assert calls == [(
+ "openrouter", "openrouter-deep", {"delivery": "written", "unattended": True},
+ )]
def test_remember_grant_loads_task_only_tool(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -198,4 +286,3 @@ def test_build_dispatcher_accepts_tool_context() -> None:
params = inspect.signature(BrainManager._build_dispatcher).parameters
assert "tool_context" in params
- assert params["tool_context"].kind is inspect.Parameter.KEYWORD_ONLY