Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(computer-use): let a mission wait for a slow screen instead of dying
Haraldb03920829d68
diff --git a/jarvis/cu/conventions.py b/jarvis/cu/conventions.pyindex 60c9f24d43..898f0c2b44 100644--- a/jarvis/cu/conventions.py+++ b/jarvis/cu/conventions.py@@ -131,7 +131,11 @@ def action_grammar_block() -> str:'"y2": <num>, "duration_ms": 400}\n'' {"action": "open_app", "name": "<app>"}\n'' {"action": "switch_window", "name": "<window-title substring>"}\n'- ' {"action": "wait", "ms": <0-10000>}\n'+ ' {"action": "wait", "ms": <0-10000>} deliberate patience while '+ "the screen is BUSY doing what you asked (page still loading, "+ "download running, installer working, spinner or progress bar, app "+ "still starting). A wait counts as progress, not as being stuck — "+ "waiting beats re-clicking a control that is already working\n"' {"action": "done", "reason": "<the on-screen proof>"}\n'' {"action": "fail", "reason": "<why the goal is impossible>"}')diff --git a/jarvis/cu/engine.py b/jarvis/cu/engine.pyindex 8235efb8fb..92fa44db8a 100644--- a/jarvis/cu/engine.py+++ b/jarvis/cu/engine.py@@ -107,8 +107,34 @@ _MAX_CONSECUTIVE_FAILURES = 4_MAX_LLM_FAILURES = 3_MAX_DONE_REJECTS = 3_MAX_GUARD_HITS = 5-_MAX_OBSERVE_FAILURES = 2+#: Consecutive observation failures before the mission gives up on the screen.+#: Most observe failures are a BUSY screen, not a broken one — a page finishing+#: its load or an installer opening its next window flips the foreground+#: mid-capture and trips the identity guard. At 2 a mission died after ~2+#: seconds of ordinary desktop churn (AU-16); 4 plus the backoff below rides+#: out a normal window transition while a genuinely unreadable screen (no+#: display, revoked capture permission) still fails within seconds.+_MAX_OBSERVE_FAILURES = 4+#: Settle beat before re-observing, so the retry does not re-probe into the+#: same churn. Scaled by ``settle_scale`` (0 in test/benchmark rigs).+_OBSERVE_RETRY_BACKOFF_S = 0.4+#: Steps with zero successful actions on an unchanged screen before the+#: dead-end verdict. Deliberate waits are exempt and bounded separately._STUCK_FRAMES = 3+#: Consecutive deliberate waits that changed nothing on screen before the wait+#: itself counts as a dead end. Anything genuinely in progress — a loading+#: page, a progress bar, an installer — repaints *something* well inside six+#: look-agains, and any visible change resets this counter. Six is also the+#: patience a human grants before concluding nothing is happening.+_MAX_STALLED_WAITS = 6+#: Total deliberate waiting per mission. Bounds patience against the mission+#: ceiling ([computer_use].mission_timeout_s, default 600s) so waiting can+#: never eat the whole budget: at most ~20% of it goes to waiting, the rest+#: stays available for actual work. Once spent, waits shrink to a beat and+#: stop being exempt from the no-progress guard.+_MAX_TOTAL_WAIT_S = 120.0+#: Length of a wait once the patience budget is spent.+_WAIT_EXHAUSTED_MS = 250_SYSTEM_BASE = ("You are the computer-use executor: you operate the user's REAL desktop "@@ -158,6 +184,14 @@ _SYSTEM_BASE = ("search boxes) instead of appending.\n""* Use open_app to launch or focus an application. Use switch_window to ""focus an open window.\n"+ "* WAITING IS WORK, not stalling. While the screen is BUSY doing what you "+ "already asked for — a page still loading, a spinner or progress bar, a "+ "download running, an installer working, an app still starting — reply "+ 'with {"action": "wait", "ms": <up to 10000>} and look again. Re-clicking '+ "a control that is already working is what breaks these tasks. But wait "+ "ONLY while something is visibly in progress: if several waits in a row "+ "change nothing on screen, stop waiting and either act differently or "+ "fail with a reason.\n""* done: ONLY when the goal's observable proof is visible in the CURRENT ""screenshot; quote that proof in reason.\n""* fail: ONLY when the goal is genuinely impossible from here; explain "@@ -837,6 +871,9 @@ async def run_cu_loop(prev_thumb: bytes | None = Nonefruitless_steps = 0 # steps with zero successful actions on an unchanged screenlast_step_had_success = False+ last_step_waited = False # the previous step deliberately waited+ stalled_waits = 0 # consecutive waits that changed nothing on screen+ total_wait_s = 0.0 # deliberate waiting spent so far (patience budget)# Normalize the work surface before the first frame and again after every# focus change (open_app / switch_window): professional computer-use# harnesses never pixel-ground on a small window floating in a big desktop@@ -951,6 +988,11 @@ async def run_cu_loop(exit_code=_EXIT_OBSERVE,)return+ # A busy desktop is the common cause (the foreground flips while a+ # page finishes loading or an installer opens its next window).+ # Give it a beat instead of re-probing straight back into the same+ # churn — an instant retry burns the whole budget in ~2 s (AU-16).+ await asyncio.sleep(_OBSERVE_RETRY_BACKOFF_S * settle_scale)continueobserve_failures = 0window_title = _foreground_title()@@ -973,17 +1015,28 @@ async def run_cu_loop(# No-progress guard: several consecutive steps that produced no# successful action while the screen never changed means we are# circling — judge once (the goal may in fact be met), then stop.- if (- prev_thumb is not None- and thumbs_similar(prev_thumb, frame.thumb)- and not last_step_had_success- ):+ #+ # A step that DELIBERATELY waited is exempt (AU-16). Waiting for a page+ # to load, a download to finish or an installer to run looks EXACTLY+ # like being stuck, so counting it here killed patient, correct work+ # after three steps. Patience still has to terminate, so no-change+ # waits accumulate on their own counter (and on the time budget spent+ # in the wait action itself); any visible change resets both.+ screen_unchanged = prev_thumb is not None and thumbs_similar(+ prev_thumb, frame.thumb+ )+ if screen_unchanged and not last_step_had_success and not last_step_waited:fruitless_steps += 1else:fruitless_steps = 0+ if not screen_unchanged:+ stalled_waits = 0+ elif last_step_waited:+ stalled_waits += 1prev_thumb = frame.thumblast_step_had_success = False- if fruitless_steps >= _STUCK_FRAMES:+ last_step_waited = False+ if fruitless_steps >= _STUCK_FRAMES or stalled_waits >= _MAX_STALLED_WAITS:done, proof = await _judge_done(ctx,goal,@@ -1003,10 +1056,15 @@ async def run_cu_loop(# "the unsubscribe link did not work" was never attempted —# the mission died scrolling).recent = "; ".join(h[:140] for h in history[-2:])+ cause = (+ f"I waited {stalled_waits} times in a row and nothing on "+ "screen moved"+ if stalled_waits >= _MAX_STALLED_WAITS+ else "the screen has not changed despite my actions"+ )yield _final(stderr=(- f"[cu] fail at step-{step_idx}: no progress — the "- "screen has not changed despite my actions."+ f"[cu] fail at step-{step_idx}: no progress — {cause}."+ (f" Last attempts: {recent}" if recent else "")+ "\n"),@@ -1683,8 +1741,26 @@ async def run_cu_loop(profiler.add("act", t0, step_idx)elif kind == "wait":- await asyncio.sleep(action["ms"] / 1000.0)- ok, detail = True, f"waited {action['ms']} ms"+ # Deliberate patience, bounded by a per-mission time budget.+ # While budget remains the step is marked as a wait, which+ # exempts it from the no-progress guard above; once the budget+ # is spent a wait shrinks to a beat, loses the exemption and+ # the model is told so in plain words.+ remaining_wait_s = _MAX_TOTAL_WAIT_S - total_wait_s+ if remaining_wait_s > 0:+ wait_ms = min(int(action["ms"]), int(remaining_wait_s * 1000))+ last_step_waited = True+ detail = f"waited {wait_ms} ms"+ else:+ wait_ms = min(int(action["ms"]), _WAIT_EXHAUSTED_MS)+ detail = (+ f"waited {wait_ms} ms — the {_MAX_TOTAL_WAIT_S:.0f}s "+ "patience budget of this mission is spent; act "+ "differently now or stop with fail"+ )+ total_wait_s += wait_ms / 1000.0+ await asyncio.sleep(wait_ms / 1000.0)+ ok = Trueprofiler.add("settle", t0, step_idx)# -- bookkeeping ---------------------------------------------------@@ -1693,8 +1769,12 @@ async def run_cu_loop(if ok:if kind != "wait":acted_since_capture = True+ # A wait changed nothing in the world, so it must not+ # claim "the screen moved because of me" — its exemption+ # from the no-progress guard rides on last_step_waited,+ # which is bounded, instead of on this unbounded reset.+ last_step_had_success = Trueconsecutive_failures = 0- last_step_had_success = Truehistory.append(f"step {step_idx}: {summary} -> OK" + (f" ({detail[:120]})" if detail else ""),)diff --git a/jarvis/plugins/harness/computer_use.py b/jarvis/plugins/harness/computer_use.pyindex 1028a69a63..b1f0e9932f 100644--- a/jarvis/plugins/harness/computer_use.py+++ b/jarvis/plugins/harness/computer_use.py@@ -56,8 +56,9 @@ _CANCEL_HEARTBEAT_S = 0.12# used to run concurrently (the tool only dedupes IDENTICAL goals) and raced# each other's pointer moves and foreground guards. Every launch route (voice,# LLM tool, scheduled task) funnels through this harness, so serializing here-# covers them all. A queued mission waits honoring its own deadline and-# cancellation; the wait is logged so "nothing happens" is diagnosable.+# covers them all. A queued mission waits on a SEPARATE queue allowance (not on+# its mission budget, AU-17) and stays cancellable throughout; the wait is+# logged so "nothing happens" is diagnosable._DESKTOP_LOCK = asyncio.Lock()@@ -153,7 +154,17 @@ class ComputerUseHarness:"""ctx = self._explicit_context or get_computer_use_context()timeout_s = max(0.001, float(task.timeout_s))- deadline = time.monotonic() + timeout_s+ # AU-17: the budget bounds the WORK, not the queue. Waiting for the+ # ONE desktop is not mission work, and charging it to the mission+ # timeout let a queued mission die of a timeout without ever having+ # started — with the 600 s ceiling of bdf15515 that is ten minutes of+ # burning nothing and then reporting failure. So the queue gets its+ # OWN allowance of the same size, and the work deadline below is armed+ # only once the desktop is actually held. One equal allowance is+ # exactly enough to outlast a full-length predecessor, because the+ # active mission is bounded by the same ceiling.+ queue_deadline = time.monotonic() + timeout_s+ deadline = queue_deadline # re-armed to the full budget on acquiret_start = time.time_ns()# BUG-CU-HANGUP-RACE (2026-05-28): if the user said "auflegen" in the# window between this mission being requested and it actually starting,@@ -218,7 +229,7 @@ class ComputerUseHarness:is_final=True,)return- remaining_s = deadline - time.monotonic()+ remaining_s = queue_deadline - time.monotonic()if remaining_s <= 0:end_reason = "timeout"final_exit_code = _TIMEOUT_EXIT_CODE@@ -239,6 +250,11 @@ class ComputerUseHarness:lock_acquired = Trueexcept TimeoutError:continue+ # The desktop is ours: the mission's own budget starts NOW,+ # undiminished by however long the queue took (AU-17).+ # ``t_start`` deliberately stays at invoke() entry — the+ # reported duration is honest wall time, queue included.+ deadline = time.monotonic() + timeout_s# Control-indicator contract (jarvis.cu.indicator): Started# fires the moment this mission may actually drive# mouse/keyboard (token registered AND desktop lock held);diff --git a/tests/unit/cu/test_engine_loop.py b/tests/unit/cu/test_engine_loop.pyindex 3fb10d66b8..fc7a49d6d0 100644--- a/tests/unit/cu/test_engine_loop.py+++ b/tests/unit/cu/test_engine_loop.py@@ -965,3 +965,137 @@ async def test_handoff_screen_fails_fast_with_speakable_reason(patched, monkeypaassert final.exit_code == 5assert "captcha challenge" in final.stderrassert not brain.calls, "no model call may happen on a handoff screen"+++# ---------------------------------------------------------------------------+# Patience vs. dead end (AU-16). Waiting for a page to load, a download to+# finish or an installer to run looks EXACTLY like being stuck. A deliberate+# wait is progress and must survive the no-progress guard; unbounded waiting+# on a frozen screen must still die.+# ---------------------------------------------------------------------------+++async def test_deliberate_wait_is_not_a_fruitless_step(patched):+ # Five waits in a row on a screen that never changes — more than+ # _STUCK_FRAMES. Before AU-16 the third one killed the mission as "no+ # progress"; a mission that patiently waits out a load must reach its goal.+ assert engine_mod._STUCK_FRAMES < 5, "the scenario must exceed the guard"+ brain = FakeBrain(+ ['{"action": "wait", "ms": 1}'] * 5+ + [+ '{"action": "done", "reason": "the page finished loading"}',+ '{"done": true, "proof": "the page is loaded"}',+ ]+ )++ chunks = await _run(_ctx(brain, FakeExecutor()))++ final = _final(chunks)+ assert final.exit_code == 0, f"patience was killed: {final.stderr!r}"+ assert "no progress" not in final.stderr+++async def test_waiting_forever_on_a_frozen_screen_still_aborts(patched):+ # The bound on patience: waits that change nothing accumulate, and once+ # _MAX_STALLED_WAITS of them pass in a row the mission ends honestly+ # instead of stalling until the mission timeout.+ brain = FakeBrain(['{"action": "wait", "ms": 1}'] * 20)++ chunks = await _run(_ctx(brain, FakeExecutor()))++ final = _final(chunks)+ assert final.exit_code == 5+ assert "no progress" in final.stderr+ assert "waited" in final.stderr, (+ f"the abort must name waiting as the cause, got: {final.stderr!r}"+ )+ waits = [c for c in chunks if "wait(" in (c.stdout or "")]+ assert len(waits) >= engine_mod._MAX_STALLED_WAITS, (+ "the mission must actually be allowed to wait before giving up"+ )+++async def test_repeated_no_op_actions_still_abort_as_a_dead_end(patched):+ # The dead-end protection stays intact for genuinely fruitless steps: an+ # action that keeps failing on an unchanged screen ends the mission.+ brain = FakeBrain(['{"action": "click_element", "name": "Next"}'] * 8)+ executor = FakeExecutor(failures={"click_element"})++ chunks = await _run(_ctx(brain, executor))++ final = _final(chunks)+ assert final.exit_code in (5, 8), f"a dead end must abort: {final.stderr!r}"+ assert len(executor.calls) <= engine_mod._STUCK_FRAMES + 2, (+ "a dead end must be cut short, not retried indefinitely"+ )+++async def test_spent_patience_budget_is_told_to_the_model(patched, monkeypatch):+ # Patience is also bounded in TIME, so waiting can never eat the whole+ # mission budget. Once spent, waits shrink to a beat and the model is told+ # in plain words that it has to act instead.+ monkeypatch.setattr(engine_mod, "_MAX_TOTAL_WAIT_S", 0.005)+ brain = FakeBrain(['{"action": "wait", "ms": 10}'] * 5)++ await _run(_ctx(brain, FakeExecutor()))++ assert any("patience budget" in user for _system, user in brain.calls), (+ "the model must learn that its waiting budget is spent"+ )+++async def test_the_model_is_told_that_waiting_is_a_legitimate_move(patched):+ # A capability nobody is told about is useless: both the discipline block+ # and the action grammar must surface the wait action and when to use it.+ brain = FakeBrain(['{"action": "fail", "reason": "stop"}'])++ await _run(_ctx(brain, FakeExecutor()))++ system = brain.calls[0][0]+ assert '"action": "wait"' in system+ assert "still loading" in system+ assert "installer" in system+++async def test_a_busy_screen_survives_more_than_two_observe_failures(patched, monkeypatch):+ # A page finishing its load or an installer opening its next window flips+ # the foreground mid-capture, which trips the capture identity guard. At+ # two tolerated failures a mission died after ~2 s of ordinary desktop+ # churn (AU-16); transient blindness must be ridden out, not fatal.+ real_capture = engine_mod.capture_stable_frame+ state = {"n": 0}++ def flaky_capture(monitor, **kw):+ state["n"] += 1+ if state["n"] <= 3:+ raise RuntimeError("foreground window changed during capture")+ return real_capture(monitor, **kw)++ monkeypatch.setattr(engine_mod, "capture_stable_frame", flaky_capture)+ brain = FakeBrain(+ [+ '{"action": "done", "reason": "the page finished loading"}',+ '{"done": true, "proof": "the page is loaded"}',+ ]+ )++ chunks = await _run(_ctx(brain, FakeExecutor()))++ final = _final(chunks)+ assert final.exit_code == 0, f"transient blindness was fatal: {final.stderr!r}"+ assert state["n"] > 3, "the retries must actually have happened"+++async def test_a_screen_that_stays_unreadable_still_fails(patched, monkeypatch):+ # The honest counterpart: a screen that never becomes readable ends the+ # mission with the observe exit code instead of grinding to the budget.+ def dead_capture(monitor, **kw):+ raise RuntimeError("no display available")++ monkeypatch.setattr(engine_mod, "capture_stable_frame", dead_capture)++ chunks = await _run(_ctx(FakeBrain([]), FakeExecutor()))++ final = _final(chunks)+ assert final.exit_code == 1+ assert "cannot see the screen" in final.stderrdiff --git a/tests/unit/harness/test_cu_desktop_lock.py b/tests/unit/harness/test_cu_desktop_lock.pyindex 063d6f7c50..d7aaa47c28 100644--- a/tests/unit/harness/test_cu_desktop_lock.py+++ b/tests/unit/harness/test_cu_desktop_lock.py@@ -149,3 +149,49 @@ async def test_queued_mission_times_out_honestly(release.set()await asyncio.wait_for(first_task, timeout=2.0)+++async def test_queued_mission_gets_its_full_budget_after_the_lock_frees(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The budget bounds the WORK, not the queue (AU-17).++ The holder occupies the desktop for 0.5 s; the queued mission then needs+ 0.6 s of work and carries a 1.0 s budget. Charging the queue wait to that+ budget left it 0.5 s — less than its work — so it died of a timeout+ without ever having started. Its budget must start when the desktop does.+ """+ release = asyncio.Event()+ worked: list[str] = []++ def loop_factory(task: HarnessTask, ctx, cancel_token=None) -> AsyncIterator[HarnessResult]:+ async def run() -> AsyncIterator[HarnessResult]:+ if task.prompt == "holder":+ await release.wait()+ else:+ await asyncio.sleep(0.6) # work that outlasts the leftover+ worked.append(task.prompt)+ yield HarnessResult(stdout="ok", exit_code=0, is_final=True)+ return run()++ monkeypatch.setattr(cu_mod, "_resolve_run_cu_loop", lambda: loop_factory)+ bus = EventBus()+ ctx = _ctx(bus)+ first = ComputerUseHarness(context=ctx)+ queued = ComputerUseHarness(context=ctx)++ async def drain(h: ComputerUseHarness, goal: str, timeout_s: float) -> list[HarnessResult]:+ return [c async for c in h.invoke(HarnessTask(prompt=goal, timeout_s=timeout_s))]++ first_task = asyncio.create_task(drain(first, "holder", 10))+ await asyncio.sleep(0.05) # the holder owns the desktop+ queued_task = asyncio.create_task(drain(queued, "queued", 1.0))+ await asyncio.sleep(0.45) # the queued mission spends this in the queue+ release.set()++ chunks = await asyncio.wait_for(queued_task, timeout=5.0)+ assert chunks[-1].exit_code == 0, (+ f"the queued mission lost budget to the queue: {chunks[-1].stderr!r}"+ )+ assert worked == ["queued"]+ await asyncio.wait_for(first_task, timeout=2.0)