Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(computer-use): let a mission wait for a slow screen instead of dying

Haraldb03920829d68
diff --git a/jarvis/cu/conventions.py b/jarvis/cu/conventions.py
index 60c9f24d43..898f0c2b44 100644
--- a/jarvis/cu/conventions.py
+++ b/jarvis/cu/conventions.py
@@ -131,7 +131,11 @@ def action_grammar_block() -> str:
'"y2": <num>, "duration_ms": 400}\n'
' {"action": "open_app", "name": "<app>"}\n'
' {"action": "switch_window", "name": "<window-title substring>"}\n'
- ' {"action": "wait", "ms": <0-10000>}\n'
+ ' {"action": "wait", "ms": <0-10000>} deliberate patience while '
+ "the screen is BUSY doing what you asked (page still loading, "
+ "download running, installer working, spinner or progress bar, app "
+ "still starting). A wait counts as progress, not as being stuck — "
+ "waiting beats re-clicking a control that is already working\n"
' {"action": "done", "reason": "<the on-screen proof>"}\n'
' {"action": "fail", "reason": "<why the goal is impossible>"}'
)
diff --git a/jarvis/cu/engine.py b/jarvis/cu/engine.py
index 8235efb8fb..92fa44db8a 100644
--- a/jarvis/cu/engine.py
+++ b/jarvis/cu/engine.py
@@ -107,8 +107,34 @@ _MAX_CONSECUTIVE_FAILURES = 4
_MAX_LLM_FAILURES = 3
_MAX_DONE_REJECTS = 3
_MAX_GUARD_HITS = 5
-_MAX_OBSERVE_FAILURES = 2
+#: Consecutive observation failures before the mission gives up on the screen.
+#: Most observe failures are a BUSY screen, not a broken one — a page finishing
+#: its load or an installer opening its next window flips the foreground
+#: mid-capture and trips the identity guard. At 2 a mission died after ~2
+#: seconds of ordinary desktop churn (AU-16); 4 plus the backoff below rides
+#: out a normal window transition while a genuinely unreadable screen (no
+#: display, revoked capture permission) still fails within seconds.
+_MAX_OBSERVE_FAILURES = 4
+#: Settle beat before re-observing, so the retry does not re-probe into the
+#: same churn. Scaled by ``settle_scale`` (0 in test/benchmark rigs).
+_OBSERVE_RETRY_BACKOFF_S = 0.4
+#: Steps with zero successful actions on an unchanged screen before the
+#: dead-end verdict. Deliberate waits are exempt and bounded separately.
_STUCK_FRAMES = 3
+#: Consecutive deliberate waits that changed nothing on screen before the wait
+#: itself counts as a dead end. Anything genuinely in progress — a loading
+#: page, a progress bar, an installer — repaints *something* well inside six
+#: look-agains, and any visible change resets this counter. Six is also the
+#: patience a human grants before concluding nothing is happening.
+_MAX_STALLED_WAITS = 6
+#: Total deliberate waiting per mission. Bounds patience against the mission
+#: ceiling ([computer_use].mission_timeout_s, default 600s) so waiting can
+#: never eat the whole budget: at most ~20% of it goes to waiting, the rest
+#: stays available for actual work. Once spent, waits shrink to a beat and
+#: stop being exempt from the no-progress guard.
+_MAX_TOTAL_WAIT_S = 120.0
+#: Length of a wait once the patience budget is spent.
+_WAIT_EXHAUSTED_MS = 250
_SYSTEM_BASE = (
"You are the computer-use executor: you operate the user's REAL desktop "
@@ -158,6 +184,14 @@ _SYSTEM_BASE = (
"search boxes) instead of appending.\n"
"* Use open_app to launch or focus an application. Use switch_window to "
"focus an open window.\n"
+ "* WAITING IS WORK, not stalling. While the screen is BUSY doing what you "
+ "already asked for — a page still loading, a spinner or progress bar, a "
+ "download running, an installer working, an app still starting — reply "
+ 'with {"action": "wait", "ms": <up to 10000>} and look again. Re-clicking '
+ "a control that is already working is what breaks these tasks. But wait "
+ "ONLY while something is visibly in progress: if several waits in a row "
+ "change nothing on screen, stop waiting and either act differently or "
+ "fail with a reason.\n"
"* done: ONLY when the goal's observable proof is visible in the CURRENT "
"screenshot; quote that proof in reason.\n"
"* fail: ONLY when the goal is genuinely impossible from here; explain "
@@ -837,6 +871,9 @@ async def run_cu_loop(
prev_thumb: bytes | None = None
fruitless_steps = 0 # steps with zero successful actions on an unchanged screen
last_step_had_success = False
+ last_step_waited = False # the previous step deliberately waited
+ stalled_waits = 0 # consecutive waits that changed nothing on screen
+ total_wait_s = 0.0 # deliberate waiting spent so far (patience budget)
# Normalize the work surface before the first frame and again after every
# focus change (open_app / switch_window): professional computer-use
# harnesses never pixel-ground on a small window floating in a big desktop
@@ -951,6 +988,11 @@ async def run_cu_loop(
exit_code=_EXIT_OBSERVE,
)
return
+ # A busy desktop is the common cause (the foreground flips while a
+ # page finishes loading or an installer opens its next window).
+ # Give it a beat instead of re-probing straight back into the same
+ # churn — an instant retry burns the whole budget in ~2 s (AU-16).
+ await asyncio.sleep(_OBSERVE_RETRY_BACKOFF_S * settle_scale)
continue
observe_failures = 0
window_title = _foreground_title()
@@ -973,17 +1015,28 @@ async def run_cu_loop(
# No-progress guard: several consecutive steps that produced no
# successful action while the screen never changed means we are
# circling — judge once (the goal may in fact be met), then stop.
- if (
- prev_thumb is not None
- and thumbs_similar(prev_thumb, frame.thumb)
- and not last_step_had_success
- ):
+ #
+ # A step that DELIBERATELY waited is exempt (AU-16). Waiting for a page
+ # to load, a download to finish or an installer to run looks EXACTLY
+ # like being stuck, so counting it here killed patient, correct work
+ # after three steps. Patience still has to terminate, so no-change
+ # waits accumulate on their own counter (and on the time budget spent
+ # in the wait action itself); any visible change resets both.
+ screen_unchanged = prev_thumb is not None and thumbs_similar(
+ prev_thumb, frame.thumb
+ )
+ if screen_unchanged and not last_step_had_success and not last_step_waited:
fruitless_steps += 1
else:
fruitless_steps = 0
+ if not screen_unchanged:
+ stalled_waits = 0
+ elif last_step_waited:
+ stalled_waits += 1
prev_thumb = frame.thumb
last_step_had_success = False
- if fruitless_steps >= _STUCK_FRAMES:
+ last_step_waited = False
+ if fruitless_steps >= _STUCK_FRAMES or stalled_waits >= _MAX_STALLED_WAITS:
done, proof = await _judge_done(
ctx,
goal,
@@ -1003,10 +1056,15 @@ async def run_cu_loop(
# "the unsubscribe link did not work" was never attempted —
# the mission died scrolling).
recent = "; ".join(h[:140] for h in history[-2:])
+ cause = (
+ f"I waited {stalled_waits} times in a row and nothing on "
+ "screen moved"
+ if stalled_waits >= _MAX_STALLED_WAITS
+ else "the screen has not changed despite my actions"
+ )
yield _final(
stderr=(
- f"[cu] fail at step-{step_idx}: no progress — the "
- "screen has not changed despite my actions."
+ f"[cu] fail at step-{step_idx}: no progress — {cause}."
+ (f" Last attempts: {recent}" if recent else "")
+ "\n"
),
@@ -1683,8 +1741,26 @@ async def run_cu_loop(
profiler.add("act", t0, step_idx)
elif kind == "wait":
- await asyncio.sleep(action["ms"] / 1000.0)
- ok, detail = True, f"waited {action['ms']} ms"
+ # Deliberate patience, bounded by a per-mission time budget.
+ # While budget remains the step is marked as a wait, which
+ # exempts it from the no-progress guard above; once the budget
+ # is spent a wait shrinks to a beat, loses the exemption and
+ # the model is told so in plain words.
+ remaining_wait_s = _MAX_TOTAL_WAIT_S - total_wait_s
+ if remaining_wait_s > 0:
+ wait_ms = min(int(action["ms"]), int(remaining_wait_s * 1000))
+ last_step_waited = True
+ detail = f"waited {wait_ms} ms"
+ else:
+ wait_ms = min(int(action["ms"]), _WAIT_EXHAUSTED_MS)
+ detail = (
+ f"waited {wait_ms} ms — the {_MAX_TOTAL_WAIT_S:.0f}s "
+ "patience budget of this mission is spent; act "
+ "differently now or stop with fail"
+ )
+ total_wait_s += wait_ms / 1000.0
+ await asyncio.sleep(wait_ms / 1000.0)
+ ok = True
profiler.add("settle", t0, step_idx)
# -- bookkeeping ---------------------------------------------------
@@ -1693,8 +1769,12 @@ async def run_cu_loop(
if ok:
if kind != "wait":
acted_since_capture = True
+ # A wait changed nothing in the world, so it must not
+ # claim "the screen moved because of me" — its exemption
+ # from the no-progress guard rides on last_step_waited,
+ # which is bounded, instead of on this unbounded reset.
+ last_step_had_success = True
consecutive_failures = 0
- last_step_had_success = True
history.append(
f"step {step_idx}: {summary} -> OK" + (f" ({detail[:120]})" if detail else ""),
)
diff --git a/jarvis/plugins/harness/computer_use.py b/jarvis/plugins/harness/computer_use.py
index 1028a69a63..b1f0e9932f 100644
--- a/jarvis/plugins/harness/computer_use.py
+++ b/jarvis/plugins/harness/computer_use.py
@@ -56,8 +56,9 @@ _CANCEL_HEARTBEAT_S = 0.12
# used to run concurrently (the tool only dedupes IDENTICAL goals) and raced
# each other's pointer moves and foreground guards. Every launch route (voice,
# LLM tool, scheduled task) funnels through this harness, so serializing here
-# covers them all. A queued mission waits honoring its own deadline and
-# cancellation; the wait is logged so "nothing happens" is diagnosable.
+# covers them all. A queued mission waits on a SEPARATE queue allowance (not on
+# its mission budget, AU-17) and stays cancellable throughout; the wait is
+# logged so "nothing happens" is diagnosable.
_DESKTOP_LOCK = asyncio.Lock()
@@ -153,7 +154,17 @@ class ComputerUseHarness:
"""
ctx = self._explicit_context or get_computer_use_context()
timeout_s = max(0.001, float(task.timeout_s))
- deadline = time.monotonic() + timeout_s
+ # AU-17: the budget bounds the WORK, not the queue. Waiting for the
+ # ONE desktop is not mission work, and charging it to the mission
+ # timeout let a queued mission die of a timeout without ever having
+ # started — with the 600 s ceiling of bdf15515 that is ten minutes of
+ # burning nothing and then reporting failure. So the queue gets its
+ # OWN allowance of the same size, and the work deadline below is armed
+ # only once the desktop is actually held. One equal allowance is
+ # exactly enough to outlast a full-length predecessor, because the
+ # active mission is bounded by the same ceiling.
+ queue_deadline = time.monotonic() + timeout_s
+ deadline = queue_deadline # re-armed to the full budget on acquire
t_start = time.time_ns()
# BUG-CU-HANGUP-RACE (2026-05-28): if the user said "auflegen" in the
# window between this mission being requested and it actually starting,
@@ -218,7 +229,7 @@ class ComputerUseHarness:
is_final=True,
)
return
- remaining_s = deadline - time.monotonic()
+ remaining_s = queue_deadline - time.monotonic()
if remaining_s <= 0:
end_reason = "timeout"
final_exit_code = _TIMEOUT_EXIT_CODE
@@ -239,6 +250,11 @@ class ComputerUseHarness:
lock_acquired = True
except TimeoutError:
continue
+ # The desktop is ours: the mission's own budget starts NOW,
+ # undiminished by however long the queue took (AU-17).
+ # ``t_start`` deliberately stays at invoke() entry — the
+ # reported duration is honest wall time, queue included.
+ deadline = time.monotonic() + timeout_s
# Control-indicator contract (jarvis.cu.indicator): Started
# fires the moment this mission may actually drive
# mouse/keyboard (token registered AND desktop lock held);
diff --git a/tests/unit/cu/test_engine_loop.py b/tests/unit/cu/test_engine_loop.py
index 3fb10d66b8..fc7a49d6d0 100644
--- a/tests/unit/cu/test_engine_loop.py
+++ b/tests/unit/cu/test_engine_loop.py
@@ -965,3 +965,137 @@ async def test_handoff_screen_fails_fast_with_speakable_reason(patched, monkeypa
assert final.exit_code == 5
assert "captcha challenge" in final.stderr
assert not brain.calls, "no model call may happen on a handoff screen"
+
+
+# ---------------------------------------------------------------------------
+# Patience vs. dead end (AU-16). Waiting for a page to load, a download to
+# finish or an installer to run looks EXACTLY like being stuck. A deliberate
+# wait is progress and must survive the no-progress guard; unbounded waiting
+# on a frozen screen must still die.
+# ---------------------------------------------------------------------------
+
+
+async def test_deliberate_wait_is_not_a_fruitless_step(patched):
+ # Five waits in a row on a screen that never changes — more than
+ # _STUCK_FRAMES. Before AU-16 the third one killed the mission as "no
+ # progress"; a mission that patiently waits out a load must reach its goal.
+ assert engine_mod._STUCK_FRAMES < 5, "the scenario must exceed the guard"
+ brain = FakeBrain(
+ ['{"action": "wait", "ms": 1}'] * 5
+ + [
+ '{"action": "done", "reason": "the page finished loading"}',
+ '{"done": true, "proof": "the page is loaded"}',
+ ]
+ )
+
+ chunks = await _run(_ctx(brain, FakeExecutor()))
+
+ final = _final(chunks)
+ assert final.exit_code == 0, f"patience was killed: {final.stderr!r}"
+ assert "no progress" not in final.stderr
+
+
+async def test_waiting_forever_on_a_frozen_screen_still_aborts(patched):
+ # The bound on patience: waits that change nothing accumulate, and once
+ # _MAX_STALLED_WAITS of them pass in a row the mission ends honestly
+ # instead of stalling until the mission timeout.
+ brain = FakeBrain(['{"action": "wait", "ms": 1}'] * 20)
+
+ chunks = await _run(_ctx(brain, FakeExecutor()))
+
+ final = _final(chunks)
+ assert final.exit_code == 5
+ assert "no progress" in final.stderr
+ assert "waited" in final.stderr, (
+ f"the abort must name waiting as the cause, got: {final.stderr!r}"
+ )
+ waits = [c for c in chunks if "wait(" in (c.stdout or "")]
+ assert len(waits) >= engine_mod._MAX_STALLED_WAITS, (
+ "the mission must actually be allowed to wait before giving up"
+ )
+
+
+async def test_repeated_no_op_actions_still_abort_as_a_dead_end(patched):
+ # The dead-end protection stays intact for genuinely fruitless steps: an
+ # action that keeps failing on an unchanged screen ends the mission.
+ brain = FakeBrain(['{"action": "click_element", "name": "Next"}'] * 8)
+ executor = FakeExecutor(failures={"click_element"})
+
+ chunks = await _run(_ctx(brain, executor))
+
+ final = _final(chunks)
+ assert final.exit_code in (5, 8), f"a dead end must abort: {final.stderr!r}"
+ assert len(executor.calls) <= engine_mod._STUCK_FRAMES + 2, (
+ "a dead end must be cut short, not retried indefinitely"
+ )
+
+
+async def test_spent_patience_budget_is_told_to_the_model(patched, monkeypatch):
+ # Patience is also bounded in TIME, so waiting can never eat the whole
+ # mission budget. Once spent, waits shrink to a beat and the model is told
+ # in plain words that it has to act instead.
+ monkeypatch.setattr(engine_mod, "_MAX_TOTAL_WAIT_S", 0.005)
+ brain = FakeBrain(['{"action": "wait", "ms": 10}'] * 5)
+
+ await _run(_ctx(brain, FakeExecutor()))
+
+ assert any("patience budget" in user for _system, user in brain.calls), (
+ "the model must learn that its waiting budget is spent"
+ )
+
+
+async def test_the_model_is_told_that_waiting_is_a_legitimate_move(patched):
+ # A capability nobody is told about is useless: both the discipline block
+ # and the action grammar must surface the wait action and when to use it.
+ brain = FakeBrain(['{"action": "fail", "reason": "stop"}'])
+
+ await _run(_ctx(brain, FakeExecutor()))
+
+ system = brain.calls[0][0]
+ assert '"action": "wait"' in system
+ assert "still loading" in system
+ assert "installer" in system
+
+
+async def test_a_busy_screen_survives_more_than_two_observe_failures(patched, monkeypatch):
+ # A page finishing its load or an installer opening its next window flips
+ # the foreground mid-capture, which trips the capture identity guard. At
+ # two tolerated failures a mission died after ~2 s of ordinary desktop
+ # churn (AU-16); transient blindness must be ridden out, not fatal.
+ real_capture = engine_mod.capture_stable_frame
+ state = {"n": 0}
+
+ def flaky_capture(monitor, **kw):
+ state["n"] += 1
+ if state["n"] <= 3:
+ raise RuntimeError("foreground window changed during capture")
+ return real_capture(monitor, **kw)
+
+ monkeypatch.setattr(engine_mod, "capture_stable_frame", flaky_capture)
+ brain = FakeBrain(
+ [
+ '{"action": "done", "reason": "the page finished loading"}',
+ '{"done": true, "proof": "the page is loaded"}',
+ ]
+ )
+
+ chunks = await _run(_ctx(brain, FakeExecutor()))
+
+ final = _final(chunks)
+ assert final.exit_code == 0, f"transient blindness was fatal: {final.stderr!r}"
+ assert state["n"] > 3, "the retries must actually have happened"
+
+
+async def test_a_screen_that_stays_unreadable_still_fails(patched, monkeypatch):
+ # The honest counterpart: a screen that never becomes readable ends the
+ # mission with the observe exit code instead of grinding to the budget.
+ def dead_capture(monitor, **kw):
+ raise RuntimeError("no display available")
+
+ monkeypatch.setattr(engine_mod, "capture_stable_frame", dead_capture)
+
+ chunks = await _run(_ctx(FakeBrain([]), FakeExecutor()))
+
+ final = _final(chunks)
+ assert final.exit_code == 1
+ assert "cannot see the screen" in final.stderr
diff --git a/tests/unit/harness/test_cu_desktop_lock.py b/tests/unit/harness/test_cu_desktop_lock.py
index 063d6f7c50..d7aaa47c28 100644
--- a/tests/unit/harness/test_cu_desktop_lock.py
+++ b/tests/unit/harness/test_cu_desktop_lock.py
@@ -149,3 +149,49 @@ async def test_queued_mission_times_out_honestly(
release.set()
await asyncio.wait_for(first_task, timeout=2.0)
+
+
+async def test_queued_mission_gets_its_full_budget_after_the_lock_frees(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The budget bounds the WORK, not the queue (AU-17).
+
+ The holder occupies the desktop for 0.5 s; the queued mission then needs
+ 0.6 s of work and carries a 1.0 s budget. Charging the queue wait to that
+ budget left it 0.5 s — less than its work — so it died of a timeout
+ without ever having started. Its budget must start when the desktop does.
+ """
+ release = asyncio.Event()
+ worked: list[str] = []
+
+ def loop_factory(task: HarnessTask, ctx, cancel_token=None) -> AsyncIterator[HarnessResult]:
+ async def run() -> AsyncIterator[HarnessResult]:
+ if task.prompt == "holder":
+ await release.wait()
+ else:
+ await asyncio.sleep(0.6) # work that outlasts the leftover
+ worked.append(task.prompt)
+ yield HarnessResult(stdout="ok", exit_code=0, is_final=True)
+ return run()
+
+ monkeypatch.setattr(cu_mod, "_resolve_run_cu_loop", lambda: loop_factory)
+ bus = EventBus()
+ ctx = _ctx(bus)
+ first = ComputerUseHarness(context=ctx)
+ queued = ComputerUseHarness(context=ctx)
+
+ async def drain(h: ComputerUseHarness, goal: str, timeout_s: float) -> list[HarnessResult]:
+ return [c async for c in h.invoke(HarnessTask(prompt=goal, timeout_s=timeout_s))]
+
+ first_task = asyncio.create_task(drain(first, "holder", 10))
+ await asyncio.sleep(0.05) # the holder owns the desktop
+ queued_task = asyncio.create_task(drain(queued, "queued", 1.0))
+ await asyncio.sleep(0.45) # the queued mission spends this in the queue
+ release.set()
+
+ chunks = await asyncio.wait_for(queued_task, timeout=5.0)
+ assert chunks[-1].exit_code == 0, (
+ f"the queued mission lost budget to the queue: {chunks[-1].stderr!r}"
+ )
+ assert worked == ["queued"]
+ await asyncio.wait_for(first_task, timeout=2.0)