Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(live): give subscription-call appshots a forwardable image reference

rubenluetke10-beep877331e83b18
diff --git a/docs/agentic-ide-visual-handoffs.md b/docs/agentic-ide-visual-handoffs.md
index 2b901e0ee7..d4f90de2a9 100644
--- a/docs/agentic-ide-visual-handoffs.md
+++ b/docs/agentic-ide-visual-handoffs.md
@@ -16,11 +16,16 @@ creating a pane or sending a task. No global latest-appshot lookup is used.
- A Jarvis chat upload registers the bytes of the images selected on that
message. Paths must resolve inside that chat's working folder. Missing,
oversized, unsupported or escaped files produce an unavailable notice.
-- Appshots attached directly to a live call belong to that call's scope.
+- Appshots attached directly to a live call belong to that call's scope, in
+ every live mode (backend thinking, native, and subscription). The ID rides
+ beside the pixels wherever the call keeps them in model context, including
+ a capture's retained snapshot after its tool output left the history.
Chat, live-call and ordinary conversation scopes cannot resolve each other's
- IDs. Closing a live call clears its capabilities. Captures retain their
- configured TTL; other temporary references expire after 120 seconds.
- One active timer removes expired bytes even when there are no later calls.
+ IDs. A live-call reference lives as long as the call keeps the picture in
+ context: closing the call clears it, and a one-hour bound expires a scope
+ whose call never closed. Other captures retain their configured TTL; other
+ temporary references expire after 120 seconds. One active timer removes
+ expired bytes even when there are no later calls.
The existing ToolExecutor authorization, capture denylist/redaction, coding
pane permissions and prompt readiness checks remain in force. Pixels are
@@ -62,8 +67,8 @@ retries cannot silently replace or duplicate an attachment.
## Verification
-`tests/contract/test_coding_image_handoff.py` exercises upload/appshot sources
-with new/existing sessions through the real workspace orchestrator, coding
+`tests/contract/test_coding_image_handoff.py` exercises upload, appshot-tool and
+live-call appshot sources with new/existing sessions through the real workspace orchestrator, coding
gateway and registry into a fake PTY. It opens the delivered PNG, compares bytes,
checks the typed file path, receipt and retry behavior, and covers multiple images,
unrelated images, expiry, conversation isolation, invalid images, path traversal,
@@ -81,6 +86,6 @@ to a user's remote host, or prove that an actual vendor model used its image too
| Scoped references and expiry | `jarvis/core/image_references.py` |
| Upload intake | `jarvis/agent_chat/attachments.py`, `service.py` |
| Conversation and tool routing | `jarvis/brain/manager.py`, `dispatcher.py`, `tool_use_loop.py`, `workspace_tool.py` |
-| Captures and live calls | `jarvis/plugins/tool/appshot.py`, `jarvis/live/tools.py`, `session.py`, `native.py` |
+| Captures and live calls | `jarvis/plugins/tool/appshot.py`, `jarvis/live/tools.py`, `session.py`, `native.py`, `subscription.py` |
| Work order and CLI delivery | `jarvis/agentic_ide/orchestration.py`, `control.py`, `session.py`, `visual_handoff.py` |
-| Regression evidence | `tests/contract/test_coding_image_handoff.py`, `tests/fakes/fake_visual_sftp.py`, `tests/unit/appshot/test_appshot_surfaces.py` |
+| Regression evidence | `tests/contract/test_coding_image_handoff.py`, `tests/fakes/fake_visual_sftp.py`, `tests/unit/appshot/test_appshot_surfaces.py`, `tests/unit/live/test_subscription_session.py` |
diff --git a/jarvis/core/image_references.py b/jarvis/core/image_references.py
index 7231e5a701..42ed09a5d5 100644
--- a/jarvis/core/image_references.py
+++ b/jarvis/core/image_references.py
@@ -17,6 +17,11 @@ from uuid import uuid4
MAX_IMAGES = 20
MAX_BYTES = 25 * 1024 * 1024
TTL_SECONDS = 120.0
+#: A picture handed to a live call stays in that call's model context until the
+#: call ends, so its reference must live as long or the model sees an image it
+#: cannot forward. The call's close clears the scope; this bound only expires a
+#: scope whose call never closed cleanly.
+LIVE_CALL_TTL_SECONDS = 3600.0
active_scope: ContextVar[str] = ContextVar("visual_reference_scope", default="")
_EXTENSIONS = {
"image/png": ".png",
@@ -60,9 +65,15 @@ def scope_for(config: dict | None = None, trace_id: object = "") -> str:
)
+def ttl_for(scope: str, config: object = None) -> float:
+ """Reference lifetime: the capture budget, or the whole call for a live scope."""
+ ttl = float(getattr(getattr(config, "screen_context", None), "ttl_s", TTL_SECONDS))
+ return max(ttl, LIVE_CALL_TTL_SECONDS) if scope.startswith("live:") else ttl
+
+
def appshot_context(session_id: str, image: bytes, mime: str, config: object) -> str:
- ttl = getattr(getattr(config, "screen_context", None), "ttl_s", TTL_SECONDS)
- ref = get_store().add("live:" + session_id, image, mime, source="appshot", ttl_s=float(ttl))
+ scope = "live:" + session_id
+ ref = get_store().add(scope, image, mime, source="appshot", ttl_s=ttl_for(scope, config))
return instruction([ref])
diff --git a/jarvis/live/subscription.py b/jarvis/live/subscription.py
index baf0701ba5..f1689c3a35 100644
--- a/jarvis/live/subscription.py
+++ b/jarvis/live/subscription.py
@@ -42,7 +42,8 @@ def _is_computer_call(call: dict) -> bool:
if name == "call_tool":
try:
name = str(json.loads(call.get("arguments") or "{}").get("name") or "")
- except (ValueError, AttributeError): # Malformed tool arguments cannot activate the extended computer budget.
+ except (ValueError, AttributeError):
+ # Malformed tool arguments cannot activate the extended computer budget.
return False
return name.rsplit(":", 1)[-1] == "computer"
@@ -584,6 +585,14 @@ class SubscriptionLiveVoiceSession(LiveVoiceSession):
}
for image in images
]
+ # Keep the reference beside its pixels: the tool
+ # output carrying it is evicted with old history.
+ output = result.get("output")
+ handoff = isinstance(output, dict) and output.get("handoff")
+ if isinstance(handoff, str) and handoff:
+ self._image_context.insert(
+ 0, {"type": "input_text", "text": handoff}
+ )
image_item = {"role": "user", "content": self._image_context}
items.append(image_item)
screenshot_items.append(image_item)
@@ -755,8 +764,17 @@ class SubscriptionLiveVoiceSession(LiveVoiceSession):
async def attach_appshot(self, image: bytes, mime: str, note: str) -> bool:
import base64
+ from jarvis.core.image_references import ImageReferenceError, appshot_context
+
if self._closing:
return False
+ # The backend sees these pixels for the rest of the call; without a
+ # scoped reference beside them it could not forward them to a coding
+ # session and would ask the user to attach an image it already has.
+ try:
+ note += "\n\n" + appshot_context(self.session_id, image, mime, self._config)
+ except ImageReferenceError as exc:
+ note += "\n\nVisual handoff unavailable: " + str(exc)
self._pending_images = [
{"type": "input_text", "text": note},
{
diff --git a/jarvis/plugins/tool/appshot.py b/jarvis/plugins/tool/appshot.py
index ec5dca0f57..0f52eaf624 100644
--- a/jarvis/plugins/tool/appshot.py
+++ b/jarvis/plugins/tool/appshot.py
@@ -134,13 +134,12 @@ async def _screen_appshot(trace_id, ctx=None):
def _picture(shot, ctx=None, config=None):
import base64
- from jarvis.core.image_references import get_store, instruction, scope_for
+ from jarvis.core.image_references import get_store, instruction, scope_for, ttl_for
from jarvis.core.protocols import ToolResult
+ scope = scope_for(getattr(ctx, "config", None), getattr(ctx, "trace_id", ""))
ref = get_store().add(
- scope_for(getattr(ctx, "config", None), getattr(ctx, "trace_id", "")),
- shot.image, shot.mime, source="appshot",
- ttl_s=float(config.screen_context.ttl_s) if config is not None else 120,
+ scope, shot.image, shot.mime, source="appshot", ttl_s=ttl_for(scope, config),
)
return ToolResult(
diff --git a/tests/contract/test_coding_image_handoff.py b/tests/contract/test_coding_image_handoff.py
index 57c168a99d..ccdbae4ed5 100644
--- a/tests/contract/test_coding_image_handoff.py
+++ b/tests/contract/test_coding_image_handoff.py
@@ -73,7 +73,7 @@ def delivery(rig, runnable, monkeypatch, tmp_path): # noqa: F811
return WorkspaceOrchestrationTool(orchestrator), registry, ctx
-async def source_ref(source, delivery, monkeypatch, tmp_path):
+async def source_ref(source, delivery, monkeypatch, tmp_path, image_store=None):
tool, registry, ctx = delivery
data = png()
if source == "upload":
@@ -106,6 +106,32 @@ async def source_ref(source, delivery, monkeypatch, tmp_path):
ref = re.search(r"img_[0-9a-f]{32}", note)[0]
# Original upload may disappear; the selected bytes remain bound.
(tmp_path / dereference(attached[0].reference)).unlink()
+ elif source == "voice":
+ # A hotkey appshot supplied into a subscription voice call: the backend
+ # sees the pixels for the whole call and must be able to forward them.
+ from jarvis.live.config import LiveConfig
+ from jarvis.live.subscription import SubscriptionLiveVoiceSession
+ from tests.fakes.fake_subscription_session import (
+ SubscriptionConnection,
+ subscription_provider,
+ )
+
+ async def send(_value):
+ pass # No client socket in this contract.
+
+ session = SubscriptionLiveVoiceSession(
+ session_id="call-A", providers=[subscription_provider()],
+ config=SimpleNamespace(
+ live=LiveConfig(configured=True, auth_mode="chatgpt_subscription"),
+ brain=SimpleNamespace(reply_language="en"),
+ screen_context=SimpleNamespace(ttl_s=120),
+ ),
+ send_binary=send, send_json=send,
+ )
+ session._connection = SubscriptionConnection()
+ assert await session.attach_appshot(data, "image/png", "Supplied appshot")
+ ref = re.search(r"img_[0-9a-f]{32}", session._pending_images[0]["text"])[0]
+ image_store[1][0] += 600 # The user keeps talking before asking for the fix.
else:
from jarvis.appshot import service
from jarvis.plugins.tool import appshot
@@ -148,12 +174,12 @@ async def assignment(delivery, ctx, action, refs, *, prompt="Fix the pictured bu
return await tool.execute(args, ctx), args
[email protected]("source", ["upload", "appshot"])
[email protected]("source", ["upload", "appshot", "voice"])
@pytest.mark.parametrize("action", ["create", "send"])
async def test_actual_pixels_reach_new_and_existing_cli(
- source, action, delivery, monkeypatch, tmp_path
+ source, action, delivery, monkeypatch, tmp_path, image_store
):
- ref, data, ctx = await source_ref(source, delivery, monkeypatch, tmp_path)
+ ref, data, ctx = await source_ref(source, delivery, monkeypatch, tmp_path, image_store)
# A screenshot from a previous topic must not be included by recency.
old = image_references.get_store().add(
image_references.scope_for(ctx.config), png("blue"), "image/png", source="unrelated"
@@ -219,6 +245,25 @@ async def test_unavailable_image_refuses_whole_task(failure, delivery, image_sto
assert result.output["status"] == "not_accepted"
[email protected]("ending", ["call_closed", "call_bound"])
+async def test_live_call_reference_ends_with_the_call(ending, delivery, image_store):
+ from jarvis.live.tools import LiveTools
+
+ store, now = image_store
+ _, registry, ctx = delivery
+ image_references.appshot_context(
+ "call-A", png(), "image/png", SimpleNamespace(screen_context=SimpleNamespace(ttl_s=120)),
+ )
+ [row] = store.available("live:call-A")
+ if ending == "call_closed":
+ await LiveTools(None, None, "call-A", language="en", backend_model="").close()
+ else:
+ now[0] = image_references.LIVE_CALL_TTL_SECONDS + 1
+ result, _ = await assignment(delivery, ctx, "send", [row["id"]])
+ assert not result.success and result.output["status"] == "not_accepted"
+ assert not registry._pty.writes
+
+
async def test_expired_create_does_not_open_empty_pane(delivery, image_store):
store, now = image_store
_, registry, ctx = delivery
diff --git a/tests/fakes/fake_subscription_session.py b/tests/fakes/fake_subscription_session.py
index 1b5c5979bd..bddeeee830 100644
--- a/tests/fakes/fake_subscription_session.py
+++ b/tests/fakes/fake_subscription_session.py
@@ -37,9 +37,10 @@ class SubscriptionGateway:
class AppshotSubscriptionGateway(SubscriptionGateway):
"""A newly requested capture returns a different image, or an explicit refusal."""
- def __init__(self, *, refused=False):
+ def __init__(self, *, refused=False, handoff=None):
super().__init__()
self.refused = refused
+ self.handoff = handoff
def catalog(self):
from jarvis.plugins.tool.appshot import AppshotTool
@@ -52,10 +53,13 @@ class AppshotSubscriptionGateway(SubscriptionGateway):
self.calls.append((name, args, operation_model.get()))
if self.refused:
return ToolResult(False, None, "Blocked by the privacy filter.")
- return ToolResult(True, {
+ output = {
"description": "A newly captured window",
"_image": {"mime": "image/png", "data": "TkVX"},
- })
+ }
+ if self.handoff:
+ output["handoff"] = self.handoff
+ return ToolResult(True, output)
class SubscriptionConnection:
diff --git a/tests/unit/live/test_subscription_session.py b/tests/unit/live/test_subscription_session.py
index 0a0ad3f2f7..8e08b7242a 100644
--- a/tests/unit/live/test_subscription_session.py
+++ b/tests/unit/live/test_subscription_session.py
@@ -385,6 +385,76 @@ async def test_existing_appshot_stays_available_for_a_followup_without_recapture
assert gateway.calls == []
+def image_store(monkeypatch):
+ from jarvis.core import image_references
+
+ now = [0.0]
+ store = image_references.ImageReferences(clock=lambda: now[0], schedule_expiry=False)
+ monkeypatch.setattr(image_references, "_STORE", store)
+ return store, now
+
+
+async def test_supplied_appshot_carries_a_forwardable_reference_for_the_call(
+ make_session, image_store,
+):
+ import re
+
+ from jarvis.core.image_references import ImageReferenceError
+
+ store, now = image_store
+ session, reasoning, _, _, _, _ = make_session([
+ completed_response("r1", [spoken_result("I can see the bug.")]),
+ completed_response("r2", [spoken_result("Starting a coding session.")]),
+ ])
+ await session.attach_appshot(b"\x89PNG-shot", "image/png", "A supplied screenshot")
+ scope = "live:" + session.session_id
+ for index, request in enumerate(("What is wrong here?", "Start a session to fix it.")):
+ await session._delegate(f"image-{index}", request)
+ sent = json.dumps(reasoning.requests[index]["input"])
+ ref = re.search(r"img_[0-9a-f]{32}", sent)[0]
+ assert "image_refs" in sent
+ # Talking past the capture budget must not orphan a picture still in context.
+ now[0] += 600
+ assert store.resolve(scope, [ref])[0].data == b"\x89PNG-shot"
+ assert [row["source"] for row in store.available(scope)] == ["appshot"]
+ with pytest.raises(ImageReferenceError):
+ store.resolve("live:another-call", [ref])
+ await session._tools.close()
+ with pytest.raises(ImageReferenceError):
+ store.resolve(scope, [ref])
+
+
+async def test_unforwardable_appshot_is_reported_instead_of_given_an_invented_reference(
+ make_session, image_store,
+):
+ session, _, _, _, _, _ = make_session([])
+ await session.attach_appshot(b"", "image/png", "An empty capture")
+ note = session._pending_images[0]["text"]
+ assert "Visual handoff unavailable" in note and "img_" not in note
+ assert image_store[0].available("live:" + session.session_id) == []
+
+
+async def test_tool_captured_appshot_keeps_its_reference_beside_the_pixels(make_session):
+ capture = {
+ "id": "fresh-shot", "type": "function_call", "call_id": "capture-1",
+ "name": "take_appshot", "arguments": '{"scope":"window"}',
+ }
+ handoff = "Visual reference IDs (same order as the attached images): img_fresh."
+ session, _, _, _, _, _ = make_session([
+ completed_response("r1", [capture]),
+ completed_response("r2", [spoken_result("Captured.")]),
+ ], gateway=AppshotSubscriptionGateway(handoff=handoff))
+ await session._delegate("fresh-request", "Take a new appshot.")
+ assert session._image_context == [
+ {"type": "input_text", "text": handoff},
+ {"type": "input_image", "image_url": "data:image/png;base64,TkVX"},
+ ]
+
+
@pytest.mark.asyncio
async def test_completion_report_uses_subscription_reasoning_without_new_tool_authority(
make_session,