Skip to content

Personal AI, On Personal Devices

Python

feat(vision): add image + screen capture input for vision models (#486)

Joshua Azmy79e23719d4b0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 83a375cc..a65e66a8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
## [Unreleased]
+### Added
+
+**Vision input for `jarvis ask`** — attach images to a query with
+`-i`/`--image` (repeatable) or capture the current screen with
+`-S`/`--screen`, for vision-capable models such as `gemma3:4b`. Images flow
+through `Message.images` into Ollama's `/api/chat` `images` field; text-only
+requests are unaffected. A privacy guard warns before any image is sent to a
+non-local engine, and the security guardrail now preserves images when it
+sanitizes a flagged prompt. Screen capture uses the built-in Windows .NET
+stack with `mss`/`Pillow` fallbacks on other platforms. Adds the
+`JARVIS_NUM_CTX` environment variable to tune the Ollama context window
+(default `16384`).
+
## [1.0.2] - 2026-05-24
A patch release that fixes a packaging bug which broke the v1.0.1
diff --git a/docs/user-guide/cli.md b/docs/user-guide/cli.md
index ab2d13d5..519c957c 100644
--- a/docs/user-guide/cli.md
+++ b/docs/user-guide/cli.md
@@ -66,6 +66,8 @@ jarvis ask "What is the capital of France?"
| `--no-context` | flag | off | Disable memory context injection |
| `-a`, `--agent AGENT` | string | none | Agent to use (`simple`, `orchestrator`) |
| `--tools TOOLS` | string | none | Comma-separated tool names to enable |
+| `-i`, `--image PATH` | path | none | Image file for a vision model (e.g. `gemma3:4b`); repeatable |
+| `-S`, `--screen` | flag | off | Capture the current screen and send it to the vision model |
### Direct Mode vs Agent Mode
@@ -105,6 +107,39 @@ jarvis ask --no-context "Tell me about Python"
jarvis ask --max-tokens 2048 "Write a detailed essay about AI"
```
+### Vision Input
+
+Vision-capable models (such as `gemma3:4b`) can read images alongside your
+text prompt. Attach one or more image files with `-i`/`--image`, or capture
+the current screen with `-S`/`--screen`:
+
+```bash
+# Ask about a local image
+jarvis ask -i screenshot.png "What is shown in this image?"
+
+# Send multiple images (the flag is repeatable)
+jarvis ask -i chart-a.png -i chart-b.png "Compare these two charts"
+
+# Capture the current screen and ask about it
+jarvis ask --screen "Summarize what's on my screen"
+```
+
+Vision runs in **direct mode** only. If you also pass `--agent`, the image is
+ignored and a note is printed — re-run with `--agent ""` to force direct mode.
+
+The Ollama context window can be tuned for large images or long prompts with
+the `JARVIS_NUM_CTX` environment variable (default `16384`):
+
+```bash
+JARVIS_NUM_CTX=8192 jarvis ask --screen "What's on my screen?"
+```
+
+!!! note "Keep vision on-device"
+ Images are sensitive. OpenJarvis prints a privacy warning before sending
+ an image to a non-local engine, so a screenshot never leaves your machine
+ unnoticed. Use a local engine (e.g. `ollama` with `gemma3:4b`) to keep
+ vision fully local.
+
### JSON Output Format
When using `--json` in **direct mode**, the output includes:
diff --git a/src/openjarvis/cli/_screen.py b/src/openjarvis/cli/_screen.py
new file mode 100644
index 00000000..e6b7dde5
--- /dev/null
+++ b/src/openjarvis/cli/_screen.py
@@ -0,0 +1,79 @@
+"""Screen capture for vision input (``jarvis ask --screen``).
+
+Captures the primary monitor to a temporary PNG so it can be handed to a
+vision-capable model. On Windows this uses the built-in .NET
+``System.Drawing`` stack (no third-party dependency). Other platforms fall
+back to ``mss`` or ``Pillow`` if installed.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import tempfile
+
+# PowerShell: capture the PRIMARY monitor (more legible for a vision model
+# than a downscaled multi-monitor grab). {path} is filled in with forward
+# slashes, which .NET accepts on Windows and which avoids backslash escaping.
+_PS_CAPTURE = """
+Add-Type -AssemblyName System.Windows.Forms, System.Drawing
+$b = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
+$bmp = New-Object System.Drawing.Bitmap($b.Width, $b.Height)
+$g = [System.Drawing.Graphics]::FromImage($bmp)
+$g.CopyFromScreen($b.X, $b.Y, 0, 0, $bmp.Size)
+$bmp.Save("{path}", [System.Drawing.Imaging.ImageFormat]::Png)
+$g.Dispose(); $bmp.Dispose()
+"""
+
+
+def capture_screen_to_temp() -> str:
+ """Capture the screen to a temp PNG and return its absolute path.
+
+ Raises ``RuntimeError`` with actionable guidance if capture fails or the
+ platform has no available backend.
+ """
+ fd, path = tempfile.mkstemp(prefix="jarvis_screen_", suffix=".png")
+ os.close(fd)
+
+ if sys.platform.startswith("win"):
+ script = _PS_CAPTURE.replace("{path}", path.replace("\\", "/"))
+ proc = subprocess.run(
+ ["powershell", "-NoProfile", "-NonInteractive", "-Command", script],
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ if (
+ proc.returncode != 0
+ or not os.path.exists(path)
+ or not os.path.getsize(path)
+ ):
+ raise RuntimeError(
+ "screen capture failed: "
+ + (proc.stderr.strip() or "empty image written")
+ )
+ return path
+
+ # Non-Windows: optional backends.
+ try:
+ import mss # type: ignore
+
+ with mss.mss() as sct:
+ sct.shot(mon=-1, output=path)
+ return path
+ except ImportError:
+ pass
+ try:
+ from PIL import ImageGrab # type: ignore
+
+ ImageGrab.grab().save(path)
+ return path
+ except Exception as exc: # noqa: BLE001
+ raise RuntimeError(
+ "screen capture on this platform needs 'mss' or 'Pillow' "
+ "(try: pip install mss)"
+ ) from exc
+
+
+__all__ = ["capture_screen_to_temp"]
diff --git a/src/openjarvis/cli/ask.py b/src/openjarvis/cli/ask.py
index 91804878..17b9ccbd 100644
--- a/src/openjarvis/cli/ask.py
+++ b/src/openjarvis/cli/ask.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import base64
import json as json_mod
import logging
import sys
@@ -619,6 +620,21 @@ def _print_profile(
"(default: ~/.openjarvis/knowledge.db)."
),
)
+ "-i",
+ "--image",
+ "image_paths",
+ multiple=True,
+ type=click.Path(exists=True, dir_okay=False),
+ help="Image file for a vision model (e.g. gemma3). Repeatable.",
+)
+ "-S",
+ "--screen",
+ "capture_screen",
+ is_flag=True,
+ help="Capture the current screen and send it to the vision model.",
+)
@click.option(
"--persona",
"persona_name",
@@ -645,6 +661,8 @@ def ask(
research_mode: bool,
knowledge_db: str | None,
persona_name: str | None,
+ image_paths: tuple[str, ...] = (),
+ capture_screen: bool = False,
) -> None:
"""Ask Jarvis a question."""
quiet = (ctx.obj or {}).get("quiet", False) or output_json
@@ -652,6 +670,27 @@ def ask(
console = Console(stderr=True)
query_text = " ".join(query)
+ # Vision: collect base64 images from --image files and/or --screen.
+ image_b64: list[str] = []
+ for _img_path in image_paths:
+ try:
+ with open(_img_path, "rb") as _fh:
+ image_b64.append(base64.b64encode(_fh.read()).decode("ascii"))
+ except OSError as exc:
+ console.print(f"[red]Could not read image {_img_path}: {exc}[/red]")
+ sys.exit(1)
+ if capture_screen:
+ try:
+ from openjarvis.cli._screen import capture_screen_to_temp
+
+ _shot = capture_screen_to_temp()
+ with open(_shot, "rb") as _fh:
+ image_b64.append(base64.b64encode(_fh.read()).decode("ascii"))
+ logger.debug("Captured screen to %s", _shot)
+ except Exception as exc: # noqa: BLE001
+ console.print(f"[red]Screen capture failed:[/red] {exc}")
+ sys.exit(1)
+
wall_start = time.monotonic() if enable_profile else None
# Load config
@@ -671,11 +710,26 @@ def ask(
# Without this fallback, `[agent].default_system_prompt` and the
# SOUL.md / MEMORY.md / USER.md persona system are silently bypassed for
# the most common command (`jarvis ask "..."`).
+ agent_explicitly_set = agent_name is not None
if agent_name is None:
configured_default = (config.agent.default_agent or "").strip()
if configured_default:
agent_name = configured_default
+ # Vision flows only through direct-to-engine mode. If an image/screenshot
+ # was supplied without an explicit --agent, route to direct mode so the
+ # picture reaches the model; if an agent was explicitly requested, say
+ # plainly that the image is being skipped rather than dropping it silently.
+ if image_b64:
+ if not agent_explicitly_set:
+ agent_name = ""
+ else:
+ console.print(
+ "[yellow]Note:[/yellow] --image/--screen only works in direct "
+ "mode; the image is ignored with --agent set. Re-run with "
+ '`--agent ""` to use vision.'
+ )
+
# Track whether the user explicitly set --max-tokens
user_set_max_tokens = max_tokens is not None
@@ -871,6 +925,27 @@ def ask(
return
# Direct-to-engine mode (no agent)
+ # Privacy guard: a screenshot/image is sensitive, and OpenJarvis is
+ # local-first. If the active engine isn't local, warn before the image
+ # leaves the machine rather than silently uploading it to a third party.
+ _LOCAL_ENGINES = {
+ "ollama",
+ "llamacpp",
+ "vllm",
+ "sglang",
+ "exo",
+ "nexa",
+ "uzu",
+ "apple_fm",
+ "gemma_cpp",
+ }
+ if image_b64 and engine_name not in _LOCAL_ENGINES:
+ console.print(
+ f"[yellow]Privacy warning:[/yellow] sending {len(image_b64)} "
+ f"image(s) to a non-local engine ('{engine_name}'). The image will "
+ "leave this machine. Use a local engine (e.g. ollama) to keep "
+ "vision on-device."
+ )
messages = [Message(role=Role.USER, content=query_text)]
# Memory-augmented context injection
@@ -897,6 +972,15 @@ def ask(
except Exception as exc:
logger.debug("Failed to inject memory context: %s", exc)
+ # Vision: attach images to the final user message *after* any context
+ # injection (which may rebuild the list). messages_to_dicts() forwards
+ # the "images" field to Ollama's /api/chat.
+ if image_b64:
+ for _m in reversed(messages):
+ if _m.role == Role.USER:
+ _m.images = image_b64
+ break
+
# Generate (InstrumentedEngine handles telemetry + energy recording)
try:
with console.status("[bold green]Generating...[/bold green]"):
diff --git a/src/openjarvis/core/types.py b/src/openjarvis/core/types.py
index 849a5269..62cdc9f4 100644
--- a/src/openjarvis/core/types.py
+++ b/src/openjarvis/core/types.py
@@ -68,6 +68,10 @@ class Message:
tool_calls: Optional[List[ToolCall]] = None
tool_call_id: Optional[str] = None
metadata: Dict[str, Any] = field(default_factory=dict)
+ # Base64-encoded image data for vision-capable models (e.g. gemma3,
+ # qwen2.5-vl). Forwarded to Ollama's /api/chat "images" field; None or
+ # empty for text-only messages (the common case).
+ images: Optional[List[str]] = None
@dataclass(slots=True)
diff --git a/src/openjarvis/engine/_base.py b/src/openjarvis/engine/_base.py
index 3b12f34f..abbad3e7 100644
--- a/src/openjarvis/engine/_base.py
+++ b/src/openjarvis/engine/_base.py
@@ -34,6 +34,10 @@ def messages_to_dicts(messages: Sequence[Message]) -> List[Dict[str, Any]]:
]
if m.tool_call_id:
d["tool_call_id"] = m.tool_call_id
+ # Vision: forward base64 images to the engine. Ollama's /api/chat
+ # accepts an "images" array on a message; text messages skip this.
+ if getattr(m, "images", None):
+ d["images"] = list(m.images)
out.append(d)
return out
diff --git a/src/openjarvis/engine/ollama.py b/src/openjarvis/engine/ollama.py
index 5b784120..8724cdcd 100644
--- a/src/openjarvis/engine/ollama.py
+++ b/src/openjarvis/engine/ollama.py
@@ -23,6 +23,19 @@ from openjarvis.engine._stubs import StreamChunk
logger = logging.getLogger(__name__)
+def _default_num_ctx() -> int:
+ """Default context window (tokens). Override with ``JARVIS_NUM_CTX``.
+
+ Raised above Ollama's 4k default so an image (which costs many tokens)
+ plus a real conversation fit. 16k is comfortable for small models on a
+ typical consumer GPU.
+ """
+ try:
+ return int(os.environ.get("JARVIS_NUM_CTX", "16384"))
+ except ValueError:
+ return 16384
+
+
@EngineRegistry.register("ollama")
class OllamaEngine(InferenceEngine):
"""Ollama backend via its native HTTP API."""
@@ -73,7 +86,7 @@ class OllamaEngine(InferenceEngine):
"options": {
"temperature": temperature,
"num_predict": max_tokens,
- "num_ctx": kwargs.get("num_ctx", 8192),
+ "num_ctx": kwargs.get("num_ctx", _default_num_ctx()),
},
}
# Disable extended thinking by default (Qwen3.5 etc.).
@@ -189,7 +202,7 @@ class OllamaEngine(InferenceEngine):
"options": {
"temperature": temperature,
"num_predict": max_tokens,
- "num_ctx": kwargs.get("num_ctx", 8192),
+ "num_ctx": kwargs.get("num_ctx", _default_num_ctx()),
},
}
# Mirror generate()'s default: disable extended thinking unless the
@@ -268,7 +281,7 @@ class OllamaEngine(InferenceEngine):
"options": {
"temperature": temperature,
"num_predict": max_tokens,
- "num_ctx": kwargs.get("num_ctx", 8192),
+ "num_ctx": kwargs.get("num_ctx", _default_num_ctx()),
},
}
if "think" not in kwargs:
diff --git a/src/openjarvis/security/guardrails.py b/src/openjarvis/security/guardrails.py
index 5d16883e..a7bdfe6e 100644
--- a/src/openjarvis/security/guardrails.py
+++ b/src/openjarvis/security/guardrails.py
@@ -192,6 +192,7 @@ class GuardrailsEngine(InferenceEngine):
tool_calls=msg.tool_calls,
tool_call_id=msg.tool_call_id,
metadata=msg.metadata,
+ images=msg.images,
)
messages = processed
diff --git a/tests/cli/test_ask_vision.py b/tests/cli/test_ask_vision.py
new file mode 100644
index 00000000..e66ec2e7
--- /dev/null
+++ b/tests/cli/test_ask_vision.py
@@ -0,0 +1,128 @@
+"""CLI-level regression tests for ``jarvis ask`` vision input.
+
+The unit tests in ``tests/test_vision.py`` cover the ``Message.images`` ->
+``messages_to_dicts`` serialization contract in isolation. These tests lock
+the *end-to-end CLI wiring*: that ``--image`` reads a file, base64-encodes it,
+attaches it to the final user ``Message``, and that the bytes actually reach
+``engine.generate()`` -- and that the local-first privacy guard fires only for
+non-local engines.
+"""
+
+from __future__ import annotations
+
+import base64
+import importlib
+from pathlib import Path
+from typing import Any
+
+from click.testing import CliRunner
+
+from openjarvis.cli import cli
+from openjarvis.core.config import JarvisConfig
+from openjarvis.core.types import Role
+
+# Import the module (not the Click command attribute) so we can monkeypatch
+# the names it looks up at call time.
+_ask_mod = importlib.import_module("openjarvis.cli.ask")
+
+# A minimal but valid 1x1 PNG so ``click.Path(exists=True)`` is satisfied and
+# the bytes are deterministic.
+_PNG_BYTES = base64.b64decode(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk"
+ "+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+)
+
+
+class _RecordingEngine:
+ """A fake engine that records the messages handed to ``generate()``."""
+
+ def __init__(self) -> None:
+ self.engine_id = "mock"
+ self.received: list[Any] = []
+
+ def health(self) -> bool:
+ return True
+
+ def list_models(self) -> list[str]:
+ return ["test-model"]
+
+ def generate(self, messages, *, model=None, **kwargs):
+ # Capture the exact Message objects the CLI built so the test can
+ # assert the image bytes reached the engine boundary.
+ self.received = list(messages)
+ return {
+ "content": "a 1x1 pixel",
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+ "model": "test-model",
+ "finish_reason": "stop",
+ }
+
+
+def _patch_ask(monkeypatch, tmp_path: Path, *, engine_name: str) -> _RecordingEngine:
+ """Wire ``jarvis ask`` to a recording engine reported under ``engine_name``."""
+ cfg = JarvisConfig()
+ cfg.telemetry.db_path = str(tmp_path / "telemetry.db")
+ # Keep memory context out of the picture so the user message we inspect is
+ # the one the CLI built directly from the query + image.
+ cfg.agent.context_from_memory = False
+ monkeypatch.setattr(_ask_mod, "load_config", lambda: cfg)
+
+ engine = _RecordingEngine()
+ monkeypatch.setattr(_ask_mod, "get_engine", lambda *a, **kw: (engine_name, engine))
+ monkeypatch.setattr(_ask_mod, "discover_engines", lambda c: [(engine_name, engine)])
+ monkeypatch.setattr(
+ _ask_mod, "discover_models", lambda e: {engine_name: ["test-model"]}
+ )
+ return engine
+
+
+def _write_png(tmp_path: Path) -> tuple[Path, str]:
+ img = tmp_path / "pixel.png"
+ img.write_bytes(_PNG_BYTES)
+ return img, base64.b64encode(_PNG_BYTES).decode("ascii")
+
+
+def test_image_reaches_engine_payload(monkeypatch, tmp_path: Path) -> None:
+ engine = _patch_ask(monkeypatch, tmp_path, engine_name="ollama")
+ img, expected_b64 = _write_png(tmp_path)
+
+ result = CliRunner().invoke(
+ cli,
+ ["ask", "-i", str(img), "--no-context", "--agent", "", "describe this"],
+ )
+
+ assert result.exit_code == 0, result.output
+ # The CLI must have routed to direct mode and called the engine.
+ assert engine.received, "engine.generate() was never called"
+ user_msgs = [m for m in engine.received if m.role == Role.USER]
+ assert user_msgs, "no USER message reached the engine"
+ assert user_msgs[-1].images == [expected_b64]
+
+
+def test_privacy_warning_for_non_local_engine(monkeypatch, tmp_path: Path) -> None:
+ engine = _patch_ask(monkeypatch, tmp_path, engine_name="openai")
+ img, expected_b64 = _write_png(tmp_path)
+
+ result = CliRunner().invoke(
+ cli,
+ ["ask", "-i", str(img), "--no-context", "--agent", "", "describe this"],
+ )
+
+ assert result.exit_code == 0, result.output
+ assert "Privacy warning" in result.output
+ # The warning is informational; the image must still be delivered.
+ user_msgs = [m for m in engine.received if m.role == Role.USER]
+ assert user_msgs and user_msgs[-1].images == [expected_b64]
+
+
+def test_no_privacy_warning_for_local_engine(monkeypatch, tmp_path: Path) -> None:
+ _patch_ask(monkeypatch, tmp_path, engine_name="ollama")
+ img, _ = _write_png(tmp_path)
+
+ result = CliRunner().invoke(
+ cli,
+ ["ask", "-i", str(img), "--no-context", "--agent", "", "describe this"],
+ )
+
+ assert result.exit_code == 0, result.output
+ assert "Privacy warning" not in result.output
diff --git a/tests/test_vision.py b/tests/test_vision.py
new file mode 100644
index 00000000..d20850b3
--- /dev/null
+++ b/tests/test_vision.py
@@ -0,0 +1,94 @@
+"""Tests for vision input support: ``Message.images`` -> Ollama payload.
+
+These cover the data-flow contract that makes vision work end to end:
+a ``Message`` can carry base64 images, the engine serializer forwards them
+to Ollama's ``/api/chat`` ``images`` field, and text-only messages are
+completely unaffected. The security guardrail must preserve images when it
+rewrites a flagged message.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import openjarvis.engine.ollama as ollama_mod
+from openjarvis.core.types import Message, Role
+from openjarvis.engine._base import messages_to_dicts
+
+
+def test_message_defaults_to_no_images() -> None:
+ assert Message(role=Role.USER, content="hi").images is None
+
+
+def test_messages_to_dicts_omits_images_for_text() -> None:
+ dicts = messages_to_dicts([Message(role=Role.USER, content="hi")])
+ assert "images" not in dicts[0]
+
+
+def test_messages_to_dicts_forwards_images() -> None:
+ b64 = "aGVsbG8=" # "hello"
+ dicts = messages_to_dicts(
+ [Message(role=Role.USER, content="what is this?", images=[b64])]
+ )
+ assert dicts[0]["role"] == "user"
+ assert dicts[0]["content"] == "what is this?"
+ assert dicts[0]["images"] == [b64]
+
+
+def test_messages_to_dicts_empty_images_treated_as_text() -> None:
+ dicts = messages_to_dicts([Message(role=Role.USER, content="hi", images=[])])
+ assert "images" not in dicts[0]
+
+
+def test_default_num_ctx_default_and_override(monkeypatch) -> None:
+ monkeypatch.delenv("JARVIS_NUM_CTX", raising=False)
+ assert ollama_mod._default_num_ctx() == 16384
+
+ monkeypatch.setenv("JARVIS_NUM_CTX", "8000")
+ assert ollama_mod._default_num_ctx() == 8000
+
+ # A non-integer override must fall back to the safe default, not crash.
+ monkeypatch.setenv("JARVIS_NUM_CTX", "not-an-int")
+ assert ollama_mod._default_num_ctx() == 16384
+
+
+def test_guardrails_preserves_images_when_sanitizing() -> None:
+ """A flagged message gets rewritten; its image must survive the rewrite."""
+ from openjarvis.security.guardrails import GuardrailsEngine
+
+ class _RecordingEngine:
+ """Captures the messages the guardrail forwards to the real engine."""
+
+ def __init__(self) -> None:
+ self.received: list[Message] = []
+
+ def generate(self, messages, *, model, **kwargs):
+ self.received = list(messages)
+ return {"content": "ok"}
+
+ class _AlwaysFlag:
+ """A scanner that flags everything, forcing the sanitize rewrite path."""
+
+ def scan(self, text: str):
+ finding = SimpleNamespace(
+ pattern_name="test",
+ threat_level=SimpleNamespace(value="low"),
+ description="always flags",
+ )
+ return SimpleNamespace(findings=[finding])
+
+ def redact(self, text: str) -> str:
+ return text
+
+ engine = _RecordingEngine()
+ guarded = GuardrailsEngine(
+ engine,
+ scanners=[_AlwaysFlag()],
+ scan_input=True,
+ scan_output=False,
+ )
+ msg = Message(role=Role.USER, content="suspicious", images=["aGVsbG8="])
+
+ guarded.generate([msg], model="x")
+
+ assert engine.received[0].images == ["aGVsbG8="]