Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let unsloth start agents send images to a vision model (#17540)
Nilayde45d3f2e28e
diff --git a/unsloth_cli/commands/start.py b/unsloth_cli/commands/start.pyindex 08bed037c65..a07d3365420 100644--- a/unsloth_cli/commands/start.py+++ b/unsloth_cli/commands/start.py@@ -5551,9 +5551,12 @@ def _openclaw_provider(key: str,model: dict,max_tokens: Optional[int] = None,+ vision: bool = False,) -> dict:# Unsloth is a generic OpenAI-compatible /v1 endpoint (the vLLM/LM Studio path).provider_model = {"id": model["id"], "name": model["id"]}+ if vision:+ provider_model["input"] = ["text", "image"]window = model.get("context_length") or model.get("max_context_length")if window:window = int(window)@@ -5593,6 +5596,7 @@ def write_openclaw_config(embedding_model: Optional[str] = None,request_body: Optional[dict] = None,max_tokens: Optional[int] = None,+ vision: bool = False,) -> None:config = _read_json_object(path)if config is None:@@ -5605,7 +5609,9 @@ def write_openclaw_config(before = json.dumps(config, sort_keys = True)models = _subdict(config, "models")models.setdefault("mode", "merge")- _subdict(models, "providers")["unsloth"] = _openclaw_provider(base, key, model, max_tokens)+ _subdict(models, "providers")["unsloth"] = _openclaw_provider(+ base, key, model, max_tokens, vision+ )# Memory search is on by default and defaults to openai, so a local session reaches OpenAI unless this block is written.search = _subdict(_subdict(config, "memory"), "search")if embedding_model:@@ -5934,8 +5940,12 @@ def _opencode_provider(model: dict,max_tokens: Optional[int] = None,request_body: Optional[dict] = None,+ vision: bool = False,) -> dict:model_entry = {"name": model["id"]}+ if vision:+ model_entry["attachment"] = True+ model_entry["modalities"] = {"input": ["text", "image"], "output": ["text"]}window = model.get("context_length") or model.get("max_context_length")if window:window = int(window)@@ -5969,6 +5979,7 @@ def write_opencode_config(as_subagent: bool = False,max_tokens: Optional[int] = None,request_body: Optional[dict] = None,+ vision: bool = False,) -> dict:config = _read_json_object(path)if config is None:@@ -5987,7 +5998,7 @@ def write_opencode_config(reserved = opencode_compaction_reserved(window, opencode_output_limit(window, max_tokens))# Keep the provider definition in this private session file. The launch path adjusts effective provider filters in the higher-priority inline overlay._subdict(config, "provider")[_OPENCODE_PROVIDER] = _opencode_provider(- base, key, model, max_tokens, request_body+ base, key, model, max_tokens, request_body, vision)# Normal mode pins this as the session model. Subagent mode leaves the user's main/small models alone and exposes the local model to @unsloth and /models.opencode_model = f"{_OPENCODE_PROVIDER}/{model['id']}"@@ -6251,9 +6262,12 @@ def _vibe_env(base: str,model: dict,request_body: Optional[dict] = None,+ vision: bool = False,) -> dict:"""VIBE_* env vars outrank config.toml, so the user's Vibe config is never written."""entry = {"name": model["id"], "provider": _VIBE_PROVIDER, "alias": _VIBE_MODEL_ALIAS}+ if vision:+ entry["supports_images"] = True# Vibe sends its own temperature (0.2 by default) with every request.temperature = (request_body or {}).get("temperature")if temperature is not None:@@ -6294,6 +6308,7 @@ def write_pi_config(*,max_tokens: Optional[int] = None,request_body: Optional[dict] = None,+ vision: bool = False,) -> None:config = _read_json_object(path)if config is None:@@ -6306,6 +6321,8 @@ def write_pi_config(before = json.dumps(config, sort_keys = True)# Pi reads custom providers from $PI_CODING_AGENT_DIR/models.json; the key lives there, not in the env, matching openclaw/opencode.provider_model = {"id": model["id"]}+ if vision:+ provider_model["input"] = ["text", "image"]window = model.get("context_length") or model.get("max_context_length")if window:window = int(window)@@ -6545,6 +6562,7 @@ def write_pi_subagent_config(approve: bool = False,max_tokens: Optional[int] = None,request_body: Optional[dict] = None,+ vision: bool = False,) -> None:"""Write private bootstrap data for the bundled Pi extension."""window = model.get("context_length") or model.get("max_context_length")@@ -6559,6 +6577,7 @@ def write_pi_subagent_config("maxTokens": _agent_output_limit(window, max_tokens),"approve": approve,**({"samplingParams": request_body} if request_body else {}),+ **({"vision": True} if vision else {}),},)@@ -6568,12 +6587,15 @@ def write_dsh_patch(model: dict,path: Path,request_body: Optional[dict] = None,+ vision: bool = False,) -> None:"""dsh 0.1.7 dropped settings.yaml, which only applies after a first boot still on the DeepSeekdefault; the --patch overlay is read at boot and rewritten whole, as it is Unsloth's own."""import yamlmodel_entry = {"id": model["id"]}+ if vision:+ model_entry["input"] = ["text", "image"]window = model.get("context_length") or model.get("max_context_length")if window:window = int(window)@@ -7219,7 +7241,9 @@ def _opencode_app_target(path = next((root / n for n in names if os.path.lexists(root / n)), root / "opencode.json")def updates(text, path, base, key, entry):- provider = _opencode_provider(base, key, entry, max_tokens, request_body)+ provider = _opencode_provider(+ base, key, entry, max_tokens, request_body, _accepts_images(base, key, entry)+ )changes = [(("provider", _OPENCODE_PROVIDER), provider)]config = _parse_app_config(path, text) or {}enabled = config.get("enabled_providers")@@ -7249,9 +7273,8 @@ def _openclaw_app_updates(defaults = agents.get("defaults") if isinstance(agents, dict) else Nonedefaults = defaults if isinstance(defaults, dict) else {}ref = f"unsloth/{entry['id']}"- changes = [- (("models", "providers", "unsloth"), _openclaw_provider(base, key, entry, max_tokens))- ]+ provider = _openclaw_provider(base, key, entry, max_tokens, _accepts_images(base, key, entry))+ changes = [(("models", "providers", "unsloth"), provider)]# The picker only offers allowlisted models once an allowlist exists; never create one.policy = defaults.get("modelPolicy")allowed = policy.get("allow") if isinstance(policy, dict) else None@@ -7495,6 +7518,10 @@ def _vscode_capabilities(base: str, key: str, entry: dict) -> tuple:)+def _accepts_images(base: str, key: str, entry: dict) -> bool:+ return _vscode_capabilities(base, key, entry)[1]++def _container_engine() -> Optional[str]:# `docker run unsloth/unsloth` sets none of the env vars below, only these markers.if os.path.exists("/.dockerenv"):@@ -8071,6 +8098,7 @@ def openclaw(embedding_model = _studio_embedding_model(base, key),request_body = server_options.request_body(),max_tokens = max_tokens,+ vision = _accepts_images(base, key, entry),)# Scope config and state so OpenClaw never touches ~/.openclaw. Shell env off, else it re-imports keys.env = {@@ -8233,6 +8261,7 @@ def opencode(as_subagent = True,max_tokens = max_tokens,request_body = server_options.request_body(),+ vision = _accepts_images(base, key, entry),)env = {"OPENCODE_CONFIG": str(config_path),@@ -8314,6 +8343,7 @@ def opencode(yolo = yolo and not native_auto,max_tokens = max_tokens,request_body = server_options.request_body(),+ vision = _accepts_images(base, key, entry),)# A project's own opencode.json outranks OPENCODE_CONFIG, so the session model pin would silently lose to a repo config; carry it in OPENCODE_CONFIG_CONTENT, which outranks project config, while the API key stays in the private file. Only the config fallback carries a permission: native --auto omits it (auto-approve asks, keep explicit denies) and a non-yolo session omits it too, honoring project rules. V1 filters are ordinary overlays, so scope that session to our provider; V2 turns filters into security policies where global/project rules intentionally win, so keep those policies intact and tell the user above that they must allow our provider. small_model is opencode's separate model for lightweight tasks; pin it to the session model too, or a user/project small_model on another (now filtered) provider would resolve a not-found error mid-session.inline_config: dict = {@@ -8502,6 +8532,7 @@ def pi(approve = yolo,max_tokens = max_tokens,request_body = server_options.request_body(),+ vision = _accepts_images(base, key, entry),)command = ["pi",@@ -8545,6 +8576,7 @@ def pi(pi_agent_dir / "models.json",max_tokens = max_tokens,request_body = server_options.request_body(),+ vision = _accepts_images(base, key, entry),)write_pi_user_resources(pi_agent_dir)env = {"PI_CODING_AGENT_DIR": str(pi_agent_dir)}@@ -8627,7 +8659,9 @@ def dsh()with _session_config("dsh", launch, persist = persist) as home:patch = home / _DSH_PATCH_FILE- write_dsh_patch(base, entry, patch, server_options.request_body())+ write_dsh_patch(+ base, entry, patch, server_options.request_body(), _accepts_images(base, key, entry)+ )# A Windows dsh under WSL gets DSH_HOME translated through WSLENV, but not argv.command = _dsh_command(ctx.args, _agent_config_path(patch, ["dsh"]))env = {@@ -8710,7 +8744,10 @@ def vibe(if status_id):request_body = {**request_body, "temperature": recommended}- env = {_VIBE_ENV_KEY: key, **_vibe_env(base, entry, request_body)}+ env = {+ _VIBE_ENV_KEY: key,+ **_vibe_env(base, entry, request_body, _accepts_images(base, key, entry)),+ }_run(base, entry, env, command, launch = launch, install_hint = install_hint)diff --git a/unsloth_cli/pi_subagent.ts b/unsloth_cli/pi_subagent.tsindex 889d30d6c8d..c4c5d399c8b 100644--- a/unsloth_cli/pi_subagent.ts+++ b/unsloth_cli/pi_subagent.ts@@ -31,6 +31,7 @@ const approve = config.approve === true;const contextWindow = positiveInt(config.contextWindow, 32768);const maxTokens = positiveInt(config.maxTokens, Math.min(Math.floor(contextWindow / 4), 8192));const samplingParams = (config.samplingParams ?? {}) as Record<string, unknown>;+const input: ("text" | "image")[] = config.vision === true ? ["text", "image"] : ["text"];let activeAgents = 0;const waitingAgents: Array<() => boolean> = [];@@ -298,7 +299,7 @@ export default function unslothSubagent(pi: ExtensionAPI): void {id: model,name: `${model} via Unsloth`,reasoning: false,- input: ["text"],+ input,cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },contextWindow,maxTokens,diff --git a/unsloth_cli/tests/test_start.py b/unsloth_cli/tests/test_start.pyindex 31774b7175e..d839aa2b73a 100644--- a/unsloth_cli/tests/test_start.py+++ b/unsloth_cli/tests/test_start.py@@ -6660,8 +6660,6 @@ def test_connect_openclaw_no_launch(fake_studio, tmp_path, monkeypatch):}_assert_env_cwd(result.output, "OPENCLAW_WORKSPACE_DIR")assert _launch_command(result.output) == ["openclaw", "tui", "--local"]- # OpenAI /v1/chat/completions works on either backend — no GGUF gate.- assert not any(c[1].split("?")[0].endswith("/api/inference/status") for c in fake_studio)@pytest.mark.skipif(os.name == "nt", reason = "WSL scenario")@@ -7091,7 +7089,6 @@ def test_connect_opencode_no_launch(fake_studio, tmp_path):"disabled_providers": [],}assert _launch_command(result.output) == ["opencode"]- assert not any(c[1].split("?")[0].endswith("/api/inference/status") for c in fake_studio)def test_connect_opencode_v2_no_launch_uses_private_server(fake_studio, monkeypatch):@@ -7852,7 +7849,6 @@ def test_connect_pi_no_launch(fake_studio, tmp_path, monkeypatch):assert config["providers"]["unsloth"]["models"] == [{"id": MODEL["id"], "contextWindow": MODEL["context_length"], "maxTokens": 32000}]- assert not any(c[1].split("?")[0].endswith("/api/inference/status") for c in fake_studio)assert (home / ".pi" / "agent" / "extensions" / "mine.ts").is_file()settings = json.loads((home / ".pi" / "agent" / "settings.json").read_text())assert settings == {"packages": ["npm:pi-mine"]}@@ -8427,6 +8423,51 @@ def test_write_dsh_patch_is_idempotent_and_follows_the_server(dsh_patch, capsys)assert entries["agent-default-model"]["config"]["model"] == "other"+def _generated_image_input(tmp_path, agent, vibe_env):+ agents = tmp_path / "agents"+ if agent == "opencode":+ config = json.loads((agents / "opencode" / "opencode.json").read_text())+ model = config["provider"][start._OPENCODE_PROVIDER]["models"][MODEL["id"]]+ return model.get("attachment", False) and "image" in model["modalities"]["input"]+ if agent == "openclaw":+ config = json.loads((agents / "openclaw" / "openclaw.json").read_text())+ return "image" in config["models"]["providers"]["unsloth"]["models"][0].get("input", ())+ if agent == "pi":+ return "image" in _pi_generated_model(tmp_path, False).get("input", ())+ if agent == "pi-subagent":+ return _pi_generated_model(tmp_path, True).get("vision", False)+ if agent == "dsh":+ entries = _dsh_entries(agents / "dsh" / start._DSH_PATCH_FILE)+ model = entries["llm-pi-ai"]["config"]["providers"]["unsloth"]["models"][0]+ return "image" in model.get("input", ())+ return json.loads(vibe_env["VIBE_MODELS"])[0].get("supports_images") is True++[email protected]("vision", [True, False])[email protected]("agent", ["opencode", "openclaw", "pi", "pi-subagent", "dsh", "vibe"])+def test_agent_config_declares_image_input_only_for_a_vision_model(+ fake_studio, tmp_path, monkeypatch, agent, vision+):+ http_json = start._http_json++ def with_vision(method, url, *args, **kwargs):+ answer = http_json(method, url, *args, **kwargs)+ if url.split("?")[0].endswith("/api/inference/status"):+ answer["is_vision"] = vision+ return answer++ monkeypatch.setattr(start, "_http_json", with_vision)+ vibe_env = {}+ vibe_env_fn = start._vibe_env+ monkeypatch.setattr(+ start, "_vibe_env", lambda *a, **k: vibe_env.update(vibe_env_fn(*a, **k)) or vibe_env+ )+ args = ["pi", "--as-subagent"] if agent == "pi-subagent" else [agent]+ result = CliRunner().invoke(start.start_app, [*args, "--no-launch"])+ assert result.exit_code == 0, result.output+ assert _generated_image_input(tmp_path, agent, vibe_env) is vision++@pytest.mark.parametrize(("args", "expected"),[diff --git a/unsloth_cli/tests/test_start_app.py b/unsloth_cli/tests/test_start_app.pyindex b88235c9e99..a2ca4f49bd8 100644--- a/unsloth_cli/tests/test_start_app.py+++ b/unsloth_cli/tests/test_start_app.py@@ -502,6 +502,31 @@ def test_openclaw_app_keeps_an_empty_allowlist_open(studio, defaults):assert json.loads(config.read_text())["agents"]["defaults"] == defaults[email protected]("vision", [True, False])+def test_app_config_declares_image_input_only_for_a_vision_model(studio, monkeypatch, vision):+ monkeypatch.delenv("XDG_CONFIG_HOME", raising = False)+ monkeypatch.setattr(start, "_opencode_command", lambda *_: ("opencode", False))+ http_json = start._http_json++ def with_vision(method, url, *args, **kwargs):+ answer = http_json(method, url, *args, **kwargs)+ if url.split("?")[0].endswith("/api/inference/status"):+ answer["is_vision"] = vision+ return answer++ monkeypatch.setattr(start, "_http_json", with_vision)+ for agent in ("opencode", "openclaw"):+ result = CliRunner().invoke(start.start_app, [agent, "--app"])+ assert result.exit_code == 0, result.output+ opencode = json.loads((studio["home"] / ".config" / "opencode" / "opencode.json").read_text())+ model = opencode["provider"]["unsloth-studio"]["models"][MODEL["id"]]+ assert model.get("attachment", False) is vision+ assert ("image" in model.get("modalities", {}).get("input", ())) is vision+ openclaw = json.loads((studio["home"] / ".openclaw" / "openclaw.json").read_text())+ model = openclaw["models"]["providers"]["unsloth"]["models"][0]+ assert ("image" in model.get("input", ())) is vision++def test_opencode_app_reads_a_commented_jsonc(studio, monkeypatch):monkeypatch.delenv("XDG_CONFIG_HOME", raising = False)monkeypatch.setattr(start, "_opencode_command", lambda *_: ("opencode", False))