Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(chat): files, web off, dictation and slash commands for the message box
trohitg078e42bb2bf3
diff --git a/server/config/chat_defaults.json b/server/config/chat_defaults.jsonindex 718c2f82..fd5f69f2 100644--- a/server/config/chat_defaults.json+++ b/server/config/chat_defaults.json@@ -19,5 +19,34 @@"branches": {"max_snapshots_per_session": 50,"max_snapshot_bytes": 1048576+ },+ "commands": {+ "_comment": "Slash commands every workflow's chat offers (get_chat_context); an employee's apps add theirs (employee_apps.json commands). fill is the text the command puts in the message box; suggest also shows it in an empty chat. {name} is the employee's name.",+ "generic": [+ {+ "command": "/summarize",+ "description": "Summarize this conversation",+ "fill": "Summarize what we’ve agreed so far.",+ "suggest": false+ },+ {+ "command": "/remember",+ "description": "Teach {name} something",+ "fill": "Remember that ",+ "suggest": false+ },+ {+ "command": "/today",+ "description": "What {name} did today",+ "fill": "What have you done today?",+ "suggest": true+ },+ {+ "command": "/plan",+ "description": "Plan something step by step",+ "fill": "Make a plan for ",+ "suggest": false+ }+ ]}}diff --git a/server/config/employee_apps.json b/server/config/employee_apps.jsonindex 698f5838..6e9cc49f 100644--- a/server/config/employee_apps.json+++ b/server/config/employee_apps.json@@ -57,7 +57,15 @@"waiting": "Waiting for new WhatsApp messages","working": "Answering a WhatsApp message","reporting": "Sending you an update on WhatsApp"- }+ },+ "commands": [+ {+ "command": "/draft",+ "description": "Draft a WhatsApp reply",+ "fill": "Draft a WhatsApp reply to ",+ "suggest": false+ }+ ]},"whatsapp_business": {"name": "WhatsApp Business",@@ -258,7 +266,15 @@"waiting": "Checking your inbox for new email","working": "Reading a new email","reporting": "Emailing you an update"- }+ },+ "commands": [+ {+ "command": "/inbox",+ "description": "Summarize unread email",+ "fill": "Summarize my unread email.",+ "suggest": true+ }+ ]},"outlook": {"name": "Outlook",@@ -317,7 +333,15 @@"waiting": "Checking your Outlook inbox","working": "Reading a new email","reporting": "Emailing you an update"- }+ },+ "commands": [+ {+ "command": "/inbox",+ "description": "Summarize unread email",+ "fill": "Summarize my unread email.",+ "suggest": true+ }+ ]},"email": {"name": "Email",@@ -399,7 +423,15 @@"side_effects": "send","phrases": {"working": "Checking your calendar"- }+ },+ "commands": [+ {+ "command": "/schedule",+ "description": "Check a day in the calendar",+ "fill": "What does tomorrow look like?",+ "suggest": true+ }+ ]},"google_drive": {"name": "Google Drive",diff --git a/server/config/llm_defaults.json b/server/config/llm_defaults.jsonindex 209a9169..11059b8b 100644--- a/server/config/llm_defaults.json+++ b/server/config/llm_defaults.json@@ -25,6 +25,8 @@"max_tokens_param": "max_tokens","vision": {"enabled": false,+ "user_images": true,+ "_user_images_note": "user_images: images the owner attaches to a chat message go to the model as image input in that user message (services/llm/media.py provider_supports_user_images; the provider's encoder renders them). Separate from enabled, which covers images in tool results.","_note": "Native image input is emitted only when enabled is true; unknown/absent providers get a text placeholder instead (a rejected image block would poison journaled history). Read by services/llm/media.py provider_supports_vision. False here until the OpenAI request encoder renders image blocks (Responses input_image in function_call_output; Chat Completions hoist-to-user-turn) — flipping it early would hydrate bytes the encoder silently drops. Anthropic ships the first encoder; text-only providers keep the visionAnalyze delegate tool."},"_models_note": "Ids, context, max output and prices follow the OpenRouter registry (2026-09-29). GPT-6 is the current generation: gpt-6-sol is the default ($2/$10 per MTok), gpt-6-luna the low tier ($0.10/$0.50) and gpt-6-astra the flagship ($10/$50); all have 1.05M context and 128K max output, and each -pro variant is priced like its base model. GPT-6 has no Terra model, so gpt-5.6-terra ($2/$12) stays the middle tier; the rest of the GPT-5.6 family, gpt-5.5, gpt-5.4 and gpt-4.1 stay curated as earlier tiers (unlike GPT-6, gpt-5.6-sol-pro costs twice its base: $4/$20). The rolling -latest aliases (gpt-astra-latest, gpt-sol-latest, gpt-terra-latest, gpt-luna-latest) are deliberately not curated: real model names only, like gemini. gpt-5.2, o3 and o4-mini are not curated because OpenAI is retiring them; the registry still describes them, so their limits resolve from it.",@@ -81,7 +83,9 @@"api_key_param": "anthropic_api_key","max_tokens_param": "max_tokens","vision": {- "enabled": true+ "enabled": true,+ "user_images": true,+ "_user_images_note": "user_images: images the owner attaches to a chat message go to the model as image input in that user message (services/llm/media.py provider_supports_user_images; the provider's encoder renders them). Separate from enabled, which covers images in tool results."},"max_output_tokens": {"claude-opus-5-5": 128000,@@ -141,6 +145,8 @@"max_tokens_param": "max_output_tokens","vision": {"enabled": false,+ "user_images": true,+ "_user_images_note": "user_images: images the owner attaches to a chat message go to the model as image input in that user message (services/llm/media.py provider_supports_user_images; the provider's encoder renders them). Separate from enabled, which covers images in tool results.","_note": "See openai.vision._note — off until the Gemini encoder renders inline_data parts beside function_response."},"_models_note": "Real (concrete) model names only — no -latest rolling aliases. The curated list (max_output_tokens keys) serves BOTH the Gemini Developer API (AIza keys) and Vertex / Agent Platform (AQ. keys); gemini-* entries live-verified on both endpoints (2026-06-12). Vertex 404s on -latest aliases and its models.list rejects API keys, which is why the dropdown is curated instead of live-fetched. Gemma exception: gemma-4-31b-it + gemma-4-26b-a4b-it are served ONLY on the Developer API (free tier, ai.google.dev/gemma/docs/core/gemma_on_gemini_api, verified 2026-07-11) — Vertex keys must not see them, hence vertex_incompatible_models below. Gemma 4 supports systemInstruction and function calling natively (unlike the gemma-3 era). Context and max output for every entry follow the OpenRouter registry (2026-09-29), including Gemma's 262K context and 16K / 236K max output.",diff --git a/server/config/speech_defaults.json b/server/config/speech_defaults.jsonindex df0cb6ad..3cddce2b 100644--- a/server/config/speech_defaults.json+++ b/server/config/speech_defaults.json@@ -210,5 +210,9 @@"billed_unit": "seconds"}}+ },+ "dictation": {+ "_comment": "Dictation in the chat's message box (nodes/speech/_handlers.py): the first of these speech-to-text providers with a stored key transcribes a recording (webm/opus from the browser). The microphone shows only when one has a key.",+ "providers": ["openai", "groq", "deepgram"]}}diff --git a/server/nodes/speech/__init__.py b/server/nodes/speech/__init__.pyindex 1285508f..b52e6d92 100644--- a/server/nodes/speech/__init__.py+++ b/server/nodes/speech/__init__.py@@ -28,12 +28,13 @@ Layout::_credentials.py speech-only credentials_base.py shared node helpers_option_loaders.py dropdown loaders+ _handlers.py dictation for the chat (dictation_status, transcribe_audio)"""from __future__ import annotationsfrom services.node_output_schemas import register_output_schema-from services.ws_handler_registry import register_option_loader+from services.ws_handler_registry import register_option_loader, register_ws_handlersfrom ._option_loaders import (load_speech_formats,@@ -58,6 +59,12 @@ register_option_loader("speechLanguages", load_speech_languages)register_option_loader("speechFormats", load_speech_formats)register_option_loader("speechSampleRates", load_speech_sample_rates)+# Dictation for the chat's message box: whether it can work, and turning a+# recording into text.+from ._handlers import WS_HANDLERS # noqa: E402++register_ws_handlers(WS_HANDLERS)+__all__ = ["SpeechToTextNode","SpeechToTextOutput",diff --git a/server/nodes/speech/_handlers.py b/server/nodes/speech/_handlers.pynew file mode 100644index 00000000..941bd6a2--- /dev/null+++ b/server/nodes/speech/_handlers.py@@ -0,0 +1,136 @@+"""Dictation for the chat's message box.++``dictation_status`` says whether a recording can be turned into text: the+first provider in ``speech_defaults.json``'s ``dictation.providers`` with a+stored key (the microphone shows only when there is one; the browser's own+speech recognition is not used, it does not work in the desktop app).+``transcribe_audio`` transcribes a recording the chat uploaded into the+workflow's workspace (``uploads/``) with that provider, records the usage,+and deletes the recording: a dictation leaves nothing behind.++Both answer only the workflow's owner, like the chat itself+(``services/chat/access.py``).+"""++from __future__ import annotations++import mimetypes+from pathlib import PurePosixPath+from types import SimpleNamespace+from typing import Any, Dict, Optional++from fastapi import WebSocket++from core.logging import get_logger+from services.plugin import NodeUserError+from services.plugin.ws import ws_response++from . import _config as speech_config+from . import _unifier+from ._base import track_usage+from ._protocol import SttRequest+from ._registry import stt_providers++logger = get_logger(__name__)+++def dictation_providers() -> list[str]:+ """The providers dictation may use, in the order it tries them."""+ configured = (speech_config.SPEECH_DEFAULTS.get("dictation") or {}).get("providers") or []+ available = set(stt_providers())+ return [str(provider) for provider in configured if str(provider) in available]+++async def dictation_provider() -> Optional[str]:+ """The first dictation provider with a stored key, or None."""+ from core.container import container++ auth = container.auth_service()+ for provider in dictation_providers():+ try:+ if await auth.get_api_key(speech_config.credential_id(provider)):+ return provider+ except Exception: # noqa: BLE001 - an unreadable key is no key+ logger.warning("Dictation key could not be read", provider=provider, exc_info=True)+ return None+++async def _owner_session(data: Dict[str, Any], websocket: WebSocket) -> str:+ from core.container import container+ from services.chat.access import ChatAccessDenied, authorize_session, session_id_of++ session_id = session_id_of(data)+ try:+ scope = await authorize_session(container.database(), websocket, session_id)+ except ChatAccessDenied as exc:+ raise NodeUserError("access_denied") from exc+ if scope.workflow_id is None:+ raise NodeUserError("Dictation needs a workflow's chat.")+ return scope.workflow_id+++@ws_response+async def handle_dictation_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:+ """``{available, provider}``: whether a recording can be turned into+ text now."""+ await _owner_session(data, websocket)+ provider = await dictation_provider()+ return {"success": True, "available": provider is not None, "provider": provider}+++@ws_response+async def handle_transcribe_audio(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:+ """Transcribe the recording at ``path`` (under ``uploads/``) and delete+ it. Answers ``{text, language, provider}``; ``speech_unavailable`` when no+ dictation provider has a key."""+ from core.container import container+ from services.media.workspace import UPLOAD_SUBDIR, resolve_media+ from services.workspace_locator import resolve_workspace_root++ workflow_id = await _owner_session(data, websocket)+ provider = await dictation_provider()+ if provider is None:+ return {"success": False, "error": "speech_unavailable"}+ rel = str(data.get("path") or "").replace("\\", "/")+ parts = PurePosixPath(rel).parts+ if len(parts) < 2 or parts[0] != UPLOAD_SUBDIR:+ raise NodeUserError("Only a recording uploaded to this chat can be transcribed.")+ database = container.database()+ root = await resolve_workspace_root(workflow_id, database, allow_default=False)+ path = resolve_media(rel, workspace_dir=str(root))+ if not path.is_file():+ raise NodeUserError("The recording is no longer there. Record it again.")+ max_bytes = speech_config.capability(provider, "stt", "max_upload_bytes", default=None)+ size = path.stat().st_size+ if isinstance(max_bytes, int) and size > max_bytes:+ raise NodeUserError("The recording is too long to transcribe. Record a shorter one.")+ try:+ api_key = await container.auth_service().get_api_key(speech_config.credential_id(provider))+ result = await _unifier.transcribe(+ provider=provider,+ api_key=str(api_key or ""),+ request=SttRequest(+ audio=path.read_bytes(),+ filename=path.name,+ mime_type=mimetypes.guess_type(path.name)[0] or "audio/webm",+ model=speech_config.default_model(provider, "stt"),+ language=str(data.get("language") or ""),+ ),+ )+ finally:+ try:+ path.unlink(missing_ok=True)+ except OSError:+ logger.warning("Dictation recording could not be deleted", path=rel, exc_info=True)+ usage = SimpleNamespace(raw={"session_id": workflow_id}, node_id="dictation", workflow_id=workflow_id)+ units = result.billed_units if result.billed_units is not None else result.duration_seconds+ await track_usage(usage, provider=provider, operation="speech_to_text", units=units, unit=result.billed_unit or "seconds")+ return {"success": True, "text": result.text, "language": result.language, "provider": provider}+++WS_HANDLERS = {+ "dictation_status": handle_dictation_status,+ "transcribe_audio": handle_transcribe_audio,+}++__all__ = ["WS_HANDLERS", "dictation_provider", "dictation_providers", "handle_dictation_status", "handle_transcribe_audio"]diff --git a/server/services/chat/attachments.py b/server/services/chat/attachments.pynew file mode 100644index 00000000..7f765e56--- /dev/null+++ b/server/services/chat/attachments.py@@ -0,0 +1,118 @@+"""Files the owner attaches to a chat message (docs-internal/chat_protocol.md,+"Attachments").++The chat uploads each file into the workflow's workspace first+(``POST /api/workspace/{workflow_id}/uploads``, which puts it under+``uploads/``) and sends the references it got back with the message.+``check_attachments`` trusts nothing in them but the path: each must name a+file under the workspace's ``uploads/`` folder that exists, and the+reference the message keeps is rebuilt here from the file itself (name,+type, size, address). At most ``MAX_ATTACHMENTS`` per message.++The employee reads them after the owner's words as one bracketed line+(``attachments_line``), with their workspace paths, so its file tools can+open them; images also travel with the message itself (``image_refs``) and+reach a model that can view them (``services/llm/media.py``).+"""++from __future__ import annotations++import json+import mimetypes+from datetime import datetime, timezone+from pathlib import PurePosixPath+from typing import Any, Dict, List, Optional, Sequence++from services.media.workspace import UPLOAD_SUBDIR++#: The most files one message carries.+MAX_ATTACHMENTS = 6++_KINDS = {"image": "image", "video": "video", "pdf": "document"}+++class AttachmentRefused(ValueError):+ """An attachment that cannot go with the message; the text says why."""+++def _relative(raw: Any) -> str:+ path = raw.get("path") if isinstance(raw, dict) else raw+ if not isinstance(path, str) or not path.strip():+ raise AttachmentRefused("each attachment names a file you uploaded")+ parts = PurePosixPath(path.replace("\\", "/")).parts+ if len(parts) < 2 or parts[0] != UPLOAD_SUBDIR or any(part in ("..", "") for part in parts):+ raise AttachmentRefused("only files you uploaded to this chat can be attached")+ return "/".join(parts)+++async def check_attachments(database: Any, workflow_id: str, raw: Any) -> List[Dict[str, Any]]:+ """The message's attachments, each rebuilt from its file. Raises+ :class:`AttachmentRefused`."""+ if raw is None:+ return []+ if not isinstance(raw, list):+ raise AttachmentRefused("attachments is a list of uploaded files")+ if len(raw) > MAX_ATTACHMENTS:+ raise AttachmentRefused(f"at most {MAX_ATTACHMENTS} files go with one message")+ if not raw:+ return []+ from services.media.preview import preview_kind+ from services.media.refs import FileRef+ from services.media.workspace import resolve_media, workspace_file_url+ from services.plugin import NodeUserError+ from services.workspace_locator import resolve_workspace_root++ try:+ root = await resolve_workspace_root(workflow_id, database, allow_default=False)+ except Exception as exc: # noqa: BLE001 - no workspace, nothing to attach+ raise AttachmentRefused("this chat has no workspace to attach files from") from exc+ refs: List[Dict[str, Any]] = []+ seen = set()+ for item in raw:+ rel = _relative(item)+ if rel in seen:+ continue+ seen.add(rel)+ try:+ path = resolve_media(rel, workspace_dir=str(root))+ except NodeUserError as exc:+ raise AttachmentRefused(str(exc)) from exc+ if not path.is_file():+ raise AttachmentRefused(f"{path.name} is no longer there; attach it again")+ stat = path.stat()+ mime_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"+ refs.append(+ FileRef(+ kind=_KINDS.get(preview_kind(mime_type) or "", "file"),+ path=rel,+ workflow_id=workflow_id,+ filename=path.name,+ mime_type=mime_type,+ size_bytes=stat.st_size,+ modified_at=datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),+ url=workspace_file_url(workflow_id, rel),+ ).model_dump(mode="json")+ )+ return refs+++def attachments_line(attachments: Sequence[Dict[str, Any]]) -> str:+ """What the employee reads about the files after the owner's words."""+ files = [+ {"path": ref.get("path"), "name": ref.get("filename"), "type": ref.get("mime_type"), "bytes": ref.get("size_bytes")}+ for ref in attachments+ if isinstance(ref, dict) and ref.get("path")+ ]+ if not files:+ return ""+ return f"[attachments]{json.dumps({'files': files}, ensure_ascii=False, separators=(', ', ': '))}[/attachments]"+++def image_refs(attachments: Optional[Sequence[Dict[str, Any]]]) -> List[Dict[str, Any]]:+ """The attached images a model that can view them is shown."""+ from services.llm.media import IMAGE_MIME_ALLOWLIST++ return [dict(ref) for ref in attachments or [] if isinstance(ref, dict) and ref.get("mime_type") in IMAGE_MIME_ALLOWLIST]+++__all__ = ["AttachmentRefused", "MAX_ATTACHMENTS", "attachments_line", "check_attachments", "image_refs"]diff --git a/server/services/chat/branches.py b/server/services/chat/branches.pyindex 9fd5bba1..9d7b795f 100644--- a/server/services/chat/branches.py+++ b/server/services/chat/branches.py@@ -493,6 +493,7 @@ class _Edit(_Operation):execution_id=self.live_root,uid=self.uid,run_id=run.run_id,+ attachments=list(original.attachments or []),meta={**self.meta, "edit_of": original.uid},after=original.parent_uid,)diff --git a/server/services/chat/guide.py b/server/services/chat/guide.pyindex 0814d3e4..1e96eb54 100644--- a/server/services/chat/guide.py+++ b/server/services/chat/guide.py@@ -16,7 +16,7 @@ from __future__ import annotationsimport jsonimport re-from typing import Any, List, Mapping, Tuple+from typing import Any, Dict, List, Mapping, Tuplefrom core.logging import get_loggerfrom services.chat.stream import FOLLOWUPS_OPEN@@ -34,7 +34,9 @@ CHAT_REPLY_GUIDE = ("[ui-event]{...}[/ui-event] is a button they pressed in an interface you showed (with what they picked), and ""[ui-state]{...}[/ui-state] is what they set in one without pressing anything; treat both as the owner's choice. ""[update]{...}[/update] says what became of something you did (a draft sent or not, a part of the chat the owner "- "went back from), and [feedback]{...}[/feedback] is how the owner rated one of your answers; learn from it.\n"+ "went back from), and [feedback]{...}[/feedback] is how the owner rated one of your answers; learn from it. "+ "[attachments]{...}[/attachments] after their words lists the files they attached, with their paths in your "+ "workspace.\n""- A search result numbered n is a source: cite the ones you rely on as [n] right after what they support.\n""- When a few short next questions would help the owner, end your reply with "'<followups>["...", "..."]</followups>: at most 3, each a short question in the owner\'s own words. The chat '@@ -67,9 +69,28 @@ async def chat_turn(database: Any, chat_stream: Mapping[str, Any], *, system_mesif claimed:lines = notes_prompt(claimed)prompt = f"{lines}\n\n{prompt}" if prompt else lines+ from services.chat.attachments import attachments_line++ attached = attachments_line(await run_attachments(database, chat_stream))+ if attached:+ prompt = f"{prompt}\n\n{attached}" if prompt else attachedreturn system, prompt+async def run_attachments(database: Any, chat_stream: Mapping[str, Any]) -> List[Dict[str, Any]]:+ """The files the owner attached to the message the run answers (none+ when it cannot be read: the turn goes on without them)."""+ from services.chat import ledger++ try:+ run = await ledger.get_run(database, str(chat_stream["run_id"]))+ message = await ledger.saved_message(database, run.user_message_uid) if run and run.user_message_uid else None+ except Exception: # noqa: BLE001+ logger.warning("Chat attachments could not be read for a turn", run_id=chat_stream.get("run_id"), exc_info=True)+ return []+ return [dict(item) for item in (message.attachments or []) if isinstance(item, dict)] if message is not None else []++def _items(body: str) -> List[str]:body = body.strip()raw: List[str]@@ -99,4 +120,12 @@ def split_followups(text: str) -> Tuple[str, List[str]]:return visible, _items(inner)-__all__ = ["CHAT_REPLY_GUIDE", "FOLLOWUPS_CLOSE", "MAX_FOLLOWUPS", "chat_turn", "split_followups", "with_chat_guide"]+__all__ = [+ "CHAT_REPLY_GUIDE",+ "FOLLOWUPS_CLOSE",+ "MAX_FOLLOWUPS",+ "chat_turn",+ "run_attachments",+ "split_followups",+ "with_chat_guide",+]diff --git a/server/services/chat/handlers.py b/server/services/chat/handlers.pyindex f5d93ee3..5a6a0579 100644--- a/server/services/chat/handlers.py+++ b/server/services/chat/handlers.py@@ -31,6 +31,7 @@ from core.container import containerfrom core.logging import get_loggerfrom services.chat import branches, ledger, reducerfrom services.chat.access import ChatAccessDenied, authorize_session, session_id_of+from services.chat.attachments import MAX_ATTACHMENTS, AttachmentRefused, check_attachmentsfrom services.chat.feedback import FeedbackRefused, feedback_for, set_feedbackfrom services.chat.events import MESSAGE_WIRE_ROUTING_KEY, dispatch_chat_message_receivedfrom services.chat.hub import get_chat_hub@@ -182,18 +183,29 @@ async def _ui_event(database: Any, session_id: str, raw: Any) -> Dict[str, Any]:async def _dispatch(- session_id: str, workflow_id: Optional[str], *, message_uid: str, prompt: str, run_id: Optional[str], timestamp: str+ session_id: str,+ workflow_id: Optional[str],+ *,+ message_uid: str,+ prompt: str,+ run_id: Optional[str],+ timestamp: str,+ attachments: Optional[List[Dict[str, Any]]] = None,) -> None:"""Send the owner's message to the workflow's chat triggers: the run it- starts is the event's id, so the trigger's child workflow is the run's."""+ starts is the event's id, so the trigger's child workflow is the run's.+ Its attachments ride along (the trigger's output carries them)."""event_data: Dict[str, Any] = {"message": prompt, "timestamp": timestamp, "session_id": session_id, "message_id": message_uid}if run_id is not None:event_data["run_id"] = run_id+ if attachments:+ event_data["attachments"] = list(attachments)await dispatch_chat_message_received(event_data, workflow_id=workflow_id, event_id=run_id)logger.info("Chat message dispatched", session_id=session_id, run_id=run_id)-@ws_handler("message")+# ``message`` is checked below: a message may be only files.+@ws_handler()async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:"""Save the owner's message, start its run, and dispatch it. Answers``{message_id, run_id, delivery, timestamp}``; ``run_in_progress`` (with@@ -203,9 +215,16 @@ async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -With ``ui_event`` the message is a button pressed in an interface theemployee showed: checked against that interface, saved as the owner'smessage reading the button's label (kind ``action``), and sent to the- employee as a ``[ui-event]`` line naming the press."""+ employee as a ``[ui-event]`` line naming the press.++ ``attachments`` are files the chat uploaded under the workspace's+ ``uploads/`` (``services/chat/attachments.py``: rebuilt from the files,+ at most six; refused with ``attachment_rejected``); a message may be+ only files. ``options.web: false`` keeps the employee off web search for+ this message."""message = data.get("message")- if not isinstance(message, str) or not message.strip():+ raw_attachments = data.get("attachments")+ if not isinstance(message, str) or not (message.strip() or raw_attachments):return {"success": False, "error": "invalid_request", "detail": "message must be text"}if data.get("role", "user") != "user":return {"success": False, "error": "invalid_request", "detail": "send_chat_message sends the owner's messages"}@@ -217,6 +236,15 @@ async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -except ChatAccessDenied:return dict(_DENIED)+ attachments: List[Dict[str, Any]] = []+ if raw_attachments:+ if scope.workflow_id is None:+ return {"success": False, "error": "invalid_request", "detail": "files can be attached only in a workflow's chat"}+ try:+ attachments = await check_attachments(database, scope.workflow_id, raw_attachments)+ except AttachmentRefused as exc:+ return {"success": False, "error": "attachment_rejected", "detail": str(exc)}+press: Optional[Dict[str, Any]] = Noneif data.get("ui_event") is not None:try:@@ -251,6 +279,8 @@ async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -message_kind="action" if press is not None else "text",client_message_id=data.get("client_message_id"),meta=press["meta"] if press is not None else None,+ options=_options(data.get("options")),+ attachments=attachments or None,)except ledger.RunInProgress as exc:return {"success": False, "error": "run_in_progress", "run_id": exc.run.run_id}@@ -270,6 +300,7 @@ async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -prompt=press["prompt"] if press is not None else message,run_id=run.run_id if run is not None else None,timestamp=timestamp,+ attachments=attachments,)response: Dict[str, Any] = {@@ -445,6 +476,73 @@ async def handle_chat_ui_state(data: Dict[str, Any], websocket: WebSocket) -> Direturn {"success": True, "part_id": part_id, "state_revision": updated["state_revision"]}+def _options(raw: Any) -> Dict[str, Any]:+ """The per-message choices kept on the run: ``web`` (a bool) only."""+ if isinstance(raw, dict) and isinstance(raw.get("web"), bool):+ return {"web": raw["web"]}+ return {}+++@ws_handler()+async def handle_get_chat_context(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:+ """What the session's message box offers: slash commands (the generic+ ones in ``chat_defaults.json`` and those its employee's apps add,+ ``employee_apps.json``; ``{name}`` is the workflow's name), whether files+ can be attached (a workflow's chat), whether the Web chip has web search+ to turn off (a search tool in the saved graph), and the limits."""+ from services.chat.config import load_chat_config+ from services.employees.apps import app_for_node_type+ from services.media.limits import MEDIA_MAX_UPLOAD_BYTES+ from services.node_registry import get_node_class++ session_id = session_id_of(data)+ database = container.database()+ try:+ scope = await authorize_session(database, websocket, session_id)+ except ChatAccessDenied:+ return dict(_DENIED)+ saved = await database.get_workflow(scope.workflow_id) if scope.workflow_id is not None else None+ graph = (getattr(saved, "data", None) or {}) if saved is not None else {}+ name = str(getattr(saved, "name", "") or "the employee")+ node_types = [str(node.get("type") or "") for node in graph.get("nodes") or [] if isinstance(node, dict)]++ configured = (load_chat_config().get("commands") or {}).get("generic") or []+ commands: List[Dict[str, Any]] = []+ seen: set = set()++ def add(command: Any) -> None:+ if not isinstance(command, dict) or command.get("command") in seen or not scope.workflow_id:+ return+ seen.add(command.get("command"))+ commands.append(+ {+ "command": str(command.get("command")),+ "description": str(command.get("description") or "").replace("{name}", name),+ "fill": str(command.get("fill") or "").replace("{name}", name),+ "suggest": bool(command.get("suggest")),+ }+ )++ for node_type in node_types:+ app = app_for_node_type(node_type)+ for command in app.commands if app is not None else ():+ add(dict(command))+ for command in configured:+ add(command)++ def searches(node_type: str) -> bool:+ cls = get_node_class(node_type)+ return cls is not None and "search" in tuple(getattr(cls, "group", ()) or ())++ return {+ "success": True,+ "session_id": session_id,+ "commands": commands,+ "capabilities": {"attachments": scope.workflow_id is not None, "web": any(searches(t) for t in node_types)},+ "limits": {"max_attachments": MAX_ATTACHMENTS, "max_upload_bytes": MEDIA_MAX_UPLOAD_BYTES},+ }++def _expected_revision(data: Dict[str, Any]) -> Optional[int]:value = data.get("expected_revision")if value is None:@@ -504,7 +602,13 @@ async def _answer_branch(database: Any, moved: "branches.Moved", context: Dict[sawait announce_chat_updated(moved.session_id, role)message, run = moved.result["message"], moved.result["run"]await _dispatch(- moved.session_id, moved.workflow_id, message_uid=message["uid"], prompt=moved.result["prompt"], run_id=run.run_id, timestamp=timestamp,+ moved.session_id,+ moved.workflow_id,+ message_uid=message["uid"],+ prompt=moved.result["prompt"],+ run_id=run.run_id,+ timestamp=timestamp,+ attachments=list(message.get("attachments") or []),)response: Dict[str, Any] = {"success": True, "message_id": message["uid"], "run_id": run.run_id}if context.get("delivery") is not None:@@ -687,6 +791,7 @@ WS_HANDLERS = {"regenerate_chat_reply": handle_regenerate_chat_reply,"switch_chat_branch": handle_switch_chat_branch,"set_chat_feedback": handle_set_chat_feedback,+ "get_chat_context": handle_get_chat_context,"clear_chat_messages": handle_clear_chat_messages,"save_chat_message": handle_save_chat_message,}diff --git a/server/services/employees/apps.py b/server/services/employees/apps.pyindex b3e46e65..510e20d0 100644--- a/server/services/employees/apps.py+++ b/server/services/employees/apps.py@@ -86,6 +86,9 @@ class AppSpec:#: agent only, never to a worker strangers write to; every call waits#: for the owner while they ask first (services/approvals/tool_calls.py).talk_send: Optional[NodeTemplate] = None+ #: Slash commands this app adds to the employee's chat+ #: (``{command, description, fill, suggest}``; get_chat_context).+ commands: Tuple[Mapping[str, Any], ...] = ()@propertydef node_types(self) -> frozenset:@@ -171,10 +174,37 @@ def _parse(raw: Mapping[str, Any]) -> Dict[str, AppSpec]:tools=tools,side_effects=effect,phrases=MappingProxyType({str(k): str(v) for k, v in (entry.get("phrases") or {}).items()}),+ commands=_commands(app_id, entry.get("commands")),)return apps+_COMMAND = re.compile(r"^/[a-z0-9-]{1,30}$")+++def _commands(app_id: str, raw: Any) -> Tuple[Mapping[str, Any], ...]:+ """An app's slash commands: ``{command, description, fill, suggest}``."""+ if raw is None:+ return ()+ if not isinstance(raw, list):+ raise AppRegistryError(f"{app_id}.commands must be a list")+ commands = []+ for item in raw:+ if not isinstance(item, dict) or not isinstance(item.get("command"), str) or not _COMMAND.match(item["command"]):+ raise AppRegistryError(f"{app_id}.commands: each needs a `command` like /schedule")+ commands.append(+ MappingProxyType(+ {+ "command": item["command"],+ "description": str(item.get("description") or ""),+ "fill": str(item.get("fill") or ""),+ "suggest": bool(item.get("suggest")),+ }+ )+ )+ return tuple(commands)++_apps: Optional[Dict[str, AppSpec]] = None_by_node_type: Optional[Dict[str, AppSpec]] = Nonediff --git a/server/services/llm/media.py b/server/services/llm/media.pyindex 47729bb6..bae840c6 100644--- a/server/services/llm/media.py+++ b/server/services/llm/media.py@@ -1,12 +1,19 @@-"""Tool-result images: refs in durable state, bytes only at the provider boundary.+"""Images for the model: refs in durable state, bytes only at the provider boundary.A tool opts in by returning ``llm_media: [{"ref": <FileRef kind=image>,"detail": "auto"}]``. The agent loop attaches ref-only image ContentBlocks to-the tool message (durable, ~450 B each). ``hydrate_image_blocks`` runs once-per LLM step on throwaway copies, loading bytes through the contained media-reader and fitting them to a budget; provider encoders then emit the-official wire shapes. Originals are never mutated, so nothing downstream of-the journal ever sees bytes.+the tool message (durable, ~450 B each). Images the owner attaches to a chat+message ride the same way on that user message (``image_blocks``).+``hydrate_image_blocks`` runs once per LLM step on throwaway copies, loading+bytes through the contained media reader and fitting them to a budget;+provider encoders then emit the official wire shapes. Originals are never+mutated, so nothing downstream of the journal ever sees bytes.++Two gates, because the encoders came at different times: images in a tool+result need ``vision.enabled``, images in the owner's message+``vision.user_images`` (``llm_defaults.json``). A model behind neither reads a+text placeholder; in the owner's message it names the file's workspace path,+so the employee can open it with a tool that can."""from __future__ import annotations@@ -38,6 +45,29 @@ def provider_supports_vision(provider: str, model: str = "") -> bool:return bool(vision.get("enabled"))+def provider_supports_user_images(provider: str, model: str = "") -> bool:+ """Capability gate for images in the owner's message: ``vision.user_images``.+ Unknown means False, like :func:`provider_supports_vision`."""+ del model+ vision = (+ LLM_DEFAULTS.get("providers", {}).get(provider, {}).get("vision", {})+ )+ return bool(vision.get("user_images"))+++def image_blocks(refs: Sequence[Any], *, detail: str = "auto") -> List[ContentBlock]:+ """Ref-only image blocks for images the owner attached (FileRef dumps+ with an image type in :data:`IMAGE_MIME_ALLOWLIST`); others are left out."""+ blocks: List[ContentBlock] = []+ for ref in refs:+ if not isinstance(ref, dict) or ref.get("mime_type") not in IMAGE_MIME_ALLOWLIST:+ continue+ if not ref.get("workflow_id") or not ref.get("path"):+ continue+ blocks.append(ContentBlock(type="image", source={"kind": "file_ref", "ref": dict(ref), "detail": detail}))+ return blocks++def image_blocks_from_tool_result(result: Any) -> List[ContentBlock]:"""Parse a tool result's ``llm_media`` opt-in into ref-only image blocks.@@ -101,12 +131,17 @@ async def hydrate_image_blocks(if not any(_ref_blocks(message) for message in messages):return list(messages)- capable = provider_supports_vision(provider, model)+ in_results = provider_supports_vision(provider, model)+ in_owner_messages = provider_supports_user_images(provider, model)hydrated: List[Message] = []for message in messages:if not _ref_blocks(message):hydrated.append(message)continue+ if message.role == "user" and not in_owner_messages:+ hydrated.append(_described(message, model))+ continue+ capable = in_owner_messages if message.role == "user" else in_resultsblocks: List[ContentBlock] = []for block in message.blocks:if not _is_ref_block(block):@@ -147,6 +182,22 @@ async def hydrate_image_blocks(return hydrated+def _described(message: Message, model: str) -> Message:+ """The owner's message for a model that cannot view its images: each+ image named in the text instead, with where it is in the workspace."""+ notes = []+ for block in _ref_blocks(message):+ ref = dict((block.source or {}).get("ref") or {})+ notes.append(+ _placeholder(+ ref,+ f"the current model ({model}) cannot view images; it is at {ref.get('path')} in the workspace",+ ).text+ )+ content = "\n\n".join(part for part in (message.content, "\n".join(notes)) if part)+ return replace(message, content=content, blocks=[ContentBlock(type="text", text=content)])++def _is_ref_block(block: ContentBlock) -> bool:return (block.type == "image"@@ -192,6 +243,8 @@ __all__ = ["LLM_MEDIA_KEY","LLM_MEDIA_MAX_PER_RESULT","hydrate_image_blocks",+ "image_blocks",+ "provider_supports_user_images","image_blocks_from_tool_result","provider_supports_vision",]diff --git a/server/services/llm/providers/anthropic.py b/server/services/llm/providers/anthropic.pyindex 72176473..056e07fb 100644--- a/server/services/llm/providers/anthropic.py+++ b/server/services/llm/providers/anthropic.py@@ -238,7 +238,15 @@ class AnthropicProvider:hydrated images ride along (the documented image-in-tool-resultshape). Ref-only blocks render nothing here — hydration eitherproduced bytes or a text placeholder before compile time."""- images = [+ images = AnthropicProvider._image_parts(m)+ if not images:+ return m.content+ return [{"type": "text", "text": m.content}, *images]++ @staticmethod+ def _image_parts(m: Message) -> List[Dict[str, Any]]:+ """The message's hydrated images as Anthropic image blocks."""+ return [{"type": "image","source": {@@ -252,9 +260,6 @@ class AnthropicProvider:and isinstance(block.source, dict)and block.source.get("kind") == "bytes"]- if not images:- return m.content- return [{"type": "text", "text": m.content}, *images]def _to_api_message(self, m: Message) -> Dict[str, Any]:role = "assistant" if m.role == "assistant" else "user"@@ -302,6 +307,12 @@ class AnthropicProvider:)return {"role": "assistant", "content": content}+ images = self._image_parts(m) if m.role == "user" else []+ if images:+ # Images the owner attached, ahead of their words (Anthropic's+ # recommended order).+ text = [{"type": "text", "text": m.content}] if m.content else []+ return {"role": "user", "content": [*images, *text]}return {"role": role, "content": m.content}@staticmethoddiff --git a/server/services/llm/providers/gemini.py b/server/services/llm/providers/gemini.pyindex e9130184..8df8927a 100644--- a/server/services/llm/providers/gemini.py+++ b/server/services/llm/providers/gemini.py@@ -256,7 +256,17 @@ class GeminiProvider:continuerole = "model" if m.role == "assistant" else "user"- contents.append({"role": role, "parts": [{"text": m.content}]})+ parts: List[Dict[str, Any]] = [{"text": m.content}] if m.content or m.role != "user" else []+ if m.role == "user":+ # Images the owner attached, as inline data parts.+ parts.extend(+ {"inline_data": {"mime_type": block.source["media_type"], "data": base64.b64decode(block.source["data_b64"])}}+ for block in m.blocks or []+ if block.type == "image"+ and isinstance(block.source, dict)+ and block.source.get("kind") == "bytes"+ )+ contents.append({"role": role, "parts": parts or [{"text": m.content}]})index += 1system = "\n\n".join(system_parts) if system_parts else Nonediff --git a/server/services/llm/providers/openai.py b/server/services/llm/providers/openai.pyindex 55ef3efd..14b8c9fe 100644--- a/server/services/llm/providers/openai.py+++ b/server/services/llm/providers/openai.py@@ -30,6 +30,26 @@ _RESPONSES_TERMINAL_EVENTS = frozenset({"response.completed", "response.incomplelogger = get_logger(__name__)+def _image_sources(m: Message) -> List[Dict[str, Any]]:+ """The message's hydrated images (``source`` dicts with ``data_b64``)."""+ return [+ block.source+ for block in m.blocks or []+ if block.type == "image"+ and isinstance(block.source, dict)+ and block.source.get("kind") == "bytes"+ ]+++def _data_url(source: Dict[str, Any]) -> str:+ return f"data:{source['media_type']};base64,{source['data_b64']}"+++def _detail(source: Dict[str, Any]) -> str:+ detail = str(source.get("detail") or "auto")+ return detail if detail in ("low", "high", "auto") else "auto"++class OpenAIProvider:provider_name = "openai"@@ -370,6 +390,16 @@ class OpenAIProvider:result[key] = payload[key]return result+ images = _image_sources(m) if m.role == "user" else []+ if images:+ # Images the owner attached: Chat Completions takes them as+ # image_url parts (a data URL) beside the text.+ parts: List[Dict[str, Any]] = [{"type": "text", "text": m.content}] if m.content else []+ parts.extend(+ {"type": "image_url", "image_url": {"url": _data_url(source), "detail": _detail(source)}}+ for source in images+ )+ return {"role": "user", "content": parts}return {"role": m.role, "content": m.content}def _to_api_tool(self, tool: ToolDef) -> Dict[str, Any]:@@ -452,6 +482,19 @@ class OpenAIProvider:)continue+ images = _image_sources(message) if message.role == "user" else []+ if images:+ # Images the owner attached: Responses takes them as+ # input_image parts beside input_text.+ content: List[Dict[str, Any]] = (+ [{"type": "input_text", "text": message.content}] if message.content else []+ )+ content.extend(+ {"type": "input_image", "image_url": _data_url(source), "detail": _detail(source)}+ for source in images+ )+ items.append({"role": "user", "content": content})+ continueitems.append({"role": message.role, "content": message.content})diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.pyindex e35b6e8c..d064f3c9 100644--- a/server/services/temporal/agent_activities.py+++ b/server/services/temporal/agent_activities.py@@ -912,6 +912,32 @@ async def store_agent_output(payload: Dict[str, Any]) -> Dict[str, Any]:return {"stored": True}+async def _without_web_tools(database: Any, context: Dict[str, Any], tool_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:+ """The tools, without the web search ones when the owner turned Web off+ for the message the chat run answers (``options.web``)."""+ from services.chat.stream import chat_run_id_of++ run_id = chat_run_id_of(context)+ if not run_id or not tool_data:+ return tool_data+ try:+ from services.chat.ledger import get_run++ run = await get_run(database, run_id)+ except Exception: # noqa: BLE001 - the run keeps its tools+ activity.logger.warning("Chat run options could not be read for %s", run_id, exc_info=True)+ return tool_data+ if run is None or (run.options or {}).get("web", True) is not False:+ return tool_data+ from services.node_registry import get_node_class++ def searches(tool: Dict[str, Any]) -> bool:+ cls = get_node_class(str(tool.get("node_type") or ""))+ return cls is not None and "search" in tuple(getattr(cls, "group", ()) or ())++ return [tool for tool in tool_data if not searches(tool)]++async def _record_chat_cursor(database: Any, context: Dict[str, Any], *, node_id: str, generation: int, stored: List[Dict[str, Any]]) -> None:@@ -1333,7 +1359,7 @@ async def prepare_agent_payload(context: Dict[str, Any]) -> Dict[str, Any]:tools_payload: List[Dict[str, Any]] = []from services.skill_runtime import skill_tool_info- effective_tool_data = list(tool_data or [])+ effective_tool_data = await _without_web_tools(database, context, list(tool_data or []))progressive_skill_tool = skill_tool_info(skill_data or [], node_id)if progressive_skill_tool:effective_tool_data.append(progressive_skill_tool)@@ -1516,6 +1542,12 @@ async def prepare_agent_payload(context: Dict[str, Any]) -> Dict[str, Any]:system_message, prompt = await chat_turn(database, chat_stream, system_message=system_message, prompt=prompt)+ from services.chat.attachments import image_refs+ from services.chat.guide import run_attachments++ user_images = image_refs(await run_attachments(database, chat_stream))+ else:+ user_images = []return {# The chat run this agent works for (stops when the owner presses@@ -1532,6 +1564,9 @@ async def prepare_agent_payload(context: Dict[str, Any]) -> Dict[str, Any]:"temperature": temperature,"system_message": system_message,"user_prompt": prompt,+ # Images the owner attached to the message this run answers: the+ # opening user message carries them (services/llm/media.py).+ "user_images": user_images,"tools": tools_payload,"memory_node_id": memory_node_id,"memory_content": memory_content,diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.pyindex e0e36d3a..9e9298aa 100644--- a/server/services/temporal/agent_workflow.py+++ b/server/services/temporal/agent_workflow.py@@ -187,6 +187,20 @@ def _native_message()+def _owner_message(content: str, images: Any) -> Dict[str, Any]:+ """The run's opening user message: the prompt, and the images the owner+ attached as ref-only image blocks (hydrated per provider call,+ ``services/llm/media.py``)."""++ from services.llm.media import image_blocks+ from services.llm.protocol import ContentBlock, Message, message_to_wire++ blocks = image_blocks(images if isinstance(images, list) else [])+ if not blocks:+ return _native_message(role="user", content=content)+ return message_to_wire(Message(role="user", content=content, blocks=[ContentBlock(type="text", text=content), *blocks]))++def _append_tool_result_message(messages: List[Dict[str, Any]],*,@@ -773,9 +787,7 @@ class AgentWorkflow:if isinstance(wire, dict) and wire.get("role") != "system")if user_prompt:- messages.append(- _native_message(role="user", content=user_prompt)- )+ messages.append(_owner_message(user_prompt, payload.get("user_images")))else:if system:messages.append(_native_message(role="system", content=system))@@ -790,9 +802,7 @@ class AgentWorkflow:)if user_prompt:- messages.append(- _native_message(role="user", content=user_prompt)- )+ messages.append(_owner_message(user_prompt, payload.get("user_images")))# Map LLM tool name -> {node_type, version, task_queue, node_id,# parameters} so the workflow can schedule the right activitydiff --git a/server/tests/llm/test_user_images.py b/server/tests/llm/test_user_images.pynew file mode 100644index 00000000..0052ae3b--- /dev/null+++ b/server/tests/llm/test_user_images.py@@ -0,0 +1,162 @@+"""Images the owner attaches to a chat message: carried as ref-only blocks on+the opening user message, hydrated only for a provider that declares+``vision.user_images`` (each encoder's own shape: Anthropic image blocks,+OpenAI image_url / input_image, Gemini inline_data), and named with their+workspace path for a model that cannot view them."""++from __future__ import annotations++import base64+from unittest.mock import patch++import pytest++from services.llm.media import hydrate_image_blocks, image_blocks, provider_supports_user_images+from services.llm.protocol import ContentBlock, Message, message_from_wire++REF = {+ "kind": "image",+ "path": "uploads/shelf.png",+ "workflow_id": "wf",+ "filename": "shelf.png",+ "mime_type": "image/png",+ "size_bytes": 64,+}+PIXEL = base64.b64encode(b"png-bytes").decode("ascii")+++def owner_message(text: str = "Is this stock right?") -> Message:+ return Message(role="user", content=text, blocks=[ContentBlock(type="text", text=text), *image_blocks([REF])])+++def hydrated_message(text: str = "Is this stock right?") -> Message:+ source = {"kind": "bytes", "media_type": "image/png", "data_b64": PIXEL, "detail": "auto"}+ return Message(role="user", content=text, blocks=[ContentBlock(type="text", text=text), ContentBlock(type="image", source=source)])+++def test_only_attached_images_become_blocks():+ blocks = image_blocks([REF, {**REF, "mime_type": "text/csv"}, {**REF, "path": ""}, "nope"])+ assert [(block.type, block.source["kind"], block.source["ref"]["path"]) for block in blocks] == [("image", "file_ref", "uploads/shelf.png")]+++def test_the_gate_names_the_providers_with_an_encoder():+ for provider in ("anthropic", "openai", "gemini"):+ assert provider_supports_user_images(provider) is True+ for provider in ("deepseek", "groq", "openrouter", "nope"):+ assert provider_supports_user_images(provider) is False+++async def test_a_capable_provider_gets_bytes_on_the_owner_message(monkeypatch):+ async def fake_hydrate(ref, detail):+ return PIXEL, "image/png"++ monkeypatch.setattr("services.llm.media._hydrate_one", fake_hydrate)+ message = owner_message()+ [hydrated] = await hydrate_image_blocks([message], provider="openai", model="gpt-6")+ images = [block for block in hydrated.blocks if block.type == "image"]+ assert images[0].source == {"kind": "bytes", "media_type": "image/png", "data_b64": PIXEL, "detail": "auto"}+ # The durable message keeps its refs.+ assert any(block.source and block.source.get("kind") == "file_ref" for block in message.blocks)+++async def test_a_model_that_cannot_view_images_reads_where_they_are(monkeypatch):+ async def never(ref, detail):+ raise AssertionError("no bytes for a model that cannot view them")++ monkeypatch.setattr("services.llm.media._hydrate_one", never)+ [hydrated] = await hydrate_image_blocks([owner_message()], provider="deepseek", model="deepseek-chat")+ assert all(block.type != "image" for block in hydrated.blocks)+ assert hydrated.content.startswith("Is this stock right?\n\n[Image attached: shelf.png")+ assert "uploads/shelf.png in the workspace" in hydrated.content+++async def test_tool_results_keep_their_own_gate(monkeypatch):+ async def fake_hydrate(ref, detail):+ return PIXEL, "image/png"++ monkeypatch.setattr("services.llm.media._hydrate_one", fake_hydrate)+ tool = Message(role="tool", content="{}", tool_call_id="t1", blocks=[ContentBlock(type="text", text="{}"), *image_blocks([REF])])+ # OpenAI views the owner's images but not yet a tool's.+ [hydrated] = await hydrate_image_blocks([tool], provider="openai", model="gpt-6")+ assert all(block.type != "image" for block in hydrated.blocks)+++def test_anthropic_puts_the_images_ahead_of_the_words():+ with patch("anthropic.AsyncAnthropic"):+ from services.llm.providers.anthropic import AnthropicProvider++ provider = AnthropicProvider("key")+ api = provider._to_api_message(hydrated_message())+ assert api == {+ "role": "user",+ "content": [+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PIXEL}},+ {"type": "text", "text": "Is this stock right?"},+ ],+ }+ # Only images: no empty text block.+ assert provider._to_api_message(hydrated_message(""))["content"] == [api["content"][0]]+ # A message with nothing to view stays a string.+ assert provider._to_api_message(Message(role="user", content="Hi")) == {"role": "user", "content": "Hi"}+++def test_openai_sends_data_urls_on_both_apis():+ with patch("openai.AsyncOpenAI"):+ from services.llm.providers.openai import OpenAIProvider++ provider = OpenAIProvider("key")+ url = f"data:image/png;base64,{PIXEL}"+ assert provider._to_api_message(hydrated_message()) == {+ "role": "user",+ "content": [+ {"type": "text", "text": "Is this stock right?"},+ {"type": "image_url", "image_url": {"url": url, "detail": "auto"}},+ ],+ }+ [item] = provider._to_responses_input([hydrated_message()])+ assert item == {+ "role": "user",+ "content": [+ {"type": "input_text", "text": "Is this stock right?"},+ {"type": "input_image", "image_url": url, "detail": "auto"},+ ],+ }+ assert provider._to_api_message(Message(role="user", content="Hi")) == {"role": "user", "content": "Hi"}+++def test_gemini_sends_inline_data():+ with patch("google.genai.Client"):+ from services.llm.providers.gemini import GeminiProvider++ provider = GeminiProvider("key")+ _, contents = provider._split_system_and_contents([hydrated_message()])+ assert contents == [+ {+ "role": "user",+ "parts": [+ {"text": "Is this stock right?"},+ {"inline_data": {"mime_type": "image/png", "data": b"png-bytes"}},+ ],+ }+ ]+ _, plain = provider._split_system_and_contents([Message(role="user", content="Hi")])+ assert plain == [{"role": "user", "parts": [{"text": "Hi"}]}]+++def test_the_opening_message_carries_the_images():+ from services.temporal.agent_workflow import _owner_message++ wire = _owner_message("Is this stock right?", [REF])+ message = message_from_wire(wire)+ assert message.content == "Is this stock right?"+ assert [(block.type, (block.source or {}).get("kind")) for block in message.blocks] == [("text", None), ("image", "file_ref")]+ # No images (or none the owner sent): the plain message as before.+ plain = message_from_wire(_owner_message("Hi", None))+ assert [block.type for block in plain.blocks] == ["text"]++[email protected]("images", [[], [{"mime_type": "text/csv", "path": "uploads/a.csv", "workflow_id": "wf"}], "nope"])+def test_no_images_no_blocks(images):+ from services.temporal.agent_workflow import _owner_message++ assert [block.type for block in message_from_wire(_owner_message("Hi", images)).blocks] == ["text"]diff --git a/server/tests/nodes/test_speech_dictation.py b/server/tests/nodes/test_speech_dictation.pynew file mode 100644index 00000000..9e0902a4--- /dev/null+++ b/server/tests/nodes/test_speech_dictation.py@@ -0,0 +1,89 @@+"""Dictation for the chat's message box (nodes/speech/_handlers.py): it can+work only with a stored key for one of the configured providers, tried in+order; a recording must be one the chat uploaded under ``uploads/``; it is+transcribed with that provider and deleted, and only the workflow's owner+may ask."""++from __future__ import annotations++from pathlib import Path+from types import SimpleNamespace+from unittest.mock import AsyncMock, MagicMock++import pytest++from nodes.speech import _handlers++from ._mocks import patched_container, patched_pricing+++def database(owner: str = "") -> MagicMock:+ db = MagicMock(name="Database")+ db.get_workflow = AsyncMock(return_value=SimpleNamespace(data={"owner_id": owner}) if owner else None)+ db.save_api_usage_metric = AsyncMock(return_value=None)+ return db+++def workspace(tmp_path: Path, monkeypatch) -> Path:+ root = tmp_path / "workspace"+ (root / "uploads").mkdir(parents=True)++ async def fake_root(workflow_id, db, *, allow_default=True):+ return root++ monkeypatch.setattr("services.workspace_locator.resolve_workspace_root", fake_root)+ return root+++def test_the_providers_come_from_the_config_in_order():+ assert _handlers.dictation_providers() == ["openai", "groq", "deepgram"]+++async def test_dictation_needs_a_stored_key():+ with patched_container(auth_api_keys={"groq": "gsk-test"}, database=database()):+ assert await _handlers.handle_dictation_status({"session_id": "wf"}, None) == {+ "success": True,+ "available": True,+ "provider": "groq",+ }+ with patched_container(auth_api_keys={}, database=database()):+ status = await _handlers.handle_dictation_status({"session_id": "wf"}, None)+ assert (status["available"], status["provider"]) == (False, None)+ # The editor's chat with no workflow has nowhere to put a recording.+ assert (await _handlers.handle_dictation_status({"session_id": "default"}, None))["success"] is False+++async def test_a_recording_becomes_text_and_is_deleted(workspace, monkeypatch):+ recording = workspace / "uploads" / "dictation-1.webm"+ recording.write_bytes(b"webm-audio")+ transcribe = AsyncMock(return_value=SimpleNamespace(text="Send reminders for tomorrow", language="en", billed_units=None, billed_unit="seconds", duration_seconds=3.2))+ monkeypatch.setattr(_handlers._unifier, "transcribe", transcribe)+ with patched_container(auth_api_keys={"openai": "sk-test"}, database=database()), patched_pricing():+ result = await _handlers.handle_transcribe_audio({"session_id": "wf", "path": "uploads/dictation-1.webm"}, None)+ assert result == {"success": True, "text": "Send reminders for tomorrow", "language": "en", "provider": "openai"}+ assert not recording.exists()+ call = transcribe.await_args.kwargs+ assert (call["provider"], call["api_key"], call["request"].audio, call["request"].filename) == ("openai", "sk-test", b"webm-audio", "dictation-1.webm")++[email protected]("path, message", [("notes/dictation.webm", "uploaded"), ("uploads/missing.webm", "no longer there")])+async def test_what_cannot_be_transcribed(workspace, path, message):+ with patched_container(auth_api_keys={"openai": "sk-test"}, database=database()):+ result = await _handlers.handle_transcribe_audio({"session_id": "wf", "path": path}, None)+ assert result["success"] is False and message in result["error"]+++async def test_without_a_key_or_access_nothing_is_transcribed(workspace):+ recording = workspace / "uploads" / "dictation-2.webm"+ recording.write_bytes(b"webm-audio")+ with patched_container(auth_api_keys={}, database=database()):+ assert await _handlers.handle_transcribe_audio({"session_id": "wf", "path": "uploads/dictation-2.webm"}, None) == {+ "success": False,+ "error": "speech_unavailable",+ }+ socket = SimpleNamespace(scope={"path": "/ws/status"}, state=SimpleNamespace(user_id="someone-else"))+ with patched_container(auth_api_keys={"openai": "sk-test"}, database=database(owner="owner")):+ denied = await _handlers.handle_transcribe_audio({"session_id": "wf", "path": "uploads/dictation-2.webm"}, socket)+ assert denied == {"success": False, "error": "access_denied"}+ assert recording.exists()diff --git a/server/tests/services/chat/test_attachments.py b/server/tests/services/chat/test_attachments.pynew file mode 100644index 00000000..27452942--- /dev/null+++ b/server/tests/services/chat/test_attachments.py@@ -0,0 +1,169 @@+"""Files the owner attaches to a chat message (services/chat/attachments.py):+rebuilt from what was uploaded under ``uploads/``, never taken from the+client; carried on the message, to the employee as a bracketed line after+the owner's words and, for images, on the message itself; and what the+message box offers (``get_chat_context``)."""++from __future__ import annotations++from pathlib import Path++import pytest++from services.chat import attachments, guide, ledger+from tests.services.chat._helpers import talking++CLAIM = dict(temporal_workflow_id="tw-1", temporal_run_id="tr-1")+++def workspace(tmp_path: Path, monkeypatch) -> Path:+ root = tmp_path / "workspace"+ (root / "uploads").mkdir(parents=True)++ async def fake_root(workflow_id, database, *, allow_default=True):+ if workflow_id != "wf":+ raise ValueError("no such workflow")+ return root++ monkeypatch.setattr("services.workspace_locator.resolve_workspace_root", fake_root)+ return root+++def upload(root: Path, name: str, data: bytes = b"data") -> str:+ (root / "uploads" / name).write_bytes(data)+ return f"uploads/{name}"+++async def test_attachments_are_rebuilt_from_the_files(database, workspace):+ photo = upload(workspace, "shelf.png", b"\x89PNG" + b"0" * 60)+ sheet = upload(workspace, "prices.txt", b"a,b\n1,2\n")+ claimed = [{"path": photo, "size_bytes": 1, "mime_type": "text/html", "url": "https://evil.example"}, sheet, photo]+ refs = await attachments.check_attachments(database, "wf", claimed)+ assert [ref["path"] for ref in refs] == [photo, sheet]+ shelf, prices = refs+ assert (shelf["kind"], shelf["mime_type"], shelf["size_bytes"], shelf["filename"]) == ("image", "image/png", 64, "shelf.png")+ assert shelf["url"] == f"/api/workspace/wf/files/{photo}" and shelf["workflow_id"] == "wf"+ assert (prices["kind"], prices["mime_type"]) == ("file", "text/plain")+ assert await attachments.check_attachments(database, "wf", None) == []+++ "raw, reason",+ [+ ("uploads/shelf.png", "list"),+ (["notes.txt"], "uploaded"),+ (["../uploads/x.png"], "uploaded"),+ (["uploads/../secret.txt"], "uploaded"),+ ([{"path": 5}], "names a file"),+ (["uploads/missing.png"], "no longer there"),+ ([f"uploads/f{i}.txt" for i in range(7)], "at most 6"),+ ],+)+async def test_what_cannot_be_attached(database, workspace, raw, reason):+ with pytest.raises(attachments.AttachmentRefused, match=reason):+ await attachments.check_attachments(database, "wf", raw)+++def test_what_the_employee_reads_and_sees():+ refs = [+ {"path": "uploads/shelf.png", "filename": "shelf.png", "mime_type": "image/png", "size_bytes": 64, "workflow_id": "wf"},+ {"path": "uploads/prices.csv", "filename": "prices.csv", "mime_type": "text/csv", "size_bytes": 8, "workflow_id": "wf"},+ ]+ line = attachments.attachments_line(refs)+ assert line.startswith("[attachments]{") and line.endswith("}[/attachments]")+ assert '"path": "uploads/prices.csv"' in line and '"type": "image/png"' in line+ assert attachments.attachments_line([]) == ""+ assert [ref["path"] for ref in attachments.image_refs(refs)] == ["uploads/shelf.png"]+++# ----- sending ------------------------------------------------------------------------+++async def test_a_message_carries_its_files_to_the_employee(chat, workspace):+ await talking(chat.database)+ photo = upload(workspace, "shelf.png", b"\x89PNG" + b"0" * 60)+ sent = await chat.handlers.handle_send_chat_message(+ {"message": "Is this stock right?", "session_id": "wf", "attachments": [{"path": photo}], "options": {"web": False}}, None+ )+ assert sent["success"] is True+ [row] = await chat.database.get_chat_messages("wf")+ assert [ref["path"] for ref in row["attachments"]] == [photo]+ assert chat.dispatched[-1]["data"]["attachments"][0]["path"] == photo+ run = await ledger.get_run(chat.database, sent["run_id"])+ assert run.options == {"web": False}++ # The answering agent reads them after the owner's words, and gets the image.+ stream = {"run_id": run.run_id, "session_id": "wf", "workflow_id": "wf"}+ _, prompt = await guide.chat_turn(chat.database, stream, system_message="", prompt="Is this stock right?")+ assert prompt.startswith("Is this stock right?\n\n[attachments]{")+ assert [ref["path"] for ref in attachments.image_refs(await guide.run_attachments(chat.database, stream))] == [photo]+++async def test_files_alone_are_a_message_but_not_in_the_default_chat(chat, workspace):+ await talking(chat.database)+ sheet = upload(workspace, "prices.csv")+ alone = await chat.handlers.handle_send_chat_message({"message": "", "session_id": "wf", "attachments": [sheet]}, None)+ assert alone["success"] is True+ empty = await chat.handlers.handle_send_chat_message({"message": " ", "session_id": "wf"}, None)+ assert empty["error"] == "invalid_request"+ default = await chat.handlers.handle_send_chat_message({"message": "x", "session_id": "default", "attachments": [sheet]}, None)+ assert default["error"] == "invalid_request"+ refused = await chat.handlers.handle_send_chat_message({"message": "x", "session_id": "wf", "attachments": ["notes.txt"]}, None)+ assert refused["error"] == "attachment_rejected"+++async def test_an_edit_keeps_the_original_files(chat, workspace):+ await talking(chat.database)+ sheet = upload(workspace, "prices.csv")+ sent = await chat.handlers.handle_send_chat_message({"message": "Check these", "session_id": "wf", "attachments": [sheet]}, None)+ run = await ledger.start_run(chat.database, run_id=sent["run_id"], **CLAIM)+ await ledger.post_reply(chat.database, run=run, node_id="n", text="Done.", execution_id="gen-1")+ await ledger.finish_run(chat.database, run_id=run.run_id, success=True, **CLAIM)+ edited = await chat.handlers.handle_edit_chat_message({"session_id": "wf", "message_id": sent["message_id"], "message": "Check these again"}, None)+ assert edited["success"] is True+ assert chat.dispatched[-1]["data"]["attachments"][0]["path"] == sheet+++# ----- what the box offers ---------------------------------------------------------------+++async def test_the_box_offers_commands_attachments_and_web(chat, monkeypatch):+ import nodes # noqa: F401 - the plugin registry++ graph = {+ "nodes": [{"id": "wf:googleGmail:1", "type": "googleGmail"}, {"id": "wf:duckduckgoSearch:1", "type": "duckduckgoSearch"}],+ "edges": [],+ }++ async def get_workflow(workflow_id):+ from types import SimpleNamespace++ return SimpleNamespace(name="Maya", data=graph) if workflow_id == "wf" else None++ monkeypatch.setattr(chat.database, "get_workflow", get_workflow)+ context = await chat.handlers.handle_get_chat_context({"session_id": "wf"}, None)+ commands = {item["command"]: item for item in context["commands"]}+ assert commands["/inbox"]["suggest"] is True+ assert commands["/remember"]["description"] == "Teach Maya something"+ assert context["capabilities"] == {"attachments": True, "web": True}+ assert context["limits"]["max_attachments"] == 6++ plain = await chat.handlers.handle_get_chat_context({"session_id": "default"}, None)+ assert plain["commands"] == [] and plain["capabilities"] == {"attachments": False, "web": False}+++async def test_web_off_keeps_the_answering_agent_off_search(database):+ import nodes # noqa: F401+ from services.temporal.agent_activities import _without_web_tools++ await talking(database)+ admission = await ledger.admit_message(+ database, session_id="wf", workflow_id="wf", execution_id="gen-1", text="Hi", track=True, options={"web": False}+ )+ tools = [{"node_type": "duckduckgoSearch"}, {"node_type": "calculatorTool"}]+ context = {"run_scope": {"run_id": admission.run.run_id, "session_id": "wf"}}+ kept = await _without_web_tools(database, context, tools)+ assert [tool["node_type"] for tool in kept] == ["calculatorTool"]+ # Web on, or no chat run: every tool stays.+ assert await _without_web_tools(database, {}, tools) == toolsdiff --git a/server/tests/services/employees/test_app_registry.py b/server/tests/services/employees/test_app_registry.pyindex 5eac60a9..80024328 100644--- a/server/tests/services/employees/test_app_registry.py+++ b/server/tests/services/employees/test_app_registry.py@@ -251,8 +251,10 @@ _TOOL = '{"type": "browser", "side_effects": "money", %s}''{"name": "X", "provider_id": "p", "side_effects": "dangerous"}','{"name": "X", "provider_id": "p", "side_effects": "money", "tools": [%s]}' % (_TOOL % '"ask_first_params": ["read_only"]'),'{"name": "X", "provider_id": "p", "side_effects": "money", "tools": [%s]}' % (_TOOL % '"role": ""'),+ '{"name": "X", "provider_id": "p", "side_effects": "read", "commands": {"command": "/x"}}',+ '{"name": "X", "provider_id": "p", "side_effects": "read", "commands": [{"command": "Inbox"}]}',],- ids=["bad side effect", "ask_first_params not an object", "empty role"],+ ids=["bad side effect", "ask_first_params not an object", "empty role", "commands not a list", "command without a slash"],)def test_a_malformed_registry_fails_loudly(tmp_path, monkeypatch, app_json):bad = tmp_path / "employee_apps.json"@@ -265,3 +267,11 @@ def test_a_malformed_registry_fails_loudly(tmp_path, monkeypatch, app_json):finally:monkeypatch.undo()apps_module.reload_apps()+++def test_apps_add_slash_commands_to_the_chat():+ commands = {command["command"]: dict(command) for app in get_apps().values() for command in app.commands}+ assert commands["/inbox"] == {"command": "/inbox", "description": "Summarize unread email", "fill": "Summarize my unread email.", "suggest": True}+ assert set(commands) >= {"/draft", "/inbox", "/schedule"}+ for command in commands.values():+ assert re.fullmatch(r"/[a-z0-9-]{1,30}", command["command"]) and command["fill"]