Skip to content

Self-improving AI Employees for everyone

feat(chat): files, web off, dictation and slash commands for the message box

trohitg078e42bb2bf3
diff --git a/server/config/chat_defaults.json b/server/config/chat_defaults.json
index 718c2f82..fd5f69f2 100644
--- a/server/config/chat_defaults.json
+++ b/server/config/chat_defaults.json
@@ -19,5 +19,34 @@
"branches": {
"max_snapshots_per_session": 50,
"max_snapshot_bytes": 1048576
+ },
+ "commands": {
+ "_comment": "Slash commands every workflow's chat offers (get_chat_context); an employee's apps add theirs (employee_apps.json commands). fill is the text the command puts in the message box; suggest also shows it in an empty chat. {name} is the employee's name.",
+ "generic": [
+ {
+ "command": "/summarize",
+ "description": "Summarize this conversation",
+ "fill": "Summarize what we’ve agreed so far.",
+ "suggest": false
+ },
+ {
+ "command": "/remember",
+ "description": "Teach {name} something",
+ "fill": "Remember that ",
+ "suggest": false
+ },
+ {
+ "command": "/today",
+ "description": "What {name} did today",
+ "fill": "What have you done today?",
+ "suggest": true
+ },
+ {
+ "command": "/plan",
+ "description": "Plan something step by step",
+ "fill": "Make a plan for ",
+ "suggest": false
+ }
+ ]
}
}
diff --git a/server/config/employee_apps.json b/server/config/employee_apps.json
index 698f5838..6e9cc49f 100644
--- a/server/config/employee_apps.json
+++ b/server/config/employee_apps.json
@@ -57,7 +57,15 @@
"waiting": "Waiting for new WhatsApp messages",
"working": "Answering a WhatsApp message",
"reporting": "Sending you an update on WhatsApp"
- }
+ },
+ "commands": [
+ {
+ "command": "/draft",
+ "description": "Draft a WhatsApp reply",
+ "fill": "Draft a WhatsApp reply to ",
+ "suggest": false
+ }
+ ]
},
"whatsapp_business": {
"name": "WhatsApp Business",
@@ -258,7 +266,15 @@
"waiting": "Checking your inbox for new email",
"working": "Reading a new email",
"reporting": "Emailing you an update"
- }
+ },
+ "commands": [
+ {
+ "command": "/inbox",
+ "description": "Summarize unread email",
+ "fill": "Summarize my unread email.",
+ "suggest": true
+ }
+ ]
},
"outlook": {
"name": "Outlook",
@@ -317,7 +333,15 @@
"waiting": "Checking your Outlook inbox",
"working": "Reading a new email",
"reporting": "Emailing you an update"
- }
+ },
+ "commands": [
+ {
+ "command": "/inbox",
+ "description": "Summarize unread email",
+ "fill": "Summarize my unread email.",
+ "suggest": true
+ }
+ ]
},
"email": {
"name": "Email",
@@ -399,7 +423,15 @@
"side_effects": "send",
"phrases": {
"working": "Checking your calendar"
- }
+ },
+ "commands": [
+ {
+ "command": "/schedule",
+ "description": "Check a day in the calendar",
+ "fill": "What does tomorrow look like?",
+ "suggest": true
+ }
+ ]
},
"google_drive": {
"name": "Google Drive",
diff --git a/server/config/llm_defaults.json b/server/config/llm_defaults.json
index 209a9169..11059b8b 100644
--- a/server/config/llm_defaults.json
+++ b/server/config/llm_defaults.json
@@ -25,6 +25,8 @@
"max_tokens_param": "max_tokens",
"vision": {
"enabled": false,
+ "user_images": true,
+ "_user_images_note": "user_images: images the owner attaches to a chat message go to the model as image input in that user message (services/llm/media.py provider_supports_user_images; the provider's encoder renders them). Separate from enabled, which covers images in tool results.",
"_note": "Native image input is emitted only when enabled is true; unknown/absent providers get a text placeholder instead (a rejected image block would poison journaled history). Read by services/llm/media.py provider_supports_vision. False here until the OpenAI request encoder renders image blocks (Responses input_image in function_call_output; Chat Completions hoist-to-user-turn) — flipping it early would hydrate bytes the encoder silently drops. Anthropic ships the first encoder; text-only providers keep the visionAnalyze delegate tool."
},
"_models_note": "Ids, context, max output and prices follow the OpenRouter registry (2026-09-29). GPT-6 is the current generation: gpt-6-sol is the default ($2/$10 per MTok), gpt-6-luna the low tier ($0.10/$0.50) and gpt-6-astra the flagship ($10/$50); all have 1.05M context and 128K max output, and each -pro variant is priced like its base model. GPT-6 has no Terra model, so gpt-5.6-terra ($2/$12) stays the middle tier; the rest of the GPT-5.6 family, gpt-5.5, gpt-5.4 and gpt-4.1 stay curated as earlier tiers (unlike GPT-6, gpt-5.6-sol-pro costs twice its base: $4/$20). The rolling -latest aliases (gpt-astra-latest, gpt-sol-latest, gpt-terra-latest, gpt-luna-latest) are deliberately not curated: real model names only, like gemini. gpt-5.2, o3 and o4-mini are not curated because OpenAI is retiring them; the registry still describes them, so their limits resolve from it.",
@@ -81,7 +83,9 @@
"api_key_param": "anthropic_api_key",
"max_tokens_param": "max_tokens",
"vision": {
- "enabled": true
+ "enabled": true,
+ "user_images": true,
+ "_user_images_note": "user_images: images the owner attaches to a chat message go to the model as image input in that user message (services/llm/media.py provider_supports_user_images; the provider's encoder renders them). Separate from enabled, which covers images in tool results."
},
"max_output_tokens": {
"claude-opus-5-5": 128000,
@@ -141,6 +145,8 @@
"max_tokens_param": "max_output_tokens",
"vision": {
"enabled": false,
+ "user_images": true,
+ "_user_images_note": "user_images: images the owner attaches to a chat message go to the model as image input in that user message (services/llm/media.py provider_supports_user_images; the provider's encoder renders them). Separate from enabled, which covers images in tool results.",
"_note": "See openai.vision._note — off until the Gemini encoder renders inline_data parts beside function_response."
},
"_models_note": "Real (concrete) model names only — no -latest rolling aliases. The curated list (max_output_tokens keys) serves BOTH the Gemini Developer API (AIza keys) and Vertex / Agent Platform (AQ. keys); gemini-* entries live-verified on both endpoints (2026-06-12). Vertex 404s on -latest aliases and its models.list rejects API keys, which is why the dropdown is curated instead of live-fetched. Gemma exception: gemma-4-31b-it + gemma-4-26b-a4b-it are served ONLY on the Developer API (free tier, ai.google.dev/gemma/docs/core/gemma_on_gemini_api, verified 2026-07-11) — Vertex keys must not see them, hence vertex_incompatible_models below. Gemma 4 supports systemInstruction and function calling natively (unlike the gemma-3 era). Context and max output for every entry follow the OpenRouter registry (2026-09-29), including Gemma's 262K context and 16K / 236K max output.",
diff --git a/server/config/speech_defaults.json b/server/config/speech_defaults.json
index df0cb6ad..3cddce2b 100644
--- a/server/config/speech_defaults.json
+++ b/server/config/speech_defaults.json
@@ -210,5 +210,9 @@
"billed_unit": "seconds"
}
}
+ },
+ "dictation": {
+ "_comment": "Dictation in the chat's message box (nodes/speech/_handlers.py): the first of these speech-to-text providers with a stored key transcribes a recording (webm/opus from the browser). The microphone shows only when one has a key.",
+ "providers": ["openai", "groq", "deepgram"]
}
}
diff --git a/server/nodes/speech/__init__.py b/server/nodes/speech/__init__.py
index 1285508f..b52e6d92 100644
--- a/server/nodes/speech/__init__.py
+++ b/server/nodes/speech/__init__.py
@@ -28,12 +28,13 @@ Layout::
_credentials.py speech-only credentials
_base.py shared node helpers
_option_loaders.py dropdown loaders
+ _handlers.py dictation for the chat (dictation_status, transcribe_audio)
"""
from __future__ import annotations
from services.node_output_schemas import register_output_schema
-from services.ws_handler_registry import register_option_loader
+from services.ws_handler_registry import register_option_loader, register_ws_handlers
from ._option_loaders import (
load_speech_formats,
@@ -58,6 +59,12 @@ register_option_loader("speechLanguages", load_speech_languages)
register_option_loader("speechFormats", load_speech_formats)
register_option_loader("speechSampleRates", load_speech_sample_rates)
+# Dictation for the chat's message box: whether it can work, and turning a
+# recording into text.
+from ._handlers import WS_HANDLERS # noqa: E402
+
+register_ws_handlers(WS_HANDLERS)
+
__all__ = [
"SpeechToTextNode",
"SpeechToTextOutput",
diff --git a/server/nodes/speech/_handlers.py b/server/nodes/speech/_handlers.py
new file mode 100644
index 00000000..941bd6a2
--- /dev/null
+++ b/server/nodes/speech/_handlers.py
@@ -0,0 +1,136 @@
+"""Dictation for the chat's message box.
+
+``dictation_status`` says whether a recording can be turned into text: the
+first provider in ``speech_defaults.json``'s ``dictation.providers`` with a
+stored key (the microphone shows only when there is one; the browser's own
+speech recognition is not used, it does not work in the desktop app).
+``transcribe_audio`` transcribes a recording the chat uploaded into the
+workflow's workspace (``uploads/``) with that provider, records the usage,
+and deletes the recording: a dictation leaves nothing behind.
+
+Both answer only the workflow's owner, like the chat itself
+(``services/chat/access.py``).
+"""
+
+from __future__ import annotations
+
+import mimetypes
+from pathlib import PurePosixPath
+from types import SimpleNamespace
+from typing import Any, Dict, Optional
+
+from fastapi import WebSocket
+
+from core.logging import get_logger
+from services.plugin import NodeUserError
+from services.plugin.ws import ws_response
+
+from . import _config as speech_config
+from . import _unifier
+from ._base import track_usage
+from ._protocol import SttRequest
+from ._registry import stt_providers
+
+logger = get_logger(__name__)
+
+
+def dictation_providers() -> list[str]:
+ """The providers dictation may use, in the order it tries them."""
+ configured = (speech_config.SPEECH_DEFAULTS.get("dictation") or {}).get("providers") or []
+ available = set(stt_providers())
+ return [str(provider) for provider in configured if str(provider) in available]
+
+
+async def dictation_provider() -> Optional[str]:
+ """The first dictation provider with a stored key, or None."""
+ from core.container import container
+
+ auth = container.auth_service()
+ for provider in dictation_providers():
+ try:
+ if await auth.get_api_key(speech_config.credential_id(provider)):
+ return provider
+ except Exception: # noqa: BLE001 - an unreadable key is no key
+ logger.warning("Dictation key could not be read", provider=provider, exc_info=True)
+ return None
+
+
+async def _owner_session(data: Dict[str, Any], websocket: WebSocket) -> str:
+ from core.container import container
+ from services.chat.access import ChatAccessDenied, authorize_session, session_id_of
+
+ session_id = session_id_of(data)
+ try:
+ scope = await authorize_session(container.database(), websocket, session_id)
+ except ChatAccessDenied as exc:
+ raise NodeUserError("access_denied") from exc
+ if scope.workflow_id is None:
+ raise NodeUserError("Dictation needs a workflow's chat.")
+ return scope.workflow_id
+
+
+@ws_response
+async def handle_dictation_status(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """``{available, provider}``: whether a recording can be turned into
+ text now."""
+ await _owner_session(data, websocket)
+ provider = await dictation_provider()
+ return {"success": True, "available": provider is not None, "provider": provider}
+
+
+@ws_response
+async def handle_transcribe_audio(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """Transcribe the recording at ``path`` (under ``uploads/``) and delete
+ it. Answers ``{text, language, provider}``; ``speech_unavailable`` when no
+ dictation provider has a key."""
+ from core.container import container
+ from services.media.workspace import UPLOAD_SUBDIR, resolve_media
+ from services.workspace_locator import resolve_workspace_root
+
+ workflow_id = await _owner_session(data, websocket)
+ provider = await dictation_provider()
+ if provider is None:
+ return {"success": False, "error": "speech_unavailable"}
+ rel = str(data.get("path") or "").replace("\\", "/")
+ parts = PurePosixPath(rel).parts
+ if len(parts) < 2 or parts[0] != UPLOAD_SUBDIR:
+ raise NodeUserError("Only a recording uploaded to this chat can be transcribed.")
+ database = container.database()
+ root = await resolve_workspace_root(workflow_id, database, allow_default=False)
+ path = resolve_media(rel, workspace_dir=str(root))
+ if not path.is_file():
+ raise NodeUserError("The recording is no longer there. Record it again.")
+ max_bytes = speech_config.capability(provider, "stt", "max_upload_bytes", default=None)
+ size = path.stat().st_size
+ if isinstance(max_bytes, int) and size > max_bytes:
+ raise NodeUserError("The recording is too long to transcribe. Record a shorter one.")
+ try:
+ api_key = await container.auth_service().get_api_key(speech_config.credential_id(provider))
+ result = await _unifier.transcribe(
+ provider=provider,
+ api_key=str(api_key or ""),
+ request=SttRequest(
+ audio=path.read_bytes(),
+ filename=path.name,
+ mime_type=mimetypes.guess_type(path.name)[0] or "audio/webm",
+ model=speech_config.default_model(provider, "stt"),
+ language=str(data.get("language") or ""),
+ ),
+ )
+ finally:
+ try:
+ path.unlink(missing_ok=True)
+ except OSError:
+ logger.warning("Dictation recording could not be deleted", path=rel, exc_info=True)
+ usage = SimpleNamespace(raw={"session_id": workflow_id}, node_id="dictation", workflow_id=workflow_id)
+ units = result.billed_units if result.billed_units is not None else result.duration_seconds
+ await track_usage(usage, provider=provider, operation="speech_to_text", units=units, unit=result.billed_unit or "seconds")
+ return {"success": True, "text": result.text, "language": result.language, "provider": provider}
+
+
+WS_HANDLERS = {
+ "dictation_status": handle_dictation_status,
+ "transcribe_audio": handle_transcribe_audio,
+}
+
+__all__ = ["WS_HANDLERS", "dictation_provider", "dictation_providers", "handle_dictation_status", "handle_transcribe_audio"]
diff --git a/server/services/chat/attachments.py b/server/services/chat/attachments.py
new file mode 100644
index 00000000..7f765e56
--- /dev/null
+++ b/server/services/chat/attachments.py
@@ -0,0 +1,118 @@
+"""Files the owner attaches to a chat message (docs-internal/chat_protocol.md,
+"Attachments").
+
+The chat uploads each file into the workflow's workspace first
+(``POST /api/workspace/{workflow_id}/uploads``, which puts it under
+``uploads/``) and sends the references it got back with the message.
+``check_attachments`` trusts nothing in them but the path: each must name a
+file under the workspace's ``uploads/`` folder that exists, and the
+reference the message keeps is rebuilt here from the file itself (name,
+type, size, address). At most ``MAX_ATTACHMENTS`` per message.
+
+The employee reads them after the owner's words as one bracketed line
+(``attachments_line``), with their workspace paths, so its file tools can
+open them; images also travel with the message itself (``image_refs``) and
+reach a model that can view them (``services/llm/media.py``).
+"""
+
+from __future__ import annotations
+
+import json
+import mimetypes
+from datetime import datetime, timezone
+from pathlib import PurePosixPath
+from typing import Any, Dict, List, Optional, Sequence
+
+from services.media.workspace import UPLOAD_SUBDIR
+
+#: The most files one message carries.
+MAX_ATTACHMENTS = 6
+
+_KINDS = {"image": "image", "video": "video", "pdf": "document"}
+
+
+class AttachmentRefused(ValueError):
+ """An attachment that cannot go with the message; the text says why."""
+
+
+def _relative(raw: Any) -> str:
+ path = raw.get("path") if isinstance(raw, dict) else raw
+ if not isinstance(path, str) or not path.strip():
+ raise AttachmentRefused("each attachment names a file you uploaded")
+ parts = PurePosixPath(path.replace("\\", "/")).parts
+ if len(parts) < 2 or parts[0] != UPLOAD_SUBDIR or any(part in ("..", "") for part in parts):
+ raise AttachmentRefused("only files you uploaded to this chat can be attached")
+ return "/".join(parts)
+
+
+async def check_attachments(database: Any, workflow_id: str, raw: Any) -> List[Dict[str, Any]]:
+ """The message's attachments, each rebuilt from its file. Raises
+ :class:`AttachmentRefused`."""
+ if raw is None:
+ return []
+ if not isinstance(raw, list):
+ raise AttachmentRefused("attachments is a list of uploaded files")
+ if len(raw) > MAX_ATTACHMENTS:
+ raise AttachmentRefused(f"at most {MAX_ATTACHMENTS} files go with one message")
+ if not raw:
+ return []
+ from services.media.preview import preview_kind
+ from services.media.refs import FileRef
+ from services.media.workspace import resolve_media, workspace_file_url
+ from services.plugin import NodeUserError
+ from services.workspace_locator import resolve_workspace_root
+
+ try:
+ root = await resolve_workspace_root(workflow_id, database, allow_default=False)
+ except Exception as exc: # noqa: BLE001 - no workspace, nothing to attach
+ raise AttachmentRefused("this chat has no workspace to attach files from") from exc
+ refs: List[Dict[str, Any]] = []
+ seen = set()
+ for item in raw:
+ rel = _relative(item)
+ if rel in seen:
+ continue
+ seen.add(rel)
+ try:
+ path = resolve_media(rel, workspace_dir=str(root))
+ except NodeUserError as exc:
+ raise AttachmentRefused(str(exc)) from exc
+ if not path.is_file():
+ raise AttachmentRefused(f"{path.name} is no longer there; attach it again")
+ stat = path.stat()
+ mime_type = mimetypes.guess_type(path.name)[0] or "application/octet-stream"
+ refs.append(
+ FileRef(
+ kind=_KINDS.get(preview_kind(mime_type) or "", "file"),
+ path=rel,
+ workflow_id=workflow_id,
+ filename=path.name,
+ mime_type=mime_type,
+ size_bytes=stat.st_size,
+ modified_at=datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
+ url=workspace_file_url(workflow_id, rel),
+ ).model_dump(mode="json")
+ )
+ return refs
+
+
+def attachments_line(attachments: Sequence[Dict[str, Any]]) -> str:
+ """What the employee reads about the files after the owner's words."""
+ files = [
+ {"path": ref.get("path"), "name": ref.get("filename"), "type": ref.get("mime_type"), "bytes": ref.get("size_bytes")}
+ for ref in attachments
+ if isinstance(ref, dict) and ref.get("path")
+ ]
+ if not files:
+ return ""
+ return f"[attachments]{json.dumps({'files': files}, ensure_ascii=False, separators=(', ', ': '))}[/attachments]"
+
+
+def image_refs(attachments: Optional[Sequence[Dict[str, Any]]]) -> List[Dict[str, Any]]:
+ """The attached images a model that can view them is shown."""
+ from services.llm.media import IMAGE_MIME_ALLOWLIST
+
+ return [dict(ref) for ref in attachments or [] if isinstance(ref, dict) and ref.get("mime_type") in IMAGE_MIME_ALLOWLIST]
+
+
+__all__ = ["AttachmentRefused", "MAX_ATTACHMENTS", "attachments_line", "check_attachments", "image_refs"]
diff --git a/server/services/chat/branches.py b/server/services/chat/branches.py
index 9fd5bba1..9d7b795f 100644
--- a/server/services/chat/branches.py
+++ b/server/services/chat/branches.py
@@ -493,6 +493,7 @@ class _Edit(_Operation):
execution_id=self.live_root,
uid=self.uid,
run_id=run.run_id,
+ attachments=list(original.attachments or []),
meta={**self.meta, "edit_of": original.uid},
after=original.parent_uid,
)
diff --git a/server/services/chat/guide.py b/server/services/chat/guide.py
index 0814d3e4..1e96eb54 100644
--- a/server/services/chat/guide.py
+++ b/server/services/chat/guide.py
@@ -16,7 +16,7 @@ from __future__ import annotations
import json
import re
-from typing import Any, List, Mapping, Tuple
+from typing import Any, Dict, List, Mapping, Tuple
from core.logging import get_logger
from services.chat.stream import FOLLOWUPS_OPEN
@@ -34,7 +34,9 @@ CHAT_REPLY_GUIDE = (
"[ui-event]{...}[/ui-event] is a button they pressed in an interface you showed (with what they picked), and "
"[ui-state]{...}[/ui-state] is what they set in one without pressing anything; treat both as the owner's choice. "
"[update]{...}[/update] says what became of something you did (a draft sent or not, a part of the chat the owner "
- "went back from), and [feedback]{...}[/feedback] is how the owner rated one of your answers; learn from it.\n"
+ "went back from), and [feedback]{...}[/feedback] is how the owner rated one of your answers; learn from it. "
+ "[attachments]{...}[/attachments] after their words lists the files they attached, with their paths in your "
+ "workspace.\n"
"- A search result numbered n is a source: cite the ones you rely on as [n] right after what they support.\n"
"- When a few short next questions would help the owner, end your reply with "
'<followups>["...", "..."]</followups>: at most 3, each a short question in the owner\'s own words. The chat '
@@ -67,9 +69,28 @@ async def chat_turn(database: Any, chat_stream: Mapping[str, Any], *, system_mes
if claimed:
lines = notes_prompt(claimed)
prompt = f"{lines}\n\n{prompt}" if prompt else lines
+ from services.chat.attachments import attachments_line
+
+ attached = attachments_line(await run_attachments(database, chat_stream))
+ if attached:
+ prompt = f"{prompt}\n\n{attached}" if prompt else attached
return system, prompt
+async def run_attachments(database: Any, chat_stream: Mapping[str, Any]) -> List[Dict[str, Any]]:
+ """The files the owner attached to the message the run answers (none
+ when it cannot be read: the turn goes on without them)."""
+ from services.chat import ledger
+
+ try:
+ run = await ledger.get_run(database, str(chat_stream["run_id"]))
+ message = await ledger.saved_message(database, run.user_message_uid) if run and run.user_message_uid else None
+ except Exception: # noqa: BLE001
+ logger.warning("Chat attachments could not be read for a turn", run_id=chat_stream.get("run_id"), exc_info=True)
+ return []
+ return [dict(item) for item in (message.attachments or []) if isinstance(item, dict)] if message is not None else []
+
+
def _items(body: str) -> List[str]:
body = body.strip()
raw: List[str]
@@ -99,4 +120,12 @@ def split_followups(text: str) -> Tuple[str, List[str]]:
return visible, _items(inner)
-__all__ = ["CHAT_REPLY_GUIDE", "FOLLOWUPS_CLOSE", "MAX_FOLLOWUPS", "chat_turn", "split_followups", "with_chat_guide"]
+__all__ = [
+ "CHAT_REPLY_GUIDE",
+ "FOLLOWUPS_CLOSE",
+ "MAX_FOLLOWUPS",
+ "chat_turn",
+ "run_attachments",
+ "split_followups",
+ "with_chat_guide",
+]
diff --git a/server/services/chat/handlers.py b/server/services/chat/handlers.py
index f5d93ee3..5a6a0579 100644
--- a/server/services/chat/handlers.py
+++ b/server/services/chat/handlers.py
@@ -31,6 +31,7 @@ from core.container import container
from core.logging import get_logger
from services.chat import branches, ledger, reducer
from services.chat.access import ChatAccessDenied, authorize_session, session_id_of
+from services.chat.attachments import MAX_ATTACHMENTS, AttachmentRefused, check_attachments
from services.chat.feedback import FeedbackRefused, feedback_for, set_feedback
from services.chat.events import MESSAGE_WIRE_ROUTING_KEY, dispatch_chat_message_received
from services.chat.hub import get_chat_hub
@@ -182,18 +183,29 @@ async def _ui_event(database: Any, session_id: str, raw: Any) -> Dict[str, Any]:
async def _dispatch(
- session_id: str, workflow_id: Optional[str], *, message_uid: str, prompt: str, run_id: Optional[str], timestamp: str
+ session_id: str,
+ workflow_id: Optional[str],
+ *,
+ message_uid: str,
+ prompt: str,
+ run_id: Optional[str],
+ timestamp: str,
+ attachments: Optional[List[Dict[str, Any]]] = None,
) -> None:
"""Send the owner's message to the workflow's chat triggers: the run it
- starts is the event's id, so the trigger's child workflow is the run's."""
+ starts is the event's id, so the trigger's child workflow is the run's.
+ Its attachments ride along (the trigger's output carries them)."""
event_data: Dict[str, Any] = {"message": prompt, "timestamp": timestamp, "session_id": session_id, "message_id": message_uid}
if run_id is not None:
event_data["run_id"] = run_id
+ if attachments:
+ event_data["attachments"] = list(attachments)
await dispatch_chat_message_received(event_data, workflow_id=workflow_id, event_id=run_id)
logger.info("Chat message dispatched", session_id=session_id, run_id=run_id)
-@ws_handler("message")
+# ``message`` is checked below: a message may be only files.
+@ws_handler()
async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Save the owner's message, start its run, and dispatch it. Answers
``{message_id, run_id, delivery, timestamp}``; ``run_in_progress`` (with
@@ -203,9 +215,16 @@ async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -
With ``ui_event`` the message is a button pressed in an interface the
employee showed: checked against that interface, saved as the owner's
message reading the button's label (kind ``action``), and sent to the
- employee as a ``[ui-event]`` line naming the press."""
+ employee as a ``[ui-event]`` line naming the press.
+
+ ``attachments`` are files the chat uploaded under the workspace's
+ ``uploads/`` (``services/chat/attachments.py``: rebuilt from the files,
+ at most six; refused with ``attachment_rejected``); a message may be
+ only files. ``options.web: false`` keeps the employee off web search for
+ this message."""
message = data.get("message")
- if not isinstance(message, str) or not message.strip():
+ raw_attachments = data.get("attachments")
+ if not isinstance(message, str) or not (message.strip() or raw_attachments):
return {"success": False, "error": "invalid_request", "detail": "message must be text"}
if data.get("role", "user") != "user":
return {"success": False, "error": "invalid_request", "detail": "send_chat_message sends the owner's messages"}
@@ -217,6 +236,15 @@ async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -
except ChatAccessDenied:
return dict(_DENIED)
+ attachments: List[Dict[str, Any]] = []
+ if raw_attachments:
+ if scope.workflow_id is None:
+ return {"success": False, "error": "invalid_request", "detail": "files can be attached only in a workflow's chat"}
+ try:
+ attachments = await check_attachments(database, scope.workflow_id, raw_attachments)
+ except AttachmentRefused as exc:
+ return {"success": False, "error": "attachment_rejected", "detail": str(exc)}
+
press: Optional[Dict[str, Any]] = None
if data.get("ui_event") is not None:
try:
@@ -251,6 +279,8 @@ async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -
message_kind="action" if press is not None else "text",
client_message_id=data.get("client_message_id"),
meta=press["meta"] if press is not None else None,
+ options=_options(data.get("options")),
+ attachments=attachments or None,
)
except ledger.RunInProgress as exc:
return {"success": False, "error": "run_in_progress", "run_id": exc.run.run_id}
@@ -270,6 +300,7 @@ async def handle_send_chat_message(data: Dict[str, Any], websocket: WebSocket) -
prompt=press["prompt"] if press is not None else message,
run_id=run.run_id if run is not None else None,
timestamp=timestamp,
+ attachments=attachments,
)
response: Dict[str, Any] = {
@@ -445,6 +476,73 @@ async def handle_chat_ui_state(data: Dict[str, Any], websocket: WebSocket) -> Di
return {"success": True, "part_id": part_id, "state_revision": updated["state_revision"]}
+def _options(raw: Any) -> Dict[str, Any]:
+ """The per-message choices kept on the run: ``web`` (a bool) only."""
+ if isinstance(raw, dict) and isinstance(raw.get("web"), bool):
+ return {"web": raw["web"]}
+ return {}
+
+
+@ws_handler()
+async def handle_get_chat_context(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """What the session's message box offers: slash commands (the generic
+ ones in ``chat_defaults.json`` and those its employee's apps add,
+ ``employee_apps.json``; ``{name}`` is the workflow's name), whether files
+ can be attached (a workflow's chat), whether the Web chip has web search
+ to turn off (a search tool in the saved graph), and the limits."""
+ from services.chat.config import load_chat_config
+ from services.employees.apps import app_for_node_type
+ from services.media.limits import MEDIA_MAX_UPLOAD_BYTES
+ from services.node_registry import get_node_class
+
+ session_id = session_id_of(data)
+ database = container.database()
+ try:
+ scope = await authorize_session(database, websocket, session_id)
+ except ChatAccessDenied:
+ return dict(_DENIED)
+ saved = await database.get_workflow(scope.workflow_id) if scope.workflow_id is not None else None
+ graph = (getattr(saved, "data", None) or {}) if saved is not None else {}
+ name = str(getattr(saved, "name", "") or "the employee")
+ node_types = [str(node.get("type") or "") for node in graph.get("nodes") or [] if isinstance(node, dict)]
+
+ configured = (load_chat_config().get("commands") or {}).get("generic") or []
+ commands: List[Dict[str, Any]] = []
+ seen: set = set()
+
+ def add(command: Any) -> None:
+ if not isinstance(command, dict) or command.get("command") in seen or not scope.workflow_id:
+ return
+ seen.add(command.get("command"))
+ commands.append(
+ {
+ "command": str(command.get("command")),
+ "description": str(command.get("description") or "").replace("{name}", name),
+ "fill": str(command.get("fill") or "").replace("{name}", name),
+ "suggest": bool(command.get("suggest")),
+ }
+ )
+
+ for node_type in node_types:
+ app = app_for_node_type(node_type)
+ for command in app.commands if app is not None else ():
+ add(dict(command))
+ for command in configured:
+ add(command)
+
+ def searches(node_type: str) -> bool:
+ cls = get_node_class(node_type)
+ return cls is not None and "search" in tuple(getattr(cls, "group", ()) or ())
+
+ return {
+ "success": True,
+ "session_id": session_id,
+ "commands": commands,
+ "capabilities": {"attachments": scope.workflow_id is not None, "web": any(searches(t) for t in node_types)},
+ "limits": {"max_attachments": MAX_ATTACHMENTS, "max_upload_bytes": MEDIA_MAX_UPLOAD_BYTES},
+ }
+
+
def _expected_revision(data: Dict[str, Any]) -> Optional[int]:
value = data.get("expected_revision")
if value is None:
@@ -504,7 +602,13 @@ async def _answer_branch(database: Any, moved: "branches.Moved", context: Dict[s
await announce_chat_updated(moved.session_id, role)
message, run = moved.result["message"], moved.result["run"]
await _dispatch(
- moved.session_id, moved.workflow_id, message_uid=message["uid"], prompt=moved.result["prompt"], run_id=run.run_id, timestamp=timestamp,
+ moved.session_id,
+ moved.workflow_id,
+ message_uid=message["uid"],
+ prompt=moved.result["prompt"],
+ run_id=run.run_id,
+ timestamp=timestamp,
+ attachments=list(message.get("attachments") or []),
)
response: Dict[str, Any] = {"success": True, "message_id": message["uid"], "run_id": run.run_id}
if context.get("delivery") is not None:
@@ -687,6 +791,7 @@ WS_HANDLERS = {
"regenerate_chat_reply": handle_regenerate_chat_reply,
"switch_chat_branch": handle_switch_chat_branch,
"set_chat_feedback": handle_set_chat_feedback,
+ "get_chat_context": handle_get_chat_context,
"clear_chat_messages": handle_clear_chat_messages,
"save_chat_message": handle_save_chat_message,
}
diff --git a/server/services/employees/apps.py b/server/services/employees/apps.py
index b3e46e65..510e20d0 100644
--- a/server/services/employees/apps.py
+++ b/server/services/employees/apps.py
@@ -86,6 +86,9 @@ class AppSpec:
#: agent only, never to a worker strangers write to; every call waits
#: for the owner while they ask first (services/approvals/tool_calls.py).
talk_send: Optional[NodeTemplate] = None
+ #: Slash commands this app adds to the employee's chat
+ #: (``{command, description, fill, suggest}``; get_chat_context).
+ commands: Tuple[Mapping[str, Any], ...] = ()
@property
def node_types(self) -> frozenset:
@@ -171,10 +174,37 @@ def _parse(raw: Mapping[str, Any]) -> Dict[str, AppSpec]:
tools=tools,
side_effects=effect,
phrases=MappingProxyType({str(k): str(v) for k, v in (entry.get("phrases") or {}).items()}),
+ commands=_commands(app_id, entry.get("commands")),
)
return apps
+_COMMAND = re.compile(r"^/[a-z0-9-]{1,30}$")
+
+
+def _commands(app_id: str, raw: Any) -> Tuple[Mapping[str, Any], ...]:
+ """An app's slash commands: ``{command, description, fill, suggest}``."""
+ if raw is None:
+ return ()
+ if not isinstance(raw, list):
+ raise AppRegistryError(f"{app_id}.commands must be a list")
+ commands = []
+ for item in raw:
+ if not isinstance(item, dict) or not isinstance(item.get("command"), str) or not _COMMAND.match(item["command"]):
+ raise AppRegistryError(f"{app_id}.commands: each needs a `command` like /schedule")
+ commands.append(
+ MappingProxyType(
+ {
+ "command": item["command"],
+ "description": str(item.get("description") or ""),
+ "fill": str(item.get("fill") or ""),
+ "suggest": bool(item.get("suggest")),
+ }
+ )
+ )
+ return tuple(commands)
+
+
_apps: Optional[Dict[str, AppSpec]] = None
_by_node_type: Optional[Dict[str, AppSpec]] = None
diff --git a/server/services/llm/media.py b/server/services/llm/media.py
index 47729bb6..bae840c6 100644
--- a/server/services/llm/media.py
+++ b/server/services/llm/media.py
@@ -1,12 +1,19 @@
-"""Tool-result images: refs in durable state, bytes only at the provider boundary.
+"""Images for the model: refs in durable state, bytes only at the provider boundary.
A tool opts in by returning ``llm_media: [{"ref": <FileRef kind=image>,
"detail": "auto"}]``. The agent loop attaches ref-only image ContentBlocks to
-the tool message (durable, ~450 B each). ``hydrate_image_blocks`` runs once
-per LLM step on throwaway copies, loading bytes through the contained media
-reader and fitting them to a budget; provider encoders then emit the
-official wire shapes. Originals are never mutated, so nothing downstream of
-the journal ever sees bytes.
+the tool message (durable, ~450 B each). Images the owner attaches to a chat
+message ride the same way on that user message (``image_blocks``).
+``hydrate_image_blocks`` runs once per LLM step on throwaway copies, loading
+bytes through the contained media reader and fitting them to a budget;
+provider encoders then emit the official wire shapes. Originals are never
+mutated, so nothing downstream of the journal ever sees bytes.
+
+Two gates, because the encoders came at different times: images in a tool
+result need ``vision.enabled``, images in the owner's message
+``vision.user_images`` (``llm_defaults.json``). A model behind neither reads a
+text placeholder; in the owner's message it names the file's workspace path,
+so the employee can open it with a tool that can.
"""
from __future__ import annotations
@@ -38,6 +45,29 @@ def provider_supports_vision(provider: str, model: str = "") -> bool:
return bool(vision.get("enabled"))
+def provider_supports_user_images(provider: str, model: str = "") -> bool:
+ """Capability gate for images in the owner's message: ``vision.user_images``.
+ Unknown means False, like :func:`provider_supports_vision`."""
+ del model
+ vision = (
+ LLM_DEFAULTS.get("providers", {}).get(provider, {}).get("vision", {})
+ )
+ return bool(vision.get("user_images"))
+
+
+def image_blocks(refs: Sequence[Any], *, detail: str = "auto") -> List[ContentBlock]:
+ """Ref-only image blocks for images the owner attached (FileRef dumps
+ with an image type in :data:`IMAGE_MIME_ALLOWLIST`); others are left out."""
+ blocks: List[ContentBlock] = []
+ for ref in refs:
+ if not isinstance(ref, dict) or ref.get("mime_type") not in IMAGE_MIME_ALLOWLIST:
+ continue
+ if not ref.get("workflow_id") or not ref.get("path"):
+ continue
+ blocks.append(ContentBlock(type="image", source={"kind": "file_ref", "ref": dict(ref), "detail": detail}))
+ return blocks
+
+
def image_blocks_from_tool_result(result: Any) -> List[ContentBlock]:
"""Parse a tool result's ``llm_media`` opt-in into ref-only image blocks.
@@ -101,12 +131,17 @@ async def hydrate_image_blocks(
if not any(_ref_blocks(message) for message in messages):
return list(messages)
- capable = provider_supports_vision(provider, model)
+ in_results = provider_supports_vision(provider, model)
+ in_owner_messages = provider_supports_user_images(provider, model)
hydrated: List[Message] = []
for message in messages:
if not _ref_blocks(message):
hydrated.append(message)
continue
+ if message.role == "user" and not in_owner_messages:
+ hydrated.append(_described(message, model))
+ continue
+ capable = in_owner_messages if message.role == "user" else in_results
blocks: List[ContentBlock] = []
for block in message.blocks:
if not _is_ref_block(block):
@@ -147,6 +182,22 @@ async def hydrate_image_blocks(
return hydrated
+def _described(message: Message, model: str) -> Message:
+ """The owner's message for a model that cannot view its images: each
+ image named in the text instead, with where it is in the workspace."""
+ notes = []
+ for block in _ref_blocks(message):
+ ref = dict((block.source or {}).get("ref") or {})
+ notes.append(
+ _placeholder(
+ ref,
+ f"the current model ({model}) cannot view images; it is at {ref.get('path')} in the workspace",
+ ).text
+ )
+ content = "\n\n".join(part for part in (message.content, "\n".join(notes)) if part)
+ return replace(message, content=content, blocks=[ContentBlock(type="text", text=content)])
+
+
def _is_ref_block(block: ContentBlock) -> bool:
return (
block.type == "image"
@@ -192,6 +243,8 @@ __all__ = [
"LLM_MEDIA_KEY",
"LLM_MEDIA_MAX_PER_RESULT",
"hydrate_image_blocks",
+ "image_blocks",
+ "provider_supports_user_images",
"image_blocks_from_tool_result",
"provider_supports_vision",
]
diff --git a/server/services/llm/providers/anthropic.py b/server/services/llm/providers/anthropic.py
index 72176473..056e07fb 100644
--- a/server/services/llm/providers/anthropic.py
+++ b/server/services/llm/providers/anthropic.py
@@ -238,7 +238,15 @@ class AnthropicProvider:
hydrated images ride along (the documented image-in-tool-result
shape). Ref-only blocks render nothing here — hydration either
produced bytes or a text placeholder before compile time."""
- images = [
+ images = AnthropicProvider._image_parts(m)
+ if not images:
+ return m.content
+ return [{"type": "text", "text": m.content}, *images]
+
+ @staticmethod
+ def _image_parts(m: Message) -> List[Dict[str, Any]]:
+ """The message's hydrated images as Anthropic image blocks."""
+ return [
{
"type": "image",
"source": {
@@ -252,9 +260,6 @@ class AnthropicProvider:
and isinstance(block.source, dict)
and block.source.get("kind") == "bytes"
]
- if not images:
- return m.content
- return [{"type": "text", "text": m.content}, *images]
def _to_api_message(self, m: Message) -> Dict[str, Any]:
role = "assistant" if m.role == "assistant" else "user"
@@ -302,6 +307,12 @@ class AnthropicProvider:
)
return {"role": "assistant", "content": content}
+ images = self._image_parts(m) if m.role == "user" else []
+ if images:
+ # Images the owner attached, ahead of their words (Anthropic's
+ # recommended order).
+ text = [{"type": "text", "text": m.content}] if m.content else []
+ return {"role": "user", "content": [*images, *text]}
return {"role": role, "content": m.content}
@staticmethod
diff --git a/server/services/llm/providers/gemini.py b/server/services/llm/providers/gemini.py
index e9130184..8df8927a 100644
--- a/server/services/llm/providers/gemini.py
+++ b/server/services/llm/providers/gemini.py
@@ -256,7 +256,17 @@ class GeminiProvider:
continue
role = "model" if m.role == "assistant" else "user"
- contents.append({"role": role, "parts": [{"text": m.content}]})
+ parts: List[Dict[str, Any]] = [{"text": m.content}] if m.content or m.role != "user" else []
+ if m.role == "user":
+ # Images the owner attached, as inline data parts.
+ parts.extend(
+ {"inline_data": {"mime_type": block.source["media_type"], "data": base64.b64decode(block.source["data_b64"])}}
+ for block in m.blocks or []
+ if block.type == "image"
+ and isinstance(block.source, dict)
+ and block.source.get("kind") == "bytes"
+ )
+ contents.append({"role": role, "parts": parts or [{"text": m.content}]})
index += 1
system = "\n\n".join(system_parts) if system_parts else None
diff --git a/server/services/llm/providers/openai.py b/server/services/llm/providers/openai.py
index 55ef3efd..14b8c9fe 100644
--- a/server/services/llm/providers/openai.py
+++ b/server/services/llm/providers/openai.py
@@ -30,6 +30,26 @@ _RESPONSES_TERMINAL_EVENTS = frozenset({"response.completed", "response.incomple
logger = get_logger(__name__)
+def _image_sources(m: Message) -> List[Dict[str, Any]]:
+ """The message's hydrated images (``source`` dicts with ``data_b64``)."""
+ return [
+ block.source
+ for block in m.blocks or []
+ if block.type == "image"
+ and isinstance(block.source, dict)
+ and block.source.get("kind") == "bytes"
+ ]
+
+
+def _data_url(source: Dict[str, Any]) -> str:
+ return f"data:{source['media_type']};base64,{source['data_b64']}"
+
+
+def _detail(source: Dict[str, Any]) -> str:
+ detail = str(source.get("detail") or "auto")
+ return detail if detail in ("low", "high", "auto") else "auto"
+
+
class OpenAIProvider:
provider_name = "openai"
@@ -370,6 +390,16 @@ class OpenAIProvider:
result[key] = payload[key]
return result
+ images = _image_sources(m) if m.role == "user" else []
+ if images:
+ # Images the owner attached: Chat Completions takes them as
+ # image_url parts (a data URL) beside the text.
+ parts: List[Dict[str, Any]] = [{"type": "text", "text": m.content}] if m.content else []
+ parts.extend(
+ {"type": "image_url", "image_url": {"url": _data_url(source), "detail": _detail(source)}}
+ for source in images
+ )
+ return {"role": "user", "content": parts}
return {"role": m.role, "content": m.content}
def _to_api_tool(self, tool: ToolDef) -> Dict[str, Any]:
@@ -452,6 +482,19 @@ class OpenAIProvider:
)
continue
+ images = _image_sources(message) if message.role == "user" else []
+ if images:
+ # Images the owner attached: Responses takes them as
+ # input_image parts beside input_text.
+ content: List[Dict[str, Any]] = (
+ [{"type": "input_text", "text": message.content}] if message.content else []
+ )
+ content.extend(
+ {"type": "input_image", "image_url": _data_url(source), "detail": _detail(source)}
+ for source in images
+ )
+ items.append({"role": "user", "content": content})
+ continue
items.append(
{"role": message.role, "content": message.content}
)
diff --git a/server/services/temporal/agent_activities.py b/server/services/temporal/agent_activities.py
index e35b6e8c..d064f3c9 100644
--- a/server/services/temporal/agent_activities.py
+++ b/server/services/temporal/agent_activities.py
@@ -912,6 +912,32 @@ async def store_agent_output(payload: Dict[str, Any]) -> Dict[str, Any]:
return {"stored": True}
+async def _without_web_tools(database: Any, context: Dict[str, Any], tool_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+ """The tools, without the web search ones when the owner turned Web off
+ for the message the chat run answers (``options.web``)."""
+ from services.chat.stream import chat_run_id_of
+
+ run_id = chat_run_id_of(context)
+ if not run_id or not tool_data:
+ return tool_data
+ try:
+ from services.chat.ledger import get_run
+
+ run = await get_run(database, run_id)
+ except Exception: # noqa: BLE001 - the run keeps its tools
+ activity.logger.warning("Chat run options could not be read for %s", run_id, exc_info=True)
+ return tool_data
+ if run is None or (run.options or {}).get("web", True) is not False:
+ return tool_data
+ from services.node_registry import get_node_class
+
+ def searches(tool: Dict[str, Any]) -> bool:
+ cls = get_node_class(str(tool.get("node_type") or ""))
+ return cls is not None and "search" in tuple(getattr(cls, "group", ()) or ())
+
+ return [tool for tool in tool_data if not searches(tool)]
+
+
async def _record_chat_cursor(
database: Any, context: Dict[str, Any], *, node_id: str, generation: int, stored: List[Dict[str, Any]]
) -> None:
@@ -1333,7 +1359,7 @@ async def prepare_agent_payload(context: Dict[str, Any]) -> Dict[str, Any]:
tools_payload: List[Dict[str, Any]] = []
from services.skill_runtime import skill_tool_info
- effective_tool_data = list(tool_data or [])
+ effective_tool_data = await _without_web_tools(database, context, list(tool_data or []))
progressive_skill_tool = skill_tool_info(skill_data or [], node_id)
if progressive_skill_tool:
effective_tool_data.append(progressive_skill_tool)
@@ -1516,6 +1542,12 @@ async def prepare_agent_payload(context: Dict[str, Any]) -> Dict[str, Any]:
system_message, prompt = await chat_turn(
database, chat_stream, system_message=system_message, prompt=prompt
)
+ from services.chat.attachments import image_refs
+ from services.chat.guide import run_attachments
+
+ user_images = image_refs(await run_attachments(database, chat_stream))
+ else:
+ user_images = []
return {
# The chat run this agent works for (stops when the owner presses
@@ -1532,6 +1564,9 @@ async def prepare_agent_payload(context: Dict[str, Any]) -> Dict[str, Any]:
"temperature": temperature,
"system_message": system_message,
"user_prompt": prompt,
+ # Images the owner attached to the message this run answers: the
+ # opening user message carries them (services/llm/media.py).
+ "user_images": user_images,
"tools": tools_payload,
"memory_node_id": memory_node_id,
"memory_content": memory_content,
diff --git a/server/services/temporal/agent_workflow.py b/server/services/temporal/agent_workflow.py
index e0e36d3a..9e9298aa 100644
--- a/server/services/temporal/agent_workflow.py
+++ b/server/services/temporal/agent_workflow.py
@@ -187,6 +187,20 @@ def _native_message(
)
+def _owner_message(content: str, images: Any) -> Dict[str, Any]:
+ """The run's opening user message: the prompt, and the images the owner
+ attached as ref-only image blocks (hydrated per provider call,
+ ``services/llm/media.py``)."""
+
+ from services.llm.media import image_blocks
+ from services.llm.protocol import ContentBlock, Message, message_to_wire
+
+ blocks = image_blocks(images if isinstance(images, list) else [])
+ if not blocks:
+ return _native_message(role="user", content=content)
+ return message_to_wire(Message(role="user", content=content, blocks=[ContentBlock(type="text", text=content), *blocks]))
+
+
def _append_tool_result_message(
messages: List[Dict[str, Any]],
*,
@@ -773,9 +787,7 @@ class AgentWorkflow:
if isinstance(wire, dict) and wire.get("role") != "system"
)
if user_prompt:
- messages.append(
- _native_message(role="user", content=user_prompt)
- )
+ messages.append(_owner_message(user_prompt, payload.get("user_images")))
else:
if system:
messages.append(_native_message(role="system", content=system))
@@ -790,9 +802,7 @@ class AgentWorkflow:
)
if user_prompt:
- messages.append(
- _native_message(role="user", content=user_prompt)
- )
+ messages.append(_owner_message(user_prompt, payload.get("user_images")))
# Map LLM tool name -> {node_type, version, task_queue, node_id,
# parameters} so the workflow can schedule the right activity
diff --git a/server/tests/llm/test_user_images.py b/server/tests/llm/test_user_images.py
new file mode 100644
index 00000000..0052ae3b
--- /dev/null
+++ b/server/tests/llm/test_user_images.py
@@ -0,0 +1,162 @@
+"""Images the owner attaches to a chat message: carried as ref-only blocks on
+the opening user message, hydrated only for a provider that declares
+``vision.user_images`` (each encoder's own shape: Anthropic image blocks,
+OpenAI image_url / input_image, Gemini inline_data), and named with their
+workspace path for a model that cannot view them."""
+
+from __future__ import annotations
+
+import base64
+from unittest.mock import patch
+
+import pytest
+
+from services.llm.media import hydrate_image_blocks, image_blocks, provider_supports_user_images
+from services.llm.protocol import ContentBlock, Message, message_from_wire
+
+REF = {
+ "kind": "image",
+ "path": "uploads/shelf.png",
+ "workflow_id": "wf",
+ "filename": "shelf.png",
+ "mime_type": "image/png",
+ "size_bytes": 64,
+}
+PIXEL = base64.b64encode(b"png-bytes").decode("ascii")
+
+
+def owner_message(text: str = "Is this stock right?") -> Message:
+ return Message(role="user", content=text, blocks=[ContentBlock(type="text", text=text), *image_blocks([REF])])
+
+
+def hydrated_message(text: str = "Is this stock right?") -> Message:
+ source = {"kind": "bytes", "media_type": "image/png", "data_b64": PIXEL, "detail": "auto"}
+ return Message(role="user", content=text, blocks=[ContentBlock(type="text", text=text), ContentBlock(type="image", source=source)])
+
+
+def test_only_attached_images_become_blocks():
+ blocks = image_blocks([REF, {**REF, "mime_type": "text/csv"}, {**REF, "path": ""}, "nope"])
+ assert [(block.type, block.source["kind"], block.source["ref"]["path"]) for block in blocks] == [("image", "file_ref", "uploads/shelf.png")]
+
+
+def test_the_gate_names_the_providers_with_an_encoder():
+ for provider in ("anthropic", "openai", "gemini"):
+ assert provider_supports_user_images(provider) is True
+ for provider in ("deepseek", "groq", "openrouter", "nope"):
+ assert provider_supports_user_images(provider) is False
+
+
+async def test_a_capable_provider_gets_bytes_on_the_owner_message(monkeypatch):
+ async def fake_hydrate(ref, detail):
+ return PIXEL, "image/png"
+
+ monkeypatch.setattr("services.llm.media._hydrate_one", fake_hydrate)
+ message = owner_message()
+ [hydrated] = await hydrate_image_blocks([message], provider="openai", model="gpt-6")
+ images = [block for block in hydrated.blocks if block.type == "image"]
+ assert images[0].source == {"kind": "bytes", "media_type": "image/png", "data_b64": PIXEL, "detail": "auto"}
+ # The durable message keeps its refs.
+ assert any(block.source and block.source.get("kind") == "file_ref" for block in message.blocks)
+
+
+async def test_a_model_that_cannot_view_images_reads_where_they_are(monkeypatch):
+ async def never(ref, detail):
+ raise AssertionError("no bytes for a model that cannot view them")
+
+ monkeypatch.setattr("services.llm.media._hydrate_one", never)
+ [hydrated] = await hydrate_image_blocks([owner_message()], provider="deepseek", model="deepseek-chat")
+ assert all(block.type != "image" for block in hydrated.blocks)
+ assert hydrated.content.startswith("Is this stock right?\n\n[Image attached: shelf.png")
+ assert "uploads/shelf.png in the workspace" in hydrated.content
+
+
+async def test_tool_results_keep_their_own_gate(monkeypatch):
+ async def fake_hydrate(ref, detail):
+ return PIXEL, "image/png"
+
+ monkeypatch.setattr("services.llm.media._hydrate_one", fake_hydrate)
+ tool = Message(role="tool", content="{}", tool_call_id="t1", blocks=[ContentBlock(type="text", text="{}"), *image_blocks([REF])])
+ # OpenAI views the owner's images but not yet a tool's.
+ [hydrated] = await hydrate_image_blocks([tool], provider="openai", model="gpt-6")
+ assert all(block.type != "image" for block in hydrated.blocks)
+
+
+def test_anthropic_puts_the_images_ahead_of_the_words():
+ with patch("anthropic.AsyncAnthropic"):
+ from services.llm.providers.anthropic import AnthropicProvider
+
+ provider = AnthropicProvider("key")
+ api = provider._to_api_message(hydrated_message())
+ assert api == {
+ "role": "user",
+ "content": [
+ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PIXEL}},
+ {"type": "text", "text": "Is this stock right?"},
+ ],
+ }
+ # Only images: no empty text block.
+ assert provider._to_api_message(hydrated_message(""))["content"] == [api["content"][0]]
+ # A message with nothing to view stays a string.
+ assert provider._to_api_message(Message(role="user", content="Hi")) == {"role": "user", "content": "Hi"}
+
+
+def test_openai_sends_data_urls_on_both_apis():
+ with patch("openai.AsyncOpenAI"):
+ from services.llm.providers.openai import OpenAIProvider
+
+ provider = OpenAIProvider("key")
+ url = f"data:image/png;base64,{PIXEL}"
+ assert provider._to_api_message(hydrated_message()) == {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Is this stock right?"},
+ {"type": "image_url", "image_url": {"url": url, "detail": "auto"}},
+ ],
+ }
+ [item] = provider._to_responses_input([hydrated_message()])
+ assert item == {
+ "role": "user",
+ "content": [
+ {"type": "input_text", "text": "Is this stock right?"},
+ {"type": "input_image", "image_url": url, "detail": "auto"},
+ ],
+ }
+ assert provider._to_api_message(Message(role="user", content="Hi")) == {"role": "user", "content": "Hi"}
+
+
+def test_gemini_sends_inline_data():
+ with patch("google.genai.Client"):
+ from services.llm.providers.gemini import GeminiProvider
+
+ provider = GeminiProvider("key")
+ _, contents = provider._split_system_and_contents([hydrated_message()])
+ assert contents == [
+ {
+ "role": "user",
+ "parts": [
+ {"text": "Is this stock right?"},
+ {"inline_data": {"mime_type": "image/png", "data": b"png-bytes"}},
+ ],
+ }
+ ]
+ _, plain = provider._split_system_and_contents([Message(role="user", content="Hi")])
+ assert plain == [{"role": "user", "parts": [{"text": "Hi"}]}]
+
+
+def test_the_opening_message_carries_the_images():
+ from services.temporal.agent_workflow import _owner_message
+
+ wire = _owner_message("Is this stock right?", [REF])
+ message = message_from_wire(wire)
+ assert message.content == "Is this stock right?"
+ assert [(block.type, (block.source or {}).get("kind")) for block in message.blocks] == [("text", None), ("image", "file_ref")]
+ # No images (or none the owner sent): the plain message as before.
+ plain = message_from_wire(_owner_message("Hi", None))
+ assert [block.type for block in plain.blocks] == ["text"]
+
+
[email protected]("images", [[], [{"mime_type": "text/csv", "path": "uploads/a.csv", "workflow_id": "wf"}], "nope"])
+def test_no_images_no_blocks(images):
+ from services.temporal.agent_workflow import _owner_message
+
+ assert [block.type for block in message_from_wire(_owner_message("Hi", images)).blocks] == ["text"]
diff --git a/server/tests/nodes/test_speech_dictation.py b/server/tests/nodes/test_speech_dictation.py
new file mode 100644
index 00000000..9e0902a4
--- /dev/null
+++ b/server/tests/nodes/test_speech_dictation.py
@@ -0,0 +1,89 @@
+"""Dictation for the chat's message box (nodes/speech/_handlers.py): it can
+work only with a stored key for one of the configured providers, tried in
+order; a recording must be one the chat uploaded under ``uploads/``; it is
+transcribed with that provider and deleted, and only the workflow's owner
+may ask."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from nodes.speech import _handlers
+
+from ._mocks import patched_container, patched_pricing
+
+
+def database(owner: str = "") -> MagicMock:
+ db = MagicMock(name="Database")
+ db.get_workflow = AsyncMock(return_value=SimpleNamespace(data={"owner_id": owner}) if owner else None)
+ db.save_api_usage_metric = AsyncMock(return_value=None)
+ return db
+
+
+def workspace(tmp_path: Path, monkeypatch) -> Path:
+ root = tmp_path / "workspace"
+ (root / "uploads").mkdir(parents=True)
+
+ async def fake_root(workflow_id, db, *, allow_default=True):
+ return root
+
+ monkeypatch.setattr("services.workspace_locator.resolve_workspace_root", fake_root)
+ return root
+
+
+def test_the_providers_come_from_the_config_in_order():
+ assert _handlers.dictation_providers() == ["openai", "groq", "deepgram"]
+
+
+async def test_dictation_needs_a_stored_key():
+ with patched_container(auth_api_keys={"groq": "gsk-test"}, database=database()):
+ assert await _handlers.handle_dictation_status({"session_id": "wf"}, None) == {
+ "success": True,
+ "available": True,
+ "provider": "groq",
+ }
+ with patched_container(auth_api_keys={}, database=database()):
+ status = await _handlers.handle_dictation_status({"session_id": "wf"}, None)
+ assert (status["available"], status["provider"]) == (False, None)
+ # The editor's chat with no workflow has nowhere to put a recording.
+ assert (await _handlers.handle_dictation_status({"session_id": "default"}, None))["success"] is False
+
+
+async def test_a_recording_becomes_text_and_is_deleted(workspace, monkeypatch):
+ recording = workspace / "uploads" / "dictation-1.webm"
+ recording.write_bytes(b"webm-audio")
+ transcribe = AsyncMock(return_value=SimpleNamespace(text="Send reminders for tomorrow", language="en", billed_units=None, billed_unit="seconds", duration_seconds=3.2))
+ monkeypatch.setattr(_handlers._unifier, "transcribe", transcribe)
+ with patched_container(auth_api_keys={"openai": "sk-test"}, database=database()), patched_pricing():
+ result = await _handlers.handle_transcribe_audio({"session_id": "wf", "path": "uploads/dictation-1.webm"}, None)
+ assert result == {"success": True, "text": "Send reminders for tomorrow", "language": "en", "provider": "openai"}
+ assert not recording.exists()
+ call = transcribe.await_args.kwargs
+ assert (call["provider"], call["api_key"], call["request"].audio, call["request"].filename) == ("openai", "sk-test", b"webm-audio", "dictation-1.webm")
+
+
[email protected]("path, message", [("notes/dictation.webm", "uploaded"), ("uploads/missing.webm", "no longer there")])
+async def test_what_cannot_be_transcribed(workspace, path, message):
+ with patched_container(auth_api_keys={"openai": "sk-test"}, database=database()):
+ result = await _handlers.handle_transcribe_audio({"session_id": "wf", "path": path}, None)
+ assert result["success"] is False and message in result["error"]
+
+
+async def test_without_a_key_or_access_nothing_is_transcribed(workspace):
+ recording = workspace / "uploads" / "dictation-2.webm"
+ recording.write_bytes(b"webm-audio")
+ with patched_container(auth_api_keys={}, database=database()):
+ assert await _handlers.handle_transcribe_audio({"session_id": "wf", "path": "uploads/dictation-2.webm"}, None) == {
+ "success": False,
+ "error": "speech_unavailable",
+ }
+ socket = SimpleNamespace(scope={"path": "/ws/status"}, state=SimpleNamespace(user_id="someone-else"))
+ with patched_container(auth_api_keys={"openai": "sk-test"}, database=database(owner="owner")):
+ denied = await _handlers.handle_transcribe_audio({"session_id": "wf", "path": "uploads/dictation-2.webm"}, socket)
+ assert denied == {"success": False, "error": "access_denied"}
+ assert recording.exists()
diff --git a/server/tests/services/chat/test_attachments.py b/server/tests/services/chat/test_attachments.py
new file mode 100644
index 00000000..27452942
--- /dev/null
+++ b/server/tests/services/chat/test_attachments.py
@@ -0,0 +1,169 @@
+"""Files the owner attaches to a chat message (services/chat/attachments.py):
+rebuilt from what was uploaded under ``uploads/``, never taken from the
+client; carried on the message, to the employee as a bracketed line after
+the owner's words and, for images, on the message itself; and what the
+message box offers (``get_chat_context``)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from services.chat import attachments, guide, ledger
+from tests.services.chat._helpers import talking
+
+CLAIM = dict(temporal_workflow_id="tw-1", temporal_run_id="tr-1")
+
+
+def workspace(tmp_path: Path, monkeypatch) -> Path:
+ root = tmp_path / "workspace"
+ (root / "uploads").mkdir(parents=True)
+
+ async def fake_root(workflow_id, database, *, allow_default=True):
+ if workflow_id != "wf":
+ raise ValueError("no such workflow")
+ return root
+
+ monkeypatch.setattr("services.workspace_locator.resolve_workspace_root", fake_root)
+ return root
+
+
+def upload(root: Path, name: str, data: bytes = b"data") -> str:
+ (root / "uploads" / name).write_bytes(data)
+ return f"uploads/{name}"
+
+
+async def test_attachments_are_rebuilt_from_the_files(database, workspace):
+ photo = upload(workspace, "shelf.png", b"\x89PNG" + b"0" * 60)
+ sheet = upload(workspace, "prices.txt", b"a,b\n1,2\n")
+ claimed = [{"path": photo, "size_bytes": 1, "mime_type": "text/html", "url": "https://evil.example"}, sheet, photo]
+ refs = await attachments.check_attachments(database, "wf", claimed)
+ assert [ref["path"] for ref in refs] == [photo, sheet]
+ shelf, prices = refs
+ assert (shelf["kind"], shelf["mime_type"], shelf["size_bytes"], shelf["filename"]) == ("image", "image/png", 64, "shelf.png")
+ assert shelf["url"] == f"/api/workspace/wf/files/{photo}" and shelf["workflow_id"] == "wf"
+ assert (prices["kind"], prices["mime_type"]) == ("file", "text/plain")
+ assert await attachments.check_attachments(database, "wf", None) == []
+
+
+ "raw, reason",
+ [
+ ("uploads/shelf.png", "list"),
+ (["notes.txt"], "uploaded"),
+ (["../uploads/x.png"], "uploaded"),
+ (["uploads/../secret.txt"], "uploaded"),
+ ([{"path": 5}], "names a file"),
+ (["uploads/missing.png"], "no longer there"),
+ ([f"uploads/f{i}.txt" for i in range(7)], "at most 6"),
+ ],
+)
+async def test_what_cannot_be_attached(database, workspace, raw, reason):
+ with pytest.raises(attachments.AttachmentRefused, match=reason):
+ await attachments.check_attachments(database, "wf", raw)
+
+
+def test_what_the_employee_reads_and_sees():
+ refs = [
+ {"path": "uploads/shelf.png", "filename": "shelf.png", "mime_type": "image/png", "size_bytes": 64, "workflow_id": "wf"},
+ {"path": "uploads/prices.csv", "filename": "prices.csv", "mime_type": "text/csv", "size_bytes": 8, "workflow_id": "wf"},
+ ]
+ line = attachments.attachments_line(refs)
+ assert line.startswith("[attachments]{") and line.endswith("}[/attachments]")
+ assert '"path": "uploads/prices.csv"' in line and '"type": "image/png"' in line
+ assert attachments.attachments_line([]) == ""
+ assert [ref["path"] for ref in attachments.image_refs(refs)] == ["uploads/shelf.png"]
+
+
+# ----- sending ------------------------------------------------------------------------
+
+
+async def test_a_message_carries_its_files_to_the_employee(chat, workspace):
+ await talking(chat.database)
+ photo = upload(workspace, "shelf.png", b"\x89PNG" + b"0" * 60)
+ sent = await chat.handlers.handle_send_chat_message(
+ {"message": "Is this stock right?", "session_id": "wf", "attachments": [{"path": photo}], "options": {"web": False}}, None
+ )
+ assert sent["success"] is True
+ [row] = await chat.database.get_chat_messages("wf")
+ assert [ref["path"] for ref in row["attachments"]] == [photo]
+ assert chat.dispatched[-1]["data"]["attachments"][0]["path"] == photo
+ run = await ledger.get_run(chat.database, sent["run_id"])
+ assert run.options == {"web": False}
+
+ # The answering agent reads them after the owner's words, and gets the image.
+ stream = {"run_id": run.run_id, "session_id": "wf", "workflow_id": "wf"}
+ _, prompt = await guide.chat_turn(chat.database, stream, system_message="", prompt="Is this stock right?")
+ assert prompt.startswith("Is this stock right?\n\n[attachments]{")
+ assert [ref["path"] for ref in attachments.image_refs(await guide.run_attachments(chat.database, stream))] == [photo]
+
+
+async def test_files_alone_are_a_message_but_not_in_the_default_chat(chat, workspace):
+ await talking(chat.database)
+ sheet = upload(workspace, "prices.csv")
+ alone = await chat.handlers.handle_send_chat_message({"message": "", "session_id": "wf", "attachments": [sheet]}, None)
+ assert alone["success"] is True
+ empty = await chat.handlers.handle_send_chat_message({"message": " ", "session_id": "wf"}, None)
+ assert empty["error"] == "invalid_request"
+ default = await chat.handlers.handle_send_chat_message({"message": "x", "session_id": "default", "attachments": [sheet]}, None)
+ assert default["error"] == "invalid_request"
+ refused = await chat.handlers.handle_send_chat_message({"message": "x", "session_id": "wf", "attachments": ["notes.txt"]}, None)
+ assert refused["error"] == "attachment_rejected"
+
+
+async def test_an_edit_keeps_the_original_files(chat, workspace):
+ await talking(chat.database)
+ sheet = upload(workspace, "prices.csv")
+ sent = await chat.handlers.handle_send_chat_message({"message": "Check these", "session_id": "wf", "attachments": [sheet]}, None)
+ run = await ledger.start_run(chat.database, run_id=sent["run_id"], **CLAIM)
+ await ledger.post_reply(chat.database, run=run, node_id="n", text="Done.", execution_id="gen-1")
+ await ledger.finish_run(chat.database, run_id=run.run_id, success=True, **CLAIM)
+ edited = await chat.handlers.handle_edit_chat_message({"session_id": "wf", "message_id": sent["message_id"], "message": "Check these again"}, None)
+ assert edited["success"] is True
+ assert chat.dispatched[-1]["data"]["attachments"][0]["path"] == sheet
+
+
+# ----- what the box offers ---------------------------------------------------------------
+
+
+async def test_the_box_offers_commands_attachments_and_web(chat, monkeypatch):
+ import nodes # noqa: F401 - the plugin registry
+
+ graph = {
+ "nodes": [{"id": "wf:googleGmail:1", "type": "googleGmail"}, {"id": "wf:duckduckgoSearch:1", "type": "duckduckgoSearch"}],
+ "edges": [],
+ }
+
+ async def get_workflow(workflow_id):
+ from types import SimpleNamespace
+
+ return SimpleNamespace(name="Maya", data=graph) if workflow_id == "wf" else None
+
+ monkeypatch.setattr(chat.database, "get_workflow", get_workflow)
+ context = await chat.handlers.handle_get_chat_context({"session_id": "wf"}, None)
+ commands = {item["command"]: item for item in context["commands"]}
+ assert commands["/inbox"]["suggest"] is True
+ assert commands["/remember"]["description"] == "Teach Maya something"
+ assert context["capabilities"] == {"attachments": True, "web": True}
+ assert context["limits"]["max_attachments"] == 6
+
+ plain = await chat.handlers.handle_get_chat_context({"session_id": "default"}, None)
+ assert plain["commands"] == [] and plain["capabilities"] == {"attachments": False, "web": False}
+
+
+async def test_web_off_keeps_the_answering_agent_off_search(database):
+ import nodes # noqa: F401
+ from services.temporal.agent_activities import _without_web_tools
+
+ await talking(database)
+ admission = await ledger.admit_message(
+ database, session_id="wf", workflow_id="wf", execution_id="gen-1", text="Hi", track=True, options={"web": False}
+ )
+ tools = [{"node_type": "duckduckgoSearch"}, {"node_type": "calculatorTool"}]
+ context = {"run_scope": {"run_id": admission.run.run_id, "session_id": "wf"}}
+ kept = await _without_web_tools(database, context, tools)
+ assert [tool["node_type"] for tool in kept] == ["calculatorTool"]
+ # Web on, or no chat run: every tool stays.
+ assert await _without_web_tools(database, {}, tools) == tools
diff --git a/server/tests/services/employees/test_app_registry.py b/server/tests/services/employees/test_app_registry.py
index 5eac60a9..80024328 100644
--- a/server/tests/services/employees/test_app_registry.py
+++ b/server/tests/services/employees/test_app_registry.py
@@ -251,8 +251,10 @@ _TOOL = '{"type": "browser", "side_effects": "money", %s}'
'{"name": "X", "provider_id": "p", "side_effects": "dangerous"}',
'{"name": "X", "provider_id": "p", "side_effects": "money", "tools": [%s]}' % (_TOOL % '"ask_first_params": ["read_only"]'),
'{"name": "X", "provider_id": "p", "side_effects": "money", "tools": [%s]}' % (_TOOL % '"role": ""'),
+ '{"name": "X", "provider_id": "p", "side_effects": "read", "commands": {"command": "/x"}}',
+ '{"name": "X", "provider_id": "p", "side_effects": "read", "commands": [{"command": "Inbox"}]}',
],
- ids=["bad side effect", "ask_first_params not an object", "empty role"],
+ ids=["bad side effect", "ask_first_params not an object", "empty role", "commands not a list", "command without a slash"],
)
def test_a_malformed_registry_fails_loudly(tmp_path, monkeypatch, app_json):
bad = tmp_path / "employee_apps.json"
@@ -265,3 +267,11 @@ def test_a_malformed_registry_fails_loudly(tmp_path, monkeypatch, app_json):
finally:
monkeypatch.undo()
apps_module.reload_apps()
+
+
+def test_apps_add_slash_commands_to_the_chat():
+ commands = {command["command"]: dict(command) for app in get_apps().values() for command in app.commands}
+ assert commands["/inbox"] == {"command": "/inbox", "description": "Summarize unread email", "fill": "Summarize my unread email.", "suggest": True}
+ assert set(commands) >= {"/draft", "/inbox", "/schedule"}
+ for command in commands.values():
+ assert re.fullmatch(r"/[a-z0-9-]{1,30}", command["command"]) and command["fill"]