Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Images for the model: refs in durable state, bytes only at the provider boundary.
A tool opts in by returning ``llm_media: [{"ref": <FileRef kind=image>,
"detail": "auto"}]``. The agent loop attaches ref-only image ContentBlocks to
the tool message (durable, ~450 B each). Images the owner attaches to a chat
message ride the same way on that user message (``image_blocks``).
``hydrate_image_blocks`` runs once per LLM step on throwaway copies, loading
bytes through the contained media reader and fitting them to a budget;
provider encoders then emit the official wire shapes. Originals are never
mutated, so nothing downstream of the journal ever sees bytes.
Two gates, because the encoders came at different times: images in a tool
result need ``vision.enabled``, images in the owner's message
``vision.user_images`` (``llm_defaults.json``). A model behind neither reads a
text placeholder; in the owner's message it names the file's workspace path,
so the employee can open it with a tool that can.
"""
from __future__ import annotations
from dataclasses import replace
from typing import Any, List, Sequence
from core.logging import get_logger
from services.llm.config import LLM_DEFAULTS
from services.llm.protocol import ContentBlock, Message
logger = get_logger(__name__)
LLM_MEDIA_KEY = "llm_media"
LLM_MEDIA_MAX_PER_RESULT = 8
IMAGE_MIME_ALLOWLIST = {"image/png", "image/jpeg", "image/webp", "image/gif"}
# detail -> visual-token budget for services.media.image_fit.
_DETAIL_BUDGETS = {"low": "small", "auto": "normal", "high": "large"}
def provider_supports_vision(provider: str, model: str = "") -> bool:
"""Capability gate. Unknown means False: a rejected image block would be
journaled and resent on every turn, so never emit one speculatively."""
del model # per-model gating arrives with registry modality data
vision = (
LLM_DEFAULTS.get("providers", {}).get(provider, {}).get("vision", {})
)
return bool(vision.get("enabled"))
def provider_supports_user_images(provider: str, model: str = "") -> bool:
"""Capability gate for images in the owner's message: ``vision.user_images``.
Unknown means False, like :func:`provider_supports_vision`."""
del model
vision = (
LLM_DEFAULTS.get("providers", {}).get(provider, {}).get("vision", {})
)
return bool(vision.get("user_images"))
def image_blocks(refs: Sequence[Any], *, detail: str = "auto") -> List[ContentBlock]:
"""Ref-only image blocks for images the owner attached (FileRef dumps
with an image type in :data:`IMAGE_MIME_ALLOWLIST`); others are left out."""
blocks: List[ContentBlock] = []
for ref in refs:
if not isinstance(ref, dict) or ref.get("mime_type") not in IMAGE_MIME_ALLOWLIST:
continue
if not ref.get("workflow_id") or not ref.get("path"):
continue
blocks.append(ContentBlock(type="image", source={"kind": "file_ref", "ref": dict(ref), "detail": detail}))
return blocks
def image_blocks_from_tool_result(result: Any) -> List[ContentBlock]:
"""Parse a tool result's ``llm_media`` opt-in into ref-only image blocks.
Invalid entries are skipped with a warning, never raised — the tool
already did its (possibly paid) work.
"""
if not isinstance(result, dict):
return []
entries = result.get(LLM_MEDIA_KEY)
if not isinstance(entries, list):
return []
blocks: List[ContentBlock] = []
for entry in entries[:LLM_MEDIA_MAX_PER_RESULT]:
ref = entry.get("ref") if isinstance(entry, dict) else None
if not isinstance(ref, dict):
logger.warning("llm_media entry without a ref; skipped")
continue
if ref.get("kind") not in ("image", "file"):
logger.warning(
"llm_media ref kind unsupported", kind=ref.get("kind")
)
continue
if ref.get("mime_type") not in IMAGE_MIME_ALLOWLIST:
logger.warning(
"llm_media mime unsupported", mime=ref.get("mime_type")
)
continue
if not ref.get("workflow_id") or not ref.get("path"):
logger.warning("llm_media ref missing workflow_id/path; skipped")
continue
detail = str(entry.get("detail") or "auto")
blocks.append(
ContentBlock(
type="image",
source={
"kind": "file_ref",
"ref": dict(ref),
"detail": detail if detail in _DETAIL_BUDGETS else "auto",
},
)
)
return blocks
def _placeholder(ref: dict, reason: str) -> ContentBlock:
filename = ref.get("filename") or ref.get("path") or "image"
return ContentBlock(
type="text", text=f"[Image attached: {filename} — {reason}]"
)
async def hydrate_image_blocks(
messages: Sequence[Message], *, provider: str, model: str
) -> List[Message]:
"""Return messages with image blocks hydrated for one provider call.
Identity (same objects, zero cost) when nothing carries an image ref.
Hydration works on copies; per-image failures degrade to a text
placeholder rather than failing an otherwise-valid turn.
"""
if not any(_ref_blocks(message) for message in messages):
return list(messages)
in_results = provider_supports_vision(provider, model)
in_owner_messages = provider_supports_user_images(provider, model)
hydrated: List[Message] = []
for message in messages:
if not _ref_blocks(message):
hydrated.append(message)
continue
if message.role == "user" and not in_owner_messages:
hydrated.append(_described(message, model))
continue
capable = in_owner_messages if message.role == "user" else in_results
blocks: List[ContentBlock] = []
for block in message.blocks:
if not _is_ref_block(block):
blocks.append(block)
continue
source = block.source or {}
ref = dict(source.get("ref") or {})
if not capable:
blocks.append(
_placeholder(
ref,
f"the current model ({model}) cannot view images; "
"metadata only",
)
)
continue
try:
data_b64, mime = await _hydrate_one(
ref, str(source.get("detail") or "auto")
)
blocks.append(
ContentBlock(
type="image",
source={
"kind": "bytes",
"media_type": mime,
"data_b64": data_b64,
"detail": source.get("detail") or "auto",
},
)
)
except Exception as exc:
logger.warning(
"image hydration failed", path=ref.get("path"), error=str(exc)
)
blocks.append(_placeholder(ref, f"unavailable: {exc}"))
hydrated.append(replace(message, blocks=blocks))
return hydrated
def _described(message: Message, model: str) -> Message:
"""The owner's message for a model that cannot view its images: each
image named in the text instead, with where it is in the workspace."""
notes = []
for block in _ref_blocks(message):
ref = dict((block.source or {}).get("ref") or {})
notes.append(
_placeholder(
ref,
f"the current model ({model}) cannot view images; it is at {ref.get('path')} in the workspace",
).text
)
content = "\n\n".join(part for part in (message.content, "\n".join(notes)) if part)
return replace(message, content=content, blocks=[ContentBlock(type="text", text=content)])
def _is_ref_block(block: ContentBlock) -> bool:
return (
block.type == "image"
and isinstance(block.source, dict)
and block.source.get("kind") == "file_ref"
)
def _ref_blocks(message: Message) -> List[ContentBlock]:
return [block for block in message.blocks or [] if _is_ref_block(block)]
async def _hydrate_one(ref: dict, detail: str) -> tuple[str, str]:
"""Resolve a FileRef to fitted base64 + mime, contained end to end."""
import asyncio
import base64
from nodes.filesystem._backend import resolve_within
from services.media.image_fit import fit_image_bytes
from services.media.limits import MEDIA_MAX_READ_BYTES
from services.plugin.deps import get_database
from services.workspace_locator import resolve_workspace_root
root = await resolve_workspace_root(
str(ref.get("workflow_id") or ""), get_database()
)
path = resolve_within(root, str(ref.get("path") or ""))
if not path.is_file():
raise FileNotFoundError(f"{ref.get('path')} not found in workspace")
if path.stat().st_size > MEDIA_MAX_READ_BYTES:
raise ValueError("image exceeds the read size cap")
budget = _DETAIL_BUDGETS.get(detail, "normal")
def fit() -> tuple[str, str]:
fitted, mime, _ = fit_image_bytes(path.read_bytes(), budget=budget)
return base64.b64encode(fitted).decode("ascii"), mime
return await asyncio.to_thread(fit)
__all__ = [
"IMAGE_MIME_ALLOWLIST",
"LLM_MEDIA_KEY",
"LLM_MEDIA_MAX_PER_RESULT",
"hydrate_image_blocks",
"image_blocks",
"provider_supports_user_images",
"image_blocks_from_tool_result",
"provider_supports_vision",
]