The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.
Rust agent-orchestrationai-agentsai-assistantdesktopllmlocal-firstmcppersonal-aiprivacyrustsecond-braintauri
/**
* Utilities for multimodal chat attachments.
*
* Originals travel as typed `ContentPart`s ({@link attachmentsToParts}) next to
* the message text; the core stages them and returns durable opaque typed
* references. {@link parseAttachmentReferences} remains for legacy marker
* records. The `[IMAGE:]`/`[FILE:]` marker grammar is an internal staging
* format of the core and is no longer built here.
*/
import debugFactory from 'debug';
import type { ContentPart } from '../services/chatWire';
const debug = debugFactory('chat:attachments');
const ALLOWED_IMAGE_MIME_TYPES = [
'image/png',
'image/jpeg',
'image/webp',
'image/gif',
'image/bmp',
] as const;
export type AllowedImageMimeType = (typeof ALLOWED_IMAGE_MIME_TYPES)[number];
const ALLOWED_FILE_MIME_TYPES = [
'application/pdf',
'text/plain',
'text/csv',
'text/markdown',
'application/zip',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/octet-stream',
] as const;
export type AllowedFileMimeType = (typeof ALLOWED_FILE_MIME_TYPES)[number];
// Known video types select a video chip; every other MIME is still accepted
// as an original file. Capability routing and extraction belong to the core.
const ALLOWED_VIDEO_MIME_TYPES = ['video/mp4', 'video/quicktime', 'video/webm'] as const;
export type AllowedVideoMimeType = (typeof ALLOWED_VIDEO_MIME_TYPES)[number];
export type AllowedAttachmentMimeType = string;
export type AttachmentKind = 'image' | 'file' | 'video';
export const ALLOWED_ATTACHMENT_MIME_TYPES = [
...ALLOWED_IMAGE_MIME_TYPES,
...ALLOWED_FILE_MIME_TYPES,
...ALLOWED_VIDEO_MIME_TYPES,
] as const;
// Original image uploads and all other original files have separate budgets.
export const ATTACHMENT_MAX_IMAGES = 4;
export const ATTACHMENT_MAX_FILES = 4;
export const ATTACHMENT_MAX_IMAGE_SIZE_BYTES = 8 * 1024 * 1024; // 8 MB
export const ATTACHMENT_MAX_FILE_SIZE_BYTES = 16 * 1024 * 1024; // 16 MB
export const ATTACHMENT_MAX_VIDEO_SIZE_BYTES = ATTACHMENT_MAX_FILE_SIZE_BYTES;
// Default number of stills for callers explicitly requesting a video preview.
export const VIDEO_FRAME_COUNT = 2;
export interface Attachment {
id: string;
kind: AttachmentKind;
file: File;
dataUri: string;
previewUri?: string;
mimeType: AllowedAttachmentMimeType;
originalSizeBytes: number;
payloadSizeBytes: number;
compressed: boolean;
// Optional preview frames only; originals are always sent as FILE markers.
frames?: string[];
}
type AttachmentError =
| { code: 'too_large'; sizeBytes: number; maxBytes: number }
| { code: 'too_many'; kind: AttachmentKind; max: number }
| { code: 'read_failed'; reason: string };
export function isAllowedMimeType(mime: string): mime is AllowedImageMimeType {
return (ALLOWED_IMAGE_MIME_TYPES as readonly string[]).includes(mime);
}
export function isVideoMimeType(mime: string): mime is AllowedVideoMimeType {
return (ALLOWED_VIDEO_MIME_TYPES as readonly string[]).includes(mime);
}
export function attachmentKindForMime(mime: AllowedAttachmentMimeType): AttachmentKind {
if (isAllowedMimeType(mime)) return 'image';
if (isVideoMimeType(mime)) return 'video';
return 'file';
}
export function fileToDataUri(file: Blob): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
const name = file instanceof File ? file.name : 'blob';
reader.onload = () => resolve(reader.result as string);
reader.onerror = () => reject(new Error(`Failed to read file: ${name}`));
reader.readAsDataURL(file);
});
}
async function blobToDataUri(blob: Blob, mimeType: string): Promise<string> {
// Blob normalizes MIME text to lowercase, including arbitrary parameters.
// Insert the encoded filename after reading so its case stays intact.
const typedBlob = new Blob([blob], { type: mimeType.split(';')[0] });
const uri = await fileToDataUri(typedBlob);
return `data:${mimeType};base64,${uri.slice(uri.indexOf(',') + 1)}`;
}
async function gzipBlob(file: File): Promise<Blob | null> {
if (!('CompressionStream' in globalThis)) return null;
try {
const compressionStream = new CompressionStream('gzip');
const compressed = file.stream().pipeThrough(compressionStream);
return await new Response(compressed).blob();
} catch (error) {
debug('[chat:attachments] gzip_failed name=%s error=%o', file.name, error);
return null;
}
}
function encodeDataUriParam(value: string): string {
return encodeURIComponent(value).replace(/'/g, '%27');
}
async function buildAttachmentDataUri(
file: File,
mimeType: AllowedAttachmentMimeType
): Promise<{ dataUri: string; payloadSizeBytes: number; compressed: boolean }> {
debug(
'[chat:attachments] compression:start name=%s mime=%s size=%d',
file.name,
mimeType,
file.size
);
const compressed = await gzipBlob(file);
if (compressed && compressed.size < file.size) {
const dataUri = await blobToDataUri(
compressed,
`application/gzip;original_mime=${encodeDataUriParam(mimeType)};name=${encodeDataUriParam(file.name)}`
);
debug(
'[chat:attachments] compression:ok name=%s original=%d compressed=%d',
file.name,
file.size,
compressed.size
);
return { dataUri, payloadSizeBytes: compressed.size, compressed: true };
}
const dataUri = await blobToDataUri(file, `${mimeType};name=${encodeDataUriParam(file.name)}`);
debug(
'[chat:attachments] compression:skipped name=%s original=%d compressed=%s',
file.name,
file.size,
compressed?.size ?? 'unavailable'
);
return { dataUri, payloadSizeBytes: file.size, compressed: false };
}
/** Evenly spread `count` sample points across the 0.1–0.9 span of the clip. */
function sampleFractions(count: number): number[] {
if (count <= 1) return [0.1];
const fractions: number[] = [];
for (let i = 0; i < count; i++) {
fractions.push(0.1 + (0.8 * i) / (count - 1));
}
return fractions;
}
function seekVideo(video: HTMLVideoElement, time: number): Promise<void> {
return new Promise((resolve, reject) => {
// Assigning currentTime to (approximately) its current value is a no-op and
// never fires `seeked` (HTML spec) — which would hang for zero-length clips
// or a first frame already at t=0. Short-circuit those.
if (Math.abs(video.currentTime - time) < 0.01) {
resolve();
return;
}
const cleanup = () => {
video.removeEventListener('seeked', onSeeked);
video.removeEventListener('error', onError);
};
const onSeeked = () => {
cleanup();
resolve();
};
const onError = () => {
cleanup();
reject(new Error('video seek failed'));
};
video.addEventListener('seeked', onSeeked);
video.addEventListener('error', onError);
video.currentTime = time;
});
}
/**
* Sample `count` still frames from a video file as JPEG data URIs by decoding it
* in a detached `<video>` element and painting each seek point onto a `<canvas>`.
* This optional preview helper does not affect original upload acceptance.
* Throws if the browser cannot decode the file. Requires a codec-capable browser;
* jsdom can't decode video, so unit tests stub {@link videoFrameExtractor}.
*/
async function extractVideoFramesImpl(
file: File,
count: number = VIDEO_FRAME_COUNT
): Promise<string[]> {
const url = URL.createObjectURL(file);
const video = document.createElement('video');
video.muted = true;
video.preload = 'auto';
video.src = url;
try {
await new Promise<void>((resolve, reject) => {
video.onloadedmetadata = () => resolve();
video.onerror = () => reject(new Error('video metadata load failed'));
});
const duration = Number.isFinite(video.duration) && video.duration > 0 ? video.duration : 0;
const canvas = document.createElement('canvas');
canvas.width = video.videoWidth || 320;
canvas.height = video.videoHeight || 240;
const ctx = canvas.getContext('2d');
if (!ctx) throw new Error('canvas 2d context unavailable');
const frames: string[] = [];
for (const fraction of sampleFractions(count)) {
const target = duration ? Math.min(duration * fraction, Math.max(duration - 0.05, 0)) : 0;
await seekVideo(video, target);
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
frames.push(canvas.toDataURL('image/jpeg', 0.7));
}
debug('[chat:attachments] video_frames name=%s count=%d', file.name, frames.length);
return frames;
} finally {
URL.revokeObjectURL(url);
video.removeAttribute('src');
}
}
/**
* Indirection seam so unit tests can stub frame extraction (jsdom has no video
* decoder). Production calls `extract` directly.
*/
export const videoFrameExtractor = { extract: extractVideoFramesImpl };
export function extractVideoFrames(file: File, count?: number): Promise<string[]> {
return videoFrameExtractor.extract(file, count);
}
/** Number of image markers sent for an original attachment. */
export function imageMarkerCost(kind: AttachmentKind): number {
return kind === 'image' ? 1 : 0;
}
export async function validateAndReadFile(
file: File,
existingImageMarkers: number,
existingFileCount = 0,
// Kept for compatibility with callers: capabilities affect core routing,
// never whether an original may be uploaded.
_allowImages = true
): Promise<{ attachment: Attachment } | { error: AttachmentError }> {
const mimeType = file.type || 'application/octet-stream';
const kind = attachmentKindForMime(mimeType);
if (kind !== 'image') {
if (existingFileCount >= ATTACHMENT_MAX_FILES) {
return { error: { code: 'too_many', kind: 'file', max: ATTACHMENT_MAX_FILES } };
}
} else if (existingImageMarkers >= ATTACHMENT_MAX_IMAGES) {
return { error: { code: 'too_many', kind: 'image', max: ATTACHMENT_MAX_IMAGES } };
}
const maxBytes =
kind === 'image'
? ATTACHMENT_MAX_IMAGE_SIZE_BYTES
: kind === 'video'
? ATTACHMENT_MAX_VIDEO_SIZE_BYTES
: ATTACHMENT_MAX_FILE_SIZE_BYTES;
if (file.size > maxBytes) {
return { error: { code: 'too_large', sizeBytes: file.size, maxBytes } };
}
try {
const { dataUri, payloadSizeBytes, compressed } = await buildAttachmentDataUri(file, mimeType);
const previewUri = kind === 'image' ? await fileToDataUri(file) : undefined;
return {
attachment: {
id: globalThis.crypto.randomUUID(),
kind,
file,
dataUri,
previewUri,
mimeType,
originalSizeBytes: file.size,
payloadSizeBytes,
compressed,
},
};
} catch (err) {
return {
error: { code: 'read_failed', reason: err instanceof Error ? err.message : String(err) },
};
}
}
/**
* The typed wire form of composer attachments (`ContentPart[]`): an `image`,
* `video` or `file` part whose media carries the original as a `data:` URI
* plus its name, type and size. Sent next to the message text through the
* `attachments` param of `channel.web_chat`, `threads.message_append` and
* `threads.edit_message`; the core stages the originals. The text is never
* touched, so no `[IMAGE:]`/`[FILE:]` marker is built here.
*/
export function attachmentsToParts(attachments: Attachment[]): ContentPart[] {
return attachments.map((a): ContentPart => {
const media = {
url: a.dataUri,
contentType: a.mimeType,
fileName: a.file.name,
sizeBytes: a.originalSizeBytes,
codec: null,
durationMs: null,
height: null,
localCachePath: null,
path: null,
platformMessageId: null,
transcribed: false,
width: null,
};
if (a.kind === 'image') {
return { kind: 'image', altText: null, caption: null, media: { ...media, kind: 'image' } };
}
if (a.kind === 'video') {
return {
kind: 'video',
caption: null,
durationMs: null,
thumbnail: null,
media: { ...media, kind: 'video' },
};
}
return {
kind: 'file',
fileName: a.file.name,
caption: null,
media: { ...media, kind: 'document' },
};
});
}
/** Workspace references returned by the core after it has saved originals. */
export interface AttachmentReference {
path: string;
name: string;
mime: string;
size_bytes: number;
}
/** Decode durable references without exposing raw JSON in message bubbles. */
export function parseAttachmentReferences(content: string): {
text: string;
attachments: AttachmentReference[];
} {
const attachments: AttachmentReference[] = [];
const text = content
.replace(/\[ATTACHMENT:([^\]]+)\]/g, (marker, encoded: string) => {
try {
const decoded = new URLSearchParams(`value=${encoded}`).get('value');
const value: unknown = JSON.parse(decoded ?? '');
if (!value || typeof value !== 'object') return marker;
const file = value as Partial<AttachmentReference>;
if (
typeof file.path !== 'string' ||
typeof file.name !== 'string' ||
typeof file.mime !== 'string' ||
typeof file.size_bytes !== 'number' ||
!Number.isSafeInteger(file.size_bytes) ||
file.size_bytes < 0
)
return marker;
// These are workspace references, never URLs or absolute browser targets.
const components = file.path.split('/');
if (
!file.path ||
file.path.includes('\\') ||
file.path.includes(':') ||
file.path.includes('\0') ||
components.some(part => part === '' || part === '.' || part === '..')
)
return marker;
attachments.push(file as AttachmentReference);
return '';
} catch {
return marker;
}
})
.replace(/ {2,}/g, ' ')
.trim();
return { text, attachments };
}
/** Display metadata excludes payload bytes; originals travel only in upload markers. */
export function attachmentMetadata(attachments: Attachment[]): Record<string, unknown> {
return attachments.length === 0
? {}
: {
attachmentCount: attachments.length,
attachmentNames: attachments.map(file => file.file.name),
attachmentKinds: attachments.map(file => file.kind),
attachmentCompressed: attachments.map(file => file.compressed),
};
}
/**
* Parse `[IMAGE:<data-uri>]` and `[FILE:<data-uri>]` markers out of a stored message string.
* Returns the clean text (markers removed) and the list of image data URIs found.
* File markers are stripped from text but not returned (file data lives in extraMetadata).
*/
export function parseMessageImages(content: string): { text: string; dataUris: string[] } {
const dataUris: string[] = [];
const text = parseAttachmentReferences(content)
.text.replace(/\[IMAGE:([^\]]+)\]/g, (_match, uri: string) => {
dataUris.push(uri);
return '';
})
.replace(/\[FILE:([^\]]+)\]/g, '') // Strip file markers
// Collapse only runs of plain spaces (not \s) left behind by marker
// removal — using \s here would also eat intentional newlines/paragraph
// breaks in the user's own text.
.replace(/ {2,}/g, ' ')
.trim();
return { text, dataUris };
}
export function formatFileSize(bytes: number): string {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}