Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance MIME type detection and result handling in asset management

konstantin-paulusb6bf78ac6490
diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.ts
index 8a5b5b3b..173aa556 100644
--- a/apps/web/src/utils/gen-ai.ts
+++ b/apps/web/src/utils/gen-ai.ts
@@ -18,7 +18,7 @@ import {
PROMPT_INPUT_VOICE_MODEL,
PROMPT_INPUT_VOICE_OPTIONS,
} from "@/components/genai/config";
-import { assert } from "@/utils";
+import { assert, mimeTypeToExtension } from "@/utils";
import { uploadBlob } from "@/lib/uploads";
import { track } from "@/lib/analytics";
import { trpc } from "@/lib/trpc";
@@ -396,7 +396,7 @@ export class EditorGenAi extends GenAi {
/**
* Downloads what a model returned and files it in the library. The name is
- * the model's, the extension the URL's — what came back decides what the
+ * the model's, the extension the result's — what came back decides what the
* file is called, not what was asked for.
*/
private async store(url: string, name: string, generation: { key: string; id?: string | null }): Promise<Asset> {
@@ -404,10 +404,11 @@ export class EditorGenAi extends GenAi {
assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);
const blob = await response.blob();
- const extension = url.split(/[?#]/)[0].split(".").pop();
- const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;
-
- return this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation });
+ return this.library.store(blob, {
+ name: name + resultExtension(url, blob),
+ folder: GENERATED_DIR,
+ generation,
+ });
}
private requestGeneration(spec: ResolvedSpec) {
@@ -479,6 +480,24 @@ export class EditorGenAi extends GenAi {
/** Whether an asset type is footage, for the calls that only take footage. */
const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";
+/**
+ * What to call a generated result: the extension its type implies, falling
+ * back to the one its URL carries. Neither is guaranteed — some models hand
+ * back a plain slug (`.../files/young-man-city-skyline`), some servers say
+ * only `application/octet-stream` — so this can come back empty, and the
+ * library reads the bytes instead. The name is what the file is identified
+ * by once on disk, so getting it right saves that guess.
+ */
+function resultExtension(url: string, blob: Blob): string {
+ const fromType = blob.type ? mimeTypeToExtension(blob.type) : ".bin";
+ if (fromType !== ".bin") return fromType;
+
+ const fileName = url.split(/[?#]/)[0]!.split("/").pop() ?? "";
+ const dot = fileName.lastIndexOf(".");
+ const fromUrl = dot > 0 ? fileName.slice(dot + 1) : "";
+ return fromUrl && fromUrl.length <= 5 ? `.${fromUrl}` : "";
+}
+
/**
* Says what a generation failed with, and hands the error on. The message is
* what the element ends up carrying too (the runtime's `SourceError`, which
diff --git a/packages/assets/src/probe.ts b/packages/assets/src/probe.ts
index 713ec516..1f832b4b 100644
--- a/packages/assets/src/probe.ts
+++ b/packages/assets/src/probe.ts
@@ -26,9 +26,11 @@ const TRANSCRIPT_TYPES = new Set(['application/json', 'application/x-subrip', 't
/**
* The MIME type of a file, or of the resource behind a URL. Subtitle files go
- * by extension (OS registries rarely map .srt); audio and video are sniffed
- * by mediabunny, since a container's declared type is often wrong or empty.
- * Null when the file is not something the library takes.
+ * by extension (OS registries rarely map .srt); images fall back to their
+ * magic bytes when nothing declared a type (a download filed without an
+ * extension has none to go by); audio and video are sniffed by mediabunny,
+ * since a container's declared type is often wrong or empty. Null when the
+ * file is not something the library takes.
*/
export async function detectMimeType(input: Blob | string): Promise<string | null> {
const name = typeof input === 'string' ? input.split(/[?#]/)[0]! : (input as File).name ?? '';
@@ -41,6 +43,11 @@ export async function detectMimeType(input: Blob | string): Promise<string | nul
if (mimeType?.startsWith('image/')) return mimeType;
if (mimeType?.startsWith('text/html')) return mimeType;
+ if (typeof input !== 'string') {
+ const sniffed = await sniffImageType(input);
+ if (sniffed) return sniffed;
+ }
+
try {
const source = typeof input === 'string' ? new UrlSource(input) : new BlobSource(input);
mimeType = await new Input({ formats: ALL_FORMATS, source }).getMimeType();
@@ -52,6 +59,59 @@ export async function detectMimeType(input: Blob | string): Promise<string | nul
return null;
}
+/** Image container brands that ride in an ISO base media `ftyp` box. */
+const IMAGE_BRANDS: Record<string, string> = {
+ avif: 'image/avif',
+ avis: 'image/avif',
+ heic: 'image/heic',
+ heix: 'image/heic',
+ heim: 'image/heic',
+ hevc: 'image/heic',
+ mif1: 'image/heic',
+ msf1: 'image/heic',
+};
+
+/**
+ * The image format of a blob, read from its first bytes. Images are the one
+ * kind mediabunny does not sniff, so without this a picture whose name and
+ * declared type say nothing (an extension-less download, an
+ * `application/octet-stream` response) looks like nothing at all.
+ */
+async function sniffImageType(input: Blob): Promise<string | null> {
+ let bytes: Uint8Array;
+ try {
+ bytes = new Uint8Array(await input.slice(0, 64).arrayBuffer());
+ } catch {
+ return null;
+ }
+ if (bytes.length < 12) return null;
+
+ const magic = (offset: number, ...signature: number[]) => signature.every((byte, index) => bytes[offset + index] === byte);
+ const ascii = (offset: number, text: string) => [...text].every((char, index) => bytes[offset + index] === char.charCodeAt(0));
+
+ if (magic(0, 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)) return 'image/png';
+ if (magic(0, 0xff, 0xd8, 0xff)) return 'image/jpeg';
+ if (ascii(0, 'GIF87a') || ascii(0, 'GIF89a')) return 'image/gif';
+ if (ascii(0, 'RIFF') && ascii(8, 'WEBP')) return 'image/webp';
+ if (magic(0, 0x42, 0x4d)) return 'image/bmp';
+ if (magic(0, 0x49, 0x49, 0x2a, 0x00) || magic(0, 0x4d, 0x4d, 0x00, 0x2a)) return 'image/tiff';
+
+ // `....ftyp<brand>` — only the still-image brands; mediabunny takes the rest.
+ if (ascii(4, 'ftyp')) {
+ const brand = String.fromCharCode(...bytes.slice(8, 12)).toLowerCase();
+ return IMAGE_BRANDS[brand] ?? null;
+ }
+
+ // SVG is text: an `<svg>` root, possibly behind an XML declaration or comments.
+ const head = new TextDecoder().decode(bytes).trimStart();
+ if (/^(<\?xml|<!--|<!doctype svg|<svg[\s>])/i.test(head)) {
+ const text = (await input.slice(0, 4096).text()).slice(0, 4096);
+ if (/<svg[\s>]/i.test(text)) return 'image/svg+xml';
+ }
+
+ return null;
+}
+
/**
* Derives the type-specific asset properties of a media file. Throws when
* the file is of no type the library takes, or is missing what its type