The VS Code of video editing, built for agents.
TypeScript
Enhance MIME type detection and result handling in asset management
konstantin-paulusb6bf78ac6490
diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.tsindex 8a5b5b3b..173aa556 100644--- a/apps/web/src/utils/gen-ai.ts+++ b/apps/web/src/utils/gen-ai.ts@@ -18,7 +18,7 @@ import {PROMPT_INPUT_VOICE_MODEL,PROMPT_INPUT_VOICE_OPTIONS,} from "@/components/genai/config";-import { assert } from "@/utils";+import { assert, mimeTypeToExtension } from "@/utils";import { uploadBlob } from "@/lib/uploads";import { track } from "@/lib/analytics";import { trpc } from "@/lib/trpc";@@ -396,7 +396,7 @@ export class EditorGenAi extends GenAi {/*** Downloads what a model returned and files it in the library. The name is- * the model's, the extension the URL's — what came back decides what the+ * the model's, the extension the result's — what came back decides what the* file is called, not what was asked for.*/private async store(url: string, name: string, generation: { key: string; id?: string | null }): Promise<Asset> {@@ -404,10 +404,11 @@ export class EditorGenAi extends GenAi {assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);const blob = await response.blob();- const extension = url.split(/[?#]/)[0].split(".").pop();- const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;-- return this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation });+ return this.library.store(blob, {+ name: name + resultExtension(url, blob),+ folder: GENERATED_DIR,+ generation,+ });}private requestGeneration(spec: ResolvedSpec) {@@ -479,6 +480,24 @@ export class EditorGenAi extends GenAi {/** Whether an asset type is footage, for the calls that only take footage. */const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";+/**+ * What to call a generated result: the extension its type implies, falling+ * back to the one its URL carries. Neither is guaranteed — some models hand+ * back a plain slug (`.../files/young-man-city-skyline`), some servers say+ * only `application/octet-stream` — so this can come back empty, and the+ * library reads the bytes instead. The name is what the file is identified+ * by once on disk, so getting it right saves that guess.+ */+function resultExtension(url: string, blob: Blob): string {+ const fromType = blob.type ? mimeTypeToExtension(blob.type) : ".bin";+ if (fromType !== ".bin") return fromType;++ const fileName = url.split(/[?#]/)[0]!.split("/").pop() ?? "";+ const dot = fileName.lastIndexOf(".");+ const fromUrl = dot > 0 ? fileName.slice(dot + 1) : "";+ return fromUrl && fromUrl.length <= 5 ? `.${fromUrl}` : "";+}+/*** Says what a generation failed with, and hands the error on. The message is* what the element ends up carrying too (the runtime's `SourceError`, whichdiff --git a/packages/assets/src/probe.ts b/packages/assets/src/probe.tsindex 713ec516..1f832b4b 100644--- a/packages/assets/src/probe.ts+++ b/packages/assets/src/probe.ts@@ -26,9 +26,11 @@ const TRANSCRIPT_TYPES = new Set(['application/json', 'application/x-subrip', 't/*** The MIME type of a file, or of the resource behind a URL. Subtitle files go- * by extension (OS registries rarely map .srt); audio and video are sniffed- * by mediabunny, since a container's declared type is often wrong or empty.- * Null when the file is not something the library takes.+ * by extension (OS registries rarely map .srt); images fall back to their+ * magic bytes when nothing declared a type (a download filed without an+ * extension has none to go by); audio and video are sniffed by mediabunny,+ * since a container's declared type is often wrong or empty. Null when the+ * file is not something the library takes.*/export async function detectMimeType(input: Blob | string): Promise<string | null> {const name = typeof input === 'string' ? input.split(/[?#]/)[0]! : (input as File).name ?? '';@@ -41,6 +43,11 @@ export async function detectMimeType(input: Blob | string): Promise<string | nulif (mimeType?.startsWith('image/')) return mimeType;if (mimeType?.startsWith('text/html')) return mimeType;+ if (typeof input !== 'string') {+ const sniffed = await sniffImageType(input);+ if (sniffed) return sniffed;+ }+try {const source = typeof input === 'string' ? new UrlSource(input) : new BlobSource(input);mimeType = await new Input({ formats: ALL_FORMATS, source }).getMimeType();@@ -52,6 +59,59 @@ export async function detectMimeType(input: Blob | string): Promise<string | nulreturn null;}+/** Image container brands that ride in an ISO base media `ftyp` box. */+const IMAGE_BRANDS: Record<string, string> = {+ avif: 'image/avif',+ avis: 'image/avif',+ heic: 'image/heic',+ heix: 'image/heic',+ heim: 'image/heic',+ hevc: 'image/heic',+ mif1: 'image/heic',+ msf1: 'image/heic',+};++/**+ * The image format of a blob, read from its first bytes. Images are the one+ * kind mediabunny does not sniff, so without this a picture whose name and+ * declared type say nothing (an extension-less download, an+ * `application/octet-stream` response) looks like nothing at all.+ */+async function sniffImageType(input: Blob): Promise<string | null> {+ let bytes: Uint8Array;+ try {+ bytes = new Uint8Array(await input.slice(0, 64).arrayBuffer());+ } catch {+ return null;+ }+ if (bytes.length < 12) return null;++ const magic = (offset: number, ...signature: number[]) => signature.every((byte, index) => bytes[offset + index] === byte);+ const ascii = (offset: number, text: string) => [...text].every((char, index) => bytes[offset + index] === char.charCodeAt(0));++ if (magic(0, 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)) return 'image/png';+ if (magic(0, 0xff, 0xd8, 0xff)) return 'image/jpeg';+ if (ascii(0, 'GIF87a') || ascii(0, 'GIF89a')) return 'image/gif';+ if (ascii(0, 'RIFF') && ascii(8, 'WEBP')) return 'image/webp';+ if (magic(0, 0x42, 0x4d)) return 'image/bmp';+ if (magic(0, 0x49, 0x49, 0x2a, 0x00) || magic(0, 0x4d, 0x4d, 0x00, 0x2a)) return 'image/tiff';++ // `....ftyp<brand>` — only the still-image brands; mediabunny takes the rest.+ if (ascii(4, 'ftyp')) {+ const brand = String.fromCharCode(...bytes.slice(8, 12)).toLowerCase();+ return IMAGE_BRANDS[brand] ?? null;+ }++ // SVG is text: an `<svg>` root, possibly behind an XML declaration or comments.+ const head = new TextDecoder().decode(bytes).trimStart();+ if (/^(<\?xml|<!--|<!doctype svg|<svg[\s>])/i.test(head)) {+ const text = (await input.slice(0, 4096).text()).slice(0, 4096);+ if (/<svg[\s>]/i.test(text)) return 'image/svg+xml';+ }++ return null;+}+/*** Derives the type-specific asset properties of a media file. Throws when* the file is of no type the library takes, or is missing what its type