Skip to content

The VS Code of video editing, built for agents.

TypeScript

Update asset management and enhance project features

konstantin-paulusac181f985edc
diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts
index 9fd08255..cd716304 100644
--- a/apps/cli/src/index.ts
+++ b/apps/cli/src/index.ts
@@ -4,7 +4,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { randomUUID } from "node:crypto";
-import { existsSync, mkdirSync, statSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, isAbsolute, join, resolve } from "node:path";
import { Command } from "commander";
@@ -122,19 +122,20 @@ async function mediaFrame(ref: string, opts: MediaFrameOptions): Promise<void> {
}
}
+/**
+ * An asset id, a local file (or frames folder) that exists, or otherwise a
+ * library path (`b-roll/clip.mp4`) for the app to look up.
+ */
function resolveAssetRef(ref: string): AssetRef {
if (/^[A-Za-z0-9]+$/.test(ref)) return { id: ref };
const absPath = isAbsolute(ref) ? ref : resolve(process.cwd(), ref);
- if (!existsSync(absPath)) {
+ if (existsSync(absPath)) return { path: absPath };
+ if (isAbsolute(ref)) {
console.error(`File not found: ${absPath}`);
process.exit(1);
}
- if (!statSync(absPath).isFile()) {
- console.error(`Not a file: ${absPath}`);
- process.exit(1);
- }
- return { path: absPath };
+ return { id: ref };
}
async function mediaProbe(ref: string): Promise<void> {
diff --git a/apps/desktop/package.json b/apps/desktop/package.json
index 3bef4ebe..4fcd10dd 100644
--- a/apps/desktop/package.json
+++ b/apps/desktop/package.json
@@ -41,6 +41,7 @@
"esbuild": "^0.28.1",
"sqids": "^0.3.0",
"ts-morph": "^28.0.0",
- "update-electron-app": "^3.0.0"
+ "update-electron-app": "^3.0.0",
+ "yaml": "^2.9.0"
}
}
diff --git a/apps/desktop/src/main-channels.ts b/apps/desktop/src/main-channels.ts
index 094f5131..2ae4b316 100644
--- a/apps/desktop/src/main-channels.ts
+++ b/apps/desktop/src/main-channels.ts
@@ -50,6 +50,11 @@ export const MAIN_CHANNELS = {
PROJECTS_WRITE: "projects:write",
PROJECTS_WATCH: "projects:watch",
PROJECTS_UNWATCH: "projects:unwatch",
+ PROJECTS_MANIFEST_READ: "projects:manifest-read",
+ PROJECTS_MANIFEST_WRITE: "projects:manifest-write",
+ PROJECTS_FS_LIST: "projects:fs-list",
+ PROJECTS_FS_STAT: "projects:fs-stat",
+ PROJECTS_FS_REMOVE: "projects:fs-remove",
// Main→Renderer events
AUTH_CALLBACK: "auth:callback",
@@ -141,7 +146,19 @@ export type MainRequestMap = {
};
[MAIN_CHANNELS.PROJECTS_WATCH]: { request: { dir: string }; response: void };
[MAIN_CHANNELS.PROJECTS_UNWATCH]: { request: { dir: string }; response: void };
+ // The asset manifest (`assets.yml`) as plain data; null when there is none.
+ [MAIN_CHANNELS.PROJECTS_MANIFEST_READ]: { request: { dir: string }; response: unknown };
+ [MAIN_CHANNELS.PROJECTS_MANIFEST_WRITE]: { request: { dir: string; manifest: unknown }; response: void };
+ // Project file system, for the asset library. `source` is project-relative
+ // or absolute; `path` is always project-relative. Writes stream through the
+ // FILE_WRITE_* channels (which create parent directories).
+ [MAIN_CHANNELS.PROJECTS_FS_LIST]: { request: { dir: string; source: string }; response: FsEntry[] };
+ [MAIN_CHANNELS.PROJECTS_FS_STAT]: { request: { dir: string; source: string }; response: FsStat | null };
+ [MAIN_CHANNELS.PROJECTS_FS_REMOVE]: { request: { dir: string; path: string }; response: void };
};
+
+export type FsEntry = { name: string; kind: "file" | "directory"; size: number; mtime: number };
+export type FsStat = { size: number; mtime: number };
export type MainRequestChannel = keyof MainRequestMap;
export type MainEventMap = {
diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts
index 367a0ffc..19dc8a0b 100644
--- a/apps/desktop/src/main.ts
+++ b/apps/desktop/src/main.ts
@@ -3,8 +3,8 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { app, BrowserWindow, nativeImage, session, shell } from "electron";
-import { join } from "node:path";
-import { open, unlink } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { mkdir, open, unlink } from "node:fs/promises";
import { randomUUID } from "node:crypto";
import type { FileHandle } from "node:fs/promises";
import { updateElectronApp } from "update-electron-app";
@@ -23,8 +23,14 @@ import {
pickRoot,
renameProject,
unwatchAll,
+ listEntries,
+ markSelfWriteAbsolute,
+ readManifest,
+ removeEntry,
+ statEntry,
unwatchProject,
watchProject,
+ writeManifest,
writeProject,
} from "./projects";
import type { DeepLinkChannel } from "./main-channels";
@@ -275,11 +281,18 @@ if (app.requestSingleInstanceLock()) {
watchProject(BrowserWindow.fromWebContents(event.sender), dir),
);
mainBridge.handle(MAIN_CHANNELS.PROJECTS_UNWATCH, ({ dir }) => unwatchProject(dir));
+ mainBridge.handle(MAIN_CHANNELS.PROJECTS_MANIFEST_READ, ({ dir }) => readManifest(dir));
+ mainBridge.handle(MAIN_CHANNELS.PROJECTS_MANIFEST_WRITE, ({ dir, manifest }) => writeManifest(dir, manifest));
+ mainBridge.handle(MAIN_CHANNELS.PROJECTS_FS_LIST, ({ dir, source }) => listEntries(dir, source));
+ mainBridge.handle(MAIN_CHANNELS.PROJECTS_FS_STAT, ({ dir, source }) => statEntry(dir, source));
+ mainBridge.handle(MAIN_CHANNELS.PROJECTS_FS_REMOVE, ({ dir, path }) => removeEntry(dir, path));
mainBridge.handle(MAIN_CHANNELS.FILE_TRANSFER, ({ selector, absolutePath }) =>
setFileInputFiles(selector, absolutePath),
);
mainBridge.handle(MAIN_CHANNELS.FILE_WRITE_OPEN, async ({ path, exclusive }) => {
+ await mkdir(dirname(path), { recursive: true });
+ markSelfWriteAbsolute(path);
const handle = await open(path, exclusive ? "wx" : "w");
const id = randomUUID();
openWrites.set(id, { handle, path });
@@ -289,6 +302,7 @@ if (app.requestSingleInstanceLock()) {
mainBridge.handle(MAIN_CHANNELS.FILE_WRITE_CHUNK, async ({ id, data, position }) => {
const entry = openWrites.get(id);
if (!entry) throw new Error(`No open file for write id ${id}`);
+ markSelfWriteAbsolute(entry.path);
await entry.handle.write(data, 0, data.byteLength, position);
});
@@ -296,6 +310,7 @@ if (app.requestSingleInstanceLock()) {
const entry = openWrites.get(id);
if (!entry) return;
openWrites.delete(id);
+ markSelfWriteAbsolute(entry.path);
await entry.handle.close();
});
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.ts
index 1cfd79e1..7cd603c8 100644
--- a/apps/desktop/src/projects.ts
+++ b/apps/desktop/src/projects.ts
@@ -15,9 +15,10 @@
import { app, dialog, shell, type BrowserWindow } from "electron";
import { watch, type FSWatcher } from "node:fs";
-import { cp, mkdir, readdir, readFile, realpath, stat, writeFile } from "node:fs/promises";
+import { cp, mkdir, readdir, readFile, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
import { createRequire } from "node:module";
-import { basename, dirname, join, relative, sep } from "node:path";
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
import type { TransformOptions } from "@babel/core";
import type { BuildOptions, Plugin } from "esbuild";
@@ -26,7 +27,7 @@ import { mainBridge } from "./main-manager";
import { MAIN_CHANNELS } from "./main-channels";
import { applyEdits, stampProject } from "./edit";
import { sourcePlugin } from "./source";
-import type { CompileResult, ProjectInfo, SourceEdit, WriteResult } from "./main-channels";
+import type { CompileResult, FsEntry, FsStat, ProjectInfo, SourceEdit, WriteResult } from "./main-channels";
import type { SourceContext } from "./edit";
/** Entry points looked up in a project folder, in order of preference. */
@@ -372,6 +373,89 @@ export async function writeProject(dir: string, edits: SourceEdit[]): Promise<Wr
}
}
+// ---------------------------------------------------------------------------
+// Assets
+
+/** The asset manifest's file name. */
+export const MANIFEST_FILE = "assets.yml";
+
+/**
+ * A `source` of the asset library as an absolute path: absolute already, or
+ * relative to the project. Refuses to leave the project for a relative one.
+ */
+function sourcePath(dir: string, source: string): string {
+ if (isAbsolute(source)) return source;
+ const path = resolve(dir, source);
+ if (relative(dir, path).startsWith("..")) throw new Error(`Path leaves the project: ${source}`);
+ return path;
+}
+
+/** The manifest as plain data, or null when the project has none. */
+export async function readManifest(dir: string): Promise<unknown> {
+ let text: string;
+ try {
+ text = await readFile(join(dir, MANIFEST_FILE), "utf8");
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
+ throw error;
+ }
+ return parseYaml(text) ?? null;
+}
+
+/**
+ * Writes the manifest as YAML. Atomic (temp file + rename), so a crash
+ * mid-write leaves the old manifest, and marked as ours so the watcher does
+ * not hand it back as a change.
+ */
+export async function writeManifest(dir: string, manifest: unknown): Promise<void> {
+ const path = join(dir, MANIFEST_FILE);
+ const temp = join(dir, `.${MANIFEST_FILE}.tmp`);
+ const text = stringifyYaml(manifest, { lineWidth: 0 });
+ markSelfWrite(dir, MANIFEST_FILE);
+ markSelfWrite(dir, `.${MANIFEST_FILE}.tmp`);
+ await writeFile(temp, `# Diffusion Studio asset library. Edited by the app; hand edits are read on the next load.
+${text}`, "utf8");
+ await rename(temp, path);
+}
+
+/** Entries of a directory; [] when it is missing or not a directory. */
+export async function listEntries(dir: string, source: string): Promise<FsEntry[]> {
+ const path = sourcePath(dir, source);
+ let names: import("node:fs").Dirent[];
+ try {
+ names = await readdir(path, { withFileTypes: true });
+ } catch {
+ return [];
+ }
+ const entries = await Promise.all(names.map(async (entry): Promise<FsEntry | null> => {
+ if (!entry.isFile() && !entry.isDirectory()) return null;
+ try {
+ const info = await stat(join(path, entry.name));
+ return { name: entry.name, kind: entry.isDirectory() ? "directory" : "file", size: info.size, mtime: info.mtimeMs };
+ } catch {
+ return null;
+ }
+ }));
+ return entries.filter((entry): entry is FsEntry => entry !== null);
+}
+
+/** Size and mtime of a file, or null when it does not exist. */
+export async function statEntry(dir: string, source: string): Promise<FsStat | null> {
+ try {
+ const info = await stat(sourcePath(dir, source));
+ return { size: info.size, mtime: info.mtimeMs };
+ } catch {
+ return null;
+ }
+}
+
+/** Removes a file or directory inside the project; missing is fine. */
+export async function removeEntry(dir: string, path: string): Promise<void> {
+ if (isAbsolute(path)) throw new Error("Only project files can be removed");
+ markSelfWrite(dir, path.split(sep).join("/"));
+ await rm(sourcePath(dir, path), { recursive: true, force: true });
+}
+
// ---------------------------------------------------------------------------
// Watch
@@ -395,6 +479,16 @@ export function markSelfWrite(dir: string, path: string): void {
selfWrites.set(writeKey(dir, path), Date.now());
}
+/** `markSelfWrite` for an absolute path, against whichever watched project holds it. */
+export function markSelfWriteAbsolute(path: string): void {
+ for (const dir of watchers.keys()) {
+ const rel = relative(dir, path);
+ if (rel && !rel.startsWith("..") && !isAbsolute(rel)) {
+ markSelfWrite(dir, rel.split(sep).join("/"));
+ }
+ }
+}
+
function isSelfWrite(dir: string, path: string): boolean {
const key = writeKey(dir, path);
const at = selfWrites.get(key);
diff --git a/apps/web/package.json b/apps/web/package.json
index 2a628d5f..2529e327 100644
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -14,6 +14,7 @@
},
"dependencies": {
"@diffusionstudio/api-contract": "^0.1.0",
+ "@diffusionstudio/assets": "*",
"@diffusionstudio/cli": "*",
"@diffusionstudio/jsx": "*",
"@diffusionstudio/koota-solid": "*",
diff --git a/apps/web/src/components/assets/actions.ts b/apps/web/src/components/assets/actions.ts
deleted file mode 100644
index a2c6d213..00000000
--- a/apps/web/src/components/assets/actions.ts
+++ /dev/null
@@ -1,30 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { toast } from "somoto";
-import { getAssetFile } from "@/components/engine";
-
-import type { Asset } from "@/components/engine/db";
-
-/**
- * Download an asset file and handle user-facing failure feedback.
- */
-export async function downloadAsset(asset: Asset): Promise<void> {
- let url: string | null = null;
-
- try {
- const file = await getAssetFile(asset);
- url = URL.createObjectURL(file);
-
- const anchor = document.createElement("a");
- anchor.href = url;
- anchor.download = asset.name;
- anchor.click();
- } catch (error) {
- const message = error instanceof Error ? error.message : "Unknown download error";
- toast("Failed to download", { description: message });
- } finally {
- if (url) URL.revokeObjectURL(url);
- }
-}
diff --git a/apps/web/src/components/canvas/canvas.tsx b/apps/web/src/components/canvas/canvas.tsx
index e9b5c9cb..e6df295a 100644
--- a/apps/web/src/components/canvas/canvas.tsx
+++ b/apps/web/src/components/canvas/canvas.tsx
@@ -2,142 +2,61 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { useEngine } from "@/context/engine";
-import { loadAsset, getAssetFile } from "@/components/engine";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { findSceneAt, screenToWorld, worldToLocal, Library } from "@diffusionstudio/runtime";
import { CameraController, EngineCanvas } from "@/engine";
+import { insertAsset } from "@/engine/insert-asset";
+import { droppedFiles, importFiles } from "@/engine/asset-actions";
import { Toolbar } from "./toolbar";
import { DrawOverlay } from "./draw-overlay";
import { DesktopAppBanner } from "./desktop-app-banner";
import { toast } from "somoto"
-import { findSceneAt, screenToWorld, worldToLocal, secondsToFrames, GeometryType, PaintType, createEntity, addComponent, appendChild, setComponent, resizeEntity } from "../engine";
import { SceneInitOverlay } from "./scene-init-overlay";
+import { ASSET_DRAG_TYPE } from "@/components/sidebar-left/folder-item";
-import { CLASSIC_PRESET_HEIGHT, CLASSIC_PRESET_WIDTH } from "../engine/decoders/caption/classic";
-
-import type { Asset } from "../engine/db";
-import type { Transcript } from "@diffusionstudio/api-contract";
+import type { Asset } from "@diffusionstudio/assets";
export const MAIN_CANVAS_ID = 'engine-canvas';
export function Canvas() {
- const engine = useEngine();
- const world = engine.world;
- const c = world.components;
+ const world = useWorld();
+ /**
+ * Drops onto the canvas: library assets (dragged from the panel) land where
+ * they were dropped, in the scene under the pointer; external files are
+ * imported into the library first, then land the same way.
+ */
const handleDropEvent = async (event: DragEvent) => {
event.preventDefault();
event.stopPropagation();
+ const library = world.get(Library);
+ if (!library) return;
+
const rect = (event.currentTarget as HTMLElement).getBoundingClientRect();
const worldPt = screenToWorld(world, event.clientX - rect.left, event.clientY - rect.top);
- const sceneEid = findSceneAt(world, worldPt.x, worldPt.y);
-
- const getTargetPosition = (size: { width: number, height: number }, eid: number | null) => {
- const localPt = eid === null
- ? worldPt
- : worldToLocal(world, eid, worldPt.x, worldPt.y);
-
- return {
- x: Math.round(localPt.x - size.width / 2),
- y: Math.round(localPt.y - size.height / 2),
- };
- };
-
- const placeAsset = async (asset: Asset) => {
- const parentEid: number | null = sceneEid;
-
- const size = { width: 400, height: 100 };
-
- if (asset.type === 'VIDEO' || asset.type === 'IMAGE' || asset.type === 'SEQUENCE') {
- size.width = Math.round(asset.width);
- size.height = Math.round(asset.height);
- }
-
- if (asset.type === 'TRANSCRIPT') {
- size.width = CLASSIC_PRESET_WIDTH;
- size.height = CLASSIC_PRESET_HEIGHT;
- }
-
- const position = getTargetPosition(size, parentEid);
-
- // Resolve async transcript data up front so all entity mutations run in
- // one synchronous, undoable transaction.
- let transcriptTrim: { start: number; end: number } | null = null;
- if (asset.type === 'TRANSCRIPT') {
- const file = await getAssetFile(asset);
- const transcript = JSON.parse(await file.text()) as Transcript;
- const lastWord = transcript.at(-1)?.words.at(-1);
- const firstWord = transcript.at(0)?.words.at(0);
- transcriptTrim = {
- start: secondsToFrames(firstWord?.start),
- end: secondsToFrames(lastWord?.end),
- };
- }
-
- world.history.transaction('Place asset', () => {
- const eid = createEntity(world);
- setComponent(world, eid, c.Position, { x: position.x, y: position.y });
- setComponent(world, eid, c.KeepAspectRatio, { width: size.width, height: size.height });
- setComponent(world, eid, c.Name, asset.name);
-
- if (asset.type === 'TRANSCRIPT') {
- setComponent(world, eid, c.Geometry, GeometryType.TEXT);
- setComponent(world, eid, c.Caption, {});
- if (transcriptTrim) setComponent(world, eid, c.Trim, transcriptTrim);
- } else {
- setComponent(world, eid, c.Geometry, GeometryType.RECT);
-
- // Other assets need a paint component
- let paint = PaintType.IMAGE;
- if (asset.type === 'SEQUENCE') paint = PaintType.SEQUENCE;
- if (asset.type === 'VIDEO') paint = PaintType.VIDEO;
- if (asset.type === 'AUDIO') paint = PaintType.WAVEFORM;
- const fillEid = createEntity(world);
- setComponent(world, fillEid, c.Paint, paint);
- setComponent(world, fillEid, c.AssetId, asset.id);
- appendChild(world, fillEid, eid);
- }
-
- if (asset.type === 'AUDIO') {
- addComponent(world, eid, c.Audio);
- }
-
- if (asset.type === 'AUDIO' || asset.type === 'TRANSCRIPT') {
- setComponent(world, eid, c.AssetId, asset.id);
- }
-
- if (asset.type === 'VIDEO' || asset.type === 'AUDIO' || asset.type === 'SEQUENCE') {
- setComponent(world, eid, c.Playback, {});
- }
-
- if (parentEid) {
- appendChild(world, eid, parentEid);
- }
-
- resizeEntity(world, eid, { width: size.width, height: size.height });
+ const scene = findSceneAt(world, worldPt.x, worldPt.y);
+ const localPt = scene ? worldToLocal(world, scene, worldPt.x, worldPt.y) : worldPt;
+
+ const place = (asset: Asset) => {
+ const size = 'width' in asset && 'height' in asset ? { width: asset.width, height: asset.height } : { width: 500, height: 150 };
+ const placed = insertAsset(world, asset, {
+ ...(scene ? { parent: scene } : {}),
+ x: localPt.x - size.width / 2,
+ y: localPt.y - size.height / 2,
});
+ if (!placed) toast("Nothing to insert into", { description: "Open a project first." });
};
- const assetIds = event.dataTransfer?.getData('application/x-asset-id')?.split(',').filter(Boolean);
- if (assetIds && assetIds.length > 0) {
- for (const id of assetIds) {
- const asset = world.assets.get(id);
- if (asset) {
- await placeAsset(asset);
- }
- }
+ const assetIds = event.dataTransfer?.getData(ASSET_DRAG_TYPE)?.split(',').filter(Boolean) ?? [];
+ for (const id of assetIds) {
+ const asset = library.get(id);
+ if (asset) place(asset);
}
-
- for (let i = 0; i < (event.dataTransfer?.items.length ?? 0); i++) {
- if (event.dataTransfer?.items[i]?.kind !== 'file') continue;
-
- try {
- const asset = await loadAsset(world, event.dataTransfer.items[i]);
- await placeAsset(asset);
- } catch (e) {
- toast("Failed to import assets", { description: (e as Error).message });
- }
+ const files = droppedFiles(event);
+ if (files.length) {
+ for (const asset of await importFiles(library, files, '')) place(asset);
}
}
diff --git a/apps/web/src/components/sidebar-left/asset-item.tsx b/apps/web/src/components/sidebar-left/asset-item.tsx
index 9bd0429e..41f8a2bc 100644
--- a/apps/web/src/components/sidebar-left/asset-item.tsx
+++ b/apps/web/src/components/sidebar-left/asset-item.tsx
@@ -4,7 +4,9 @@
import { createSignal, onMount, onCleanup, Show, createMemo } from "solid-js";
import { toast } from "somoto";
-import { downloadAsset } from "@/components/assets/actions";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { assetName } from "@diffusionstudio/assets";
+import { pickFiles, saveAssetAs } from "@/engine/asset-actions";
import {
ContextMenu,
ContextMenuContent,
@@ -16,11 +18,11 @@ import {
} from "../ui/context-menu";
import { AssetThumbnail } from "../ui/asset-thumbnail";
import { formatAssetDuration } from "@/utils";
-import { useEngine } from "@/context/engine";
-import { insertAssetInTimeline, replaceAssetHandle, removeAsset, saveAsset } from "@/components/engine";
-
-import type { Asset } from "@/components/engine/db";
+import { useLibrary } from "@/engine/library";
+import { insertAsset } from "@/engine/insert-asset";
+import { ASSET_DRAG_TYPE } from "./folder-item";
+import type { Asset } from "@diffusionstudio/assets";
export type LazyAssetItemProps = {
asset: Asset;
@@ -32,11 +34,13 @@ export type LazyAssetItemProps = {
* A lazy loaded asset item. Clears the buffer when not visible.
*/
export function LazyAssetItem(props: LazyAssetItemProps) {
- const { world } = useEngine();
+ const world = useWorld();
+ const library = useLibrary();
const [isVisible, setIsVisible] = createSignal(false);
const [isRenaming, setIsRenaming] = createSignal(false);
const assetDuration = createMemo(() => formatAssetDuration(props.asset));
+ const name = () => assetName(props.asset);
let ref: HTMLDivElement | undefined;
@@ -54,35 +58,25 @@ export function LazyAssetItem(props: LazyAssetItemProps) {
const handleDeleteAsset = async () => {
try {
- await removeAsset(world, props.asset.id);
+ await library()?.remove([props.asset]);
} catch (e) {
toast.error("Failed to delete", { description: (e as Error).message });
}
};
const handleDragStart = (event: DragEvent) => {
- event.dataTransfer?.setData("application/x-asset-id", props.asset.id);
- };
-
- const handleRename = () => {
- setIsRenaming(true);
+ event.dataTransfer?.setData(ASSET_DRAG_TYPE, props.asset.id);
};
- const commitRename = async (nextName: string) => {
+ const commitRename = (nextName: string) => {
if (!isRenaming()) return;
setIsRenaming(false);
const trimmed = nextName.trim();
- if (!trimmed || trimmed === props.asset.name) return;
- try {
- await saveAsset(world, { ...props.asset, name: trimmed });
- } catch (e) {
- toast.error("Failed to rename", { description: (e as Error).message });
- }
+ if (!trimmed || trimmed === name()) return;
+ library()?.rename(props.asset, trimmed);
};
- const handleRenameKeyDown = (
- event: KeyboardEvent & { currentTarget: HTMLInputElement }
- ) => {
+ const handleRenameKeyDown = (event: KeyboardEvent & { currentTarget: HTMLInputElement }) => {
event.stopPropagation();
if (event.key === "Enter") {
event.preventDefault();
@@ -100,9 +94,25 @@ export function LazyAssetItem(props: LazyAssetItemProps) {
});
};
- const handleDownloadMedia = () => downloadAsset(props.asset);
- const handleReplaceMedia = () => replaceAssetHandle(world, props.asset);
- const handleInsertToTimeline = () => insertAssetInTimeline(world, props.asset, 'playhead');
+ /** Points the asset at another file; the JSX keeps naming it by path. */
+ const handleReplaceMedia = async () => {
+ const lib = library();
+ const [file] = await pickFiles({ multiple: false });
+ const path = file && lib ? lib.fs.pathOf?.(file) : null;
+ if (!lib || !path) return;
+ try {
+ await lib.relink(props.asset, path);
+ } catch (e) {
+ toast.error("Failed to replace", { description: (e as Error).message });
+ }
+ };
+
+ const handleSaveAs = () => saveAssetAs(props.asset);
+ const handleInsertToTimeline = () => {
+ if (!insertAsset(world, props.asset)) {
+ toast("Nothing to insert into", { description: "Open a project first." });
+ }
+ };
return (
<ContextMenu>
@@ -135,8 +145,8 @@ export function LazyAssetItem(props: LazyAssetItemProps) {
<Show
when={isRenaming()}
fallback={
- <div class="text-xs text-foreground truncate select-none">
- {props.asset.name}
+ <div class="text-xs text-foreground truncate select-none" title={props.asset.source}>
+ {name()}
</div>
}
>
@@ -145,7 +155,7 @@ export function LazyAssetItem(props: LazyAssetItemProps) {
type="text"
name="asset-name"
autocomplete="off"
- value={props.asset.name}
+ value={name()}
onKeyDown={handleRenameKeyDown}
onBlur={(e) => commitRename(e.currentTarget.value)}
onClick={(e) => e.stopPropagation()}
@@ -161,14 +171,14 @@ export function LazyAssetItem(props: LazyAssetItemProps) {
Insert at playhead
</ContextMenuItem>
<ContextMenuSeparator />
- <ContextMenuItem onSelect={handleRename}>
+ <ContextMenuItem onSelect={() => setIsRenaming(true)}>
Rename
</ContextMenuItem>
<ContextMenuItem onSelect={handleReplaceMedia}>
Replace
</ContextMenuItem>
- <ContextMenuItem onSelect={handleDownloadMedia}>
- Download
+ <ContextMenuItem onSelect={handleSaveAs}>
+ Save as
</ContextMenuItem>
<ContextMenuSeparator />
<ContextMenuItem onSelect={handleDeleteAsset}>
diff --git a/apps/web/src/components/sidebar-left/assets.tsx b/apps/web/src/components/sidebar-left/assets.tsx
index b520a6ff..60af0026 100644
--- a/apps/web/src/components/sidebar-left/assets.tsx
+++ b/apps/web/src/components/sidebar-left/assets.tsx
@@ -2,19 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { useEngine } from "@/context/engine";
-import {
- useAssets,
- loadAsset,
- removeAsset,
- selectAsset,
- clearSelectedAssets,
- useFolders,
- openFolder,
- createFolder,
- assetFolderId,
- nextFolderName,
-} from "@/components/engine";
+import { basename, dirname } from "@diffusionstudio/assets";
import { usePromptInput } from "@/context/prompt-input";
import { createDefaultConfig } from "@/components/genai/prompt-input";
import { createEffect, createMemo, createSignal, For, onCleanup, onMount, Show } from "solid-js";
@@ -28,7 +16,6 @@ import {
DropdownMenuShortcut,
DropdownMenuTrigger,
} from "../ui/dropdown-menu";
-import { showFileDialog } from "@/utils";
import { toast } from "somoto";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
import {
@@ -40,26 +27,34 @@ import {
BreadcrumbsSeparator,
} from "../ui/breadcrumbs";
import { LazyAssetItem } from "./asset-item";
-import { FolderItem, handleFolderDrop, ASSET_DRAG_TYPE, FOLDER_DRAG_TYPE } from "./folder-item";
+import { FolderItem, handleFolderDrop, isAssetOrFolderDrag, ASSET_DRAG_TYPE, FOLDER_DRAG_TYPE } from "./folder-item";
+import { useLibrary } from "@/engine/library";
+import { droppedFiles, importFiles, pickAndImport } from "@/engine/asset-actions";
+
+import type { AssetLibrary } from "@diffusionstudio/assets";
-import type { EngineWorld } from "@/components/engine";
-import type { ElectronFileHandle } from "@/lib/electron-file-handle";
+/** The ancestors of a folder path, root excluded, nearest last. */
+function ancestorsOf(path: string): string[] {
+ const chain: string[] = [];
+ for (let folder = path; folder; folder = dirname(folder)) chain.unshift(folder);
+ return chain;
+}
export function Assets() {
let root: HTMLDivElement | undefined;
let dragCounter = 0;
- const { world } = useEngine();
- const assets = useAssets(world);
- const folders = useFolders(world);
+ const library = useLibrary();
const { openPromptInput } = usePromptInput();
const [query, setQuery] = createSignal("");
const [assetFilter, setAssetFilter] = createSignal<AssetFilter>("ALL");
const [isDragging, setIsDragging] = createSignal(false);
- const [renamingFolderId, setRenamingFolderId] = createSignal<string | null>(null);
-
+ const [renamingFolder, setRenamingFolder] = createSignal<string | null>(null);
+ const [selectedAssetId, setSelectedAssetId] = createSignal<string | null>(null);
+ const [currentFolder, setCurrentFolder] = createSignal("");
- const selectedAssetId = () => assets.selected().keys().next().value ?? null;
+ const allAssets = createMemo(() => library()?.list().filter((asset) => asset.type !== "SCRIPT") ?? []);
+ const allFolders = createMemo(() => [...(library()?.folders() ?? [])].sort());
const activeFilterLabel = () => {
const value = assetFilter();
@@ -69,46 +64,46 @@ export function Assets() {
const filteredAssets = createMemo(() => {
const q = query().trim().toLowerCase();
- const list = assets.all();
const selectedFilter = assetFilter();
+ const folder = currentFolder();
- return list.filter((asset) => {
- if (asset.type === "SCRIPT") return false;
+ return allAssets().filter((asset) => {
if (selectedFilter !== "ALL" && asset.type !== selectedFilter) return false;
- if (q) return asset.name.toLowerCase().includes(q);
- return assetFolderId(world, asset) === folders.currentId();
+ if (q) return basename(asset.path).toLowerCase().includes(q);
+ return dirname(asset.path) === folder;
});
});
const visibleFolders = createMemo(() => {
const q = query().trim().toLowerCase();
if (q) {
- return folders
- .all()
- .filter((folder) => folder.name.toLowerCase().includes(q))
- .sort((a, b) => a.name.localeCompare(b.name));
+ return allFolders().filter((folder) => basename(folder).toLowerCase().includes(q));
}
- return folders.childrenOf(folders.currentId());
+ return library()?.childrenOf(currentFolder()).folders ?? [];
});
- const panelTitle = createMemo(() => folders.get(folders.currentId() ?? "")?.name ?? "Assets");
+ const panelTitle = createMemo(() => (currentFolder() ? basename(currentFolder()) : "Assets"));
const itemCount = createMemo(() => visibleFolders().length + filteredAssets().length);
// Deep paths collapse like the breadcrumbs docs example:
// All assets / … / parent / current, with the hidden folders in a dropdown.
+ const folderPath = createMemo(() => ancestorsOf(currentFolder()));
const collapsedFolders = createMemo(() => {
- const path = folders.path();
+ const path = folderPath();
return path.length > 2 ? path.slice(0, -2) : [];
});
const tailFolders = createMemo(() => {
- const path = folders.path();
+ const path = folderPath();
return path.length > 2 ? path.slice(-2) : path;
});
+ const openFolder = (path: string) => {
+ setQuery("");
+ setCurrentFolder(path);
+ };
+
const handleGoToParent = () => {
- const id = folders.currentId();
- if (id === null) return;
- openFolder(world, folders.get(id)?.parentId ?? null);
+ if (currentFolder()) openFolder(dirname(currentFolder()));
};
const getAssetIdFromTarget = (target: EventTarget | null) => {
@@ -116,34 +111,30 @@ export function Assets() {
return element?.closest("[data-asset-id]")?.getAttribute("data-asset-id") ?? null;
};
+ // A folder that went away (deleted, moved) closes to the nearest one left.
+ createEffect(() => {
+ const folders = library()?.folders();
+ let folder = currentFolder();
+ if (!folders || !folder || folders.has(folder)) return;
+ while (folder && !folders.has(folder)) folder = dirname(folder);
+ setCurrentFolder(folder);
+ });
+
createEffect(() => {
const selected = selectedAssetId();
- if (!selected) return;
- if (!assets.get(selected)) {
- clearSelectedAssets(world);
- }
+ if (selected && !library()?.get(selected)) setSelectedAssetId(null);
});
- const importAssets = async (
- handles: ReadonlyArray<File | FileSystemFileHandle | ElectronFileHandle | DataTransferItem>,
- ) => {
- if (handles.length === 0) return;
-
- try {
- await Promise.all(handles.map((handle) => {
- return loadAsset(world, handle, { folderId: folders.currentId() });
- }));
- } catch (e) {
- toast("Failed to import assets", {
- description: (e as Error).message,
- });
+ const withLibrary = (run: (library: AssetLibrary) => void | Promise<void>) => {
+ const lib = library();
+ if (!lib) {
+ toast("No project open", { description: "Open a project to manage its assets." });
+ return;
}
+ return run(lib);
};
- const handleImportAssets = async () => {
- const files = await showFileDialog();
- await importAssets(files);
- };
+ const handleImportAssets = () => withLibrary((lib) => pickAndImport(lib, currentFolder()).then(() => {}));
const handleDrop = async (event: DragEvent) => {
event.preventDefault();
@@ -153,14 +144,10 @@ export function Assets() {
document.body.style.cursor = "default";
- if (await handleFolderDrop(world, event, folders.currentId())) {
- return;
- }
-
- const items = Array.from(event.dataTransfer?.items ?? []).filter(
- (item) => item.kind === "file"
- );
- await importAssets(items);
+ const lib = library();
+ if (!lib) return;
+ if (handleFolderDrop(lib, event, currentFolder())) return;
+ await importFiles(lib, droppedFiles(event), currentFolder());
};
const handleDragOver = (event: DragEvent) => {
@@ -198,7 +185,7 @@ export function Assets() {
const handleSelectAsset = (assetId: string) => {
- selectAsset(world, assetId);
+ setSelectedAssetId(assetId);
root?.focus();
};
@@ -222,7 +209,7 @@ export function Assets() {
const nextId = nextItem?.dataset.assetId;
if (!nextId) return;
- selectAsset(world, nextId);
+ setSelectedAssetId(nextId);
nextItem.scrollIntoView({ block: "nearest" });
};
@@ -237,18 +224,18 @@ export function Assets() {
}
if (event.key === "Backspace" || event.key === "Delete") {
- const selectedAsset = selectedAssetId();
- if (selectedAsset) {
+ const selected = selectedAssetId();
+ const asset = selected ? library()?.get(selected) : undefined;
+ if (asset) {
event.preventDefault();
event.stopPropagation();
- removeAsset(world, selectedAsset);
+ void library()?.remove([asset]);
return;
}
- const selectedFolder = folders.currentId();
- if (selectedFolder) {
+ if (currentFolder()) {
event.preventDefault();
- openFolder(world, folders.get(selectedFolder)?.parentId ?? null);
+ handleGoToParent();
}
return;
@@ -270,29 +257,21 @@ export function Assets() {
const handleBackgroundClick = (event: MouseEvent) => {
if (!getAssetIdFromTarget(event.target)) {
- clearSelectedAssets(world);
+ setSelectedAssetId(null);
}
};
- const hasAssets = () => assets.all().length > 0;
- const hasContent = () => hasAssets() || folders.all().length > 0;
+ const hasAssets = () => allAssets().length > 0;
+ const hasContent = () => hasAssets() || allFolders().length > 0;
const isFiltering = () => query().trim().length > 0 || assetFilter() !== "ALL";
const isEmptyView = () => visibleFolders().length === 0 && filteredAssets().length === 0;
- const handleCreateFolder = async () => {
- try {
- const parentId = folders.currentId();
- const folder = await createFolder(world, nextFolderName(world, parentId), parentId);
- setRenamingFolderId(folder.id);
- } catch (e) {
- toast.error("Failed to create folder", { description: (e as Error).message });
- }
- };
-
- const handleOpenFolder = (folderId: string) => {
- setQuery("");
- openFolder(world, folderId);
- };
+ const handleCreateFolder = () => withLibrary((lib) => {
+ const parent = currentFolder();
+ const name = lib.uniqueFolderName(parent, "New folder");
+ const folder = lib.createFolder(parent ? `${parent}/${name}` : name);
+ setRenamingFolder(folder);
+ });
onMount(() => {
const onCreateFolder = () => handleCreateFolder();
@@ -313,7 +292,7 @@ export function Assets() {
>
<div class="h-12 shrink-0 flex items-center gap-2 px-4 border-y border-border">
<div class="flex-1 min-w-0 flex items-center gap-0.5 text-[12px] leading-5 font-strong text-foreground">
- <Show when={folders.currentId() !== null}>
+ <Show when={currentFolder() !== ""}>
<Button
size="icon"
variant="ghost"
@@ -414,10 +393,10 @@ export function Assets() {
<Show when={hasContent() && !isDragging()}>
<div class="shrink-0 px-4 pt-4 pb-1 flex flex-col gap-4">
- <Show when={folders.currentId() !== null}>
+ <Show when={currentFolder() !== ""}>
<Breadcrumbs separator="/">
<BreadcrumbList class="text-xs gap-1 sm:gap-1">
- <BreadcrumbCrumb world={world} label="All assets" folderId={null} />
+ <BreadcrumbCrumb label="All assets" folder="" onOpen={openFolder} />
<Show when={collapsedFolders().length > 0}>
<BreadcrumbsSeparator />
<BreadcrumbsItem>
@@ -432,9 +411,9 @@ export function Assets() {
{(folder) => (
<DropdownMenuItem
tone="neutral"
- onSelect={() => openFolder(world, folder.id)}
+ onSelect={() => openFolder(folder)}
>
- <span class="truncate">{folder.name}</span>
+ <span class="truncate">{basename(folder)}</span>
</DropdownMenuItem>
)}
</For>
@@ -448,10 +427,10 @@ export function Assets() {
<>
<BreadcrumbsSeparator />
<BreadcrumbCrumb
- world={world}
- label={folder.name}
- folderId={folder.id}
- current={folder.id === folders.currentId()}
+ label={basename(folder)}
+ folder={folder}
+ current={folder === currentFolder()}
+ onOpen={openFolder}
/>
</>
)}
@@ -530,11 +509,11 @@ export function Assets() {
<For each={visibleFolders()}>
{(folder) => (
<FolderItem
- folder={folder}
- renaming={renamingFolderId() === folder.id}
- onRenameStart={() => setRenamingFolderId(folder.id)}
- onRenameEnd={() => setRenamingFolderId(null)}
- onOpen={() => handleOpenFolder(folder.id)}
+ path={folder}
+ renaming={renamingFolder() === folder}
+ onRenameStart={() => setRenamingFolder(folder)}
+ onRenameEnd={() => setRenamingFolder(null)}
+ onOpen={() => openFolder(folder)}
/>
)}
</For>
@@ -589,10 +568,10 @@ function FilterIconStack(props: FilterIconStackProps) {
}
type BreadcrumbCrumbProps = {
- world: EngineWorld;
label: string;
- folderId: string | null;
+ folder: string;
current?: boolean;
+ onOpen(folder: string): void;
};
/**
@@ -601,25 +580,21 @@ type BreadcrumbCrumbProps = {
*/
function BreadcrumbCrumb(props: BreadcrumbCrumbProps) {
const [isDropTarget, setIsDropTarget] = createSignal(false);
-
- const isInternalDrag = (event: DragEvent) => {
- const types = event.dataTransfer?.types ?? [];
- return types.includes(ASSET_DRAG_TYPE) || types.includes(FOLDER_DRAG_TYPE);
- };
+ const library = useLibrary();
const handleDragOver = (event: DragEvent) => {
- if (!isInternalDrag(event)) return;
+ if (!isAssetOrFolderDrag(event)) return;
event.preventDefault();
event.stopPropagation();
setIsDropTarget(true);
};
- const handleDrop = async (event: DragEvent) => {
- if (!isInternalDrag(event)) return;
+ const handleDrop = (event: DragEvent) => {
+ if (!isAssetOrFolderDrag(event)) return;
event.preventDefault();
event.stopPropagation();
setIsDropTarget(false);
- await handleFolderDrop(props.world, event, props.folderId);
+ handleFolderDrop(library(), event, props.folder);
};
return (
@@ -629,7 +604,7 @@ function BreadcrumbCrumb(props: BreadcrumbCrumbProps) {
current={props.current}
data-drop-target={isDropTarget()}
class="max-w-32 truncate rounded px-1 outline-none focus-ring data-[current]:text-muted-foreground hover:text-foreground data-[drop-target=true]:ring-2 data-[drop-target=true]:ring-inset data-[drop-target=true]:ring-ring"
- onClick={() => openFolder(props.world, props.folderId)}
+ onClick={() => props.onOpen(props.folder)}
on:dragover={handleDragOver}
on:dragleave={() => setIsDropTarget(false)}
on:drop={handleDrop}
diff --git a/apps/web/src/components/sidebar-left/folder-item.tsx b/apps/web/src/components/sidebar-left/folder-item.tsx
index 2ffdf76f..3b5efad4 100644
--- a/apps/web/src/components/sidebar-left/folder-item.tsx
+++ b/apps/web/src/components/sidebar-left/folder-item.tsx
@@ -4,6 +4,7 @@
import { createMemo, createSignal, Show } from "solid-js";
import { toast } from "somoto";
+import { basename } from "@diffusionstudio/assets";
import {
ContextMenu,
ContextMenuContent,
@@ -13,54 +14,46 @@ import {
ContextMenuTrigger,
} from "../ui/context-menu";
import { Icon } from "../ui/icon";
-import { useEngine } from "@/context/engine";
-import {
- renameFolder,
- deleteFolder,
- moveFolder,
- moveAssetsToFolder,
- useAssets,
- useFolders,
- assetFolderId,
-} from "@/components/engine";
+import { useLibrary } from "@/engine/library";
-import type { Folder } from "@/components/engine/db";
+import type { AssetLibrary } from "@diffusionstudio/assets";
+/** Drag payloads: comma-separated asset ids, or one folder path. */
export const ASSET_DRAG_TYPE = "application/x-asset-id";
-export const FOLDER_DRAG_TYPE = "application/x-folder-id";
+export const FOLDER_DRAG_TYPE = "application/x-folder-path";
-function isAssetOrFolderDrag(event: DragEvent): boolean {
+export function isAssetOrFolderDrag(event: DragEvent): boolean {
const types = event.dataTransfer?.types ?? [];
return types.includes(ASSET_DRAG_TYPE) || types.includes(FOLDER_DRAG_TYPE);
}
/**
* Handles a drop of an internal asset/folder drag onto the target folder
- * (null = root). Returns true when the event was an internal drag.
+ * ('' = root). Returns true when the event was an internal drag.
*/
-export async function handleFolderDrop(
- world: ReturnType<typeof useEngine>["world"],
- event: DragEvent,
- targetFolderId: string | null,
-): Promise<boolean> {
- const assetId = event.dataTransfer?.getData(ASSET_DRAG_TYPE);
- const folderId = event.dataTransfer?.getData(FOLDER_DRAG_TYPE);
- if (!assetId && !folderId) return false;
-
- if (assetId) {
- await moveAssetsToFolder(world, [assetId], targetFolderId);
- }
- if (folderId && folderId !== targetFolderId) {
- const moved = await moveFolder(world, folderId, targetFolderId);
- if (!moved) {
+export function handleFolderDrop(library: AssetLibrary | undefined, event: DragEvent, target: string): boolean {
+ if (!library) return false;
+
+ const assetIds = event.dataTransfer?.getData(ASSET_DRAG_TYPE)?.split(",").filter(Boolean) ?? [];
+ const folder = event.dataTransfer?.getData(FOLDER_DRAG_TYPE);
+ if (!assetIds.length && !folder) return false;
+
+ const assets = assetIds.map((id) => library.get(id)).filter((asset) => asset !== undefined);
+ if (assets.length) library.move(assets, target);
+
+ if (folder && folder !== target) {
+ if (target === folder || target.startsWith(`${folder}/`)) {
toast("Cannot move a folder into itself");
+ } else {
+ library.moveFolder(folder, target);
}
}
return true;
}
export type FolderItemProps = {
- folder: Folder;
+ /** The folder's library path. */
+ path: string;
renaming: boolean;
onRenameStart(): void;
onRenameEnd(): void;
@@ -68,19 +61,17 @@ export type FolderItemProps = {
};
export function FolderItem(props: FolderItemProps) {
- const { world } = useEngine();
- const assets = useAssets(world);
- const folders = useFolders(world);
-
+ const library = useLibrary();
const [isDropTarget, setIsDropTarget] = createSignal(false);
- const childCount = createMemo(() =>
- folders.childrenOf(props.folder.id).length +
- assets.all().filter((asset) => assetFolderId(world, asset) === props.folder.id).length
- );
+ const name = () => basename(props.path);
+ const childCount = createMemo(() => {
+ const children = library()?.childrenOf(props.path);
+ return children ? children.folders.length + children.assets.length : 0;
+ });
const handleDragStart = (event: DragEvent) => {
- event.dataTransfer?.setData(FOLDER_DRAG_TYPE, props.folder.id);
+ event.dataTransfer?.setData(FOLDER_DRAG_TYPE, props.path);
};
const handleDragOver = (event: DragEvent) => {
@@ -90,33 +81,23 @@ export function FolderItem(props: FolderItemProps) {
setIsDropTarget(true);
};
- const handleDragLeave = () => {
- setIsDropTarget(false);
- };
-
- const handleDrop = async (event: DragEvent) => {
+ const handleDrop = (event: DragEvent) => {
if (!isAssetOrFolderDrag(event)) return;
event.preventDefault();
event.stopPropagation();
setIsDropTarget(false);
- await handleFolderDrop(world, event, props.folder.id);
+ handleFolderDrop(library(), event, props.path);
};
- const commitRename = async (nextName: string) => {
+ const commitRename = (nextName: string) => {
if (!props.renaming) return;
props.onRenameEnd();
const trimmed = nextName.trim();
- if (!trimmed || trimmed === props.folder.name) return;
- try {
- await renameFolder(world, props.folder.id, trimmed);
- } catch (e) {
- toast.error("Failed to rename", { description: (e as Error).message });
- }
+ if (!trimmed || trimmed === name()) return;
+ library()?.renameFolder(props.path, trimmed);
};
- const handleRenameKeyDown = (
- event: KeyboardEvent & { currentTarget: HTMLInputElement }
- ) => {
+ const handleRenameKeyDown = (event: KeyboardEvent & { currentTarget: HTMLInputElement }) => {
event.stopPropagation();
if (event.key === "Enter") {
event.preventDefault();
@@ -136,7 +117,7 @@ export function FolderItem(props: FolderItemProps) {
const handleDelete = async () => {
try {
- await deleteFolder(world, props.folder.id);
+ await library()?.deleteFolder(props.path);
} catch (e) {
toast.error("Failed to delete", { description: (e as Error).message });
}
@@ -147,11 +128,11 @@ export function FolderItem(props: FolderItemProps) {
<ContextMenuTrigger
as="div"
class="flex flex-col gap-1 text-left"
- data-folder-id={props.folder.id}
+ data-folder-path={props.path}
draggable={true}
onDragStart={handleDragStart}
on:dragover={handleDragOver}
- on:dragleave={handleDragLeave}
+ on:dragleave={() => setIsDropTarget(false)}
on:drop={handleDrop}
onClick={props.onOpen}
>
@@ -162,26 +143,20 @@ export function FolderItem(props: FolderItemProps) {
<Icon name="folder-thumbnail" class="w-8 h-6 text-muted-foreground pointer-events-none" />
<Show when={childCount() > 0}>
<div class="absolute left-1 top-1 z-20 flex h-4 items-center justify-center rounded bg-overlay px-1">
- <span class="text-xxs text-primary-foreground">
- {childCount()}
- </span>
+ <span class="text-xxs text-primary-foreground">{childCount()}</span>
</div>
</Show>
</div>
<Show
when={props.renaming}
- fallback={
- <div class="text-xs text-foreground truncate select-none">
- {props.folder.name}
- </div>
- }
+ fallback={<div class="text-xs text-foreground truncate select-none">{name()}</div>}
>
<input
ref={handleFocusElement}
type="text"
name="folder-name"
autocomplete="off"
- value={props.folder.name}
+ value={name()}
onKeyDown={handleRenameKeyDown}
onBlur={(e) => commitRename(e.currentTarget.value)}
onClick={(e) => e.stopPropagation()}
@@ -193,17 +168,11 @@ export function FolderItem(props: FolderItemProps) {
</ContextMenuTrigger>
<ContextMenuPortal>
<ContextMenuContent class="w-[220px]">
- <ContextMenuItem onSelect={props.onOpen}>
- Open
- </ContextMenuItem>
+ <ContextMenuItem onSelect={props.onOpen}>Open</ContextMenuItem>
<ContextMenuSeparator />
- <ContextMenuItem onSelect={props.onRenameStart}>
- Rename
- </ContextMenuItem>
+ <ContextMenuItem onSelect={props.onRenameStart}>Rename</ContextMenuItem>
<ContextMenuSeparator />
- <ContextMenuItem onSelect={handleDelete}>
- Delete
- </ContextMenuItem>
+ <ContextMenuItem onSelect={handleDelete}>Delete</ContextMenuItem>
</ContextMenuContent>
</ContextMenuPortal>
</ContextMenu>
diff --git a/apps/web/src/components/sidebar-left/project-menu/file-menu.tsx b/apps/web/src/components/sidebar-left/project-menu/file-menu.tsx
index 55d0fa97..304f6e80 100644
--- a/apps/web/src/components/sidebar-left/project-menu/file-menu.tsx
+++ b/apps/web/src/components/sidebar-left/project-menu/file-menu.tsx
@@ -13,23 +13,27 @@ import {
DropdownMenuGroup,
} from "@/components/ui/dropdown-menu";
import { useNavigate } from "@solidjs/router";
-import { getAllEntities } from "bitecs";
import { For, Show, createMemo } from "solid-js";
import { toast } from "somoto";
import { generateProjectName } from "@/components/engine/db";
import { createProject, deleteProject, duplicateProject, pickProjectsRoot, projectsRoot } from "@/projects";
import { Not } from "bitecs";
-import { ChildOf, isScene, useQuery, loadAsset, changeAssetDirectory, removeAsset, getAssetFile } from "@/components/engine";
+import { ChildOf, isScene, useQuery } from "@/components/engine";
+import { AssetId, getAssetFile } from "@diffusionstudio/runtime";
+import { assetName } from "@diffusionstudio/assets";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { useLibrary } from "@/engine/library";
+import { pickAndImport } from "@/engine/asset-actions";
import { projectRoute, useProjectId } from "@/hooks/use-project-id";
import { useEngine } from "@/context/engine";
import { useExport } from "@/context/export";
import { getDefaultExportTemplate } from "@/components/sidebar-right/inspector/export-templates";
-import { showFileDialog, mimeTypeToExtension } from "@/utils";
+import { mimeTypeToExtension } from "@/utils";
export function FileMenu() {
const navigate = useNavigate();
const projectId = useProjectId();
- const { world } = useEngine();
+ const library = useLibrary();
const handleNewProject = async () => {
try {
@@ -62,26 +66,13 @@ export function FileMenu() {
}
};
- const handleChangeProjectFolder = async () => {
- try {
- await changeAssetDirectory(world);
- } catch (e) {
- toast.error("Failed to change project folder", {
- description: (e as Error).message,
- });
- }
- };
-
const handleImportFromComputer = async () => {
- try {
- const files = await showFileDialog();
- if (files.length === 0) return;
- await Promise.all(files.map((file) => loadAsset(world, file)));
- } catch (e) {
- toast.error("Failed to import assets", {
- description: (e as Error).message,
- });
+ const lib = library();
+ if (!lib) {
+ toast("No project open");
+ return;
}
+ await pickAndImport(lib, "");
};
return (
@@ -97,14 +88,6 @@ export function FileMenu() {
<DropdownMenuSeparator />
- <DropdownMenuGroup>
- <DropdownMenuItem onSelect={handleChangeProjectFolder}>
- Change project folder...
- </DropdownMenuItem>
- </DropdownMenuGroup>
-
- <DropdownMenuSeparator />
-
<DropdownMenuGroup>
<DropdownMenuItem onSelect={handleImportFromComputer}>
Import from computer...
@@ -145,10 +128,11 @@ export function FileMenu() {
}
export function FileAssetMenu() {
- const { world } = useEngine();
+ const world = useWorld();
+ const library = useLibrary();
const handleDownloadAll = async () => {
- const all = Array.from(world.assets.values());
+ const all = library()?.list() ?? [];
if (all.length === 0) {
toast("No assets to download");
return;
@@ -173,9 +157,10 @@ export function FileAssetMenu() {
await Promise.all(
all.map(async (asset) => {
const file = await getAssetFile(asset);
- const base = asset.name.match(/\.[^.]+$/)
- ? asset.name
- : asset.name + mimeTypeToExtension(asset.mimeType);
+ const assetFileName = assetName(asset);
+ const base = assetFileName.match(/\.[^.]+$/)
+ ? assetFileName
+ : assetFileName + mimeTypeToExtension(asset.mimeType);
let name = base;
let i = 1;
@@ -205,58 +190,24 @@ export function FileAssetMenu() {
}
};
- const handleRemoveDuplicates = async () => {
- const all = Array.from(world.assets.values());
- const byHash = new Map<string, typeof all>();
- for (const asset of all) {
- const list = byHash.get(asset.hash) ?? [];
- list.push(asset);
- byHash.set(asset.hash, list);
- }
-
- const toRemove: string[] = [];
- for (const list of byHash.values()) {
- if (list.length <= 1) continue;
- list.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
- for (const asset of list.slice(1)) toRemove.push(asset.id);
- }
-
- if (toRemove.length === 0) {
- toast("No duplicated media found");
- return;
- }
-
- try {
- await removeAsset(world, ...toRemove);
- toast.success(
- `Removed ${toRemove.length} duplicate${toRemove.length === 1 ? "" : "s"}`
- );
- } catch (e) {
- toast.error("Failed to remove duplicated media", {
- description: (e as Error).message,
- });
- }
- };
-
const handleRemoveUnused = async () => {
+ const lib = library();
+ if (!lib) return;
+
const referenced = new Set<string>();
- const AssetId = world.components.AssetId;
- for (const eid of getAllEntities(world)) {
- const id = AssetId[eid];
+ for (const entity of world.query(AssetId)) {
+ const id = entity.get(AssetId)?.value;
if (id) referenced.add(id);
}
- const toRemove = Array.from(world.assets.values())
- .filter((asset) => !referenced.has(asset.id))
- .map((asset) => asset.id);
-
+ const toRemove = lib.list().filter((asset) => !referenced.has(asset.id));
if (toRemove.length === 0) {
toast("No unused media found");
return;
}
try {
- await removeAsset(world, ...toRemove);
+ await lib.remove(toRemove);
toast.success(
`Removed ${toRemove.length} unused asset${toRemove.length === 1 ? "" : "s"}`
);
@@ -278,9 +229,6 @@ export function FileAssetMenu() {
<DropdownMenuSeparator />
<DropdownMenuGroup>
- <DropdownMenuItem onSelect={handleRemoveDuplicates}>
- Remove duplicated media...
- </DropdownMenuItem>
<DropdownMenuItem onSelect={handleRemoveUnused}>
Remove unused media...
</DropdownMenuItem>
diff --git a/apps/web/src/components/sidebar-right/inspector/asset-picker.tsx b/apps/web/src/components/sidebar-right/inspector/asset-picker.tsx
index 89fe7a83..23f7445d 100644
--- a/apps/web/src/components/sidebar-right/inspector/asset-picker.tsx
+++ b/apps/web/src/components/sidebar-right/inspector/asset-picker.tsx
@@ -5,7 +5,7 @@
import { For, Show, createMemo, createSignal, type JSX } from "solid-js";
import { toast } from "somoto";
-import { downloadAsset } from "@/components/assets/actions";
+import { saveAssetAs } from "@/engine/asset-actions";
import { AssetThumbnail } from "@/components/ui/asset-thumbnail";
import { Button } from "@/components/ui/button";
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
@@ -248,9 +248,11 @@ function AssetCard(props: AssetCardProps) {
toast("Replace media", { description: props.asset.name });
};
- const handleDownload = async () => {
- await downloadAsset(props.asset);
- };
+ const handleSaveAs = () => saveAssetAs({
+ handle: props.asset.handle,
+ mimeType: props.asset.mimeType,
+ path: props.asset.name
+ });
const handleDelete = async () => {
try {
@@ -307,20 +309,20 @@ function AssetCard(props: AssetCardProps) {
<ContextMenuSeparator />
- <ContextMenuItem onSelect={() => void handleRename()}>
+ <ContextMenuItem onSelect={handleRename}>
<MenuItemLabel>Rename</MenuItemLabel>
<ContextMenuShortcut>⌘R</ContextMenuShortcut>
</ContextMenuItem>
<ContextMenuItem onSelect={handleReplace}>
<MenuItemLabel>Replace</MenuItemLabel>
</ContextMenuItem>
- <ContextMenuItem onSelect={() => void handleDownload()}>
- <MenuItemLabel>Download</MenuItemLabel>
+ <ContextMenuItem onSelect={handleSaveAs}>
+ <MenuItemLabel>Save as…</MenuItemLabel>
</ContextMenuItem>
<ContextMenuSeparator />
- <ContextMenuItem onSelect={() => void handleDelete()}>
+ <ContextMenuItem onSelect={handleDelete}>
<MenuItemLabel>Delete</MenuItemLabel>
<ContextMenuShortcut>⌫</ContextMenuShortcut>
</ContextMenuItem>
diff --git a/apps/web/src/components/ui/asset-thumbnail.tsx b/apps/web/src/components/ui/asset-thumbnail.tsx
index 0fcf651c..4641f0f1 100644
--- a/apps/web/src/components/ui/asset-thumbnail.tsx
+++ b/apps/web/src/components/ui/asset-thumbnail.tsx
@@ -5,12 +5,29 @@
import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
import { Show, For, createResource } from 'solid-js';
import { cx } from '@/lib/cva';
-import { getAudioPeaksAsync } from '@/components/engine/decoders/audio-peaks';
-import { useEngine } from '@/context/engine';
-import { assetsVersion, getAssetFile } from '@/components/engine';
+import { getAssetFile, getAudioPeaksAsync } from '@diffusionstudio/runtime';
+
+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
+
+/**
+ * What a thumbnail needs of an asset: the runtime's `Asset` and the legacy
+ * engine's both qualify.
+ */
+export type ThumbnailAsset = {
+ id: string;
+ type: string;
+ mimeType: string;
+ handle: { getFile(): Promise<File> };
+ width?: number;
+ height?: number;
+ stat?: { mtime: number };
+ lastModified?: number;
+};
+
+type Asset = ThumbnailAsset;
-import type { AudioAsset, ImageAsset, VideoAsset } from '@/components/engine/db';
-import type { Asset } from '@/components/engine/db';
+/** Re-renders a thumbnail when the asset, or the file behind it, changes. */
+const keyOf = (asset: Asset): string => `${asset.id}:${asset.stat?.mtime ?? asset.lastModified ?? ''}`;
const DEFAULT_THUMBNAIL_SIZE = {
width: 300,
@@ -23,9 +40,8 @@ type ThumbnailSize = {
};
function ImageThumbnail(props: { asset: Asset; size: ThumbnailSize }) {
- const engine = useEngine();
const [url] = createResource(
- () => `${props.asset.id}:${assetsVersion(engine.world)}`,
+ () => keyOf(props.asset),
async () => {
const objectUrl = URL.createObjectURL(await getAssetFile(props.asset));
try {
@@ -41,9 +57,8 @@ function ImageThumbnail(props: { asset: Asset; size: ThumbnailSize }) {
canvas.height = props.size.height;
const ctx = canvas.getContext('2d')!;
- const asset = props.asset as ImageAsset;
- const imgW = image.naturalWidth || asset.width;
- const imgH = image.naturalHeight || asset.height;
+ const imgW = image.naturalWidth || props.asset.width || props.size.width;
+ const imgH = image.naturalHeight || props.asset.height || props.size.height;
const scale = Math.max(props.size.width / imgW, props.size.height / imgH);
const scaledW = imgW * scale;
const scaledH = imgH * scale;
@@ -69,7 +84,8 @@ function ImageThumbnail(props: { asset: Asset; size: ThumbnailSize }) {
<Show when={url()}>
<img
src={url()!}
- alt={props.asset.name}
+ alt=""
+
class="w-full h-full object-cover select-none"
/>
</Show>
@@ -77,9 +93,8 @@ function ImageThumbnail(props: { asset: Asset; size: ThumbnailSize }) {
}
function VideoThumbnail(props: { asset: Asset; size: ThumbnailSize }) {
- const engine = useEngine();
const [src] = createResource(
- () => `${props.asset.id}:${assetsVersion(engine.world)}`,
+ () => keyOf(props.asset),
async () => {
try {
const file = await getAssetFile(props.asset);
@@ -120,7 +135,8 @@ function VideoThumbnail(props: { asset: Asset; size: ThumbnailSize }) {
<Show when={src()}>
<img
src={src()!}
- alt={props.asset.name}
+ alt=""
+
class="w-full h-full object-cover select-none"
/>
</Show>
@@ -144,10 +160,9 @@ function TranscriptThumbnail() {
}
function AudioThumbnail(props: { asset: Asset }) {
- const engine = useEngine();
const [bins] = createResource(
- () => `${props.asset.id}:${assetsVersion(engine.world)}`,
- () => getAudioPeaksAsync(props.asset as AudioAsset | VideoAsset),
+ () => keyOf(props.asset),
+ () => getAudioPeaksAsync(props.asset as unknown as AudioAsset | VideoAsset),
);
return (
diff --git a/apps/web/src/context/dapi/api.tsx b/apps/web/src/context/dapi/api.tsx
index 9418550e..5f6931ce 100644
--- a/apps/web/src/context/dapi/api.tsx
+++ b/apps/web/src/context/dapi/api.tsx
@@ -4,6 +4,7 @@
import { createEffect, createContext, useContext, onCleanup, onMount, createResource } from "solid-js";
import { useEngine } from '@/context/engine';
+import { useWorld } from '@diffusionstudio/koota-solid';
import { useProjectId } from '@/hooks/use-project-id';
import { useAuth } from '@/context/auth';
import { t, q, q0 } from "@/lib/cli-rpc";
@@ -22,6 +23,7 @@ import { handleGetFullscreenState, handleWindowFullscreenChange, handleWindowScr
import type { JSX, Accessor } from 'solid-js';
import type { User } from '@supabase/supabase-js';
import type { Engine } from '@/components/engine';
+import type { World } from 'koota';
type EditorApiProviderProps = {
children: JSX.Element;
@@ -52,6 +54,7 @@ export function EditorApiProvider(props: EditorApiProviderProps) {
};
const getEngine = () => engine;
+ const world = useWorld();
createEffect(() => {
document.documentElement.dataset.fullscreen = String(isFullscreen());
@@ -66,7 +69,7 @@ export function EditorApiProvider(props: EditorApiProviderProps) {
if (!window.desktop || !engine.initialized() || projectId() !== engine.world.projectId) return;
- const router = createAppRouter({ getEngine, getUser, requireAuth });
+ const router = createAppRouter({ getEngine, world, getUser, requireAuth });
onCleanup(cliBridge.register(createRouterCaller(router)));
});
@@ -85,11 +88,12 @@ export function EditorApiProvider(props: EditorApiProviderProps) {
type AppRouterDeps = {
getEngine: () => Engine;
+ world: World;
getUser: () => User;
requireAuth: <I, O>(fn: (data: I) => Promise<O>) => (data: I) => Promise<O>;
};
-function createAppRouter({ getEngine, getUser, requireAuth }: AppRouterDeps) {
+function createAppRouter({ getEngine, world, getUser, requireAuth }: AppRouterDeps) {
return t.router({
ping: t.procedure.query(() => {}),
whoami: t.procedure.query(() => getUser()),
@@ -100,12 +104,12 @@ function createAppRouter({ getEngine, getUser, requireAuth }: AppRouterDeps) {
screenshot: q0(handleWindowScreenshot()),
voices: q0(handleVoices()),
media: t.router({
- probe: q(handleMediaProbe(getEngine)),
- frame: q(handleMediaFrame(getEngine)),
- transcribe: q(handleMediaTranscribe(getEngine)),
- filmstrip: q(handleMediaFilmstrip(getEngine)),
- waveform: q(handleMediaWaveform(getEngine)),
- listen: q(requireAuth(handleMediaListen(getEngine))),
+ probe: q(handleMediaProbe(world)),
+ frame: q(handleMediaFrame(world)),
+ transcribe: q(handleMediaTranscribe(world)),
+ filmstrip: q(handleMediaFilmstrip(world)),
+ waveform: q(handleMediaWaveform(world)),
+ listen: q(requireAuth(handleMediaListen(world))),
}),
});
}
diff --git a/apps/web/src/context/dapi/media.ts b/apps/web/src/context/dapi/media.ts
index 1c094275..e0c953a1 100644
--- a/apps/web/src/context/dapi/media.ts
+++ b/apps/web/src/context/dapi/media.ts
@@ -3,46 +3,41 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { ALL_FORMATS, BlobSource, CanvasSink, Input } from 'mediabunny';
-import { ElectronFileHandle } from '@/lib/electron-file-handle';
import { pickInformativeTimes } from './frame-triage';
import { trpc } from '@/lib/trpc';
-import { uploadBlob, filmstripAsset, waveformAsset, describeFileAsset, getAssetFile, formatTimecode, composeSheet, planSheet, planSheetSizes, sheetTimecode } from '@/components/engine';
+import { uploadBlob, composeSheet, planSheet, planSheetSizes, sheetTimecode } from '@/components/engine';
import { assert } from '@/utils';
-import {
- transcodeForTranscription,
- transcodeForAnalysis,
- startResumableSession,
- uploadResumableStream,
-} from '@/components/engine';
-
-import type { Engine } from "@/components/engine";
-import type { Asset } from "@/components/engine/db";
+import { startResumableSession, uploadResumableStream } from '@/components/engine';
+import { filmstripAsset, formatTimecode, getAsset, getAssetFile, getLibrary, Project, transcodeForAnalysis, transcodeForTranscription, waveformAsset } from '@diffusionstudio/runtime';
+import { assetName } from '@diffusionstudio/assets';
+
+import type { World } from 'koota';
+import type { Asset } from '@diffusionstudio/assets';
import type { MediaListenRequest, MediaListenResult, MediaFrameRequest, MediaFrameResult, TimecodedImage, MediaProbeRequest, AssetRef, MediaTranscribeRequest, MediaTranscribeResult, MediaFilmstripRequest, MediaFilmstripResult, MediaWaveformRequest, MediaWaveformResult, TranscriptSegment } from "@diffusionstudio/cli/channels";
-import type { Accessor } from "solid-js";
/**
- * Resolves a command target: an id looks up the project's asset library, a
- * path describes the file in place as an ephemeral asset (never added to the
- * library). Ephemeral assets carry the path as their id.
+ * Resolves a command target: an id (or library path) looks up the project's
+ * asset library; a path is described in place through the library without
+ * being added to it (a transient asset).
*/
-async function resolveAssetRef(world: Engine["world"], ref: AssetRef): Promise<Asset> {
- if ("path" in ref) return describeFileAsset(new ElectronFileHandle(ref.path));
- const asset = world.assets.get(ref.id);
+async function resolveAssetRef(world: World, ref: AssetRef): Promise<Asset> {
+ if ("path" in ref) return getLibrary(world).resolve(ref.path);
+ const asset = getAsset(world, ref.id);
assert(asset, `Asset ${ref.id} not found.`);
return asset;
}
const PROBE_SAMPLE_PACKETS = 200;
-export function handleMediaProbe(engine: Accessor<Engine>) {
+export function handleMediaProbe(world: World) {
return async (req: MediaProbeRequest): Promise<unknown> => {
- const { world } = engine();
const asset = await resolveAssetRef(world, req);
const blob = await getAssetFile(asset);
const base = {
id: asset.id,
- name: asset.name,
+ name: assetName(asset),
+ path: asset.path,
type: asset.type,
mimeType: asset.mimeType,
size: blob.size,
@@ -111,11 +106,10 @@ const FRAME_QUALITY_BUDGETS = {
// Default cap on frames returned by auto selection when `count` is not given.
const AUTO_MAX_FRAMES = 30;
-export function handleMediaFrame(engine: Accessor<Engine>) {
+export function handleMediaFrame(world: World) {
return async (req: MediaFrameRequest): Promise<MediaFrameResult> => {
const { times, count, start, end, quality, auto } = req;
const combine = req.combine ?? true;
- const { world } = engine();
const asset = await resolveAssetRef(world, req);
const id = asset.id;
assert(asset.type === "VIDEO", `Asset ${id} is not a video.`);
@@ -249,9 +243,8 @@ export function handleMediaFrame(engine: Accessor<Engine>) {
const transcripts = new Map<string, TranscriptSegment[]>();
-export function handleMediaTranscribe(engine: Accessor<Engine>) {
+export function handleMediaTranscribe(world: World) {
return async (req: MediaTranscribeRequest): Promise<MediaTranscribeResult> => {
- const { world } = engine();
const asset = await resolveAssetRef(world, req);
const id = asset.id;
assert(
@@ -259,7 +252,7 @@ export function handleMediaTranscribe(engine: Accessor<Engine>) {
`Asset ${id} is not a video or audio asset.`,
);
- let transcript = transcripts.get(asset.hash);
+ let transcript = transcripts.get(asset.id);
if (!transcript) {
const uploadId = crypto.randomUUID();
const audioFile = await transcodeForTranscription(asset);
@@ -271,16 +264,15 @@ export function handleMediaTranscribe(engine: Accessor<Engine>) {
throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");
}
- transcripts.set(asset.hash, transcript);
+ transcripts.set(asset.id, transcript);
}
return { segments: transcript };
};
}
-export function handleMediaFilmstrip(engine: Accessor<Engine>) {
+export function handleMediaFilmstrip(world: World) {
return async (req: MediaFilmstripRequest): Promise<MediaFilmstripResult> => {
- const { world } = engine();
const asset = await resolveAssetRef(world, req);
const { dataUrl, ...rest } = await filmstripAsset(asset, { start: req.start, end: req.end, scale: req.scale });
const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);
@@ -288,9 +280,8 @@ export function handleMediaFilmstrip(engine: Accessor<Engine>) {
};
}
-export function handleMediaWaveform(engine: Accessor<Engine>) {
+export function handleMediaWaveform(world: World) {
return async (req: MediaWaveformRequest): Promise<MediaWaveformResult> => {
- const { world } = engine();
const asset = await resolveAssetRef(world, req);
const { dataUrl, ...rest } = await waveformAsset(asset, { start: req.start, end: req.end, scale: req.scale });
const base64 = dataUrl.slice(dataUrl.indexOf(",") + 1);
@@ -298,11 +289,10 @@ export function handleMediaWaveform(engine: Accessor<Engine>) {
};
}
-export function handleMediaListen(engine: Accessor<Engine>) {
+export function handleMediaListen(world: World) {
return async (req: MediaListenRequest): Promise<MediaListenResult> => {
const { prompt, start, end } = req;
let { stripVideo } = req;
- const { world } = engine();
const asset = await resolveAssetRef(world, req);
const id = asset.id;
assert(
@@ -317,8 +307,7 @@ export function handleMediaListen(engine: Accessor<Engine>) {
asset.type === "VIDEO" ? (stripVideo ? "audio/ogg" : "video/mp4") : "audio/ogg";
const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";
- const key = world.assets.has(id) ? id : asset.hash;
- const uploadId = `${world.projectId}-${key}-analyze${stripVideo ? "-audio" : ""}${window}`
+ const uploadId = `${world.get(Project)?.id ?? "project"}-${id}-analyze${stripVideo ? "-audio" : ""}${window}`
.replace(/[^A-Za-z0-9._-]/g, "_");
const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({
action: "resumable",
diff --git a/apps/web/src/engine/asset-actions.ts b/apps/web/src/engine/asset-actions.ts
new file mode 100644
index 00000000..52d2daab
--- /dev/null
+++ b/apps/web/src/engine/asset-actions.ts
@@ -0,0 +1,40 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// File actions on the library, with the user told how they went: the
+// mechanics live in @diffusionstudio/assets.
+
+import { toast } from "somoto";
+import { importFiles as importFilesInto, pickFiles, saveAssetAs as saveAs } from "@diffusionstudio/assets";
+
+import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
+
+export { droppedFiles, pickFiles } from "@diffusionstudio/assets";
+
+/** Saves a copy of an asset's file where the user says; reports failure. */
+export async function saveAssetAs(asset: Pick<Asset, "handle" | "mimeType" | "path">): Promise<void> {
+ try {
+ await saveAs(asset);
+ } catch (error) {
+ toast.error("Failed to save", { description: (error as Error).message });
+ }
+}
+
+/** Links files into the library at `folder` and reports whatever was skipped or refused. */
+export async function importFiles(library: AssetLibrary, files: ReadonlyArray<File>, folder: string): Promise<Asset[]> {
+ const report = await importFilesInto(library, files, folder);
+
+ if (report.unnamed.length) {
+ toast("Some files could not be imported", { description: "Only files on this computer can be added to the library." });
+ }
+ for (const { source, error } of report.failed) {
+ toast.error(`Could not import ${source.split(/[\\/]/).pop()}`, { description: error.message });
+ }
+ return report.assets;
+}
+
+/** Opens the file picker and imports what the user picks into `folder`. */
+export async function pickAndImport(library: AssetLibrary, folder: string): Promise<Asset[]> {
+ return importFiles(library, await pickFiles(), folder);
+}
diff --git a/apps/web/src/engine/index.ts b/apps/web/src/engine/index.ts
index a1cdfe0c..1dbc959f 100644
--- a/apps/web/src/engine/index.ts
+++ b/apps/web/src/engine/index.ts
@@ -11,3 +11,5 @@ export * from './editor';
export * from './traits';
export * from './input';
export * from './hud';
+export * from './library';
+export * from './asset-actions';
diff --git a/apps/web/src/engine/insert-asset.tsx b/apps/web/src/engine/insert-asset.tsx
new file mode 100644
index 00000000..ebe42be1
--- /dev/null
+++ b/apps/web/src/engine/insert-asset.tsx
@@ -0,0 +1,96 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Putting an asset on the canvas: an element in the project's JSX whose
+// `src` is the asset's library path, inserted the way every drawn element
+// is (see DocumentEditor.insertElement), so it lands in the file too.
+
+import { Audio, Captions, Image, Video } from '@diffusionstudio/reconciler';
+import { Computed, getActiveEntity, getNextName, Root, Source, store } from '@diffusionstudio/runtime';
+import { assetName } from '@diffusionstudio/assets';
+
+import { getDocumentEditor } from './editor';
+
+import type { Asset } from '@diffusionstudio/assets';
+import type { Entity, World } from 'koota';
+
+export interface InsertAssetOptions {
+ /** The scene (or group) to insert into; the active scene by default. */
+ parent?: Entity;
+ /** Top-left corner in the parent's space; centered by default. */
+ x?: number;
+ y?: number;
+ /** Where on the timeline the clip starts, in seconds; the playhead by default. */
+ start?: number;
+}
+
+const AUDIO_WIDTH = 500;
+const AUDIO_HEIGHT = 150;
+
+/**
+ * Inserts `asset` into the project as the element of its type and returns
+ * the entity, or null when there is nothing to insert into (no project is
+ * mounted, or the target has no source to be written under).
+ */
+export function insertAsset(world: World, asset: Asset, options: InsertAssetOptions = {}): Entity | null {
+ const parent = options.parent ?? getActiveEntity(world) ?? world.get(Root)!;
+ if (!parent.get(Source)?.value) return null;
+
+ const editor = getDocumentEditor(world);
+ const src = asset.path;
+ const name = getNextName(world, assetName(asset).replace(/\.[^.]+$/, ''));
+ const start = options.start ?? (store(world, Computed).localTimeInSeconds[parent.id()] ?? 0);
+
+ const size = sizeOf(asset);
+ const position = size ? placement(world, parent, size, options) : {};
+ const timing = start > 0 ? { start } : {};
+
+ const [entity] = editor.insertElement(parent, () => {
+ switch (asset.type) {
+ case 'VIDEO':
+ case 'SEQUENCE':
+ return <Video name={name} src={src} {...position} {...size} {...timing} />;
+ case 'IMAGE':
+ return <Image name={name} src={src} {...position} {...size} {...timing} />;
+ case 'AUDIO':
+ return <Audio name={name} src={src} {...position} {...size} {...timing} />;
+ case 'TRANSCRIPT':
+ return <Captions src={src} {...timing} />;
+ default:
+ return null;
+ }
+ });
+
+ if (entity) editor.select(entity);
+ return entity ?? null;
+}
+
+function sizeOf(asset: Asset): { width: number; height: number } | undefined {
+ switch (asset.type) {
+ case 'VIDEO':
+ case 'IMAGE':
+ case 'SEQUENCE':
+ return { width: Math.round(asset.width), height: Math.round(asset.height) };
+ case 'AUDIO':
+ return { width: AUDIO_WIDTH, height: AUDIO_HEIGHT };
+ default:
+ return undefined;
+ }
+}
+
+/** Where a new element of `size` goes: as asked, or centered in its parent. */
+function placement(
+ world: World,
+ parent: Entity,
+ size: { width: number; height: number },
+ options: InsertAssetOptions,
+): { x?: number; y?: number } {
+ if (options.x !== undefined && options.y !== undefined) {
+ return { x: Math.round(options.x), y: Math.round(options.y) };
+ }
+ const bounds = store(world, Computed);
+ const width = bounds.width[parent.id()] ?? size.width;
+ const height = bounds.height[parent.id()] ?? size.height;
+ return { x: Math.round((width - size.width) / 2), y: Math.round((height - size.height) / 2) };
+}
diff --git a/apps/web/src/engine/library.ts b/apps/web/src/engine/library.ts
new file mode 100644
index 00000000..f28e7265
--- /dev/null
+++ b/apps/web/src/engine/library.ts
@@ -0,0 +1,74 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The asset library in the app: attached to the world for the project on
+// disk and kept in step with the JSX (a rename in the library is a `src`
+// edit in the file).
+
+import { AssetId, Library, Project } from '@diffusionstudio/runtime';
+import { AssetLibrary, MANIFEST_FILE, ASSETS_DIR } from '@diffusionstudio/assets';
+import { useTrait, useWorld } from '@diffusionstudio/koota-solid';
+import { authoredElement } from '@diffusionstudio/reconciler';
+import type { Accessor } from 'solid-js';
+
+import { createProjectFS } from '@/projects/fs';
+import { generateAsset } from '@/utils/jsx-generation';
+import { getDocumentEditor } from './editor';
+
+import type { Asset } from '@diffusionstudio/assets';
+import type { World } from 'koota';
+
+/**
+ * Creates the library of the project at `dir`, attaches it to the world and
+ * starts loading it. Renames in the library are written through to every
+ * element whose `src` named the old path. Returns the library and a
+ * disposer that flushes the manifest and detaches it.
+ */
+export function attachLibrary(world: World, dir: string) {
+ const projectId = world.get(Project)?.id ?? 'project';
+
+ const library = new AssetLibrary(createProjectFS(dir), {
+ generate: (ref, library) => generateAsset(ref, library, projectId),
+ onRename: (asset, from) => followRename(world, asset, from),
+ onRelink: (asset, from) => followRelink(world, asset, from),
+ });
+
+ world.set(Library, library);
+
+ return library;
+}
+
+/** Whether a changed project file is the library's business rather than the JSX's. */
+export function isLibraryFile(path: string): boolean {
+ return path === MANIFEST_FILE || path === ASSETS_DIR || path.startsWith(`${ASSETS_DIR}/`);
+}
+
+/** Rewrites `src` on every element that named an asset by its old path. */
+function followRename(world: World, asset: Asset, from: string): void {
+ const editor = getDocumentEditor(world);
+ for (const entity of world.query(AssetId)) {
+ if (entity.get(AssetId)?.value !== asset.id) continue;
+ const src = authoredElement(entity)?.props.src;
+ if (src !== from) continue;
+ editor.editProperty(entity, 'src', asset.path);
+ }
+}
+
+/** Rebinds every element bound to a relinked asset's old id to its new one. */
+function followRelink(world: World, asset: Asset, from: string): void {
+ for (const entity of world.query(AssetId)) {
+ if (entity.get(AssetId)?.value === from) entity.set(AssetId, { value: asset.id });
+ }
+}
+
+/**
+ * The world's library, or undefined until a project attaches one. The
+ * library's own state is reactive (`assets()`, `folders()`, `childrenOf()`),
+ * so readers inside a tracking scope follow it.
+ */
+export function useLibrary(): Accessor<AssetLibrary | undefined> {
+ const world = useWorld();
+ const attached = useTrait(world, Library);
+ return () => attached() ?? undefined;
+}
diff --git a/apps/web/src/pages/editor.tsx b/apps/web/src/pages/editor.tsx
index 680ae01c..37ab8223 100644
--- a/apps/web/src/pages/editor.tsx
+++ b/apps/web/src/pages/editor.tsx
@@ -15,6 +15,7 @@ import { toast } from 'somoto';
import { useWorld } from '@diffusionstudio/koota-solid';
import { mount } from '@diffusionstudio/reconciler';
import { getDocumentEditor } from '@/engine/editor';
+import { attachLibrary, isLibraryFile } from '@/engine/library';
import { createEditWriter } from '@/projects/edits';
import { compileProject, watchProject, projectDir } from '@/projects/host';
import { useProjectId } from "@/hooks/use-project-id";
@@ -41,6 +42,9 @@ export function EditorPage() {
let disposed = false;
let generation = 0;
+ // The library first: a mounted project's `src` values name its assets.
+ const library = attachLibrary(world, dir);
+
const unmount = (): void => {
// Before the entities go: what the editor changed is still owed to the
// file, whatever happens to the scene that showed it.
@@ -52,9 +56,9 @@ export function EditorPage() {
mounted = undefined;
};
- const load = async (): Promise<void> => {
+ const loadProject = async (): Promise<void> => {
const current = ++generation;
- const result = await compileProject(dir);
+ const [result] = await Promise.all([compileProject(dir), library.load()]);
if (disposed || current !== generation) return;
// A broken edit keeps the last good render on the canvas.
@@ -77,13 +81,21 @@ export function EditorPage() {
}
};
- void load();
- const unwatch = watchProject(dir, () => void load());
+ loadProject();
+
+ const unwatch = watchProject(dir, (path) => {
+ if (isLibraryFile(path)) {
+ library.load();
+ } else {
+ loadProject();
+ }
+ });
onCleanup(() => {
disposed = true;
unwatch();
unmount();
+ library.dispose();
});
});
diff --git a/apps/web/src/projects/fs.ts b/apps/web/src/projects/fs.ts
new file mode 100644
index 00000000..2455e908
--- /dev/null
+++ b/apps/web/src/projects/fs.ts
@@ -0,0 +1,56 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The asset library's view of a project folder on desktop: the main process
+// reads and writes the manifest (as YAML) and lists and stats files; bytes
+// come in as real Files (see ElectronFileHandle) and go out through the
+// streaming FILE_WRITE_* channels.
+
+import { MAIN_CHANNELS } from '@desktop/main-channels';
+import { mainBridge } from '@/lib/ipc';
+import { ElectronFileHandle } from '@/lib/electron-file-handle';
+import { ElectronWritableFileHandle } from '@/lib/electron-file-writable';
+import { isAbsoluteSource } from '@diffusionstudio/assets';
+
+import type { Manifest, ProjectFS } from '@diffusionstudio/assets';
+
+/** Streams `blob` to an absolute path in chunks. */
+async function writeBlob(path: string, blob: Blob): Promise<void> {
+ const target = new ElectronWritableFileHandle(path);
+ const writable = await target.createWritable();
+ const writer = writable.getWriter();
+ try {
+ let position = 0;
+ const reader = blob.stream().getReader();
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ await writer.write({ type: 'write', data: value, position });
+ position += value.byteLength;
+ }
+ await writer.close();
+ } catch (error) {
+ await writer.abort().catch(() => {});
+ throw error;
+ }
+}
+
+/** The `ProjectFS` of the project folder at `dir`. */
+export function createProjectFS(dir: string): ProjectFS {
+ const separator = dir.includes('\\') ? '\\' : '/';
+ const absolute = (source: string): string =>
+ isAbsoluteSource(source) ? source : `${dir}${separator}${source.split('/').join(separator)}`;
+
+ return {
+ absolute,
+ readManifest: () => mainBridge.call(MAIN_CHANNELS.PROJECTS_MANIFEST_READ, { dir }),
+ writeManifest: (manifest: Manifest) => mainBridge.call(MAIN_CHANNELS.PROJECTS_MANIFEST_WRITE, { dir, manifest }),
+ list: (source) => mainBridge.call(MAIN_CHANNELS.PROJECTS_FS_LIST, { dir, source }),
+ stat: (source) => mainBridge.call(MAIN_CHANNELS.PROJECTS_FS_STAT, { dir, source }),
+ file: (source) => new ElectronFileHandle(absolute(source)).getFile(),
+ write: (path, blob) => writeBlob(absolute(path), blob),
+ remove: (path) => mainBridge.call(MAIN_CHANNELS.PROJECTS_FS_REMOVE, { dir, path }),
+ pathOf: (file) => window.desktop?.getPathForFile(file) || null,
+ };
+}
diff --git a/apps/web/src/utils/common.ts b/apps/web/src/utils/common.ts
index 9f2cfb93..c26b5aba 100644
--- a/apps/web/src/utils/common.ts
+++ b/apps/web/src/utils/common.ts
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import type { Asset } from "@/components/engine/db";
+import type { Asset } from "@diffusionstudio/assets";
/**
* clip assert replacement for the browser
@@ -75,12 +75,12 @@ export class AsyncMutex {
* @param asset - The asset to format.
* @returns The formatted duration string, or null if the asset is not a video.
*/
-export function formatAssetDuration(asset: Asset | null | undefined): string | null {
+export function formatAssetDuration(asset: Pick<Asset, 'type'> & { duration?: number } | null | undefined): string | null {
if (asset?.type != "VIDEO" && asset?.type != "AUDIO" && asset?.type != "SEQUENCE") {
return null;
}
- const seconds = asset.duration;
+ const seconds = asset.duration ?? NaN;
if (!Number.isFinite(seconds)) return "0:00";
const totalSeconds = Math.max(0, Math.floor(seconds));
diff --git a/apps/web/src/utils/jsx-generation.ts b/apps/web/src/utils/jsx-generation.ts
index afceb8a3..1bc56f99 100644
--- a/apps/web/src/utils/jsx-generation.ts
+++ b/apps/web/src/utils/jsx-generation.ts
@@ -2,47 +2,44 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+// `generate.*` declarations in a project's JSX, made real: the asset
+// library's generator (see `attachLibrary`). A declaration resolves to an
+// asset by content — the same spec is the same asset, in this session and
+// the next, since the finished file lands in the library under
+// `generated/` with the spec's key in the manifest.
+
import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";
-import { loadAsset, uploadAsset } from "@/components/engine";
+import { getAssetFile } from "@diffusionstudio/runtime";
+import { GENERATED_DIR } from "@diffusionstudio/assets";
import {
PROMPT_INPUT_AUDIO_MODEL_OPTIONS,
PROMPT_INPUT_IMAGE_MODEL_OPTIONS,
PROMPT_INPUT_VIDEO_MODEL_OPTIONS,
PROMPT_INPUT_VOICE_OPTIONS,
} from "@/components/genai/config";
-import { ensureGenerationFolder } from "@/utils/genai";
import { assert } from "@/utils";
-import { ElectronFileHandle } from "@/lib/electron-file-handle";
+import { uploadBlob } from "@/components/engine";
import { trpc } from "@/lib/trpc";
import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";
-import type { EngineWorld } from "@/components/engine";
-import type { Asset } from "@/components/engine/db";
+import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
-const URL_RE = /^https?:\/\//;
+const memo: GenerationMemo = new Map();
+const inflight: Inflight = new Map();
/**
- * Resolves a plain `src` — a remote URL, a global filesystem path, or an existing asset id.
+ * A library's `generate` option for the project `projectId` (which prefixes
+ * upload keys). Identical concurrent declarations collapse to one request.
*/
-export async function resolveAsset(world: EngineWorld, src: string): Promise<Asset> {
- const value = src.trim();
-
- if (URL_RE.test(value)) {
- return await loadAsset(world, value);
- }
-
- // A global filesystem path (absolute or containing a separator).
- if (value.startsWith("/") || value.includes("/") || value.includes("\\")) {
- return await loadAsset(world, new ElectronFileHandle(value));
- }
-
- // Otherwise treat it as an existing asset id.
- const existing = world.assets.get(value);
- assert(existing, `Could not resolve media src: ${src}`);
+export function generateAsset(ref: object, library: AssetLibrary, projectId: string): Promise<Asset> {
+ assert(isAssetRef(ref), "Not a generate.* declaration");
- return existing;
+ return resolveGeneratedAsset({ library, projectId, memo, inflight }, ref);
}
+/** What one generation run has to hand: the library it fills and its caches. */
+type Generation = { library: AssetLibrary; projectId: string; memo: GenerationMemo; inflight: Inflight };
+
// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).
const VOICE_MODEL = "elevenlabs-v3";
@@ -57,61 +54,52 @@ type ResolvedSpec =
| { type: "audio"; model: string; prompt: string; duration?: number };
/**
- * In-flight generations per world, keyed by `generationKey`, so identical
- * concurrent declarations collapse to one request. Entries are removed once
- * settled; finished generations are found via `asset.generationKey`.
+ * Declarations already resolved, keyed by ref identity — a ref consumed by
+ * several elements resolves (and validates) once. A failed one is forgotten,
+ * so a remount tries again.
*/
-const inflight = new WeakMap<EngineWorld, Map<string, Promise<Asset>>>();
+type GenerationMemo = Map<AssetRef, Promise<Asset>>;
-/**
- * Declarations already resolved within one `generateAssets` run, keyed by ref
- * identity — a ref consumed by several elements resolves (and validates) once.
- */
-export type GenerationMemo = Map<AssetRef, Promise<Asset>>;
+/** In-flight generations keyed by `generationKey`. */
+type Inflight = Map<string, Promise<Asset>>;
-/** Resolves a `generate.*` declaration to a finished, imported asset. */
-export function resolveGeneratedAsset(world: EngineWorld, ref: AssetRef, memo: GenerationMemo): Promise<Asset> {
- let promise = memo.get(ref);
+function resolveGeneratedAsset(gen: Generation, ref: AssetRef): Promise<Asset> {
+ let promise = gen.memo.get(ref);
if (!promise) {
- promise = generateFromRef(world, ref, memo);
- memo.set(ref, promise);
+ promise = generateFromRef(gen, ref);
+ promise.catch(() => gen.memo.delete(ref));
+ gen.memo.set(ref, promise);
}
return promise;
}
-function resolveInput(world: EngineWorld, input: AssetInput, memo: GenerationMemo): Promise<Asset> {
- return isAssetRef(input) ? resolveGeneratedAsset(world, input, memo) : resolveAsset(world, input);
+function resolveInput(gen: Generation, input: AssetInput): Promise<Asset> {
+ return isAssetRef(input) ? resolveGeneratedAsset(gen, input) : gen.library.resolve(input);
}
-async function generateFromRef(world: EngineWorld, ref: AssetRef, memo: GenerationMemo): Promise<Asset> {
- const resolved = await resolveSpec(world, getAssetSpec(ref), memo);
+async function generateFromRef(gen: Generation, ref: AssetRef): Promise<Asset> {
+ const resolved = await resolveSpec(gen, getAssetSpec(ref));
const generationKey = JSON.stringify(resolved);
- const cached = Array.from(world.assets.values()).find((asset) => asset.generationKey === generationKey);
+ const cached = gen.library.list().find((asset) => asset.generation?.key === generationKey);
if (cached) return cached;
- let pending = inflight.get(world);
- if (!pending) {
- pending = new Map();
- inflight.set(world, pending);
- }
-
- const running = pending.get(generationKey);
+ const running = gen.inflight.get(generationKey);
if (running) return await running;
- const promise = runGeneration(world, resolved, generationKey);
- pending.set(generationKey, promise);
+ const promise = runGeneration(gen, resolved, generationKey);
+ gen.inflight.set(generationKey, promise);
try {
return await promise;
} finally {
- pending.delete(generationKey);
+ gen.inflight.delete(generationKey);
}
}
-async function resolveSpec(world: EngineWorld, spec: AssetSpecInput, memo: GenerationMemo): Promise<ResolvedSpec> {
+async function resolveSpec(gen: Generation, spec: AssetSpecInput): Promise<ResolvedSpec> {
switch (spec.type) {
case "image": {
- const refs = await Promise.all((spec.refs ?? []).map((ref) => resolveInput(world, ref, memo)));
+ const refs = await Promise.all((spec.refs ?? []).map((ref) => resolveInput(gen, ref)));
return {
type: "image",
model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,
@@ -123,8 +111,8 @@ async function resolveSpec(world: EngineWorld, spec: AssetSpecInput, memo: Gener
}
case "video": {
const [startFrame, endFrame] = await Promise.all([
- spec.startFrame !== undefined ? resolveInput(world, spec.startFrame, memo) : undefined,
- spec.endFrame !== undefined ? resolveInput(world, spec.endFrame, memo) : undefined,
+ spec.startFrame !== undefined ? resolveInput(gen, spec.startFrame) : undefined,
+ spec.endFrame !== undefined ? resolveInput(gen, spec.endFrame) : undefined,
]);
const resolved = {
type: "video",
@@ -172,20 +160,27 @@ function checkVideoConstraints(spec: Extract<ResolvedSpec, { type: "video" }>):
assert(spec.endFrameId === undefined || model.features.includes("end-frame"), `${spec.model} does not support an end frame`);
}
-async function runGeneration(world: EngineWorld, spec: ResolvedSpec, generationKey: string): Promise<Asset> {
- const { name, results, generationId } = await requestGeneration(world, spec);
+/** Runs a generation and stores its first result under `generated/`. */
+async function runGeneration(gen: Generation, spec: ResolvedSpec, generationKey: string): Promise<Asset> {
+ const { name, results, generationId } = await requestGeneration(gen, spec);
assert(results.length > 0, "No results returned from the model");
- const folderId = await ensureGenerationFolder(world, name, 1);
- return await loadAsset(world, results[0].url, { name, generationId, generationKey, folderId });
+ const url = results[0].url;
+ const response = await fetch(url);
+ assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);
+ const blob = await response.blob();
+ const extension = url.split(/[?#]/)[0].split(".").pop();
+ const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;
+
+ return gen.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });
}
-function requestGeneration(world: EngineWorld, spec: ResolvedSpec) {
+function requestGeneration(gen: Generation, spec: ResolvedSpec) {
switch (spec.type) {
case "image": {
return (async () => {
const images = spec.refIds.length > 0
- ? await Promise.all(spec.refIds.map((id) => uploadInput(world, id)))
+ ? await Promise.all(spec.refIds.map((id) => uploadInput(gen, id)))
: undefined;
return await trpc.generateImage.mutate({
@@ -201,8 +196,8 @@ function requestGeneration(world: EngineWorld, spec: ResolvedSpec) {
case "video": {
return (async () => {
const [startFrame, endFrame] = await Promise.all([
- spec.startFrameId ? uploadInput(world, spec.startFrameId) : undefined,
- spec.endFrameId ? uploadInput(world, spec.endFrameId) : undefined,
+ spec.startFrameId ? uploadInput(gen, spec.startFrameId) : undefined,
+ spec.endFrameId ? uploadInput(gen, spec.endFrameId) : undefined,
]);
return await trpc.generateVideo.mutate({
@@ -234,8 +229,11 @@ function requestGeneration(world: EngineWorld, spec: ResolvedSpec) {
}
}
-async function uploadInput(world: EngineWorld, assetId: string) {
- const uploaded = await uploadAsset(world, assetId);
+/** Uploads a referenced asset for the model to read; the bucket key is project-unique. */
+async function uploadInput(gen: Generation, assetId: string) {
+ const asset = gen.library.get(assetId);
+ assert(asset, `Referenced asset ${assetId} not found`);
+ const uploaded = await uploadBlob(await getAssetFile(asset), `${gen.projectId}-${assetId}`);
assert(uploaded, `Failed to upload referenced asset ${assetId}`);
return uploaded;
}
diff --git a/package-lock.json b/package-lock.json
index 3aa93217..eed01533 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -54,7 +54,8 @@
"esbuild": "^0.28.1",
"sqids": "^0.3.0",
"ts-morph": "^28.0.0",
- "update-electron-app": "^3.0.0"
+ "update-electron-app": "^3.0.0",
+ "yaml": "^2.9.0"
},
"devDependencies": {
"@electron-forge/cli": "^7.11.1",
@@ -596,6 +597,10 @@
"zod": "^4.3.6"
}
},
+ "node_modules/@diffusionstudio/assets": {
+ "resolved": "packages/assets",
+ "link": true
+ },
"node_modules/@diffusionstudio/cli": {
"resolved": "apps/cli",
"link": true
@@ -12266,7 +12271,6 @@
"version": "2.9.0",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz",
"integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==",
- "dev": true,
"license": "ISC",
"bin": {
"yaml": "bin.mjs"
@@ -12385,6 +12389,18 @@
"url": "https://github.com/sponsors/colinhacks"
}
},
+ "packages/assets": {
+ "name": "@diffusionstudio/assets",
+ "version": "0.1.0",
+ "license": "MPL-2.0",
+ "dependencies": {
+ "mediabunny": "^1.32.2",
+ "solid-js": "^1.9.0"
+ },
+ "devDependencies": {
+ "typescript": "~5.9.3"
+ }
+ },
"packages/encoder": {
"name": "@diffusionstudio/encoder",
"version": "0.1.0",
@@ -12447,11 +12463,11 @@
"version": "0.1.0",
"license": "MPL-2.0",
"dependencies": {
+ "@diffusionstudio/assets": "*",
"animejs": "^4.4.1",
"colord": "^2.9.3",
"koota": "^0.6.6",
- "mediabunny": "^1.32.2",
- "sqids": "^0.3.0"
+ "mediabunny": "^1.32.2"
},
"devDependencies": {
"@webgpu/types": "^0.1.71",
diff --git a/packages/assets/package.json b/packages/assets/package.json
new file mode 100644
index 00000000..4a49acd1
--- /dev/null
+++ b/packages/assets/package.json
@@ -0,0 +1,26 @@
+{
+ "name": "@diffusionstudio/assets",
+ "version": "0.1.0",
+ "description": "A project's asset library: the assets.yml manifest, content hashing, media probing, and the AssetLibrary that links, stores, resolves and organizes assets over a host-provided ProjectFS. World-free: reactive state is solid-js signals; no koota, no DOM beyond media decoding.",
+ "license": "MPL-2.0",
+ "type": "module",
+ "sideEffects": false,
+ "exports": {
+ ".": "./src/index.ts"
+ },
+ "scripts": {
+ "check": "tsc --noEmit"
+ },
+ "keywords": [
+ "assets",
+ "manifest",
+ "media"
+ ],
+ "dependencies": {
+ "mediabunny": "^1.32.2",
+ "solid-js": "^1.9.0"
+ },
+ "devDependencies": {
+ "typescript": "~5.9.3"
+ }
+}
diff --git a/packages/assets/src/browser.ts b/packages/assets/src/browser.ts
new file mode 100644
index 00000000..e0e61db4
--- /dev/null
+++ b/packages/assets/src/browser.ts
@@ -0,0 +1,85 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The browser side of getting files into and out of a library: pickers,
+// drops, and the save dialog. Nothing here talks to the user; a failure is
+// thrown or returned for the host to report.
+
+import { assetName } from './types';
+
+import type { AssetLibrary, ImportResult } from './library';
+import type { Asset } from './types';
+
+/** Opens the file picker; resolves to what was picked ([] on cancel). */
+export function pickFiles(options: { multiple?: boolean; accept?: string } = {}): Promise<File[]> {
+ return new Promise((resolve) => {
+ const input = document.createElement('input');
+ input.type = 'file';
+ input.multiple = options.multiple ?? true;
+ input.accept = options.accept ?? 'image/*,video/*,audio/*,text/*,application/json';
+ input.onchange = () => resolve(Array.from(input.files ?? []));
+ input.oncancel = () => resolve([]);
+ input.click();
+ });
+}
+
+/** The files of a drop, when it carried any from outside the page. */
+export function droppedFiles(event: DragEvent): File[] {
+ return Array.from(event.dataTransfer?.items ?? [])
+ .filter((item) => item.kind === 'file')
+ .map((item) => item.getAsFile())
+ .filter((file): file is File => file !== null);
+}
+
+export interface ImportReport extends ImportResult {
+ /** Files the host could not name by path, so nothing could be linked. */
+ unnamed: File[];
+}
+
+/**
+ * Links files (or frame folders) into the library at `folder`, leaving them
+ * where they are. Every file is tried; the report says what happened to each.
+ */
+export async function importFiles(library: AssetLibrary, files: ReadonlyArray<File>, folder: string): Promise<ImportReport> {
+ const unnamed: File[] = [];
+ const paths: string[] = [];
+ for (const file of files) {
+ const path = library.fs.pathOf?.(file) ?? null;
+ if (path) paths.push(path);
+ else unnamed.push(file);
+ }
+ return { ...(await library.import(paths, { folder })), unnamed };
+}
+
+/**
+ * Asks where to save a copy of an asset's file and writes it there. Resolves
+ * to false when the dialog was cancelled; throws when the write failed.
+ */
+export async function saveAssetAs(asset: Pick<Asset, 'handle' | 'mimeType' | 'path'>): Promise<boolean> {
+ const suggestedName = assetName(asset);
+ const extension = suggestedName.match(/\.[^.]+$/)?.[0];
+
+ let target: FileSystemFileHandle;
+ try {
+ target = await window.showSaveFilePicker({
+ suggestedName,
+ ...(extension && asset.mimeType.includes('/')
+ ? { types: [{ description: asset.mimeType, accept: { [asset.mimeType]: [extension] } as Record<`${string}/${string}`, `.${string}`[]> }] }
+ : {}),
+ });
+ } catch (error) {
+ if ((error as Error).name === 'AbortError') return false;
+ throw error;
+ }
+
+ const file = await asset.handle.getFile();
+ const writable = await target.createWritable();
+ try {
+ await file.stream().pipeTo(writable);
+ } catch (error) {
+ await writable.abort().catch(() => {});
+ throw error;
+ }
+ return true;
+}
diff --git a/packages/assets/src/fs.ts b/packages/assets/src/fs.ts
new file mode 100644
index 00000000..444b693c
--- /dev/null
+++ b/packages/assets/src/fs.ts
@@ -0,0 +1,44 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The file system a project lives in, as far as the asset library needs it.
+// The desktop implements this over IPC to the main process (real files, the
+// manifest as YAML); a browser host would implement it over a
+// FileSystemDirectoryHandle. The library never joins paths itself: a `path`
+// is project-relative, a `source` is either that or an absolute OS path, and
+// the host knows which is which.
+
+import type { Manifest } from './manifest';
+import type { AssetStat } from './types';
+
+export interface FsEntry {
+ name: string;
+ kind: 'file' | 'directory';
+ size: number;
+ /** Modification time in ms since the epoch. */
+ mtime: number;
+}
+
+export interface ProjectFS {
+ /** The manifest as plain data, or null when the project has none yet. */
+ readManifest(): Promise<unknown>;
+ writeManifest(manifest: Manifest): Promise<void>;
+ /** Entries of a directory (project-relative or absolute); [] when missing. */
+ list(source: string): Promise<FsEntry[]>;
+ /** Size and mtime of a file, or null when it does not exist. */
+ stat(source: string): Promise<AssetStat | null>;
+ /** The bytes of a file (project-relative or absolute). */
+ file(source: string): Promise<File>;
+ /** Writes a file inside the project, creating parent directories. */
+ write(path: string, data: Blob): Promise<void>;
+ /** Removes a file or directory inside the project; missing is fine. */
+ remove(path: string): Promise<void>;
+ /** The absolute location of a project-relative path, when the host has one. */
+ absolute?(path: string): string;
+ /**
+ * The path of a File the user picked or dropped, when the host can tell
+ * (Electron can; a browser cannot). What `import` links to.
+ */
+ pathOf?(file: File): string | null;
+}
diff --git a/packages/assets/src/hash.ts b/packages/assets/src/hash.ts
new file mode 100644
index 00000000..13b05267
--- /dev/null
+++ b/packages/assets/src/hash.ts
@@ -0,0 +1,51 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Content identity for assets. Footage runs to gigabytes, and hashing all of
+// it on every import would make dropping a clip into the project take
+// seconds; the id instead is a SHA-256 over a sample — the size and the head,
+// middle and tail of the bytes — which is stable for the same file, tells
+// apart anything a user is likely to have two of, and costs milliseconds.
+// Truncated to 16 hex characters: plenty for one project's library, short
+// enough to read in a manifest.
+
+/** Bytes hashed from each of the head, middle and tail of a file. */
+const SAMPLE_BYTES = 1024 * 1024;
+
+export const ID_LENGTH = 16;
+
+/** The content id of `blob`. */
+export async function hashBlob(blob: Blob): Promise<string> {
+ const size = blob.size;
+ const parts: Blob[] = [new Blob([String(size)])];
+
+ if (size <= SAMPLE_BYTES * 3) {
+ parts.push(blob);
+ } else {
+ const middle = Math.floor(size / 2 - SAMPLE_BYTES / 2);
+ parts.push(
+ blob.slice(0, SAMPLE_BYTES),
+ blob.slice(middle, middle + SAMPLE_BYTES),
+ blob.slice(size - SAMPLE_BYTES),
+ );
+ }
+
+ const bytes = await new Blob(parts).arrayBuffer();
+ return hexOf(await crypto.subtle.digest('SHA-256', bytes)).slice(0, ID_LENGTH);
+}
+
+/** The id of a sequence: the hash of its frame names and sizes, not their bytes. */
+export async function hashSequence(frames: { name: string; size: number }[]): Promise<string> {
+ const text = frames.map((frame) => `${frame.name}:${frame.size}`).join('\n');
+ const bytes = new TextEncoder().encode(`seq\n${text}`);
+ return hexOf(await crypto.subtle.digest('SHA-256', bytes)).slice(0, ID_LENGTH);
+}
+
+/** Whether a string could be an asset id (and not, say, a library path). */
+export const looksLikeId = (value: string): boolean =>
+ value.length === ID_LENGTH && /^[0-9a-f]+$/.test(value);
+
+function hexOf(buffer: ArrayBuffer): string {
+ return Array.from(new Uint8Array(buffer), (byte) => byte.toString(16).padStart(2, '0')).join('');
+}
diff --git a/packages/assets/src/index.ts b/packages/assets/src/index.ts
new file mode 100644
index 00000000..641972bc
--- /dev/null
+++ b/packages/assets/src/index.ts
@@ -0,0 +1,11 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export * from './types';
+export * from './manifest';
+export * from './fs';
+export * from './hash';
+export * from './probe';
+export * from './library';
+export * from './browser';
diff --git a/packages/assets/src/library.ts b/packages/assets/src/library.ts
new file mode 100644
index 00000000..e3fc12ac
--- /dev/null
+++ b/packages/assets/src/library.ts
@@ -0,0 +1,714 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The asset library of a project: the manifest, loaded with handles attached,
+// plus everything that changes it — imports, renames, folders, generated
+// output — and the one place a `src` is turned into an asset. Files the user
+// imports stay where they are (the manifest links to them by absolute path);
+// only bytes the app itself produces are written into the project's
+// `assets/` directory, and anything found there that the manifest does not
+// know is taken into the library.
+//
+// Knows nothing of the host: files come and go through a `ProjectFS`, and
+// what it cannot do itself (generate an asset, follow a rename into whatever
+// names assets by path, rebind whatever is bound to a relinked asset's old
+// id) it asks of `LibraryOptions`. Its state is solid signals (`assets()`,
+// `folders()`), so readers inside a tracking scope follow it.
+
+import { createSignal, type Accessor } from 'solid-js';
+
+import { hashBlob, hashSequence } from './hash';
+import {
+ ASSETS_DIR, isAbsoluteSource, isProjectSource, isUrlSource, normalizeManifest, toRecord,
+} from './manifest';
+import { detectMimeType, DEFAULT_SEQUENCE_FPS, isSequenceListing, probeMedia, sortFrames } from './probe';
+import { assetFolder, assetName, basename, dirname, joinPath, normalizePath } from './types';
+
+import type { FsEntry, ProjectFS } from './fs';
+import type { AssetRecord, Manifest } from './manifest';
+import type { Asset, AssetDirectoryHandle, AssetFileHandle, AssetGeneration, SequenceAsset } from './types';
+
+/** How long changes pile up before the manifest is written. */
+const SAVE_DEBOUNCE = 200;
+
+export interface LibraryOptions {
+ /**
+ * Turns what the runtime cannot — a `generate.*` declaration — into an
+ * asset of this library; `resolve` hands it any non-string source. Without
+ * one, such sources fail to resolve.
+ */
+ generate?: AssetGenerator;
+ /**
+ * Called when an asset's library path changes (renamed or moved), so the
+ * host can follow in whatever names assets by path — the JSX `src` props.
+ */
+ onRename?: (asset: Asset, from: string) => void;
+ /**
+ * Called when an asset is pointed at other bytes and so has a new id, so
+ * the host can rebind whatever held the old one (`from`).
+ */
+ onRelink?: (asset: Asset, from: string) => void;
+}
+
+export interface ImportResult {
+ /** What was taken in, in the order it finished. */
+ assets: Asset[];
+ /** Sources the library refused, with why. */
+ failed: { source: string; error: Error }[];
+}
+
+export interface ImportOptions {
+ /** Library folder to place the asset in; root by default. */
+ folder?: string;
+ /** Library name; the source's file name by default. */
+ name?: string;
+ generation?: AssetGeneration;
+}
+
+/** See `LibraryOptions.generate`. */
+export type AssetGenerator = (ref: object, library: AssetLibrary) => Promise<Asset>;
+
+export class AssetLibrary {
+ public readonly fs: ProjectFS;
+ /** Library assets, newest first; transient ones excluded. Reactive. */
+ public readonly assets: Accessor<Asset[]>;
+ /** Every folder: declared ones, those implied by asset paths, and their ancestors. Reactive. */
+ public readonly folders: Accessor<ReadonlySet<string>>;
+
+ private readonly setAssets: (assets: Asset[]) => void;
+ private readonly setFolders: (folders: ReadonlySet<string>) => void;
+ /** Folders declared in the manifest (the empty ones need declaring). */
+ private declared = new Set<string>();
+ /** Every asset by id: the library's, and transient ones resolved from outside it. */
+ private readonly map = new Map<string, Asset>();
+ private readonly generator: LibraryOptions['generate'];
+ private readonly onRename: LibraryOptions['onRename'];
+ private readonly onRelink: LibraryOptions['onRelink'];
+ private inflight = new Map<string, Promise<Asset>>();
+ private saveTimer: ReturnType<typeof setTimeout> | undefined;
+ private saving: Promise<void> = Promise.resolve();
+ private dirty = false;
+ private disposed = false;
+
+ public constructor(fs: ProjectFS, options: LibraryOptions = {}) {
+ this.fs = fs;
+ this.generator = options.generate;
+ this.onRename = options.onRename;
+ this.onRelink = options.onRelink;
+ [this.assets, this.setAssets] = createSignal<Asset[]>([]);
+ [this.folders, this.setFolders] = createSignal<ReadonlySet<string>>(new Set());
+ }
+
+ // -----------------------------------------------------------------------
+ // Reading
+
+ /** Library assets, newest first; transient ones excluded. Same as `assets()`. */
+ public list(): Asset[] {
+ return this.assets();
+ }
+
+ /** The library's assets as of now, straight from the map. */
+ private listNow(): Asset[] {
+ return Array.from(this.map.values()).filter((asset) => !asset.transient);
+ }
+
+ /** The asset with `id`, or at library path `path`; undefined otherwise. */
+ public get(pathOrId: string): Asset | undefined {
+ const byId = this.map.get(pathOrId);
+ if (byId) return byId;
+ const path = normalizePath(pathOrId);
+ for (const asset of this.map.values()) {
+ if (asset.path === path && !asset.transient) return asset;
+ }
+ return undefined;
+ }
+
+ /** The asset whose bytes are `source`, if the library links to it. */
+ public bySource(source: string): Asset | undefined {
+ for (const asset of this.map.values()) {
+ if (asset.source === source && !asset.transient) return asset;
+ }
+ return undefined;
+ }
+
+ /** Direct children of a folder ('' for the root): its subfolders and assets. Reactive. */
+ public childrenOf(folder: string): { folders: string[]; assets: Asset[] } {
+ const prefix = folder ? `${folder}/` : '';
+ const folders = new Set<string>();
+ for (const path of this.folders()) {
+ if (path.startsWith(prefix) && path !== folder && !path.slice(prefix.length).includes('/')) {
+ folders.add(path);
+ }
+ }
+ const assets = this.assets().filter((asset) => assetFolder(asset) === folder);
+ return { folders: [...folders].sort(), assets };
+ }
+
+ /** Every folder as of now: declared, implied by asset paths, and all their ancestors. */
+ private foldersNow(): Set<string> {
+ const folders = new Set<string>();
+ const declare = (path: string): void => {
+ for (let folder = path; folder && !folders.has(folder); folder = dirname(folder)) folders.add(folder);
+ };
+ for (const folder of this.declared) declare(folder);
+ for (const asset of this.listNow()) declare(assetFolder(asset));
+ return folders;
+ }
+
+ /** Publishes the map's state to the signals. */
+ private publish(): void {
+ this.setAssets(this.listNow());
+ this.setFolders(this.foldersNow());
+ }
+
+ // -----------------------------------------------------------------------
+ // Loading
+
+ /**
+ * Reads the manifest in, then takes in whatever the project's
+ * `assets/` directory holds that the manifest does not know. Assets whose
+ * source changed since they were recorded (by size or mtime) are probed
+ * again. Safe to call again: it reconciles rather than replaces.
+ */
+ public async load(): Promise<void> {
+ const manifest = normalizeManifest(await this.fs.readManifest());
+ this.declared = new Set(manifest.folders);
+
+ const next = new Map<string, Asset>();
+ await Promise.all(manifest.assets.map(async (record) => {
+ const asset = await this.revive(record).catch((error: unknown) => {
+ console.warn(`[assets] could not load ${record.path}:`, error);
+ return this.attach(record);
+ });
+ next.set(asset.id, asset);
+ }));
+
+ // Keep transient assets and the same-id instances entities already hold.
+ for (const [id, asset] of this.map) {
+ if (asset.transient && !next.has(id)) next.set(id, asset);
+ }
+ this.map.clear();
+ for (const [id, asset] of next) this.map.set(id, asset);
+
+ this.publish();
+ await this.scanAssetsDir();
+ this.publish();
+ }
+
+ /** Attaches handles to a record; re-examines it when its source changed. */
+ private async revive(record: AssetRecord): Promise<Asset> {
+ if (isUrlSource(record.source)) return this.attach(record);
+
+ if (record.type === 'SEQUENCE') {
+ const frames = sortFrames(await this.fs.list(record.source)).filter((entry) => entry.kind === 'file');
+ const id = await hashSequence(frames);
+ if (id === record.id) return this.attach(record);
+ return this.describeSequence(record.source, frames, { path: record.path, createdAt: record.createdAt });
+ }
+
+ const stat = await this.fs.stat(record.source);
+ if (!stat || (record.stat && stat.size === record.stat.size && stat.mtime === record.stat.mtime)) {
+ return this.attach(record);
+ }
+ return this.describeFile(record.source, {
+ path: record.path,
+ createdAt: record.createdAt,
+ generation: record.generation,
+ });
+ }
+
+ /** Registers files under `assets/` that the manifest does not link to. */
+ private async scanAssetsDir(): Promise<void> {
+ const known = new Set(this.listNow().map((asset) => asset.source));
+ const walk = async (dir: string): Promise<void> => {
+ const entries = await this.fs.list(dir);
+ if (dir !== ASSETS_DIR && isSequenceListing(entries.map((entry) => entry.name))) {
+ if (!known.has(dir)) await this.link(dir, { folder: dirname(dir.slice(ASSETS_DIR.length + 1)) }).catch(() => {});
+ return;
+ }
+ for (const entry of entries) {
+ if (entry.name.startsWith('.')) continue;
+ const source = joinPath(dir, entry.name);
+ if (entry.kind === 'directory') {
+ await walk(source);
+ } else if (!known.has(source)) {
+ await this.link(source, { folder: dirname(source.slice(ASSETS_DIR.length + 1)) }).catch(() => {});
+ }
+ }
+ };
+ await walk(ASSETS_DIR);
+ }
+
+ // -----------------------------------------------------------------------
+ // Resolving
+
+ /**
+ * The asset a `src` names: a library path or id, an absolute path or URL
+ * (resolved on the fly, kept in memory only), or a `generate.*` ref (left
+ * to the host's generator).
+ */
+ public async resolve(input: unknown): Promise<Asset> {
+ if (typeof input !== 'string') {
+ if (!this.generator) throw new Error('This host cannot generate assets');
+ return this.generator(input as object, this);
+ }
+
+ const value = input.trim();
+ const known = this.get(value);
+ if (known) return known;
+
+ if (isUrlSource(value) || isAbsoluteSource(value)) {
+ return this.transient(value);
+ }
+
+ // A project-relative path that exists is a file of the project.
+ if (value.includes('/') && (await this.fs.stat(value))) {
+ return this.transient(value);
+ }
+
+ throw new Error(`Could not resolve media src: ${input}`);
+ }
+
+ /** Resolves a source outside the library without adding it to the manifest. */
+ private transient(source: string): Promise<Asset> {
+ const existing = Array.from(this.map.values()).find((asset) => asset.transient && asset.source === source);
+ if (existing) return Promise.resolve(existing);
+
+ return this.once(`transient:${source}`, async () => {
+ const asset = isUrlSource(source)
+ ? await this.describeUrl(source, { path: basename(source.split(/[?#]/)[0]!) })
+ : await this.describeSource(source, { path: basename(source) });
+ asset.transient = true;
+ // A transient asset that turns out to be one the library has is that one.
+ const owned = this.map.get(asset.id);
+ if (owned && !owned.transient) return owned;
+ this.map.set(asset.id, asset);
+ return asset;
+ });
+ }
+
+ /** The File backing an asset. */
+ public file(asset: Asset): Promise<File> {
+ return asset.handle.getFile();
+ }
+
+ // -----------------------------------------------------------------------
+ // Importing
+
+ /**
+ * Imports sources into the library, all at once, leaving their bytes where
+ * they are: a file (absolute path, or a path inside the project) is linked,
+ * a frames directory becomes one sequence asset, any other directory
+ * becomes a folder with its files linked inside (recursively), and a URL
+ * is downloaded into `assets/`. The same bytes imported twice are one
+ * asset. Every source is tried; the result says which were refused and why.
+ */
+ public async import(sources: readonly string[], options: ImportOptions = {}): Promise<ImportResult> {
+ const result: ImportResult = { assets: [], failed: [] };
+ await Promise.all(sources.map(async (source) => {
+ try {
+ result.assets.push(...await this.importOne(source, options));
+ } catch (error) {
+ result.failed.push({ source, error: error instanceof Error ? error : new Error(String(error)) });
+ }
+ }));
+ return result;
+ }
+
+ private async importOne(source: string, options: ImportOptions): Promise<Asset[]> {
+ if (isUrlSource(source)) {
+ const response = await fetch(source);
+ if (!response.ok) throw new Error(`Failed to fetch ${source}: ${response.status}`);
+ const blob = await response.blob();
+ const name = options.name ?? basename(source.split(/[?#]/)[0]!) ?? 'download';
+ return [await this.store(blob, { ...options, name })];
+ }
+
+ const entries = await this.fs.list(source);
+ if (!entries.length || isSequenceListing(entries.map((entry) => entry.name))) {
+ return [await this.link(source, options)];
+ }
+
+ // A folder of files: a folder here, with each of them in it.
+ const folder = this.createFolder(joinPath(options.folder ?? '', options.name ?? basename(source)));
+ const children = entries.filter((entry) => !entry.name.startsWith('.')).map((entry) => joinPath(source, entry.name));
+ return (await this.import(children, { folder })).assets;
+ }
+
+ /** Links a file at `source` (absolute, or project-relative) into the library. */
+ private link(source: string, options: ImportOptions): Promise<Asset> {
+ const existing = this.bySource(source);
+ if (existing) return Promise.resolve(existing);
+
+ return this.once(`link:${source}`, async () => {
+ const name = options.name ?? basename(source);
+ const path = this.uniquePath(joinPath(options.folder ?? '', name));
+ const asset = await this.describeSource(source, { path, generation: options.generation });
+ return this.add(asset);
+ });
+ }
+
+ /**
+ * Writes bytes the app produced into `assets/<folder>/<name>` and takes
+ * the file into the library at the same library path.
+ */
+ public async store(blob: Blob, options: ImportOptions & { name: string }): Promise<Asset> {
+ const path = this.uniquePath(joinPath(options.folder ?? '', options.name));
+ const source = joinPath(ASSETS_DIR, path);
+ await this.fs.write(source, blob);
+ const asset = await this.describeFile(source, { path, generation: options.generation });
+ return this.add(asset);
+ }
+
+ /** Puts an asset into the library, deduplicating by content. */
+ private add(asset: Asset): Asset {
+ const existing = this.map.get(asset.id);
+ if (existing && !existing.transient) return existing;
+ // A transient asset the library now takes in becomes a real one; the
+ // instance entities hold stays valid, so mutate rather than replace.
+ if (existing) {
+ Object.assign(existing, asset);
+ delete existing.transient;
+ this.reorder(existing);
+ this.changed();
+ return existing;
+ }
+ this.reorder(asset);
+ this.changed();
+ return asset;
+ }
+
+ /** Puts `asset` at the front of the map (newest first). */
+ private reorder(asset: Asset): void {
+ const rest = Array.from(this.map.values()).filter((other) => other.id !== asset.id);
+ this.map.clear();
+ this.map.set(asset.id, asset);
+ for (const other of rest) this.map.set(other.id, other);
+ }
+
+ // -----------------------------------------------------------------------
+ // Editing
+
+ /** Replaces fields of an asset (a transcript, a corrected frame rate). Keeps identity. */
+ public update<T extends Asset>(asset: T, patch: Partial<Omit<T, 'id' | 'handle' | 'type'>>): T {
+ Object.assign(asset, patch);
+ this.changed();
+ return asset;
+ }
+
+ /**
+ * Points an asset at other bytes: same library path (so the JSX is
+ * untouched), new source, new content id. Whatever is bound to the old id
+ * is the host's to move over (`onRelink`).
+ */
+ public async relink(asset: Asset, source: string): Promise<Asset> {
+ const next = await this.describeSource(source, {
+ path: asset.path,
+ createdAt: asset.createdAt,
+ generation: asset.generation,
+ });
+ const from = asset.id;
+ if (next.id === from) {
+ Object.assign(asset, next);
+ this.changed();
+ return asset;
+ }
+ this.map.delete(from);
+ this.reorder(next);
+ this.changed();
+ this.onRelink?.(next, from);
+ return next;
+ }
+
+ /** Renames an asset within its folder; the source file is untouched. */
+ public rename(asset: Asset, name: string): void {
+ const clean = normalizePath(name).replace(/\//g, '-');
+ if (!clean || clean === assetName(asset)) return;
+ this.setPath(asset, this.uniquePath(joinPath(assetFolder(asset), clean), asset));
+ }
+
+ /** Moves assets into a folder ('' for the root). */
+ public move(assets: Asset[], folder: string): void {
+ const target = normalizePath(folder);
+ if (target) this.declared.add(target);
+ for (const asset of assets) {
+ if (assetFolder(asset) === target) continue;
+ this.setPath(asset, this.uniquePath(joinPath(target, assetName(asset)), asset));
+ }
+ }
+
+ private setPath(asset: Asset, path: string): void {
+ const from = asset.path;
+ if (from === path) return;
+ asset.path = path;
+ this.changed();
+ this.onRename?.(asset, from);
+ }
+
+ /** Removes assets from the library. Bytes the app wrote into `assets/` go too. */
+ public async remove(assets: Asset[]): Promise<void> {
+ for (const asset of assets) {
+ this.map.delete(asset.id);
+ if (isProjectSource(asset.source) && asset.source.startsWith(`${ASSETS_DIR}/`)) {
+ await this.fs.remove(asset.source).catch(() => {});
+ }
+ }
+ this.changed();
+ }
+
+ public createFolder(path: string): string {
+ const folder = normalizePath(path);
+ if (folder) {
+ this.declared.add(folder);
+ this.changed();
+ }
+ return folder;
+ }
+
+ /** A folder name free under `parent`, from `base` (`base`, `base 2`, …). */
+ public uniqueFolderName(parent: string, base: string): string {
+ const prefix = parent ? `${parent}/` : '';
+ const taken = [...this.foldersNow()]
+ .filter((path) => path.startsWith(prefix) && !path.slice(prefix.length).includes('/'))
+ .map(basename);
+ let name = base;
+ for (let n = 2; taken.includes(name); n++) name = `${base} ${n}`;
+ return name;
+ }
+
+ /** Renames a folder in place; everything under it follows. */
+ public renameFolder(path: string, name: string): string {
+ const clean = normalizePath(name).replace(/\//g, '-');
+ if (!clean) return path;
+ return this.moveFolderTo(path, joinPath(dirname(path), clean));
+ }
+
+ /** Moves a folder into another ('' for the root); refuses to move it into itself. */
+ public moveFolder(path: string, into: string): string {
+ const target = normalizePath(into);
+ if (target === path || target.startsWith(`${path}/`)) return path;
+ return this.moveFolderTo(path, joinPath(target, basename(path)));
+ }
+
+ private moveFolderTo(from: string, to: string): string {
+ if (from === to || !from) return from;
+ if (this.foldersNow().has(to)) to = joinPath(dirname(to), this.uniqueFolderName(dirname(to), basename(to)));
+
+ const prefix = `${from}/`;
+ for (const folder of [...this.declared]) {
+ if (folder === from || folder.startsWith(prefix)) {
+ this.declared.delete(folder);
+ this.declared.add(to + folder.slice(from.length));
+ }
+ }
+ this.declared.add(to);
+ for (const asset of this.listNow()) {
+ if (asset.path.startsWith(prefix)) this.setPath(asset, to + asset.path.slice(from.length));
+ }
+ this.changed();
+ return to;
+ }
+
+ /** Deletes a folder and everything in it. Returns the removed assets. */
+ public async deleteFolder(path: string): Promise<Asset[]> {
+ const prefix = `${path}/`;
+ for (const folder of [...this.declared]) {
+ if (folder === path || folder.startsWith(prefix)) this.declared.delete(folder);
+ }
+ const doomed = this.listNow().filter((asset) => asset.path.startsWith(prefix));
+ await this.remove(doomed);
+ return doomed;
+ }
+
+ // -----------------------------------------------------------------------
+ // Persisting
+
+ private changed(): void {
+ if (this.disposed) return;
+ this.dirty = true;
+ this.publish();
+ clearTimeout(this.saveTimer);
+ this.saveTimer = setTimeout(() => void this.flush(), SAVE_DEBOUNCE);
+ }
+
+ /** The manifest as it would be written now. */
+ public manifest(): Manifest {
+ return {
+ version: 1,
+ folders: [...this.declared].sort(),
+ assets: this.listNow().map(toRecord),
+ };
+ }
+
+ /** Writes the manifest if anything changed. Writes are serialized. */
+ public flush(): Promise<void> {
+ clearTimeout(this.saveTimer);
+ if (!this.dirty) return this.saving;
+ this.dirty = false;
+ this.saving = this.saving
+ .then(() => this.fs.writeManifest(this.manifest()))
+ .catch((error: unknown) => console.error('[assets] could not write the manifest:', error));
+ return this.saving;
+ }
+
+ /** Writes what is pending and stops. */
+ public dispose(): Promise<void> {
+ const done = this.flush();
+ this.disposed = true;
+ return done;
+ }
+
+ // -----------------------------------------------------------------------
+ // Describing
+
+ /** A library path not taken by any other asset: `name`, `name 2`, … */
+ private uniquePath(path: string, except?: Asset): string {
+ const taken = new Set(this.listNow().filter((asset) => asset !== except).map((asset) => asset.path));
+ if (!taken.has(path)) return path;
+ const folder = dirname(path);
+ const name = basename(path);
+ const dot = name.lastIndexOf('.');
+ const stem = dot > 0 ? name.slice(0, dot) : name;
+ const ext = dot > 0 ? name.slice(dot) : '';
+ for (let n = 2; ; n++) {
+ const candidate = joinPath(folder, `${stem} ${n}${ext}`);
+ if (!taken.has(candidate)) return candidate;
+ }
+ }
+
+ private once(key: string, run: () => Promise<Asset>): Promise<Asset> {
+ const running = this.inflight.get(key);
+ if (running) return running;
+ const promise = run().finally(() => this.inflight.delete(key));
+ this.inflight.set(key, promise);
+ return promise;
+ }
+
+ /** Describes a file or frames directory at `source`. */
+ private async describeSource(source: string, meta: DescribeMeta): Promise<Asset> {
+ const entries = await this.fs.list(source);
+ if (entries.length && isSequenceListing(entries.map((entry) => entry.name))) {
+ return this.describeSequence(source, sortFrames(entries).filter((entry) => entry.kind === 'file'), meta);
+ }
+ return this.describeFile(source, meta);
+ }
+
+ private async describeFile(source: string, meta: DescribeMeta): Promise<Asset> {
+ const handle = this.fileHandle(source);
+ const file = await handle.getFile();
+ const mimeType = await detectMimeType(file);
+ if (!mimeType) throw new Error(`Unsupported file: ${basename(source)}`);
+ const [id, probe] = await Promise.all([hashBlob(file), probeMedia(file, mimeType)]);
+
+ return {
+ id,
+ path: meta.path,
+ source,
+ createdAt: meta.createdAt ?? new Date().toISOString(),
+ mimeType,
+ stat: { size: file.size, mtime: file.lastModified },
+ ...(meta.generation ? { generation: meta.generation } : {}),
+ handle,
+ ...probe,
+ };
+ }
+
+ private async describeUrl(url: string, meta: DescribeMeta): Promise<Asset> {
+ const mimeType = await detectMimeType(url);
+ if (!mimeType) throw new Error(`Unsupported resource: ${url}`);
+ const handle = this.urlHandle(url);
+ const file = await handle.getFile();
+ const [id, probe] = await Promise.all([hashBlob(file), probeMedia(file, mimeType)]);
+
+ return {
+ id,
+ path: meta.path,
+ source: url,
+ createdAt: meta.createdAt ?? new Date().toISOString(),
+ mimeType,
+ handle,
+ ...probe,
+ };
+ }
+
+ private async describeSequence(source: string, frames: FsEntry[], meta: DescribeMeta): Promise<SequenceAsset> {
+ if (!frames.length) throw new Error(`Empty sequence: ${basename(source)}`);
+ const first = frames[0]!;
+ const handle = this.fileHandle(joinPath(source, first.name));
+ const file = await handle.getFile();
+ const mimeType = await detectMimeType(file);
+ if (!mimeType?.startsWith('image/')) throw new Error(`Not an image sequence: ${basename(source)}`);
+ const probe = await probeMedia(file, mimeType);
+ if (probe.type !== 'IMAGE') throw new Error(`Not an image sequence: ${basename(source)}`);
+
+ return {
+ id: await hashSequence(frames),
+ type: 'SEQUENCE',
+ path: meta.path,
+ source,
+ createdAt: meta.createdAt ?? new Date().toISOString(),
+ mimeType,
+ width: probe.width,
+ height: probe.height,
+ frameRate: DEFAULT_SEQUENCE_FPS,
+ duration: frames.length / DEFAULT_SEQUENCE_FPS,
+ handle,
+ directoryHandle: this.directoryHandle(source),
+ };
+ }
+
+ /** Attaches handles to a manifest record without touching its bytes. */
+ private attach(record: AssetRecord): Asset {
+ if (record.type === 'SEQUENCE') {
+ const first = this.fs.list(record.source).then((entries) => sortFrames(entries).find((entry) => entry.kind === 'file'));
+ return {
+ ...record,
+ handle: { getFile: async () => this.fs.file(joinPath(record.source, (await first)?.name ?? '')) },
+ directoryHandle: this.directoryHandle(record.source),
+ };
+ }
+ return {
+ ...record,
+ handle: isUrlSource(record.source) ? this.urlHandle(record.source) : this.fileHandle(record.source),
+ };
+ }
+
+ private fileHandle(source: string): AssetFileHandle {
+ return { getFile: () => this.fs.file(source) };
+ }
+
+ private urlHandle(url: string): AssetFileHandle {
+ let cached: Promise<File> | undefined;
+ return {
+ getFile: () => {
+ cached ??= fetch(url).then(async (response) => {
+ if (!response.ok) throw new Error(`Failed to fetch ${url}: ${response.status}`);
+ const blob = await response.blob();
+ return new File([blob], basename(url.split(/[?#]/)[0]!), { type: blob.type });
+ });
+ cached.catch(() => (cached = undefined));
+ return cached;
+ },
+ };
+ }
+
+ private directoryHandle(source: string): AssetDirectoryHandle {
+ const fs = this.fs;
+ return {
+ async *entries() {
+ const frames = sortFrames(await fs.list(source)).filter((entry) => entry.kind === 'file');
+ for (const frame of frames) {
+ const path = joinPath(source, frame.name);
+ yield [frame.name, { kind: 'file', getFile: () => fs.file(path) }] as [string, { kind: string; getFile: () => Promise<File> }];
+ }
+ },
+ };
+ }
+}
+
+interface DescribeMeta {
+ path: string;
+ createdAt?: string;
+ generation?: AssetGeneration;
+}
diff --git a/packages/assets/src/manifest.ts b/packages/assets/src/manifest.ts
new file mode 100644
index 00000000..5f77d1b9
--- /dev/null
+++ b/packages/assets/src/manifest.ts
@@ -0,0 +1,109 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The asset manifest: the project's record of its library. On desktop it is
+// `assets.yml` at the project root (the main process converts to and from
+// YAML); elsewhere the same shape as JSON. It holds what cannot be derived
+// from the files themselves — which files belong to the project, where the
+// project keeps them (`path`), where their bytes are (`source`), and what
+// they were found to be — so opening a project never re-probes footage that
+// has not changed.
+
+import { normalizePath } from './types';
+
+import type { Asset } from './types';
+
+export const MANIFEST_VERSION = 1;
+
+/** The manifest's file name (with the YAML extension; the codec is the host's). */
+export const MANIFEST_FILE = 'assets.yml';
+
+/** The project directory the app writes its own asset bytes into. */
+export const ASSETS_DIR = 'assets';
+
+/** Where generated assets are stored inside `ASSETS_DIR`. */
+export const GENERATED_DIR = 'generated';
+
+type DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K> : never;
+
+/** An asset as written to the manifest: the model minus its runtime handles. */
+export type AssetRecord = DistributiveOmit<Asset, 'handle' | 'directoryHandle' | 'transient'>;
+
+export interface Manifest {
+ version: number;
+ /**
+ * Every folder of the library, including empty ones (a folder with assets
+ * in it is also implied by their paths). `/`-separated, no leading slash.
+ */
+ folders: string[];
+ assets: AssetRecord[];
+}
+
+const RECORD_KEYS = new Set(['handle', 'directoryHandle', 'transient']);
+
+/** The manifest record of an asset: its data, none of its handles. */
+export function toRecord(asset: Asset): AssetRecord {
+ const record: Record<string, unknown> = {};
+ for (const [key, value] of Object.entries(asset)) {
+ if (RECORD_KEYS.has(key) || value === undefined) continue;
+ record[key] = value;
+ }
+ return record as AssetRecord;
+}
+
+/** Whether `record` has what every asset needs; unrecognized fields are kept. */
+export function isAssetRecord(record: unknown): record is AssetRecord {
+ if (!record || typeof record !== 'object') return false;
+ const r = record as Record<string, unknown>;
+ return (
+ typeof r.id === 'string' && r.id.length > 0 &&
+ typeof r.path === 'string' && r.path.length > 0 &&
+ typeof r.source === 'string' && r.source.length > 0 &&
+ typeof r.type === 'string' &&
+ typeof r.mimeType === 'string'
+ );
+}
+
+/**
+ * Reads whatever the host handed back as a manifest into a well-formed one.
+ * Tolerant: a missing or malformed file is an empty library, and a broken
+ * record is dropped rather than taking the rest of the library with it.
+ */
+export function normalizeManifest(input: unknown): Manifest {
+ const manifest: Manifest = { version: MANIFEST_VERSION, folders: [], assets: [] };
+ if (!input || typeof input !== 'object') return manifest;
+
+ const raw = input as Record<string, unknown>;
+ if (Array.isArray(raw.folders)) {
+ for (const folder of raw.folders) {
+ if (typeof folder !== 'string') continue;
+ const path = normalizePath(folder);
+ if (path && !manifest.folders.includes(path)) manifest.folders.push(path);
+ }
+ }
+
+ if (Array.isArray(raw.assets)) {
+ const seen = new Set<string>();
+ for (const record of raw.assets) {
+ if (!isAssetRecord(record)) continue;
+ const path = normalizePath(record.path);
+ if (!path || seen.has(record.id)) continue;
+ seen.add(record.id);
+ manifest.assets.push({ ...record, path });
+ }
+ }
+
+ return manifest;
+}
+
+/** Whether a source is a URL rather than a path. */
+export const isUrlSource = (source: string): boolean => /^https?:\/\//i.test(source);
+
+/** Whether a source is an absolute OS path (POSIX or Windows). */
+export const isAbsoluteSource = (source: string): boolean =>
+ source.startsWith('/') || /^[a-zA-Z]:[\\/]/.test(source) || source.startsWith('\\\\');
+
+/** Whether a source is a path inside the project (neither URL nor absolute). */
+export const isProjectSource = (source: string): boolean =>
+ !isUrlSource(source) && !isAbsoluteSource(source);
diff --git a/packages/assets/src/probe.ts b/packages/assets/src/probe.ts
new file mode 100644
index 00000000..713ec516
--- /dev/null
+++ b/packages/assets/src/probe.ts
@@ -0,0 +1,190 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// What a file is: its MIME type and the type-specific metadata the runtime
+// needs before it can size or time an asset (dimensions, duration, tracks).
+
+import { ALL_FORMATS, BlobSource, Input, UrlSource } from 'mediabunny';
+
+import type { InputTrack } from 'mediabunny';
+import type { Asset } from './types';
+
+export const DEFAULT_SEQUENCE_FPS = 30;
+
+/** Type-specific fields of an asset, as found by `probeMedia`. */
+export type ProbeResult = Extract<
+ | { type: 'IMAGE'; width: number; height: number }
+ | { type: 'AUDIO'; duration: number; sampleRate: number; channels: number }
+ | { type: 'VIDEO'; duration: number; width: number; height: number; frameRate: number; bitRate: number; sampleRate?: number; channels?: number }
+ | { type: 'TRANSCRIPT' }
+ | { type: 'SCRIPT' },
+ { type: Asset['type'] }
+>;
+
+const TRANSCRIPT_TYPES = new Set(['application/json', 'application/x-subrip', 'text/vtt']);
+
+/**
+ * The MIME type of a file, or of the resource behind a URL. Subtitle files go
+ * by extension (OS registries rarely map .srt); audio and video are sniffed
+ * by mediabunny, since a container's declared type is often wrong or empty.
+ * Null when the file is not something the library takes.
+ */
+export async function detectMimeType(input: Blob | string): Promise<string | null> {
+ const name = typeof input === 'string' ? input.split(/[?#]/)[0]! : (input as File).name ?? '';
+ let mimeType = typeof input === 'string' ? await fetchMimeType(input) : input.type;
+
+ if (/\.srt$/i.test(name) || mimeType === 'application/x-subrip') return 'application/x-subrip';
+ if (/\.vtt$/i.test(name) || mimeType === 'text/vtt') return 'text/vtt';
+ if (/\.json$/i.test(name) || mimeType === 'application/json') return 'application/json';
+
+ if (mimeType?.startsWith('image/')) return mimeType;
+ if (mimeType?.startsWith('text/html')) return mimeType;
+
+ try {
+ const source = typeof input === 'string' ? new UrlSource(input) : new BlobSource(input);
+ mimeType = await new Input({ formats: ALL_FORMATS, source }).getMimeType();
+ } catch {
+ return null;
+ }
+
+ if (mimeType?.startsWith('audio/') || mimeType?.startsWith('video/')) return mimeType;
+ return null;
+}
+
+/**
+ * Derives the type-specific asset properties of a media file. Throws when
+ * the file is of no type the library takes, or is missing what its type
+ * needs (a video without a video track, say).
+ */
+export async function probeMedia(file: Blob, mimeType: string): Promise<ProbeResult> {
+ if (mimeType.startsWith('image/')) {
+ const { width, height } = mimeType === 'image/svg+xml'
+ ? await getSvgDimensions(file)
+ : await getRasterDimensions(file);
+ return { type: 'IMAGE', width, height };
+ }
+
+ if (TRANSCRIPT_TYPES.has(mimeType)) return { type: 'TRANSCRIPT' };
+
+ if (mimeType.startsWith('audio/') || mimeType.startsWith('video/')) {
+ const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(file) });
+ const audioTrack = await input.getPrimaryAudioTrack();
+ const channels = audioTrack?.numberOfChannels;
+ const sampleRate = audioTrack?.sampleRate;
+
+ if (mimeType.startsWith('audio/')) {
+ if (!sampleRate || !channels) throw new Error('Audio track not found');
+ return { type: 'AUDIO', duration: await trackContentDuration(audioTrack), sampleRate, channels };
+ }
+
+ const videoTrack = await input.getPrimaryVideoTrack();
+ if (!videoTrack) throw new Error('Video track not found');
+ const stats = await videoTrack.computePacketStats();
+
+ return {
+ type: 'VIDEO',
+ width: videoTrack.displayWidth,
+ height: videoTrack.displayHeight,
+ frameRate: stats.averagePacketRate,
+ bitRate: stats.averageBitrate,
+ duration: await trackContentDuration(videoTrack),
+ ...(sampleRate === undefined ? {} : { sampleRate }),
+ ...(channels === undefined ? {} : { channels }),
+ };
+ }
+
+ throw new Error(`Unsupported file type: ${mimeType}`);
+}
+
+const FRAME_NAME = /^(.*?)(\d+)\.(png|jpe?g|webp|gif|avif|bmp)$/i;
+
+/**
+ * Whether a directory listing looks like an image sequence: two or more
+ * numbered image files sharing one prefix and extension (`frame_0001.png`,
+ * `frame_0002.png`, …) and nothing else. A folder of unrelated stills is a
+ * folder, not a clip.
+ */
+export function isSequenceListing(names: string[]): boolean {
+ const files = names.filter((name) => !name.startsWith('.'));
+ if (files.length < 2) return false;
+ let prefix: string | undefined;
+ let extension: string | undefined;
+ for (const name of files) {
+ const match = FRAME_NAME.exec(name);
+ if (!match) return false;
+ const [, stem, , ext] = match;
+ if (prefix === undefined) {
+ prefix = stem;
+ extension = ext!.toLowerCase();
+ } else if (stem !== prefix || ext!.toLowerCase() !== extension) {
+ return false;
+ }
+ }
+ return true;
+}
+
+/** The frames of a sequence in play order: natural sort, so 2 comes before 10. */
+export function sortFrames<T extends { name: string }>(frames: T[]): T[] {
+ const collator = new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' });
+ return [...frames].sort((a, b) => collator.compare(a.name, b.name));
+}
+
+async function getRasterDimensions(file: Blob): Promise<{ width: number; height: number }> {
+ const bitmap = await createImageBitmap(file);
+ const { width, height } = bitmap;
+ bitmap.close();
+ return { width, height };
+}
+
+async function getSvgDimensions(file: Blob): Promise<{ width: number; height: number }> {
+ const text = await file.text();
+ const root = new DOMParser().parseFromString(text, 'image/svg+xml').documentElement;
+
+ const parseLength = (value: string | null): number => {
+ if (!value) return 0;
+ const n = parseFloat(value);
+ return Number.isFinite(n) ? n : 0;
+ };
+
+ let width = parseLength(root.getAttribute('width'));
+ let height = parseLength(root.getAttribute('height'));
+
+ if ((!width || !height) && root.getAttribute('viewBox')) {
+ const parts = root.getAttribute('viewBox')!.split(/[\s,]+/).map(Number);
+ if (parts.length === 4 && Number.isFinite(parts[2]) && Number.isFinite(parts[3])) {
+ width = width || parts[2]!;
+ height = height || parts[3]!;
+ }
+ }
+
+ // SVG spec default when neither width/height nor viewBox is declared.
+ return { width: width || 300, height: height || 150 };
+}
+
+async function fetchMimeType(url: string): Promise<string | null> {
+ let response: Response;
+ try {
+ response = await fetch(url, { method: 'HEAD' });
+ } catch {
+ const controller = new AbortController();
+ response = await fetch(url, { signal: controller.signal });
+ controller.abort();
+ }
+ if (!response.ok) return null;
+ return response.headers.get('Content-Type');
+}
+
+/**
+ * Playable content length of a media track, in seconds: the end timestamp
+ * minus the track's first *presented* timestamp. A track may carry a global
+ * time offset (a non-zero first timestamp); using the raw end timestamp as
+ * the duration would over-report the length by that offset. A negative first
+ * timestamp means the head has been trimmed by an edit list; those pre-roll
+ * samples are not presented, so the start is clamped to 0.
+ */
+async function trackContentDuration(track?: InputTrack | null): Promise<number> {
+ if (!track) return 0;
+ const [start, end] = await Promise.all([track.getFirstTimestamp(), track.computeDuration()]);
+ return Math.max(0, end - Math.max(0, start));
+}
diff --git a/packages/assets/src/types.ts b/packages/assets/src/types.ts
new file mode 100644
index 00000000..f302acd8
--- /dev/null
+++ b/packages/assets/src/types.ts
@@ -0,0 +1,147 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The asset model. An asset is a manifest record (see ./manifest.ts) plus a
+// runtime handle to its bytes, attached by the AssetLibrary. Everything an
+// asset *is* lives in the record: what the file was found to be (probe
+// results), where its bytes are (`source`), and where the project keeps it in
+// the library (`path`). Its `id` is a content hash, so the same bytes imported
+// twice are one asset, and a relinked file keeps its metadata.
+
+/** Whatever hands out the asset's bytes: a real File on desktop, an OPFS
+ * file on the web, a fetched blob for a URL. */
+export interface AssetFileHandle {
+ getFile(): Promise<File>;
+}
+
+/** Directory of numbered frame files backing a SEQUENCE asset. Typed to the
+ * web FileSystemDirectoryHandle surface the decoders iterate. */
+export interface AssetDirectoryHandle {
+ entries(): AsyncIterableIterator<[string, { kind: string; getFile?: () => Promise<File> }]>;
+}
+
+export type TranscriptWord = { text: string; start: number; end: number };
+
+export type WordGroup = TranscriptWord[];
+
+export type Transcript = { text: string; words: TranscriptWord[] }[];
+
+/** Size and mtime of a source when it was last hashed/probed; a mismatch
+ * means the bytes may have changed and the asset is re-examined. */
+export interface AssetStat {
+ size: number;
+ mtime: number;
+}
+
+/** How a generated asset came to be: the content key of the fully-resolved
+ * `generate.*` spec (dedup across runs) and the backend generation id. */
+export interface AssetGeneration {
+ key: string;
+ id?: string | null;
+}
+
+interface AssetBase {
+ /** Short content hash; identity of the asset. */
+ id: string;
+ /**
+ * Library path: `folder/sub/name.ext`, `/`-separated, no leading slash.
+ * What JSX `src` names, and what the asset panel shows. Folders are the
+ * prefixes; renaming or moving an asset changes only this.
+ */
+ path: string;
+ /**
+ * Where the bytes are: an absolute OS path (a linked file, left where the
+ * user had it), an `http(s)://` URL, or a project-relative path (`assets/…`,
+ * bytes the app itself produced: generations, downloads, transcodes).
+ */
+ source: string;
+ createdAt: string;
+ mimeType: string;
+ stat?: AssetStat;
+ generation?: AssetGeneration;
+ /**
+ * Resolved on the fly for a `src` that names a path or URL outside the
+ * library; lives in memory only and is never written to the manifest.
+ */
+ transient?: boolean;
+ handle: AssetFileHandle;
+}
+
+export interface ImageAsset extends AssetBase {
+ type: 'IMAGE';
+ width: number;
+ height: number;
+}
+
+export interface AudioAsset extends AssetBase {
+ type: 'AUDIO';
+ duration: number;
+ sampleRate: number;
+ channels: number;
+ transcript?: Transcript;
+}
+
+export interface VideoAsset extends AssetBase {
+ type: 'VIDEO';
+ duration: number;
+ width: number;
+ height: number;
+ frameRate: number;
+ bitRate: number;
+ sampleRate?: number;
+ channels?: number;
+ transcript?: Transcript;
+}
+
+export interface TranscriptAsset extends AssetBase {
+ type: 'TRANSCRIPT';
+}
+
+export interface ScriptAsset extends AssetBase {
+ type: 'SCRIPT';
+}
+
+// `source` is the frames directory; `handle` points to the first frame's
+// file so generic preview code works.
+export interface SequenceAsset extends AssetBase {
+ type: 'SEQUENCE';
+ width: number;
+ height: number;
+ frameRate: number;
+ duration: number;
+ directoryHandle: AssetDirectoryHandle;
+}
+
+export type Asset =
+ | ImageAsset
+ | AudioAsset
+ | VideoAsset
+ | TranscriptAsset
+ | ScriptAsset
+ | SequenceAsset;
+
+export type AssetType = Asset['type'];
+
+/** The file name of an asset: the last segment of its library path. */
+export const assetName = (asset: Pick<Asset, 'path'>): string => basename(asset.path);
+
+/** The folder of an asset: its library path without the name, '' at root. */
+export const assetFolder = (asset: Pick<Asset, 'path'>): string => dirname(asset.path);
+
+export function basename(path: string): string {
+ const at = path.lastIndexOf('/');
+ return at < 0 ? path : path.slice(at + 1);
+}
+
+export function dirname(path: string): string {
+ const at = path.lastIndexOf('/');
+ return at < 0 ? '' : path.slice(0, at);
+}
+
+export const joinPath = (...parts: string[]): string =>
+ parts.filter(Boolean).join('/').replace(/\/+/g, '/');
+
+/** Normalizes a library path: `/`-separated, no leading/trailing/double slashes. */
+export const normalizePath = (path: string): string =>
+ path.replace(/\\/g, '/').split('/').filter((part) => part && part !== '.').join('/');
diff --git a/packages/assets/tsconfig.json b/packages/assets/tsconfig.json
new file mode 100644
index 00000000..09fafae6
--- /dev/null
+++ b/packages/assets/tsconfig.json
@@ -0,0 +1,9 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo",
+ "erasableSyntaxOnly": false,
+ "lib": ["es2024", "DOM", "DOM.Iterable"]
+ },
+ "include": ["src"]
+}
diff --git a/packages/reconciler/package.json b/packages/reconciler/package.json
index dcc4a9c7..fea74292 100644
--- a/packages/reconciler/package.json
+++ b/packages/reconciler/package.json
@@ -18,6 +18,7 @@
"koota"
],
"dependencies": {
+ "@diffusionstudio/assets": "*",
"@diffusionstudio/jsx": "*",
"@diffusionstudio/runtime": "*",
"koota": "^0.6.6"
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 0cb41b71..770b01f8 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,95 +3,12 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import {
- Active,
- Animation,
- AnimationPhase,
- AnimationType,
- appendChild,
- AssetId,
- Assets,
- Audio,
- Background,
- BlendMode,
- BlendModeType,
- Blur,
- Chars,
- ClipsContent,
- CornerRadius,
- createEntity,
- DEFAULT_BACKGROUND,
- Color,
- ColorStop,
- Effect,
- EffectType,
- End,
- FontStyle,
- FrameRate,
- Loop,
- Geometry,
- GeometryType,
- getEntityTree,
- getParentEntity,
- getParentNode,
- Hidden,
- IsMask,
- isText,
- ItemIndex,
- Keyframe,
- KeyframeTrack,
- Muted,
- Name,
- Offset,
- Opacity,
- Paint,
- PaintType,
- parseColor,
- Playback,
- PlaybackRate,
- Position,
- removeChild,
- resizeEntity,
- Scale,
- ScaleMode,
- ScaleModeType,
- secondsToFrames,
- getEntityChildren,
- Group,
- Sequential,
- Shader,
- Size,
- Stage,
- Root,
- Rotation,
- Scene,
- Selected,
- Shadow,
- Source,
- SourceIn,
- SourceOut,
- setCameraMatrix,
- Start,
- Stroke,
- StrokeCap,
- StrokeJoin,
- StrokeStyle,
- SurfaceHost,
- SurfaceHostHandle,
- TextAlign,
- TextBaseline,
- TextCase,
- TextRange,
- TextStyle,
- Transition,
- TransitionType,
- UniformScale,
- Volume,
-} from '@diffusionstudio/runtime';
+import { Active, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, BlendMode, BlendModeType, Blur, Chars, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, FontStyle, FrameRate, Loop, Geometry, GeometryType, Library, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getAssetFile, getEntityChildren, getLibrary, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';
-import type { PropValue } from '@diffusionstudio/jsx';
+import type { Asset } from '@diffusionstudio/assets';
+import type { AssetInput, PropValue } from '@diffusionstudio/jsx';
import { trait, type Entity, type World } from 'koota';
import type { ProjectDocument } from './host';
@@ -145,6 +62,13 @@ const TextParts = trait(() => new Set<TextNode>());
*/
const Authored = trait(() => ({ tag: '', props: {} as Record<string, PropValue> }));
+/**
+ * The `src` an element is waiting on: set while the library resolves a
+ * source the world does not know yet, so a resolution that arrives after the
+ * element was given another source (or none) is dropped.
+ */
+const SourceRequest = trait(() => ({ value: undefined as unknown }));
+
/** Props that address or wire an element rather than describe it. */
const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);
@@ -950,13 +874,7 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return;
}
case 'src': {
- if (typeof value !== 'string' || !this.world.get(Assets)?.has(value)) {
- entity.remove(AssetId);
- return;
- }
-
- entity.add(AssetId);
- entity.set(AssetId, { value });
+ this.setSource(entity, value);
return;
}
case 'objectFit': {
@@ -1232,6 +1150,71 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return next === undefined ? undefined : { entity: next };
}
+ /**
+ * Binds an element to the asset its `src` names. A library path or id the
+ * world already knows binds at once; anything else — a path or URL outside
+ * the library, a `generate.*` ref — is resolved through the library, and
+ * binds when it arrives, unless the element has moved on to another source
+ * or gone away in the meantime.
+ */
+ private setSource(entity: Entity, value: unknown): void {
+ const library = this.world.get(Library);
+ const known = typeof value === 'string' ? getAsset(this.world, value) : undefined;
+
+ if (known) {
+ entity.remove(SourceRequest);
+ this.bindAsset(entity, known);
+ return;
+ }
+
+ if (!library || value === undefined || value === null || value === '') {
+ entity.remove(SourceRequest);
+ entity.remove(AssetId);
+ return;
+ }
+
+ // Whatever it showed before is wrong until the new source arrives.
+ entity.remove(AssetId);
+ entity.add(SourceRequest);
+ entity.set(SourceRequest, { value });
+
+ void library.resolve(value).then(
+ (asset) => {
+ if (!entity.isAlive() || entity.get(SourceRequest)?.value !== value) return;
+ entity.remove(SourceRequest);
+ this.bindAsset(entity, asset);
+ },
+ (error: unknown) => {
+ if (!entity.isAlive() || entity.get(SourceRequest)?.value !== value) return;
+ entity.remove(SourceRequest);
+ console.error(`[reconciler] could not resolve src:`, error);
+ },
+ );
+ }
+
+ /**
+ * Binds an element to an asset. A frames directory on a `<video>` or
+ * `<image>` plays as a sequence: the element's paint follows the asset.
+ */
+ private bindAsset(entity: Entity, asset: Asset): void {
+ entity.add(AssetId);
+ entity.set(AssetId, { value: asset.id });
+
+ const paint = entity.get(Paint)?.value;
+ if (asset.type === 'SEQUENCE' && (paint === PaintType.VIDEO || paint === PaintType.IMAGE)) {
+ entity.set(Paint, { value: PaintType.SEQUENCE });
+ } else if (asset.type !== 'SEQUENCE' && paint === PaintType.SEQUENCE) {
+ entity.set(Paint, { value: asset.type === 'IMAGE' ? PaintType.IMAGE : PaintType.VIDEO });
+ }
+ }
+
+ /** The `useFile` of a project: what its `src` would bind to, as a File. */
+ public async loadFile(input: AssetInput): Promise<File> {
+ const known = typeof input === 'string' ? getAsset(this.world, input) : undefined;
+ const asset = known ?? await getLibrary(this.world).resolve(input);
+ return getAssetFile(asset);
+ }
+
public dispose(): void {
this.removeNode(this.stage, this.stage);
}
diff --git a/packages/runtime/package.json b/packages/runtime/package.json
index e036b811..8d616b23 100644
--- a/packages/runtime/package.json
+++ b/packages/runtime/package.json
@@ -21,11 +21,11 @@
"video"
],
"dependencies": {
+ "@diffusionstudio/assets": "*",
"animejs": "^4.4.1",
"colord": "^2.9.3",
"koota": "^0.6.6",
- "mediabunny": "^1.32.2",
- "sqids": "^0.3.0"
+ "mediabunny": "^1.32.2"
},
"devDependencies": {
"@webgpu/types": "^0.1.71",
diff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.ts
index e738bad8..d270b2de 100644
--- a/packages/runtime/src/actions/assets.ts
+++ b/packages/runtime/src/actions/assets.ts
@@ -2,85 +2,25 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Asset model actions (was api/assets.ts, model only). Loading, probing,
-// OPFS/db persistence, uploads, and pickers stay app-side; the runtime owns
-// the in-memory metadata store, id allocation, and timeline insertion.
+// Reading assets out of the world. Everything that changes the asset set
+// goes through the AssetLibrary (@diffusionstudio/assets); these are the
+// lookups the decoders and hosts share.
-import Sqids from 'sqids';
+import { Library } from '../traits';
-import { GeometryType, PaintType } from '../constants';
-import {
- Geometry, Paint, Audio, Caption, Hidden, Name, AssetId, Start, SourceIn,
- SourceOut, KeepAspectRatio, Computed, Assets, AssetIds,
-} from '../traits';
-import { store } from '../world/store';
-import { secondsToFrames } from '../utils/time';
-import { createEntity } from './entities';
-import { appendChild } from './hierarchy';
-import { resizeEntity } from './resize';
-import { getActiveEntity } from './frame';
+import type { World } from 'koota';
+import type { Asset, AssetLibrary } from '@diffusionstudio/assets';
-import type { Entity, World } from 'koota';
-import type { Asset, Transcript } from '../assets/types';
-
-const sqids = new Sqids({ minLength: 4 });
-
-/**
- * Allocates the next id in the project's shared asset/folder id space, so an
- * id never refers to both an asset and a folder.
- */
-export function allocateId(world: World): string {
- const counter = (world.get(AssetIds)?.counter ?? 0) + 1;
- world.set(AssetIds, { counter });
- return sqids.encode([counter]);
-}
-
-/**
- * Raises the id counter past every sqid in `ids`. Non-sqid ids (e.g. legacy
- * "fld-…" folder ids) are ignored. Restores may run in parallel, so this only
- * ever raises the counter.
- */
-export function raiseIdCounter(world: World, ids: Iterable<string>): void {
- let counter = world.get(AssetIds)?.counter ?? 0;
- for (const id of ids) {
- const decoded = sqids.decode(id);
- const num = decoded[0];
- if (decoded.length === 1 && Number.isFinite(num) && num! > counter) {
- counter = num!;
- }
- }
- world.set(AssetIds, { counter });
-}
-
-/**
- * Appends one or more assets to the in-memory store. New assets are ordered
- * first; existing records win on id conflict.
- */
-export function appendAssets(world: World, ...incoming: Asset[]): void {
- const map = world.get(Assets)!;
- const existing = Array.from(map.values());
- map.clear();
- for (const asset of incoming) map.set(asset.id, asset);
- for (const asset of existing) map.set(asset.id, asset);
-}
-
-/**
- * Inserts or replaces an asset at the front of the in-memory store (newest
- * first). The app persists it and refreshes its reactive views.
- */
-export function upsertAsset(world: World, asset: Asset): Asset {
- const map = world.get(Assets)!;
- const existing = Array.from(map.values()).filter(a => a.id !== asset.id);
- map.clear();
- map.set(asset.id, asset);
- existing.forEach(a => map.set(a.id, a));
- return asset;
+/** The world's asset library; throws when the host attached none. */
+export function getLibrary(world: World): AssetLibrary {
+ const library = world.get(Library);
+ if (!library) throw new Error('This world has no asset library');
+ return library;
}
-/** Removes one or more assets from the in-memory store. */
-export function removeAssets(world: World, ...ids: string[]): void {
- const map = world.get(Assets)!;
- for (const id of ids) map.delete(id);
+/** The asset with `id` (or at that library path), or undefined without a library. */
+export function getAsset(world: World, idOrPath: string): Asset | undefined {
+ return world.get(Library)?.get(idOrPath);
}
const assetFileCache = new WeakMap<Asset['handle'], Promise<File>>();
@@ -106,95 +46,8 @@ export function getAssetFile(asset: Pick<Asset, 'handle'>): Promise<File> {
/** Returns the File/Blob backing an asset, or null if the asset is missing. */
export async function getAssetBlob(world: World, id: string): Promise<Blob | null> {
- const asset = world.get(Assets)?.get(id);
+ const asset = getAsset(world, id);
if (!asset) return null;
return await getAssetFile(asset);
}
-
-/**
- * Composes the entity tree for an asset dropped into the active scene: a
- * geometry sized to the asset with the matching paint/caption/audio setup.
- * Returns the created node, or null when there is no active scene.
- */
-export async function insertAssetInTimeline(
- world: World,
- asset: Asset,
- mode: 'playhead' | 'start',
-): Promise<Entity | null> {
- const scene = getActiveEntity(world);
- if (scene === null) return null;
-
- const start = mode === 'playhead' ? store(world, Computed).localTime[scene.id()] ?? 0 : 0;
-
- // Resolve any async asset data up front so all the entity mutations below
- // run synchronously (the app wraps them in a single undo step).
- let spokenWords: { start: number; end: number } | null = null;
- if (asset.type === 'TRANSCRIPT') {
- // Transcript should be trimmed to the span of recognized words
- const file = await getAssetFile(asset);
- const transcript = JSON.parse(await file.text()) as Transcript;
- const lastWord = transcript.at(-1)?.words.at(-1);
- const firstWord = transcript.at(0)?.words.at(0);
- spokenWords = {
- start: secondsToFrames(firstWord?.start),
- end: secondsToFrames(lastWord?.end),
- };
- }
-
- const entity = createEntity(world);
- entity.add(Name);
- entity.set(Name, { value: asset.name });
- entity.add(Start({ value: start }));
-
- if (asset.type === 'IMAGE' || asset.type === 'VIDEO' || asset.type === 'SEQUENCE') {
- const width = Math.round(asset.width);
- const height = Math.round(asset.height);
- resizeEntity(world, entity, { width, height });
- entity.add(KeepAspectRatio);
- entity.set(KeepAspectRatio, { width, height });
- }
-
- if (asset.type === 'TRANSCRIPT') {
- entity.add(Geometry);
- entity.set(Geometry, { value: GeometryType.TEXT });
- entity.add(Caption);
- if (spokenWords) {
- entity.set(Start, { value: start + spokenWords.start });
- entity.add(SourceIn({ value: spokenWords.start }));
- entity.add(SourceOut({ value: spokenWords.end }));
- }
- } else {
- entity.add(Geometry);
- entity.set(Geometry, { value: GeometryType.RECT });
-
- // Other assets need a paint sub-entity
- let paint = PaintType.IMAGE;
- if (asset.type === 'SEQUENCE') paint = PaintType.SEQUENCE;
- if (asset.type === 'VIDEO') paint = PaintType.VIDEO;
- if (asset.type === 'AUDIO') paint = PaintType.WAVEFORM;
- const fill = createEntity(world);
- fill.add(Paint);
- fill.set(Paint, { value: paint });
- fill.add(AssetId);
- fill.set(AssetId, { value: asset.id });
- if (asset.type === 'AUDIO') {
- fill.add(Hidden);
- }
- appendChild(world, fill, entity);
- }
-
- if (asset.type === 'AUDIO') {
- entity.add(Audio);
- resizeEntity(world, entity, { width: 500, height: 150 });
- }
-
- if (asset.type === 'AUDIO' || asset.type === 'TRANSCRIPT') {
- entity.add(AssetId);
- entity.set(AssetId, { value: asset.id });
- }
-
- appendChild(world, entity, scene);
-
- return entity;
-}
diff --git a/packages/runtime/src/actions/folders.ts b/packages/runtime/src/actions/folders.ts
deleted file mode 100644
index 7a51f751..00000000
--- a/packages/runtime/src/actions/folders.ts
+++ /dev/null
@@ -1,188 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// Folder model actions (was api/folders.ts, model only). Pure in-memory tree
-// operations over the Folders/Assets world traits; the app persists the
-// returned records and keeps its own panel navigation state.
-
-import { Assets, Folders } from '../traits';
-import { allocateId } from './assets';
-
-import type { World } from 'koota';
-import type { Asset, Folder } from '../assets/types';
-
-/**
- * A folder's effective parent: its stored parentId, or root when the parent
- * record no longer exists (dangling references degrade to root instead of
- * hiding the folder).
- */
-export function effectiveParentId(world: World, folder: Folder): string | null {
- const folders = world.get(Folders)!;
- return folder.parentId !== null && folders.has(folder.parentId)
- ? folder.parentId
- : null;
-}
-
-/**
- * The folder an asset effectively lives in: its stored folderId, or root when
- * that folder no longer exists.
- */
-export function assetFolderId(world: World, asset: Asset): string | null {
- const folderId = asset.folderId ?? null;
- return folderId !== null && world.get(Folders)!.has(folderId) ? folderId : null;
-}
-
-/**
- * True when `id` is a descendant of `ancestorId` in the folder tree.
- */
-export function isDescendantFolder(world: World, id: string, ancestorId: string): boolean {
- const folders = world.get(Folders)!;
- const visited = new Set<string>();
- let current = folders.get(id);
- while (current && current.parentId !== null && !visited.has(current.id)) {
- if (current.parentId === ancestorId) return true;
- visited.add(current.id);
- current = folders.get(current.parentId);
- }
- return false;
-}
-
-/**
- * The next free `base`/`base N` name among a parent's children.
- */
-export function uniqueFolderName(world: World, base: string, parentId: string | null): string {
- const names = new Set(
- Array.from(world.get(Folders)!.values())
- .filter(folder => effectiveParentId(world, folder) === parentId)
- .map(folder => folder.name),
- );
- let name = base;
- for (let i = 2; names.has(name); i++) name = `${base} ${i}`;
- return name;
-}
-
-/**
- * The next free "New folder"/"New folder N" name among a parent's children.
- */
-export function nextFolderName(world: World, parentId: string | null): string {
- return uniqueFolderName(world, 'New folder', parentId);
-}
-
-/**
- * Finds a direct child folder of `parentId` by name (case-insensitive),
- * creating it when missing.
- */
-export function ensureFolder(world: World, name: string, parentId: string | null = null): Folder {
- const target = name.trim().toLowerCase();
- for (const folder of world.get(Folders)!.values()) {
- if (
- effectiveParentId(world, folder) === parentId &&
- folder.name.trim().toLowerCase() === target
- ) {
- return folder;
- }
- }
- return createFolder(world, name, parentId);
-}
-
-/**
- * Creates a folder under `parentId` (null = root).
- */
-export function createFolder(world: World, name: string, parentId: string | null = null): Folder {
- const folders = world.get(Folders)!;
- const folder: Folder = {
- id: allocateId(world),
- name,
- parentId: parentId !== null && folders.has(parentId) ? parentId : null,
- createdAt: new Date().toISOString(),
- };
-
- folders.set(folder.id, folder);
- return folder;
-}
-
-/**
- * Renames a folder. Returns the updated record, or null when nothing changed.
- */
-export function renameFolder(world: World, id: string, name: string): Folder | null {
- const folders = world.get(Folders)!;
- const folder = folders.get(id);
- if (!folder || folder.name === name) return null;
-
- const next = { ...folder, name };
- folders.set(id, next);
- return next;
-}
-
-/**
- * Moves assets into a folder (null = root). Ignores unknown asset ids and
- * unknown target folders. Returns the updated assets.
- */
-export function moveAssetsToFolder(world: World, ids: string[], folderId: string | null): Asset[] {
- if (folderId !== null && !world.get(Folders)!.has(folderId)) return [];
-
- const assets = world.get(Assets)!;
- const moved: Asset[] = [];
- for (const id of ids) {
- const asset = assets.get(id);
- if (!asset || (asset.folderId ?? null) === folderId) continue;
- const next = { ...asset, folderId };
- assets.set(id, next);
- moved.push(next);
- }
- return moved;
-}
-
-/**
- * Re-parents a folder (null = root). Returns the updated record, or null when
- * the move is invalid (unknown folder/target, or a cycle: dropping a folder
- * into itself or one of its descendants).
- */
-export function moveFolder(world: World, id: string, parentId: string | null): Folder | null {
- const folders = world.get(Folders)!;
- const folder = folders.get(id);
- if (!folder) return null;
- if (parentId === id) return null;
- if (parentId !== null) {
- if (!folders.has(parentId)) return null;
- if (isDescendantFolder(world, parentId, id)) return null;
- }
- if (folder.parentId === parentId) return folder;
-
- const next = { ...folder, parentId };
- folders.set(id, next);
- return next;
-}
-
-/**
- * Deletes a folder subtree with its contents from the in-memory model.
- * Returns the removed folder and asset ids so the app can persist the
- * deletions (and navigate out of the doomed subtree first).
- */
-export function deleteFolder(world: World, id: string): { folderIds: string[]; assetIds: string[] } {
- const folders = world.get(Folders)!;
- if (!folders.has(id)) return { folderIds: [], assetIds: [] };
-
- const subtree = new Set([id]);
- let grew = true;
- while (grew) {
- grew = false;
- for (const candidate of folders.values()) {
- if (candidate.parentId !== null && subtree.has(candidate.parentId) && !subtree.has(candidate.id)) {
- subtree.add(candidate.id);
- grew = true;
- }
- }
- }
-
- const assets = world.get(Assets)!;
- const assetIds = Array.from(assets.values())
- .filter(asset => asset.folderId != null && subtree.has(asset.folderId))
- .map(asset => asset.id);
-
- for (const assetId of assetIds) assets.delete(assetId);
- for (const folderId of subtree) folders.delete(folderId);
-
- return { folderIds: [...subtree], assetIds };
-}
diff --git a/packages/runtime/src/actions/index.ts b/packages/runtime/src/actions/index.ts
index 1ae92300..d3df237b 100644
--- a/packages/runtime/src/actions/index.ts
+++ b/packages/runtime/src/actions/index.ts
@@ -21,5 +21,4 @@ export * from './group';
export * from './frame';
export * from './clipboard';
export * from './assets';
-export * from './folders';
export * from './interactive';
diff --git a/packages/runtime/src/assets/types.ts b/packages/runtime/src/assets/types.ts
deleted file mode 100644
index ef3ac168..00000000
--- a/packages/runtime/src/assets/types.ts
+++ /dev/null
@@ -1,98 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// Asset and folder metadata model. The runtime only needs what resolves
-// durations, dimensions, and file bytes; the app layers persistence (zod
-// schemas, IndexedDB records) on top with structurally compatible shapes.
-
-/** Whatever hands out the asset's bytes: a FileSystemFileHandle in the
- * browser, a path-backed handle in Electron or node. */
-export interface AssetFileHandle {
- getFile(): Promise<File>;
-}
-
-/** Directory of numbered frame files backing a SEQUENCE asset. Typed to the
- * web FileSystemDirectoryHandle surface the decoders iterate. */
-export interface AssetDirectoryHandle {
- entries(): AsyncIterableIterator<[string, { kind: string; getFile?: () => Promise<File> }]>;
-}
-
-export type TranscriptWord = { text: string; start: number; end: number };
-
-export type WordGroup = TranscriptWord[];
-
-export type Transcript = { text: string; words: TranscriptWord[] }[];
-
-interface AssetBase {
- id: string;
- hash: string;
- createdAt: string;
- lastModified?: number;
- name: string;
- mimeType: string;
- size?: number;
- handle: AssetFileHandle;
- generationId?: string | null;
- generationKey?: string | null;
- folderId?: string | null;
-}
-
-export interface ImageAsset extends AssetBase {
- type: 'IMAGE';
- width: number;
- height: number;
-}
-
-export interface AudioAsset extends AssetBase {
- type: 'AUDIO';
- duration: number;
- sampleRate: number;
- channels: number;
- transcript?: Transcript;
-}
-
-export interface VideoAsset extends AssetBase {
- type: 'VIDEO';
- duration: number;
- width: number;
- height: number;
- frameRate: number;
- bitRate: number;
- sampleRate?: number;
- channels?: number;
- transcript?: Transcript;
-}
-
-export interface TranscriptAsset extends AssetBase {
- type: 'TRANSCRIPT';
-}
-
-export interface ScriptAsset extends AssetBase {
- type: 'SCRIPT';
-}
-
-// `handle` points to the first frame's file so generic preview code works.
-export interface SequenceAsset extends AssetBase {
- type: 'SEQUENCE';
- width: number;
- height: number;
- frameRate: number;
- duration: number;
- directoryHandle: AssetDirectoryHandle;
-}
-
-export type Asset =
- | ImageAsset
- | AudioAsset
- | VideoAsset
- | TranscriptAsset
- | ScriptAsset
- | SequenceAsset;
-
-export interface Folder {
- id: string;
- name: string;
- parentId: string | null;
- createdAt: string;
-}
diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts
index 7c8faa44..21ba3118 100644
--- a/packages/runtime/src/index.ts
+++ b/packages/runtime/src/index.ts
@@ -6,7 +6,6 @@ export * from './world/serialize';
export * from './world/store';
export * from './queries';
export * from './actions';
-export * from './assets/types';
export * from './systems';
export * from './math';
export * from './media';
diff --git a/packages/runtime/src/media/audio-peaks.ts b/packages/runtime/src/media/audio-peaks.ts
index 71450578..b4d8d4de 100644
--- a/packages/runtime/src/media/audio-peaks.ts
+++ b/packages/runtime/src/media/audio-peaks.ts
@@ -6,7 +6,7 @@ import { AudioSampleSink } from 'mediabunny';
import { getAudioTrack } from './audio';
-import type { AudioAsset, VideoAsset } from '../assets/types';
+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
const peakCache = new Map<string, Uint8ClampedArray>();
const inflight = new Map<string, Promise<Uint8ClampedArray | null>>();
diff --git a/packages/runtime/src/media/audio-sync.ts b/packages/runtime/src/media/audio-sync.ts
index 5fcca06e..c7e4c091 100644
--- a/packages/runtime/src/media/audio-sync.ts
+++ b/packages/runtime/src/media/audio-sync.ts
@@ -7,7 +7,7 @@ import { ALL_FORMATS, AudioSampleSink, BlobSource, Input } from 'mediabunny';
import { assert } from '../utils/assert';
import { getAssetFile } from '../actions/assets';
-import type { Asset } from '../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
// Envelope sample rate. 500 Hz → 2 ms resolution (well under one video frame),
// and keeps the FFT small enough for multi-minute clips. Parabolic
@@ -149,7 +149,7 @@ function prefixSums(x: Float32Array): { sum: Float64Array; sumSq: Float64Array }
const offsetCache = new Map<string, Promise<AudioSyncResult>>();
export function computeAudioSyncOffsetCached(a: Asset, b: Asset): Promise<AudioSyncResult> {
- const key = `${a.hash}:${b.hash}`;
+ const key = `${a.id}:${b.id}`;
const cached = offsetCache.get(key);
if (cached) return cached;
const promise = computeAudioSyncOffset(a, b);
diff --git a/packages/runtime/src/media/audio.ts b/packages/runtime/src/media/audio.ts
index faedff1f..df05a8dd 100644
--- a/packages/runtime/src/media/audio.ts
+++ b/packages/runtime/src/media/audio.ts
@@ -4,14 +4,14 @@
import { Input, ALL_FORMATS, BlobSource, AudioBufferSink, type WrappedAudioBuffer, type InputAudioTrack } from 'mediabunny';
-import { AssetId, AudioDecoderHandle, Assets } from '../traits';
+import { AssetId, AudioDecoderHandle } from '../traits';
import { AsyncMutex } from '../utils/async';
import { assert } from '../utils/assert';
-import { getAssetFile } from '../actions/assets';
+import { getAsset, getAssetFile } from '../actions/assets';
import { TimeStretcher } from './time-stretcher';
import type { Entity, World } from 'koota';
-import type { AudioAsset, VideoAsset } from '../assets/types';
+import type { AudioAsset, VideoAsset } from '@diffusionstudio/assets';
import type { AudioBus } from './audio-bus';
const audioTrackCache = new Map<string, Promise<InputAudioTrack | null>>();
@@ -263,7 +263,7 @@ export function resolveAudioDecoder(world: World, entity: Entity): ResolvedAudio
// Asset changed: reset old decoder and create a new one.
if (existing) existing.reset();
- const asset = world.get(Assets)?.get(assetId);
+ const asset = getAsset(world, assetId);
if (!asset || (asset.type !== 'AUDIO' && asset.type !== 'VIDEO')) return null;
if (asset.type === 'VIDEO' && !asset.channels) return null;
diff --git a/packages/runtime/src/media/caption/cascade.ts b/packages/runtime/src/media/caption/cascade.ts
index 33d9ab37..4953c035 100644
--- a/packages/runtime/src/media/caption/cascade.ts
+++ b/packages/runtime/src/media/caption/cascade.ts
@@ -12,7 +12,7 @@ import { createEntity } from '../../actions/entities';
import { appendChild } from '../../actions/hierarchy';
import type { Entity, World } from 'koota';
-import type { Asset } from '../../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
import type { CaptionDecoder } from './types';
const WIDTH = 800;
diff --git a/packages/runtime/src/media/caption/classic.ts b/packages/runtime/src/media/caption/classic.ts
index 8c03ea54..1bf029c6 100644
--- a/packages/runtime/src/media/caption/classic.ts
+++ b/packages/runtime/src/media/caption/classic.ts
@@ -12,7 +12,7 @@ import { createEntity } from '../../actions/entities';
import { appendChild } from '../../actions/hierarchy';
import type { Entity, World } from 'koota';
-import type { Asset } from '../../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
import type { CaptionDecoder } from './types';
export const CLASSIC_PRESET_WIDTH = 600;
diff --git a/packages/runtime/src/media/caption/guinea.ts b/packages/runtime/src/media/caption/guinea.ts
index 69c3504d..e16d3226 100644
--- a/packages/runtime/src/media/caption/guinea.ts
+++ b/packages/runtime/src/media/caption/guinea.ts
@@ -13,7 +13,7 @@ import { createEntity } from '../../actions/entities';
import { appendChild } from '../../actions/hierarchy';
import type { Entity, World } from 'koota';
-import type { Asset } from '../../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
import type { CaptionDecoder } from './types';
const WIDTH = 700;
diff --git a/packages/runtime/src/media/caption/index.ts b/packages/runtime/src/media/caption/index.ts
index 457608ca..053aeb1d 100644
--- a/packages/runtime/src/media/caption/index.ts
+++ b/packages/runtime/src/media/caption/index.ts
@@ -3,7 +3,8 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { CaptionType } from '../../constants';
-import { AssetId, Caption, TextStyle, CaptionDecoderHandle, Assets } from '../../traits';
+import { AssetId, Caption, TextStyle, CaptionDecoderHandle } from '../../traits';
+import { getAsset } from '../../actions/assets';
import { ClassicCaptionDecoder } from './classic';
import { CascadeCaptionDecoder } from './cascade';
import { SpotlightCaptionDecoder } from './spotlight';
@@ -13,7 +14,7 @@ import { GuineaCaptionDecoder } from './guinea';
import { StarkCaptionDecoder } from './stark';
import type { Entity, World } from 'koota';
-import type { Asset } from '../../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
import type { CaptionDecoder } from './types';
export type { CaptionDecoder } from './types';
@@ -66,7 +67,7 @@ export function resolveCaptionDecoder(world: World, entity: Entity): CaptionDeco
const typeChanged = existing != null;
existing?.dispose();
- const asset = world.get(Assets)?.get(assetId);
+ const asset = getAsset(world, assetId);
if (!asset) return null;
const decoder = createCaptionDecoder(captionType, asset);
diff --git a/packages/runtime/src/media/caption/paper.ts b/packages/runtime/src/media/caption/paper.ts
index cedceb5b..76f6fc61 100644
--- a/packages/runtime/src/media/caption/paper.ts
+++ b/packages/runtime/src/media/caption/paper.ts
@@ -12,7 +12,7 @@ import { createEntity } from '../../actions/entities';
import { appendChild } from '../../actions/hierarchy';
import type { Entity, World } from 'koota';
-import type { Asset } from '../../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
import type { CaptionDecoder } from './types';
const WIDTH = 700;
diff --git a/packages/runtime/src/media/caption/spotlight.ts b/packages/runtime/src/media/caption/spotlight.ts
index c90ffd25..52eb67d5 100644
--- a/packages/runtime/src/media/caption/spotlight.ts
+++ b/packages/runtime/src/media/caption/spotlight.ts
@@ -13,7 +13,7 @@ import { createEntity } from '../../actions/entities';
import { appendChild } from '../../actions/hierarchy';
import type { Entity, World } from 'koota';
-import type { Asset } from '../../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
import type { CaptionDecoder } from './types';
diff --git a/packages/runtime/src/media/caption/stark.ts b/packages/runtime/src/media/caption/stark.ts
index 452d310d..ad7ac963 100644
--- a/packages/runtime/src/media/caption/stark.ts
+++ b/packages/runtime/src/media/caption/stark.ts
@@ -12,7 +12,7 @@ import { createEntity } from '../../actions/entities';
import { appendChild } from '../../actions/hierarchy';
import type { Entity, World } from 'koota';
-import type { Asset } from '../../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
import type { CaptionDecoder } from './types';
const WIDTH = 700;
diff --git a/packages/runtime/src/media/caption/subtitles.ts b/packages/runtime/src/media/caption/subtitles.ts
index 8f9ab2a2..342d8ccd 100644
--- a/packages/runtime/src/media/caption/subtitles.ts
+++ b/packages/runtime/src/media/caption/subtitles.ts
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import type { Transcript } from '../../assets/types';
+import type { Transcript } from '@diffusionstudio/assets';
const TIMESTAMP_RE = /(?:\d{1,2}:)?\d{1,2}:\d{2}[.,]\d{1,3}/g;
diff --git a/packages/runtime/src/media/caption/types.ts b/packages/runtime/src/media/caption/types.ts
index c4f3a0e6..402036e0 100644
--- a/packages/runtime/src/media/caption/types.ts
+++ b/packages/runtime/src/media/caption/types.ts
@@ -4,7 +4,7 @@
import type { Entity, World } from 'koota';
import type { CaptionType } from '../../constants';
-import type { WordGroup } from '../../assets/types';
+import type { WordGroup } from '@diffusionstudio/assets';
export interface CaptionDecoder {
readonly type: CaptionType;
diff --git a/packages/runtime/src/media/caption/utils.ts b/packages/runtime/src/media/caption/utils.ts
index 57627a0d..a36db190 100644
--- a/packages/runtime/src/media/caption/utils.ts
+++ b/packages/runtime/src/media/caption/utils.ts
@@ -9,7 +9,7 @@ import { getAssetFile } from '../../actions/assets';
import { parseSubtitles } from './subtitles';
import type { Entity, World } from 'koota';
-import type { Asset, Transcript, WordGroup } from '../../assets/types';
+import type { Asset, Transcript, WordGroup } from '@diffusionstudio/assets';
export type TextRangeOverride = {
start: number;
diff --git a/packages/runtime/src/media/caption/whisper.ts b/packages/runtime/src/media/caption/whisper.ts
index 1ed00959..9bb04b95 100644
--- a/packages/runtime/src/media/caption/whisper.ts
+++ b/packages/runtime/src/media/caption/whisper.ts
@@ -12,7 +12,7 @@ import { createEntity } from '../../actions/entities';
import { appendChild } from '../../actions/hierarchy';
import type { Entity, World } from 'koota';
-import type { Asset } from '../../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
import type { CaptionDecoder } from './types';
const WIDTH = 1000;
diff --git a/packages/runtime/src/media/image.ts b/packages/runtime/src/media/image.ts
index 4bd45a3f..84c0a7e3 100644
--- a/packages/runtime/src/media/image.ts
+++ b/packages/runtime/src/media/image.ts
@@ -2,11 +2,11 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { AssetId, ImageDecoderHandle, Assets } from '../traits';
-import { getAssetFile } from '../actions/assets';
+import { AssetId, ImageDecoderHandle } from '../traits';
+import { getAsset, getAssetFile } from '../actions/assets';
import type { Entity, World } from 'koota';
-import type { ImageAsset } from '../assets/types';
+import type { ImageAsset } from '@diffusionstudio/assets';
export type DecodedImage = ImageBitmap | HTMLImageElement;
@@ -236,7 +236,7 @@ export function resolveImageDecoder(world: World, entity: Entity): ResolvedImage
// Asset changed: release the old decoder and acquire one for the new asset.
if (existing) existing.dispose();
- const asset = world.get(Assets)?.get(assetId);
+ const asset = getAsset(world, assetId);
if (!asset || asset.type !== 'IMAGE') return null;
const cache = getDecoderCache(world);
diff --git a/packages/runtime/src/media/sequence.ts b/packages/runtime/src/media/sequence.ts
index a646a475..bfd16ef5 100644
--- a/packages/runtime/src/media/sequence.ts
+++ b/packages/runtime/src/media/sequence.ts
@@ -2,12 +2,13 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { AssetId, SequenceDecoderHandle, Assets } from '../traits';
+import { AssetId, SequenceDecoderHandle } from '../traits';
+import { getAsset } from '../actions/assets';
import { assert } from '../utils/assert';
import { FrameCache } from './frame-cache';
import type { Entity, World } from 'koota';
-import type { SequenceAsset } from '../assets/types';
+import type { SequenceAsset } from '@diffusionstudio/assets';
const FRAME_EXTENSIONS = /\.(png|jpe?g|webp|avif|bmp|gif)$/i;
@@ -246,7 +247,7 @@ export function resolveSequenceDecoder(world: World, entity: Entity, hasCache =
const assetId = entity.get(AssetId)?.value;
if (!assetId) return null;
- const asset = world.get(Assets)?.get(assetId);
+ const asset = getAsset(world, assetId);
if (!asset || asset.type !== 'SEQUENCE') return null;
// Identity check (not just id) so decoders rebuild when asset metadata —
diff --git a/packages/runtime/src/media/transcode.ts b/packages/runtime/src/media/transcode.ts
index 36e3cc01..cce17f00 100644
--- a/packages/runtime/src/media/transcode.ts
+++ b/packages/runtime/src/media/transcode.ts
@@ -13,7 +13,7 @@ import { formatTimestamp } from '../utils/time';
import { getAssetFile } from '../actions/assets';
import type { StreamTargetChunk } from 'mediabunny';
-import type { Asset, VideoAsset, AudioAsset } from '../assets/types';
+import type { Asset, VideoAsset, AudioAsset } from '@diffusionstudio/assets';
type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
diff --git a/packages/runtime/src/media/video.ts b/packages/runtime/src/media/video.ts
index b8aa86bc..2abeb518 100644
--- a/packages/runtime/src/media/video.ts
+++ b/packages/runtime/src/media/video.ts
@@ -4,15 +4,15 @@
import { BlobSource, ALL_FORMATS, Input, InputVideoTrack, EncodedPacketSink, EncodedPacket, CanvasSink, type WrappedCanvas } from 'mediabunny';
-import { AssetId, VideoDecoderHandle, Assets, Mode } from '../traits';
+import { AssetId, VideoDecoderHandle, Mode } from '../traits';
import { assert } from '../utils/assert';
-import { getAssetFile } from '../actions/assets';
+import { getAsset, getAssetFile } from '../actions/assets';
import { FrameCache } from './frame-cache';
import { getKeyframeIndex } from './keyframe-index';
import type { Entity, World } from 'koota';
import type { KeyframeIndex } from './keyframe-index';
-import type { VideoAsset } from '../assets/types';
+import type { VideoAsset } from '@diffusionstudio/assets';
export type VideoBufferMode = 'discarded' | 'idle' | 'alive';
@@ -719,7 +719,7 @@ export function resolveVideoDecoder(world: World, entity: Entity): VideoDecoderI
// Asset changed — dispose old decoder and create a new one.
existing?.dispose();
- const asset = world.get(Assets)?.get(assetId);
+ const asset = getAsset(world, assetId);
if (!asset || asset.type !== 'VIDEO') return null;
const decoder = world.get(Mode)?.value === 'realtime'
diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.ts
index 9c675ad2..85554fb0 100644
--- a/packages/runtime/src/systems/render.ts
+++ b/packages/runtime/src/systems/render.ts
@@ -22,10 +22,11 @@ import {
BlendMode, Effect, AssetId, Transition, MixedCornerRadius,
LocalTransform, WorldTransform, Computed, Cache,
HtmlHostHandle, SurfaceHostHandle,
- Mode, Time, FrameRate, Camera, Background, RenderSurface, Assets,
+ Mode, Time, FrameRate, Camera, Background, RenderSurface,
HitRegions,
Root,
} from '../traits';
+import { getAsset } from '../actions/assets';
import { getParentNode } from '../queries/hierarchy';
import { getViewMatrix } from '../queries/camera';
import { colorToHex } from '../utils/color';
@@ -622,7 +623,7 @@ function renderWaveform(world: World, entity: Entity, fill: Entity): void {
const assetId = store(world, AssetId).value[fill.id()];
if (!assetId) return;
- const asset = world.get(Assets)?.get(assetId);
+ const asset = getAsset(world, assetId);
// Both AUDIO and VIDEO assets carry pre-computed peaks (video has audio).
if (asset?.type !== 'AUDIO' && asset?.type !== 'VIDEO') return;
diff --git a/packages/runtime/src/traits/world.ts b/packages/runtime/src/traits/world.ts
index 1d8aaadc..d99290c2 100644
--- a/packages/runtime/src/traits/world.ts
+++ b/packages/runtime/src/traits/world.ts
@@ -7,7 +7,7 @@ import { trait, type Entity, type World } from 'koota';
import { createLiveMounts } from '../utils/live-mounts';
import type { FontSource } from '../fonts/types';
-import type { Asset, Folder } from '../assets/types';
+import type { AssetLibrary } from '@diffusionstudio/assets';
import type { Quad } from '../math/aabb';
// Singleton state attached to the world itself (world.get/world.set).
@@ -64,15 +64,10 @@ export const AudioEngine = trait({
// Fonts registered for text layout.
export const Fonts = trait({ list: () => [] as FontSource[] });
-// Asset metadata by id (model only; the app persists and rehydrates it).
-export const Assets = trait(() => new Map<string, Asset>());
-
-// Asset-library folder tree by id (model only, like Assets).
-export const Folders = trait(() => new Map<string, Folder>());
-
-// Allocation state for the project's shared asset/folder id space (sqids
-// over a monotonic counter), so an id never refers to both kinds.
-export const AssetIds = trait({ counter: 0 });
+// The project's asset library, once the host has attached one (it needs the
+// project's file system): every asset an entity's AssetId can name, library
+// and transient alike. Null in a world without a project.
+export const Library = trait(() => null as AssetLibrary | null);
// Live JSX mount registry; running graphs advance once per frame.
export const Mounts = trait(() => createLiveMounts());
diff --git a/packages/runtime/src/utils/time.ts b/packages/runtime/src/utils/time.ts
index 75a9f019..60e9f3a7 100644
--- a/packages/runtime/src/utils/time.ts
+++ b/packages/runtime/src/utils/time.ts
@@ -4,12 +4,12 @@
import { CONONICAL_TIME_BASE, PaintType } from '../constants';
import {
- Audio, AssetId, Cache, Computed, Geometry, Paint, SourceIn, Assets, FrameRate,
+ Audio, AssetId, Cache, Computed, Geometry, Paint, SourceIn, Library, FrameRate,
} from '../traits';
import { getParentNode } from '../queries/hierarchy';
import type { Entity, World } from 'koota';
-import type { Asset } from '../assets/types';
+import type { Asset } from '@diffusionstudio/assets';
export function snapToMs(seconds: number) {
return Math.round(seconds * CONONICAL_TIME_BASE) / CONONICAL_TIME_BASE;
@@ -110,7 +110,7 @@ export function isPaintEntity(entity: Entity): boolean {
/** The asset backing a geometry: its own AssetId, or the first fill's. */
export function findGeometryAsset(world: World, entity: Entity): Asset | null {
- const assets = world.get(Assets);
+ const assets = world.get(Library);
if (!assets) return null;
const ownId = entity.get(AssetId)?.value;
@@ -138,7 +138,7 @@ export function findGeometryAsset(world: World, entity: Entity): Asset | null {
* absent (for a handler of its removal).
*/
export function findAssetDuration(world: World, entity: Entity, ignoreOwn = false): number | null {
- const assets = world.get(Assets);
+ const assets = world.get(Library);
if (!assets) return null;
const frameRate = world.get(FrameRate)?.value ?? 30;
diff --git a/packages/runtime/src/world/create-world.ts b/packages/runtime/src/world/create-world.ts
index 59c49ff5..1834b7d0 100644
--- a/packages/runtime/src/world/create-world.ts
+++ b/packages/runtime/src/world/create-world.ts
@@ -17,9 +17,7 @@ import {
RenderSurface,
AudioEngine,
Fonts,
- Assets,
- Folders,
- AssetIds,
+ Library,
Mounts,
FramePromises,
HitRegions,
@@ -41,9 +39,7 @@ export function createRuntimeWorld(projectId: string) {
RenderSurface,
AudioEngine,
Fonts,
- Assets,
- Folders,
- AssetIds,
+ Library,
Mounts,
FramePromises,
HitRegions,
diff --git a/reference/README.md b/reference/README.md
index 80e139fa..90cc1dee 100644
--- a/reference/README.md
+++ b/reference/README.md
@@ -56,7 +56,7 @@ How the surface is divided:
## Shared types
```ts
-Asset = { id: string; name: string; type: string } // asset ids are opaque strings (sqids)
+Asset = { id: string; path: string; type: string } // asset ids are content hashes; `path` is the library path
Time = number | `${number}f` | "MM:SS" // seconds, frames at 30 fps ("45f"), or a clock string; see jsx/timing.md
```
diff --git a/reference/jsx/generate.md b/reference/jsx/generate.md
index b3ad1718..834e119b 100644
--- a/reference/jsx/generate.md
+++ b/reference/jsx/generate.md
@@ -76,4 +76,4 @@ generate.audio(opts: {
## Caching and idempotency
-Generation is long-running and consumes credits, so results are **cached by content**: a key derived from the fully-resolved spec (`type`, `model`, `prompt`, resolved references, `seed`, …). Within a session, re-running an unchanged project reuses cached assets instead of regenerating, and two declarations with identical specs collapse to a single asset; changing any option produces a new asset. Set `seed` to make a spec reproducible. The cache is per-session and is not persisted across app restarts.
+Generation is long-running and consumes credits, so results are **cached by content**: a key derived from the fully-resolved spec (`type`, `model`, `prompt`, resolved references, `seed`, …). Within a session, re-running an unchanged project reuses cached assets instead of regenerating, and two declarations with identical specs collapse to a single asset; changing any option produces a new asset. Set `seed` to make a spec reproducible. Finished generations are stored in the project's library under `generated/` with their spec key, so the cache survives app restarts; deleting the asset from the library regenerates it on the next mount.
diff --git a/reference/jsx/media.md b/reference/jsx/media.md
index cc1f2fd4..ff64c529 100644
--- a/reference/jsx/media.md
+++ b/reference/jsx/media.md
@@ -2,12 +2,15 @@
`src` (on [`<video>`](./video.md), [`<image>`](./image.md), [`<audio>`](./audio.md), and the DOM [`<img>`](./html.md#images) inside `<html>`) accepts:
-- **Global path**: e.g. `"/Movies/video.mp4"`, resolved against the user's OS.
-- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, registered as a remote asset.
-- **Asset id**: e.g. `"gbHJ"`, an imported asset (discover ids with `dapi asset ls` or `dapi asset tree`).
-- **`AssetRef`**: the value returned by a `generate.*` declaration (see [generate.md](./generate.md)). The node is inserted immediately as a placeholder and its paint is attached once the asset has generated.
+- **Library path**: e.g. `"b-roll/drone.mp4"`, an asset of the project's library by its path there (folder + name, as shown in the asset panel and recorded in the project's `assets.yml`). The preferred form: it is portable and survives the file being relinked. A directory of numbered images is an image sequence and plays on `<video>` or `<image>`.
+- **Asset id**: e.g. `"9f3a2c1d7e4b8a01"`, the content hash of a library asset (`assets.yml` lists them).
+- **Global path**: e.g. `"/Movies/video.mp4"`, resolved against the user's OS. Not added to the library.
+- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, fetched on mount. Not added to the library.
+- **`AssetRef`**: the value returned by a `generate.*` declaration (see [generate.md](./generate.md)). The node is inserted immediately as a placeholder and its paint is attached once the asset has generated; the result is stored under the library's `generated/` folder.
-In `dapi node patch`, `src` takes a path, URL, or asset id; `generate.*` declarations are JSX-only.
+## The library
+
+A project's assets are recorded in `assets.yml` next to its entry file: for each asset, its library `path`, where its bytes are (`source`: the absolute path of a file imported from disk — imports never move or copy files — or a project-relative path under `assets/` for files the app produced), and what it was found to be. Folders are virtual: they are the prefixes of the paths. Renaming or moving an asset in the panel rewrites the `src` props that named it. Files placed under `assets/` by hand are taken into the library on the next load.
An `<img>` additionally takes a `data:` or `blob:` URL, which goes to the browser as it is.