Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor media selection and enhance prompt input component

konstantin-paulus2367cb78daa0
diff --git a/apps/web/src/components/genai/prompt-input.tsx b/apps/web/src/components/genai/prompt-input.tsx
index 39b75f69..5fd46d02 100644
--- a/apps/web/src/components/genai/prompt-input.tsx
+++ b/apps/web/src/components/genai/prompt-input.tsx
@@ -468,7 +468,7 @@ export function PromptInput(props: PromptInputProps) {
return (
<div
- class="absolute w-xl bottom-16 z-10 mx-auto left-1/2 -translate-x-1/2 flex flex-col gap-2 rounded-xl border border-border bg-background p-2"
+ class="absolute w-xl max-w-[calc(100%-2rem)] bottom-16 z-10 mx-auto left-1/2 -translate-x-1/2 flex flex-col gap-2 rounded-xl border border-border bg-background p-2"
onDragOver={handleDragOver}
onDragEnter={handleDragEnter}
onDragLeave={handleDragLeave}
diff --git a/apps/web/src/components/genai/selection.ts b/apps/web/src/components/genai/selection.ts
index 1cc4fac6..df6a01c9 100644
--- a/apps/web/src/components/genai/selection.ts
+++ b/apps/web/src/components/genai/selection.ts
@@ -2,55 +2,79 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// What the prompt box has to work with. Two readings of the selection, for
-// two different questions:
-//
-// `imageNodes`/`videoNodes` are what the element *is*, which is what a
-// transform applies to — a picture that is still generating is still a
-// picture, and `removeBackground` can be asked of it before it exists.
-//
-// `images`/`videos` are what the element currently *shows*, which is what a
-// reference needs: a library asset with a path to name it by.
-
import { createMemo } from "solid-js";
-import { AssetId } from "@diffusionstudio/runtime";
-import { authoredElement } from "@diffusionstudio/reconciler";
+import { AssetId, Cache, Paint, PaintType, getIntrinsicPaint } from "@diffusionstudio/runtime";
import { useSelection } from "@/engine/hooks";
import { useLibrary } from "@/engine/library";
import type { Asset } from "@diffusionstudio/assets";
import type { Entity } from "koota";
-/** A selected node and the library asset it is bound to. */
-export interface BoundNode {
+/** The kinds of paint the prompt box has anything to say about. */
+type MediaPaint = PaintType.IMAGE | PaintType.VIDEO;
+
+/** A selected node and the element its source props live on. */
+export interface SelectedNode {
entity: Entity;
+ source: Entity;
+ paint?: MediaPaint;
+}
+
+/** A selected node and the library asset its source element is bound to. */
+export interface BoundNode extends SelectedNode {
asset: Asset;
}
+function isMediaPaint(paint: PaintType | undefined): paint is MediaPaint {
+ return paint === PaintType.IMAGE || paint === PaintType.VIDEO;
+}
+
+/**
+ * Where `entity` keeps its source: itself when its own paint is the media,
+ * else its first media fill. A node that paints no media at all is still its
+ * own source element — an `<audio>` carries its `src` like any other.
+ */
+function resolve(entity: Entity): SelectedNode {
+ const intrinsic = getIntrinsicPaint(entity);
+ if (isMediaPaint(intrinsic)) return { entity, source: entity, paint: intrinsic };
+
+ for (const fill of entity.get(Cache)?.fills ?? []) {
+ const paint = fill.get(Paint)?.value;
+ if (isMediaPaint(paint)) return { entity, source: fill, paint };
+ }
+
+ return { entity, source: entity };
+}
+
export function useMediaSelection() {
const library = useLibrary();
const { nodes } = useSelection();
- const tagged = (tag: string) =>
- createMemo(() => nodes().filter((entity) => authoredElement(entity)?.tag === tag));
+ const selected = createMemo(() => nodes().map(resolve));
+
+ const painted = (paint: MediaPaint) =>
+ createMemo(() => selected().filter((node) => node.paint === paint).map((node) => node.source));
+
+ const imageNodes = painted(PaintType.IMAGE);
+ const videoNodes = painted(PaintType.VIDEO);
- const imageNodes = tagged("image");
- const videoNodes = tagged("video");
+ /** Every selected node as the element its source props live on. */
+ const sources = createMemo(() => selected().map((node) => node.source));
const bound = createMemo(() => {
const lib = library();
if (!lib) return [];
const entries: BoundNode[] = [];
- for (const entity of nodes()) {
- const id = entity.get(AssetId)?.value;
+ for (const node of selected()) {
+ const id = node.source.get(AssetId)?.value;
const asset = id ? lib.get(id) : undefined;
- if (asset) entries.push({ entity, asset });
+ if (asset) entries.push({ ...node, asset });
}
return entries;
});
const images = createMemo(() => bound().filter((entry) => entry.asset.type === "IMAGE"));
- return { bound, images, imageNodes, videoNodes };
+ return { bound, images, imageNodes, videoNodes, sources };
}
diff --git a/apps/web/src/components/genai/use-generation-records.ts b/apps/web/src/components/genai/use-generation-records.ts
index 2b91461e..766e99ee 100644
--- a/apps/web/src/components/genai/use-generation-records.ts
+++ b/apps/web/src/components/genai/use-generation-records.ts
@@ -5,7 +5,6 @@
import { createMemo, createResource } from "solid-js";
import { getAssetSpec, isAssetRef, isTransformSpec } from "@diffusionstudio/jsx";
import { authoredElement } from "@diffusionstudio/reconciler";
-import { useSelection } from "@/engine/hooks";
import { useLibrary } from "@/engine/library";
import { supabase } from "@/lib/supabase";
import { useMediaSelection } from "./selection";
@@ -113,15 +112,14 @@ function generationUnder(ref: AssetRef): GenerateSpec | undefined {
export function useGenerationRecords() {
const library = useLibrary();
- const { nodes } = useSelection();
- const { bound } = useMediaSelection();
+ const { bound, sources } = useMediaSelection();
// What the selected elements declare their source to be. An element made by
// the prompt box carries the whole spec, so this answers before the asset
// exists — and without asking the server what it was asked for. Transforms
// are looked through: the generation under them is what the box made.
const declarations = createMemo(() =>
- nodes()
+ sources()
.map((entity) => authoredElement(entity)?.props.src)
.filter(isAssetRef)
.map(generationUnder)
diff --git a/apps/web/src/components/sidebar-left/partial-asset-item.tsx b/apps/web/src/components/sidebar-left/partial-asset-item.tsx
index 309a56a0..4e990c92 100644
--- a/apps/web/src/components/sidebar-left/partial-asset-item.tsx
+++ b/apps/web/src/components/sidebar-left/partial-asset-item.tsx
@@ -73,8 +73,8 @@ export function PartialAssetItem(props: PartialAssetItemProps) {
classList={{ "animate-pulse": !failed() }}
>
<div class="absolute inset-0 flex items-center justify-center text-muted-foreground">
- <Show when={failed()} fallback={<Icon name="spinner-loader" class="size-5 animate-spin" />}>
- <Icon name="alert-warning" class="size-5 text-destructive" />
+ <Show when={failed()} fallback={<Icon name="spinner-loader" class="size-6 animate-spin" />}>
+ <Icon name="alert-warning" class="size-6 text-destructive" />
</Show>
</div>
</TooltipTrigger>
diff --git a/apps/web/src/components/ui/asset-thumbnail.tsx b/apps/web/src/components/ui/asset-thumbnail.tsx
index 7fd8eada..ae16a82e 100644
--- a/apps/web/src/components/ui/asset-thumbnail.tsx
+++ b/apps/web/src/components/ui/asset-thumbnail.tsx
@@ -34,6 +34,37 @@ type ThumbnailSize = {
height: number;
};
+/**
+ * The widths a thumbnail is derived at. A request is rounded up to one of
+ * these so the cache keeps a handful of variants rather than one per layout,
+ * and `DEFAULT_THUMBNAIL_WIDTH` is among them: the width the asset bar asks
+ * for is the one stored without a variant, and the small boxes mostly land on
+ * it too.
+ */
+const THUMBNAIL_WIDTHS = [64, 128, DEFAULT_THUMBNAIL_WIDTH, 600, 1200];
+
+/**
+ * How wide to derive for a box `size` CSS pixels across. Two things the box
+ * itself does not say: the display's pixels are not CSS pixels, and the
+ * picture is cropped to fill (`object-cover`), so a 16:9 asset in a square
+ * box is scaled by its *height* and its width overflows. Both are upscales of
+ * the derived bitmap, and they multiply — a 64px-wide thumbnail in a 64px
+ * square is drawn at about 3.5× on a retina screen, which is the blur.
+ *
+ * Never wider than the asset itself: there is nothing above its own
+ * resolution to derive.
+ */
+function sourceWidth(asset: Asset, size?: ThumbnailSize): number {
+ if (!size) return DEFAULT_THUMBNAIL_WIDTH;
+
+ const dpr = typeof window === 'undefined' ? 1 : window.devicePixelRatio || 1;
+ const aspect = asset.width && asset.height ? asset.width / asset.height : 1;
+ const needed = dpr * Math.max(size.width, size.height * aspect);
+
+ const width = THUMBNAIL_WIDTHS.find(candidate => candidate >= needed) ?? THUMBNAIL_WIDTHS.at(-1)!;
+ return asset.width ? Math.min(width, Math.ceil(asset.width)) : width;
+}
+
/**
* The thumbnail of an image or video: from the library's cache when there is
* one (kept in the project's `cache/` across sessions), derived on the spot
@@ -140,7 +171,7 @@ type AssetThumbnailProps = {
}
export function AssetThumbnail(props: AssetThumbnailProps) {
- const width = () => props.size?.width ?? DEFAULT_THUMBNAIL_WIDTH;
+ const width = () => sourceWidth(props.asset, props.size);
return (
<div class={cx('relative', props.class)} draggable={props.draggable}>