Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Sharper mood board video frames and a visual frame picker

Claude7975950d91fa
diff --git a/client/src/components/moodBoard/VideoFramePicker.jsx b/client/src/components/moodBoard/VideoFramePicker.jsx
new file mode 100644
index 0000000000..e3425fe398
--- /dev/null
+++ b/client/src/components/moodBoard/VideoFramePicker.jsx
@@ -0,0 +1,218 @@
+/**
+ * Visual frame picker for a mood board video pin.
+ *
+ * A large preview with a zoomable filmstrip under it: drag across the strip to
+ * scrub, zoom in to spread the frames out, step one frame at a time, then tap
+ * "Pick" to keep the moment on screen. "Add" sends the picked times to the
+ * server, which extracts each one at source resolution.
+ */
+
+import { useEffect, useRef, useState } from 'react';
+import { ChevronLeft, ChevronRight, Pause, Play, Plus, X, ZoomIn, ZoomOut } from 'lucide-react';
+import Modal from '../ui/Modal';
+
+const MAX_PICKS = 24;
+const FRAME_SEC = 1 / 30;
+const ZOOMS = [1, 2, 4, 8];
+const THUMB_HEIGHT = 56;
+
+const formatFrameTime = (sec) => `${Math.floor(sec / 60)}:${(sec % 60).toFixed(2).padStart(5, '0')}`;
+
+// Draw evenly spaced thumbnails from a second, hidden video so the preview
+// never jumps while the strip fills in. `isCurrent` drops a superseded run.
+async function drawThumbnails(src, duration, count, isCurrent, onThumb) {
+ const video = document.createElement('video');
+ video.muted = true;
+ video.preload = 'auto';
+ video.playsInline = true;
+ video.src = src;
+ const canvas = document.createElement('canvas');
+ for (let i = 0; i < count && isCurrent(); i++) {
+ const time = ((i + 0.5) / count) * duration;
+ const seeked = new Promise((resolve) => { video.onseeked = resolve; video.onerror = resolve; });
+ video.currentTime = time;
+ await seeked;
+ if (!isCurrent() || !video.videoWidth) break;
+ canvas.height = THUMB_HEIGHT * 2;
+ canvas.width = Math.round((video.videoWidth / video.videoHeight) * canvas.height);
+ canvas.getContext('2d')?.drawImage(video, 0, 0, canvas.width, canvas.height);
+ onThumb(i, canvas.toDataURL('image/jpeg', 0.7));
+ }
+ video.removeAttribute('src');
+ video.load();
+}
+
+export default function VideoFramePicker({ src, poster, busy = false, onAdd, onClose }) {
+ const videoRef = useRef(null);
+ const stripRef = useRef(null);
+ const scrubbingRef = useRef(false);
+ const [duration, setDuration] = useState(0);
+ const [time, setTime] = useState(0);
+ const [playing, setPlaying] = useState(false);
+ const [zoom, setZoom] = useState(1);
+ const [thumbs, setThumbs] = useState([]);
+ const [picked, setPicked] = useState([]);
+
+ const thumbCount = Math.min(64, 8 * zoom);
+ useEffect(() => {
+ if (!duration) return undefined;
+ let active = true;
+ setThumbs(Array(thumbCount).fill(null));
+ drawThumbnails(src, duration, thumbCount, () => active, (i, url) => {
+ if (active) setThumbs((prev) => prev.map((t, j) => (j === i ? url : t)));
+ });
+ return () => { active = false; };
+ }, [src, duration, thumbCount]);
+
+ // Keep the playhead in view on a zoomed strip.
+ useEffect(() => {
+ const strip = stripRef.current;
+ if (!strip || !duration || scrubbingRef.current) return;
+ const x = (time / duration) * strip.scrollWidth;
+ if (x < strip.scrollLeft + 24 || x > strip.scrollLeft + strip.clientWidth - 24) strip.scrollLeft = x - strip.clientWidth / 2;
+ }, [time, duration, zoom]);
+
+ // A streamed file can report Infinity at first and its real length later.
+ const readDuration = (e) => setDuration(Number.isFinite(e.currentTarget.duration) ? e.currentTarget.duration : 0);
+ const seek = (t) => {
+ const video = videoRef.current;
+ if (!video || !duration) return;
+ const next = Math.min(Math.max(0, t), Math.max(0, duration - FRAME_SEC));
+ video.currentTime = next;
+ setTime(next);
+ };
+ const pause = () => { videoRef.current?.pause(); setPlaying(false); };
+ const togglePlay = () => {
+ const video = videoRef.current;
+ if (!video) return;
+ if (video.paused) { video.play()?.catch(() => setPlaying(false)); setPlaying(true); } else pause();
+ };
+ const seekFromPointer = (e) => {
+ const inner = e.currentTarget.getBoundingClientRect();
+ if (!inner.width) return;
+ seek(((e.clientX - inner.left) / inner.width) * duration);
+ };
+ const isPicked = (t) => picked.some((p) => Math.abs(p - t) < FRAME_SEC / 2);
+ const pick = () => {
+ const t = Math.round(time * 1000) / 1000;
+ if (isPicked(t) || picked.length >= MAX_PICKS) return;
+ setPicked((prev) => [...prev, t].sort((a, b) => a - b));
+ };
+
+ const button = 'min-h-[44px] min-w-[40px] inline-flex items-center justify-center rounded text-gray-300 hover:text-white disabled:opacity-40';
+ return (
+ <Modal open usePortal size="3xl" onClose={onClose} ariaLabel="Pick frames" panelClassName="bg-port-card border border-port-border rounded-xl p-3 sm:p-4 space-y-3">
+ <div className="flex items-center justify-between gap-2">
+ <h2 className="text-sm font-semibold text-white">Pick frames</h2>
+ <button type="button" onClick={onClose} aria-label="Close frame picker" className={button}>
+ <X className="w-4 h-4" aria-hidden="true" />
+ </button>
+ </div>
+
+ {/* eslint-disable-next-line jsx-a11y/media-has-caption -- reference clips have no caption track */}
+ <video
+ ref={videoRef}
+ src={src}
+ poster={poster || undefined}
+ playsInline
+ muted
+ preload="auto"
+ data-testid="frame-picker-video"
+ onLoadedMetadata={readDuration}
+ onDurationChange={readDuration}
+ onTimeUpdate={(e) => { if (!scrubbingRef.current) setTime(e.currentTarget.currentTime); }}
+ onPause={() => setPlaying(false)}
+ onClick={togglePlay}
+ className="w-full max-h-[50dvh] object-contain bg-black rounded"
+ />
+
+ <div className="flex items-center">
+ <button type="button" onClick={() => { pause(); seek(time - FRAME_SEC); }} aria-label="Back one frame" className={button}>
+ <ChevronLeft className="w-4 h-4" aria-hidden="true" />
+ </button>
+ <button type="button" onClick={togglePlay} aria-label={playing ? 'Pause' : 'Play'} className={button}>
+ {playing ? <Pause className="w-4 h-4" aria-hidden="true" /> : <Play className="w-4 h-4" aria-hidden="true" />}
+ </button>
+ <button type="button" onClick={() => { pause(); seek(time + FRAME_SEC); }} aria-label="Forward one frame" className={button}>
+ <ChevronRight className="w-4 h-4" aria-hidden="true" />
+ </button>
+ <span className="text-[11px] tabular-nums whitespace-nowrap text-gray-400">{formatFrameTime(time)} / {formatFrameTime(duration)}</span>
+ <span className="ml-auto inline-flex items-center">
+ <button type="button" onClick={() => setZoom((z) => ZOOMS[Math.max(0, ZOOMS.indexOf(z) - 1)])} disabled={zoom === ZOOMS[0]} aria-label="Zoom out of the filmstrip" className={button}>
+ <ZoomOut className="w-4 h-4" aria-hidden="true" />
+ </button>
+ <span className="text-[11px] tabular-nums text-gray-400 w-5 text-center">{zoom}×</span>
+ <button type="button" onClick={() => setZoom((z) => ZOOMS[Math.min(ZOOMS.length - 1, ZOOMS.indexOf(z) + 1)])} disabled={zoom === ZOOMS.at(-1)} aria-label="Zoom into the filmstrip" className={button}>
+ <ZoomIn className="w-4 h-4" aria-hidden="true" />
+ </button>
+ </span>
+ </div>
+
+ <div ref={stripRef} className="overflow-x-auto rounded border border-port-border bg-port-bg">
+ <div
+ role="slider"
+ tabIndex={0}
+ aria-label="Filmstrip"
+ aria-valuemin={0}
+ aria-valuemax={Math.round(duration * 1000) / 1000}
+ aria-valuenow={Math.round(time * 1000) / 1000}
+ aria-valuetext={formatFrameTime(time)}
+ data-testid="frame-picker-strip"
+ className="relative flex touch-none select-none cursor-pointer"
+ style={{ width: `${zoom * 100}%`, height: THUMB_HEIGHT }}
+ onPointerDown={(e) => { pause(); scrubbingRef.current = true; e.currentTarget.setPointerCapture?.(e.pointerId); seekFromPointer(e); }}
+ onPointerMove={(e) => { if (scrubbingRef.current) seekFromPointer(e); }}
+ onPointerUp={() => { scrubbingRef.current = false; }}
+ onPointerCancel={() => { scrubbingRef.current = false; }}
+ onKeyDown={(e) => {
+ if (e.key === 'ArrowLeft' || e.key === 'ArrowRight') { e.preventDefault(); pause(); seek(time + (e.key === 'ArrowLeft' ? -FRAME_SEC : FRAME_SEC)); }
+ }}
+ >
+ {thumbs.map((url, i) => (
+ <div key={i} className="flex-1 min-w-0 h-full bg-port-card border-r border-port-bg/60 overflow-hidden">
+ {url ? <img src={url} alt="" className="w-full h-full object-cover pointer-events-none" draggable={false} /> : null}
+ </div>
+ ))}
+ {duration ? picked.map((t) => (
+ <span key={t} className="absolute top-0 bottom-0 w-1 -ml-0.5 bg-port-success pointer-events-none" style={{ left: `${(t / duration) * 100}%` }} />
+ )) : null}
+ {duration ? (
+ <span className="absolute top-0 bottom-0 w-1 -ml-0.5 bg-white ring-1 ring-black/60 pointer-events-none" style={{ left: `${(time / duration) * 100}%` }} />
+ ) : null}
+ </div>
+ </div>
+
+ <div className="flex flex-wrap items-center gap-2">
+ <button
+ type="button"
+ onClick={pick}
+ disabled={!duration || isPicked(time) || picked.length >= MAX_PICKS}
+ className="min-h-[44px] inline-flex items-center gap-1.5 px-3 rounded bg-port-bg border border-port-border text-xs text-white hover:border-port-accent disabled:opacity-40"
+ >
+ <Plus className="w-3.5 h-3.5" aria-hidden="true" /> Pick this frame
+ </button>
+ {picked.map((t) => (
+ <span key={t} className="inline-flex items-center rounded-full border border-port-border bg-port-bg text-xs text-gray-300">
+ <button type="button" onClick={() => { pause(); seek(t); }} className="min-h-[36px] pl-3 pr-1 tabular-nums" aria-label={`Go to ${formatFrameTime(t)}`}>
+ {formatFrameTime(t)}
+ </button>
+ <button type="button" onClick={() => setPicked((prev) => prev.filter((p) => p !== t))} className="min-h-[36px] min-w-[36px] inline-flex items-center justify-center text-gray-500 hover:text-port-error" aria-label={`Remove ${formatFrameTime(t)}`}>
+ <X className="w-3 h-3" aria-hidden="true" />
+ </button>
+ </span>
+ ))}
+ </div>
+
+ <div className="flex justify-end">
+ <button
+ type="button"
+ onClick={() => onAdd(picked)}
+ disabled={!picked.length || busy}
+ className="min-h-[44px] px-4 rounded bg-port-accent text-white text-sm font-medium disabled:opacity-40"
+ >
+ {busy ? 'Adding…' : picked.length ? `Add ${picked.length} frame${picked.length === 1 ? '' : 's'} to board` : 'Add frames to board'}
+ </button>
+ </div>
+ </Modal>
+ );
+}
diff --git a/client/src/components/moodBoard/VideoFramePicker.test.jsx b/client/src/components/moodBoard/VideoFramePicker.test.jsx
new file mode 100644
index 0000000000..8f9df9ca70
--- /dev/null
+++ b/client/src/components/moodBoard/VideoFramePicker.test.jsx
@@ -0,0 +1,37 @@
+import { describe, it, expect, vi } from 'vitest';
+import { render, screen, fireEvent } from '@testing-library/react';
+import VideoFramePicker from './VideoFramePicker';
+
+function openWithDuration(onAdd) {
+ render(<VideoFramePicker src="/data/videos/clip.mp4" onAdd={onAdd} onClose={() => {}} />);
+ const video = screen.getByTestId('frame-picker-video');
+ Object.defineProperty(video, 'duration', { configurable: true, value: 10 });
+ fireEvent.loadedMetadata(video);
+}
+
+describe('VideoFramePicker', () => {
+ it('picks frames by stepping and keyboard scrubbing, then adds them in timeline order', () => {
+ const onAdd = vi.fn();
+ openWithDuration(onAdd);
+ const add = screen.getByRole('button', { name: 'Add frames to board' });
+ expect(add).toBeDisabled();
+
+ fireEvent.keyDown(screen.getByRole('slider', { name: 'Filmstrip' }), { key: 'ArrowRight' });
+ fireEvent.click(screen.getByRole('button', { name: 'Pick this frame' }));
+ // The same frame can't be picked twice.
+ expect(screen.getByRole('button', { name: 'Pick this frame' })).toBeDisabled();
+ fireEvent.click(screen.getByRole('button', { name: 'Back one frame' }));
+ fireEvent.click(screen.getByRole('button', { name: 'Pick this frame' }));
+
+ expect(screen.getByRole('button', { name: 'Go to 0:00.03' })).toBeInTheDocument();
+ fireEvent.click(screen.getByRole('button', { name: 'Add 2 frames to board' }));
+ expect(onAdd).toHaveBeenCalledWith([0, 0.033]);
+ });
+
+ it('removes a picked frame', () => {
+ openWithDuration(vi.fn());
+ fireEvent.click(screen.getByRole('button', { name: 'Pick this frame' }));
+ fireEvent.click(screen.getByRole('button', { name: 'Remove 0:00.00' }));
+ expect(screen.getByRole('button', { name: 'Add frames to board' })).toBeDisabled();
+ });
+});
diff --git a/client/src/pages/MoodBoardDetail.jsx b/client/src/pages/MoodBoardDetail.jsx
index 980879d9f1..b4d7003f18 100644
--- a/client/src/pages/MoodBoardDetail.jsx
+++ b/client/src/pages/MoodBoardDetail.jsx
@@ -10,8 +10,8 @@
*/
import { useEffect, useState, useCallback, useMemo, useRef } from 'react';
-import { useParams, useNavigate, Link } from 'react-router';
-import { ArrowLeft, ChevronDown, ImageIcon, FileText, Trash2, Plus, Save, Link2, Unlink, RefreshCw, Images, Film, Play, ScanEye, Copy, AtSign, Download, Sparkles, Clapperboard, Paintbrush, Wand2 } from 'lucide-react';
+import { useParams, useNavigate, useSearchParams, Link } from 'react-router';
+import { ArrowLeft, ChevronDown, ImageIcon, FileText, Trash2, Plus, Save, Link2, Unlink, RefreshCw, Images, Film, Play, ScanEye, Copy, AtSign, Download, Sparkles, Clapperboard, Paintbrush, Wand2, GalleryHorizontal } from 'lucide-react';
import PageSkeleton from '../components/ui/PageSkeleton';
import toast from '../components/ui/Toast';
import TabPills from '../components/ui/TabPills';
@@ -23,6 +23,7 @@ import MediaLightbox from '../components/media/MediaLightbox';
import MoodBoardGeneratePanel from '../components/moodBoard/MoodBoardGeneratePanel';
import MoodBoardStylePanel from '../components/moodBoard/MoodBoardStylePanel';
import MoodBoardCollagePanel from '../components/moodBoard/MoodBoardCollagePanel';
+import VideoFramePicker from '../components/moodBoard/VideoFramePicker';
import usePreviewRoute from '../hooks/usePreviewRoute';
import { copyToClipboard } from '../lib/clipboard';
import {
@@ -38,6 +39,7 @@ import {
importMoodBoardXPost,
localizeMoodBoardMedia,
extractMoodBoardItemFrames,
+ extractMoodBoardItemFramesAt,
renderMoodBoardItem,
} from '../services/api';
import socket from '../services/socket';
@@ -88,6 +90,14 @@ function MoodBoardEditor({ id }) {
// Per-video frame extraction: requested frame count + the item in flight.
const [frameCount, setFrameCount] = useState(4);
+ // The frame picker's video is in the URL (?frames=<itemId>), so it is reload-safe and linkable.
+ const [searchParams, setSearchParams] = useSearchParams();
+ const pickingFramesItemId = searchParams.get('frames');
+ const setPickingFramesItemId = useCallback((itemId) => setSearchParams((prev) => {
+ const next = new URLSearchParams(prev);
+ if (itemId) next.set('frames', itemId); else next.delete('frames');
+ return next;
+ }, { replace: !itemId }), [setSearchParams]);
const [extractingItemId, setExtractingItemId] = useState(null);
// Text note → image render (#10531): the note whose request is in flight.
@@ -184,6 +194,9 @@ function MoodBoardEditor({ id }) {
// when its `initialSource` identity changes, so a fresh object every render
// would wipe an in-flight run on unrelated re-renders. (Lives above the
// loading/not-found early returns — hooks must run unconditionally.)
+ const pickingFramesItem = pickingFramesItemId
+ ? ((Array.isArray(board?.items) ? board.items : []).find((it) => it.id === pickingFramesItemId && moodBoardItemVideoSrc(it)) || null)
+ : null;
const analyzeItem = analyzeItemId
? ((Array.isArray(board?.items) ? board.items : []).find((it) => it.id === analyzeItemId) || null)
: null;
@@ -353,6 +366,19 @@ function MoodBoardEditor({ id }) {
toast.success(res.added ? `Added ${res.added} frame${res.added === 1 ? '' : 's'} to the board` : 'Those frames are already on the board');
};
+ const handleAddPickedFrames = async (itemId, times) => {
+ if (!mountedRef.current || !times.length) return;
+ setExtractingItemId(itemId);
+ const res = await extractMoodBoardItemFramesAt(id, itemId, times, { silent: true })
+ .catch((err) => { toast.error(err?.message || 'Could not add those frames'); return null; });
+ if (!mountedRef.current) return;
+ setExtractingItemId(null);
+ if (!res?.board) return;
+ setBoard(res.board);
+ setPickingFramesItemId(null);
+ toast.success(res.added ? `Added ${res.added} frame${res.added === 1 ? '' : 's'} to the board` : 'Those frames are already on the board');
+ };
+
const handleRenderItem = async (itemId) => {
if (!mountedRef.current) return;
setRenderingItemId(itemId);
@@ -1002,6 +1028,15 @@ function MoodBoardEditor({ id }) {
>
<Clapperboard className={`w-3.5 h-3.5 ${extractingItemId === item.id ? 'animate-pulse' : ''}`} aria-hidden="true" />
</button>
+ <button
+ type="button"
+ onClick={() => setPickingFramesItemId(item.id)}
+ title="Pick frames on a timeline and add them to the board"
+ aria-label="Pick frames"
+ className="min-h-[44px] min-w-[44px] shrink-0 inline-flex items-center justify-center p-1 text-gray-500 hover:text-white transition-colors"
+ >
+ <GalleryHorizontal className="w-3.5 h-3.5" aria-hidden="true" />
+ </button>
</span>
) : null}
{item.type === 'text' ? (
@@ -1148,6 +1183,15 @@ function MoodBoardEditor({ id }) {
) : null}
</PromptFromMediaModal>
) : null}
+ {pickingFramesItem ? (
+ <VideoFramePicker
+ src={moodBoardItemVideoSrc(pickingFramesItem)}
+ poster={moodBoardItemSrc(pickingFramesItem)}
+ busy={extractingItemId === pickingFramesItem.id}
+ onAdd={(times) => handleAddPickedFrames(pickingFramesItem.id, times)}
+ onClose={() => setPickingFramesItemId(null)}
+ />
+ ) : null}
</div>
);
}
diff --git a/client/src/services/apiMoodBoard.js b/client/src/services/apiMoodBoard.js
index 445046087e..038666ef90 100644
--- a/client/src/services/apiMoodBoard.js
+++ b/client/src/services/apiMoodBoard.js
@@ -141,6 +141,14 @@ export const extractMoodBoardItemFrames = (id, itemId, count, options) =>
...options,
});
+// Extract the exact moments (seconds) picked in the frame picker.
+export const extractMoodBoardItemFramesAt = (id, itemId, times, options) =>
+ request(`/mood-boards/${encodeURIComponent(id)}/items/${encodeURIComponent(itemId)}/extract-frames`, {
+ method: 'POST',
+ body: JSON.stringify({ times }),
+ ...options,
+ });
+
// Render a text note into an image (#10531) on the install's default image
// backend. Resolves to `{ item, jobId }` once queued; the note becomes an image
// item when the job completes (`mood-board:item-render` follows it).
diff --git a/server/lib/moodBoardValidation.js b/server/lib/moodBoardValidation.js
index 9feebaf4bc..bef3650915 100644
--- a/server/lib/moodBoardValidation.js
+++ b/server/lib/moodBoardValidation.js
@@ -196,9 +196,12 @@ export const moodBoardCollageSchema = z.object({
cellSize: z.number().int().min(128).max(1024).optional().default(512),
}).strict();
+// Either `count` evenly spaced frames, or the exact `times` (seconds) picked
+// in the frame picker.
export const moodBoardExtractFramesSchema = z.object({
- count: z.number().int().min(1).max(24),
-}).strict();
+ count: z.number().int().min(1).max(24).optional(),
+ times: z.array(z.number().finite().min(0).max(86400)).min(1).max(24).optional(),
+}).strict().refine((v) => (v.count == null) !== (v.times == null), { message: 'Send either count or times' });
// Render a text note into an image (#10531). Both optional: an omitted backend
// renders on the install's default image backend.
diff --git a/server/lib/xPostMedia.js b/server/lib/xPostMedia.js
index 13e148205d..e3688488dd 100644
--- a/server/lib/xPostMedia.js
+++ b/server/lib/xPostMedia.js
@@ -78,23 +78,42 @@ export function buildSyndicationUrl(statusId) {
* both (X's own constraint), but this reads whichever is present without
* assuming which. `video.variants` may list multiple encodings (mp4 +
* HLS/`application/x-mpegURL`); only mp4 is downloadable via a plain fetch, so
- * the highest-bitrate mp4 variant is picked. Returns `{ images: [], video:
+ * the sharpest mp4 (by bitrate, then URL resolution) is picked. Returns `{ images: [], video:
* null }` for a malformed/empty payload rather than throwing — the caller
* decides that's "no media" (400), not a parse crash.
*/
+// Pixel count from a CDN path like `/vid/avc1/720x1280/…mp4`, 0 when absent.
+const variantPixels = (url) => {
+ const m = String(url).match(/\/(\d{2,5})x(\d{2,5})\//);
+ return m ? Number(m[1]) * Number(m[2]) : 0;
+};
+
+// The syndication `video.variants` list carries no bitrate (and lists the
+// smallest mp4 first); `mediaDetails[].video_info.variants` does. Rank every
+// mp4 by bitrate, then by the resolution in its URL, so the sharpest wins.
+function bestMp4(json) {
+ const candidates = [
+ ...(Array.isArray(json.video?.variants) ? json.video.variants : [])
+ .filter((v) => v?.type === 'video/mp4').map((v) => ({ url: v.src, bitrate: v.bitrate })),
+ ...(Array.isArray(json.mediaDetails) ? json.mediaDetails : [])
+ .flatMap((m) => (Array.isArray(m?.video_info?.variants) ? m.video_info.variants : []))
+ .filter((v) => v?.content_type === 'video/mp4').map((v) => ({ url: v.url, bitrate: v.bitrate })),
+ ].filter((v) => typeof v.url === 'string' && v.url);
+ if (!candidates.length) return null;
+ const rank = (v) => [Number(v.bitrate) || 0, variantPixels(v.url)];
+ return candidates.reduce((a, b) => {
+ const [ra, rb] = [rank(a), rank(b)];
+ return rb[0] > ra[0] || (rb[0] === ra[0] && rb[1] > ra[1]) ? b : a;
+ }).url;
+}
+
export function extractXPostMedia(json) {
if (!json || typeof json !== 'object') return { images: [], video: null };
const images = Array.isArray(json.photos)
? json.photos.map((p) => p?.url).filter((u) => typeof u === 'string' && u)
: [];
- let video = null;
- const variants = json.video?.variants;
- if (Array.isArray(variants)) {
- const mp4Variants = variants.filter((v) => v?.type === 'video/mp4' && typeof v.src === 'string' && v.src);
- if (mp4Variants.length) {
- const best = mp4Variants.reduce((a, b) => ((b.bitrate || 0) > (a.bitrate || 0) ? b : a));
- video = { url: best.src, poster: typeof json.video.poster === 'string' ? json.video.poster : null };
- }
- }
- return { images, video };
+ const url = bestMp4(json);
+ const poster = typeof json.video?.poster === 'string' ? json.video.poster
+ : (Array.isArray(json.mediaDetails) ? json.mediaDetails.find((m) => m?.video_info)?.media_url_https : null) || null;
+ return { images, video: url ? { url, poster } : null };
}
diff --git a/server/lib/xPostMedia.test.js b/server/lib/xPostMedia.test.js
index 2412b04f4e..26fbb403bf 100644
--- a/server/lib/xPostMedia.test.js
+++ b/server/lib/xPostMedia.test.js
@@ -111,6 +111,27 @@ describe('extractXPostMedia', () => {
expect(video).toEqual({ url: 'https://video.twimg.com/high.mp4', poster: 'https://pbs.twimg.com/poster.jpg' });
});
+ it('picks the sharpest mp4 when the syndication variants carry no bitrate', () => {
+ // video.variants lists the smallest mp4 first with no bitrate; mediaDetails has the bitrates.
+ const json = {
+ video: {
+ poster: 'https://pbs.twimg.com/poster.jpg',
+ variants: [
+ { type: 'application/x-mpegURL', src: 'https://video.twimg.com/pl/x.m3u8' },
+ { type: 'video/mp4', src: 'https://video.twimg.com/vid/avc1/320x568/a.mp4' },
+ { type: 'video/mp4', src: 'https://video.twimg.com/vid/avc1/720x1280/c.mp4' },
+ { type: 'video/mp4', src: 'https://video.twimg.com/vid/avc1/480x852/b.mp4' },
+ ],
+ },
+ };
+ expect(extractXPostMedia(json).video.url).toBe('https://video.twimg.com/vid/avc1/720x1280/c.mp4');
+ json.mediaDetails = [{ video_info: { variants: [
+ { content_type: 'video/mp4', url: 'https://video.twimg.com/vid/avc1/720x1280/c.mp4', bitrate: 2176000 },
+ { content_type: 'video/mp4', url: 'https://video.twimg.com/vid/avc1/1080x1920/d.mp4', bitrate: 10368000 },
+ ] } }];
+ expect(extractXPostMedia(json).video).toEqual({ url: 'https://video.twimg.com/vid/avc1/1080x1920/d.mp4', poster: 'https://pbs.twimg.com/poster.jpg' });
+ });
+
it('returns a null video when only a non-mp4 variant is present', () => {
const { video } = extractXPostMedia({
video: { variants: [{ type: 'application/x-mpegURL', src: 'https://video.twimg.com/x.m3u8' }] },
diff --git a/server/routes/moodBoard.test.js b/server/routes/moodBoard.test.js
index 734d61ace7..851c067f23 100644
--- a/server/routes/moodBoard.test.js
+++ b/server/routes/moodBoard.test.js
@@ -203,12 +203,15 @@ describe('mood-board routes', () => {
expect(bad.status).toBe(400);
});
- it('requires a count for per-item frame extraction', async () => {
+ it('takes a count or picked times for per-item frame extraction', async () => {
svc.extractItemFrames.mockResolvedValueOnce({ added: 4 });
const ok = await request(makeApp()).post('/api/mood-boards/mb-1/items/i1/extract-frames').send({ count: 4 });
expect(ok.status).toBe(200);
expect(svc.extractItemFrames).toHaveBeenCalledWith('mb-1', 'i1', { count: 4 });
expect((await request(makeApp()).post('/api/mood-boards/mb-1/items/i1/extract-frames').send({})).status).toBe(400);
+ expect((await request(makeApp()).post('/api/mood-boards/mb-1/items/i1/extract-frames').send({ times: [1.25, 3] })).status).toBe(200);
+ expect(svc.extractItemFrames).toHaveBeenLastCalledWith('mb-1', 'i1', { times: [1.25, 3] });
+ expect((await request(makeApp()).post('/api/mood-boards/mb-1/items/i1/extract-frames').send({ count: 2, times: [1] })).status).toBe(400);
});
it('queues a note render (202) on an optional backend pin', async () => {
diff --git a/server/services/moodBoard/collage.js b/server/services/moodBoard/collage.js
index 9e4bb2efdf..cd9a561425 100644
--- a/server/services/moodBoard/collage.js
+++ b/server/services/moodBoard/collage.js
@@ -50,7 +50,7 @@ async function requireBoard(boardId) {
* Returns the written paths in timeline order. Throws a 422 when ffmpeg is
* missing, the file is unreadable, or no frame could be decoded.
*/
-async function sampleVideoFrames(videoFile, count, outDir, prefix) {
+async function sampleVideoFrames(videoFile, count, outDir, prefix, picked = null) {
const videoPath = safeUnder(PATHS.videos, videoFile);
if (!videoPath) throw new ServerError('Invalid video reference', { status: 400, code: 'VALIDATION_ERROR' });
const ffmpeg = await findFfmpeg();
@@ -58,40 +58,60 @@ async function sampleVideoFrames(videoFile, count, outDir, prefix) {
const duration = await probeVideoDuration(videoPath);
if (!duration) throw new ServerError('Could not read that video (missing or unreadable)', { status: 422, code: 'VIDEO_UNREADABLE' });
- const times = frameSampleTimes(duration, count);
+ // Picked times are clamped just inside the end so the last frame still decodes.
+ const times = picked ? picked.map((t) => Math.min(Math.max(0, t), Math.max(0, duration - 0.05))) : frameSampleTimes(duration, count);
const paths = [];
for (let i = 0; i < times.length; i++) {
const out = join(outDir, `${prefix}${i + 1}.jpg`);
+ // Input seek decodes forward from the prior keyframe to the exact time;
+ // the frame is written at source resolution at JPEG's top quality.
const result = await runFfmpegProcess({
bin: ffmpeg,
- args: ['-ss', times[i].toFixed(3), '-i', videoPath, '-frames:v', '1', '-q:v', '3', '-y', out],
+ args: ['-ss', times[i].toFixed(3), '-i', videoPath, '-frames:v', '1', '-q:v', '1', '-qmin', '1', '-y', out],
stderrTailBytes: 0,
});
- if (result.ok) paths.push(out);
+ if (result.ok) paths.push({ path: out, time: times[i] });
}
if (!paths.length) throw new ServerError('No frames could be extracted from that video', { status: 422, code: 'FRAME_EXTRACT_FAILED' });
return paths;
}
/** Copy sampled frames into the gallery under stable, content-keyed names. */
-async function storeFramesInGallery(paths, videoFile, count) {
+async function storeFramesInGallery(frames, videoFile, count) {
await ensureDir(PATHS.images);
const key = createHash('sha1').update(`${videoFile}|${count}`).digest('hex').slice(0, 12);
const names = [];
- for (let i = 0; i < paths.length; i++) {
+ for (let i = 0; i < frames.length; i++) {
const name = `board-frame-${key}-${i + 1}of${count}.jpg`;
- await atomicWrite(join(PATHS.images, name), await readFile(paths[i]));
+ await atomicWrite(join(PATHS.images, name), await readFile(frames[i].path));
names.push(name);
}
return names;
}
+// Picked frames are keyed by their time (to the millisecond), so picking the
+// same moment twice dedupes and different picks never overwrite each other.
+async function storePickedFramesInGallery(frames, videoFile) {
+ await ensureDir(PATHS.images);
+ const key = createHash('sha1').update(videoFile).digest('hex').slice(0, 12);
+ const names = [];
+ for (const frame of frames) {
+ const name = `board-frame-${key}-at${Math.round(frame.time * 1000)}ms.jpg`;
+ await atomicWrite(join(PATHS.images, name), await readFile(frame.path));
+ names.push(name);
+ }
+ return names;
+}
+
+const clock = (sec) => `${Math.floor(sec / 60)}:${(sec % 60).toFixed(2).padStart(5, '0')}`;
+
/**
- * Extract `count` frames from one video pin and append them to the board as
- * image items. Re-running with the same count is a no-op (source-deduped).
+ * Extract frames from one video pin and append them to the board as image
+ * items: `count` evenly spaced frames, or the exact `times` (seconds) picked in
+ * the frame picker. Re-running the same request is a no-op (source-deduped).
* Returns `{ board, added, frames }`.
*/
-export async function extractItemFrames(boardId, itemId, { count }) {
+export async function extractItemFrames(boardId, itemId, { count, times }) {
const board = await requireBoard(boardId);
const item = (board.items || []).find((it) => it.id === itemId);
if (!item) throw new ServerError('Item not found', { status: 404, code: 'NOT_FOUND' });
@@ -100,13 +120,15 @@ export async function extractItemFrames(boardId, itemId, { count }) {
const tmp = await mkdtemp(join(tmpdir(), 'portos-board-frames-'));
try {
- const paths = await sampleVideoFrames(file, count, tmp, 'f');
- const names = await storeFramesInGallery(paths, file, count);
+ const picked = Array.isArray(times) ? [...new Set(times.map((t) => Math.round(t * 1000) / 1000))].sort((a, b) => a - b) : null;
+ const frames = await sampleVideoFrames(file, picked ? picked.length : count, tmp, 'f', picked);
+ const names = picked ? await storePickedFramesInGallery(frames, file) : await storeFramesInGallery(frames, file, count);
+ const label = (i) => (picked ? `frame at ${clock(frames[i].time)}` : `frame ${i + 1}/${count}`);
const imported = names.map((name, i) => ({
type: 'image',
imageUrl: `/data/images/${name}`,
- source: `frame:${file}#${i + 1}of${count}`,
- caption: item.caption ? `${item.caption} (frame ${i + 1}/${count})` : `Frame ${i + 1}/${count} of ${file}`,
+ source: picked ? `frame:${file}@${Math.round(frames[i].time * 1000)}ms` : `frame:${file}#${i + 1}of${count}`,
+ caption: item.caption ? `${item.caption} (${label(i)})` : `${label(i)[0].toUpperCase()}${label(i).slice(1)} of ${file}`,
}));
// The append first names the frame files, so it commits under the backup lease (#9982).
const { board: next, added } = await withBackupAssetPublication(() => store.appendImportedItems(boardId, imported));
@@ -158,7 +180,7 @@ export async function composeBoardCollage(boardId, { framesPerVideo, addFramesTo
return [];
});
if (!frames.length) { skipped += 1; continue; }
- sources.push(...frames);
+ sources.push(...frames.map((f) => f.path));
if (addFramesToBoard) {
const names = await storeFramesInGallery(frames, file, framesPerVideo);
names.forEach((name, i) => imported.push({
diff --git a/server/services/moodBoard/collage.test.js b/server/services/moodBoard/collage.test.js
new file mode 100644
index 0000000000..571073d7f2
--- /dev/null
+++ b/server/services/moodBoard/collage.test.js
@@ -0,0 +1,64 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import sharp from 'sharp';
+
+const state = vi.hoisted(() => ({ root: null, board: null }));
+vi.mock('../../lib/fileUtils.js', async (importOriginal) => {
+ const original = await importOriginal();
+ const { mkdtemp } = await import('node:fs/promises');
+ const { tmpdir } = await import('node:os');
+ const { join } = await import('node:path');
+ state.root = await mkdtemp(join(tmpdir(), 'board-frames-test-'));
+ return { ...original, PATHS: { ...original.PATHS, data: state.root, videos: join(state.root, 'videos'), images: join(state.root, 'images') } };
+});
+vi.mock('./db.js', () => ({
+ getBoard: async () => state.board,
+ appendImportedItems: async (_id, imported) => {
+ const seen = new Set(state.board.items.map((it) => it.source));
+ const fresh = imported.filter((it) => !seen.has(it.source));
+ state.board = { ...state.board, items: [...state.board.items, ...fresh] };
+ return { board: state.board, added: fresh.length };
+ },
+}));
+vi.mock('../sharing/recordEvents.js', () => ({ emitRecordUpdated: vi.fn() }));
+
+const { extractItemFrames } = await import('./collage.js');
+const { findFfmpeg } = await import('../../lib/ffmpeg.js');
+const exec = promisify(execFile);
+let ffmpeg = null;
+
+beforeAll(async () => {
+ ffmpeg = await findFfmpeg();
+ if (!ffmpeg) return;
+ await mkdir(join(state.root, 'videos'), { recursive: true });
+ await exec(ffmpeg, ['-y', '-f', 'lavfi', '-i', 'testsrc=s=640x360:r=24:d=2', '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-an', join(state.root, 'videos', 'clip.mp4')]);
+}, 30000);
+afterAll(() => rm(state.root, { recursive: true, force: true }));
+beforeEach(() => {
+ state.board = { id: 'mb-1', items: [{ id: 'v1', type: 'video', mediaKey: 'video:clip.mp4', caption: null }] };
+});
+
+describe('extractItemFrames', () => {
+ it('adds the picked moments once each, in timeline order, at source resolution', async (context) => {
+ if (!ffmpeg) context.skip();
+ const res = await extractItemFrames('mb-1', 'v1', { times: [1.5, 0.25, 0.25] });
+ expect(res.added).toBe(2);
+ const frames = state.board.items.filter((it) => it.type === 'image');
+ expect(frames.map((it) => it.source)).toEqual(['frame:clip.mp4@250ms', 'frame:clip.mp4@1500ms']);
+ expect(frames[0].caption).toBe('Frame at 0:00.25 of clip.mp4');
+ const meta = await sharp(join(state.root, 'images', frames[1].imageUrl.split('/').pop())).metadata();
+ expect([meta.width, meta.height]).toEqual([640, 360]);
+ // Picking the same moment again adds nothing; a new pick adds one more.
+ expect((await extractItemFrames('mb-1', 'v1', { times: [1.5, 1] })).added).toBe(1);
+ });
+
+ it('still samples evenly spaced frames from a count', async (context) => {
+ if (!ffmpeg) context.skip();
+ const res = await extractItemFrames('mb-1', 'v1', { count: 3 });
+ expect(res.added).toBe(3);
+ expect(state.board.items.at(-1).source).toBe('frame:clip.mp4#3of3');
+ });
+});