Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Sharper mood board video frames and a visual frame picker
Claude7975950d91fa
diff --git a/client/src/components/moodBoard/VideoFramePicker.jsx b/client/src/components/moodBoard/VideoFramePicker.jsxnew file mode 100644index 0000000000..e3425fe398--- /dev/null+++ b/client/src/components/moodBoard/VideoFramePicker.jsx@@ -0,0 +1,218 @@+/**+ * Visual frame picker for a mood board video pin.+ *+ * A large preview with a zoomable filmstrip under it: drag across the strip to+ * scrub, zoom in to spread the frames out, step one frame at a time, then tap+ * "Pick" to keep the moment on screen. "Add" sends the picked times to the+ * server, which extracts each one at source resolution.+ */++import { useEffect, useRef, useState } from 'react';+import { ChevronLeft, ChevronRight, Pause, Play, Plus, X, ZoomIn, ZoomOut } from 'lucide-react';+import Modal from '../ui/Modal';++const MAX_PICKS = 24;+const FRAME_SEC = 1 / 30;+const ZOOMS = [1, 2, 4, 8];+const THUMB_HEIGHT = 56;++const formatFrameTime = (sec) => `${Math.floor(sec / 60)}:${(sec % 60).toFixed(2).padStart(5, '0')}`;++// Draw evenly spaced thumbnails from a second, hidden video so the preview+// never jumps while the strip fills in. `isCurrent` drops a superseded run.+async function drawThumbnails(src, duration, count, isCurrent, onThumb) {+ const video = document.createElement('video');+ video.muted = true;+ video.preload = 'auto';+ video.playsInline = true;+ video.src = src;+ const canvas = document.createElement('canvas');+ for (let i = 0; i < count && isCurrent(); i++) {+ const time = ((i + 0.5) / count) * duration;+ const seeked = new Promise((resolve) => { video.onseeked = resolve; video.onerror = resolve; });+ video.currentTime = time;+ await seeked;+ if (!isCurrent() || !video.videoWidth) break;+ canvas.height = THUMB_HEIGHT * 2;+ canvas.width = Math.round((video.videoWidth / video.videoHeight) * canvas.height);+ canvas.getContext('2d')?.drawImage(video, 0, 0, canvas.width, canvas.height);+ onThumb(i, canvas.toDataURL('image/jpeg', 0.7));+ }+ video.removeAttribute('src');+ video.load();+}++export default function VideoFramePicker({ src, poster, busy = false, onAdd, onClose }) {+ const videoRef = useRef(null);+ const stripRef = useRef(null);+ const scrubbingRef = useRef(false);+ const [duration, setDuration] = useState(0);+ const [time, setTime] = useState(0);+ const [playing, setPlaying] = useState(false);+ const [zoom, setZoom] = useState(1);+ const [thumbs, setThumbs] = useState([]);+ const [picked, setPicked] = useState([]);++ const thumbCount = Math.min(64, 8 * zoom);+ useEffect(() => {+ if (!duration) return undefined;+ let active = true;+ setThumbs(Array(thumbCount).fill(null));+ drawThumbnails(src, duration, thumbCount, () => active, (i, url) => {+ if (active) setThumbs((prev) => prev.map((t, j) => (j === i ? url : t)));+ });+ return () => { active = false; };+ }, [src, duration, thumbCount]);++ // Keep the playhead in view on a zoomed strip.+ useEffect(() => {+ const strip = stripRef.current;+ if (!strip || !duration || scrubbingRef.current) return;+ const x = (time / duration) * strip.scrollWidth;+ if (x < strip.scrollLeft + 24 || x > strip.scrollLeft + strip.clientWidth - 24) strip.scrollLeft = x - strip.clientWidth / 2;+ }, [time, duration, zoom]);++ // A streamed file can report Infinity at first and its real length later.+ const readDuration = (e) => setDuration(Number.isFinite(e.currentTarget.duration) ? e.currentTarget.duration : 0);+ const seek = (t) => {+ const video = videoRef.current;+ if (!video || !duration) return;+ const next = Math.min(Math.max(0, t), Math.max(0, duration - FRAME_SEC));+ video.currentTime = next;+ setTime(next);+ };+ const pause = () => { videoRef.current?.pause(); setPlaying(false); };+ const togglePlay = () => {+ const video = videoRef.current;+ if (!video) return;+ if (video.paused) { video.play()?.catch(() => setPlaying(false)); setPlaying(true); } else pause();+ };+ const seekFromPointer = (e) => {+ const inner = e.currentTarget.getBoundingClientRect();+ if (!inner.width) return;+ seek(((e.clientX - inner.left) / inner.width) * duration);+ };+ const isPicked = (t) => picked.some((p) => Math.abs(p - t) < FRAME_SEC / 2);+ const pick = () => {+ const t = Math.round(time * 1000) / 1000;+ if (isPicked(t) || picked.length >= MAX_PICKS) return;+ setPicked((prev) => [...prev, t].sort((a, b) => a - b));+ };++ const button = 'min-h-[44px] min-w-[40px] inline-flex items-center justify-center rounded text-gray-300 hover:text-white disabled:opacity-40';+ return (+ <Modal open usePortal size="3xl" onClose={onClose} ariaLabel="Pick frames" panelClassName="bg-port-card border border-port-border rounded-xl p-3 sm:p-4 space-y-3">+ <div className="flex items-center justify-between gap-2">+ <h2 className="text-sm font-semibold text-white">Pick frames</h2>+ <button type="button" onClick={onClose} aria-label="Close frame picker" className={button}>+ <X className="w-4 h-4" aria-hidden="true" />+ </button>+ </div>++ {/* eslint-disable-next-line jsx-a11y/media-has-caption -- reference clips have no caption track */}+ <video+ ref={videoRef}+ src={src}+ poster={poster || undefined}+ playsInline+ muted+ preload="auto"+ data-testid="frame-picker-video"+ onLoadedMetadata={readDuration}+ onDurationChange={readDuration}+ onTimeUpdate={(e) => { if (!scrubbingRef.current) setTime(e.currentTarget.currentTime); }}+ onPause={() => setPlaying(false)}+ onClick={togglePlay}+ className="w-full max-h-[50dvh] object-contain bg-black rounded"+ />++ <div className="flex items-center">+ <button type="button" onClick={() => { pause(); seek(time - FRAME_SEC); }} aria-label="Back one frame" className={button}>+ <ChevronLeft className="w-4 h-4" aria-hidden="true" />+ </button>+ <button type="button" onClick={togglePlay} aria-label={playing ? 'Pause' : 'Play'} className={button}>+ {playing ? <Pause className="w-4 h-4" aria-hidden="true" /> : <Play className="w-4 h-4" aria-hidden="true" />}+ </button>+ <button type="button" onClick={() => { pause(); seek(time + FRAME_SEC); }} aria-label="Forward one frame" className={button}>+ <ChevronRight className="w-4 h-4" aria-hidden="true" />+ </button>+ <span className="text-[11px] tabular-nums whitespace-nowrap text-gray-400">{formatFrameTime(time)} / {formatFrameTime(duration)}</span>+ <span className="ml-auto inline-flex items-center">+ <button type="button" onClick={() => setZoom((z) => ZOOMS[Math.max(0, ZOOMS.indexOf(z) - 1)])} disabled={zoom === ZOOMS[0]} aria-label="Zoom out of the filmstrip" className={button}>+ <ZoomOut className="w-4 h-4" aria-hidden="true" />+ </button>+ <span className="text-[11px] tabular-nums text-gray-400 w-5 text-center">{zoom}×</span>+ <button type="button" onClick={() => setZoom((z) => ZOOMS[Math.min(ZOOMS.length - 1, ZOOMS.indexOf(z) + 1)])} disabled={zoom === ZOOMS.at(-1)} aria-label="Zoom into the filmstrip" className={button}>+ <ZoomIn className="w-4 h-4" aria-hidden="true" />+ </button>+ </span>+ </div>++ <div ref={stripRef} className="overflow-x-auto rounded border border-port-border bg-port-bg">+ <div+ role="slider"+ tabIndex={0}+ aria-label="Filmstrip"+ aria-valuemin={0}+ aria-valuemax={Math.round(duration * 1000) / 1000}+ aria-valuenow={Math.round(time * 1000) / 1000}+ aria-valuetext={formatFrameTime(time)}+ data-testid="frame-picker-strip"+ className="relative flex touch-none select-none cursor-pointer"+ style={{ width: `${zoom * 100}%`, height: THUMB_HEIGHT }}+ onPointerDown={(e) => { pause(); scrubbingRef.current = true; e.currentTarget.setPointerCapture?.(e.pointerId); seekFromPointer(e); }}+ onPointerMove={(e) => { if (scrubbingRef.current) seekFromPointer(e); }}+ onPointerUp={() => { scrubbingRef.current = false; }}+ onPointerCancel={() => { scrubbingRef.current = false; }}+ onKeyDown={(e) => {+ if (e.key === 'ArrowLeft' || e.key === 'ArrowRight') { e.preventDefault(); pause(); seek(time + (e.key === 'ArrowLeft' ? -FRAME_SEC : FRAME_SEC)); }+ }}+ >+ {thumbs.map((url, i) => (+ <div key={i} className="flex-1 min-w-0 h-full bg-port-card border-r border-port-bg/60 overflow-hidden">+ {url ? <img src={url} alt="" className="w-full h-full object-cover pointer-events-none" draggable={false} /> : null}+ </div>+ ))}+ {duration ? picked.map((t) => (+ <span key={t} className="absolute top-0 bottom-0 w-1 -ml-0.5 bg-port-success pointer-events-none" style={{ left: `${(t / duration) * 100}%` }} />+ )) : null}+ {duration ? (+ <span className="absolute top-0 bottom-0 w-1 -ml-0.5 bg-white ring-1 ring-black/60 pointer-events-none" style={{ left: `${(time / duration) * 100}%` }} />+ ) : null}+ </div>+ </div>++ <div className="flex flex-wrap items-center gap-2">+ <button+ type="button"+ onClick={pick}+ disabled={!duration || isPicked(time) || picked.length >= MAX_PICKS}+ className="min-h-[44px] inline-flex items-center gap-1.5 px-3 rounded bg-port-bg border border-port-border text-xs text-white hover:border-port-accent disabled:opacity-40"+ >+ <Plus className="w-3.5 h-3.5" aria-hidden="true" /> Pick this frame+ </button>+ {picked.map((t) => (+ <span key={t} className="inline-flex items-center rounded-full border border-port-border bg-port-bg text-xs text-gray-300">+ <button type="button" onClick={() => { pause(); seek(t); }} className="min-h-[36px] pl-3 pr-1 tabular-nums" aria-label={`Go to ${formatFrameTime(t)}`}>+ {formatFrameTime(t)}+ </button>+ <button type="button" onClick={() => setPicked((prev) => prev.filter((p) => p !== t))} className="min-h-[36px] min-w-[36px] inline-flex items-center justify-center text-gray-500 hover:text-port-error" aria-label={`Remove ${formatFrameTime(t)}`}>+ <X className="w-3 h-3" aria-hidden="true" />+ </button>+ </span>+ ))}+ </div>++ <div className="flex justify-end">+ <button+ type="button"+ onClick={() => onAdd(picked)}+ disabled={!picked.length || busy}+ className="min-h-[44px] px-4 rounded bg-port-accent text-white text-sm font-medium disabled:opacity-40"+ >+ {busy ? 'Adding…' : picked.length ? `Add ${picked.length} frame${picked.length === 1 ? '' : 's'} to board` : 'Add frames to board'}+ </button>+ </div>+ </Modal>+ );+}diff --git a/client/src/components/moodBoard/VideoFramePicker.test.jsx b/client/src/components/moodBoard/VideoFramePicker.test.jsxnew file mode 100644index 0000000000..8f9df9ca70--- /dev/null+++ b/client/src/components/moodBoard/VideoFramePicker.test.jsx@@ -0,0 +1,37 @@+import { describe, it, expect, vi } from 'vitest';+import { render, screen, fireEvent } from '@testing-library/react';+import VideoFramePicker from './VideoFramePicker';++function openWithDuration(onAdd) {+ render(<VideoFramePicker src="/data/videos/clip.mp4" onAdd={onAdd} onClose={() => {}} />);+ const video = screen.getByTestId('frame-picker-video');+ Object.defineProperty(video, 'duration', { configurable: true, value: 10 });+ fireEvent.loadedMetadata(video);+}++describe('VideoFramePicker', () => {+ it('picks frames by stepping and keyboard scrubbing, then adds them in timeline order', () => {+ const onAdd = vi.fn();+ openWithDuration(onAdd);+ const add = screen.getByRole('button', { name: 'Add frames to board' });+ expect(add).toBeDisabled();++ fireEvent.keyDown(screen.getByRole('slider', { name: 'Filmstrip' }), { key: 'ArrowRight' });+ fireEvent.click(screen.getByRole('button', { name: 'Pick this frame' }));+ // The same frame can't be picked twice.+ expect(screen.getByRole('button', { name: 'Pick this frame' })).toBeDisabled();+ fireEvent.click(screen.getByRole('button', { name: 'Back one frame' }));+ fireEvent.click(screen.getByRole('button', { name: 'Pick this frame' }));++ expect(screen.getByRole('button', { name: 'Go to 0:00.03' })).toBeInTheDocument();+ fireEvent.click(screen.getByRole('button', { name: 'Add 2 frames to board' }));+ expect(onAdd).toHaveBeenCalledWith([0, 0.033]);+ });++ it('removes a picked frame', () => {+ openWithDuration(vi.fn());+ fireEvent.click(screen.getByRole('button', { name: 'Pick this frame' }));+ fireEvent.click(screen.getByRole('button', { name: 'Remove 0:00.00' }));+ expect(screen.getByRole('button', { name: 'Add frames to board' })).toBeDisabled();+ });+});diff --git a/client/src/pages/MoodBoardDetail.jsx b/client/src/pages/MoodBoardDetail.jsxindex 980879d9f1..b4d7003f18 100644--- a/client/src/pages/MoodBoardDetail.jsx+++ b/client/src/pages/MoodBoardDetail.jsx@@ -10,8 +10,8 @@*/import { useEffect, useState, useCallback, useMemo, useRef } from 'react';-import { useParams, useNavigate, Link } from 'react-router';-import { ArrowLeft, ChevronDown, ImageIcon, FileText, Trash2, Plus, Save, Link2, Unlink, RefreshCw, Images, Film, Play, ScanEye, Copy, AtSign, Download, Sparkles, Clapperboard, Paintbrush, Wand2 } from 'lucide-react';+import { useParams, useNavigate, useSearchParams, Link } from 'react-router';+import { ArrowLeft, ChevronDown, ImageIcon, FileText, Trash2, Plus, Save, Link2, Unlink, RefreshCw, Images, Film, Play, ScanEye, Copy, AtSign, Download, Sparkles, Clapperboard, Paintbrush, Wand2, GalleryHorizontal } from 'lucide-react';import PageSkeleton from '../components/ui/PageSkeleton';import toast from '../components/ui/Toast';import TabPills from '../components/ui/TabPills';@@ -23,6 +23,7 @@ import MediaLightbox from '../components/media/MediaLightbox';import MoodBoardGeneratePanel from '../components/moodBoard/MoodBoardGeneratePanel';import MoodBoardStylePanel from '../components/moodBoard/MoodBoardStylePanel';import MoodBoardCollagePanel from '../components/moodBoard/MoodBoardCollagePanel';+import VideoFramePicker from '../components/moodBoard/VideoFramePicker';import usePreviewRoute from '../hooks/usePreviewRoute';import { copyToClipboard } from '../lib/clipboard';import {@@ -38,6 +39,7 @@ import {importMoodBoardXPost,localizeMoodBoardMedia,extractMoodBoardItemFrames,+ extractMoodBoardItemFramesAt,renderMoodBoardItem,} from '../services/api';import socket from '../services/socket';@@ -88,6 +90,14 @@ function MoodBoardEditor({ id }) {// Per-video frame extraction: requested frame count + the item in flight.const [frameCount, setFrameCount] = useState(4);+ // The frame picker's video is in the URL (?frames=<itemId>), so it is reload-safe and linkable.+ const [searchParams, setSearchParams] = useSearchParams();+ const pickingFramesItemId = searchParams.get('frames');+ const setPickingFramesItemId = useCallback((itemId) => setSearchParams((prev) => {+ const next = new URLSearchParams(prev);+ if (itemId) next.set('frames', itemId); else next.delete('frames');+ return next;+ }, { replace: !itemId }), [setSearchParams]);const [extractingItemId, setExtractingItemId] = useState(null);// Text note → image render (#10531): the note whose request is in flight.@@ -184,6 +194,9 @@ function MoodBoardEditor({ id }) {// when its `initialSource` identity changes, so a fresh object every render// would wipe an in-flight run on unrelated re-renders. (Lives above the// loading/not-found early returns — hooks must run unconditionally.)+ const pickingFramesItem = pickingFramesItemId+ ? ((Array.isArray(board?.items) ? board.items : []).find((it) => it.id === pickingFramesItemId && moodBoardItemVideoSrc(it)) || null)+ : null;const analyzeItem = analyzeItemId? ((Array.isArray(board?.items) ? board.items : []).find((it) => it.id === analyzeItemId) || null): null;@@ -353,6 +366,19 @@ function MoodBoardEditor({ id }) {toast.success(res.added ? `Added ${res.added} frame${res.added === 1 ? '' : 's'} to the board` : 'Those frames are already on the board');};+ const handleAddPickedFrames = async (itemId, times) => {+ if (!mountedRef.current || !times.length) return;+ setExtractingItemId(itemId);+ const res = await extractMoodBoardItemFramesAt(id, itemId, times, { silent: true })+ .catch((err) => { toast.error(err?.message || 'Could not add those frames'); return null; });+ if (!mountedRef.current) return;+ setExtractingItemId(null);+ if (!res?.board) return;+ setBoard(res.board);+ setPickingFramesItemId(null);+ toast.success(res.added ? `Added ${res.added} frame${res.added === 1 ? '' : 's'} to the board` : 'Those frames are already on the board');+ };+const handleRenderItem = async (itemId) => {if (!mountedRef.current) return;setRenderingItemId(itemId);@@ -1002,6 +1028,15 @@ function MoodBoardEditor({ id }) {><Clapperboard className={`w-3.5 h-3.5 ${extractingItemId === item.id ? 'animate-pulse' : ''}`} aria-hidden="true" /></button>+ <button+ type="button"+ onClick={() => setPickingFramesItemId(item.id)}+ title="Pick frames on a timeline and add them to the board"+ aria-label="Pick frames"+ className="min-h-[44px] min-w-[44px] shrink-0 inline-flex items-center justify-center p-1 text-gray-500 hover:text-white transition-colors"+ >+ <GalleryHorizontal className="w-3.5 h-3.5" aria-hidden="true" />+ </button></span>) : null}{item.type === 'text' ? (@@ -1148,6 +1183,15 @@ function MoodBoardEditor({ id }) {) : null}</PromptFromMediaModal>) : null}+ {pickingFramesItem ? (+ <VideoFramePicker+ src={moodBoardItemVideoSrc(pickingFramesItem)}+ poster={moodBoardItemSrc(pickingFramesItem)}+ busy={extractingItemId === pickingFramesItem.id}+ onAdd={(times) => handleAddPickedFrames(pickingFramesItem.id, times)}+ onClose={() => setPickingFramesItemId(null)}+ />+ ) : null}</div>);}diff --git a/client/src/services/apiMoodBoard.js b/client/src/services/apiMoodBoard.jsindex 445046087e..038666ef90 100644--- a/client/src/services/apiMoodBoard.js+++ b/client/src/services/apiMoodBoard.js@@ -141,6 +141,14 @@ export const extractMoodBoardItemFrames = (id, itemId, count, options) =>...options,});+// Extract the exact moments (seconds) picked in the frame picker.+export const extractMoodBoardItemFramesAt = (id, itemId, times, options) =>+ request(`/mood-boards/${encodeURIComponent(id)}/items/${encodeURIComponent(itemId)}/extract-frames`, {+ method: 'POST',+ body: JSON.stringify({ times }),+ ...options,+ });+// Render a text note into an image (#10531) on the install's default image// backend. Resolves to `{ item, jobId }` once queued; the note becomes an image// item when the job completes (`mood-board:item-render` follows it).diff --git a/server/lib/moodBoardValidation.js b/server/lib/moodBoardValidation.jsindex 9feebaf4bc..bef3650915 100644--- a/server/lib/moodBoardValidation.js+++ b/server/lib/moodBoardValidation.js@@ -196,9 +196,12 @@ export const moodBoardCollageSchema = z.object({cellSize: z.number().int().min(128).max(1024).optional().default(512),}).strict();+// Either `count` evenly spaced frames, or the exact `times` (seconds) picked+// in the frame picker.export const moodBoardExtractFramesSchema = z.object({- count: z.number().int().min(1).max(24),-}).strict();+ count: z.number().int().min(1).max(24).optional(),+ times: z.array(z.number().finite().min(0).max(86400)).min(1).max(24).optional(),+}).strict().refine((v) => (v.count == null) !== (v.times == null), { message: 'Send either count or times' });// Render a text note into an image (#10531). Both optional: an omitted backend// renders on the install's default image backend.diff --git a/server/lib/xPostMedia.js b/server/lib/xPostMedia.jsindex 13e148205d..e3688488dd 100644--- a/server/lib/xPostMedia.js+++ b/server/lib/xPostMedia.js@@ -78,23 +78,42 @@ export function buildSyndicationUrl(statusId) {* both (X's own constraint), but this reads whichever is present without* assuming which. `video.variants` may list multiple encodings (mp4 +* HLS/`application/x-mpegURL`); only mp4 is downloadable via a plain fetch, so- * the highest-bitrate mp4 variant is picked. Returns `{ images: [], video:+ * the sharpest mp4 (by bitrate, then URL resolution) is picked. Returns `{ images: [], video:* null }` for a malformed/empty payload rather than throwing — the caller* decides that's "no media" (400), not a parse crash.*/+// Pixel count from a CDN path like `/vid/avc1/720x1280/…mp4`, 0 when absent.+const variantPixels = (url) => {+ const m = String(url).match(/\/(\d{2,5})x(\d{2,5})\//);+ return m ? Number(m[1]) * Number(m[2]) : 0;+};++// The syndication `video.variants` list carries no bitrate (and lists the+// smallest mp4 first); `mediaDetails[].video_info.variants` does. Rank every+// mp4 by bitrate, then by the resolution in its URL, so the sharpest wins.+function bestMp4(json) {+ const candidates = [+ ...(Array.isArray(json.video?.variants) ? json.video.variants : [])+ .filter((v) => v?.type === 'video/mp4').map((v) => ({ url: v.src, bitrate: v.bitrate })),+ ...(Array.isArray(json.mediaDetails) ? json.mediaDetails : [])+ .flatMap((m) => (Array.isArray(m?.video_info?.variants) ? m.video_info.variants : []))+ .filter((v) => v?.content_type === 'video/mp4').map((v) => ({ url: v.url, bitrate: v.bitrate })),+ ].filter((v) => typeof v.url === 'string' && v.url);+ if (!candidates.length) return null;+ const rank = (v) => [Number(v.bitrate) || 0, variantPixels(v.url)];+ return candidates.reduce((a, b) => {+ const [ra, rb] = [rank(a), rank(b)];+ return rb[0] > ra[0] || (rb[0] === ra[0] && rb[1] > ra[1]) ? b : a;+ }).url;+}+export function extractXPostMedia(json) {if (!json || typeof json !== 'object') return { images: [], video: null };const images = Array.isArray(json.photos)? json.photos.map((p) => p?.url).filter((u) => typeof u === 'string' && u): [];- let video = null;- const variants = json.video?.variants;- if (Array.isArray(variants)) {- const mp4Variants = variants.filter((v) => v?.type === 'video/mp4' && typeof v.src === 'string' && v.src);- if (mp4Variants.length) {- const best = mp4Variants.reduce((a, b) => ((b.bitrate || 0) > (a.bitrate || 0) ? b : a));- video = { url: best.src, poster: typeof json.video.poster === 'string' ? json.video.poster : null };- }- }- return { images, video };+ const url = bestMp4(json);+ const poster = typeof json.video?.poster === 'string' ? json.video.poster+ : (Array.isArray(json.mediaDetails) ? json.mediaDetails.find((m) => m?.video_info)?.media_url_https : null) || null;+ return { images, video: url ? { url, poster } : null };}diff --git a/server/lib/xPostMedia.test.js b/server/lib/xPostMedia.test.jsindex 2412b04f4e..26fbb403bf 100644--- a/server/lib/xPostMedia.test.js+++ b/server/lib/xPostMedia.test.js@@ -111,6 +111,27 @@ describe('extractXPostMedia', () => {expect(video).toEqual({ url: 'https://video.twimg.com/high.mp4', poster: 'https://pbs.twimg.com/poster.jpg' });});+ it('picks the sharpest mp4 when the syndication variants carry no bitrate', () => {+ // video.variants lists the smallest mp4 first with no bitrate; mediaDetails has the bitrates.+ const json = {+ video: {+ poster: 'https://pbs.twimg.com/poster.jpg',+ variants: [+ { type: 'application/x-mpegURL', src: 'https://video.twimg.com/pl/x.m3u8' },+ { type: 'video/mp4', src: 'https://video.twimg.com/vid/avc1/320x568/a.mp4' },+ { type: 'video/mp4', src: 'https://video.twimg.com/vid/avc1/720x1280/c.mp4' },+ { type: 'video/mp4', src: 'https://video.twimg.com/vid/avc1/480x852/b.mp4' },+ ],+ },+ };+ expect(extractXPostMedia(json).video.url).toBe('https://video.twimg.com/vid/avc1/720x1280/c.mp4');+ json.mediaDetails = [{ video_info: { variants: [+ { content_type: 'video/mp4', url: 'https://video.twimg.com/vid/avc1/720x1280/c.mp4', bitrate: 2176000 },+ { content_type: 'video/mp4', url: 'https://video.twimg.com/vid/avc1/1080x1920/d.mp4', bitrate: 10368000 },+ ] } }];+ expect(extractXPostMedia(json).video).toEqual({ url: 'https://video.twimg.com/vid/avc1/1080x1920/d.mp4', poster: 'https://pbs.twimg.com/poster.jpg' });+ });+it('returns a null video when only a non-mp4 variant is present', () => {const { video } = extractXPostMedia({video: { variants: [{ type: 'application/x-mpegURL', src: 'https://video.twimg.com/x.m3u8' }] },diff --git a/server/routes/moodBoard.test.js b/server/routes/moodBoard.test.jsindex 734d61ace7..851c067f23 100644--- a/server/routes/moodBoard.test.js+++ b/server/routes/moodBoard.test.js@@ -203,12 +203,15 @@ describe('mood-board routes', () => {expect(bad.status).toBe(400);});- it('requires a count for per-item frame extraction', async () => {+ it('takes a count or picked times for per-item frame extraction', async () => {svc.extractItemFrames.mockResolvedValueOnce({ added: 4 });const ok = await request(makeApp()).post('/api/mood-boards/mb-1/items/i1/extract-frames').send({ count: 4 });expect(ok.status).toBe(200);expect(svc.extractItemFrames).toHaveBeenCalledWith('mb-1', 'i1', { count: 4 });expect((await request(makeApp()).post('/api/mood-boards/mb-1/items/i1/extract-frames').send({})).status).toBe(400);+ expect((await request(makeApp()).post('/api/mood-boards/mb-1/items/i1/extract-frames').send({ times: [1.25, 3] })).status).toBe(200);+ expect(svc.extractItemFrames).toHaveBeenLastCalledWith('mb-1', 'i1', { times: [1.25, 3] });+ expect((await request(makeApp()).post('/api/mood-boards/mb-1/items/i1/extract-frames').send({ count: 2, times: [1] })).status).toBe(400);});it('queues a note render (202) on an optional backend pin', async () => {diff --git a/server/services/moodBoard/collage.js b/server/services/moodBoard/collage.jsindex 9e4bb2efdf..cd9a561425 100644--- a/server/services/moodBoard/collage.js+++ b/server/services/moodBoard/collage.js@@ -50,7 +50,7 @@ async function requireBoard(boardId) {* Returns the written paths in timeline order. Throws a 422 when ffmpeg is* missing, the file is unreadable, or no frame could be decoded.*/-async function sampleVideoFrames(videoFile, count, outDir, prefix) {+async function sampleVideoFrames(videoFile, count, outDir, prefix, picked = null) {const videoPath = safeUnder(PATHS.videos, videoFile);if (!videoPath) throw new ServerError('Invalid video reference', { status: 400, code: 'VALIDATION_ERROR' });const ffmpeg = await findFfmpeg();@@ -58,40 +58,60 @@ async function sampleVideoFrames(videoFile, count, outDir, prefix) {const duration = await probeVideoDuration(videoPath);if (!duration) throw new ServerError('Could not read that video (missing or unreadable)', { status: 422, code: 'VIDEO_UNREADABLE' });- const times = frameSampleTimes(duration, count);+ // Picked times are clamped just inside the end so the last frame still decodes.+ const times = picked ? picked.map((t) => Math.min(Math.max(0, t), Math.max(0, duration - 0.05))) : frameSampleTimes(duration, count);const paths = [];for (let i = 0; i < times.length; i++) {const out = join(outDir, `${prefix}${i + 1}.jpg`);+ // Input seek decodes forward from the prior keyframe to the exact time;+ // the frame is written at source resolution at JPEG's top quality.const result = await runFfmpegProcess({bin: ffmpeg,- args: ['-ss', times[i].toFixed(3), '-i', videoPath, '-frames:v', '1', '-q:v', '3', '-y', out],+ args: ['-ss', times[i].toFixed(3), '-i', videoPath, '-frames:v', '1', '-q:v', '1', '-qmin', '1', '-y', out],stderrTailBytes: 0,});- if (result.ok) paths.push(out);+ if (result.ok) paths.push({ path: out, time: times[i] });}if (!paths.length) throw new ServerError('No frames could be extracted from that video', { status: 422, code: 'FRAME_EXTRACT_FAILED' });return paths;}/** Copy sampled frames into the gallery under stable, content-keyed names. */-async function storeFramesInGallery(paths, videoFile, count) {+async function storeFramesInGallery(frames, videoFile, count) {await ensureDir(PATHS.images);const key = createHash('sha1').update(`${videoFile}|${count}`).digest('hex').slice(0, 12);const names = [];- for (let i = 0; i < paths.length; i++) {+ for (let i = 0; i < frames.length; i++) {const name = `board-frame-${key}-${i + 1}of${count}.jpg`;- await atomicWrite(join(PATHS.images, name), await readFile(paths[i]));+ await atomicWrite(join(PATHS.images, name), await readFile(frames[i].path));names.push(name);}return names;}+// Picked frames are keyed by their time (to the millisecond), so picking the+// same moment twice dedupes and different picks never overwrite each other.+async function storePickedFramesInGallery(frames, videoFile) {+ await ensureDir(PATHS.images);+ const key = createHash('sha1').update(videoFile).digest('hex').slice(0, 12);+ const names = [];+ for (const frame of frames) {+ const name = `board-frame-${key}-at${Math.round(frame.time * 1000)}ms.jpg`;+ await atomicWrite(join(PATHS.images, name), await readFile(frame.path));+ names.push(name);+ }+ return names;+}++const clock = (sec) => `${Math.floor(sec / 60)}:${(sec % 60).toFixed(2).padStart(5, '0')}`;+/**- * Extract `count` frames from one video pin and append them to the board as- * image items. Re-running with the same count is a no-op (source-deduped).+ * Extract frames from one video pin and append them to the board as image+ * items: `count` evenly spaced frames, or the exact `times` (seconds) picked in+ * the frame picker. Re-running the same request is a no-op (source-deduped).* Returns `{ board, added, frames }`.*/-export async function extractItemFrames(boardId, itemId, { count }) {+export async function extractItemFrames(boardId, itemId, { count, times }) {const board = await requireBoard(boardId);const item = (board.items || []).find((it) => it.id === itemId);if (!item) throw new ServerError('Item not found', { status: 404, code: 'NOT_FOUND' });@@ -100,13 +120,15 @@ export async function extractItemFrames(boardId, itemId, { count }) {const tmp = await mkdtemp(join(tmpdir(), 'portos-board-frames-'));try {- const paths = await sampleVideoFrames(file, count, tmp, 'f');- const names = await storeFramesInGallery(paths, file, count);+ const picked = Array.isArray(times) ? [...new Set(times.map((t) => Math.round(t * 1000) / 1000))].sort((a, b) => a - b) : null;+ const frames = await sampleVideoFrames(file, picked ? picked.length : count, tmp, 'f', picked);+ const names = picked ? await storePickedFramesInGallery(frames, file) : await storeFramesInGallery(frames, file, count);+ const label = (i) => (picked ? `frame at ${clock(frames[i].time)}` : `frame ${i + 1}/${count}`);const imported = names.map((name, i) => ({type: 'image',imageUrl: `/data/images/${name}`,- source: `frame:${file}#${i + 1}of${count}`,- caption: item.caption ? `${item.caption} (frame ${i + 1}/${count})` : `Frame ${i + 1}/${count} of ${file}`,+ source: picked ? `frame:${file}@${Math.round(frames[i].time * 1000)}ms` : `frame:${file}#${i + 1}of${count}`,+ caption: item.caption ? `${item.caption} (${label(i)})` : `${label(i)[0].toUpperCase()}${label(i).slice(1)} of ${file}`,}));// The append first names the frame files, so it commits under the backup lease (#9982).const { board: next, added } = await withBackupAssetPublication(() => store.appendImportedItems(boardId, imported));@@ -158,7 +180,7 @@ export async function composeBoardCollage(boardId, { framesPerVideo, addFramesToreturn [];});if (!frames.length) { skipped += 1; continue; }- sources.push(...frames);+ sources.push(...frames.map((f) => f.path));if (addFramesToBoard) {const names = await storeFramesInGallery(frames, file, framesPerVideo);names.forEach((name, i) => imported.push({diff --git a/server/services/moodBoard/collage.test.js b/server/services/moodBoard/collage.test.jsnew file mode 100644index 0000000000..571073d7f2--- /dev/null+++ b/server/services/moodBoard/collage.test.js@@ -0,0 +1,64 @@+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';+import { execFile } from 'node:child_process';+import { promisify } from 'node:util';+import { mkdir, rm } from 'node:fs/promises';+import { join } from 'node:path';+import sharp from 'sharp';++const state = vi.hoisted(() => ({ root: null, board: null }));+vi.mock('../../lib/fileUtils.js', async (importOriginal) => {+ const original = await importOriginal();+ const { mkdtemp } = await import('node:fs/promises');+ const { tmpdir } = await import('node:os');+ const { join } = await import('node:path');+ state.root = await mkdtemp(join(tmpdir(), 'board-frames-test-'));+ return { ...original, PATHS: { ...original.PATHS, data: state.root, videos: join(state.root, 'videos'), images: join(state.root, 'images') } };+});+vi.mock('./db.js', () => ({+ getBoard: async () => state.board,+ appendImportedItems: async (_id, imported) => {+ const seen = new Set(state.board.items.map((it) => it.source));+ const fresh = imported.filter((it) => !seen.has(it.source));+ state.board = { ...state.board, items: [...state.board.items, ...fresh] };+ return { board: state.board, added: fresh.length };+ },+}));+vi.mock('../sharing/recordEvents.js', () => ({ emitRecordUpdated: vi.fn() }));++const { extractItemFrames } = await import('./collage.js');+const { findFfmpeg } = await import('../../lib/ffmpeg.js');+const exec = promisify(execFile);+let ffmpeg = null;++beforeAll(async () => {+ ffmpeg = await findFfmpeg();+ if (!ffmpeg) return;+ await mkdir(join(state.root, 'videos'), { recursive: true });+ await exec(ffmpeg, ['-y', '-f', 'lavfi', '-i', 'testsrc=s=640x360:r=24:d=2', '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-an', join(state.root, 'videos', 'clip.mp4')]);+}, 30000);+afterAll(() => rm(state.root, { recursive: true, force: true }));+beforeEach(() => {+ state.board = { id: 'mb-1', items: [{ id: 'v1', type: 'video', mediaKey: 'video:clip.mp4', caption: null }] };+});++describe('extractItemFrames', () => {+ it('adds the picked moments once each, in timeline order, at source resolution', async (context) => {+ if (!ffmpeg) context.skip();+ const res = await extractItemFrames('mb-1', 'v1', { times: [1.5, 0.25, 0.25] });+ expect(res.added).toBe(2);+ const frames = state.board.items.filter((it) => it.type === 'image');+ expect(frames.map((it) => it.source)).toEqual(['frame:clip.mp4@250ms', 'frame:clip.mp4@1500ms']);+ expect(frames[0].caption).toBe('Frame at 0:00.25 of clip.mp4');+ const meta = await sharp(join(state.root, 'images', frames[1].imageUrl.split('/').pop())).metadata();+ expect([meta.width, meta.height]).toEqual([640, 360]);+ // Picking the same moment again adds nothing; a new pick adds one more.+ expect((await extractItemFrames('mb-1', 'v1', { times: [1.5, 1] })).added).toBe(1);+ });++ it('still samples evenly spaced frames from a count', async (context) => {+ if (!ffmpeg) context.skip();+ const res = await extractItemFrames('mb-1', 'v1', { count: 3 });+ expect(res.added).toBe(3);+ expect(state.board.items.at(-1).source).toBe('frame:clip.mp4#3of3');+ });+});