A self-hosted media server and file management system
Vue ffmpegfile-serverjellyfinlaravelmediamedia-playermedia-servermediabrowsermediaservermusicmusic-librarymusic-playermusicplayerself-hostedvuejs
feat: storyboard generation job
aminnausinacd520797ae7
diff --git a/app/Enums/HardwareType.php b/app/Enums/HardwareType.phpnew file mode 100644index 00000000..4afb4107--- /dev/null+++ b/app/Enums/HardwareType.php@@ -0,0 +1,18 @@+<?php++namespace App\Enums;++enum HardwareType: int {+ case CPU = 0;+ case CUDA = 1;+ case QSV = 2;+ case VAAPI = 3;++ public function label(): string {+ return strtolower($this->name);+ }++ public static function fromLabel(string $label): ?self {+ return collect(self::cases())->first(fn ($case) => strtolower($case->name) === strtolower($label));+ }+}diff --git a/app/Jobs/Metadata/GenerateStoryboard.php b/app/Jobs/Metadata/GenerateStoryboard.phpnew file mode 100644index 00000000..cdf72318--- /dev/null+++ b/app/Jobs/Metadata/GenerateStoryboard.php@@ -0,0 +1,128 @@+<?php++namespace App\Jobs\Metadata;++use App\Enums\TaskStatus;+use App\Jobs\ManagedSubTask;+use App\Models\Metadata;+use App\Models\Storyboard;+use App\Models\SubTask;+use App\Services\Ffmpeg\FFmpegCommandBuilder;+use App\Services\Images\Storyboard\StoryboardOptions;+use App\Services\TaskService;+use Illuminate\Support\Facades\Log;+use Illuminate\Support\Facades\Storage;+use Symfony\Component\Process\Process;++class GenerateStoryboard extends ManagedSubTask {+ protected string $filePath;++ protected string $uuid;++ public int $timeout = 600;++ public int $tries = 1;++ public int $backoff = 0;++ /**+ * Create a new job instance.+ *+ * @return void+ */+ public function __construct(string $filePath, string $uuid, int $taskId) {+ $this->filePath = $filePath;+ $this->uuid = $uuid;++ $subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Generate storyboard for video file ' . basename(dirname($filePath)) . '/' . basename($filePath)]);++ $this->taskId = $taskId;+ $this->subTaskId = $subTask->id;+ }++ /**+ * Execute the job.+ */+ public function handle(TaskService $taskService, FFmpegCommandBuilder $builder): void {+ $this->beginSubTask($taskService, "Generating storyboard for $this->filePath");++ try {+ $summary = $this->handleGenerateStoryboard($builder);+ $this->completeSubTask($taskService, $summary);+ } catch (\Throwable $th) {+ Log::warning('Storyboard generation failed', [+ 'uuid' => $this->uuid,+ 'file' => basename($this->filePath),+ 'error' => $th->getMessage(),+ ]);+ $this->failSubTask($taskService, $th);+ // throw $th; Don't stop subsequent jobs if this one fails?+ }+ }++ private function handleGenerateStoryboard(FFmpegCommandBuilder $builder): string {+ $outputDir = 'metadata/' . substr($this->uuid, 0, 2) . "/{$this->uuid}/storyboard";++ $publicDisk = Storage::disk('public');+ $publicDisk->makeDirectory($outputDir);++ $metadata = Metadata::where('uuid', $this->uuid)->firstOrFail();+ $metadata->load('video');++ $filePath = $publicDisk->path(str_replace('storage/', '', $metadata->video->path));+ $options = StoryboardOptions::fromMetadata($metadata, $filePath);++ $tile_count = ceil($metadata->duration / config('media.storyboard.default_interval_seconds', 10));++ $command = $builder->storyboard(+ filePath: $filePath,+ outputPattern: $publicDisk->path($outputDir) . '/%d.jpg',+ options: $options,+ );++ $start = microtime(true);++ $process = new Process($command);+ $process->setTimeout(600);+ $process->run();++ if (! $process->isSuccessful()) {+ Log::warning('Storyboard command failed', [+ 'uuid' => $this->uuid,+ 'file' => basename($this->filePath),+ 'command' => str_replace('"^%"', '%', str_replace('\\', '/', $process->getCommandLine())),+ ]);+ throw new \RuntimeException('FFmpeg failed: ' . $process->getErrorOutput());+ }++ $timeElapsed = round(microtime(true) - $start, 2);++ $sheets = $publicDisk->files($outputDir);+ $sheetCount = count($sheets);++ if ($sheetCount === 0) {+ throw new \RuntimeException('FFmpeg produced no output files');+ }++ Storyboard::updateOrCreate(['metadata_uuid' => $this->uuid], [+ 'tile_rows' => 10,+ 'tile_cols' => 10,+ 'tile_width' => $options->width,+ 'tile_height' => $options->height,+ 'tile_count' => $tile_count,+ 'interval_seconds' => 10,+ 'modified_at' => now(),+ ]);++ Log::info('Storyboard generated', [+ 'uuid' => $this->uuid,+ 'file' => basename($filePath),+ 'sheets' => $sheetCount,+ 'tile_count' => $tile_count,+ 'time' => $timeElapsed . 's',+ 'command' => str_replace('"^%"', '%', str_replace('\\', '/', $process->getCommandLine())),+ ]);++ return "Generated storyboard for {$this->uuid} in {$timeElapsed}s";+ }+}diff --git a/app/Jobs/Metadata/ScheduledGenerateStoryboards.php b/app/Jobs/Metadata/ScheduledGenerateStoryboards.phpnew file mode 100644index 00000000..09b62bff--- /dev/null+++ b/app/Jobs/Metadata/ScheduledGenerateStoryboards.php@@ -0,0 +1,19 @@+<?php++namespace App\Jobs\Metadata;++use App\Services\FileJobService;+use Illuminate\Contracts\Queue\ShouldQueue;+use Illuminate\Foundation\Bus\Dispatchable;+use Illuminate\Foundation\Queue\Queueable;+use Illuminate\Queue\InteractsWithQueue;++class ScheduledGenerateStoryboards implements ShouldQueue {+ use Dispatchable, InteractsWithQueue, Queueable;++ public function handle(FileJobService $fileJobService): void {+ $fileJobService->generateStoryboards(+ data: ['userId' => null, 'namePrefix' => 'Scheduled '],+ );+ }+}diff --git a/app/Jobs/VerifyFiles.php b/app/Jobs/VerifyFiles.phpindex dc46f10c..3f2713fd 100755--- a/app/Jobs/VerifyFiles.php+++ b/app/Jobs/VerifyFiles.php@@ -5,6 +5,7 @@ namespace App\Jobs;use App\Data\Subtitles\SubtitleScanTarget;use App\Enums\MediaType;use App\Enums\TaskStatus;+use App\Jobs\Metadata\GenerateStoryboard;use App\Jobs\Utility\Subtitles\ScanSubtitles;use App\Models\Metadata;use App\Models\Record;@@ -26,6 +27,8 @@ class VerifyFiles extends ManagedSubTask {protected $embedChain = [];+ protected $storyboardChain = [];+protected $scannedDirectories = []; // this could go in the indexer but realistically it is only scanning each directory once and caching the "result" to later batch actual subtitle indexingprotected $fileMetaData = [];@@ -35,7 +38,7 @@ class VerifyFiles extends ManagedSubTask {** @param Collection<int, Video> $videos*/- public function __construct(public Collection $videos, int $taskId) {+ public function __construct(public Collection $videos, int $taskId, public bool $generateImageTasks = false) {$subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Verify ' . count($videos) . ' Files']);$this->taskId = $taskId;@@ -63,7 +66,7 @@ class VerifyFiles extends ManagedSubTask {);}- $additionalTaskChain = array_merge($this->embedChain, $this->subtitleScanChain);+ $additionalTaskChain = array_merge($this->embedChain, $this->subtitleScanChain, $this->storyboardChain);$additionalTaskCount = count($additionalTaskChain);$taskCountUpdates = [@@ -327,6 +330,14 @@ class VerifyFiles extends ManagedSubTask {$changes['raw_metadata'] = json_encode($this->fileMetaData);}+ if ($this->generateImageTasks && ! $is_audio && (! $metadata->storyBoard || $fileUpdated)) {+ $this->storyboardChain[] = new GenerateStoryboard(+ filePath: $filePath,+ uuid: $uuid,+ taskId: $this->taskId,+ );+ }+if (! empty($changes)) {$changes['file_scanned_at'] = now();@@ -635,4 +646,6 @@ class VerifyFiles extends ManagedSubTask {return $uuid;}++ private function checkStoryboard() {}}diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.phpindex a3c21b54..0400e436 100755--- a/app/Providers/AppServiceProvider.php+++ b/app/Providers/AppServiceProvider.php@@ -3,6 +3,7 @@namespace App\Providers;use App\Models\User;+use App\Services\Hardware\HardwareDetectionService;use App\Services\Server\ServerConfigService;use Dedoc\Scramble\Scramble;use GuzzleHttp\Client;@@ -28,6 +29,7 @@ class AppServiceProvider extends ServiceProvider {}$this->app->singleton(ServerConfigService::class);+ $this->app->singleton(HardwareDetectionService::class);Scramble::ignoreDefaultRoutes();}diff --git a/app/Services/Ffmpeg/FFmpegCommandBuilder.php b/app/Services/Ffmpeg/FFmpegCommandBuilder.phpnew file mode 100644index 00000000..4aa9d978--- /dev/null+++ b/app/Services/Ffmpeg/FFmpegCommandBuilder.php@@ -0,0 +1,90 @@+<?php++namespace App\Services\Ffmpeg;++use App\Enums\HardwareType;+use App\Services\Hardware\HardwareDetectionService;+use App\Services\Images\Storyboard\StoryboardOptions;++class FFmpegCommandBuilder {+ public function __construct(+ private HardwareDetectionService $hardware,+ ) {}++ public function storyboard(string $filePath, string $outputPattern, StoryboardOptions $options): array {+ $hw = $this->hardware->detect()->best();++ $defaultEncode = [+ 'mjpeg',+ ];++ $hardwareOptions = match ($hw) {+ HardwareType::CUDA => [+ 'decode' => ['-hwaccel', 'cuda'],+ 'decode_flags' => ['-noautoscale'],+ 'encode' => $defaultEncode,+ ],+ HardwareType::QSV => [+ 'decode' => [],+ 'decode_flags' => ['-noautoscale'],+ 'encode' => ['mjpeg_qsv'],+ ],+ default => [+ 'decode' => [],+ 'decode_flags' => [],+ 'encode' => $defaultEncode,+ ],+ };++ $w = $options->width;+ $h = $options->height;++ $scale = match ($hw) {+ HardwareType::QSV => "vpp_qsv=w={$w}:h={$h}:format=nv12:passthrough=0:out_range=pc:scale_mode=hq",+ default => "scale='min({$w},iw)':-2:flags=lanczos",+ };++ $qualityArgs = match ($hw) {+ HardwareType::QSV => ['-global_quality:v', '91'],+ default => ['-q:v', '3'],+ };++ return [+ 'ffmpeg',+ ...$hardwareOptions['decode'],+ '-skip_frame',+ 'nokey',+ '-threads',+ '1',+ '-i',+ "file:{$filePath}",+ '-an',+ '-sn',+ '-vf',+ "fps={$options->fps},{$scale},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709,tile={$options->cols}x{$options->rows}",+ '-threads',+ '1',+ '-c:v',+ ...$hardwareOptions['encode'],+ ...$qualityArgs,+ '-fps_mode',+ 'passthrough',+ '-f',+ 'image2',+ $outputPattern,+ ];+ }++ // region+ // TODO: Implement+ // public function posterFrame(string $filePath, string $outputPath, int $offset): array {+ // return [];+ // }+ // public function extractAudio(string $filePath, string $outputPath): array {+ // return [];+ // }+ // public function transcode(string $filePath, string $outputPath, TranscodeOptions $options): array {+ // return [];+ // }+ // endregion+}diff --git a/app/Services/FileJobService.php b/app/Services/FileJobService.phpindex cea2acfe..6d6e2121 100644--- a/app/Services/FileJobService.php+++ b/app/Services/FileJobService.php@@ -2,11 +2,13 @@namespace App\Services;+use App\Enums\MediaType;use App\Enums\TaskStatus;use App\Jobs\EmbedUidInMetadata;use App\Jobs\GeneratePreviewImage;use App\Jobs\IndexFiles;use App\Jobs\Maintenance\PurgeStaleGuestData;+use App\Jobs\Metadata\GenerateStoryboard;use App\Jobs\SyncFiles;use App\Jobs\Utility\Paths\CleanFolderPaths;use App\Jobs\Utility\Paths\CleanVideoPaths;@@ -14,6 +16,7 @@ use App\Jobs\VerifyFiles;use App\Jobs\VerifyFolders;use App\Models\Category;use App\Models\Folder;+use App\Models\Metadata;use App\Models\Task;use App\Models\Video;use Illuminate\Bus\Batch;@@ -101,7 +104,7 @@ class FileJobService {chain: function ($task) use ($category) {$chain = [];- $videoQuery = Video::orderBy('id');+ $videoQuery = Video::orderBy('id')->with(['metadata.storyboard']);$folderQuery = Folder::orderBy('id');if ($category) {@@ -114,8 +117,8 @@ class FileJobService {})->with('category');}- $videoQuery->chunk(100, function ($chunk) use (&$chain, $task) {- $chain[] = new VerifyFiles($chunk, $task->id);+ $videoQuery->chunk(100, function ($chunk) use (&$chain, $task, $category) {+ $chain[] = new VerifyFiles($chunk, $task->id, generateImageTasks: $category !== null);});$folderQuery->chunk(100, function ($chunk) use (&$chain, $task) {@@ -255,6 +258,55 @@ class FileJobService {);}+ public function generateStoryboards(array $data, ?Category $category = null): Task {+ $name = 'Generate Storyboards';+ $description = 'Generates storyboard sprite sheets for all videos missing them.';++ if (isset($category)) {+ $name .= " for library \"$category->name\"";+ $description = "Generates storyboard sprite sheets for the specified library \"$category->name\"";+ }++ return $this->executeBatchOperation(+ userId: $data['userId'] ?? null,+ name: ($data['namePrefix'] ?? '') . $name,+ description: $description,+ chain: function ($task) use ($category) {+ $chain = [];++ $query = Metadata::query()+ ->select('metadata.*', 'videos.path as video_path')+ ->join('videos', 'videos.id', '=', 'metadata.video_id')+ ->where('metadata.media_type', MediaType::VIDEO)+ ->whereNotNull('metadata.uuid')+ ->whereDoesntHave('storyboard')+ ->latest('metadata.updated_at');++ if ($category) {+ $query->join('folders', 'folders.id', '=', 'videos.folder_id')+ ->join('categories', 'categories.id', '=', 'folders.category_id')+ ->where('categories.id', $category->id);+ }++ $limit = config('media.storyboard.daily_limit', 200);++ if ($limit > 0) {+ $query->limit($limit);+ }++ $query->each(function ($metadata) use (&$chain, $task) {+ $chain[] = new GenerateStoryboard(+ filePath: VerifyFiles::getAbsoluteMediaPath($metadata->video),+ uuid: $metadata->uuid,+ taskId: $task->id,+ );+ });++ return $chain;+ },+ );+ }+public function executeBatchOperation(?int $userId,string $name,diff --git a/app/Services/Hardware/HardwareDetectionService.php b/app/Services/Hardware/HardwareDetectionService.phpnew file mode 100644index 00000000..f40d4df9--- /dev/null+++ b/app/Services/Hardware/HardwareDetectionService.php@@ -0,0 +1,42 @@+<?php++namespace App\Services\Hardware;++use Symfony\Component\Process\Process;++class HardwareDetectionService {+ private ?HardwareProfile $cached = null;++ public function detect(): HardwareProfile {+ if ($this->cached) {+ return $this->cached;+ }++ $hwaccels = $this->getHwaccels();+ $encoders = $this->getEncoders();++ return $this->cached = new HardwareProfile(+ qsv: in_array('qsv', $hwaccels) && in_array('mjpeg_qsv', $encoders),+ cuda: in_array('cuda', $hwaccels),+ vaapi: in_array('vaapi', $hwaccels),+ );+ }++ private function getHwaccels(): array {+ $process = new Process(['ffmpeg', '-hwaccels', '-hide_banner']);+ $process->run();+ $output = $process->getOutput();++ $lines = array_slice(explode("\n", trim($output)), 1);++ return array_values(array_filter(array_map('trim', $lines)));+ }++ private function getEncoders(): array {+ $process = new Process(['ffmpeg', '-encoders', '-hide_banner']);+ $process->run();+ preg_match_all('/\s(\w+)\s/', $process->getOutput(), $matches);++ return $matches[1] ?? [];+ }+}diff --git a/app/Services/Hardware/HardwareProfile.php b/app/Services/Hardware/HardwareProfile.phpnew file mode 100644index 00000000..ffc595a5--- /dev/null+++ b/app/Services/Hardware/HardwareProfile.php@@ -0,0 +1,27 @@+<?php++namespace App\Services\Hardware;++use App\Enums\HardwareType;++class HardwareProfile {+ public function __construct(+ public bool $cuda,+ public bool $qsv,+ public bool $vaapi,+ ) {}++ public function best(): HardwareType {+ if ($this->cuda) {+ return HardwareType::CUDA;+ }+ if ($this->qsv) {+ return HardwareType::QSV;+ }+ if ($this->vaapi) {+ return HardwareType::VAAPI;+ }++ return HardwareType::CPU;+ }+}diff --git a/app/Services/Images/Storyboard/StoryboardOptions.php b/app/Services/Images/Storyboard/StoryboardOptions.phpnew file mode 100644index 00000000..3be1fcb6--- /dev/null+++ b/app/Services/Images/Storyboard/StoryboardOptions.php@@ -0,0 +1,78 @@+<?php++namespace App\Services\Images\Storyboard;++use App\Models\Metadata;++class StoryboardOptions {+ public function __construct(+ public int $width,+ public int $height,+ public int $cols,+ public int $rows,+ public float $fps,+ public bool $tile,+ public int $sourceWidth,+ public int $sourceHeight,+ ) {}++ public static function fromMetadata(Metadata $metadata, string $filePath): self {+ self::assertSupported($metadata, $filePath);++ $sourceW = $metadata->resolution_width ?? 1920;+ $sourceH = $metadata->resolution_height ?? 1080;++ $maxTileSize = 320;++ $cellWidth = $maxTileSize;+ $cellHeight = $maxTileSize;++ if ($sourceW > $sourceH) {+ $cellWidth = $maxTileSize;+ $cellHeight = (int) max(2, floor(($maxTileSize * $sourceH / $sourceW) / 2) * 2);+ } elseif ($sourceH > $sourceW) {+ $cellHeight = $maxTileSize;+ $cellWidth = (int) max(2, floor(($maxTileSize * $sourceW / $sourceH) / 2) * 2);+ }++ return new self(+ width: $cellWidth,+ height: $cellHeight,+ cols: 10,+ rows: 10,+ fps: 0.1,+ tile: true,+ sourceWidth: $sourceW,+ sourceHeight: $sourceH,+ );+ }++ private static function assertSupported(Metadata $metadata, string $filePath): void {+ // Unsupported codecs for hardware accelerated storyboard generation+ $unsupportedCodecs = ['vp9', 'vp8', 'av1'];++ if (empty($metadata->codec)) {+ throw new \RuntimeException('No codec detected for storyboard generation');+ }++ if (in_array($metadata->codec, $unsupportedCodecs)) {+ throw new \RuntimeException("Codec {$metadata->codec} not supported for storyboard generation");+ }++ if (empty($metadata->duration) || $metadata->duration < 10) {+ throw new \RuntimeException('Duration too short or missing for storyboard generation');+ }++ if (empty($metadata->resolution_width) || empty($metadata->resolution_height)) {+ throw new \RuntimeException('Resolution missing for storyboard generation');+ }++ if (str_starts_with($metadata->mime_type ?? '', 'audio/')) {+ throw new \RuntimeException('Audio files not supported for storyboard generation');+ }++ if (! file_exists($filePath)) {+ throw new \RuntimeException("File not found: {$filePath}");+ }+ }+}diff --git a/config/media.php b/config/media.phpindex 49a65047..4715ca45 100644--- a/config/media.php+++ b/config/media.php@@ -7,6 +7,10 @@ return ['storage' => ['prefix' => 'storage/media/',],+ 'storyboard' => [+ 'default_interval_seconds' => 10,+ 'daily_limit' => 200,+ ],'format_map' => ['mp4' => 'mp4','m4a' => 'mp4',diff --git a/database/migrations/2026_05_21_210603_change_interval_to_seconds_on_storyboards.php b/database/migrations/2026_05_21_210603_change_interval_to_seconds_on_storyboards.phpindex 6517628a..d51392ea 100644--- a/database/migrations/2026_05_21_210603_change_interval_to_seconds_on_storyboards.php+++ b/database/migrations/2026_05_21_210603_change_interval_to_seconds_on_storyboards.php@@ -11,13 +11,13 @@ return new class extends Migration {*/public function up(): void {Schema::table('storyboards', function (Blueprint $table) {- $table->integer("interval_seconds")->default(10);+ $table->integer('interval_seconds')->default(10);});DB::statement('UPDATE storyboards SET interval_seconds = interval_ms / 1000');Schema::table('storyboards', function (Blueprint $table) {- $table->dropColumn("interval_ms");+ $table->dropColumn('interval_ms');});}@@ -26,13 +26,13 @@ return new class extends Migration {*/public function down(): void {Schema::table('storyboards', function (Blueprint $table) {- $table->integer("interval_ms")->default(10);+ $table->integer('interval_ms')->default(10);});DB::statement('UPDATE storyboards SET interval_ms = interval_seconds * 1000');Schema::table('storyboards', function (Blueprint $table) {- $table->dropColumn("interval_seconds");+ $table->dropColumn('interval_seconds');});}};diff --git a/routes/console.php b/routes/console.phpindex 93be5e17..52b7b6d1 100755--- a/routes/console.php+++ b/routes/console.php@@ -1,11 +1,13 @@<?phpuse App\Jobs\Maintenance\ScheduledPurgeStaleData;+use App\Jobs\Metadata\ScheduledGenerateStoryboards;use App\Jobs\ScheduledIndexFiles;use Illuminate\Support\Facades\Schedule;Schedule::job(new ScheduledIndexFiles)->twiceDaily()->withoutOverlapping()->environments(['staging', 'production']);-Schedule::job(new ScheduledPurgeStaleData)->daily()->withoutOverlapping();+Schedule::job(new ScheduledGenerateStoryboards)->dailyAt('03:00')->withoutOverlapping();+Schedule::job(new ScheduledPurgeStaleData)->weekly()->withoutOverlapping();Schedule::command('auth:clear-resets')->everyFifteenMinutes();Schedule::command('sanctum:prune-expired --hours=2')->daily();