Skip to content

A self-hosted media server and file management system

feat: storyboard generation job

aminnausinacd520797ae7
diff --git a/app/Enums/HardwareType.php b/app/Enums/HardwareType.php
new file mode 100644
index 00000000..4afb4107
--- /dev/null
+++ b/app/Enums/HardwareType.php
@@ -0,0 +1,18 @@
+<?php
+
+namespace App\Enums;
+
+enum HardwareType: int {
+ case CPU = 0;
+ case CUDA = 1;
+ case QSV = 2;
+ case VAAPI = 3;
+
+ public function label(): string {
+ return strtolower($this->name);
+ }
+
+ public static function fromLabel(string $label): ?self {
+ return collect(self::cases())->first(fn ($case) => strtolower($case->name) === strtolower($label));
+ }
+}
diff --git a/app/Jobs/Metadata/GenerateStoryboard.php b/app/Jobs/Metadata/GenerateStoryboard.php
new file mode 100644
index 00000000..cdf72318
--- /dev/null
+++ b/app/Jobs/Metadata/GenerateStoryboard.php
@@ -0,0 +1,128 @@
+<?php
+
+namespace App\Jobs\Metadata;
+
+use App\Enums\TaskStatus;
+use App\Jobs\ManagedSubTask;
+use App\Models\Metadata;
+use App\Models\Storyboard;
+use App\Models\SubTask;
+use App\Services\Ffmpeg\FFmpegCommandBuilder;
+use App\Services\Images\Storyboard\StoryboardOptions;
+use App\Services\TaskService;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
+use Symfony\Component\Process\Process;
+
+class GenerateStoryboard extends ManagedSubTask {
+ protected string $filePath;
+
+ protected string $uuid;
+
+ public int $timeout = 600;
+
+ public int $tries = 1;
+
+ public int $backoff = 0;
+
+ /**
+ * Create a new job instance.
+ *
+ * @return void
+ */
+ public function __construct(string $filePath, string $uuid, int $taskId) {
+ $this->filePath = $filePath;
+ $this->uuid = $uuid;
+
+ $subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Generate storyboard for video file ' . basename(dirname($filePath)) . '/' . basename($filePath)]);
+
+ $this->taskId = $taskId;
+ $this->subTaskId = $subTask->id;
+ }
+
+ /**
+ * Execute the job.
+ */
+ public function handle(TaskService $taskService, FFmpegCommandBuilder $builder): void {
+ $this->beginSubTask($taskService, "Generating storyboard for $this->filePath");
+
+ try {
+ $summary = $this->handleGenerateStoryboard($builder);
+ $this->completeSubTask($taskService, $summary);
+ } catch (\Throwable $th) {
+ Log::warning('Storyboard generation failed', [
+ 'uuid' => $this->uuid,
+ 'file' => basename($this->filePath),
+ 'error' => $th->getMessage(),
+ ]);
+ $this->failSubTask($taskService, $th);
+ // throw $th; Don't stop subsequent jobs if this one fails?
+ }
+ }
+
+ private function handleGenerateStoryboard(FFmpegCommandBuilder $builder): string {
+ $outputDir = 'metadata/' . substr($this->uuid, 0, 2) . "/{$this->uuid}/storyboard";
+
+ $publicDisk = Storage::disk('public');
+ $publicDisk->makeDirectory($outputDir);
+
+ $metadata = Metadata::where('uuid', $this->uuid)->firstOrFail();
+ $metadata->load('video');
+
+ $filePath = $publicDisk->path(str_replace('storage/', '', $metadata->video->path));
+ $options = StoryboardOptions::fromMetadata($metadata, $filePath);
+
+ $tile_count = ceil($metadata->duration / config('media.storyboard.default_interval_seconds', 10));
+
+ $command = $builder->storyboard(
+ filePath: $filePath,
+ outputPattern: $publicDisk->path($outputDir) . '/%d.jpg',
+ options: $options,
+ );
+
+ $start = microtime(true);
+
+ $process = new Process($command);
+ $process->setTimeout(600);
+ $process->run();
+
+ if (! $process->isSuccessful()) {
+ Log::warning('Storyboard command failed', [
+ 'uuid' => $this->uuid,
+ 'file' => basename($this->filePath),
+ 'command' => str_replace('"^%"', '%', str_replace('\\', '/', $process->getCommandLine())),
+ ]);
+ throw new \RuntimeException('FFmpeg failed: ' . $process->getErrorOutput());
+ }
+
+ $timeElapsed = round(microtime(true) - $start, 2);
+
+ $sheets = $publicDisk->files($outputDir);
+ $sheetCount = count($sheets);
+
+ if ($sheetCount === 0) {
+ throw new \RuntimeException('FFmpeg produced no output files');
+ }
+
+ Storyboard::updateOrCreate(['metadata_uuid' => $this->uuid], [
+ 'tile_rows' => 10,
+ 'tile_cols' => 10,
+ 'tile_width' => $options->width,
+ 'tile_height' => $options->height,
+ 'tile_count' => $tile_count,
+ 'interval_seconds' => 10,
+ 'modified_at' => now(),
+ ]);
+
+ Log::info('Storyboard generated', [
+ 'uuid' => $this->uuid,
+ 'file' => basename($filePath),
+ 'sheets' => $sheetCount,
+ 'tile_count' => $tile_count,
+ 'time' => $timeElapsed . 's',
+ 'command' => str_replace('"^%"', '%', str_replace('\\', '/', $process->getCommandLine())),
+ ]);
+
+ return "Generated storyboard for {$this->uuid} in {$timeElapsed}s";
+ }
+}
diff --git a/app/Jobs/Metadata/ScheduledGenerateStoryboards.php b/app/Jobs/Metadata/ScheduledGenerateStoryboards.php
new file mode 100644
index 00000000..09b62bff
--- /dev/null
+++ b/app/Jobs/Metadata/ScheduledGenerateStoryboards.php
@@ -0,0 +1,19 @@
+<?php
+
+namespace App\Jobs\Metadata;
+
+use App\Services\FileJobService;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Foundation\Queue\Queueable;
+use Illuminate\Queue\InteractsWithQueue;
+
+class ScheduledGenerateStoryboards implements ShouldQueue {
+ use Dispatchable, InteractsWithQueue, Queueable;
+
+ public function handle(FileJobService $fileJobService): void {
+ $fileJobService->generateStoryboards(
+ data: ['userId' => null, 'namePrefix' => 'Scheduled '],
+ );
+ }
+}
diff --git a/app/Jobs/VerifyFiles.php b/app/Jobs/VerifyFiles.php
index dc46f10c..3f2713fd 100755
--- a/app/Jobs/VerifyFiles.php
+++ b/app/Jobs/VerifyFiles.php
@@ -5,6 +5,7 @@ namespace App\Jobs;
use App\Data\Subtitles\SubtitleScanTarget;
use App\Enums\MediaType;
use App\Enums\TaskStatus;
+use App\Jobs\Metadata\GenerateStoryboard;
use App\Jobs\Utility\Subtitles\ScanSubtitles;
use App\Models\Metadata;
use App\Models\Record;
@@ -26,6 +27,8 @@ class VerifyFiles extends ManagedSubTask {
protected $embedChain = [];
+ protected $storyboardChain = [];
+
protected $scannedDirectories = []; // this could go in the indexer but realistically it is only scanning each directory once and caching the "result" to later batch actual subtitle indexing
protected $fileMetaData = [];
@@ -35,7 +38,7 @@ class VerifyFiles extends ManagedSubTask {
*
* @param Collection<int, Video> $videos
*/
- public function __construct(public Collection $videos, int $taskId) {
+ public function __construct(public Collection $videos, int $taskId, public bool $generateImageTasks = false) {
$subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Verify ' . count($videos) . ' Files']);
$this->taskId = $taskId;
@@ -63,7 +66,7 @@ class VerifyFiles extends ManagedSubTask {
);
}
- $additionalTaskChain = array_merge($this->embedChain, $this->subtitleScanChain);
+ $additionalTaskChain = array_merge($this->embedChain, $this->subtitleScanChain, $this->storyboardChain);
$additionalTaskCount = count($additionalTaskChain);
$taskCountUpdates = [
@@ -327,6 +330,14 @@ class VerifyFiles extends ManagedSubTask {
$changes['raw_metadata'] = json_encode($this->fileMetaData);
}
+ if ($this->generateImageTasks && ! $is_audio && (! $metadata->storyBoard || $fileUpdated)) {
+ $this->storyboardChain[] = new GenerateStoryboard(
+ filePath: $filePath,
+ uuid: $uuid,
+ taskId: $this->taskId,
+ );
+ }
+
if (! empty($changes)) {
$changes['file_scanned_at'] = now();
@@ -635,4 +646,6 @@ class VerifyFiles extends ManagedSubTask {
return $uuid;
}
+
+ private function checkStoryboard() {}
}
diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php
index a3c21b54..0400e436 100755
--- a/app/Providers/AppServiceProvider.php
+++ b/app/Providers/AppServiceProvider.php
@@ -3,6 +3,7 @@
namespace App\Providers;
use App\Models\User;
+use App\Services\Hardware\HardwareDetectionService;
use App\Services\Server\ServerConfigService;
use Dedoc\Scramble\Scramble;
use GuzzleHttp\Client;
@@ -28,6 +29,7 @@ class AppServiceProvider extends ServiceProvider {
}
$this->app->singleton(ServerConfigService::class);
+ $this->app->singleton(HardwareDetectionService::class);
Scramble::ignoreDefaultRoutes();
}
diff --git a/app/Services/Ffmpeg/FFmpegCommandBuilder.php b/app/Services/Ffmpeg/FFmpegCommandBuilder.php
new file mode 100644
index 00000000..4aa9d978
--- /dev/null
+++ b/app/Services/Ffmpeg/FFmpegCommandBuilder.php
@@ -0,0 +1,90 @@
+<?php
+
+namespace App\Services\Ffmpeg;
+
+use App\Enums\HardwareType;
+use App\Services\Hardware\HardwareDetectionService;
+use App\Services\Images\Storyboard\StoryboardOptions;
+
+class FFmpegCommandBuilder {
+ public function __construct(
+ private HardwareDetectionService $hardware,
+ ) {}
+
+ public function storyboard(string $filePath, string $outputPattern, StoryboardOptions $options): array {
+ $hw = $this->hardware->detect()->best();
+
+ $defaultEncode = [
+ 'mjpeg',
+ ];
+
+ $hardwareOptions = match ($hw) {
+ HardwareType::CUDA => [
+ 'decode' => ['-hwaccel', 'cuda'],
+ 'decode_flags' => ['-noautoscale'],
+ 'encode' => $defaultEncode,
+ ],
+ HardwareType::QSV => [
+ 'decode' => [],
+ 'decode_flags' => ['-noautoscale'],
+ 'encode' => ['mjpeg_qsv'],
+ ],
+ default => [
+ 'decode' => [],
+ 'decode_flags' => [],
+ 'encode' => $defaultEncode,
+ ],
+ };
+
+ $w = $options->width;
+ $h = $options->height;
+
+ $scale = match ($hw) {
+ HardwareType::QSV => "vpp_qsv=w={$w}:h={$h}:format=nv12:passthrough=0:out_range=pc:scale_mode=hq",
+ default => "scale='min({$w},iw)':-2:flags=lanczos",
+ };
+
+ $qualityArgs = match ($hw) {
+ HardwareType::QSV => ['-global_quality:v', '91'],
+ default => ['-q:v', '3'],
+ };
+
+ return [
+ 'ffmpeg',
+ ...$hardwareOptions['decode'],
+ '-skip_frame',
+ 'nokey',
+ '-threads',
+ '1',
+ '-i',
+ "file:{$filePath}",
+ '-an',
+ '-sn',
+ '-vf',
+ "fps={$options->fps},{$scale},setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709,tile={$options->cols}x{$options->rows}",
+ '-threads',
+ '1',
+ '-c:v',
+ ...$hardwareOptions['encode'],
+ ...$qualityArgs,
+ '-fps_mode',
+ 'passthrough',
+ '-f',
+ 'image2',
+ $outputPattern,
+ ];
+ }
+
+ // region
+ // TODO: Implement
+ // public function posterFrame(string $filePath, string $outputPath, int $offset): array {
+ // return [];
+ // }
+ // public function extractAudio(string $filePath, string $outputPath): array {
+ // return [];
+ // }
+ // public function transcode(string $filePath, string $outputPath, TranscodeOptions $options): array {
+ // return [];
+ // }
+ // endregion
+}
diff --git a/app/Services/FileJobService.php b/app/Services/FileJobService.php
index cea2acfe..6d6e2121 100644
--- a/app/Services/FileJobService.php
+++ b/app/Services/FileJobService.php
@@ -2,11 +2,13 @@
namespace App\Services;
+use App\Enums\MediaType;
use App\Enums\TaskStatus;
use App\Jobs\EmbedUidInMetadata;
use App\Jobs\GeneratePreviewImage;
use App\Jobs\IndexFiles;
use App\Jobs\Maintenance\PurgeStaleGuestData;
+use App\Jobs\Metadata\GenerateStoryboard;
use App\Jobs\SyncFiles;
use App\Jobs\Utility\Paths\CleanFolderPaths;
use App\Jobs\Utility\Paths\CleanVideoPaths;
@@ -14,6 +16,7 @@ use App\Jobs\VerifyFiles;
use App\Jobs\VerifyFolders;
use App\Models\Category;
use App\Models\Folder;
+use App\Models\Metadata;
use App\Models\Task;
use App\Models\Video;
use Illuminate\Bus\Batch;
@@ -101,7 +104,7 @@ class FileJobService {
chain: function ($task) use ($category) {
$chain = [];
- $videoQuery = Video::orderBy('id');
+ $videoQuery = Video::orderBy('id')->with(['metadata.storyboard']);
$folderQuery = Folder::orderBy('id');
if ($category) {
@@ -114,8 +117,8 @@ class FileJobService {
})->with('category');
}
- $videoQuery->chunk(100, function ($chunk) use (&$chain, $task) {
- $chain[] = new VerifyFiles($chunk, $task->id);
+ $videoQuery->chunk(100, function ($chunk) use (&$chain, $task, $category) {
+ $chain[] = new VerifyFiles($chunk, $task->id, generateImageTasks: $category !== null);
});
$folderQuery->chunk(100, function ($chunk) use (&$chain, $task) {
@@ -255,6 +258,55 @@ class FileJobService {
);
}
+ public function generateStoryboards(array $data, ?Category $category = null): Task {
+ $name = 'Generate Storyboards';
+ $description = 'Generates storyboard sprite sheets for all videos missing them.';
+
+ if (isset($category)) {
+ $name .= " for library \"$category->name\"";
+ $description = "Generates storyboard sprite sheets for the specified library \"$category->name\"";
+ }
+
+ return $this->executeBatchOperation(
+ userId: $data['userId'] ?? null,
+ name: ($data['namePrefix'] ?? '') . $name,
+ description: $description,
+ chain: function ($task) use ($category) {
+ $chain = [];
+
+ $query = Metadata::query()
+ ->select('metadata.*', 'videos.path as video_path')
+ ->join('videos', 'videos.id', '=', 'metadata.video_id')
+ ->where('metadata.media_type', MediaType::VIDEO)
+ ->whereNotNull('metadata.uuid')
+ ->whereDoesntHave('storyboard')
+ ->latest('metadata.updated_at');
+
+ if ($category) {
+ $query->join('folders', 'folders.id', '=', 'videos.folder_id')
+ ->join('categories', 'categories.id', '=', 'folders.category_id')
+ ->where('categories.id', $category->id);
+ }
+
+ $limit = config('media.storyboard.daily_limit', 200);
+
+ if ($limit > 0) {
+ $query->limit($limit);
+ }
+
+ $query->each(function ($metadata) use (&$chain, $task) {
+ $chain[] = new GenerateStoryboard(
+ filePath: VerifyFiles::getAbsoluteMediaPath($metadata->video),
+ uuid: $metadata->uuid,
+ taskId: $task->id,
+ );
+ });
+
+ return $chain;
+ },
+ );
+ }
+
public function executeBatchOperation(
?int $userId,
string $name,
diff --git a/app/Services/Hardware/HardwareDetectionService.php b/app/Services/Hardware/HardwareDetectionService.php
new file mode 100644
index 00000000..f40d4df9
--- /dev/null
+++ b/app/Services/Hardware/HardwareDetectionService.php
@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Services\Hardware;
+
+use Symfony\Component\Process\Process;
+
+class HardwareDetectionService {
+ private ?HardwareProfile $cached = null;
+
+ public function detect(): HardwareProfile {
+ if ($this->cached) {
+ return $this->cached;
+ }
+
+ $hwaccels = $this->getHwaccels();
+ $encoders = $this->getEncoders();
+
+ return $this->cached = new HardwareProfile(
+ qsv: in_array('qsv', $hwaccels) && in_array('mjpeg_qsv', $encoders),
+ cuda: in_array('cuda', $hwaccels),
+ vaapi: in_array('vaapi', $hwaccels),
+ );
+ }
+
+ private function getHwaccels(): array {
+ $process = new Process(['ffmpeg', '-hwaccels', '-hide_banner']);
+ $process->run();
+ $output = $process->getOutput();
+
+ $lines = array_slice(explode("\n", trim($output)), 1);
+
+ return array_values(array_filter(array_map('trim', $lines)));
+ }
+
+ private function getEncoders(): array {
+ $process = new Process(['ffmpeg', '-encoders', '-hide_banner']);
+ $process->run();
+ preg_match_all('/\s(\w+)\s/', $process->getOutput(), $matches);
+
+ return $matches[1] ?? [];
+ }
+}
diff --git a/app/Services/Hardware/HardwareProfile.php b/app/Services/Hardware/HardwareProfile.php
new file mode 100644
index 00000000..ffc595a5
--- /dev/null
+++ b/app/Services/Hardware/HardwareProfile.php
@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Services\Hardware;
+
+use App\Enums\HardwareType;
+
+class HardwareProfile {
+ public function __construct(
+ public bool $cuda,
+ public bool $qsv,
+ public bool $vaapi,
+ ) {}
+
+ public function best(): HardwareType {
+ if ($this->cuda) {
+ return HardwareType::CUDA;
+ }
+ if ($this->qsv) {
+ return HardwareType::QSV;
+ }
+ if ($this->vaapi) {
+ return HardwareType::VAAPI;
+ }
+
+ return HardwareType::CPU;
+ }
+}
diff --git a/app/Services/Images/Storyboard/StoryboardOptions.php b/app/Services/Images/Storyboard/StoryboardOptions.php
new file mode 100644
index 00000000..3be1fcb6
--- /dev/null
+++ b/app/Services/Images/Storyboard/StoryboardOptions.php
@@ -0,0 +1,78 @@
+<?php
+
+namespace App\Services\Images\Storyboard;
+
+use App\Models\Metadata;
+
+class StoryboardOptions {
+ public function __construct(
+ public int $width,
+ public int $height,
+ public int $cols,
+ public int $rows,
+ public float $fps,
+ public bool $tile,
+ public int $sourceWidth,
+ public int $sourceHeight,
+ ) {}
+
+ public static function fromMetadata(Metadata $metadata, string $filePath): self {
+ self::assertSupported($metadata, $filePath);
+
+ $sourceW = $metadata->resolution_width ?? 1920;
+ $sourceH = $metadata->resolution_height ?? 1080;
+
+ $maxTileSize = 320;
+
+ $cellWidth = $maxTileSize;
+ $cellHeight = $maxTileSize;
+
+ if ($sourceW > $sourceH) {
+ $cellWidth = $maxTileSize;
+ $cellHeight = (int) max(2, floor(($maxTileSize * $sourceH / $sourceW) / 2) * 2);
+ } elseif ($sourceH > $sourceW) {
+ $cellHeight = $maxTileSize;
+ $cellWidth = (int) max(2, floor(($maxTileSize * $sourceW / $sourceH) / 2) * 2);
+ }
+
+ return new self(
+ width: $cellWidth,
+ height: $cellHeight,
+ cols: 10,
+ rows: 10,
+ fps: 0.1,
+ tile: true,
+ sourceWidth: $sourceW,
+ sourceHeight: $sourceH,
+ );
+ }
+
+ private static function assertSupported(Metadata $metadata, string $filePath): void {
+ // Unsupported codecs for hardware accelerated storyboard generation
+ $unsupportedCodecs = ['vp9', 'vp8', 'av1'];
+
+ if (empty($metadata->codec)) {
+ throw new \RuntimeException('No codec detected for storyboard generation');
+ }
+
+ if (in_array($metadata->codec, $unsupportedCodecs)) {
+ throw new \RuntimeException("Codec {$metadata->codec} not supported for storyboard generation");
+ }
+
+ if (empty($metadata->duration) || $metadata->duration < 10) {
+ throw new \RuntimeException('Duration too short or missing for storyboard generation');
+ }
+
+ if (empty($metadata->resolution_width) || empty($metadata->resolution_height)) {
+ throw new \RuntimeException('Resolution missing for storyboard generation');
+ }
+
+ if (str_starts_with($metadata->mime_type ?? '', 'audio/')) {
+ throw new \RuntimeException('Audio files not supported for storyboard generation');
+ }
+
+ if (! file_exists($filePath)) {
+ throw new \RuntimeException("File not found: {$filePath}");
+ }
+ }
+}
diff --git a/config/media.php b/config/media.php
index 49a65047..4715ca45 100644
--- a/config/media.php
+++ b/config/media.php
@@ -7,6 +7,10 @@ return [
'storage' => [
'prefix' => 'storage/media/',
],
+ 'storyboard' => [
+ 'default_interval_seconds' => 10,
+ 'daily_limit' => 200,
+ ],
'format_map' => [
'mp4' => 'mp4',
'm4a' => 'mp4',
diff --git a/database/migrations/2026_05_21_210603_change_interval_to_seconds_on_storyboards.php b/database/migrations/2026_05_21_210603_change_interval_to_seconds_on_storyboards.php
index 6517628a..d51392ea 100644
--- a/database/migrations/2026_05_21_210603_change_interval_to_seconds_on_storyboards.php
+++ b/database/migrations/2026_05_21_210603_change_interval_to_seconds_on_storyboards.php
@@ -11,13 +11,13 @@ return new class extends Migration {
*/
public function up(): void {
Schema::table('storyboards', function (Blueprint $table) {
- $table->integer("interval_seconds")->default(10);
+ $table->integer('interval_seconds')->default(10);
});
DB::statement('UPDATE storyboards SET interval_seconds = interval_ms / 1000');
Schema::table('storyboards', function (Blueprint $table) {
- $table->dropColumn("interval_ms");
+ $table->dropColumn('interval_ms');
});
}
@@ -26,13 +26,13 @@ return new class extends Migration {
*/
public function down(): void {
Schema::table('storyboards', function (Blueprint $table) {
- $table->integer("interval_ms")->default(10);
+ $table->integer('interval_ms')->default(10);
});
DB::statement('UPDATE storyboards SET interval_ms = interval_seconds * 1000');
Schema::table('storyboards', function (Blueprint $table) {
- $table->dropColumn("interval_seconds");
+ $table->dropColumn('interval_seconds');
});
}
};
diff --git a/routes/console.php b/routes/console.php
index 93be5e17..52b7b6d1 100755
--- a/routes/console.php
+++ b/routes/console.php
@@ -1,11 +1,13 @@
<?php
use App\Jobs\Maintenance\ScheduledPurgeStaleData;
+use App\Jobs\Metadata\ScheduledGenerateStoryboards;
use App\Jobs\ScheduledIndexFiles;
use Illuminate\Support\Facades\Schedule;
Schedule::job(new ScheduledIndexFiles)->twiceDaily()->withoutOverlapping()->environments(['staging', 'production']);
-Schedule::job(new ScheduledPurgeStaleData)->daily()->withoutOverlapping();
+Schedule::job(new ScheduledGenerateStoryboards)->dailyAt('03:00')->withoutOverlapping();
+Schedule::job(new ScheduledPurgeStaleData)->weekly()->withoutOverlapping();
Schedule::command('auth:clear-resets')->everyFifteenMinutes();
Schedule::command('sanctum:prune-expired --hours=2')->daily();