Skip to content

A self-hosted media server and file management system

feat: scan external subtitles

aminnausin12bb737d325e
diff --git a/app/Enums/SubtitleSource.php b/app/Enums/SubtitleSource.php
new file mode 100644
index 00000000..1db21eee
--- /dev/null
+++ b/app/Enums/SubtitleSource.php
@@ -0,0 +1,12 @@
+<?php
+
+namespace App\Enums;
+
+enum SubtitleSource: string {
+ case EMBEDDED = 'embedded';
+ case EXTERNAL = 'external';
+
+ public function makeKey(string|int $value): string {
+ return "{$this->value}:{$value}";
+ }
+}
diff --git a/app/Jobs/VerifyFiles.php b/app/Jobs/VerifyFiles.php
index b3c8a10d..291f8503 100755
--- a/app/Jobs/VerifyFiles.php
+++ b/app/Jobs/VerifyFiles.php
@@ -79,6 +79,7 @@ class VerifyFiles extends ManagedSubTask {
private function verifyFiles(TaskService $taskService, SubtitleScanner $subtitleScanner) {
$metadataTransactions = [];
$subtitleTransactions = [];
+ $scannedDirectories = []; // TODO: this should really go in the indexer or a broken down part of the indexer
$error = false;
$index = 0;
@@ -89,43 +90,46 @@ class VerifyFiles extends ManagedSubTask {
try {
foreach ($this->videos as $video) {
- $stored = []; // Metadata from db
- $changes = []; // Changes -> stored + changes . length has to be the same for every video so must generate defaults
+ $baseName = basename($video->path);
+ $fileName = pathinfo($video->path, PATHINFO_FILENAME);
+ $compositeId = $video->folder->path . '/' . $baseName;
- $compositeId = $video->folder->path . '/' . basename($video->path);
- $filePath = str_replace('\\', '/', Storage::disk('public')->path('')) . 'media/' . $video->folder->path . '/' . basename($video->path);
+ // absolute file path in storage
+ $filePath = $this->getAbsoluteMediaPath($video);
+ $folderPath = dirname($filePath);
/**
* @disregard P1013 Undefined method but it actually exists
*/
- if (! Storage::disk('public')->fileExists('media/' . $video->folder->path . '/' . basename($video->path))) {
- throw new \Exception('Video "media/' . $video->folder->path . '/' . basename($video->path) . '" no longer exists. Index your videos before running this task again.');
+ if (! Storage::disk('public')->fileExists("media/{$video->folder->path}/{$baseName}")) {
+ throw new \Exception("File media/{$video->folder->path}/{$baseName} no longer exists. Index your videos before running this task again.");
}
- // For use with private storage -> $filePath = str_replace('\\', '/', Storage::path('app/private/')) . 'media/' . $video->folder->path . "/" . basename($video->path);
- $this->fileMetaData = is_null($video->uuid) ? $this->getFileMetadata($filePath) : []; // Empty unless uuid is missing or duration is missing
- $uuid = $video->uuid ?? ''; // video has ? file has
+ // enforce loading file metadata if uuid is missing
+ $this->fileMetaData = is_null($video->uuid) ? $this->getFileMetadata($filePath, "uuid missing") : [];
+ $uuid = $video->uuid ?? '';
- // if the video in db or file does not have a valid uuid, it will add it in both the db and on the file.
- if (! Uuid::isValid($uuid ?? '')) {
- if (! isset($this->fileMetaData['tags']['uid']) && ! isset($this->fileMetaData['tags']['uuid'])) {
- $uuid = Str::uuid()->toString();
- $this->embedChain[] = new EmbedUidInMetadata($filePath, $uuid, $this->taskId, $video->id);
- } else {
- $uuid = $this->fileMetaData['tags']['uuid'];
- dump("Found UUID {$uuid}");
- $video->update(['uuid' => $uuid]); // If embedding, video is updated in the embed job
- }
+ // handle missing or invalid Uuid
+ if (! Uuid::isValid($uuid)) {
+ $uuid = $this->resolveMediaUuid($video, $filePath);
}
- $metadata = Metadata::where('uuid', $uuid)->orWhere('composite_id', $compositeId)->first();
+ $metadata = Metadata::firstOrCreate(
+ ['uuid' => $uuid],
+ ['video_id' => $video->id, 'composite_id' => $compositeId]
+ );
- if (! $metadata) {
- $metadata = Metadata::create(['uuid' => $uuid, 'composite_id' => $compositeId, 'video_id' => $video->id]);
- }
+ // $metadata = Metadata::where('uuid', $uuid)->orWhere('composite_id', $compositeId)->first();
- $stored = $metadata->toArray();
- $fileUpdated = ! is_null($metadata->date_scanned) && filemtime($filePath) > strtotime($metadata->date_scanned);
+ // if (! $metadata) {
+ // $metadata = Metadata::create(['uuid' => $uuid, 'composite_id' => $compositeId, 'video_id' => $video->id]);
+ // }
+
+ $stored = $metadata->toArray(); // Metadata from db
+ $changes = []; // Changes -> stored + changes . length has to be the same for every video so must generate defaults
+
+ $lastScannedAt = is_null($metadata->date_scanned) ? 0 : strtotime($metadata->date_scanned);
+ $fileUpdated = $metadata->date_scanned && filemtime($filePath) > $lastScannedAt;
if (is_null($metadata->uuid) || $fileUpdated) {
$changes['uuid'] = $uuid;
@@ -177,28 +181,69 @@ class VerifyFiles extends ManagedSubTask {
preg_match('!\d+!', $episodeRaw[0] ?? '', $episode);
if (is_null($metadata->duration) || $fileUpdated) {
- $this->confirmMetadata($filePath);
+ $this->confirmMetadata($filePath, "Duration is missing or file was updated");
$duration = $this->fileMetaData['format']['duration'] ?? $this->fileMetaData['streams'][0]['duration'] ?? $metadata->duration;
$changes['duration'] = is_numeric($duration) ? floor($duration) : null;
}
- if (is_null($metadata->subtitles_scanned_at) || $fileUpdated) {
- $this->confirmMetadata($filePath);
- $subtitleStreams = $subtitleScanner->extractSubtitleStreams($this->fileMetaData);
+ // Embedded Subtitles
+
+ $subtitleScanNeeded = is_null($metadata->subtitles_scanned_at) || $fileUpdated;
+
+ if ($subtitleScanNeeded) {
+ $this->confirmMetadata($filePath, "Subtitle scann date is missing or file was updated");
+ $embeddedSubtitleTransactions = $subtitleScanner->scanEmbeddedSubtitles($uuid, $this->fileMetaData);
+
+ foreach ($embeddedSubtitleTransactions as $tx) {
+ $subtitleTransactions[] = $tx;
+ }
+ }
+
+ // #region External Subtitles
+
+ // check if directory subtitle scan is needed
+
+ if (! isset($scannedDirectories[$folderPath])) {
+ $scannedDirectories[$folderPath] = [
+ 'last_modified' => filemtime($folderPath),
+ 'external_subtitles' => null,
+ ];
+ }
+
+ $dirUpdated = $scannedDirectories[$folderPath]['last_modified'] > $lastScannedAt;
+
+ // if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0
+
+ // Scan each directory once per batch
+ if (($subtitleScanNeeded || $dirUpdated) && is_null($scannedDirectories[$folderPath]['external_subtitles'])) {
+ $scannedDirectories[$folderPath]['external_subtitles'] = $subtitleScanner->findExternalSubtitlesInDirectory($folderPath);
+ }
+
+ // if subtitles in this directory have been scanned, check for matches
+ if (! is_null($scannedDirectories[$folderPath]['external_subtitles'])) {
+ $relevantSubtitles = array_filter(
+ $scannedDirectories[$folderPath]['external_subtitles'],
+ fn($sub) => strtolower($sub['media_filename']) === strtolower($fileName)
+ );
+
+ $externalSubtitleTransactions = $subtitleScanner->buildSubtitleTransactions(
+ $uuid,
+ $relevantSubtitles
+ );
- if (count($subtitleStreams) > 0) {
- $fileSubtitleTransactions = $subtitleScanner->buildSubtitleTransactions(
- $uuid,
- $subtitleStreams
- );
- $subtitleTransactions = array_merge($subtitleTransactions, $fileSubtitleTransactions);
+ foreach ($externalSubtitleTransactions as $tx) {
+ $subtitleTransactions[] = $tx;
}
+ }
+ if ($subtitleScanNeeded || $dirUpdated) {
$changes['subtitles_scanned_at'] = now();
}
+ // #endregion
+
if (! $is_audio && (is_null($metadata->resolution_height) || is_null($metadata->codec) || $fileUpdated)) {
- $this->confirmMetadata($filePath);
+ $this->confirmMetadata($filePath, "Not audio and missing resolution or codec or because fileUpdated was {$fileUpdated}");
foreach ($this->fileMetaData['streams'] as $stream) {
if (! isset($stream['codec_type']) || $stream['codec_type'] !== 'video' || ! isset($stream['width'])) {
continue;
@@ -349,7 +394,7 @@ class VerifyFiles extends ManagedSubTask {
$summary = 'Updated ' . count($metadataTransactions) . ' videos from id ' . ($metadataTransactions[0]['video_id']) . ' to ' . ($metadataTransactions[count($metadataTransactions) - 1]['video_id']);
if (count($subtitleTransactions) > 0) {
- Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'track_id'], ['language', 'codec']);
+ Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'source_key'], ['language', 'codec', 'is_default', 'is_forced', 'external_path']);
$summary .= ' and found ' . count($subtitleTransactions) . ' subtitle track(s)';
}
@@ -360,13 +405,13 @@ class VerifyFiles extends ManagedSubTask {
}
}
- public static function getFileMetadata($filePath) {
+ public static function getFileMetadata($filePath, $reason = "und") {
try {
// ? FFMPEG module with 6 test folders takes 35+ seconds but running the commands through shell takes 18 seconds
$ext = pathinfo($filePath, PATHINFO_EXTENSION);
if (config('app.env') === 'local') {
- dump('PULLING METADATA ' . $filePath);
+ dump('PULLING METADATA ' . $filePath . ' reason: ' . $reason);
}
$command = [
'ffprobe',
@@ -423,14 +468,14 @@ class VerifyFiles extends ManagedSubTask {
}
// Call get metadata when needed
- private function confirmMetadata($filePath) {
+ private function confirmMetadata(string $filePath, string $reason = 'und') {
if (is_null($this->fileMetaData) || count($this->fileMetaData) == 0) {
- $this->fileMetaData = $this->getFileMetadata($filePath);
+ $this->fileMetaData = $this->getFileMetadata($filePath, $reason);
}
}
private function getAudioDescription($filePath) {
- $this->confirmMetadata($filePath);
+ $this->confirmMetadata($filePath, "Get audio description");
$tags = $this->fileMetaData['tags'] ?? [];
$streams = $this->fileMetaData['streams'] ?? [];
@@ -560,4 +605,22 @@ class VerifyFiles extends ManagedSubTask {
return $mimeType;
}
+
+ protected function getAbsoluteMediaPath($media) {
+ return str_replace('\\', '/', Storage::disk('public')->path('')) . "media/{$media->folder->path}/" . basename($media->path);
+ }
+
+ protected function resolveMediaUuid($media, $filePath) {
+ // if the media in db or file does not have a valid uuid, it will add it in both the db and on the file.
+ if (! isset($this->fileMetaData['tags']['uuid'])) {
+ $uuid = Str::uuid()->toString();
+ $this->embedChain[] = new EmbedUidInMetadata($filePath, $uuid, $this->taskId, $media->id);
+ } else {
+ $uuid = $this->fileMetaData['tags']['uuid'];
+ dump("Found UUID on file {$uuid}");
+ $media->update(['uuid' => $uuid]); // If embedding, media file is updated in the embed job
+ }
+
+ return $uuid;
+ }
}
diff --git a/app/Services/Subtitles/SubtitleScanner.php b/app/Services/Subtitles/SubtitleScanner.php
index 9a5e0d09..43b3179a 100644
--- a/app/Services/Subtitles/SubtitleScanner.php
+++ b/app/Services/Subtitles/SubtitleScanner.php
@@ -2,14 +2,24 @@
namespace App\Services\Subtitles;
+use App\Enums\SubtitleSource;
use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Storage;
// Per Metadata Row
class SubtitleScanner {
- public function extractSubtitleStreams(array $fileMetadata): array {
+
+ public function scanEmbeddedSubtitles(string $uuid, array $fileMetaData): array {
+ $subtitleStreams = $this->filterSubtitleStreams($fileMetaData);
+ return count($subtitleStreams) > 0
+ ? $this->buildSubtitleTransactions($uuid, $subtitleStreams)
+ : [];
+ }
+
+ public function filterSubtitleStreams(array $fileMetaData): array {
return array_filter(
- $fileMetadata['streams'] ?? [],
- fn ($stream) => ($stream['codec_type'] ?? null) === 'subtitle'
+ $fileMetaData['streams'] ?? [],
+ fn($stream) => ($stream['codec_type'] ?? null) === 'subtitle'
);
}
@@ -25,16 +35,91 @@ class SubtitleScanner {
continue;
}
- $transactions[] = [
+ $transaction = [
'metadata_uuid' => $uuid,
- 'track_id' => $stream['index'],
+ 'track_id' => $stream['index'], // is 0 if external
'language' => $stream['tags']['language'] ?? 'und',
'codec' => $stream['codec_name'],
'is_default' => ($stream['disposition']['default'] ?? 0) === 1,
'is_forced' => ($stream['disposition']['forced'] ?? 0) === 1,
+ 'external_path' => null,
+ 'source_key' => SubtitleSource::EMBEDDED->makeKey($stream['index'])
];
+
+ if (isset($stream['external_path'])) {
+ $transaction['external_path'] = $stream['external_path'];
+ $transaction['source_key'] = SubtitleSource::EXTERNAL->makeKey($stream['external_path']);
+ }
+
+ $transactions[] = $transaction;
}
return $transactions;
}
+
+ /**
+ * @param string $directoryPath the relative directory of the given metadata row /library/folder
+ * @return array a list of arrays that resemble ffprobe output for each matching subtitle file
+ */
+ public function findExternalSubtitlesInDirectory(string $directoryPath): array {
+ $extensions = ['srt', 'vtt', 'ass', 'ssa', 'sub'];
+ $externalStreams = [];
+
+ if (config('app.env') === 'local') dump("SCANNING FOR SUBTITLES IN $directoryPath");
+
+ foreach ($extensions as $ext) {
+ // match pattern basename.*.ext (ex/ movie.*.srt)
+ $pattern = "{$directoryPath}/*.{$ext}";
+ $files = glob($pattern);
+
+ foreach ($files as $file) {
+ $filename = basename($file);
+ $parts = explode('.', $filename);
+
+ // Parse: movie.en.forced.srt
+ // parts: [movie, en, forced, srt]
+ $media_filename = array_shift($parts); // remove related media file name
+ $extension = array_pop($parts); // remove extension
+
+ if (count($parts) < 1) {
+ continue;
+ }
+
+ $language = $parts[0];
+ $flags = array_slice($parts, 1);
+
+ $isForced = in_array('forced', $flags);
+ $isDefault = in_array('default', $flags);
+
+ // Build ffprobe output
+ $externalStreams[] = [
+ 'media_filename' => $media_filename,
+ 'index' => 0,
+ 'codec_type' => 'subtitle',
+ 'codec_name' => $this->getCodecFromExtension($extension),
+ 'tags' => [
+ 'language' => $language,
+ ],
+ 'disposition' => [
+ 'forced' => $isForced ? 1 : 0,
+ 'default' => $isDefault ? 1 : 0,
+ ],
+ 'external_path' => str_replace(str_replace('\\', '/', Storage::disk('public')->path('')), '', $file), // store relative location
+ ];
+ }
+ }
+
+ return $externalStreams;
+ }
+
+ private function getCodecFromExtension(string $ext): string {
+ return match ($ext) {
+ 'vtt' => 'webvtt',
+ 'srt' => 'subrip',
+ 'ass' => 'ass',
+ 'ssa' => 'ssa',
+ 'sub' => 'dvd_subtitle',
+ default => 'unknown',
+ };
+ }
}
diff --git a/database/migrations/2026_01_19_195008_add_external_subtitle_identifier_to_subtitles.php b/database/migrations/2026_01_19_195008_add_external_subtitle_identifier_to_subtitles.php
new file mode 100644
index 00000000..e596597e
--- /dev/null
+++ b/database/migrations/2026_01_19_195008_add_external_subtitle_identifier_to_subtitles.php
@@ -0,0 +1,53 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration {
+ /**
+ * Run the migrations.
+ */
+ public function up(): void {
+ Schema::table('subtitles', function (Blueprint $table) {
+ $table->text('external_path')->nullable();
+ $table->text('source_key')->nullable();
+ });
+
+ DB::statement("
+ UPDATE subtitles
+ SET source_key =
+ CASE
+ WHEN track_id > 0 THEN 'embedded:' || track_id
+ ELSE 'external:' || COALESCE(external_path, 'unknown')
+ END
+ ");
+
+ Schema::table('subtitles', function (Blueprint $table) {
+ $table->text('source_key')->nullable(false)->change();
+ });
+
+ Schema::table('subtitles', function (Blueprint $table) {
+ $table->dropUnique(['metadata_uuid', 'track_id']);
+ $table->unique(['metadata_uuid', 'source_key']);
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ */
+ public function down(): void {
+ Schema::table('subtitles', function (Blueprint $table) {
+ $table->dropUnique(['metadata_uuid', 'source_key']);
+ });
+
+ Schema::table('subtitles', function (Blueprint $table) {
+ $table->unique(['metadata_uuid', 'track_id']);
+ });
+
+ Schema::table('subtitles', function (Blueprint $table) {
+ $table->dropColumn(['external_path', 'source_key']);
+ });
+ }
+};
diff --git a/tests/Unit/Services/Subtitles/SubtitleScannerTest.php b/tests/Unit/Services/Subtitles/SubtitleScannerTest.php
index 80496e3f..75c1edbb 100644
--- a/tests/Unit/Services/Subtitles/SubtitleScannerTest.php
+++ b/tests/Unit/Services/Subtitles/SubtitleScannerTest.php
@@ -23,7 +23,7 @@ class SubtitleScannerTest extends TestCase {
],
];
- $streams = $this->scanner->extractSubtitleStreams($metadata);
+ $streams = $this->scanner->filterSubtitleStreams($metadata);
$this->assertCount(1, $streams);
foreach ($streams as $stream) {