A self-hosted media server and file management system
Vue ffmpegfile-serverjellyfinlaravelmediamedia-playermedia-servermediabrowsermediaservermusicmusic-librarymusic-playermusicplayerself-hostedvuejs
feat: optimise subtitle scanning process
aminnausin9336465652b0
diff --git a/app/Data/Subtitles/SubtitleScanTarget.php b/app/Data/Subtitles/SubtitleScanTarget.phpnew file mode 100644index 00000000..41d8e3a1--- /dev/null+++ b/app/Data/Subtitles/SubtitleScanTarget.php@@ -0,0 +1,11 @@+<?php++namespace App\Data\Subtitles;++class SubtitleScanTarget {+ public function __construct(+ public readonly string $uuid,+ public readonly bool $fileUpdated,+ ) {+ }+}diff --git a/app/Jobs/Utility/Subtitles/ScanSubtitles.php b/app/Jobs/Utility/Subtitles/ScanSubtitles.phpindex 74b57d4e..1df2362d 100644--- a/app/Jobs/Utility/Subtitles/ScanSubtitles.php+++ b/app/Jobs/Utility/Subtitles/ScanSubtitles.php@@ -2,6 +2,7 @@namespace App\Jobs\Utility\Subtitles;+use App\Data\Subtitles\SubtitleScanTarget;use App\Enums\TaskStatus;use App\Jobs\ManagedSubTask;use App\Jobs\VerifyFiles;@@ -11,19 +12,25 @@ use App\Models\Subtitle;use App\Services\Subtitles\SubtitleManager;use App\Services\Subtitles\SubtitleScanner;use App\Services\TaskService;-use Illuminate\Database\Eloquent\ModelNotFoundException;+use Illuminate\Support\Facades\DB;+use Illuminate\Support\Facades\Log;class ScanSubtitles extends ManagedSubTask {- private string $uuid;+ private array $targets;+ private string $folderPath;- private bool $externalOnly;+ private bool $scanExternal;++ /** @param SubtitleScanTarget[] $targets */+ public function __construct(int $taskId, array $targets, string $folderPath, bool $scanExternal) {+ $subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Scan Subtitles for folder ' . $folderPath]);- public function __construct(int $taskId, string $uuid, bool $externalOnly = false) {- $subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Scan Subtitles for uuid ' . $uuid]);$this->taskId = $taskId;$this->subTaskId = $subTask->id;- $this->uuid = $uuid;- $this->externalOnly = $externalOnly;++ $this->targets = $targets;+ $this->folderPath = $folderPath;+ $this->scanExternal = $scanExternal;}public function handle(TaskService $taskService, SubtitleScanner $subtitleScanner, SubtitleManager $manager): void {@@ -61,51 +68,84 @@ class ScanSubtitles extends ManagedSubTask {* generate font information (as a json on metadata?)*/private function handleScanSubtitles(SubtitleScanner $subtitleScanner, SubtitleManager $manager): string {- $metadata = Metadata::with('video.folder')->where('uuid', $this->uuid)->firstOrFail();- $media = $metadata->video;+ $summary = "Scanning for subtitle tracks on " . count($this->targets) . " targets";+ $batchTransactions = [];+ $scannedUuids = [];- if (is_null($media)) {- throw new ModelNotFoundException("Media file missing for uuid {$this->uuid}");- }+ $metadataMap = Metadata::with('video.folder')+ ->whereIn('uuid', array_column($this->targets, 'uuid'))+ ->get()+ ->keyBy('uuid');++ $externalSubtitles = $this->scanExternal+ ? $subtitleScanner->findExternalSubtitlesInDirectory($this->folderPath)+ : [];- $manager->purgeSubtitles($metadata, $this->externalOnly);- // absolute file path in storage- $filePath = VerifyFiles::getAbsoluteMediaPath($media);- $fileName = pathinfo($filePath, PATHINFO_FILENAME);- $folderPath = dirname($filePath);- $fileMetaData = VerifyFiles::getFileMetadata($filePath, 'Subtitle Scan');+ foreach ($this->targets as $target) {+ $subtitleTransactions = [];- // Embedded Subtitles+ $uuid = $target->uuid;+ $fileUpdated = $target->fileUpdated;- $subtitleTransactions = $this->externalOnly ? [] : $subtitleScanner->scanEmbeddedSubtitles($this->uuid, $fileMetaData);+ $metadata = $metadataMap[$uuid] ?? null;+ $media = $metadata?->video;- // External Subtitles+ if (is_null($metadata)) {+ Log::error("Metadata missing when scanning for subtitles", ["uuid" => $uuid]);+ continue;+ }++ if (is_null($media)) {+ Log::error("Media file missing on metadata instance when scanning for subtitles", ["uuid" => $metadata->uuid, "title" => $metadata->title]);+ $summary .= "\n\nMedia file missing for " . $metadata->title . " with uuid " . $metadata->uuid;+ continue;+ }- // Previous Optimisation: if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0- // Now: every directory is scanned again per file. The scan is fast and should not have a high cost until you hit a large file count. Ideally this should be optimised again but without the batch structure it is difficult?- // If it runs with horizon, it should be fast. In a synchronous queue it will probably be slow- // Maybe I can batch this job by directory? or separate the external subtitle portion- $externalSubtitles = $subtitleScanner->findExternalSubtitlesInDirectory($folderPath);- $relevantExternal = array_filter(- $externalSubtitles,- fn($sub) => strtolower($sub['media_filename']) === strtolower($fileName)- );+ $manager->purgeSubtitles($metadata, externalOnly: !$fileUpdated); // only purge external subtitle files if the file was not updated. Getting to this point- $externalTransactions = $subtitleScanner->buildSubtitleTransactions($this->uuid, $relevantExternal);- $subtitleTransactions = array_merge($subtitleTransactions, $externalTransactions);+ // absolute file path in storage+ $filePath = VerifyFiles::getAbsoluteMediaPath($media);+ $fileName = pathinfo($filePath, PATHINFO_FILENAME);+ // $folderPath = dirname($filePath);++ // Embedded Subtitles - scanned only if file was updated+ if ($fileUpdated) {+ // TODO: Cache this output from index or subsequent verify jobs as json in the model and never call from here+ $fileMetaData = VerifyFiles::getFileMetadata($filePath, 'Subtitle Scan');+ $subtitleTransactions = $subtitleScanner->scanEmbeddedSubtitles($uuid, $fileMetaData);+ }++ // External Subtitles - directory scanned once per job and filtered per media file+ if (!empty($externalSubtitles)) {+ $relevantExternal = array_filter(+ $externalSubtitles,+ fn($sub) => strtolower($sub['media_filename']) === strtolower($fileName)+ );++ $subtitleTransactions = array_merge($subtitleTransactions, $subtitleScanner->buildSubtitleTransactions($uuid, $relevantExternal));+ }- try {if (!empty($subtitleTransactions)) {- Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'source_key'], ['language', 'title', 'codec', 'is_default', 'is_forced', 'external_path']);+ $summary .= "\n\nFound " . count($subtitleTransactions) . ' subtitle track(s) for uuid ' . $uuid;+ $batchTransactions = array_merge($batchTransactions, $subtitleTransactions);}+ $scannedUuids[] = $uuid;+ }++ try {+ DB::transaction(function () use ($batchTransactions, $scannedUuids) {+ if (!empty($batchTransactions)) {+ Subtitle::upsert($batchTransactions, ['metadata_uuid', 'source_key'], ['language', 'title', 'codec', 'is_default', 'is_forced', 'external_path']);+ }- $metadata->update(['subtitles_scanned_at' => now()]);+ Metadata::whereIn('uuid', $scannedUuids)+ ->update(['subtitles_scanned_at' => now()]);+ });- return 'Found ' . count($subtitleTransactions) . ' subtitle track(s) for uuid ' . $this->uuid;+ return $summary . "\n\n\nTotal subtitle tracks upserted: " . count($batchTransactions);} catch (\Throwable $th) {- $keys = array_column($subtitleTransactions, 'source_key');- $this->handleError('Error inserting subtitles for ' . $this->uuid, $th, $keys, count($subtitleTransactions));+ $this->handleError('Error inserting subtitle tracks', $th, $scannedUuids, count($batchTransactions));}}}diff --git a/app/Jobs/VerifyFiles.php b/app/Jobs/VerifyFiles.phpindex 4007ff53..8b0aef49 100755--- a/app/Jobs/VerifyFiles.php+++ b/app/Jobs/VerifyFiles.php@@ -2,6 +2,7 @@namespace App\Jobs;+use App\Data\Subtitles\SubtitleScanTarget;use App\Enums\MediaType;use App\Enums\TaskStatus;use App\Jobs\Utility\Subtitles\ScanSubtitles;@@ -18,9 +19,10 @@ use Symfony\Component\Process\Exception\ProcessFailedException;use Symfony\Component\Process\Process;class VerifyFiles extends ManagedSubTask {+ protected $subtitleScanChain = [];protected $embedChain = [];- protected $subtitleScanChain = [];+ protected $scannedDirectories = []; // TODO: this should really go in the indexer or a broken down part of the indexerprotected $fileMetaData = [];@@ -41,6 +43,18 @@ class VerifyFiles extends ManagedSubTask {try {$summary = $this->verifyFiles($taskService);++ foreach ($this->scannedDirectories as $folderPath => $dirData) {+ if (empty($dirData['targets'])) continue;++ $this->subtitleScanChain[] = new ScanSubtitles(+ taskId: $this->taskId,+ targets: $dirData['targets'],+ folderPath: $folderPath,+ scanExternal: $dirData['dir_updated'],+ );+ }+$taskCountUpdates = ['sub_tasks_complete' => '++','sub_tasks_total' => count($this->embedChain) + count($this->subtitleScanChain),@@ -69,7 +83,6 @@ class VerifyFiles extends ManagedSubTask {private function verifyFiles(TaskService $taskService) {$metadataTransactions = [];- $scannedDirectories = []; // TODO: this should really go in the indexer or a broken down part of the indexer$error = false;$index = 0;@@ -185,22 +198,29 @@ class VerifyFiles extends ManagedSubTask {// check if directory subtitle scan is needed- if (! isset($scannedDirectories[$folderPath])) {- $scannedDirectories[$folderPath] = [+ if (! isset($this->scannedDirectories[$folderPath])) {+ $this->scannedDirectories[$folderPath] = ['last_modified' => filemtime($folderPath),+ 'targets' => [],+ 'dir_updated' => false,];}// if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0- $dirUpdated = $metadata->file_scanned_at && $scannedDirectories[$folderPath]['last_modified'] > $metadata->file_scanned_at->timestamp;+ $dirUpdated = $metadata->file_scanned_at && $this->scannedDirectories[$folderPath]['last_modified'] > $metadata->file_scanned_at->timestamp;$subtitleScanNeeded = ! $is_audio && (is_null($metadata->subtitles_scanned_at) || $fileUpdated || $dirUpdated);if ($subtitleScanNeeded) {- // $changes['subtitles_scanned_at'] = null;- $purgeExternalOnly = $dirUpdated && ! $fileUpdated;- dump("Scan Subtitles Job", ["last scan date" => $metadata->subtitles_scanned_at, "fupdate" => $fileUpdated, "dupdate" => $dirUpdated, "name" => $fileName]);- $this->subtitleScanChain[] = new ScanSubtitles($this->taskId, $uuid, $purgeExternalOnly);+ $changes['subtitles_scanned_at'] = null;+ $this->scannedDirectories[$folderPath]['targets'][] = new SubtitleScanTarget(+ uuid: $uuid,+ fileUpdated: $fileUpdated,+ );++ if ($dirUpdated) {+ $this->scannedDirectories[$folderPath]['dir_updated'] = true;+ }}// #endregion