A self-hosted media server and file management system
Vue ffmpegfile-serverjellyfinlaravelmediamedia-playermedia-servermediabrowsermediaservermusicmusic-librarymusic-playermusicplayerself-hostedvuejs
feat: clear old subtitles on new subtitle scan v1
aminnausin96cd5dae6ebf
diff --git a/app/Jobs/Utility/Subtitles/ScanSubtitles.php b/app/Jobs/Utility/Subtitles/ScanSubtitles.phpnew file mode 100644index 00000000..74b57d4e--- /dev/null+++ b/app/Jobs/Utility/Subtitles/ScanSubtitles.php@@ -0,0 +1,111 @@+<?php++namespace App\Jobs\Utility\Subtitles;++use App\Enums\TaskStatus;+use App\Jobs\ManagedSubTask;+use App\Jobs\VerifyFiles;+use App\Models\Metadata;+use App\Models\SubTask;+use App\Models\Subtitle;+use App\Services\Subtitles\SubtitleManager;+use App\Services\Subtitles\SubtitleScanner;+use App\Services\TaskService;+use Illuminate\Database\Eloquent\ModelNotFoundException;++class ScanSubtitles extends ManagedSubTask {+ private string $uuid;++ private bool $externalOnly;++ public function __construct(int $taskId, string $uuid, bool $externalOnly = false) {+ $subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Scan Subtitles for uuid ' . $uuid]);+ $this->taskId = $taskId;+ $this->subTaskId = $subTask->id;+ $this->uuid = $uuid;+ $this->externalOnly = $externalOnly;+ }++ public function handle(TaskService $taskService, SubtitleScanner $subtitleScanner, SubtitleManager $manager): void {+ if (! $this->beginSubTask($taskService)) {+ return;+ }++ try {+ $summary = $this->handleScanSubtitles($subtitleScanner, $manager);+ $this->completeSubTask($taskService, $summary);+ } catch (\Throwable $th) {+ $this->failSubTask($taskService, $th);+ throw $th;+ }+ }++ /**+ * This will handle clearing stale subtitles and indexing new or updated files+ *+ * Responsibilities+ *+ * Clear existing subtitle files (purge directory)+ * Clear related subtitle rows in db+ *+ * Grab extended metadata+ * Extract info for all embeded subtitle tracks and build transactions+ * Check file directory for related subtitles+ * build external subtitle transactions+ * bulk upsert embedded and external subtitle transactions+ *+ * Optional/Future Responsibilities+ *+ * bulk extract subtitle files from media file (based on user config)+ * bulk extract font files from media file (based on user config)+ * generate font information (as a json on metadata?)+ */+ private function handleScanSubtitles(SubtitleScanner $subtitleScanner, SubtitleManager $manager): string {+ $metadata = Metadata::with('video.folder')->where('uuid', $this->uuid)->firstOrFail();+ $media = $metadata->video;++ if (is_null($media)) {+ throw new ModelNotFoundException("Media file missing for uuid {$this->uuid}");+ }++ $manager->purgeSubtitles($metadata, $this->externalOnly);++ // absolute file path in storage+ $filePath = VerifyFiles::getAbsoluteMediaPath($media);+ $fileName = pathinfo($filePath, PATHINFO_FILENAME);+ $folderPath = dirname($filePath);+ $fileMetaData = VerifyFiles::getFileMetadata($filePath, 'Subtitle Scan');++ // Embedded Subtitles++ $subtitleTransactions = $this->externalOnly ? [] : $subtitleScanner->scanEmbeddedSubtitles($this->uuid, $fileMetaData);++ // External Subtitles++ // Previous Optimisation: if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0+ // Now: every directory is scanned again per file. The scan is fast and should not have a high cost until you hit a large file count. Ideally this should be optimised again but without the batch structure it is difficult?+ // If it runs with horizon, it should be fast. In a synchronous queue it will probably be slow+ // Maybe I can batch this job by directory? or separate the external subtitle portion+ $externalSubtitles = $subtitleScanner->findExternalSubtitlesInDirectory($folderPath);+ $relevantExternal = array_filter(+ $externalSubtitles,+ fn($sub) => strtolower($sub['media_filename']) === strtolower($fileName)+ );++ $externalTransactions = $subtitleScanner->buildSubtitleTransactions($this->uuid, $relevantExternal);+ $subtitleTransactions = array_merge($subtitleTransactions, $externalTransactions);++ try {+ if (!empty($subtitleTransactions)) {+ Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'source_key'], ['language', 'title', 'codec', 'is_default', 'is_forced', 'external_path']);+ }++ $metadata->update(['subtitles_scanned_at' => now()]);++ return 'Found ' . count($subtitleTransactions) . ' subtitle track(s) for uuid ' . $this->uuid;+ } catch (\Throwable $th) {+ $keys = array_column($subtitleTransactions, 'source_key');+ $this->handleError('Error inserting subtitles for ' . $this->uuid, $th, $keys, count($subtitleTransactions));+ }+ }+}diff --git a/app/Jobs/VerifyFiles.php b/app/Jobs/VerifyFiles.phpindex a11292d8..4007ff53 100755--- a/app/Jobs/VerifyFiles.php+++ b/app/Jobs/VerifyFiles.php@@ -4,14 +4,12 @@ namespace App\Jobs;use App\Enums\MediaType;use App\Enums\TaskStatus;+use App\Jobs\Utility\Subtitles\ScanSubtitles;use App\Models\Metadata;use App\Models\Record;use App\Models\SubTask;-use App\Models\Subtitle;-use App\Services\Subtitles\SubtitleScanner;use App\Services\TaskService;use Carbon\Carbon;-use Illuminate\Support\Facades\Bus;use Illuminate\Support\Facades\Log;use Illuminate\Support\Facades\Storage;use Illuminate\Support\Str;@@ -22,6 +20,8 @@ use Symfony\Component\Process\Process;class VerifyFiles extends ManagedSubTask {protected $embedChain = [];+ protected $subtitleScanChain = [];+protected $fileMetaData = [];/**@@ -34,21 +34,32 @@ class VerifyFiles extends ManagedSubTask {$this->subTaskId = $subTask->id;}- public function handle(TaskService $taskService, SubtitleScanner $subtitleScanner): void {+ public function handle(TaskService $taskService): void {if (! $this->beginSubTask($taskService)) {return;}try {- $summary = $this->verifyFiles($taskService, $subtitleScanner);- $taskCountUpdates = count($this->embedChain) ? ['sub_tasks_complete' => '++', 'sub_tasks_total' => count($this->embedChain), 'sub_tasks_pending' => count($this->embedChain)] : ['sub_tasks_complete' => '++'];+ $summary = $this->verifyFiles($taskService);+ $taskCountUpdates = [+ 'sub_tasks_complete' => '++',+ 'sub_tasks_total' => count($this->embedChain) + count($this->subtitleScanChain),+ 'sub_tasks_pending' => count($this->embedChain) + count($this->subtitleScanChain),+ ];++ if (count($this->subtitleScanChain)) {+ $summary .= ' and queued ' . count($this->subtitleScanChain) . ' subtitle scans';+ }$this->completeSubTask($taskService, $summary, $taskCountUpdates);// Starts other subtasks after updating current subtask and parent subtask states// The parent task "ends" after the batch empties so in theory, this should delay that anyways and does not need to run before the task updateforeach ($this->embedChain as $embedTask) {- Bus::dispatch($embedTask);+ $this->batch()->add($embedTask);+ }+ foreach ($this->subtitleScanChain as $scanTask) {+ $this->batch()->add($scanTask);}} catch (\Throwable $th) {$this->failSubTask($taskService, $th);@@ -56,10 +67,8 @@ class VerifyFiles extends ManagedSubTask {}}- private function verifyFiles(TaskService $taskService, SubtitleScanner $subtitleScanner) {+ private function verifyFiles(TaskService $taskService) {$metadataTransactions = [];- $subtitleTransactions = [];- $scannedSubtitleUuids = [];$scannedDirectories = []; // TODO: this should really go in the indexer or a broken down part of the indexer$error = false;@@ -172,65 +181,32 @@ class VerifyFiles extends ManagedSubTask {$changes['duration'] = is_numeric($duration) ? floor($duration) : null;}- // Embedded Subtitles-- $subtitleScanNeeded = ! $is_audio && is_null($metadata->subtitles_scanned_at) || $fileUpdated;-- if ($subtitleScanNeeded) {- $scannedSubtitleUuids[] = $uuid;- $this->confirmMetadata($filePath, 'Subtitle scan date is missing or file was updated');- $embeddedSubtitleTransactions = $subtitleScanner->scanEmbeddedSubtitles($uuid, $this->fileMetaData);-- foreach ($embeddedSubtitleTransactions as $tx) {- $subtitleTransactions[] = $tx;- }- }-- // #region External Subtitles+ // #region Subtitles// check if directory subtitle scan is neededif (! isset($scannedDirectories[$folderPath])) {$scannedDirectories[$folderPath] = ['last_modified' => filemtime($folderPath),- 'external_subtitles' => null,];}- $dirUpdated = $metadata->file_scanned_at && $scannedDirectories[$folderPath]['last_modified'] > $metadata->file_scanned_at->timestamp;-// if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0+ $dirUpdated = $metadata->file_scanned_at && $scannedDirectories[$folderPath]['last_modified'] > $metadata->file_scanned_at->timestamp;- // Scan each directory once per batch- if (! $is_audio && ($subtitleScanNeeded || $dirUpdated) && is_null($scannedDirectories[$folderPath]['external_subtitles'])) {- $scannedDirectories[$folderPath]['external_subtitles'] = $subtitleScanner->findExternalSubtitlesInDirectory($folderPath);- }-- // if subtitles in this directory have been scanned, check for matches- if (! is_null($scannedDirectories[$folderPath]['external_subtitles'])) {- $relevantSubtitles = array_filter(- $scannedDirectories[$folderPath]['external_subtitles'],- fn ($sub) => strtolower($sub['media_filename']) === strtolower($fileName)- );-- $externalSubtitleTransactions = $subtitleScanner->buildSubtitleTransactions(- $uuid,- $relevantSubtitles- );-- foreach ($externalSubtitleTransactions as $tx) {- $subtitleTransactions[] = $tx;- }- }+ $subtitleScanNeeded = ! $is_audio && (is_null($metadata->subtitles_scanned_at) || $fileUpdated || $dirUpdated);- if ($subtitleScanNeeded || $dirUpdated) {- $changes['subtitles_scanned_at'] = now();+ if ($subtitleScanNeeded) {+ // $changes['subtitles_scanned_at'] = null;+ $purgeExternalOnly = $dirUpdated && ! $fileUpdated;+ dump("Scan Subtitles Job", ["last scan date" => $metadata->subtitles_scanned_at, "fupdate" => $fileUpdated, "dupdate" => $dirUpdated, "name" => $fileName]);+ $this->subtitleScanChain[] = new ScanSubtitles($this->taskId, $uuid, $purgeExternalOnly);}// #endregionif (! $is_audio && (is_null($metadata->resolution_height) || is_null($metadata->codec) || $fileUpdated)) {- $this->confirmMetadata($filePath, "Not audio and missing resolution or codec or because fileUpdated was {$fileUpdated}");+ $this->confirmMetadata($filePath, "Not audio and missing resolution or codec or because fileUpdated was {" . ($fileUpdated ? 'true' : 'false') . "}");foreach ($this->fileMetaData['streams'] as $stream) {if (! isset($stream['codec_type']) || $stream['codec_type'] !== 'video' || ! isset($stream['width'])) {continue;@@ -373,11 +349,6 @@ class VerifyFiles extends ManagedSubTask {$summary = 'Updated ' . count($metadataTransactions) . ' videos from id ' . ($metadataTransactions[0]['video_id']) . ' to ' . ($metadataTransactions[count($metadataTransactions) - 1]['video_id']);- if (! empty($subtitleTransactions)) {- Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'source_key'], ['language', 'title', 'codec', 'is_default', 'is_forced', 'external_path']);- $summary .= ' and found ' . count($subtitleTransactions) . ' subtitle track(s)';- }-return $summary;} catch (\Throwable $th) {$ids = array_column($metadataTransactions, 'id');@@ -565,50 +536,6 @@ class VerifyFiles extends ManagedSubTask {return $result;}- /**- * Unimplemented- *- * In theory, it goes through every metadata by uuid which was updated, and checks to see which of its existing subtitle rows were seen in the scan- * It deletes any rows and associated files that were not seen- *- * This would then clear old subtitle files if the media file was updated to something that did not include them- *- * I am not sure how or why of if this should even be part of the process, so it is unimplemented for now- * Code left as reference- */- private function pruneSubtitles(array $subtitleTransactions, array $scannedSubtitleUuids): string {- $subtitlesByUuid = collect($subtitleTransactions)->groupBy('metadata_uuid');- $deletedSubtitles = 0;-- foreach ($scannedSubtitleUuids as $uuid) {- $validKeys = $subtitlesByUuid[$uuid]?->pluck('source_key') ?? collect(); // Get seen source_keys (unique)-- $stale = $validKeys->isEmpty()- ? Subtitle::where('metadata_uuid', $uuid)->get() // If none were seen, delete every associated subtitle row- : Subtitle::where('metadata_uuid', $uuid)->whereNotIn('source_key', $validKeys)->get(); // Otherwise only unseen rows-- foreach ($stale as $subtitle) {- if ($subtitle->path) {- $dir = dirname($subtitle->path);- $base = pathinfo($subtitle->path, PATHINFO_FILENAME); // "0.en" or "2"-- // should probably limit by top directory of private/metadata/media/... and have some handling for malicious paths- // this would have been the first place in the entire app to delete files- foreach (glob("{$dir}/{$base}.*") as $file) {- dump('Would delete file', $file);- // if (file_exists($file)) unlink($file);- }- }- // also the file deletion is not atomic with the db deletion and can fail inbetween, but that should not truly matter because the file is regenerated if not found anyway- Log::info('Deleted subtitle', ['path' => $subtitle->path, 'track' => $subtitle->track_id, 'title' => $subtitle->title]);- // $subtitle->delete();- }- $deletedSubtitles += count($stale);- }-- return $deletedSubtitles > 0 ? " and deleted $deletedSubtitles subtitle track(s)" : '';- }-protected function extractMimeType($filePath) {$finfo = new \finfo(FILEINFO_MIME_TYPE);$mimeType = $finfo->file($filePath);@@ -630,7 +557,7 @@ class VerifyFiles extends ManagedSubTask {return $mimeType;}- protected function getAbsoluteMediaPath($media) {+ public static function getAbsoluteMediaPath($media) {return str_replace('\\', '/', Storage::disk('public')->path('')) . "media/{$media->folder->path}/" . basename($media->path);}diff --git a/app/Models/Metadata.php b/app/Models/Metadata.phpindex 24f94d1f..3172d644 100644--- a/app/Models/Metadata.php+++ b/app/Models/Metadata.php@@ -99,6 +99,7 @@ class Metadata extends Model {// Date Id'file_scanned_at','file_modified_at',+ 'subtitles_scanned_at','first_file_modified_at','edited_at',];diff --git a/app/Services/Subtitles/SubtitleManager.php b/app/Services/Subtitles/SubtitleManager.phpindex 2e0310f0..caf92481 100644--- a/app/Services/Subtitles/SubtitleManager.php+++ b/app/Services/Subtitles/SubtitleManager.php@@ -3,11 +3,12 @@namespace App\Services\Subtitles;use App\Models\Metadata;+use App\Models\Subtitle;use Illuminate\Support\Facades\Log;use Illuminate\Support\Facades\Storage;class SubtitleManager {- public function reset(Metadata $metadata): void {+ public function resetSubtitleFiles(Metadata $metadata): void {$this->deleteSubtitleFiles($metadata);$metadata->subtitles()->update([@@ -15,6 +16,28 @@ class SubtitleManager {]);}+ public function purgeSubtitles(Metadata $metadata, bool $externalOnly = false): void {+ if ($externalOnly) {+ $this->deleteExternalSubtitleFiles($metadata);+ } else {+ // atomically delete all subtitle rows and files related to metadata+ $this->deleteSubtitleFiles($metadata);+ $metadata->subtitles()->delete();+ }+ }++ private function deleteExternalSubtitleFiles(Metadata $metadata): void {+ $metadata->subtitles()+ ->whereNotNull('external_path')+ ->each(function (Subtitle $subtitle) {+ if ($subtitle->path) {+ Storage::disk('local')->delete($subtitle->path);+ }+ });++ $metadata->subtitles()->whereNotNull('external_path')->delete();+ }+private function deleteSubtitleFiles(Metadata $metadata): void {$directory = SubtitlePath::buildDirectory($metadata->uuid);Log::info('Cleared Subtitle Directory', ['uuid' => $metadata->uuid, 'title' => $metadata->title, 'dir' => $directory]);