Skip to content

A self-hosted media server and file management system

feat: clear old subtitles on new subtitle scan v1

aminnausin96cd5dae6ebf
diff --git a/app/Jobs/Utility/Subtitles/ScanSubtitles.php b/app/Jobs/Utility/Subtitles/ScanSubtitles.php
new file mode 100644
index 00000000..74b57d4e
--- /dev/null
+++ b/app/Jobs/Utility/Subtitles/ScanSubtitles.php
@@ -0,0 +1,111 @@
+<?php
+
+namespace App\Jobs\Utility\Subtitles;
+
+use App\Enums\TaskStatus;
+use App\Jobs\ManagedSubTask;
+use App\Jobs\VerifyFiles;
+use App\Models\Metadata;
+use App\Models\SubTask;
+use App\Models\Subtitle;
+use App\Services\Subtitles\SubtitleManager;
+use App\Services\Subtitles\SubtitleScanner;
+use App\Services\TaskService;
+use Illuminate\Database\Eloquent\ModelNotFoundException;
+
+class ScanSubtitles extends ManagedSubTask {
+ private string $uuid;
+
+ private bool $externalOnly;
+
+ public function __construct(int $taskId, string $uuid, bool $externalOnly = false) {
+ $subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Scan Subtitles for uuid ' . $uuid]);
+ $this->taskId = $taskId;
+ $this->subTaskId = $subTask->id;
+ $this->uuid = $uuid;
+ $this->externalOnly = $externalOnly;
+ }
+
+ public function handle(TaskService $taskService, SubtitleScanner $subtitleScanner, SubtitleManager $manager): void {
+ if (! $this->beginSubTask($taskService)) {
+ return;
+ }
+
+ try {
+ $summary = $this->handleScanSubtitles($subtitleScanner, $manager);
+ $this->completeSubTask($taskService, $summary);
+ } catch (\Throwable $th) {
+ $this->failSubTask($taskService, $th);
+ throw $th;
+ }
+ }
+
+ /**
+ * This will handle clearing stale subtitles and indexing new or updated files
+ *
+ * Responsibilities
+ *
+ * Clear existing subtitle files (purge directory)
+ * Clear related subtitle rows in db
+ *
+ * Grab extended metadata
+ * Extract info for all embeded subtitle tracks and build transactions
+ * Check file directory for related subtitles
+ * build external subtitle transactions
+ * bulk upsert embedded and external subtitle transactions
+ *
+ * Optional/Future Responsibilities
+ *
+ * bulk extract subtitle files from media file (based on user config)
+ * bulk extract font files from media file (based on user config)
+ * generate font information (as a json on metadata?)
+ */
+ private function handleScanSubtitles(SubtitleScanner $subtitleScanner, SubtitleManager $manager): string {
+ $metadata = Metadata::with('video.folder')->where('uuid', $this->uuid)->firstOrFail();
+ $media = $metadata->video;
+
+ if (is_null($media)) {
+ throw new ModelNotFoundException("Media file missing for uuid {$this->uuid}");
+ }
+
+ $manager->purgeSubtitles($metadata, $this->externalOnly);
+
+ // absolute file path in storage
+ $filePath = VerifyFiles::getAbsoluteMediaPath($media);
+ $fileName = pathinfo($filePath, PATHINFO_FILENAME);
+ $folderPath = dirname($filePath);
+ $fileMetaData = VerifyFiles::getFileMetadata($filePath, 'Subtitle Scan');
+
+ // Embedded Subtitles
+
+ $subtitleTransactions = $this->externalOnly ? [] : $subtitleScanner->scanEmbeddedSubtitles($this->uuid, $fileMetaData);
+
+ // External Subtitles
+
+ // Previous Optimisation: if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0
+ // Now: every directory is scanned again per file. The scan is fast and should not have a high cost until you hit a large file count. Ideally this should be optimised again but without the batch structure it is difficult?
+ // If it runs with horizon, it should be fast. In a synchronous queue it will probably be slow
+ // Maybe I can batch this job by directory? or separate the external subtitle portion
+ $externalSubtitles = $subtitleScanner->findExternalSubtitlesInDirectory($folderPath);
+ $relevantExternal = array_filter(
+ $externalSubtitles,
+ fn($sub) => strtolower($sub['media_filename']) === strtolower($fileName)
+ );
+
+ $externalTransactions = $subtitleScanner->buildSubtitleTransactions($this->uuid, $relevantExternal);
+ $subtitleTransactions = array_merge($subtitleTransactions, $externalTransactions);
+
+ try {
+ if (!empty($subtitleTransactions)) {
+ Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'source_key'], ['language', 'title', 'codec', 'is_default', 'is_forced', 'external_path']);
+ }
+
+ $metadata->update(['subtitles_scanned_at' => now()]);
+
+ return 'Found ' . count($subtitleTransactions) . ' subtitle track(s) for uuid ' . $this->uuid;
+ } catch (\Throwable $th) {
+ $keys = array_column($subtitleTransactions, 'source_key');
+ $this->handleError('Error inserting subtitles for ' . $this->uuid, $th, $keys, count($subtitleTransactions));
+ }
+ }
+}
diff --git a/app/Jobs/VerifyFiles.php b/app/Jobs/VerifyFiles.php
index a11292d8..4007ff53 100755
--- a/app/Jobs/VerifyFiles.php
+++ b/app/Jobs/VerifyFiles.php
@@ -4,14 +4,12 @@ namespace App\Jobs;
use App\Enums\MediaType;
use App\Enums\TaskStatus;
+use App\Jobs\Utility\Subtitles\ScanSubtitles;
use App\Models\Metadata;
use App\Models\Record;
use App\Models\SubTask;
-use App\Models\Subtitle;
-use App\Services\Subtitles\SubtitleScanner;
use App\Services\TaskService;
use Carbon\Carbon;
-use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
@@ -22,6 +20,8 @@ use Symfony\Component\Process\Process;
class VerifyFiles extends ManagedSubTask {
protected $embedChain = [];
+ protected $subtitleScanChain = [];
+
protected $fileMetaData = [];
/**
@@ -34,21 +34,32 @@ class VerifyFiles extends ManagedSubTask {
$this->subTaskId = $subTask->id;
}
- public function handle(TaskService $taskService, SubtitleScanner $subtitleScanner): void {
+ public function handle(TaskService $taskService): void {
if (! $this->beginSubTask($taskService)) {
return;
}
try {
- $summary = $this->verifyFiles($taskService, $subtitleScanner);
- $taskCountUpdates = count($this->embedChain) ? ['sub_tasks_complete' => '++', 'sub_tasks_total' => count($this->embedChain), 'sub_tasks_pending' => count($this->embedChain)] : ['sub_tasks_complete' => '++'];
+ $summary = $this->verifyFiles($taskService);
+ $taskCountUpdates = [
+ 'sub_tasks_complete' => '++',
+ 'sub_tasks_total' => count($this->embedChain) + count($this->subtitleScanChain),
+ 'sub_tasks_pending' => count($this->embedChain) + count($this->subtitleScanChain),
+ ];
+
+ if (count($this->subtitleScanChain)) {
+ $summary .= ' and queued ' . count($this->subtitleScanChain) . ' subtitle scans';
+ }
$this->completeSubTask($taskService, $summary, $taskCountUpdates);
// Starts other subtasks after updating current subtask and parent subtask states
// The parent task "ends" after the batch empties so in theory, this should delay that anyways and does not need to run before the task update
foreach ($this->embedChain as $embedTask) {
- Bus::dispatch($embedTask);
+ $this->batch()->add($embedTask);
+ }
+ foreach ($this->subtitleScanChain as $scanTask) {
+ $this->batch()->add($scanTask);
}
} catch (\Throwable $th) {
$this->failSubTask($taskService, $th);
@@ -56,10 +67,8 @@ class VerifyFiles extends ManagedSubTask {
}
}
- private function verifyFiles(TaskService $taskService, SubtitleScanner $subtitleScanner) {
+ private function verifyFiles(TaskService $taskService) {
$metadataTransactions = [];
- $subtitleTransactions = [];
- $scannedSubtitleUuids = [];
$scannedDirectories = []; // TODO: this should really go in the indexer or a broken down part of the indexer
$error = false;
@@ -172,65 +181,32 @@ class VerifyFiles extends ManagedSubTask {
$changes['duration'] = is_numeric($duration) ? floor($duration) : null;
}
- // Embedded Subtitles
-
- $subtitleScanNeeded = ! $is_audio && is_null($metadata->subtitles_scanned_at) || $fileUpdated;
-
- if ($subtitleScanNeeded) {
- $scannedSubtitleUuids[] = $uuid;
- $this->confirmMetadata($filePath, 'Subtitle scan date is missing or file was updated');
- $embeddedSubtitleTransactions = $subtitleScanner->scanEmbeddedSubtitles($uuid, $this->fileMetaData);
-
- foreach ($embeddedSubtitleTransactions as $tx) {
- $subtitleTransactions[] = $tx;
- }
- }
-
- // #region External Subtitles
+ // #region Subtitles
// check if directory subtitle scan is needed
if (! isset($scannedDirectories[$folderPath])) {
$scannedDirectories[$folderPath] = [
'last_modified' => filemtime($folderPath),
- 'external_subtitles' => null,
];
}
- $dirUpdated = $metadata->file_scanned_at && $scannedDirectories[$folderPath]['last_modified'] > $metadata->file_scanned_at->timestamp;
-
// if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0
+ $dirUpdated = $metadata->file_scanned_at && $scannedDirectories[$folderPath]['last_modified'] > $metadata->file_scanned_at->timestamp;
- // Scan each directory once per batch
- if (! $is_audio && ($subtitleScanNeeded || $dirUpdated) && is_null($scannedDirectories[$folderPath]['external_subtitles'])) {
- $scannedDirectories[$folderPath]['external_subtitles'] = $subtitleScanner->findExternalSubtitlesInDirectory($folderPath);
- }
-
- // if subtitles in this directory have been scanned, check for matches
- if (! is_null($scannedDirectories[$folderPath]['external_subtitles'])) {
- $relevantSubtitles = array_filter(
- $scannedDirectories[$folderPath]['external_subtitles'],
- fn ($sub) => strtolower($sub['media_filename']) === strtolower($fileName)
- );
-
- $externalSubtitleTransactions = $subtitleScanner->buildSubtitleTransactions(
- $uuid,
- $relevantSubtitles
- );
-
- foreach ($externalSubtitleTransactions as $tx) {
- $subtitleTransactions[] = $tx;
- }
- }
+ $subtitleScanNeeded = ! $is_audio && (is_null($metadata->subtitles_scanned_at) || $fileUpdated || $dirUpdated);
- if ($subtitleScanNeeded || $dirUpdated) {
- $changes['subtitles_scanned_at'] = now();
+ if ($subtitleScanNeeded) {
+ // $changes['subtitles_scanned_at'] = null;
+ $purgeExternalOnly = $dirUpdated && ! $fileUpdated;
+ dump("Scan Subtitles Job", ["last scan date" => $metadata->subtitles_scanned_at, "fupdate" => $fileUpdated, "dupdate" => $dirUpdated, "name" => $fileName]);
+ $this->subtitleScanChain[] = new ScanSubtitles($this->taskId, $uuid, $purgeExternalOnly);
}
// #endregion
if (! $is_audio && (is_null($metadata->resolution_height) || is_null($metadata->codec) || $fileUpdated)) {
- $this->confirmMetadata($filePath, "Not audio and missing resolution or codec or because fileUpdated was {$fileUpdated}");
+ $this->confirmMetadata($filePath, "Not audio and missing resolution or codec or because fileUpdated was {" . ($fileUpdated ? 'true' : 'false') . "}");
foreach ($this->fileMetaData['streams'] as $stream) {
if (! isset($stream['codec_type']) || $stream['codec_type'] !== 'video' || ! isset($stream['width'])) {
continue;
@@ -373,11 +349,6 @@ class VerifyFiles extends ManagedSubTask {
$summary = 'Updated ' . count($metadataTransactions) . ' videos from id ' . ($metadataTransactions[0]['video_id']) . ' to ' . ($metadataTransactions[count($metadataTransactions) - 1]['video_id']);
- if (! empty($subtitleTransactions)) {
- Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'source_key'], ['language', 'title', 'codec', 'is_default', 'is_forced', 'external_path']);
- $summary .= ' and found ' . count($subtitleTransactions) . ' subtitle track(s)';
- }
-
return $summary;
} catch (\Throwable $th) {
$ids = array_column($metadataTransactions, 'id');
@@ -565,50 +536,6 @@ class VerifyFiles extends ManagedSubTask {
return $result;
}
- /**
- * Unimplemented
- *
- * In theory, it goes through every metadata by uuid which was updated, and checks to see which of its existing subtitle rows were seen in the scan
- * It deletes any rows and associated files that were not seen
- *
- * This would then clear old subtitle files if the media file was updated to something that did not include them
- *
- * I am not sure how or why of if this should even be part of the process, so it is unimplemented for now
- * Code left as reference
- */
- private function pruneSubtitles(array $subtitleTransactions, array $scannedSubtitleUuids): string {
- $subtitlesByUuid = collect($subtitleTransactions)->groupBy('metadata_uuid');
- $deletedSubtitles = 0;
-
- foreach ($scannedSubtitleUuids as $uuid) {
- $validKeys = $subtitlesByUuid[$uuid]?->pluck('source_key') ?? collect(); // Get seen source_keys (unique)
-
- $stale = $validKeys->isEmpty()
- ? Subtitle::where('metadata_uuid', $uuid)->get() // If none were seen, delete every associated subtitle row
- : Subtitle::where('metadata_uuid', $uuid)->whereNotIn('source_key', $validKeys)->get(); // Otherwise only unseen rows
-
- foreach ($stale as $subtitle) {
- if ($subtitle->path) {
- $dir = dirname($subtitle->path);
- $base = pathinfo($subtitle->path, PATHINFO_FILENAME); // "0.en" or "2"
-
- // should probably limit by top directory of private/metadata/media/... and have some handling for malicious paths
- // this would have been the first place in the entire app to delete files
- foreach (glob("{$dir}/{$base}.*") as $file) {
- dump('Would delete file', $file);
- // if (file_exists($file)) unlink($file);
- }
- }
- // also the file deletion is not atomic with the db deletion and can fail inbetween, but that should not truly matter because the file is regenerated if not found anyway
- Log::info('Deleted subtitle', ['path' => $subtitle->path, 'track' => $subtitle->track_id, 'title' => $subtitle->title]);
- // $subtitle->delete();
- }
- $deletedSubtitles += count($stale);
- }
-
- return $deletedSubtitles > 0 ? " and deleted $deletedSubtitles subtitle track(s)" : '';
- }
-
protected function extractMimeType($filePath) {
$finfo = new \finfo(FILEINFO_MIME_TYPE);
$mimeType = $finfo->file($filePath);
@@ -630,7 +557,7 @@ class VerifyFiles extends ManagedSubTask {
return $mimeType;
}
- protected function getAbsoluteMediaPath($media) {
+ public static function getAbsoluteMediaPath($media) {
return str_replace('\\', '/', Storage::disk('public')->path('')) . "media/{$media->folder->path}/" . basename($media->path);
}
diff --git a/app/Models/Metadata.php b/app/Models/Metadata.php
index 24f94d1f..3172d644 100644
--- a/app/Models/Metadata.php
+++ b/app/Models/Metadata.php
@@ -99,6 +99,7 @@ class Metadata extends Model {
// Date Id
'file_scanned_at',
'file_modified_at',
+ 'subtitles_scanned_at',
'first_file_modified_at',
'edited_at',
];
diff --git a/app/Services/Subtitles/SubtitleManager.php b/app/Services/Subtitles/SubtitleManager.php
index 2e0310f0..caf92481 100644
--- a/app/Services/Subtitles/SubtitleManager.php
+++ b/app/Services/Subtitles/SubtitleManager.php
@@ -3,11 +3,12 @@
namespace App\Services\Subtitles;
use App\Models\Metadata;
+use App\Models\Subtitle;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
class SubtitleManager {
- public function reset(Metadata $metadata): void {
+ public function resetSubtitleFiles(Metadata $metadata): void {
$this->deleteSubtitleFiles($metadata);
$metadata->subtitles()->update([
@@ -15,6 +16,28 @@ class SubtitleManager {
]);
}
+ public function purgeSubtitles(Metadata $metadata, bool $externalOnly = false): void {
+ if ($externalOnly) {
+ $this->deleteExternalSubtitleFiles($metadata);
+ } else {
+ // atomically delete all subtitle rows and files related to metadata
+ $this->deleteSubtitleFiles($metadata);
+ $metadata->subtitles()->delete();
+ }
+ }
+
+ private function deleteExternalSubtitleFiles(Metadata $metadata): void {
+ $metadata->subtitles()
+ ->whereNotNull('external_path')
+ ->each(function (Subtitle $subtitle) {
+ if ($subtitle->path) {
+ Storage::disk('local')->delete($subtitle->path);
+ }
+ });
+
+ $metadata->subtitles()->whereNotNull('external_path')->delete();
+ }
+
private function deleteSubtitleFiles(Metadata $metadata): void {
$directory = SubtitlePath::buildDirectory($metadata->uuid);
Log::info('Cleared Subtitle Directory', ['uuid' => $metadata->uuid, 'title' => $metadata->title, 'dir' => $directory]);