Skip to content

A self-hosted media server and file management system

feat: optimise subtitle scanning process

aminnausin9336465652b0
diff --git a/app/Data/Subtitles/SubtitleScanTarget.php b/app/Data/Subtitles/SubtitleScanTarget.php
new file mode 100644
index 00000000..41d8e3a1
--- /dev/null
+++ b/app/Data/Subtitles/SubtitleScanTarget.php
@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Data\Subtitles;
+
+class SubtitleScanTarget {
+ public function __construct(
+ public readonly string $uuid,
+ public readonly bool $fileUpdated,
+ ) {
+ }
+}
diff --git a/app/Jobs/Utility/Subtitles/ScanSubtitles.php b/app/Jobs/Utility/Subtitles/ScanSubtitles.php
index 74b57d4e..1df2362d 100644
--- a/app/Jobs/Utility/Subtitles/ScanSubtitles.php
+++ b/app/Jobs/Utility/Subtitles/ScanSubtitles.php
@@ -2,6 +2,7 @@
namespace App\Jobs\Utility\Subtitles;
+use App\Data\Subtitles\SubtitleScanTarget;
use App\Enums\TaskStatus;
use App\Jobs\ManagedSubTask;
use App\Jobs\VerifyFiles;
@@ -11,19 +12,25 @@ use App\Models\Subtitle;
use App\Services\Subtitles\SubtitleManager;
use App\Services\Subtitles\SubtitleScanner;
use App\Services\TaskService;
-use Illuminate\Database\Eloquent\ModelNotFoundException;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
class ScanSubtitles extends ManagedSubTask {
- private string $uuid;
+ private array $targets;
+ private string $folderPath;
- private bool $externalOnly;
+ private bool $scanExternal;
+
+ /** @param SubtitleScanTarget[] $targets */
+ public function __construct(int $taskId, array $targets, string $folderPath, bool $scanExternal) {
+ $subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Scan Subtitles for folder ' . $folderPath]);
- public function __construct(int $taskId, string $uuid, bool $externalOnly = false) {
- $subTask = SubTask::create(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => 'Scan Subtitles for uuid ' . $uuid]);
$this->taskId = $taskId;
$this->subTaskId = $subTask->id;
- $this->uuid = $uuid;
- $this->externalOnly = $externalOnly;
+
+ $this->targets = $targets;
+ $this->folderPath = $folderPath;
+ $this->scanExternal = $scanExternal;
}
public function handle(TaskService $taskService, SubtitleScanner $subtitleScanner, SubtitleManager $manager): void {
@@ -61,51 +68,84 @@ class ScanSubtitles extends ManagedSubTask {
* generate font information (as a json on metadata?)
*/
private function handleScanSubtitles(SubtitleScanner $subtitleScanner, SubtitleManager $manager): string {
- $metadata = Metadata::with('video.folder')->where('uuid', $this->uuid)->firstOrFail();
- $media = $metadata->video;
+ $summary = "Scanning for subtitle tracks on " . count($this->targets) . " targets";
+ $batchTransactions = [];
+ $scannedUuids = [];
- if (is_null($media)) {
- throw new ModelNotFoundException("Media file missing for uuid {$this->uuid}");
- }
+ $metadataMap = Metadata::with('video.folder')
+ ->whereIn('uuid', array_column($this->targets, 'uuid'))
+ ->get()
+ ->keyBy('uuid');
+
+ $externalSubtitles = $this->scanExternal
+ ? $subtitleScanner->findExternalSubtitlesInDirectory($this->folderPath)
+ : [];
- $manager->purgeSubtitles($metadata, $this->externalOnly);
- // absolute file path in storage
- $filePath = VerifyFiles::getAbsoluteMediaPath($media);
- $fileName = pathinfo($filePath, PATHINFO_FILENAME);
- $folderPath = dirname($filePath);
- $fileMetaData = VerifyFiles::getFileMetadata($filePath, 'Subtitle Scan');
+ foreach ($this->targets as $target) {
+ $subtitleTransactions = [];
- // Embedded Subtitles
+ $uuid = $target->uuid;
+ $fileUpdated = $target->fileUpdated;
- $subtitleTransactions = $this->externalOnly ? [] : $subtitleScanner->scanEmbeddedSubtitles($this->uuid, $fileMetaData);
+ $metadata = $metadataMap[$uuid] ?? null;
+ $media = $metadata?->video;
- // External Subtitles
+ if (is_null($metadata)) {
+ Log::error("Metadata missing when scanning for subtitles", ["uuid" => $uuid]);
+ continue;
+ }
+
+ if (is_null($media)) {
+ Log::error("Media file missing on metadata instance when scanning for subtitles", ["uuid" => $metadata->uuid, "title" => $metadata->title]);
+ $summary .= "\n\nMedia file missing for " . $metadata->title . " with uuid " . $metadata->uuid;
+ continue;
+ }
- // Previous Optimisation: if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0
- // Now: every directory is scanned again per file. The scan is fast and should not have a high cost until you hit a large file count. Ideally this should be optimised again but without the batch structure it is difficult?
- // If it runs with horizon, it should be fast. In a synchronous queue it will probably be slow
- // Maybe I can batch this job by directory? or separate the external subtitle portion
- $externalSubtitles = $subtitleScanner->findExternalSubtitlesInDirectory($folderPath);
- $relevantExternal = array_filter(
- $externalSubtitles,
- fn($sub) => strtolower($sub['media_filename']) === strtolower($fileName)
- );
+ $manager->purgeSubtitles($metadata, externalOnly: !$fileUpdated); // only purge external subtitle files if the file was not updated. Getting to this point
- $externalTransactions = $subtitleScanner->buildSubtitleTransactions($this->uuid, $relevantExternal);
- $subtitleTransactions = array_merge($subtitleTransactions, $externalTransactions);
+ // absolute file path in storage
+ $filePath = VerifyFiles::getAbsoluteMediaPath($media);
+ $fileName = pathinfo($filePath, PATHINFO_FILENAME);
+ // $folderPath = dirname($filePath);
+
+ // Embedded Subtitles - scanned only if file was updated
+ if ($fileUpdated) {
+ // TODO: Cache this output from index or subsequent verify jobs as json in the model and never call from here
+ $fileMetaData = VerifyFiles::getFileMetadata($filePath, 'Subtitle Scan');
+ $subtitleTransactions = $subtitleScanner->scanEmbeddedSubtitles($uuid, $fileMetaData);
+ }
+
+ // External Subtitles - directory scanned once per job and filtered per media file
+ if (!empty($externalSubtitles)) {
+ $relevantExternal = array_filter(
+ $externalSubtitles,
+ fn($sub) => strtolower($sub['media_filename']) === strtolower($fileName)
+ );
+
+ $subtitleTransactions = array_merge($subtitleTransactions, $subtitleScanner->buildSubtitleTransactions($uuid, $relevantExternal));
+ }
- try {
if (!empty($subtitleTransactions)) {
- Subtitle::upsert($subtitleTransactions, ['metadata_uuid', 'source_key'], ['language', 'title', 'codec', 'is_default', 'is_forced', 'external_path']);
+ $summary .= "\n\nFound " . count($subtitleTransactions) . ' subtitle track(s) for uuid ' . $uuid;
+ $batchTransactions = array_merge($batchTransactions, $subtitleTransactions);
}
+ $scannedUuids[] = $uuid;
+ }
+
+ try {
+ DB::transaction(function () use ($batchTransactions, $scannedUuids) {
+ if (!empty($batchTransactions)) {
+ Subtitle::upsert($batchTransactions, ['metadata_uuid', 'source_key'], ['language', 'title', 'codec', 'is_default', 'is_forced', 'external_path']);
+ }
- $metadata->update(['subtitles_scanned_at' => now()]);
+ Metadata::whereIn('uuid', $scannedUuids)
+ ->update(['subtitles_scanned_at' => now()]);
+ });
- return 'Found ' . count($subtitleTransactions) . ' subtitle track(s) for uuid ' . $this->uuid;
+ return $summary . "\n\n\nTotal subtitle tracks upserted: " . count($batchTransactions);
} catch (\Throwable $th) {
- $keys = array_column($subtitleTransactions, 'source_key');
- $this->handleError('Error inserting subtitles for ' . $this->uuid, $th, $keys, count($subtitleTransactions));
+ $this->handleError('Error inserting subtitle tracks', $th, $scannedUuids, count($batchTransactions));
}
}
}
diff --git a/app/Jobs/VerifyFiles.php b/app/Jobs/VerifyFiles.php
index 4007ff53..8b0aef49 100755
--- a/app/Jobs/VerifyFiles.php
+++ b/app/Jobs/VerifyFiles.php
@@ -2,6 +2,7 @@
namespace App\Jobs;
+use App\Data\Subtitles\SubtitleScanTarget;
use App\Enums\MediaType;
use App\Enums\TaskStatus;
use App\Jobs\Utility\Subtitles\ScanSubtitles;
@@ -18,9 +19,10 @@ use Symfony\Component\Process\Exception\ProcessFailedException;
use Symfony\Component\Process\Process;
class VerifyFiles extends ManagedSubTask {
+ protected $subtitleScanChain = [];
protected $embedChain = [];
- protected $subtitleScanChain = [];
+ protected $scannedDirectories = []; // TODO: this should really go in the indexer or a broken down part of the indexer
protected $fileMetaData = [];
@@ -41,6 +43,18 @@ class VerifyFiles extends ManagedSubTask {
try {
$summary = $this->verifyFiles($taskService);
+
+ foreach ($this->scannedDirectories as $folderPath => $dirData) {
+ if (empty($dirData['targets'])) continue;
+
+ $this->subtitleScanChain[] = new ScanSubtitles(
+ taskId: $this->taskId,
+ targets: $dirData['targets'],
+ folderPath: $folderPath,
+ scanExternal: $dirData['dir_updated'],
+ );
+ }
+
$taskCountUpdates = [
'sub_tasks_complete' => '++',
'sub_tasks_total' => count($this->embedChain) + count($this->subtitleScanChain),
@@ -69,7 +83,6 @@ class VerifyFiles extends ManagedSubTask {
private function verifyFiles(TaskService $taskService) {
$metadataTransactions = [];
- $scannedDirectories = []; // TODO: this should really go in the indexer or a broken down part of the indexer
$error = false;
$index = 0;
@@ -185,22 +198,29 @@ class VerifyFiles extends ManagedSubTask {
// check if directory subtitle scan is needed
- if (! isset($scannedDirectories[$folderPath])) {
- $scannedDirectories[$folderPath] = [
+ if (! isset($this->scannedDirectories[$folderPath])) {
+ $this->scannedDirectories[$folderPath] = [
'last_modified' => filemtime($folderPath),
+ 'targets' => [],
+ 'dir_updated' => false,
];
}
// if directory updated, check directory for related subtitle files and make subtitle transactions for them with stream 0
- $dirUpdated = $metadata->file_scanned_at && $scannedDirectories[$folderPath]['last_modified'] > $metadata->file_scanned_at->timestamp;
+ $dirUpdated = $metadata->file_scanned_at && $this->scannedDirectories[$folderPath]['last_modified'] > $metadata->file_scanned_at->timestamp;
$subtitleScanNeeded = ! $is_audio && (is_null($metadata->subtitles_scanned_at) || $fileUpdated || $dirUpdated);
if ($subtitleScanNeeded) {
- // $changes['subtitles_scanned_at'] = null;
- $purgeExternalOnly = $dirUpdated && ! $fileUpdated;
- dump("Scan Subtitles Job", ["last scan date" => $metadata->subtitles_scanned_at, "fupdate" => $fileUpdated, "dupdate" => $dirUpdated, "name" => $fileName]);
- $this->subtitleScanChain[] = new ScanSubtitles($this->taskId, $uuid, $purgeExternalOnly);
+ $changes['subtitles_scanned_at'] = null;
+ $this->scannedDirectories[$folderPath]['targets'][] = new SubtitleScanTarget(
+ uuid: $uuid,
+ fileUpdated: $fileUpdated,
+ );
+
+ if ($dirUpdated) {
+ $this->scannedDirectories[$folderPath]['dir_updated'] = true;
+ }
}
// #endregion