Skip to content

A self-hosted media server and file management system

fix: generate preview job fix and optimise websocket broadcast jobs

aminnausine9209e4c3a41
diff --git a/app/Events/LibraryUpdated.php b/app/Events/LibraryUpdated.php
index 71b9cf8d..095a1d3f 100644
--- a/app/Events/LibraryUpdated.php
+++ b/app/Events/LibraryUpdated.php
@@ -23,7 +23,7 @@ class LibraryUpdated implements ShouldBroadcast {
/**
* Create a new event instance.
*/
- public function __construct(public Category $library) {
+ public function __construct(public int $libraryId) {
//
}
@@ -42,6 +42,8 @@ class LibraryUpdated implements ShouldBroadcast {
* Get the data to broadcast.
*/
public function broadcastWith(): array {
- return ['library' => new CategoryResource($this->library->withCount('folders'))];
+ $library = Category::findOrFail($this->libraryId);
+
+ return ['library' => new CategoryResource($library->withCount('folders'))];
}
}
diff --git a/app/Events/SubTaskUpdated.php b/app/Events/SubTaskUpdated.php
index d8b30408..8109f271 100644
--- a/app/Events/SubTaskUpdated.php
+++ b/app/Events/SubTaskUpdated.php
@@ -9,6 +9,7 @@ use Illuminate\Broadcasting\PrivateChannel;
use Illuminate\Contracts\Broadcasting\ShouldBroadcast;
use Illuminate\Foundation\Events\Dispatchable;
use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
class SubTaskUpdated implements ShouldBroadcast {
use Dispatchable, InteractsWithSockets, SerializesModels;
@@ -23,7 +24,7 @@ class SubTaskUpdated implements ShouldBroadcast {
/**
* Create a new event instance.
*/
- public function __construct(public SubTask $subTask) {
+ public function __construct(public int $subTaskId, public int $taskId) {
//
}
@@ -35,10 +36,11 @@ class SubTaskUpdated implements ShouldBroadcast {
public function broadcastOn(): array {
try {
return [
- new PrivateChannel("tasks.{$this->subTask->task_id}.subtasks"),
+ new PrivateChannel("tasks.{$this->taskId}.subtasks"),
];
} catch (\Throwable $th) {
- dump($th->getMessage());
+ Log::error('Unable to broadcast subtask update', ['error' => $th->getMessage()]);
+ throw $th;
}
}
@@ -46,6 +48,13 @@ class SubTaskUpdated implements ShouldBroadcast {
* Get the data to broadcast.
*/
public function broadcastWith(): array {
- return ['subTask' => new SubTasksResource($this->subTask)];
+ try {
+ $subTask = SubTask::findOrFail($this->subTaskId);
+
+ return ['subTask' => new SubTasksResource($subTask)];
+ } catch (\Throwable $th) {
+ Log::error('Unable to broadcast subtask update', ['error' => $th->getMessage()]);
+ throw $th;
+ }
}
}
diff --git a/app/Events/TaskEnded.php b/app/Events/TaskEnded.php
index b6dd2718..f5b82dfc 100644
--- a/app/Events/TaskEnded.php
+++ b/app/Events/TaskEnded.php
@@ -25,7 +25,7 @@ class TaskEnded implements ShouldBroadcast {
/**
* Create a new event instance.
*/
- public function __construct(public Task $task) {
+ public function __construct(public int $taskId) {
//
}
@@ -37,7 +37,7 @@ class TaskEnded implements ShouldBroadcast {
public function broadcastOn(): array {
try {
return [
- new PrivateChannel("tasks.{$this->task->id}"),
+ new PrivateChannel("tasks.{$this->taskId}"),
];
} catch (\Throwable $th) {
dump($th->getMessage());
@@ -51,7 +51,9 @@ class TaskEnded implements ShouldBroadcast {
*/
public function broadcastWith(): array {
try {
- return ['task' => new TasksResource($this->task)];
+ $task = Task::findOrFail($this->taskId);
+
+ return ['task' => new TasksResource($task)];
} catch (\Throwable $th) {
dump($th->getMessage());
Log::error('Unable to broadcast task ended update', ['error' => $th->getMessage()]);
diff --git a/app/Events/TaskUpdated.php b/app/Events/TaskUpdated.php
index b7a8ee47..4387bf37 100644
--- a/app/Events/TaskUpdated.php
+++ b/app/Events/TaskUpdated.php
@@ -24,7 +24,7 @@ class TaskUpdated implements ShouldBroadcast {
/**
* Create a new event instance.
*/
- public function __construct(public Task $task) {
+ public function __construct(public int $taskId) {
//
}
@@ -50,7 +50,9 @@ class TaskUpdated implements ShouldBroadcast {
*/
public function broadcastWith(): array {
try {
- return ['task' => new TasksResource($this->task)];
+ $task = Task::findOrFail($this->taskId);
+
+ return ['task' => new TasksResource($task)];
} catch (\Throwable $th) {
dump($th->getMessage());
Log::error('Unable to broadcast task update', ['error' => $th->getMessage()]);
diff --git a/app/Jobs/GeneratePreviewImage.php b/app/Jobs/GeneratePreviewImage.php
index dccec19f..8aea1379 100644
--- a/app/Jobs/GeneratePreviewImage.php
+++ b/app/Jobs/GeneratePreviewImage.php
@@ -2,23 +2,82 @@
namespace App\Jobs;
+use App\Enums\TaskStatus;
use App\Services\PreviewGeneratorService;
+use App\Services\TaskService;
+use Exception;
+use Illuminate\Bus\Batchable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Foundation\Queue\Queueable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\App;
class GeneratePreviewImage implements ShouldQueue {
- use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+ use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+ protected $subTaskId;
+
+ protected $startedAt;
+
+ protected $itemTitle;
+
+ public $timeout = 3600;
+
+ /**
+ * Create a new job instance.
+ *
+ * @return void
+ */
public function __construct(
public array $data,
public string $path,
- public ?string $updatedAt = null
- ) {}
+ public int $taskId,
+ ) {
+ $taskService = App::make(TaskService::class);
+ $this->itemTitle = $data['title'] ?? $path;
+
+ $subTask = $taskService->createSubTask(['task_id' => $taskId, 'status' => TaskStatus::PENDING, 'name' => "Regenerate Preview Image for '{$this->itemTitle}'"]);
+
+ $this->taskId = $taskId;
+ $this->subTaskId = $subTask->id;
+ }
+
+ public function handle(PreviewGeneratorService $previewGenerator, TaskService $taskService) {
+ if ($this->batch()->cancelled()) {
+ // Determine if the batch has been cancelled...
+ $taskService->updateSubTask($this->subTaskId, ['status' => TaskStatus::CANCELLED, 'summary' => 'Parent Task was Cancelled']);
+
+ return;
+ }
+
+ $this->startedAt = now();
+ $taskService->updateTaskCounts($this->taskId, ['sub_tasks_pending' => '--']);
+ $taskService->updateSubTask($this->subTaskId, ['status' => TaskStatus::PROCESSING, 'started_at' => $this->startedAt]);
+
+ try {
+ $result = $previewGenerator->generateImage($this->data, $this->path, true);
+ if (! $result) {
+ throw new Exception('Preview image generation failed. View logs for error.');
+ }
+ $endedAt = now();
+ $duration = (int) $this->startedAt->diffInSeconds($endedAt);
- public function handle(PreviewGeneratorService $previewGenerator) {
- $previewGenerator->generateImage($this->data, $this->path);
+ $taskService->updateTaskCounts($this->taskId, ['sub_tasks_complete' => '++'], false);
+ $taskService->updateSubTask($this->subTaskId, [
+ 'status' => TaskStatus::COMPLETED,
+ 'summary' => 'Generated preview image.',
+ 'progress' => 100,
+ 'ended_at' => $endedAt,
+ 'duration' => $duration,
+ ]);
+ } catch (\Throwable $th) {
+ $endedAt = now();
+ $duration = (int) $this->startedAt->diffInSeconds($endedAt);
+ $taskService->updateTaskCounts($this->taskId, ['sub_tasks_failed' => '++']);
+ $taskService->updateSubTask($this->subTaskId, ['status' => TaskStatus::FAILED, 'summary' => 'Error: ' . $th->getMessage(), 'ended_at' => $endedAt, 'duration' => $duration]);
+ throw $th;
+ }
}
}
diff --git a/app/Services/FileJobService.php b/app/Services/FileJobService.php
index 8d08a9b7..2eeb0ca6 100644
--- a/app/Services/FileJobService.php
+++ b/app/Services/FileJobService.php
@@ -6,6 +6,7 @@ use App\Enums\TaskStatus;
use App\Jobs\CleanFolderPaths;
use App\Jobs\CleanVideoPaths;
use App\Jobs\EmbedUidInMetadata;
+use App\Jobs\GeneratePreviewImage;
use App\Jobs\IndexFiles;
use App\Jobs\SyncFiles;
use App\Jobs\VerifyFiles;
@@ -209,6 +210,35 @@ class FileJobService {
);
}
+ /**
+ * Regenerate open graph preview images
+ *
+ * @param array<array{array: data, path: string}> $rows
+ * @return \Illuminate\Http\JsonResponse
+ */
+ public function regeneratePreviewImages(array $rows, ?int $userId = null) {
+ $name = 'Regenerate Preview Images';
+ $description = 'Regenerates open graph preview images.';
+
+ return $this->executeBatchOperation(
+ userId: $userId ?? null,
+ name: $name,
+ description: $description,
+ chain: function ($task) use ($rows) {
+ $chain = [];
+
+ foreach ($rows as $row) {
+ if (! isset($row['data']) || ! isset($row['path'])) {
+ continue;
+ }
+ $chain[] = new GeneratePreviewImage($row['data'], $row['path'], $task->id);
+ }
+
+ return $chain;
+ },
+ );
+ }
+
public function executeBatchOperation(
?int $userId,
string $name,
diff --git a/app/Services/PreviewGeneratorService.php b/app/Services/PreviewGeneratorService.php
index 2316c5ab..2d5aea2d 100644
--- a/app/Services/PreviewGeneratorService.php
+++ b/app/Services/PreviewGeneratorService.php
@@ -4,7 +4,6 @@ namespace App\Services;
use App\Http\Resources\FolderResource;
use App\Http\Resources\VideoResource;
-use App\Jobs\GeneratePreviewImage;
use App\Jobs\VerifyFiles;
use App\Models\Category;
use App\Models\Folder;
@@ -20,6 +19,7 @@ use Symfony\Component\HttpFoundation\Response;
class PreviewGeneratorService {
public function __construct(
protected PathResolverService $pathResolver,
+ protected FileJobService $fileJobService,
) {}
public function handle(Request $request): Response {
@@ -50,9 +50,9 @@ class PreviewGeneratorService {
}
}
- public function handleGenerateImage(array $data, string $relativePath, ?int $dataLastUpdated = 0, bool $queued = false): ?string {
+ public function handleGenerateImage(array $data, string $relativePath, ?int $dataLastUpdated = 0, bool $queued = true): ?string {
try {
- $override = Config('services.preview_generator.override');
+ $override = false; // config('services.preview_generator.override');
$relativePath = trim('previews/' . $relativePath, '/') . '.png';
$fullPath = Storage::disk('public')->path($relativePath);
@@ -61,9 +61,8 @@ class PreviewGeneratorService {
} elseif (! $override && Storage::disk('public')->exists($relativePath) && ! is_null($dataLastUpdated) && filemtime($fullPath) > $dataLastUpdated) {
return VerifyFiles::getPathUrl($relativePath);
}
-
if (! $override && $queued) {
- GeneratePreviewImage::dispatch($data, $relativePath);
+ $this->fileJobService->regeneratePreviewImages([['data' => $data, 'path' => $relativePath]]);
return VerifyFiles::getPathUrl($relativePath);
}
@@ -153,7 +152,7 @@ class PreviewGeneratorService {
return $baseData;
}
- public function generateImage(array $data, string $relativePath) {
+ public function generateImage(array $data, string $relativePath, bool $selfStore = false) {
try {
$tempRelativePath = 'previews/' . uniqid('og-', true) . '.png';
$tempPath = Storage::disk('public')->path($tempRelativePath);
@@ -185,10 +184,16 @@ class PreviewGeneratorService {
$imageContents = file_get_contents($tempPath);
Storage::disk('public')->delete($tempRelativePath);
- return $imageContents;
+ if (! $selfStore) {
+ return $imageContents;
+ }
+
+ Storage::disk('public')->put($relativePath, $imageContents);
+
+ return VerifyFiles::getPathUrl($relativePath);
} catch (\Throwable $th) { // Cannot catch the specific spatie/image exception since it throws a generic one
$message = $th->getMessage();
- if ($message !== 'The spatie/image package is required to perform image manipulations. Please install it by running `composer require spatie/image') {
+ if ($message !== 'The spatie/image package is required to perform image manipulations. Please install it by running `composer require spatie/image`') {
Log::warning('Error during OG image generation', ['error' => $th->getMessage()]);
}
if (file_exists($tempPath)) {
@@ -196,7 +201,13 @@ class PreviewGeneratorService {
Storage::disk('public')->delete($tempRelativePath);
}
- return $imageContents ?? false;
+ if (! $selfStore || ! $imageContents) {
+ return $imageContents ?? false;
+ }
+
+ Storage::disk('public')->put($relativePath, $imageContents);
+
+ return VerifyFiles::getPathUrl($relativePath);
}
}
diff --git a/app/Services/TaskService.php b/app/Services/TaskService.php
index 1ca63913..c87427dc 100644
--- a/app/Services/TaskService.php
+++ b/app/Services/TaskService.php
@@ -14,7 +14,7 @@ class TaskService {
$task = Task::create($attrs);
try {
- broadcast(new TaskUpdated($task));
+ broadcast(new TaskUpdated($task->id));
} catch (\Throwable $th) {
dump($th->getMessage());
Log::error('Unable to broadcast task creation', ['error' => $th->getMessage()]);
@@ -44,9 +44,9 @@ class TaskService {
try {
if ($taskEnded) {
- broadcast(new TaskEnded($task));
+ broadcast(new TaskEnded($task->id));
} else {
- broadcast(new TaskUpdated($task));
+ broadcast(new TaskUpdated($task->id));
}
} catch (\Throwable $th) {
dump($th->getMessage());
@@ -72,7 +72,7 @@ class TaskService {
$subTask->save();
if ($broadcast) {
try {
- broadcast(new SubTaskUpdated($subTask));
+ broadcast(new SubTaskUpdated($subTask->id, $subTask->task->id));
} catch (\Throwable $th) {
dump($th->getMessage());
Log::error('Unable to broadcast subTask update', ['error' => $th->getMessage()]);
@@ -101,7 +101,7 @@ class TaskService {
$task->save();
if ($broadcast) {
try {
- broadcast(new TaskUpdated($task));
+ broadcast(new TaskUpdated($task->id));
} catch (\Throwable $th) {
dump($th->getMessage());
Log::error('Unable to broadcast task count update', ['error' => $th->getMessage()]);
diff --git a/config/broadcasting.php b/config/broadcasting.php
index b748872b..3a6b5019 100644
--- a/config/broadcasting.php
+++ b/config/broadcasting.php
@@ -40,6 +40,7 @@ return [
'port' => env('REVERB_PORT', 443),
'scheme' => env('REVERB_SCHEME', 'https'),
'useTLS' => env('REVERB_SCHEME', 'https') === 'https',
+ 'verify' => env('APP_ENV') !== 'local',
],
'client_options' => [
// Guzzle client options: https://docs.guzzlephp.org/en/stable/request-options.html
diff --git a/resources/views/og-preview.blade.php b/resources/views/og-preview.blade.php
index e178673a..29149e95 100644
--- a/resources/views/og-preview.blade.php
+++ b/resources/views/og-preview.blade.php
@@ -4,6 +4,25 @@
@php
$thumbnail_url = str_replace('http://', 'https://', $thumbnail_url ?? '');
$raw = str_replace('http://', 'https://', $raw ?? '');
+
+function highlight_json($json) {
+$json = e($json); // escape for HTML safety
+
+// Highlight keys
+$json = preg_replace('/(&quot;[^&]+&quot;)(\s*:\s*)/', '<span class="text-blue-400">$1</span>$2', $json);
+// Highlight string values
+$json = preg_replace('/:\s*(&quot;[^&]+&quot;)/', ': <span class="text-orange-400">$1</span>', $json);
+// Highlight numbers
+$json = preg_replace('/:\s*(\d+)/', ': <span class="text-green-400">$1</span>', $json);
+// Highlight booleans
+$json = preg_replace('/\b(true|false)\b/', '<span class="text-pink-400">$1</span>', $json);
+// Highlight null
+$json = preg_replace('/\bnull\b/', '<span class="text-gray-400">null</span>', $json);
+// Highlight brackets
+$json = preg_replace('/([{}\[\]\(\)])/','<span class="text-yellow-400">$1</span>', $json);
+
+return $json;
+}
@endphp
<head>
@@ -35,6 +54,15 @@ $raw = str_replace('http://', 'https://', $raw ?? '');
<!-- <meta property="og:image" content="https://img.anili.st/media/176301" data-vue-meta="true"> -->
@vite('resources/css/app.css')
+ <style>
+ html {
+ scrollbar-gutter: stable;
+ }
+
+ html.fullscreen {
+ scrollbar-gutter: auto;
+ }
+ </style>
</head>
<body class="flex flex-col gap-2 text-sm p-4">
@@ -47,9 +75,14 @@ $raw = str_replace('http://', 'https://', $raw ?? '');
@else
<p>🎥 Video Content</p>
@endif
+ @if($updated_at ?? false)
+ <p>{{$updated_at}}</p>
+ @endif
<img src="{{ $raw ?: $thumbnail_url }}" width="1200" class="rounded-xl overflow-clip" />
@if (app()->environment('local'))
- <pre>{{ json_encode(get_defined_vars(), JSON_PRETTY_PRINT) }}</pre>
+ <pre class="max-w-full whitespace-pre-wrap bg-neutral-900 text-white p-2 overflow-x-auto text-sm leading-tight">
+{!! highlight_json(json_encode(get_defined_vars(), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)) !!}
+ </pre>
@endif
</body>