Skip to content

A self-hosted media server and file management system

18.0 KB
Raw Blame
<?php

namespace App\Services;

use App\Enums\MediaType;
use App\Enums\TaskStatus;
use App\Jobs\EmbedUidInMetadata;
use App\Jobs\GeneratePreviewImage;
use App\Jobs\IndexFiles;
use App\Jobs\Maintenance\PurgeStaleGuestData;
use App\Jobs\Metadata\ExtractFonts;
use App\Jobs\Metadata\GenerateStoryboard;
use App\Jobs\SyncFiles;
use App\Jobs\Utility\Paths\CleanFolderPaths;
use App\Jobs\Utility\Paths\CleanVideoPaths;
use App\Jobs\VerifyFiles;
use App\Jobs\VerifyFolders;
use App\Models\Category;
use App\Models\Folder;
use App\Models\Metadata;
use App\Models\Task;
use App\Models\Video;
use Illuminate\Bus\Batch;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Log;

class FileJobService {
    public function __construct(protected TaskService $taskService) {}

    public function scanFiles(array $data, ?Category $category = null) {
        $name = 'Scan Files';
        $description = 'Scans for file changes and loads metadata from all libraries.';
        $categoryId = null;

        if (isset($category)) {
            $name .= " from the library \"$category->name\"";
            $description = "Scans for file changes and loads metadata from the specified library \"$category->name\"";
            $categoryId = $category->id;
        }

        return $this->executeBatchOperation(
            userId: $data['userId'] ?? null,
            name: ($data['namePrefix'] ?? '') . $name,
            description: $description,
            chain: function ($task) {
                return [
                    new SyncFiles($task->id),
                    new IndexFiles($task->id),
                ];
            },
            callback: function ($task) use ($data, $categoryId) {
                $category = $categoryId ? Category::findOrFail($categoryId) : null;
                $this->verifyFiles($data, $category, $task->id);
            }
        );
    }

    public function indexFiles(array $data, ?Category $category = null) {
        $name = 'Index Files';
        $description = 'Looks for folder and video changes in in all libraries.';

        if (isset($category)) {
            $name .= " for library $category->name";
            $description = "Looks for folder and video changes in the specified library $category->name";
        }

        return $this->executeBatchOperation(
            userId: $data['userId'] ?? null,
            name: ($data['namePrefix'] ?? '') . $name,
            description: $description,
            chain: function ($task) {
                return [
                    new SyncFiles($task->id),
                    new IndexFiles($task->id),
                ];
            },
        );
    }

    public function syncFiles(array $data) {
        return $this->executeBatchOperation(
            userId: $data['userId'] ?? null,
            name: 'Sync Files',
            description: 'Syncs local file structure with database.',
            chain: function ($task) {
                return [
                    new SyncFiles($task->id),
                ];
            },
        );
    }

    public function verifyFiles(array $data, ?Category $category = null, ?int $taskId = null) {
        $name = 'Verify Files';
        $description = 'Verifies folder and video metadata for all libraries.';
        if (isset($category)) {
            $name .= " for library \"$category->name\"";
            $description = "Verifies folder and video metadata for the specified library \"$category->name\"";
        }

        return $this->executeBatchOperation(
            userId: $data['userId'] ?? null,
            name: $name,
            description: $description,
            chain: function ($task) use ($category) {
                $chain = [];

                $videoQuery = Video::orderBy('id')->with(['metadata.storyboard', 'folder.category']);
                $folderQuery = Folder::orderBy('id');

                if ($category) {
                    $videoQuery = $videoQuery->whereHas('folder.category', function ($query) use ($category) {
                        $query->where('id', $category->id);
                    })->with('folder.category');

                    $folderQuery = $folderQuery->whereHas('category', function ($query) use ($category) {
                        $query->where('id', $category->id);
                    })->with('category');
                }

                $videoQuery->chunk(100, function ($chunk) use (&$chain, $task, $category) {
                    $chain[] = new VerifyFiles($chunk, $task->id, generateImageTasks: $category !== null);
                });

                $folderQuery->chunk(100, function ($chunk) use (&$chain, $task) {
                    $chain[] = new VerifyFolders($chunk, $task->id);
                });

                return $chain;
            },
            initialTaskId: $taskId,
        );
    }

    public function verifyFolders(array $data, ?Category $category = null) {
        $name = 'Verify Folders';
        $description = 'Verifies folder metadata for all libraries.';

        if (isset($category)) {
            $name .= " for library \"$category->name\"";
            $description = "Verifies folder metadata for the specified library \"$category->name\"";
        }

        return $this->executeBatchOperation(
            userId: $data['userId'] ?? null,
            name: $name,
            description: $description,
            chain: function ($task) use ($category) {
                $chain = [];
                $query = Folder::orderBy('id');

                if ($category) {
                    $query->whereHas('category', fn ($q) => $q->where('id', $category->id))
                        ->with('category');
                }

                $query->chunk(100, function ($chunk) use (&$chain, $task) {
                    $chain[] = new VerifyFolders($chunk, $task->id);
                });

                return $chain;
            },
        );
    }

    public function cleanPaths(array $data) {
        $name = 'Clean Paths';
        $description = 'Cleans file and folder paths for all libraries.';

        return $this->executeBatchOperation(
            userId: $data['userId'] ?? null,
            name: $name,
            description: $description,
            chain: function ($task) {
                $chain = [];

                Video::orderBy('id')->chunk(20)->each(function ($chunk) use (&$chain, $task) {
                    $chain[] = new CleanVideoPaths($chunk, $task->id);
                });

                Folder::orderBy('id')->chunk(20)->each(function ($chunk) use (&$chain, $task) {
                    $chain[] = new CleanFolderPaths($chunk, $task->id);
                });

                return $chain;
            },
        );
    }

    /**
     * Embed UIDs in file metadata
     * Unused?
     *
     * @param  array<array{path: string, uuid: string}>  $files
     * @return \Illuminate\Http\JsonResponse
     */
    public function embedUIDs(int $taskId, string $name, array $files) {
        $description = 'Embeds UIDs in file metadata.';
        $lastTask = Task::find($taskId);

        return $this->executeBatchOperation(
            userId: $lastTask ? $lastTask->user_id : null,
            name: $name,
            description: $description,
            chain: function ($task) use ($files) {
                $chain = [];

                foreach ($files as $file) {
                    $chain[] = new EmbedUidInMetadata($file['path'], $file['uuid'], $task->id);
                }

                return $chain;
            },
        );
    }

    /**
     * Regenerate open graph preview images
     *
     * @param  array<array{array: data, path: string}>  $rows
     * @return \Illuminate\Http\JsonResponse
     */
    public function regeneratePreviewImages(array $rows, ?int $userId = null) {
        $name = 'Regenerate Preview Images';
        $description = 'Regenerates open graph preview images.';

        return $this->executeBatchOperation(
            userId: $userId ?? null,
            name: $name,
            description: $description,
            chain: function ($task) use ($rows) {
                $chain = [];

                foreach ($rows as $row) {
                    if (! isset($row['data']) || ! isset($row['owner'])) {
                        continue;
                    }
                    $chain[] = new GeneratePreviewImage($row['data'], $row['owner'], $task->id);
                }

                return $chain;
            },
            queue: 'encode'
        );
    }

    public function purgeStaleData(array $data) {
        $name = 'Purge Stale Data';
        $description = 'Purges stale data older than 30 days such as guest data.';

        return $this->executeBatchOperation(
            userId: $data['userId'] ?? null,
            name: ($data['namePrefix'] ?? '') . $name,
            description: $description,
            chain: function ($task) {
                return [
                    new PurgeStaleGuestData($task->id),
                ];
            },
        );
    }

    public function generateStoryboards(array $data, ?Category $category = null): Task {
        $name = 'Generate Storyboards';
        $description = 'Generates storyboard sprite sheets for all videos missing them.';

        if (isset($category)) {
            $name .= " for library \"$category->name\"";
            $description = "Generates storyboard sprite sheets for the specified library \"$category->name\"";
        }

        return $this->executeBatchOperation(
            userId: $data['userId'] ?? null,
            name: ($data['namePrefix'] ?? '') . $name,
            description: $description,
            chain: function ($task) use ($category) {
                $chain = [];

                $query = Metadata::query()
                    ->select('metadata.*', 'videos.path as video_path')
                    ->join('videos', 'videos.id', '=', 'metadata.video_id')
                    ->join('folders', 'folders.id', '=', 'videos.folder_id')
                    ->join('categories', 'categories.id', '=', 'folders.category_id')
                    ->where('metadata.media_type', MediaType::VIDEO)
                    ->whereNotNull('metadata.uuid')
                    ->whereDoesntHave('storyboard')
                    ->where(function ($q) {
                        $q->whereNull('metadata.storyboard_scanned_at')->orWhereColumn('metadata.storyboard_scanned_at', '<', 'metadata.file_modified_at');
                    })
                    ->where('categories.storyboard_enabled', true)
                    ->latest('metadata.updated_at');

                if ($category) {
                    $query->where('categories.id', $category->id);
                }

                $limit = config('media.storyboard.daily_limit', 200);

                if ($limit > 0) {
                    $query->limit($limit);
                }

                $query->each(function ($metadata) use (&$chain, $task) {
                    $chain[] = new GenerateStoryboard(
                        filePath: VerifyFiles::getAbsoluteMediaPath($metadata->video),
                        uuid: $metadata->uuid,
                        taskId: $task->id,
                    );
                });

                return $chain;
            },
            queue: 'encode'
        );
    }

    public function regenerateStoryboard(int $userId, Metadata $metadata): Task {
        $name = 'Regenerate Storyboard';
        $description = 'Generates storyboard sprite sheets for ' . $metadata->composite_id;

        return $this->executeBatchOperation(
            userId: $userId,
            name: $name,
            description: $description,
            chain: function ($task) use ($metadata) {
                return [
                    new GenerateStoryboard(
                        filePath: VerifyFiles::getAbsoluteMediaPath($metadata->video),
                        uuid: $metadata->uuid,
                        taskId: $task->id,
                    ),
                ];
            },
            queue: 'encode'
        );
    }

    public function regenerateFonts(int $userId, Metadata $metadata): Task {
        $name = 'Regenerate Fonts';
        $description = 'Re-extracts embedded fonts for ' . $metadata->composite_id;

        return $this->executeBatchOperation(
            userId: $userId,
            name: $name,
            description: $description,
            chain: function ($task) use ($metadata) {
                return [
                    new ExtractFonts(
                        filePath: VerifyFiles::getAbsoluteMediaPath($metadata->video),
                        uuid: $metadata->uuid,
                        taskId: $task->id,
                    ),
                ];
            },
            queue: 'encode'
        );
    }

    public function executeBatchOperation(
        ?int $userId,
        string $name,
        string $description,
        array|callable $chain,
        ?callable $callback = null,
        array $initialTaskData = [],
        ?int $initialTaskId = null,
        ?string $queue = 'default',
    ) {
        $task = $initialTaskId ? Task::findOrFail($initialTaskId) : $this->setupTask($userId, $name, $description);

        try {
            $finalChain = is_callable($chain) ? $chain($task) : $chain;
            $chainLength = count($finalChain);

            if ($chainLength === 0) {
                $this->taskService->updateTask($task->id, [
                    'status' => TaskStatus::COMPLETED,
                    'ended_at' => now(),
                    'duration' => 0,
                ]);

                return $task;
            }

            $taskData = array_merge([
                'sub_tasks_total' => $chainLength + $task->sub_tasks_total,
                'sub_tasks_pending' => $chainLength,
                'sub_tasks_current' => $chainLength,
            ], $initialTaskData);

            $this->setupBatch(
                $finalChain,
                $task,
                $callback,
                $taskData,
                $queue
            );

            return $task;
        } catch (\Throwable $th) {
            $this->handleOperationFailure($task, $th);
            throw $th;
        }
    }

    public function setupTask(?int $userId, string $name, $description = '', $taskCount = 0) {
        return $this->taskService->createTask([
            'user_id' => $userId,
            'name' => $name,
            'description' => $description,
            'sub_tasks_total' => $taskCount,
            'sub_tasks_pending' => $taskCount,
        ]);
    }

    public function setupBatch(array $chain, Task $task, ?callable $callback, ?array $taskData, string $queue) {
        return Bus::batch($chain)
            ->catch(fn (Batch $batch, \Throwable $e) => $this->handleOperationFailure($task, $e))
            ->finally(fn (Batch $batch) => $this->finalizeBatch($batch, $task, $callback))
            ->before(fn (Batch $batch) => $this->taskService->updateTask($task->id, array_merge([
                'batch_id' => $batch->id,
            ], $taskData)))
            ->name($task->name)
            ->onQueue($queue)
            ->dispatch();
    }

    protected function finalizeBatch(Batch $batch, Task $task, ?callable $callback) {
        $task->refresh();
        // if task is not still processing, don't change what the current status is
        // otherwise base status on batch info

        // Track with total subtask count instead of current count
        // This should work because now I just add to the same batch rather than start new batches
        // This does not work
        // TODO: Make this setup work and get rid of current task count
        // if ($task->status !== TaskStatus::PROCESSING || $task->sub_tasks_total !== $batch->totalJobs) {
        // Log::warning('Early finalize batch return', ['Status' => $task->status, 'Total Task Count' => $task->sub_tasks_total, 'Batch Task Count' => $batch->totalJobs, 'Task' => $task]);
        // return;
        // }

        // if not processing or numbers dont match, task was ended elsewhere or has not started ?
        if ($task->status !== TaskStatus::PROCESSING || $task->sub_tasks_current !== $batch->totalJobs) {
            Log::warning('Early finalize batch return', ['Status' => $task->status, 'Current Task Count' => $task->sub_tasks_current, 'Batch Task Count' => $batch->totalJobs, 'Task' => $task]);

            return;
            // Might be incorrect
            // The idea is if the status was updated elsewhere, let them finish the job.
            // Same goes for total tasks being different from the batch, meaning tasks were added externally.
        }
        $initialStatus = $batch->processedJobs() > $task->sub_tasks_total
            ? TaskStatus::INCOMPLETE
            : TaskStatus::COMPLETED;
        $status = $batch->cancelled()
            ? TaskStatus::CANCELLED : $initialStatus;

        try {
            if ($callback) {
                $callback($task);

                return;
            }

            $duration = $task->started_at
                ? (int) now()->diffInSeconds($task->started_at)
                : 0;

            $this->taskService->updateTask($task->id, [
                'status' => $status,
                'ended_at' => now(),
                'duration' => abs($duration),
            ], $status === TaskStatus::COMPLETED);
        } catch (\Throwable $th) {
            Log::error('Error finalizing batch', [
                'task_id' => $task->id,
                'error' => $th->getMessage(),
            ]);
            throw $th;
        }
    }

    protected function handleOperationFailure(?Task $task, \Throwable $exception) {
        if (! $task) {
            return;
        }

        try {
            $this->taskService->updateTask($task->id, [
                'status' => TaskStatus::FAILED,
                'ended_at' => now(),
            ]);

            Log::error('Batch operation failed', [
                'task_id' => $task->id,
                'error' => $exception->getMessage(),
            ]);
        } catch (\Throwable $updateError) {
            Log::error('Failed to update task status', [
                'task_id' => $task->id,
                'original_error' => $exception->getMessage(),
                'update_error' => $updateError->getMessage(),
            ]);
        }
    }
}