A self-hosted media server and file management system
Vue ffmpegfile-serverjellyfinlaravelmediamedia-playermedia-servermediabrowsermediaservermusicmusic-librarymusic-playermusicplayerself-hostedvuejs
<?php
namespace App\Services;
use App\Enums\MediaType;
use App\Enums\TaskStatus;
use App\Jobs\EmbedUidInMetadata;
use App\Jobs\GeneratePreviewImage;
use App\Jobs\IndexFiles;
use App\Jobs\Maintenance\PurgeStaleGuestData;
use App\Jobs\Metadata\ExtractFonts;
use App\Jobs\Metadata\GenerateStoryboard;
use App\Jobs\SyncFiles;
use App\Jobs\Utility\Paths\CleanFolderPaths;
use App\Jobs\Utility\Paths\CleanVideoPaths;
use App\Jobs\VerifyFiles;
use App\Jobs\VerifyFolders;
use App\Models\Category;
use App\Models\Folder;
use App\Models\Metadata;
use App\Models\Task;
use App\Models\Video;
use Illuminate\Bus\Batch;
use Illuminate\Support\Facades\Bus;
use Illuminate\Support\Facades\Log;
class FileJobService {
public function __construct(protected TaskService $taskService) {}
public function scanFiles(array $data, ?Category $category = null) {
$name = 'Scan Files';
$description = 'Scans for file changes and loads metadata from all libraries.';
$categoryId = null;
if (isset($category)) {
$name .= " from the library \"$category->name\"";
$description = "Scans for file changes and loads metadata from the specified library \"$category->name\"";
$categoryId = $category->id;
}
return $this->executeBatchOperation(
userId: $data['userId'] ?? null,
name: ($data['namePrefix'] ?? '') . $name,
description: $description,
chain: function ($task) {
return [
new SyncFiles($task->id),
new IndexFiles($task->id),
];
},
callback: function ($task) use ($data, $categoryId) {
$category = $categoryId ? Category::findOrFail($categoryId) : null;
$this->verifyFiles($data, $category, $task->id);
}
);
}
public function indexFiles(array $data, ?Category $category = null) {
$name = 'Index Files';
$description = 'Looks for folder and video changes in in all libraries.';
if (isset($category)) {
$name .= " for library $category->name";
$description = "Looks for folder and video changes in the specified library $category->name";
}
return $this->executeBatchOperation(
userId: $data['userId'] ?? null,
name: ($data['namePrefix'] ?? '') . $name,
description: $description,
chain: function ($task) {
return [
new SyncFiles($task->id),
new IndexFiles($task->id),
];
},
);
}
public function syncFiles(array $data) {
return $this->executeBatchOperation(
userId: $data['userId'] ?? null,
name: 'Sync Files',
description: 'Syncs local file structure with database.',
chain: function ($task) {
return [
new SyncFiles($task->id),
];
},
);
}
public function verifyFiles(array $data, ?Category $category = null, ?int $taskId = null) {
$name = 'Verify Files';
$description = 'Verifies folder and video metadata for all libraries.';
if (isset($category)) {
$name .= " for library \"$category->name\"";
$description = "Verifies folder and video metadata for the specified library \"$category->name\"";
}
return $this->executeBatchOperation(
userId: $data['userId'] ?? null,
name: $name,
description: $description,
chain: function ($task) use ($category) {
$chain = [];
$videoQuery = Video::orderBy('id')->with(['metadata.storyboard', 'folder.category']);
$folderQuery = Folder::orderBy('id');
if ($category) {
$videoQuery = $videoQuery->whereHas('folder.category', function ($query) use ($category) {
$query->where('id', $category->id);
})->with('folder.category');
$folderQuery = $folderQuery->whereHas('category', function ($query) use ($category) {
$query->where('id', $category->id);
})->with('category');
}
$videoQuery->chunk(100, function ($chunk) use (&$chain, $task, $category) {
$chain[] = new VerifyFiles($chunk, $task->id, generateImageTasks: $category !== null);
});
$folderQuery->chunk(100, function ($chunk) use (&$chain, $task) {
$chain[] = new VerifyFolders($chunk, $task->id);
});
return $chain;
},
initialTaskId: $taskId,
);
}
public function verifyFolders(array $data, ?Category $category = null) {
$name = 'Verify Folders';
$description = 'Verifies folder metadata for all libraries.';
if (isset($category)) {
$name .= " for library \"$category->name\"";
$description = "Verifies folder metadata for the specified library \"$category->name\"";
}
return $this->executeBatchOperation(
userId: $data['userId'] ?? null,
name: $name,
description: $description,
chain: function ($task) use ($category) {
$chain = [];
$query = Folder::orderBy('id');
if ($category) {
$query->whereHas('category', fn ($q) => $q->where('id', $category->id))
->with('category');
}
$query->chunk(100, function ($chunk) use (&$chain, $task) {
$chain[] = new VerifyFolders($chunk, $task->id);
});
return $chain;
},
);
}
public function cleanPaths(array $data) {
$name = 'Clean Paths';
$description = 'Cleans file and folder paths for all libraries.';
return $this->executeBatchOperation(
userId: $data['userId'] ?? null,
name: $name,
description: $description,
chain: function ($task) {
$chain = [];
Video::orderBy('id')->chunk(20)->each(function ($chunk) use (&$chain, $task) {
$chain[] = new CleanVideoPaths($chunk, $task->id);
});
Folder::orderBy('id')->chunk(20)->each(function ($chunk) use (&$chain, $task) {
$chain[] = new CleanFolderPaths($chunk, $task->id);
});
return $chain;
},
);
}
/**
* Embed UIDs in file metadata
* Unused?
*
* @param array<array{path: string, uuid: string}> $files
* @return \Illuminate\Http\JsonResponse
*/
public function embedUIDs(int $taskId, string $name, array $files) {
$description = 'Embeds UIDs in file metadata.';
$lastTask = Task::find($taskId);
return $this->executeBatchOperation(
userId: $lastTask ? $lastTask->user_id : null,
name: $name,
description: $description,
chain: function ($task) use ($files) {
$chain = [];
foreach ($files as $file) {
$chain[] = new EmbedUidInMetadata($file['path'], $file['uuid'], $task->id);
}
return $chain;
},
);
}
/**
* Regenerate open graph preview images
*
* @param array<array{array: data, path: string}> $rows
* @return \Illuminate\Http\JsonResponse
*/
public function regeneratePreviewImages(array $rows, ?int $userId = null) {
$name = 'Regenerate Preview Images';
$description = 'Regenerates open graph preview images.';
return $this->executeBatchOperation(
userId: $userId ?? null,
name: $name,
description: $description,
chain: function ($task) use ($rows) {
$chain = [];
foreach ($rows as $row) {
if (! isset($row['data']) || ! isset($row['owner'])) {
continue;
}
$chain[] = new GeneratePreviewImage($row['data'], $row['owner'], $task->id);
}
return $chain;
},
queue: 'encode'
);
}
public function purgeStaleData(array $data) {
$name = 'Purge Stale Data';
$description = 'Purges stale data older than 30 days such as guest data.';
return $this->executeBatchOperation(
userId: $data['userId'] ?? null,
name: ($data['namePrefix'] ?? '') . $name,
description: $description,
chain: function ($task) {
return [
new PurgeStaleGuestData($task->id),
];
},
);
}
public function generateStoryboards(array $data, ?Category $category = null): Task {
$name = 'Generate Storyboards';
$description = 'Generates storyboard sprite sheets for all videos missing them.';
if (isset($category)) {
$name .= " for library \"$category->name\"";
$description = "Generates storyboard sprite sheets for the specified library \"$category->name\"";
}
return $this->executeBatchOperation(
userId: $data['userId'] ?? null,
name: ($data['namePrefix'] ?? '') . $name,
description: $description,
chain: function ($task) use ($category) {
$chain = [];
$query = Metadata::query()
->select('metadata.*', 'videos.path as video_path')
->join('videos', 'videos.id', '=', 'metadata.video_id')
->join('folders', 'folders.id', '=', 'videos.folder_id')
->join('categories', 'categories.id', '=', 'folders.category_id')
->where('metadata.media_type', MediaType::VIDEO)
->whereNotNull('metadata.uuid')
->whereDoesntHave('storyboard')
->where(function ($q) {
$q->whereNull('metadata.storyboard_scanned_at')->orWhereColumn('metadata.storyboard_scanned_at', '<', 'metadata.file_modified_at');
})
->where('categories.storyboard_enabled', true)
->latest('metadata.updated_at');
if ($category) {
$query->where('categories.id', $category->id);
}
$limit = config('media.storyboard.daily_limit', 200);
if ($limit > 0) {
$query->limit($limit);
}
$query->each(function ($metadata) use (&$chain, $task) {
$chain[] = new GenerateStoryboard(
filePath: VerifyFiles::getAbsoluteMediaPath($metadata->video),
uuid: $metadata->uuid,
taskId: $task->id,
);
});
return $chain;
},
queue: 'encode'
);
}
public function regenerateStoryboard(int $userId, Metadata $metadata): Task {
$name = 'Regenerate Storyboard';
$description = 'Generates storyboard sprite sheets for ' . $metadata->composite_id;
return $this->executeBatchOperation(
userId: $userId,
name: $name,
description: $description,
chain: function ($task) use ($metadata) {
return [
new GenerateStoryboard(
filePath: VerifyFiles::getAbsoluteMediaPath($metadata->video),
uuid: $metadata->uuid,
taskId: $task->id,
),
];
},
queue: 'encode'
);
}
public function regenerateFonts(int $userId, Metadata $metadata): Task {
$name = 'Regenerate Fonts';
$description = 'Re-extracts embedded fonts for ' . $metadata->composite_id;
return $this->executeBatchOperation(
userId: $userId,
name: $name,
description: $description,
chain: function ($task) use ($metadata) {
return [
new ExtractFonts(
filePath: VerifyFiles::getAbsoluteMediaPath($metadata->video),
uuid: $metadata->uuid,
taskId: $task->id,
),
];
},
queue: 'encode'
);
}
public function executeBatchOperation(
?int $userId,
string $name,
string $description,
array|callable $chain,
?callable $callback = null,
array $initialTaskData = [],
?int $initialTaskId = null,
?string $queue = 'default',
) {
$task = $initialTaskId ? Task::findOrFail($initialTaskId) : $this->setupTask($userId, $name, $description);
try {
$finalChain = is_callable($chain) ? $chain($task) : $chain;
$chainLength = count($finalChain);
if ($chainLength === 0) {
$this->taskService->updateTask($task->id, [
'status' => TaskStatus::COMPLETED,
'ended_at' => now(),
'duration' => 0,
]);
return $task;
}
$taskData = array_merge([
'sub_tasks_total' => $chainLength + $task->sub_tasks_total,
'sub_tasks_pending' => $chainLength,
'sub_tasks_current' => $chainLength,
], $initialTaskData);
$this->setupBatch(
$finalChain,
$task,
$callback,
$taskData,
$queue
);
return $task;
} catch (\Throwable $th) {
$this->handleOperationFailure($task, $th);
throw $th;
}
}
public function setupTask(?int $userId, string $name, $description = '', $taskCount = 0) {
return $this->taskService->createTask([
'user_id' => $userId,
'name' => $name,
'description' => $description,
'sub_tasks_total' => $taskCount,
'sub_tasks_pending' => $taskCount,
]);
}
public function setupBatch(array $chain, Task $task, ?callable $callback, ?array $taskData, string $queue) {
return Bus::batch($chain)
->catch(fn (Batch $batch, \Throwable $e) => $this->handleOperationFailure($task, $e))
->finally(fn (Batch $batch) => $this->finalizeBatch($batch, $task, $callback))
->before(fn (Batch $batch) => $this->taskService->updateTask($task->id, array_merge([
'batch_id' => $batch->id,
], $taskData)))
->name($task->name)
->onQueue($queue)
->dispatch();
}
protected function finalizeBatch(Batch $batch, Task $task, ?callable $callback) {
$task->refresh();
// if task is not still processing, don't change what the current status is
// otherwise base status on batch info
// Track with total subtask count instead of current count
// This should work because now I just add to the same batch rather than start new batches
// This does not work
// TODO: Make this setup work and get rid of current task count
// if ($task->status !== TaskStatus::PROCESSING || $task->sub_tasks_total !== $batch->totalJobs) {
// Log::warning('Early finalize batch return', ['Status' => $task->status, 'Total Task Count' => $task->sub_tasks_total, 'Batch Task Count' => $batch->totalJobs, 'Task' => $task]);
// return;
// }
// if not processing or numbers dont match, task was ended elsewhere or has not started ?
if ($task->status !== TaskStatus::PROCESSING || $task->sub_tasks_current !== $batch->totalJobs) {
Log::warning('Early finalize batch return', ['Status' => $task->status, 'Current Task Count' => $task->sub_tasks_current, 'Batch Task Count' => $batch->totalJobs, 'Task' => $task]);
return;
// Might be incorrect
// The idea is if the status was updated elsewhere, let them finish the job.
// Same goes for total tasks being different from the batch, meaning tasks were added externally.
}
$initialStatus = $batch->processedJobs() > $task->sub_tasks_total
? TaskStatus::INCOMPLETE
: TaskStatus::COMPLETED;
$status = $batch->cancelled()
? TaskStatus::CANCELLED : $initialStatus;
try {
if ($callback) {
$callback($task);
return;
}
$duration = $task->started_at
? (int) now()->diffInSeconds($task->started_at)
: 0;
$this->taskService->updateTask($task->id, [
'status' => $status,
'ended_at' => now(),
'duration' => abs($duration),
], $status === TaskStatus::COMPLETED);
} catch (\Throwable $th) {
Log::error('Error finalizing batch', [
'task_id' => $task->id,
'error' => $th->getMessage(),
]);
throw $th;
}
}
protected function handleOperationFailure(?Task $task, \Throwable $exception) {
if (! $task) {
return;
}
try {
$this->taskService->updateTask($task->id, [
'status' => TaskStatus::FAILED,
'ended_at' => now(),
]);
Log::error('Batch operation failed', [
'task_id' => $task->id,
'error' => $exception->getMessage(),
]);
} catch (\Throwable $updateError) {
Log::error('Failed to update task status', [
'task_id' => $task->id,
'original_error' => $exception->getMessage(),
'update_error' => $updateError->getMessage(),
]);
}
}
}