Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

15.0 KB
Raw Blame
<?php

use App\Enums\ActivityTypes;
use App\Enums\ProcessStatus;
use App\Helpers\SshMultiplexingHelper;
use App\Helpers\SshRetryHandler;
use App\Jobs\CoolifyTask;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Services\Security\SensitiveDataRedactor;
use App\Support\RemoteProcessCommand;
use Carbon\Carbon;
use Illuminate\Contracts\Process\ProcessResult;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Process\Exceptions\ProcessTimedOutException;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Process;
use Spatie\Activitylog\Contracts\Activity;

function remote_process(
    Collection|array $command,
    Server $server,
    ?string $type = null,
    ?string $type_uuid = null,
    ?Model $model = null,
    bool $ignore_errors = false,
    $callEventOnFinish = null,
    $callEventData = null,
    array $properties = [],
    ?int $timeout = null,
    string $queue = 'high',
): Activity {
    $type = $type ?? ActivityTypes::INLINE->value;
    $command = $command instanceof Collection ? $command->toArray() : $command;

    if ($server->isNonRoot()) {
        $command = parseCommandsByLineForSudo(collect($command), $server);
    }

    $command_string = implode("\n", $command);

    if (Auth::check()) {
        $teams = Auth::user()->teams->pluck('id');
        if (! $teams->contains($server->team_id) && ! $teams->contains(0)) {
            throw new Exception('User is not part of the team that owns this server');
        }
    }

    SshMultiplexingHelper::ensureMultiplexedConnection($server);

    $properties = [
        ...$properties,
        'server_uuid' => $server->uuid,
        ...RemoteProcessCommand::properties($command_string),
        'type' => $type,
        'type_uuid' => $type_uuid,
        'status' => ProcessStatus::QUEUED->value,
        'team_id' => $server->team_id,
    ];

    $activityLog = activity()
        ->withProperties($properties)
        ->event($type);

    if ($model) {
        $activityLog->performedOn($model);
    }

    $activity = $activityLog->log('[]');

    dispatch(new CoolifyTask(
        activity: $activity,
        ignore_errors: $ignore_errors,
        call_event_on_finish: $callEventOnFinish,
        call_event_data: $callEventData,
        timeout: $timeout,
        queue: $queue,
    ));

    $activity->refresh();

    return $activity;
}

function instant_scp(string $source, string $dest, Server $server, $throwError = true)
{
    return SshRetryHandler::retry(
        function () use ($source, $dest, $server) {
            $scp_command = SshMultiplexingHelper::generateScpCommand($server, $source, $dest);
            $process = Process::timeout(config('constants.ssh.command_timeout'))->run($scp_command);

            $output = trim($process->output());
            $exitCode = $process->exitCode();

            if ($exitCode !== 0) {
                excludeCertainErrors($process->errorOutput(), $exitCode);
            }

            return $output === 'null' ? null : $output;
        },
        [
            'server' => $server->ip,
            'source' => $source,
            'dest' => $dest,
            'function' => 'instant_scp',
        ],
        $throwError
    );
}

/**
 * Download a remote file from a managed server onto the Coolify host via SCP.
 */
function instant_scp_from_server(string $remoteSource, string $localDest, Server $server, $throwError = true)
{
    return SshRetryHandler::retry(
        function () use ($remoteSource, $localDest, $server) {
            $scp_command = SshMultiplexingHelper::generateScpDownloadCommand($server, $remoteSource, $localDest);
            $process = Process::timeout(config('constants.ssh.command_timeout'))->run($scp_command);

            $output = trim($process->output());
            $exitCode = $process->exitCode();

            if ($exitCode !== 0) {
                excludeCertainErrors($process->errorOutput(), $exitCode);
            }

            return $output === 'null' ? null : $output;
        },
        [
            'server' => $server->ip,
            'source' => $remoteSource,
            'dest' => $localDest,
            'function' => 'instant_scp_from_server',
        ],
        $throwError
    );
}

/**
 * Runs an SSH command. The timeout error of the process contains the complete command line, and with it
 * the secrets of the remote script (database passwords, S3 keys), so it is replaced with a generic error.
 * Exit code 124 (as from `timeout`) marks it as a remote command failure, so the batch is not run again.
 */
function runSshProcess(string $sshCommand, int $timeout, ?string $input = null): ProcessResult
{
    try {
        return Process::timeout($timeout)->input($input)->run($sshCommand);
    } catch (ProcessTimedOutException) {
        throw new RuntimeException("The command on the server timed out after {$timeout} seconds.", 124);
    }
}

/**
 * Write content to a file on a server through SSH stdin. The content never becomes part of a command
 * line, so it is not limited by the 128 KiB per-argument limit of Linux. A non-root SSH user writes
 * with `sudo tee` because Coolify directories such as /data/coolify are root-only.
 */
function instant_remote_write_file(Server $server, string $path, string $content, bool $throwError = true): void
{
    $remoteCommand = ($server->isNonRoot() ? 'sudo ' : '').'tee '.escapeshellarg($path).' > /dev/null';
    $timeout = (int) config('constants.ssh.command_timeout');

    SshRetryHandler::retry(
        function () use ($server, $remoteCommand, $content, $timeout) {
            $sshCommand = SshMultiplexingHelper::generateSshStdinCommand($server, $remoteCommand, commandTimeout: $timeout);
            $process = runSshProcess($sshCommand, $timeout, $content);

            if ($process->exitCode() !== 0) {
                excludeCertainErrors($process->errorOutput(), $process->exitCode());
            }
        },
        [
            'server' => $server->ip,
            'dest' => $path,
            'function' => 'instant_remote_write_file',
        ],
        $throwError
    );
}

function instant_remote_process_with_timeout(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false): ?string
{
    $command = $command instanceof Collection ? $command->toArray() : $command;
    if ($server->isNonRoot() && ! $no_sudo) {
        $command = parseCommandsByLineForSudo(collect($command), $server);
    }
    $command_string = implode("\n", $command);

    return SshRetryHandler::retry(
        function () use ($server, $command_string) {
            $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string);
            $process = runSshProcess($sshCommand, 30);

            $output = trim($process->output());
            $exitCode = $process->exitCode();

            if ($exitCode !== 0) {
                excludeCertainErrors($process->errorOutput(), $exitCode);
            }

            // Sanitize output to ensure valid UTF-8 encoding
            $output = $output === 'null' ? null : sanitize_utf8_text($output);

            return $output;
        },
        [
            'server' => $server->ip,
            'command_preview' => substr($command_string, 0, 100),
            'function' => 'instant_remote_process_with_timeout',
        ],
        $throwError
    );
}

function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false, ?int $timeout = null, bool $disableMultiplexing = false, ?int $maxAttempts = null): ?string
{
    $command = $command instanceof Collection ? $command->toArray() : $command;

    if ($server->isNonRoot() && ! $no_sudo) {
        $command = parseCommandsByLineForSudo(collect($command), $server);
    }
    $command_string = implode("\n", $command);
    $effectiveTimeout = $timeout ?? config('constants.ssh.command_timeout');

    return SshRetryHandler::retry(
        function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {
            $sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);
            $process = runSshProcess($sshCommand, (int) $effectiveTimeout);

            $output = trim($process->output());
            $exitCode = $process->exitCode();

            if ($exitCode !== 0) {
                excludeCertainErrors($process->errorOutput(), $exitCode);
            }

            // Sanitize output to ensure valid UTF-8 encoding
            $output = $output === 'null' ? null : sanitize_utf8_text($output);

            return $output;
        },
        [
            'server' => $server->ip,
            'command_preview' => substr($command_string, 0, 100),
            'function' => 'instant_remote_process',
        ],
        $throwError,
        $maxAttempts
    );
}

/**
 * Throws the remote command error.
 */
function excludeCertainErrors(string $errorOutput, ?int $exitCode = null): never
{
    $errorMessage = trim($errorOutput);
    if (empty($errorMessage)) {
        $errorMessage = "SSH command failed with exit code: $exitCode";
    }

    throw new RuntimeException($errorMessage, $exitCode);
}

function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null, bool $includeAll = false): Collection
{
    if (is_null($application_deployment_queue)) {
        return collect([]);
    }
    $application = Application::find(data_get($application_deployment_queue, 'application_id'));
    $is_debug_enabled = data_get($application, 'settings.is_debug_enabled');
    $serverTimezone = getServerTimezone(data_get($application, 'destination.server'));

    // Members should never see debug logs, even if an admin enabled debug mode
    if ($is_debug_enabled && auth()->check()) {
        $teamId = $application?->team()?->id;
        if (is_null($teamId) || ! auth()->user()->isAdminOfTeam($teamId)) {
            $is_debug_enabled = false;
        }
    }

    $logs = data_get($application_deployment_queue, 'logs');
    if (empty($logs)) {
        return collect([]);
    }

    try {
        $decoded = json_decode(
            $logs,
            associative: true,
            flags: JSON_THROW_ON_ERROR
        );
    } catch (JsonException $e) {
        // If JSON decoding fails, try to clean up the logs and retry
        try {
            // Ensure valid UTF-8 encoding
            $cleaned_logs = sanitize_utf8_text($logs);
            $decoded = json_decode(
                $cleaned_logs,
                associative: true,
                flags: JSON_THROW_ON_ERROR
            );
        } catch (JsonException $e) {
            // If it still fails, return empty collection to prevent crashes
            return collect([]);
        }
    }

    if (! is_array($decoded)) {
        return collect([]);
    }

    $seenCommands = collect();
    $formatted = collect($decoded);
    if (! $is_debug_enabled && ! $includeAll) {
        $formatted = $formatted->filter(fn ($i) => $i['hidden'] === false ?? false);
    }

    return $formatted
        ->sortBy(fn ($i) => data_get($i, 'order'))
        ->map(function ($i) use ($serverTimezone) {
            $timestamp = Carbon::parse(data_get($i, 'timestamp'));
            try {
                $timestamp->setTimezone($serverTimezone);
            } catch (Exception) {
                $timestamp->setTimezone('UTC');
            }
            data_set($i, 'timestamp', $timestamp->format('Y-M-d H:i:s'));

            return $i;
        })
        ->reduce(function ($deploymentLogLines, $logItem) use ($seenCommands) {
            $command = data_get($logItem, 'command');
            $isStderr = data_get($logItem, 'type') === 'stderr';
            $isNewCommand = ! is_null($command) && ! $seenCommands->first(function ($seenCommand) use ($logItem) {
                return data_get($seenCommand, 'command') === data_get($logItem, 'command') && data_get($seenCommand, 'batch') === data_get($logItem, 'batch');
            });

            if ($isNewCommand) {
                $deploymentLogLines->push([
                    'line' => $command,
                    'timestamp' => data_get($logItem, 'timestamp'),
                    'stderr' => $isStderr,
                    'hidden' => data_get($logItem, 'hidden'),
                    'command' => true,
                ]);

                $seenCommands->push([
                    'command' => $command,
                    'batch' => data_get($logItem, 'batch'),
                ]);
            }

            $lines = explode(PHP_EOL, data_get($logItem, 'output'));

            foreach ($lines as $line) {
                $deploymentLogLines->push([
                    'line' => $line,
                    'timestamp' => data_get($logItem, 'timestamp'),
                    'stderr' => $isStderr,
                    'hidden' => data_get($logItem, 'hidden'),
                ]);
            }

            return $deploymentLogLines;
        }, collect());
}

function remove_iip($text)
{
    return app(SensitiveDataRedactor::class)->redactText((string) $text);
}

/**
 * Sanitizes text to ensure it contains valid UTF-8 encoding.
 *
 * This function is crucial for preventing "Malformed UTF-8 characters" errors
 * that can occur when Docker build output contains binary data mixed with text,
 * especially during image processing or builds with many assets.
 *
 * @param  string|null  $text  The text to sanitize
 * @return string Valid UTF-8 encoded text
 */
function sanitize_utf8_text(?string $text): string
{
    if (empty($text)) {
        return '';
    }

    // Convert to UTF-8, replacing invalid sequences
    $sanitized = mb_convert_encoding($text, 'UTF-8', 'UTF-8');

    // Additional fallback: use SUBSTITUTE flag to replace invalid sequences with substitution character
    if (! mb_check_encoding($sanitized, 'UTF-8')) {
        $sanitized = mb_convert_encoding($text, 'UTF-8', mb_detect_encoding($text, mb_detect_order(), true) ?: 'UTF-8');
    }

    return $sanitized;
}

function refresh_server_connection(?PrivateKey $private_key = null)
{
    if (is_null($private_key)) {
        return;
    }
    foreach ($private_key->servers as $server) {
        SshMultiplexingHelper::removeMuxFile($server);
    }
}

function checkRequiredCommands(Server $server)
{
    $commands = collect(['jq', 'jc']);
    foreach ($commands as $command) {
        $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false);
        if ($commandFound) {
            continue;
        }
        try {
            instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'apt update && apt install -y {$command}'"], $server);
        } catch (Throwable) {
            break;
        }
        $commandFound = instant_remote_process(["docker run --rm --privileged --net=host --pid=host --ipc=host --volume /:/host busybox chroot /host bash -c 'command -v {$command}'"], $server, false);
        if (! $commandFound) {
            break;
        }
    }
}