Skip to content

An open-source, self-hostable PaaS alternative to Vercel, Heroku & Netlify that lets you easily deploy static sites, databases, full-stack applications and 360+ one-click services on your own servers.

fix(patches): show held back package updates apart from the updates

Andras Bacsai5ed8565edfdb
diff --git a/app/Actions/Server/CheckUpdates.php b/app/Actions/Server/CheckUpdates.php
index d0586f2d16..08775f9e6d 100644
--- a/app/Actions/Server/CheckUpdates.php
+++ b/app/Actions/Server/CheckUpdates.php
@@ -24,6 +24,28 @@ class CheckUpdates
*/
public const DNF_UPGRADES_COMMAND = 'LANG=C dnf repoquery -q --refresh --upgrades --latest-limit 1 --qf "%{name} %{arch} %{evr} %{repoid}\n"';
+ /**
+ * Simulations of `apt-get upgrade` without and with the phased updates. apt holds back an update in a phased rollout
+ * until this machine is in the rollout, but `apt list --upgradable` still shows it. The difference of the two runs
+ * gives the phased updates. Text matching is not possible: apt 2.4 (Ubuntu 22.04) prints them as "kept back".
+ *
+ * @see https://documentation.ubuntu.com/server/explanation/software/about-apt-upgrade-and-phased-updates/
+ */
+ public const APT_SIMULATE_UPGRADE_COMMAND = 'LANG=C apt-get -s upgrade 2>/dev/null';
+
+ public const APT_SIMULATE_PHASED_UPGRADE_COMMAND = 'LANG=C apt-get -s -o APT::Get::Always-Include-Phased-Updates=true upgrade 2>/dev/null';
+
+ /**
+ * Packages held with `apt-mark hold`. `apt list --upgradable` shows their updates, but apt does not install them.
+ */
+ public const APT_HELD_COMMAND = 'apt-mark showhold 2>/dev/null';
+
+ /**
+ * Simulation of `apk upgrade`. A version pin in /etc/apk/world (`name=1.2-r0`, `name<2`, `name~1.2`) blocks an
+ * update that `apk list --upgradable` still shows, and also the updates of packages that depend on that version.
+ */
+ public const APK_SIMULATE_UPGRADE_COMMAND = 'LANG=C apk upgrade -s 2>/dev/null';
+
public const RPM_INSTALLED_COMMAND = 'rpm -qa --qf "%{NAME} %{ARCH} %{VERSION}-%{RELEASE}\n"';
/**
@@ -103,6 +125,11 @@ SH;
'system' => 'Restarts core system services',
];
+ public const HELD_REASON_LABELS = [
+ 'phased' => 'Phased rollout',
+ 'held' => 'Held',
+ ];
+
public function handle(Server $server)
{
$osId = 'unknown';
@@ -179,7 +206,14 @@ SH;
break;
case 'apt':
instant_remote_process(['apt-get update -qq'], $server);
- $out = $this->parseAptOutput(instant_remote_process(['LANG=C apt list --upgradable 2>/dev/null'], $server));
+ $out = $this->separateHeldUpdates(
+ $this->parseAptOutput(instant_remote_process(['LANG=C apt list --upgradable 2>/dev/null'], $server)),
+ $this->parseAptHeldPackages(
+ (string) instant_remote_process([self::APT_HELD_COMMAND], $server, false),
+ (string) instant_remote_process([self::APT_SIMULATE_UPGRADE_COMMAND], $server, false),
+ (string) instant_remote_process([self::APT_SIMULATE_PHASED_UPGRADE_COMMAND], $server, false),
+ ),
+ );
break;
case 'pacman':
$out = $this->parsePacmanOutput(instant_remote_process([self::PACMAN_CHECK_COMMAND], $server));
@@ -187,6 +221,7 @@ SH;
case 'apk':
instant_remote_process(['apk update -q'], $server);
$out = $this->parseApkOutput(instant_remote_process(['LANG=C apk list --upgradable 2>/dev/null'], $server));
+ $out = $this->separateHeldUpdates($out, $this->parseApkHeldPackages($out, (string) instant_remote_process([self::APK_SIMULATE_UPGRADE_COMMAND], $server, false)));
break;
default:
return [
@@ -414,6 +449,62 @@ SH;
];
}
+ /**
+ * Moves the updates that the package manager does not install to `held_updates`, with the reason in `held_reason`.
+ *
+ * @param array<string, 'phased'|'held'> $heldPackages Package name => reason.
+ */
+ public function separateHeldUpdates(array $out, array $heldPackages): array
+ {
+ $updates = collect($out['updates'] ?? [])->partition(fn (array $update) => isset($heldPackages[$update['package']]));
+ $out['updates'] = $updates[1]->values()->all();
+ $out['held_updates'] = $updates[0]->map(fn (array $update) => [...$update, 'held_reason' => $heldPackages[$update['package']]])->values()->all();
+ $out['total_updates'] = count($out['updates']);
+
+ return $out;
+ }
+
+ /**
+ * @param string $held Output of APT_HELD_COMMAND.
+ * @param string $upgrade Output of APT_SIMULATE_UPGRADE_COMMAND.
+ * @param string $upgradeWithPhased Output of APT_SIMULATE_PHASED_UPGRADE_COMMAND.
+ * @return array<string, 'phased'|'held'>
+ */
+ public function parseAptHeldPackages(string $held, string $upgrade, string $upgradeWithPhased): array
+ {
+ $installs = function (string $output): ?array {
+ // Without the summary line, the simulation failed and tells nothing.
+ if (! preg_match('/^\d+ upgraded,/m', $output)) {
+ return null;
+ }
+ preg_match_all('/^Inst ([^\s:]+)/m', $output, $matches);
+
+ return $matches[1];
+ };
+ $installed = $installs($upgrade);
+ $installedWithPhased = $installs($upgradeWithPhased);
+ $phased = $installed === null || $installedWithPhased === null ? [] : array_diff($installedWithPhased, $installed);
+ preg_match_all('/^([^\s:]+)/m', $held, $heldMatches);
+
+ return [...array_fill_keys($phased, 'phased'), ...array_fill_keys($heldMatches[1], 'held')];
+ }
+
+ /**
+ * @param string $simulation Output of APK_SIMULATE_UPGRADE_COMMAND.
+ * @return array<string, 'held'>
+ */
+ public function parseApkHeldPackages(array $out, string $simulation): array
+ {
+ // Without the summary line, the simulation failed and tells nothing.
+ if (! preg_match('/^OK: /m', $simulation)) {
+ return [];
+ }
+ preg_match_all('/^\(\s*\d+\/\d+\) Upgrading (\S+) \(/m', $simulation, $matches);
+ $held = array_diff(array_column($out['updates'] ?? [], 'package'), $matches[1]);
+
+ return array_fill_keys($held, 'held');
+ }
+
public function parsePacmanOutput(string $output): array
{
$updates = [];
diff --git a/resources/views/livewire/server/security/patches.blade.php b/resources/views/livewire/server/security/patches.blade.php
index 7a668013d8..7a137bf7fa 100644
--- a/resources/views/livewire/server/security/patches.blade.php
+++ b/resources/views/livewire/server/security/patches.blade.php
@@ -20,6 +20,7 @@
->values()
->all();
$totalUpdates = count($updates);
+ $heldUpdates = collect(data_get($result, 'held_updates', []))->sortBy('package')->values()->all();
$packageManager = data_get($result, 'package_manager');
$checkedAt = $server->settings->patch_checked_at;
$fullUpgradeOnly = in_array($packageManager, \App\Actions\Server\CheckUpdates::FULL_UPGRADE_ONLY, true);
@@ -271,6 +272,39 @@
</x-application.settings-section>
@endif
+ @if (! $checking && ! $error && count($heldUpdates) > 0)
+ <x-application.settings-section id="server-held-updates-section" title="Held back updates"
+ helper="The package manager does not install these updates. Phased rollout: Ubuntu releases the update to a part of all servers at a time, and this server gets it in a later phase. Held: a package hold (apt-mark hold) or a version pin (/etc/apk/world) on the server blocks the update."
+ flush>
+ <div class="data-table">
+ <div class="data-table-header package-updates-table-grid">
+ <span>Package</span>
+ <span>New version</span>
+ <span class="text-right">Reason</span>
+ </div>
+ @foreach ($heldUpdates as $update)
+ <div wire:key="held-update-{{ data_get($update, 'package') }}-{{ data_get($update, 'architecture') }}"
+ class="data-table-row package-updates-table-grid border-b border-neutral-200 last:border-b-0 dark:border-white/[0.08]">
+ <span class="min-w-0 truncate font-mono text-[12px] text-neutral-950 dark:text-fg">
+ {{ data_get($update, 'package') }}
+ </span>
+ <div class="min-w-0">
+ <p class="truncate font-mono text-[12px] text-neutral-700 dark:text-fg-dim">
+ {{ data_get($update, 'new_version') }}
+ </p>
+ <p class="mt-0.5 truncate text-[10px] text-neutral-500 dark:text-fg-faint">
+ Current: {{ data_get($update, 'current_version') }}
+ </p>
+ </div>
+ <div class="flex justify-end">
+ <x-status-badge :status="\App\Actions\Server\CheckUpdates::HELD_REASON_LABELS[data_get($update, 'held_reason')] ?? 'Held'" type="warning" />
+ </div>
+ </div>
+ @endforeach
+ </div>
+ </x-application.settings-section>
+ @endif
+
</div>
</div>
</div>
diff --git a/tests/Feature/Server/ServerPatchesTest.php b/tests/Feature/Server/ServerPatchesTest.php
index b65c74c4fc..8661dd33a0 100644
--- a/tests/Feature/Server/ServerPatchesTest.php
+++ b/tests/Feature/Server/ServerPatchesTest.php
@@ -205,6 +205,25 @@ describe('patches page', function () {
->assertSee('openssl');
});
+ it('shows held back updates apart from the updates and does not update them', function () {
+ $result = [...patchesTestResult(0), 'held_updates' => [
+ ['package' => 'cloud-init', 'current_version' => '25.1.4-0ubuntu0~24.04.1', 'new_version' => '26.2-0ubuntu1~24.04.2', 'architecture' => 'all', 'repository' => 'noble-updates', 'held_reason' => 'phased'],
+ ['package' => 'tzdata', 'current_version' => '2025b-0ubuntu0.24.04', 'new_version' => '2025b-0ubuntu0.24.04.1', 'architecture' => 'all', 'repository' => 'noble-updates', 'held_reason' => 'held'],
+ ]];
+ $this->server->settings->forceFill(['patch_check_result' => $result, 'patch_checked_at' => now()])->saveQuietly();
+ $this->actingAs($this->admin);
+ session(['currentTeam' => $this->team]);
+
+ Livewire::test(Patches::class, ['server_uuid' => $this->server->uuid])
+ ->assertSee('Server is up to date')
+ ->assertSee('Held back updates')
+ ->assertSee('Phased rollout')
+ ->assertSee('26.2-0ubuntu1~24.04.2')
+ ->assertSee('2025b-0ubuntu0.24.04.1')
+ ->call('updatePackage', 'tzdata')
+ ->assertDispatched('error', message: 'This package has no update in the last check.');
+ });
+
it('updates only a package from the saved result', function () {
$this->actingAs($this->admin);
session(['currentTeam' => $this->team]);
diff --git a/tests/Unit/Actions/Server/ServerPatchCommandsTest.php b/tests/Unit/Actions/Server/ServerPatchCommandsTest.php
index 1d9c777eeb..f1287cb066 100644
--- a/tests/Unit/Actions/Server/ServerPatchCommandsTest.php
+++ b/tests/Unit/Actions/Server/ServerPatchCommandsTest.php
@@ -121,6 +121,102 @@ OUTPUT;
->and($result['updates'][0]['risk'])->toBe('system');
});
+it('moves the apt updates that a phased rollout holds back to their own list', function () {
+ // Ubuntu 24.04: cloud-init 26.2 is phased at 80% and this machine is not in the rollout yet.
+ $list = <<<'OUTPUT'
+Listing...
+cloud-init/noble-updates 26.2-0ubuntu1~24.04.2 all [upgradable from: 25.1.4-0ubuntu0~24.04.1]
+zlib1g/noble-updates,noble-security 1:1.3.dfsg-3.1ubuntu2.3 amd64 [upgradable from: 1:1.3.dfsg-3.1ubuntu2.2]
+OUTPUT;
+ $upgrade = <<<'OUTPUT'
+Calculating upgrade...
+The following upgrades have been deferred due to phasing:
+ cloud-init
+The following packages will be upgraded:
+ zlib1g
+1 upgraded, 0 newly installed, 0 to remove and 1 not upgraded.
+Inst zlib1g [1:1.3.dfsg-3.1ubuntu2.2] (1:1.3.dfsg-3.1ubuntu2.3 Ubuntu:24.04/noble-updates, Ubuntu:24.04/noble-security [amd64])
+Conf zlib1g (1:1.3.dfsg-3.1ubuntu2.3 Ubuntu:24.04/noble-updates, Ubuntu:24.04/noble-security [amd64])
+OUTPUT;
+ $upgradeWithPhased = <<<'OUTPUT'
+Calculating upgrade...
+The following packages will be upgraded:
+ cloud-init zlib1g
+2 upgraded, 0 newly installed, 0 to remove and 0 not upgraded.
+Inst zlib1g [1:1.3.dfsg-3.1ubuntu2.2] (1:1.3.dfsg-3.1ubuntu2.3 Ubuntu:24.04/noble-updates, Ubuntu:24.04/noble-security [amd64])
+Inst cloud-init [25.1.4-0ubuntu0~24.04.1] (26.2-0ubuntu1~24.04.2 Ubuntu:24.04/noble-updates [all])
+Conf zlib1g (1:1.3.dfsg-3.1ubuntu2.3 Ubuntu:24.04/noble-updates, Ubuntu:24.04/noble-security [amd64])
+Conf cloud-init (26.2-0ubuntu1~24.04.2 Ubuntu:24.04/noble-updates [all])
+OUTPUT;
+ $action = new CheckUpdates;
+
+ $result = $action->separateHeldUpdates($action->parseAptOutput($list), $action->parseAptHeldPackages('', $upgrade, $upgradeWithPhased));
+
+ expect(collect($result['updates'])->pluck('package')->all())->toBe(['zlib1g'])
+ ->and($result['total_updates'])->toBe(1)
+ ->and($result['held_updates'])->toBe([[
+ 'package' => 'cloud-init',
+ 'repository' => 'noble-updates',
+ 'new_version' => '26.2-0ubuntu1~24.04.2',
+ 'architecture' => 'all',
+ 'current_version' => '25.1.4-0ubuntu0~24.04.1',
+ 'held_reason' => 'phased',
+ ]]);
+
+ // A failed simulation must not mark every update as phased.
+ expect($action->parseAptHeldPackages('', '', $upgradeWithPhased))->toBe([]);
+});
+
+it('moves the apt updates of held packages to the held list', function () {
+ // Debian 12 after `apt-mark hold tzdata`: `apt list --upgradable` still shows tzdata.
+ $list = <<<'OUTPUT'
+Listing...
+tzdata/stable-updates 2025b-0+deb12u2 all [upgradable from: 2025b-0+deb12u1]
+OUTPUT;
+ $action = new CheckUpdates;
+
+ $result = $action->separateHeldUpdates($action->parseAptOutput($list), $action->parseAptHeldPackages("tzdata\n", '', ''));
+
+ expect($result['updates'])->toBe([])
+ ->and($result['total_updates'])->toBe(0)
+ ->and(collect($result['held_updates'])->pluck('held_reason', 'package')->all())->toBe(['tzdata' => 'held']);
+});
+
+it('moves the apk updates that a version pin blocks to the held list', function () {
+ // Alpine 3.20.0 with `busybox=1.36.1-r28` in /etc/apk/world: busybox-binsh needs the same busybox version.
+ $list = <<<'OUTPUT'
+busybox-1.36.1-r31 x86_64 {busybox} (GPL-2.0-only) [upgradable from: busybox-1.36.1-r28]
+busybox-binsh-1.36.1-r31 x86_64 {busybox} (GPL-2.0-only) [upgradable from: busybox-binsh-1.36.1-r28]
+zlib-1.3.2-r0 x86_64 {zlib} (Zlib) [upgradable from: zlib-1.3.1-r1]
+OUTPUT;
+ $simulation = <<<'OUTPUT'
+WARNING: This simulation is not reliable as apk-tools upgrade is available.
+(1/1) Upgrading zlib (1.3.1-r1 -> 1.3.2-r0)
+OK: 8 MiB in 14 packages
+OUTPUT;
+ $action = new CheckUpdates;
+ $out = $action->parseApkOutput($list);
+
+ $result = $action->separateHeldUpdates($out, $action->parseApkHeldPackages($out, $simulation));
+
+ expect(collect($result['updates'])->pluck('package')->all())->toBe(['zlib'])
+ ->and(collect($result['held_updates'])->pluck('held_reason', 'package')->all())->toBe(['busybox' => 'held', 'busybox-binsh' => 'held'])
+ // apk-tools 3 pads the step number; a failed simulation has no OK line and marks nothing.
+ ->and($action->parseApkHeldPackages($out, "( 1/3) Upgrading busybox (1.36.1-r28 -> 1.36.1-r31)\n( 2/3) Upgrading busybox-binsh (1.36.1-r28 -> 1.36.1-r31)\n( 3/3) Upgrading zlib (1.3.1-r1 -> 1.3.2-r0)\nOK: 8 MiB in 14 packages\n"))->toBe([])
+ ->and($action->parseApkHeldPackages($out, "ERROR: unable to lock database\n"))->toBe([]);
+});
+
+it('runs the held package checks with sudo for a non-root user', function () {
+ $commands = [CheckUpdates::APT_HELD_COMMAND, CheckUpdates::APT_SIMULATE_UPGRADE_COMMAND, CheckUpdates::APT_SIMULATE_PHASED_UPGRADE_COMMAND, CheckUpdates::APK_SIMULATE_UPGRADE_COMMAND];
+
+ expect(parseCommandsByLineForSudo(collect($commands), patchCommandsNonRootServer()))->toBe([
+ 'sudo apt-mark showhold 2>/dev/null',
+ 'sudo LANG=C apt-get -s upgrade 2>/dev/null',
+ 'sudo LANG=C apt-get -s -o APT::Get::Always-Include-Phased-Updates=true upgrade 2>/dev/null',
+ 'sudo LANG=C apk upgrade -s 2>/dev/null',
+ ]);
+});
+
it('reads the security packages of dnf4 and dnf5 advisories', function () {
// Rocky Linux 9 (dnf4) and Fedora 42 (dnf5) `dnf updateinfo list --security`.
$dnf4 = <<<'OUTPUT'