Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Read Docker packaging layers from their archive manifest
pulse-triage[bot]2312172e9a32
diff --git a/scripts/test-container-packaging.py b/scripts/test-container-packaging.pyindex 8a55f886b5..789d8c8cef 100644--- a/scripts/test-container-packaging.py+++ b/scripts/test-container-packaging.py@@ -5,6 +5,7 @@ No publication, registry credentials or release qualification. The immutablepublished fixture is deliberately independent of the proposed application code."""import argparse+import gzipimport hashlibimport jsonimport subprocess@@ -61,25 +62,53 @@ def layers(stream):or a multi-platform index size. Neither number includes writable volumes."""result = []- with tarfile.open(fileobj=stream, mode="r|*") as outer:- for entry in outer:- if not entry.name.endswith("/layer.tar"):- continue+ # Classic Docker archives use <id>/layer.tar; containerd-backed Docker+ # uses blobs/sha256/<digest>. Both name the retained layers in manifest.json.+ # Spool opaque blobs under generated names, never archive-selected paths.+ with tempfile.TemporaryDirectory(prefix="pulse-image-blobs-") as directory:+ blobs = {}+ with tarfile.open(fileobj=stream, mode="r|*") as outer:+ for index, entry in enumerate(outer):+ if not entry.isfile():+ continue+ if entry.name in blobs:+ raise ValueError("duplicate Docker-save blob")+ target = Path(directory)/str(index)+ with target.open("wb") as destination:+ source = outer.extractfile(entry)+ for chunk in iter(lambda: source.read(1024 * 1024), b""):+ destination.write(chunk)+ blobs[entry.name] = target+ manifest_file = blobs.get("manifest.json")+ if manifest_file is None or manifest_file.stat().st_size > 1024 * 1024:+ raise ValueError("missing or oversized Docker-save manifest")+ manifest = json.loads(manifest_file.read_bytes())+ if (not isinstance(manifest, list) or len(manifest) != 1 or+ not isinstance(manifest[0], dict) or+ not isinstance(manifest[0].get("Layers"), list) or not manifest[0]["Layers"]):+ raise ValueError("expected exactly one image with filesystem layers")+ names = manifest[0]["Layers"]+ if not all(isinstance(n, str) for n in names) or len(set(names)) != len(names):+ raise ValueError("invalid or duplicate Docker-save layer identity")+ for name in names:+ if name not in blobs:+ raise ValueError("manifest layer blob is missing")+ path = blobs[name]compressor = zlib.compressobj(level=6, wbits=31)- compressed = 0- contents = outer.extractfile(entry)- # Stream once into an owned temporary file; never expand its paths.- with tempfile.TemporaryFile() as raw:- for chunk in iter(lambda: contents.read(1024 * 1024), b""):+ compressed = uncompressed = 0+ with path.open("rb") as raw:+ magic = raw.read(2)+ raw.seek(0)+ decoded = gzip.GzipFile(fileobj=raw) if magic == b"\x1f\x8b" else raw+ for chunk in iter(lambda: decoded.read(1024 * 1024), b""):+ uncompressed += len(chunk)compressed += len(compressor.compress(chunk))- raw.write(chunk)compressed += len(compressor.flush())- raw.seek(0)+ with path.open("rb") as raw:files = inventory(raw)- result.append(dict(uncompressed_tar_bytes=entry.size,+ result.append(dict(archive_blob_bytes=path.stat().st_size,+ uncompressed_tar_bytes=uncompressed,reencoded_gzip_bytes=compressed, files=files))- if not result:- raise ValueError("Docker save returned no filesystem layers")return resultdiff --git a/scripts/tests/test_container_packaging.py b/scripts/tests/test_container_packaging.pyindex 68d209ec7d..2b206cf047 100644--- a/scripts/tests/test_container_packaging.py+++ b/scripts/tests/test_container_packaging.py@@ -2,6 +2,7 @@"""Packaging's normal smoke-suite controls; no Docker daemon is used here."""import base64import contextlib+import gzipimport hashlibimport importlib.utilimport io@@ -55,7 +56,8 @@ class InventoryTests(unittest.TestCase):def test_layer_inventory_counts_repeated_payload_and_explicit_reencoding(self):layer = archive([('app/pulse', b'server', 'file'), ('opt/pulse/bin/agent', b'agent', 'file')])- image = archive([('one/layer.tar', layer, 'file'), ('two/layer.tar', layer, 'file')])+ image = archive([('one/layer.tar', layer, 'file'), ('two/layer.tar', layer, 'file'),+ ('manifest.json', json.dumps([{'Layers':['one/layer.tar','two/layer.tar']}]).encode(), 'file')])measured = packaging.layers(io.BytesIO(image))self.assertEqual(len(measured), 2)self.assertEqual(sum(l['uncompressed_tar_bytes'] for l in measured), len(layer)*2)@@ -65,11 +67,29 @@ class InventoryTests(unittest.TestCase):packaging.no_payload_copyups(measured)def test_absent_layer_or_payload_cannot_pass(self):- with self.assertRaisesRegex(ValueError, 'no filesystem layers'):+ with self.assertRaisesRegex(ValueError, 'manifest'):packaging.layers(io.BytesIO(archive([])))with self.assertRaisesRegex(ValueError, 'payload missing'):packaging.no_payload_copyups([])+ def test_containerd_blob_paths_and_gzip_measure_uncompressed_bytes(self):+ layer = archive([('app/pulse', b'server', 'file'), ('opt/pulse/bin/agent', b'agent', 'file')])+ blob = gzip.compress(layer)+ path = 'blobs/sha256/'+hashlib.sha256(blob).hexdigest()+ image = archive([(path, blob, 'file'), ('unreferenced/layer.tar', layer, 'file'),+ ('manifest.json', json.dumps([{'Layers':[path]}]).encode(), 'file')])+ measured = packaging.layers(io.BytesIO(image))+ self.assertEqual(len(measured), 1)+ self.assertEqual(measured[0]['archive_blob_bytes'], len(blob))+ self.assertEqual(measured[0]['uncompressed_tar_bytes'], len(layer))+ packaging.no_payload_copyups(measured)++ def test_manifest_identity_missing_duplicate_and_multiimage_fail_closed(self):+ for manifest in [[{'Layers':['missing']}], [{'Layers':['same','same']}],+ [{'Layers':[]}], [{'Layers':['a']},{'Layers':['b']}], {}]:+ with self.subTest(manifest=manifest), self.assertRaises(ValueError):+ packaging.layers(io.BytesIO(archive([('manifest.json', json.dumps(manifest).encode(), 'file')])))+def test_every_compatibility_dimension_is_checked(self):baseline = packaging.inventory(io.BytesIO(archive([('app/pulse', b'server', 'file')])))for key, value in [('sha256', 'different'), ('mode', 0o644), ('uid', 0),