Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Read Docker packaging layers from their archive manifest

pulse-triage[bot]2312172e9a32
diff --git a/scripts/test-container-packaging.py b/scripts/test-container-packaging.py
index 8a55f886b5..789d8c8cef 100644
--- a/scripts/test-container-packaging.py
+++ b/scripts/test-container-packaging.py
@@ -5,6 +5,7 @@ No publication, registry credentials or release qualification. The immutable
published fixture is deliberately independent of the proposed application code.
"""
import argparse
+import gzip
import hashlib
import json
import subprocess
@@ -61,25 +62,53 @@ def layers(stream):
or a multi-platform index size. Neither number includes writable volumes.
"""
result = []
- with tarfile.open(fileobj=stream, mode="r|*") as outer:
- for entry in outer:
- if not entry.name.endswith("/layer.tar"):
- continue
+ # Classic Docker archives use <id>/layer.tar; containerd-backed Docker
+ # uses blobs/sha256/<digest>. Both name the retained layers in manifest.json.
+ # Spool opaque blobs under generated names, never archive-selected paths.
+ with tempfile.TemporaryDirectory(prefix="pulse-image-blobs-") as directory:
+ blobs = {}
+ with tarfile.open(fileobj=stream, mode="r|*") as outer:
+ for index, entry in enumerate(outer):
+ if not entry.isfile():
+ continue
+ if entry.name in blobs:
+ raise ValueError("duplicate Docker-save blob")
+ target = Path(directory)/str(index)
+ with target.open("wb") as destination:
+ source = outer.extractfile(entry)
+ for chunk in iter(lambda: source.read(1024 * 1024), b""):
+ destination.write(chunk)
+ blobs[entry.name] = target
+ manifest_file = blobs.get("manifest.json")
+ if manifest_file is None or manifest_file.stat().st_size > 1024 * 1024:
+ raise ValueError("missing or oversized Docker-save manifest")
+ manifest = json.loads(manifest_file.read_bytes())
+ if (not isinstance(manifest, list) or len(manifest) != 1 or
+ not isinstance(manifest[0], dict) or
+ not isinstance(manifest[0].get("Layers"), list) or not manifest[0]["Layers"]):
+ raise ValueError("expected exactly one image with filesystem layers")
+ names = manifest[0]["Layers"]
+ if not all(isinstance(n, str) for n in names) or len(set(names)) != len(names):
+ raise ValueError("invalid or duplicate Docker-save layer identity")
+ for name in names:
+ if name not in blobs:
+ raise ValueError("manifest layer blob is missing")
+ path = blobs[name]
compressor = zlib.compressobj(level=6, wbits=31)
- compressed = 0
- contents = outer.extractfile(entry)
- # Stream once into an owned temporary file; never expand its paths.
- with tempfile.TemporaryFile() as raw:
- for chunk in iter(lambda: contents.read(1024 * 1024), b""):
+ compressed = uncompressed = 0
+ with path.open("rb") as raw:
+ magic = raw.read(2)
+ raw.seek(0)
+ decoded = gzip.GzipFile(fileobj=raw) if magic == b"\x1f\x8b" else raw
+ for chunk in iter(lambda: decoded.read(1024 * 1024), b""):
+ uncompressed += len(chunk)
compressed += len(compressor.compress(chunk))
- raw.write(chunk)
compressed += len(compressor.flush())
- raw.seek(0)
+ with path.open("rb") as raw:
files = inventory(raw)
- result.append(dict(uncompressed_tar_bytes=entry.size,
+ result.append(dict(archive_blob_bytes=path.stat().st_size,
+ uncompressed_tar_bytes=uncompressed,
reencoded_gzip_bytes=compressed, files=files))
- if not result:
- raise ValueError("Docker save returned no filesystem layers")
return result
diff --git a/scripts/tests/test_container_packaging.py b/scripts/tests/test_container_packaging.py
index 68d209ec7d..2b206cf047 100644
--- a/scripts/tests/test_container_packaging.py
+++ b/scripts/tests/test_container_packaging.py
@@ -2,6 +2,7 @@
"""Packaging's normal smoke-suite controls; no Docker daemon is used here."""
import base64
import contextlib
+import gzip
import hashlib
import importlib.util
import io
@@ -55,7 +56,8 @@ class InventoryTests(unittest.TestCase):
def test_layer_inventory_counts_repeated_payload_and_explicit_reencoding(self):
layer = archive([('app/pulse', b'server', 'file'), ('opt/pulse/bin/agent', b'agent', 'file')])
- image = archive([('one/layer.tar', layer, 'file'), ('two/layer.tar', layer, 'file')])
+ image = archive([('one/layer.tar', layer, 'file'), ('two/layer.tar', layer, 'file'),
+ ('manifest.json', json.dumps([{'Layers':['one/layer.tar','two/layer.tar']}]).encode(), 'file')])
measured = packaging.layers(io.BytesIO(image))
self.assertEqual(len(measured), 2)
self.assertEqual(sum(l['uncompressed_tar_bytes'] for l in measured), len(layer)*2)
@@ -65,11 +67,29 @@ class InventoryTests(unittest.TestCase):
packaging.no_payload_copyups(measured)
def test_absent_layer_or_payload_cannot_pass(self):
- with self.assertRaisesRegex(ValueError, 'no filesystem layers'):
+ with self.assertRaisesRegex(ValueError, 'manifest'):
packaging.layers(io.BytesIO(archive([])))
with self.assertRaisesRegex(ValueError, 'payload missing'):
packaging.no_payload_copyups([])
+ def test_containerd_blob_paths_and_gzip_measure_uncompressed_bytes(self):
+ layer = archive([('app/pulse', b'server', 'file'), ('opt/pulse/bin/agent', b'agent', 'file')])
+ blob = gzip.compress(layer)
+ path = 'blobs/sha256/'+hashlib.sha256(blob).hexdigest()
+ image = archive([(path, blob, 'file'), ('unreferenced/layer.tar', layer, 'file'),
+ ('manifest.json', json.dumps([{'Layers':[path]}]).encode(), 'file')])
+ measured = packaging.layers(io.BytesIO(image))
+ self.assertEqual(len(measured), 1)
+ self.assertEqual(measured[0]['archive_blob_bytes'], len(blob))
+ self.assertEqual(measured[0]['uncompressed_tar_bytes'], len(layer))
+ packaging.no_payload_copyups(measured)
+
+ def test_manifest_identity_missing_duplicate_and_multiimage_fail_closed(self):
+ for manifest in [[{'Layers':['missing']}], [{'Layers':['same','same']}],
+ [{'Layers':[]}], [{'Layers':['a']},{'Layers':['b']}], {}]:
+ with self.subTest(manifest=manifest), self.assertRaises(ValueError):
+ packaging.layers(io.BytesIO(archive([('manifest.json', json.dumps(manifest).encode(), 'file')])))
+
def test_every_compatibility_dimension_is_checked(self):
baseline = packaging.inventory(io.BytesIO(archive([('app/pulse', b'server', 'file')])))
for key, value in [('sha256', 'different'), ('mode', 0o644), ('uid', 0),