Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(lume): pull parameterized OCI tar disk parts

injaneity9c97c20ad890
diff --git a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift
index 8cae1cd20c..ead0957945 100644
--- a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift
+++ b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift
@@ -91,7 +91,13 @@ func gunzipChunkAndWriteSparse(
throw PullError.layerDownloadFailed(inputPath.lastPathComponent)
}
- let readHandle = try FileHandle(forReadingFrom: decompressedPath)
+ return try writeFileSparse(decompressedPath, to: outputHandle, startOffsets: startOffsets)
+}
+
+func writeFileSparse(
+ _ url: URL, to outputHandle: FileHandle, startOffsets: [UInt64]
+) throws -> UInt64 {
+ let readHandle = try FileHandle(forReadingFrom: url)
defer { try? readHandle.close() }
let blockSize = sparseHoleGranularityBytes
@@ -277,12 +283,45 @@ enum OCIMediaType {
static let config = "application/vnd.trycua.lume.config.v1+json"
static let disk = "application/vnd.trycua.lume.disk.v1"
static let nvram = "application/vnd.trycua.lume.nvram.v1"
+ static let standardTarLayer = "application/vnd.oci.image.layer.v1.tar"
static func detect(_ manifest: Manifest) -> ImageFormat {
let hasOCIDisk = manifest.layers.contains { $0.mediaType == disk }
let hasOCIConfig = manifest.config?.mediaType == config
return (hasOCIDisk || hasOCIConfig) ? .oci : .legacy
}
+
+ static func rawDiskPart(_ layer: Layer) -> (number: Int, total: Int)? {
+ let fields = layer.mediaType.lowercased().split(separator: ";").map {
+ $0.trimmingCharacters(in: .whitespaces)
+ }
+ guard fields.first == standardTarLayer,
+ layer.annotations?["org.opencontainers.image.title"]?.hasPrefix("disk.img.part.")
+ == true
+ else {
+ return nil
+ }
+ var parameters: [String: Int] = [:]
+ for field in fields.dropFirst() {
+ let pair = field.split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false)
+ .map { $0.trimmingCharacters(in: .whitespaces) }
+ guard pair.count == 2, !pair[0].isEmpty else { return (0, 0) }
+ parameters[pair[0]] = Int(pair[1])
+ }
+ return (parameters["part.number"] ?? 0, parameters["part.total"] ?? 0)
+ }
+
+ static func validateRawDiskParts(_ manifest: Manifest) throws {
+ let parts = manifest.layers.compactMap(rawDiskPart)
+ guard !parts.isEmpty else { return }
+ guard parts.map({ $0.number }).sorted() == Array(1...parts.count),
+ parts.allSatisfy({ $0.total == parts.count }),
+ !manifest.layers.contains(where: { $0.mediaType == "application/octet-stream+lz4" })
+ else {
+ throw PullError.reassemblyFailed(
+ "raw OCI disk parts must be numbered 1...part.total without LZ4 parts")
+ }
+ }
}
/// Annotation keys for multi-layer chunked disk images.
@@ -376,9 +415,9 @@ actor DiskPartsCollector {
private var partCounter = 0
// Adds a part and returns its assigned sequential number
- func addPart(url: URL) -> Int {
- partCounter += 1 // Use counter logic
- let partNum = partCounter
+ func addPart(url: URL, number: Int? = nil) -> Int {
+ let partNum = number ?? partCounter + 1
+ partCounter = max(partCounter, partNum)
diskParts.append((partNum, url)) // Store sequential number
return partNum // Return assigned sequential number
}
@@ -1229,6 +1268,8 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {
Logger.info("Cache miss or invalid cache, setting up new cache")
}
+ try OCIMediaType.validateRawDiskParts(manifest)
+
// Clean up old versions of this repository before setting up new cache if caching is enabled
if cachingEnabled {
try cleanupOldVersions(currentManifestId: manifestId, image: imageName)
@@ -1300,7 +1341,9 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {
}
// Identify disk parts by media type
- if layer.mediaType == "application/octet-stream+lz4" {
+ if layer.mediaType == "application/octet-stream+lz4"
+ || OCIMediaType.rawDiskPart(layer) != nil
+ {
// --- Handle LZ4 Disk Part Layer ---
lz4LayerCount += 1 // Increment count
let currentPartNum = lz4LayerCount // Use the current count as the logical number for logging
@@ -1995,6 +2038,7 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {
// If we don't have a reassembled image, proceed with legacy part handling
Logger.debug("No reassembled image found, using part-based reassembly")
+ try OCIMediaType.validateRawDiskParts(manifest)
// Instantiate collector
let diskPartsCollector = DiskPartsCollector()
var lz4LayerCount = 0 // Count lz4 layers found
@@ -2005,19 +2049,21 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {
for layer in manifest.layers {
let cachedLayer = getCachedLayerPath(manifestId: manifestId, digest: layer.digest)
+ let rawPart = OCIMediaType.rawDiskPart(layer)
// Identify disk parts simply by media type
- if layer.mediaType == "application/octet-stream+lz4" {
+ if layer.mediaType == "application/octet-stream+lz4" || rawPart != nil {
lz4LayerCount += 1 // Increment count
// When caching is disabled, the file might not exist with the cache path name
// Check if the file exists before trying to use it
- if !FileManager.default.fileExists(atPath: cachedLayer.path) {
+ if rawPart == nil && !FileManager.default.fileExists(atPath: cachedLayer.path) {
Logger.info("Layer file not found in cache: \(cachedLayer.path) - skipping")
continue
}
// Add to collector. It will assign the sequential part number.
- let collectorPartNum = await diskPartsCollector.addPart(url: cachedLayer)
+ let collectorPartNum = await diskPartsCollector.addPart(
+ url: cachedLayer, number: rawPart?.number)
Logger.info(
"Adding cached lz4 layer (part \(lz4LayerCount)) -> Collector #\(collectorPartNum): \(cachedLayer.lastPathComponent)"
)
@@ -2061,6 +2107,7 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {
// --- Safely retrieve parts AFTER loop ---
let diskPartSources = await diskPartsCollector.getSortedParts() // Sorted by assigned sequential number
let totalParts = await diskPartsCollector.getTotalParts() // Get total count from collector
+ let rawParts = manifest.layers.contains { OCIMediaType.rawDiskPart($0) != nil }
Logger.info("Found \(totalParts) lz4 disk parts in cache to reassemble.")
// --- End retrieving parts ---
@@ -2121,7 +2168,7 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {
}
// If we have just one disk part, use the shared function
- if totalParts == 1 {
+ if totalParts == 1 && !rawParts {
// Single part - use shared function
let sourceURL = diskPartSources[0].1 // Get the first source URL (index 1 of the tuple)
try createDiskImageFromSource(
@@ -2186,11 +2233,15 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {
)
// Always use the correct sparse decompression function
- let decompressedBytesWritten = try decompressChunkAndWriteSparse(
- inputPath: sourceURL.path,
- outputHandle: outputHandle,
- startOffset: currentOffset
- )
+ let decompressedBytesWritten =
+ try rawParts
+ ? writeFileSparse(
+ sourceURL, to: outputHandle, startOffsets: [currentOffset])
+ : decompressChunkAndWriteSparse(
+ inputPath: sourceURL.path,
+ outputHandle: outputHandle,
+ startOffset: currentOffset
+ )
currentOffset += decompressedBytesWritten
// Update progress (using sizeForTruncate which should be available)
reassemblyProgressLogger.logProgress(
diff --git a/libs/lume/tests/ImageContainerRegistryTests.swift b/libs/lume/tests/ImageContainerRegistryTests.swift
new file mode 100644
index 0000000000..d67f922bfc
--- /dev/null
+++ b/libs/lume/tests/ImageContainerRegistryTests.swift
@@ -0,0 +1,84 @@
+import Foundation
+import Testing
+
+@testable import lume
+
+private func tarLayer(_ parameters: String, title: String? = "disk.img.part.ab") -> Layer {
+ Layer(
+ mediaType: "application/vnd.oci.image.layer.v1.tar\(parameters)",
+ digest: "sha256:\(UUID().uuidString)", size: 1,
+ annotations: title.map { ["org.opencontainers.image.title": $0] })
+}
+
+private func rawPart(_ number: String, of total: String) -> Layer {
+ tarLayer(";part.number=\(number);part.total=\(total)")
+}
+
+private func manifest(_ layers: [Layer]) -> Manifest {
+ Manifest(
+ layers: layers, config: nil, mediaType: "application/vnd.oci.image.manifest.v1+json",
+ schemaVersion: 2, annotations: nil)
+}
+
+@Test("parameterized OCI tar disk layers are recognized as raw disk parts",
+ arguments: [
+ "application/vnd.oci.image.layer.v1.tar;part.number=2;part.total=41",
+ "Application/Vnd.Oci.Image.Layer.V1.Tar; Part.Number = 2 ; PART.TOTAL = 41",
+ ])
+func recognizesRawDiskPart(mediaType: String) {
+ let layer = Layer(
+ mediaType: mediaType, digest: "sha256:part2", size: 512,
+ annotations: ["org.opencontainers.image.title": "disk.img.part.ab"])
+ let part = OCIMediaType.rawDiskPart(layer)
+ #expect(part?.number == 2)
+ #expect(part?.total == 41)
+}
+
+@Test("layers without a disk part title or tar base type are not raw disk parts",
+ arguments: [
+ tarLayer(";part.number=1;part.total=1", title: "metadata.tar"),
+ tarLayer(";part.number=1;part.total=1", title: nil),
+ tarLayer("+gzip;part.number=1;part.total=1"),
+ ])
+func ignoresNonDiskLayers(layer: Layer) {
+ #expect(OCIMediaType.rawDiskPart(layer)?.number == nil)
+}
+
+@Test("complete raw disk part sets and manifests without raw parts are accepted")
+func acceptsValidRawDiskParts() throws {
+ try OCIMediaType.validateRawDiskParts(manifest([rawPart("2", of: "2"), rawPart("1", of: "2")]))
+ try OCIMediaType.validateRawDiskParts(manifest([
+ Layer(mediaType: "application/octet-stream+lz4", digest: "sha256:lz4", size: 1, annotations: nil)
+ ]))
+}
+
+@Test("invalid raw disk part sets are rejected as reassembly failures",
+ arguments: [
+ [rawPart("1", of: "2"), rawPart("2", of: "3")],
+ [rawPart("1", of: "2"), rawPart("1", of: "2")],
+ [rawPart("1", of: "3"), rawPart("2", of: "3")],
+ [rawPart("2", of: "2"), rawPart("3", of: "2")],
+ [rawPart("2", of: "1")],
+ [rawPart("0", of: "1")],
+ [rawPart("-1", of: "1")],
+ [rawPart("x", of: "1")],
+ [rawPart("1", of: "99999999999999999999")],
+ [rawPart("0 1", of: "1")],
+ [rawPart("1", of: "0 1")],
+ [tarLayer(";=;part.number=1;part.total=1")],
+ [tarLayer(";part.total=1")],
+ [tarLayer("")],
+ [
+ rawPart("1", of: "1"),
+ Layer(mediaType: "application/octet-stream+lz4", digest: "sha256:lz4", size: 1, annotations: nil),
+ ],
+ ])
+func rejectsInvalidRawDiskParts(layers: [Layer]) {
+ do {
+ try OCIMediaType.validateRawDiskParts(manifest(layers))
+ Issue.record("expected reassemblyFailed")
+ } catch PullError.reassemblyFailed {
+ } catch {
+ Issue.record("expected reassemblyFailed, got \(error)")
+ }
+}
diff --git a/libs/lume/tests/LegacyDiskAssemblyTests.swift b/libs/lume/tests/LegacyDiskAssemblyTests.swift
index 9749aa3c7d..d6fe880269 100644
--- a/libs/lume/tests/LegacyDiskAssemblyTests.swift
+++ b/libs/lume/tests/LegacyDiskAssemblyTests.swift
@@ -7,6 +7,51 @@ struct LegacyDiskAssemblyTests {
@Test("legacy LZ4 assembly preserves disk bytes and cached output",
arguments: [false, true], [false, true])
func preservesBytes(cachingEnabled: Bool, trailingZeroChunk: Bool) async throws {
+ let first = Data((0..<64).map { UInt8($0 + 1) })
+ let last = trailingZeroChunk
+ ? Data(count: 4 * 1024 * 1024)
+ : Data((0..<73).map { UInt8(255 - $0) })
+ let parts = try [first, last].enumerated().map { index, bytes in
+ let compressed = try (bytes as NSData).compressed(using: .lz4) as Data
+ let layer = Layer(
+ mediaType: "application/octet-stream+lz4", digest: "sha256:part-\(index)",
+ size: compressed.count, annotations: nil)
+ return (layer, compressed)
+ }
+ try await assertAssembles(
+ parts: parts, expected: first + last, cachingEnabled: cachingEnabled)
+ }
+
+ @Test("raw OCI tar parts assemble in part.number order",
+ arguments: [false, true], ["single", "multi", "trailingZero"])
+ func assemblesRawTarParts(cachingEnabled: Bool, layout: String) async throws {
+ let first = Data((0..<64).map { UInt8($0 + 1) })
+ let middle = Data((0..<73).map { UInt8(255 - $0) })
+ let payloads = [
+ "single": [first],
+ "multi": [first, middle, Data(repeating: 7, count: 5)],
+ "trailingZero": [first, middle, Data(count: 4 * 1024 * 1024)],
+ ][layout]!
+ let titleSuffixes = ["zz", "yy", "xx"]
+ let parts = payloads.enumerated().reversed().map { index, bytes in
+ let layer = Layer(
+ mediaType: "application/vnd.oci.image.layer.v1.tar;part.number=\(index + 1);part.total=\(payloads.count)",
+ digest: "sha256:raw-\(index)", size: bytes.count,
+ annotations: ["org.opencontainers.image.title": "disk.img.part.\(titleSuffixes[index])"])
+ return (layer, bytes)
+ }
+ let emptyConfig = Layer(
+ mediaType: "application/vnd.oci.empty.v1+json", digest: "sha256:empty",
+ size: 2, annotations: nil)
+ try await assertAssembles(
+ parts: parts, expected: payloads.reduce(Data(), +),
+ cachingEnabled: cachingEnabled, configDescriptor: emptyConfig)
+ }
+
+ private func assertAssembles(
+ parts: [(Layer, Data)], expected: Data, cachingEnabled: Bool,
+ configDescriptor: Layer? = nil
+ ) async throws {
let root = FileManager.default.temporaryDirectory
.appendingPathComponent("lume-legacy-assembly-\(UUID().uuidString)")
try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
@@ -16,22 +61,13 @@ struct LegacyDiskAssemblyTests {
let registry = ImageContainerRegistry(
registry: "unused.invalid", organization: "fixture",
cacheDirectory: cache, cachingEnabled: cachingEnabled)
- let manifestId = "legacy-lz4-fixture"
- let first = Data((0..<64).map { UInt8($0 + 1) })
- let last = trailingZeroChunk
- ? Data(count: 4 * 1024 * 1024)
- : Data((0..<73).map { UInt8(255 - $0) })
- let expected = first + last
+ let manifestId = "legacy-disk-fixture"
var layers: [Layer] = []
- for (index, bytes) in [first, last].enumerated() {
- let compressed = try (bytes as NSData).compressed(using: .lz4) as Data
- let layer = Layer(
- mediaType: "application/octet-stream+lz4", digest: "sha256:part-\(index)",
- size: compressed.count, annotations: nil)
+ for (layer, bytes) in parts {
let path = registry.getCachedLayerPath(manifestId: manifestId, digest: layer.digest)
try FileManager.default.createDirectory(
at: path.deletingLastPathComponent(), withIntermediateDirectories: true)
- try compressed.write(to: path)
+ try bytes.write(to: path)
layers.append(layer)
}
let config = try JSONSerialization.data(withJSONObject: [
@@ -44,9 +80,10 @@ struct LegacyDiskAssemblyTests {
manifestId: manifestId, digest: configLayer.digest))
layers.append(configLayer)
let manifest = Manifest(
- layers: layers, config: configLayer,
+ layers: layers, config: configDescriptor ?? configLayer,
mediaType: "application/vnd.oci.image.manifest.v1+json",
schemaVersion: 2, annotations: nil)
+ #expect(OCIMediaType.detect(manifest) == .legacy)
let cold = root.appendingPathComponent("cold")
try FileManager.default.createDirectory(at: cold, withIntermediateDirectories: true)
@@ -59,7 +96,7 @@ struct LegacyDiskAssemblyTests {
if cachingEnabled {
try assertDisk(assembled, equals: expected)
// The warm path must serve the assembled artifact, not re-read the parts.
- for layer in layers where layer.mediaType == "application/octet-stream+lz4" {
+ for (layer, _) in parts {
#expect(!FileManager.default.fileExists(atPath: registry.getCachedLayerPath(
manifestId: manifestId, digest: layer.digest).path))
}