Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(lume): pull parameterized OCI tar disk parts
injaneity9c97c20ad890
diff --git a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swiftindex 8cae1cd20c..ead0957945 100644--- a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift+++ b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift@@ -91,7 +91,13 @@ func gunzipChunkAndWriteSparse(throw PullError.layerDownloadFailed(inputPath.lastPathComponent)}- let readHandle = try FileHandle(forReadingFrom: decompressedPath)+ return try writeFileSparse(decompressedPath, to: outputHandle, startOffsets: startOffsets)+}++func writeFileSparse(+ _ url: URL, to outputHandle: FileHandle, startOffsets: [UInt64]+) throws -> UInt64 {+ let readHandle = try FileHandle(forReadingFrom: url)defer { try? readHandle.close() }let blockSize = sparseHoleGranularityBytes@@ -277,12 +283,45 @@ enum OCIMediaType {static let config = "application/vnd.trycua.lume.config.v1+json"static let disk = "application/vnd.trycua.lume.disk.v1"static let nvram = "application/vnd.trycua.lume.nvram.v1"+ static let standardTarLayer = "application/vnd.oci.image.layer.v1.tar"static func detect(_ manifest: Manifest) -> ImageFormat {let hasOCIDisk = manifest.layers.contains { $0.mediaType == disk }let hasOCIConfig = manifest.config?.mediaType == configreturn (hasOCIDisk || hasOCIConfig) ? .oci : .legacy}++ static func rawDiskPart(_ layer: Layer) -> (number: Int, total: Int)? {+ let fields = layer.mediaType.lowercased().split(separator: ";").map {+ $0.trimmingCharacters(in: .whitespaces)+ }+ guard fields.first == standardTarLayer,+ layer.annotations?["org.opencontainers.image.title"]?.hasPrefix("disk.img.part.")+ == true+ else {+ return nil+ }+ var parameters: [String: Int] = [:]+ for field in fields.dropFirst() {+ let pair = field.split(separator: "=", maxSplits: 1, omittingEmptySubsequences: false)+ .map { $0.trimmingCharacters(in: .whitespaces) }+ guard pair.count == 2, !pair[0].isEmpty else { return (0, 0) }+ parameters[pair[0]] = Int(pair[1])+ }+ return (parameters["part.number"] ?? 0, parameters["part.total"] ?? 0)+ }++ static func validateRawDiskParts(_ manifest: Manifest) throws {+ let parts = manifest.layers.compactMap(rawDiskPart)+ guard !parts.isEmpty else { return }+ guard parts.map({ $0.number }).sorted() == Array(1...parts.count),+ parts.allSatisfy({ $0.total == parts.count }),+ !manifest.layers.contains(where: { $0.mediaType == "application/octet-stream+lz4" })+ else {+ throw PullError.reassemblyFailed(+ "raw OCI disk parts must be numbered 1...part.total without LZ4 parts")+ }+ }}/// Annotation keys for multi-layer chunked disk images.@@ -376,9 +415,9 @@ actor DiskPartsCollector {private var partCounter = 0// Adds a part and returns its assigned sequential number- func addPart(url: URL) -> Int {- partCounter += 1 // Use counter logic- let partNum = partCounter+ func addPart(url: URL, number: Int? = nil) -> Int {+ let partNum = number ?? partCounter + 1+ partCounter = max(partCounter, partNum)diskParts.append((partNum, url)) // Store sequential numberreturn partNum // Return assigned sequential number}@@ -1229,6 +1268,8 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {Logger.info("Cache miss or invalid cache, setting up new cache")}+ try OCIMediaType.validateRawDiskParts(manifest)+// Clean up old versions of this repository before setting up new cache if caching is enabledif cachingEnabled {try cleanupOldVersions(currentManifestId: manifestId, image: imageName)@@ -1300,7 +1341,9 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {}// Identify disk parts by media type- if layer.mediaType == "application/octet-stream+lz4" {+ if layer.mediaType == "application/octet-stream+lz4"+ || OCIMediaType.rawDiskPart(layer) != nil+ {// --- Handle LZ4 Disk Part Layer ---lz4LayerCount += 1 // Increment countlet currentPartNum = lz4LayerCount // Use the current count as the logical number for logging@@ -1995,6 +2038,7 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {// If we don't have a reassembled image, proceed with legacy part handlingLogger.debug("No reassembled image found, using part-based reassembly")+ try OCIMediaType.validateRawDiskParts(manifest)// Instantiate collectorlet diskPartsCollector = DiskPartsCollector()var lz4LayerCount = 0 // Count lz4 layers found@@ -2005,19 +2049,21 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {for layer in manifest.layers {let cachedLayer = getCachedLayerPath(manifestId: manifestId, digest: layer.digest)+ let rawPart = OCIMediaType.rawDiskPart(layer)// Identify disk parts simply by media type- if layer.mediaType == "application/octet-stream+lz4" {+ if layer.mediaType == "application/octet-stream+lz4" || rawPart != nil {lz4LayerCount += 1 // Increment count// When caching is disabled, the file might not exist with the cache path name// Check if the file exists before trying to use it- if !FileManager.default.fileExists(atPath: cachedLayer.path) {+ if rawPart == nil && !FileManager.default.fileExists(atPath: cachedLayer.path) {Logger.info("Layer file not found in cache: \(cachedLayer.path) - skipping")continue}// Add to collector. It will assign the sequential part number.- let collectorPartNum = await diskPartsCollector.addPart(url: cachedLayer)+ let collectorPartNum = await diskPartsCollector.addPart(+ url: cachedLayer, number: rawPart?.number)Logger.info("Adding cached lz4 layer (part \(lz4LayerCount)) -> Collector #\(collectorPartNum): \(cachedLayer.lastPathComponent)")@@ -2061,6 +2107,7 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {// --- Safely retrieve parts AFTER loop ---let diskPartSources = await diskPartsCollector.getSortedParts() // Sorted by assigned sequential numberlet totalParts = await diskPartsCollector.getTotalParts() // Get total count from collector+ let rawParts = manifest.layers.contains { OCIMediaType.rawDiskPart($0) != nil }Logger.info("Found \(totalParts) lz4 disk parts in cache to reassemble.")// --- End retrieving parts ---@@ -2121,7 +2168,7 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {}// If we have just one disk part, use the shared function- if totalParts == 1 {+ if totalParts == 1 && !rawParts {// Single part - use shared functionlet sourceURL = diskPartSources[0].1 // Get the first source URL (index 1 of the tuple)try createDiskImageFromSource(@@ -2186,11 +2233,15 @@ class ImageContainerRegistry: ImageRegistry, @unchecked Sendable {)// Always use the correct sparse decompression function- let decompressedBytesWritten = try decompressChunkAndWriteSparse(- inputPath: sourceURL.path,- outputHandle: outputHandle,- startOffset: currentOffset- )+ let decompressedBytesWritten =+ try rawParts+ ? writeFileSparse(+ sourceURL, to: outputHandle, startOffsets: [currentOffset])+ : decompressChunkAndWriteSparse(+ inputPath: sourceURL.path,+ outputHandle: outputHandle,+ startOffset: currentOffset+ )currentOffset += decompressedBytesWritten// Update progress (using sizeForTruncate which should be available)reassemblyProgressLogger.logProgress(diff --git a/libs/lume/tests/ImageContainerRegistryTests.swift b/libs/lume/tests/ImageContainerRegistryTests.swiftnew file mode 100644index 0000000000..d67f922bfc--- /dev/null+++ b/libs/lume/tests/ImageContainerRegistryTests.swift@@ -0,0 +1,84 @@+import Foundation+import Testing++@testable import lume++private func tarLayer(_ parameters: String, title: String? = "disk.img.part.ab") -> Layer {+ Layer(+ mediaType: "application/vnd.oci.image.layer.v1.tar\(parameters)",+ digest: "sha256:\(UUID().uuidString)", size: 1,+ annotations: title.map { ["org.opencontainers.image.title": $0] })+}++private func rawPart(_ number: String, of total: String) -> Layer {+ tarLayer(";part.number=\(number);part.total=\(total)")+}++private func manifest(_ layers: [Layer]) -> Manifest {+ Manifest(+ layers: layers, config: nil, mediaType: "application/vnd.oci.image.manifest.v1+json",+ schemaVersion: 2, annotations: nil)+}++@Test("parameterized OCI tar disk layers are recognized as raw disk parts",+ arguments: [+ "application/vnd.oci.image.layer.v1.tar;part.number=2;part.total=41",+ "Application/Vnd.Oci.Image.Layer.V1.Tar; Part.Number = 2 ; PART.TOTAL = 41",+ ])+func recognizesRawDiskPart(mediaType: String) {+ let layer = Layer(+ mediaType: mediaType, digest: "sha256:part2", size: 512,+ annotations: ["org.opencontainers.image.title": "disk.img.part.ab"])+ let part = OCIMediaType.rawDiskPart(layer)+ #expect(part?.number == 2)+ #expect(part?.total == 41)+}++@Test("layers without a disk part title or tar base type are not raw disk parts",+ arguments: [+ tarLayer(";part.number=1;part.total=1", title: "metadata.tar"),+ tarLayer(";part.number=1;part.total=1", title: nil),+ tarLayer("+gzip;part.number=1;part.total=1"),+ ])+func ignoresNonDiskLayers(layer: Layer) {+ #expect(OCIMediaType.rawDiskPart(layer)?.number == nil)+}++@Test("complete raw disk part sets and manifests without raw parts are accepted")+func acceptsValidRawDiskParts() throws {+ try OCIMediaType.validateRawDiskParts(manifest([rawPart("2", of: "2"), rawPart("1", of: "2")]))+ try OCIMediaType.validateRawDiskParts(manifest([+ Layer(mediaType: "application/octet-stream+lz4", digest: "sha256:lz4", size: 1, annotations: nil)+ ]))+}++@Test("invalid raw disk part sets are rejected as reassembly failures",+ arguments: [+ [rawPart("1", of: "2"), rawPart("2", of: "3")],+ [rawPart("1", of: "2"), rawPart("1", of: "2")],+ [rawPart("1", of: "3"), rawPart("2", of: "3")],+ [rawPart("2", of: "2"), rawPart("3", of: "2")],+ [rawPart("2", of: "1")],+ [rawPart("0", of: "1")],+ [rawPart("-1", of: "1")],+ [rawPart("x", of: "1")],+ [rawPart("1", of: "99999999999999999999")],+ [rawPart("0 1", of: "1")],+ [rawPart("1", of: "0 1")],+ [tarLayer(";=;part.number=1;part.total=1")],+ [tarLayer(";part.total=1")],+ [tarLayer("")],+ [+ rawPart("1", of: "1"),+ Layer(mediaType: "application/octet-stream+lz4", digest: "sha256:lz4", size: 1, annotations: nil),+ ],+ ])+func rejectsInvalidRawDiskParts(layers: [Layer]) {+ do {+ try OCIMediaType.validateRawDiskParts(manifest(layers))+ Issue.record("expected reassemblyFailed")+ } catch PullError.reassemblyFailed {+ } catch {+ Issue.record("expected reassemblyFailed, got \(error)")+ }+}diff --git a/libs/lume/tests/LegacyDiskAssemblyTests.swift b/libs/lume/tests/LegacyDiskAssemblyTests.swiftindex 9749aa3c7d..d6fe880269 100644--- a/libs/lume/tests/LegacyDiskAssemblyTests.swift+++ b/libs/lume/tests/LegacyDiskAssemblyTests.swift@@ -7,6 +7,51 @@ struct LegacyDiskAssemblyTests {@Test("legacy LZ4 assembly preserves disk bytes and cached output",arguments: [false, true], [false, true])func preservesBytes(cachingEnabled: Bool, trailingZeroChunk: Bool) async throws {+ let first = Data((0..<64).map { UInt8($0 + 1) })+ let last = trailingZeroChunk+ ? Data(count: 4 * 1024 * 1024)+ : Data((0..<73).map { UInt8(255 - $0) })+ let parts = try [first, last].enumerated().map { index, bytes in+ let compressed = try (bytes as NSData).compressed(using: .lz4) as Data+ let layer = Layer(+ mediaType: "application/octet-stream+lz4", digest: "sha256:part-\(index)",+ size: compressed.count, annotations: nil)+ return (layer, compressed)+ }+ try await assertAssembles(+ parts: parts, expected: first + last, cachingEnabled: cachingEnabled)+ }++ @Test("raw OCI tar parts assemble in part.number order",+ arguments: [false, true], ["single", "multi", "trailingZero"])+ func assemblesRawTarParts(cachingEnabled: Bool, layout: String) async throws {+ let first = Data((0..<64).map { UInt8($0 + 1) })+ let middle = Data((0..<73).map { UInt8(255 - $0) })+ let payloads = [+ "single": [first],+ "multi": [first, middle, Data(repeating: 7, count: 5)],+ "trailingZero": [first, middle, Data(count: 4 * 1024 * 1024)],+ ][layout]!+ let titleSuffixes = ["zz", "yy", "xx"]+ let parts = payloads.enumerated().reversed().map { index, bytes in+ let layer = Layer(+ mediaType: "application/vnd.oci.image.layer.v1.tar;part.number=\(index + 1);part.total=\(payloads.count)",+ digest: "sha256:raw-\(index)", size: bytes.count,+ annotations: ["org.opencontainers.image.title": "disk.img.part.\(titleSuffixes[index])"])+ return (layer, bytes)+ }+ let emptyConfig = Layer(+ mediaType: "application/vnd.oci.empty.v1+json", digest: "sha256:empty",+ size: 2, annotations: nil)+ try await assertAssembles(+ parts: parts, expected: payloads.reduce(Data(), +),+ cachingEnabled: cachingEnabled, configDescriptor: emptyConfig)+ }++ private func assertAssembles(+ parts: [(Layer, Data)], expected: Data, cachingEnabled: Bool,+ configDescriptor: Layer? = nil+ ) async throws {let root = FileManager.default.temporaryDirectory.appendingPathComponent("lume-legacy-assembly-\(UUID().uuidString)")try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)@@ -16,22 +61,13 @@ struct LegacyDiskAssemblyTests {let registry = ImageContainerRegistry(registry: "unused.invalid", organization: "fixture",cacheDirectory: cache, cachingEnabled: cachingEnabled)- let manifestId = "legacy-lz4-fixture"- let first = Data((0..<64).map { UInt8($0 + 1) })- let last = trailingZeroChunk- ? Data(count: 4 * 1024 * 1024)- : Data((0..<73).map { UInt8(255 - $0) })- let expected = first + last+ let manifestId = "legacy-disk-fixture"var layers: [Layer] = []- for (index, bytes) in [first, last].enumerated() {- let compressed = try (bytes as NSData).compressed(using: .lz4) as Data- let layer = Layer(- mediaType: "application/octet-stream+lz4", digest: "sha256:part-\(index)",- size: compressed.count, annotations: nil)+ for (layer, bytes) in parts {let path = registry.getCachedLayerPath(manifestId: manifestId, digest: layer.digest)try FileManager.default.createDirectory(at: path.deletingLastPathComponent(), withIntermediateDirectories: true)- try compressed.write(to: path)+ try bytes.write(to: path)layers.append(layer)}let config = try JSONSerialization.data(withJSONObject: [@@ -44,9 +80,10 @@ struct LegacyDiskAssemblyTests {manifestId: manifestId, digest: configLayer.digest))layers.append(configLayer)let manifest = Manifest(- layers: layers, config: configLayer,+ layers: layers, config: configDescriptor ?? configLayer,mediaType: "application/vnd.oci.image.manifest.v1+json",schemaVersion: 2, annotations: nil)+ #expect(OCIMediaType.detect(manifest) == .legacy)let cold = root.appendingPathComponent("cold")try FileManager.default.createDirectory(at: cold, withIntermediateDirectories: true)@@ -59,7 +96,7 @@ struct LegacyDiskAssemblyTests {if cachingEnabled {try assertDisk(assembled, equals: expected)// The warm path must serve the assembled artifact, not re-read the parts.- for layer in layers where layer.mediaType == "application/octet-stream+lz4" {+ for (layer, _) in parts {#expect(!FileManager.default.fileExists(atPath: registry.getCachedLayerPath(manifestId: manifestId, digest: layer.digest).path))}