Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Fix first pull

f-trycua5b0ff2f21878
diff --git a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift
index 9ae6f65fc8..ddf5ea1f95 100644
--- a/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift
+++ b/libs/lume/src/ContainerRegistry/ImageContainerRegistry.swift
@@ -1,9 +1,9 @@
import ArgumentParser
+import CommonCrypto
+import Compression // Add this import
import Darwin
import Foundation
import Swift
-import CommonCrypto
-import Compression // Add this import
// Extension to calculate SHA256 hash
extension Data {
@@ -25,14 +25,14 @@ enum PushError: Error {
case authenticationFailed
case missingToken
case invalidURL
- case lz4NotFound // Added error case
- case invalidMediaType // Added during part refactoring
- case missingUncompressedSizeAnnotation // Added for sparse file handling
- case fileCreationFailed(String) // Added for sparse file handling
- case reassemblySetupFailed(path: String, underlyingError: Error?) // Added for sparse file handling
- case missingPart(Int) // Added for sparse file handling
- case layerDownloadFailed(String) // Added for download retries
- case manifestFetchFailed // Added for manifest fetching
+ case lz4NotFound // Added error case
+ case invalidMediaType // Added during part refactoring
+ case missingUncompressedSizeAnnotation // Added for sparse file handling
+ case fileCreationFailed(String) // Added for sparse file handling
+ case reassemblySetupFailed(path: String, underlyingError: Error?) // Added for sparse file handling
+ case missingPart(Int) // Added for sparse file handling
+ case layerDownloadFailed(String) // Added for download retries
+ case manifestFetchFailed // Added for manifest fetching
}
// Define a specific error type for when no underlying error exists
@@ -54,8 +54,11 @@ struct OCIManifestLayer {
let digest: String
let uncompressedSize: UInt64?
let uncompressedContentDigest: String?
-
- init(mediaType: String, size: Int, digest: String, uncompressedSize: UInt64? = nil, uncompressedContentDigest: String? = nil) {
+
+ init(
+ mediaType: String, size: Int, digest: String, uncompressedSize: UInt64? = nil,
+ uncompressedContentDigest: String? = nil
+ ) {
self.mediaType = mediaType
self.size = size
self.digest = digest
@@ -119,21 +122,21 @@ actor DiskPartsCollector {
// Store tuples of (sequentialPartNum, url)
private var diskParts: [(Int, URL)] = []
// Restore internal counter
- private var partCounter = 0
+ private var partCounter = 0
// Adds a part and returns its assigned sequential number
func addPart(url: URL) -> Int {
- partCounter += 1 // Use counter logic
- let partNum = partCounter
- diskParts.append((partNum, url)) // Store sequential number
- return partNum // Return assigned sequential number
+ partCounter += 1 // Use counter logic
+ let partNum = partCounter
+ diskParts.append((partNum, url)) // Store sequential number
+ return partNum // Return assigned sequential number
}
// Sort by the sequential part number (index 0 of tuple)
func getSortedParts() -> [(Int, URL)] {
return diskParts.sorted { $0.0 < $1.0 }
}
-
+
// Restore getTotalParts
func getTotalParts() -> Int {
return partCounter
@@ -363,7 +366,7 @@ struct DownloadStats {
// Renamed struct
struct UploadStats {
let totalBytes: Int64
- let uploadedBytes: Int64 // Renamed
+ let uploadedBytes: Int64 // Renamed
let elapsedTime: TimeInterval
let averageSpeed: Double
let peakSpeed: Double
@@ -391,9 +394,13 @@ struct UploadStats {
let hours = Int(seconds) / 3600
let minutes = (Int(seconds) % 3600) / 60
let secs = Int(seconds) % 60
- if hours > 0 { return String(format: "%d hours, %d minutes, %d seconds", hours, minutes, secs) }
- else if minutes > 0 { return String(format: "%d minutes, %d seconds", minutes, secs) }
- else { return String(format: "%d seconds", secs) }
+ if hours > 0 {
+ return String(format: "%d hours, %d minutes, %d seconds", hours, minutes, secs)
+ } else if minutes > 0 {
+ return String(format: "%d minutes, %d seconds", minutes, secs)
+ } else {
+ return String(format: "%d seconds", secs)
+ }
}
}
@@ -408,15 +415,15 @@ actor TaskCounter {
class ImageContainerRegistry: @unchecked Sendable {
private let registry: String
private let organization: String
- private let downloadProgress = ProgressTracker() // Renamed for clarity
- private let uploadProgress = UploadProgressTracker() // Added upload tracker
+ private let downloadProgress = ProgressTracker() // Renamed for clarity
+ private let uploadProgress = UploadProgressTracker() // Added upload tracker
private let cacheDirectory: URL
private let downloadLock = NSLock()
private var activeDownloads: [String] = []
private let cachingEnabled: Bool
// Constants for zero-skipping write logic
- private static let holeGranularityBytes = 4 * 1024 * 1024 // 4MB block size for checking zeros
+ private static let holeGranularityBytes = 4 * 1024 * 1024 // 4MB block size for checking zeros
private static let zeroChunk = Data(count: holeGranularityBytes)
// Add the createProgressBar function here as a private method
@@ -768,9 +775,7 @@ class ImageContainerRegistry: @unchecked Sendable {
)
let counter = TaskCounter()
- // Remove totalDiskParts
- // var totalDiskParts: Int? = nil
- var lz4LayerCount = 0 // Count lz4 layers found
+ var lz4LayerCount = 0 // Count lz4 layers found
try await withThrowingTaskGroup(of: Int64.self) { group in
for layer in manifest.layers {
@@ -785,45 +790,57 @@ class ImageContainerRegistry: @unchecked Sendable {
// Identify disk parts by media type
if layer.mediaType == "application/octet-stream+lz4" {
- // --- Handle LZ4 Disk Part Layer ---
- lz4LayerCount += 1 // Increment count
- let currentPartNum = lz4LayerCount // Use the current count as the logical number for logging
-
+ // --- Handle LZ4 Disk Part Layer ---
+ lz4LayerCount += 1 // Increment count
+ let currentPartNum = lz4LayerCount // Use the current count as the logical number for logging
+
let cachedLayer = getCachedLayerPath(
manifestId: manifestId, digest: layer.digest)
let digest = layer.digest
let size = layer.size
- if memoryConstrained && FileManager.default.fileExists(atPath: cachedLayer.path) {
+ if memoryConstrained
+ && FileManager.default.fileExists(atPath: cachedLayer.path)
+ {
// Add to collector, get sequential number assigned by collector
- let collectorPartNum = await diskPartsCollector.addPart(url: cachedLayer)
+ let collectorPartNum = await diskPartsCollector.addPart(
+ url: cachedLayer)
// Log using the sequential number from collector for clarity if needed, or the lz4LayerCount
- Logger.info("Using cached lz4 layer (part \(currentPartNum)) directly: \(cachedLayer.lastPathComponent) -> Collector #\(collectorPartNum)")
+ Logger.info(
+ "Using cached lz4 layer (part \(currentPartNum)) directly: \(cachedLayer.lastPathComponent) -> Collector #\(collectorPartNum)"
+ )
await downloadProgress.addProgress(Int64(size))
- continue
+ continue
} else {
// Download/Copy Path (Task Group)
group.addTask { [self] in
await counter.increment()
let finalPath: URL
if FileManager.default.fileExists(atPath: cachedLayer.path) {
- let tempPartURL = tempDownloadDir.appendingPathComponent("disk.img.part.\(UUID().uuidString)")
- try FileManager.default.copyItem(at: cachedLayer, to: tempPartURL)
+ let tempPartURL = tempDownloadDir.appendingPathComponent(
+ "disk.img.part.\(UUID().uuidString)")
+ try FileManager.default.copyItem(
+ at: cachedLayer, to: tempPartURL)
await downloadProgress.addProgress(Int64(size))
finalPath = tempPartURL
} else {
- let tempPartURL = tempDownloadDir.appendingPathComponent("disk.img.part.\(UUID().uuidString)")
+ let tempPartURL = tempDownloadDir.appendingPathComponent(
+ "disk.img.part.\(UUID().uuidString)")
if isDownloading(digest) {
- try await waitForExistingDownload(digest, cachedLayer: cachedLayer)
- if FileManager.default.fileExists(atPath: cachedLayer.path) {
- try FileManager.default.copyItem(at: cachedLayer, to: tempPartURL)
+ try await waitForExistingDownload(
+ digest, cachedLayer: cachedLayer)
+ if FileManager.default.fileExists(atPath: cachedLayer.path)
+ {
+ try FileManager.default.copyItem(
+ at: cachedLayer, to: tempPartURL)
await downloadProgress.addProgress(Int64(size))
finalPath = tempPartURL
} else {
markDownloadStarted(digest)
try await self.downloadLayer(
repository: "\(self.organization)/\(imageName)",
- digest: digest, mediaType: layer.mediaType, token: token,
+ digest: digest, mediaType: layer.mediaType,
+ token: token,
to: tempPartURL, maxRetries: 5,
progress: downloadProgress, manifestId: manifestId
)
@@ -833,7 +850,8 @@ class ImageContainerRegistry: @unchecked Sendable {
markDownloadStarted(digest)
try await self.downloadLayer(
repository: "\(self.organization)/\(imageName)",
- digest: digest, mediaType: layer.mediaType, token: token,
+ digest: digest, mediaType: layer.mediaType,
+ token: token,
to: tempPartURL, maxRetries: 5,
progress: downloadProgress, manifestId: manifestId
)
@@ -841,15 +859,18 @@ class ImageContainerRegistry: @unchecked Sendable {
}
}
// Add to collector, get sequential number assigned by collector
- let collectorPartNum = await diskPartsCollector.addPart(url: finalPath)
+ let collectorPartNum = await diskPartsCollector.addPart(
+ url: finalPath)
// Log using the sequential number from collector
- Logger.info("Assigned path for lz4 layer (part \(currentPartNum)): \(finalPath.lastPathComponent) -> Collector #\(collectorPartNum)")
+ Logger.info(
+ "Assigned path for lz4 layer (part \(currentPartNum)): \(finalPath.lastPathComponent) -> Collector #\(collectorPartNum)"
+ )
await counter.decrement()
return Int64(size)
}
}
} else {
- // --- Handle Non-Disk-Part Layer ---
+ // --- Handle Non-Disk-Part Layer ---
let mediaType = layer.mediaType
let digest = layer.digest
let size = layer.size
@@ -858,39 +879,42 @@ class ImageContainerRegistry: @unchecked Sendable {
let outputURL: URL
switch mediaType {
case "application/vnd.oci.image.layer.v1.tar",
- "application/octet-stream+gzip": // Might be compressed disk.img single file?
- outputURL = tempDownloadDir.appendingPathComponent("disk.img")
+ "application/octet-stream+gzip": // Might be compressed disk.img single file?
+ outputURL = tempDownloadDir.appendingPathComponent("disk.img")
case "application/vnd.oci.image.config.v1+json":
outputURL = tempDownloadDir.appendingPathComponent("config.json")
- case "application/octet-stream": // Could be nvram or uncompressed single disk.img
- // Heuristic: If a config.json already exists or is expected, assume this is nvram.
- // This might need refinement if single disk images use octet-stream.
- if manifest.config != nil {
+ case "application/octet-stream": // Could be nvram or uncompressed single disk.img
+ // Heuristic: If a config.json already exists or is expected, assume this is nvram.
+ // This might need refinement if single disk images use octet-stream.
+ if manifest.config != nil {
outputURL = tempDownloadDir.appendingPathComponent("nvram.bin")
- } else {
+ } else {
// Assume it's a single-file disk image if no config layer is present
outputURL = tempDownloadDir.appendingPathComponent("disk.img")
- }
+ }
default:
- Logger.info("Skipping unsupported layer media type: \(mediaType)")
- continue // Skip to the next layer
+ Logger.info("Skipping unsupported layer media type: \(mediaType)")
+ continue // Skip to the next layer
}
// Add task to download/copy the non-disk-part layer
group.addTask { [self] in
await counter.increment()
- let cachedLayer = getCachedLayerPath(manifestId: manifestId, digest: digest)
+ let cachedLayer = getCachedLayerPath(
+ manifestId: manifestId, digest: digest)
if FileManager.default.fileExists(atPath: cachedLayer.path) {
try FileManager.default.copyItem(at: cachedLayer, to: outputURL)
await downloadProgress.addProgress(Int64(size))
} else {
if isDownloading(digest) {
- try await waitForExistingDownload(digest, cachedLayer: cachedLayer)
+ try await waitForExistingDownload(
+ digest, cachedLayer: cachedLayer)
if FileManager.default.fileExists(atPath: cachedLayer.path) {
- try FileManager.default.copyItem(at: cachedLayer, to: outputURL)
+ try FileManager.default.copyItem(
+ at: cachedLayer, to: outputURL)
await downloadProgress.addProgress(Int64(size))
- await counter.decrement() // Decrement before returning
+ await counter.decrement() // Decrement before returning
return Int64(size)
}
}
@@ -908,304 +932,62 @@ class ImageContainerRegistry: @unchecked Sendable {
return Int64(size)
}
}
- } // End for layer in manifest.layers
+ } // End for layer in manifest.layers
// Wait for remaining tasks
for try await _ in group {}
- } // End TaskGroup
-
- // --- Safely retrieve parts AFTER TaskGroup ---
- let diskParts = await diskPartsCollector.getSortedParts() // Already sorted by logicalPartNum
- // Check if totalDiskParts was set (meaning at least one lz4 layer was processed)
- // Get total parts from the collector
- let totalPartsFromCollector = await diskPartsCollector.getTotalParts()
- // Change guard to if for logging only, as the later if condition handles the logic
- if totalPartsFromCollector == 0 {
- // If totalParts is 0, it means no layers matched the lz4 format.
- Logger.info("No lz4 disk part layers found. Assuming single-part image or non-lz4 parts.")
- // Reassembly logic below will be skipped if diskParts is empty.
- // Explicitly set totalParts to 0 to prevent entering the reassembly block if diskParts might somehow be non-empty but totalParts was 0
- // This ensures consistency if the collector logic changes.
- }
- Logger.info("Finished processing layers. Found \(diskParts.count) disk parts to reassemble (Total Lz4 Layers: \(totalPartsFromCollector)).")
- // --- End retrieving parts ---
-
- // Add detailed logging for debugging
- Logger.info("Disk part numbers collected and sorted: \(diskParts.map { $0.0 })")
-
- Logger.info("") // New line after progress
+ } // End TaskGroup
// Display download statistics
let stats = await downloadProgress.getDownloadStats()
+ Logger.info("") // New line after progress
Logger.info(stats.formattedSummary())
- // Parse config.json to get uncompressed size *before* reassembly
- let configURL = tempDownloadDir.appendingPathComponent("config.json")
- let uncompressedSize = getUncompressedSizeFromConfig(configPath: configURL)
-
- // Now also try to get disk size from VM config if OCI annotation not found
- var vmConfigDiskSize: UInt64? = nil
- if uncompressedSize == nil && FileManager.default.fileExists(atPath: configURL.path) {
- do {
- let configData = try Data(contentsOf: configURL)
- let decoder = JSONDecoder()
- if let vmConfig = try? decoder.decode(VMConfig.self, from: configData) {
- vmConfigDiskSize = vmConfig.diskSize
- if let size = vmConfigDiskSize {
- Logger.info("Found diskSize from VM config.json: \(size) bytes")
- }
- }
- } catch {
- Logger.error("Failed to parse VM config.json for diskSize: \(error)")
- }
- }
-
- // Force explicit use
- if uncompressedSize != nil {
- Logger.info(
- "Will use uncompressed size from annotation for sparse file: \(uncompressedSize!) bytes"
- )
- } else if vmConfigDiskSize != nil {
- Logger.info(
- "Will use diskSize from VM config for sparse file: \(vmConfigDiskSize!) bytes")
- }
-
- // Handle disk parts if present
- if !diskParts.isEmpty && totalPartsFromCollector > 0 {
- // Use totalPartsFromCollector here
- Logger.info("Reassembling \(totalPartsFromCollector) disk image parts using sparse file technique...")
- let outputURL = tempVMDir.appendingPathComponent("disk.img")
-
- // Wrap setup in do-catch for better error reporting
- let outputHandle: FileHandle
- do {
- // 1. Ensure parent directory exists
- try FileManager.default.createDirectory(
- at: outputURL.deletingLastPathComponent(), withIntermediateDirectories: true
- )
-
- // 2. Explicitly create the file first, removing old one if needed
- if FileManager.default.fileExists(atPath: outputURL.path) {
- try FileManager.default.removeItem(at: outputURL)
- }
- guard FileManager.default.createFile(atPath: outputURL.path, contents: nil)
- else {
- throw PullError.fileCreationFailed(outputURL.path)
- }
-
- // 3. Now open the handle for writing
- outputHandle = try FileHandle(forWritingTo: outputURL)
-
- } catch {
- // Catch errors during directory/file creation or handle opening
- Logger.error(
- "Failed during setup for disk image reassembly: \(error.localizedDescription)",
- metadata: ["path": outputURL.path])
- throw PullError.reassemblySetupFailed(
- path: outputURL.path, underlyingError: error)
- }
-
- // Calculate expected size from the manifest layers (sum of compressed parts - for logging only now)
- // Filter based on the correct media type now
- let expectedCompressedTotalSize = UInt64(
- manifest.layers.filter { $0.mediaType == "application/octet-stream+lz4" }.reduce(0)
- { $0 + $1.size }
- )
- Logger.info(
- "Total compressed parts size: \(ByteCountFormatter.string(fromByteCount: Int64(expectedCompressedTotalSize), countStyle: .file))"
- )
-
- // Calculate fallback size (sum of compressed parts)
- let _: UInt64 = diskParts.reduce(UInt64(0)) {
- (acc: UInt64, element) -> UInt64 in
- let fileSize =
- (try? FileManager.default.attributesOfItem(atPath: element.1.path)[.size]
- as? UInt64 ?? 0) ?? 0
- return acc + fileSize
- }
-
- // Use: annotation size > VM config diskSize > fallback size
- let sizeForTruncate: UInt64
- if let size = uncompressedSize {
- Logger.info("Using uncompressed size from annotation: \(size) bytes")
- sizeForTruncate = size
- } else if let size = vmConfigDiskSize {
- Logger.info("Using diskSize from VM config: \(size) bytes")
- sizeForTruncate = size
- } else {
- Logger.error(
- "Missing both uncompressed size annotation and VM config diskSize for multi-part image."
- )
- throw PullError.missingUncompressedSizeAnnotation
- }
-
- defer { try? outputHandle.close() }
-
- // Set the file size without writing data (creates a sparse file)
- try outputHandle.truncate(atOffset: sizeForTruncate)
-
- // Verify the sparse file was created with the correct size
- let initialSize =
- (try? FileManager.default.attributesOfItem(atPath: outputURL.path)[.size]
- as? UInt64) ?? 0
- Logger.info(
- "Sparse file initialized with size: \(ByteCountFormatter.string(fromByteCount: Int64(initialSize), countStyle: .file))"
- )
-
- // Add a simple test pattern at the beginning and end of the file to verify it's writable
- try outputHandle.seek(toOffset: 0)
- let testPattern = "LUME_TEST_PATTERN".data(using: .utf8)!
- try outputHandle.write(contentsOf: testPattern)
-
- try outputHandle.seek(toOffset: sizeForTruncate - UInt64(testPattern.count))
- try outputHandle.write(contentsOf: testPattern)
- try outputHandle.synchronize()
-
- Logger.info("Test patterns written to sparse file. File is ready for writing.")
-
- var reassemblyProgressLogger = ProgressLogger(threshold: 0.05)
- var currentOffset: UInt64 = 0 // Track position in the final *decompressed* file
-
- // Iterate using the reliable totalParts count from media type
- // Use totalPartsFromCollector for the loop range
- for partNum in 1...totalPartsFromCollector {
- // Find the part URL from our collected parts using the logical partNum
- guard let partInfo = diskParts.first(where: { $0.0 == partNum }) else {
- // This error should now be less likely, but good to keep
- Logger.error("Missing required part number \(partNum) in collected parts during reassembly.")
- // Add current state log on error
- Logger.error("Current disk part numbers available: \(diskParts.map { $0.0 })")
- throw PullError.missingPart(partNum)
- }
- let partURL = partInfo.1 // Get the URL from the tuple
-
- Logger.info(
- "Processing part \(partNum) of \(totalPartsFromCollector): \(partURL.lastPathComponent)")
-
- // Seek to the correct offset in the output sparse file
- try outputHandle.seek(toOffset: currentOffset)
-
- // Check if this chunk might be all zeros (sparse data) by sampling the compressed data
- // Skip this check for now as it's an optimization we can add later if needed
- let isLikelySparse = false
-
- // Always attempt decompression using decompressChunkAndWriteSparse for LZ4 parts
- if isLikelySparse {
- // For sparse chunks, we don't need to write anything - just advance the offset
- // We determine the uncompressed size from the chunk metadata or estimation
-
- // For now, we'll still decompress to ensure correct behavior, and optimize later
- Logger.info("Chunk appears to be sparse, but decompressing for reliability")
- let decompressedBytesWritten = try decompressChunkAndWriteSparse(
- inputPath: partURL.path,
- outputHandle: outputHandle,
- startOffset: currentOffset
- )
- currentOffset += decompressedBytesWritten
- } else {
- Logger.info("Decompressing part \(partNum)")
- let decompressedBytesWritten = try decompressChunkAndWriteSparse(
- inputPath: partURL.path,
- outputHandle: outputHandle,
- startOffset: currentOffset
+ // Now that we've downloaded everything to the cache, use copyFromCache to create final VM files
+ if cachingEnabled {
+ Logger.info("Using copyFromCache method to properly preserve partition tables")
+ try await copyFromCache(manifest: manifest, manifestId: manifestId, to: tempVMDir)
+ } else {
+ // If caching is disabled, just copy files directly to tempVMDir
+ Logger.info("Caching disabled - copying downloaded files directly to VM directory")
+
+ // Copy non-disk files first
+ for file in ["config.json", "nvram.bin"] {
+ let sourceURL = tempDownloadDir.appendingPathComponent(file)
+ if FileManager.default.fileExists(atPath: sourceURL.path) {
+ try FileManager.default.copyItem(
+ at: sourceURL,
+ to: tempVMDir.appendingPathComponent(file)
)
- currentOffset += decompressedBytesWritten
}
-
- reassemblyProgressLogger.logProgress(
- current: Double(currentOffset) / Double(sizeForTruncate),
- context: "Reassembling"
- )
-
- // Ensure data is written before processing next part
- try outputHandle.synchronize()
}
- // Finalize progress, close handle (done by defer)
- reassemblyProgressLogger.logProgress(current: 1.0, context: "Reassembly Complete")
- Logger.info("") // Newline
-
- // Optimize sparseness after completing reassembly
- try outputHandle.close() // Close handle to ensure all data is flushed
-
- // Verify final size
- let finalSize =
- (try? FileManager.default.attributesOfItem(atPath: outputURL.path)[.size]
- as? UInt64) ?? 0
- Logger.info(
- "Final disk image size: \(ByteCountFormatter.string(fromByteCount: Int64(finalSize), countStyle: .file))"
- )
-
- // Optimize sparseness if on macOS
- if FileManager.default.fileExists(atPath: "/bin/cp") {
- Logger.info("Optimizing sparse file representation...")
- let optimizedPath = outputURL.path + ".optimized"
-
- let process = Process()
- process.executableURL = URL(fileURLWithPath: "/bin/cp")
- process.arguments = ["-c", outputURL.path, optimizedPath]
-
- do {
- try process.run()
- process.waitUntilExit()
-
- if process.terminationStatus == 0 {
- // Get size of optimized file
- let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] as? UInt64) ?? 0
- let originalUsage = getActualDiskUsage(path: outputURL.path)
- let optimizedUsage = getActualDiskUsage(path: optimizedPath)
-
- Logger.info(
- "Sparse optimization results: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))"
- )
-
- // Replace the original with the optimized version
- try FileManager.default.removeItem(at: outputURL)
- try FileManager.default.moveItem(at: URL(fileURLWithPath: optimizedPath), to: outputURL)
- Logger.info("Replaced with optimized sparse version")
- } else {
- Logger.info("Sparse optimization failed, using original file")
- try? FileManager.default.removeItem(atPath: optimizedPath)
- }
- } catch {
- Logger.info("Error during sparse optimization: \(error.localizedDescription)")
- try? FileManager.default.removeItem(atPath: optimizedPath)
- }
- }
-
- if finalSize != sizeForTruncate {
- Logger.info(
- "Warning: Final reported size (\(finalSize) bytes) differs from expected size (\(sizeForTruncate) bytes), but this doesn't affect functionality"
- )
- }
-
- Logger.info("Disk image reassembly completed")
- } else {
- // Copy single disk image if it exists
+ // For the disk image, we have two cases - either a single file or parts
let diskURL = tempDownloadDir.appendingPathComponent("disk.img")
if FileManager.default.fileExists(atPath: diskURL.path) {
+ // Single file disk image
try FileManager.default.copyItem(
at: diskURL,
to: tempVMDir.appendingPathComponent("disk.img")
)
+ Logger.info("Copied single disk.img file to VM directory")
+ } else {
+ // Multiple parts case - use the partitioned disk.img from reassembly
+ let diskParts = await diskPartsCollector.getSortedParts()
+ if !diskParts.isEmpty {
+ Logger.info("Using most recently assembled disk image for VM")
+ let assembledDiskURL = tempVMDir.appendingPathComponent("disk.img")
+ if FileManager.default.fileExists(atPath: assembledDiskURL.path) {
+ Logger.info("Assembled disk.img already exists in VM directory")
+ } else {
+ Logger.error(
+ "Could not find assembled disk image - VM may not boot properly")
+ }
+ } else {
+ Logger.error("No disk image found - VM may not boot properly")
+ }
}
}
-
- // Copy config and nvram files if they exist
- for file in ["config.json", "nvram.bin"] {
- let sourceURL = tempDownloadDir.appendingPathComponent(file)
- if FileManager.default.fileExists(atPath: sourceURL.path) {
- try FileManager.default.copyItem(
- at: sourceURL,
- to: tempVMDir.appendingPathComponent(file)
- )
- }
- }
- }
-
- // Simulate cache pull behavior if this is a first pull
- if !cachingEnabled || !validateCache(manifest: manifest, manifestId: manifestId) {
- try simulateCachePull(tempVMDir: tempVMDir)
}
// Only move to final location once everything is complete
@@ -1242,10 +1024,10 @@ class ImageContainerRegistry: @unchecked Sendable {
private func createDiskImageFromSource(
sourceURL: URL, // Source data to decompress
destinationURL: URL, // Where to create the disk image
- diskSize: UInt64 // Total size for the sparse file
+ diskSize: UInt64 // Total size for the sparse file
) throws {
Logger.info("Creating sparse disk image...")
-
+
// Create empty destination file
if FileManager.default.fileExists(atPath: destinationURL.path) {
try FileManager.default.removeItem(at: destinationURL)
@@ -1253,11 +1035,11 @@ class ImageContainerRegistry: @unchecked Sendable {
guard FileManager.default.createFile(atPath: destinationURL.path, contents: nil) else {
throw PullError.fileCreationFailed(destinationURL.path)
}
-
+
// Create sparse file
let outputHandle = try FileHandle(forWritingTo: destinationURL)
try outputHandle.truncate(atOffset: diskSize)
-
+
// Write test patterns at beginning and end
Logger.info("Writing test patterns to verify writability...")
let testPattern = "LUME_TEST_PATTERN".data(using: .utf8)!
@@ -1266,7 +1048,7 @@ class ImageContainerRegistry: @unchecked Sendable {
try outputHandle.seek(toOffset: diskSize - UInt64(testPattern.count))
try outputHandle.write(contentsOf: testPattern)
try outputHandle.synchronize()
-
+
// Decompress the source data at offset 0
Logger.info("Decompressing source data...")
let bytesWritten = try decompressChunkAndWriteSparse(
@@ -1274,57 +1056,62 @@ class ImageContainerRegistry: @unchecked Sendable {
outputHandle: outputHandle,
startOffset: 0
)
- Logger.info("Decompressed \(ByteCountFormatter.string(fromByteCount: Int64(bytesWritten), countStyle: .file)) of data")
-
+ Logger.info(
+ "Decompressed \(ByteCountFormatter.string(fromByteCount: Int64(bytesWritten), countStyle: .file)) of data"
+ )
+
// Ensure data is written and close handle
try outputHandle.synchronize()
try outputHandle.close()
-
+
// Run sync to flush filesystem
let syncProcess = Process()
syncProcess.executableURL = URL(fileURLWithPath: "/bin/sync")
try syncProcess.run()
syncProcess.waitUntilExit()
-
+
// Optimize with cp -c
if FileManager.default.fileExists(atPath: "/bin/cp") {
Logger.info("Optimizing sparse file representation...")
let optimizedPath = destinationURL.path + ".optimized"
-
+
let process = Process()
process.executableURL = URL(fileURLWithPath: "/bin/cp")
process.arguments = ["-c", destinationURL.path, optimizedPath]
-
+
try process.run()
process.waitUntilExit()
-
+
if process.terminationStatus == 0 {
// Get optimization results
- let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] as? UInt64) ?? 0
+ let optimizedSize =
+ (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size]
+ as? UInt64) ?? 0
let originalUsage = getActualDiskUsage(path: destinationURL.path)
let optimizedUsage = getActualDiskUsage(path: optimizedPath)
-
+
Logger.info(
"Sparse optimization results: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))"
)
-
+
// Replace original with optimized
try FileManager.default.removeItem(at: destinationURL)
- try FileManager.default.moveItem(at: URL(fileURLWithPath: optimizedPath), to: destinationURL)
+ try FileManager.default.moveItem(
+ at: URL(fileURLWithPath: optimizedPath), to: destinationURL)
Logger.info("Replaced with optimized sparse version")
} else {
Logger.info("Sparse optimization failed, using original file")
try? FileManager.default.removeItem(atPath: optimizedPath)
}
}
-
+
// Set permissions to 0644
let chmodProcess = Process()
chmodProcess.executableURL = URL(fileURLWithPath: "/bin/chmod")
chmodProcess.arguments = ["0644", destinationURL.path]
try chmodProcess.run()
chmodProcess.waitUntilExit()
-
+
// Final sync
let finalSyncProcess = Process()
finalSyncProcess.executableURL = URL(fileURLWithPath: "/bin/sync")
@@ -1335,94 +1122,95 @@ class ImageContainerRegistry: @unchecked Sendable {
// Function to simulate cache pull behavior for freshly downloaded images
private func simulateCachePull(tempVMDir: URL) throws {
Logger.info("Simulating cache pull behavior for freshly downloaded image...")
-
+
// Find disk.img in tempVMDir
let diskImgPath = tempVMDir.appendingPathComponent("disk.img")
guard FileManager.default.fileExists(atPath: diskImgPath.path) else {
Logger.info("No disk.img found to simulate cache pull behavior")
return
}
-
+
// Get file attributes and size
let attributes = try FileManager.default.attributesOfItem(atPath: diskImgPath.path)
guard let diskSize = attributes[.size] as? UInt64, diskSize > 0 else {
Logger.error("Could not determine disk.img size for simulation")
return
}
-
+
Logger.info("Creating true disk image clone with partition table preserved...")
-
+
// Create backup of original file
let backupPath = tempVMDir.appendingPathComponent("disk.img.original")
try FileManager.default.moveItem(at: diskImgPath, to: backupPath)
-
+
// Let's first check if the original image has a partition table
Logger.info("Checking if source image has a partition table...")
let checkProcess = Process()
checkProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil")
checkProcess.arguments = ["imageinfo", backupPath.path]
-
+
let checkPipe = Pipe()
checkProcess.standardOutput = checkPipe
-
+
try checkProcess.run()
checkProcess.waitUntilExit()
-
+
let checkData = checkPipe.fileHandleForReading.readDataToEndOfFile()
let checkOutput = String(data: checkData, encoding: .utf8) ?? ""
Logger.info("Source image info: \(checkOutput)")
-
+
// Try different methods in sequence until one works
var success = false
-
+
// Method 1: Use hdiutil convert to convert the image while preserving all data
if !success {
Logger.info("Trying hdiutil convert...")
let tempPath = tempVMDir.appendingPathComponent("disk.img.temp")
-
+
let convertProcess = Process()
convertProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil")
convertProcess.arguments = [
- "convert",
- backupPath.path,
- "-format", "UDRO", // Read-only first to preserve partition table
- "-o", tempPath.path
+ "convert",
+ backupPath.path,
+ "-format", "UDRO", // Read-only first to preserve partition table
+ "-o", tempPath.path,
]
-
+
let convertOutPipe = Pipe()
let convertErrPipe = Pipe()
convertProcess.standardOutput = convertOutPipe
convertProcess.standardError = convertErrPipe
-
+
do {
try convertProcess.run()
convertProcess.waitUntilExit()
-
+
let errData = convertErrPipe.fileHandleForReading.readDataToEndOfFile()
let errOutput = String(data: errData, encoding: .utf8) ?? ""
-
+
if convertProcess.terminationStatus == 0 {
Logger.info("hdiutil convert succeeded. Converting to writable format...")
// Now convert to writable format
let convertBackProcess = Process()
convertBackProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil")
convertBackProcess.arguments = [
- "convert",
- tempPath.path,
- "-format", "UDRW", // Read-write format
- "-o", diskImgPath.path
+ "convert",
+ tempPath.path,
+ "-format", "UDRW", // Read-write format
+ "-o", diskImgPath.path,
]
-
+
try convertBackProcess.run()
convertBackProcess.waitUntilExit()
-
+
if convertBackProcess.terminationStatus == 0 {
- Logger.info("Successfully converted to writable format with partition table")
+ Logger.info(
+ "Successfully converted to writable format with partition table")
success = true
} else {
Logger.error("hdiutil convert to writable format failed")
}
-
+
// Clean up temporary image
try? FileManager.default.removeItem(at: tempPath)
} else {
@@ -1432,33 +1220,33 @@ class ImageContainerRegistry: @unchecked Sendable {
Logger.error("Error executing hdiutil convert: \(error)")
}
}
-
+
// Method 2: Try direct raw copy method
if !success {
Logger.info("Trying direct raw copy with dd...")
-
+
// Create empty file first
FileManager.default.createFile(atPath: diskImgPath.path, contents: nil)
-
+
let ddProcess = Process()
ddProcess.executableURL = URL(fileURLWithPath: "/bin/dd")
ddProcess.arguments = [
"if=\(backupPath.path)",
"of=\(diskImgPath.path)",
- "bs=1m", // Large block size
- "count=81920" // Ensure we copy everything (80GB+ should be sufficient)
+ "bs=1m", // Large block size
+ "count=81920", // Ensure we copy everything (80GB+ should be sufficient)
]
-
+
let ddErrPipe = Pipe()
ddProcess.standardError = ddErrPipe
-
+
do {
try ddProcess.run()
ddProcess.waitUntilExit()
-
+
let errData = ddErrPipe.fileHandleForReading.readDataToEndOfFile()
let errOutput = String(data: errData, encoding: .utf8) ?? ""
-
+
if ddProcess.terminationStatus == 0 {
Logger.info("Raw dd copy completed: \(errOutput)")
success = true
@@ -1469,34 +1257,36 @@ class ImageContainerRegistry: @unchecked Sendable {
Logger.error("Error executing dd: \(error)")
}
}
-
+
// Method 3: Use a more complex approach with disk mounting
if !success {
Logger.info("Trying advanced disk attach/detach approach...")
-
+
// Mount the source disk image
let attachProcess = Process()
attachProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil")
attachProcess.arguments = ["attach", backupPath.path, "-nomount"]
-
+
let attachPipe = Pipe()
attachProcess.standardOutput = attachPipe
-
+
try attachProcess.run()
attachProcess.waitUntilExit()
-
+
let attachData = attachPipe.fileHandleForReading.readDataToEndOfFile()
let attachOutput = String(data: attachData, encoding: .utf8) ?? ""
-
+
// Extract the disk device from output (/dev/diskN)
var diskDevice: String? = nil
- if let diskMatch = attachOutput.range(of: "/dev/disk[0-9]+", options: .regularExpression) {
+ if let diskMatch = attachOutput.range(
+ of: "/dev/disk[0-9]+", options: .regularExpression)
+ {
diskDevice = String(attachOutput[diskMatch])
}
-
+
if let device = diskDevice {
Logger.info("Source disk attached at \(device)")
-
+
// Create a bootable disk image clone
let createProcess = Process()
createProcess.executableURL = URL(fileURLWithPath: "/usr/sbin/asr")
@@ -1505,19 +1295,22 @@ class ImageContainerRegistry: @unchecked Sendable {
"--source", device,
"--target", diskImgPath.path,
"--erase",
- "--noprompt"
+ "--noprompt",
]
-
+
let createPipe = Pipe()
createProcess.standardOutput = createPipe
-
+
do {
try createProcess.run()
createProcess.waitUntilExit()
-
- let createOutput = String(data: createPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) ?? ""
+
+ let createOutput =
+ String(
+ data: createPipe.fileHandleForReading.readDataToEndOfFile(),
+ encoding: .utf8) ?? ""
Logger.info("asr output: \(createOutput)")
-
+
if createProcess.terminationStatus == 0 {
Logger.info("Successfully created bootable disk image clone!")
success = true
@@ -1527,7 +1320,7 @@ class ImageContainerRegistry: @unchecked Sendable {
} catch {
Logger.error("Error executing asr: \(error)")
}
-
+
// Always detach the disk when done
let detachProcess = Process()
detachProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil")
@@ -1538,98 +1331,102 @@ class ImageContainerRegistry: @unchecked Sendable {
Logger.error("Failed to extract disk device from hdiutil attach output")
}
}
-
+
// Fallback: If none of the methods worked, revert to our previous method just to ensure we have a usable image
if !success {
Logger.info("All specialized methods failed. Reverting to basic copy...")
-
+
// If the disk image file exists (from a failed attempt), remove it
if FileManager.default.fileExists(atPath: diskImgPath.path) {
try FileManager.default.removeItem(at: diskImgPath)
}
-
+
// Attempt a basic file copy which will at least give us something to work with
try FileManager.default.copyItem(at: backupPath, to: diskImgPath)
}
-
+
// Optimize sparseness if possible
if FileManager.default.fileExists(atPath: "/bin/cp") {
Logger.info("Optimizing sparse file representation...")
let optimizedPath = diskImgPath.path + ".optimized"
-
+
let process = Process()
process.executableURL = URL(fileURLWithPath: "/bin/cp")
process.arguments = ["-c", diskImgPath.path, optimizedPath]
-
+
try process.run()
process.waitUntilExit()
-
+
if process.terminationStatus == 0 {
- let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] as? UInt64) ?? 0
+ let optimizedSize =
+ (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size]
+ as? UInt64) ?? 0
let originalUsage = getActualDiskUsage(path: diskImgPath.path)
let optimizedUsage = getActualDiskUsage(path: optimizedPath)
-
+
Logger.info(
"Sparse optimization results: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))"
)
-
+
// Replace with optimized version
try FileManager.default.removeItem(at: diskImgPath)
- try FileManager.default.moveItem(at: URL(fileURLWithPath: optimizedPath), to: diskImgPath)
+ try FileManager.default.moveItem(
+ at: URL(fileURLWithPath: optimizedPath), to: diskImgPath)
Logger.info("Replaced with optimized sparse version")
} else {
Logger.info("Sparse optimization failed, using original file")
try? FileManager.default.removeItem(atPath: optimizedPath)
}
}
-
+
// Set permissions to 0644
let chmodProcess = Process()
chmodProcess.executableURL = URL(fileURLWithPath: "/bin/chmod")
chmodProcess.arguments = ["0644", diskImgPath.path]
try chmodProcess.run()
chmodProcess.waitUntilExit()
-
+
// Final sync
let finalSyncProcess = Process()
finalSyncProcess.executableURL = URL(fileURLWithPath: "/bin/sync")
try finalSyncProcess.run()
finalSyncProcess.waitUntilExit()
-
+
// Verify the final disk image
Logger.info("Verifying final disk image partition information...")
let verifyProcess = Process()
verifyProcess.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil")
verifyProcess.arguments = ["imageinfo", diskImgPath.path]
-
+
let verifyOutputPipe = Pipe()
verifyProcess.standardOutput = verifyOutputPipe
-
+
try verifyProcess.run()
verifyProcess.waitUntilExit()
-
+
let verifyOutputData = verifyOutputPipe.fileHandleForReading.readDataToEndOfFile()
let verifyOutput = String(data: verifyOutputData, encoding: .utf8) ?? ""
Logger.info("Final disk image verification:\n\(verifyOutput)")
-
+
// Clean up backup file
try FileManager.default.removeItem(at: backupPath)
-
- Logger.info("Cache pull simulation completed successfully with partition table preservation")
+
+ Logger.info(
+ "Cache pull simulation completed successfully with partition table preservation")
}
private func copyFromCache(manifest: Manifest, manifestId: String, to destination: URL)
async throws
{
Logger.info("Copying from cache...")
-
+
// Define output URL and expected size variable scope here
let outputURL = destination.appendingPathComponent("disk.img")
- var expectedTotalSize: UInt64? = nil // Use optional to handle missing config
+ var expectedTotalSize: UInt64? = nil // Use optional to handle missing config
// Instantiate collector
let diskPartsCollector = DiskPartsCollector()
- var lz4LayerCount = 0 // Count lz4 layers found
+ var lz4LayerCount = 0 // Count lz4 layers found
// First identify disk parts and non-disk files
for layer in manifest.layers {
@@ -1637,13 +1434,14 @@ class ImageContainerRegistry: @unchecked Sendable {
// Identify disk parts simply by media type
if layer.mediaType == "application/octet-stream+lz4" {
- lz4LayerCount += 1 // Increment count
- // Add to collector. It will assign the sequential part number.
- let collectorPartNum = await diskPartsCollector.addPart(url: cachedLayer)
- Logger.info("Adding cached lz4 layer (part \(lz4LayerCount)) -> Collector #\(collectorPartNum): \(cachedLayer.lastPathComponent)")
- }
- else {
- // --- Handle Non-Disk-Part Layer (from cache) ---
+ lz4LayerCount += 1 // Increment count
+ // Add to collector. It will assign the sequential part number.
+ let collectorPartNum = await diskPartsCollector.addPart(url: cachedLayer)
+ Logger.info(
+ "Adding cached lz4 layer (part \(lz4LayerCount)) -> Collector #\(collectorPartNum): \(cachedLayer.lastPathComponent)"
+ )
+ } else {
+ // --- Handle Non-Disk-Part Layer (from cache) ---
let fileName: String
switch layer.mediaType {
case "application/vnd.oci.image.config.v1+json":
@@ -1651,12 +1449,12 @@ class ImageContainerRegistry: @unchecked Sendable {
case "application/octet-stream":
// Assume nvram if config layer exists, otherwise assume single disk image
fileName = manifest.config != nil ? "nvram.bin" : "disk.img"
- case "application/vnd.oci.image.layer.v1.tar",
- "application/octet-stream+gzip":
- // Assume disk image for these types as well if encountered in cache scenario
- fileName = "disk.img"
+ case "application/vnd.oci.image.layer.v1.tar",
+ "application/octet-stream+gzip":
+ // Assume disk image for these types as well if encountered in cache scenario
+ fileName = "disk.img"
default:
- Logger.info("Skipping unsupported cached layer media type: \(layer.mediaType)")
+ Logger.info("Skipping unsupported cached layer media type: \(layer.mediaType)")
continue
}
// Copy the non-disk file directly from cache to destination
@@ -1667,19 +1465,20 @@ class ImageContainerRegistry: @unchecked Sendable {
}
}
- // --- Safely retrieve parts AFTER loop ---
- let diskPartSources = await diskPartsCollector.getSortedParts() // Sorted by assigned sequential number
- let totalParts = await diskPartsCollector.getTotalParts() // Get total count from collector
+ // --- Safely retrieve parts AFTER loop ---
+ let diskPartSources = await diskPartsCollector.getSortedParts() // Sorted by assigned sequential number
+ let totalParts = await diskPartsCollector.getTotalParts() // Get total count from collector
Logger.info("Found \(totalParts) lz4 disk parts in cache to reassemble.")
- // --- End retrieving parts ---
+ // --- End retrieving parts ---
// Reassemble disk parts if needed
// Use the count from the collector
if !diskPartSources.isEmpty {
// Use totalParts from collector directly
- Logger.info("Reassembling \(totalParts) disk image parts using sparse file technique...")
-
+ Logger.info(
+ "Reassembling \(totalParts) disk image parts using sparse file technique...")
+
// Get uncompressed size from cached config file (needs to be copied first)
let configURL = destination.appendingPathComponent("config.json")
// Parse config.json to get uncompressed size *before* reassembly
@@ -1713,24 +1512,25 @@ class ImageContainerRegistry: @unchecked Sendable {
} else {
// If neither is found in cache scenario, throw error as we cannot determine the size
Logger.error(
- "Missing both uncompressed size annotation and VM config diskSize for cached multi-part image."
- + " Cannot reassemble."
+ "Missing both uncompressed size annotation and VM config diskSize for cached multi-part image."
+ + " Cannot reassemble."
)
throw PullError.missingUncompressedSizeAnnotation
}
// Now that expectedTotalSize is guaranteed to be non-nil, proceed with setup
guard let sizeForTruncate = expectedTotalSize else {
- // This should not happen due to the checks above, but safety first
- let nilError: Error? = nil
- // Use nil-coalescing to provide a default error, appeasing the compiler
- throw PullError.reassemblySetupFailed(path: outputURL.path, underlyingError: nilError ?? NoSpecificUnderlyingError())
+ // This should not happen due to the checks above, but safety first
+ let nilError: Error? = nil
+ // Use nil-coalescing to provide a default error, appeasing the compiler
+ throw PullError.reassemblySetupFailed(
+ path: outputURL.path, underlyingError: nilError ?? NoSpecificUnderlyingError())
}
// If we have just one disk part, use the shared function
if totalParts == 1 {
// Single part - use shared function
- let sourceURL = diskPartSources[0].1 // Get the first source URL (index 1 of the tuple)
+ let sourceURL = diskPartSources[0].1 // Get the first source URL (index 1 of the tuple)
try createDiskImageFromSource(
sourceURL: sourceURL,
destinationURL: outputURL,
@@ -1742,22 +1542,30 @@ class ImageContainerRegistry: @unchecked Sendable {
let outputHandle: FileHandle
do {
// Ensure parent directory exists
- try FileManager.default.createDirectory(at: outputURL.deletingLastPathComponent(), withIntermediateDirectories: true)
+ try FileManager.default.createDirectory(
+ at: outputURL.deletingLastPathComponent(), withIntermediateDirectories: true
+ )
// Explicitly create the file first, removing old one if needed
if FileManager.default.fileExists(atPath: outputURL.path) {
try FileManager.default.removeItem(at: outputURL)
}
- guard FileManager.default.createFile(atPath: outputURL.path, contents: nil) else {
+ guard FileManager.default.createFile(atPath: outputURL.path, contents: nil)
+ else {
throw PullError.fileCreationFailed(outputURL.path)
}
// Open handle for writing
outputHandle = try FileHandle(forWritingTo: outputURL)
// Set the file size (creates sparse file)
try outputHandle.truncate(atOffset: sizeForTruncate)
- Logger.info("Sparse file initialized for cache reassembly with size: \(ByteCountFormatter.string(fromByteCount: Int64(sizeForTruncate), countStyle: .file))")
+ Logger.info(
+ "Sparse file initialized for cache reassembly with size: \(ByteCountFormatter.string(fromByteCount: Int64(sizeForTruncate), countStyle: .file))"
+ )
} catch {
- Logger.error("Failed during setup for cached disk image reassembly: \(error.localizedDescription)", metadata: ["path": outputURL.path])
- throw PullError.reassemblySetupFailed(path: outputURL.path, underlyingError: error)
+ Logger.error(
+ "Failed during setup for cached disk image reassembly: \(error.localizedDescription)",
+ metadata: ["path": outputURL.path])
+ throw PullError.reassemblySetupFailed(
+ path: outputURL.path, underlyingError: error)
}
// Ensure handle is closed when exiting this scope
@@ -1769,11 +1577,15 @@ class ImageContainerRegistry: @unchecked Sendable {
// Iterate from 1 up to the total number of parts found by the collector
for collectorPartNum in 1...totalParts {
// Find the source URL from our collected parts using the sequential collectorPartNum
- guard let sourceInfo = diskPartSources.first(where: { $0.0 == collectorPartNum }) else {
- Logger.error("Missing required cached part number \(collectorPartNum) in collected parts during reassembly.")
+ guard
+ let sourceInfo = diskPartSources.first(where: { $0.0 == collectorPartNum })
+ else {
+ Logger.error(
+ "Missing required cached part number \(collectorPartNum) in collected parts during reassembly."
+ )
throw PullError.missingPart(collectorPartNum)
}
- let sourceURL = sourceInfo.1 // Get URL from tuple
+ let sourceURL = sourceInfo.1 // Get URL from tuple
// Log using the sequential collector part number
Logger.info(
@@ -1789,10 +1601,10 @@ class ImageContainerRegistry: @unchecked Sendable {
currentOffset += decompressedBytesWritten
// Update progress (using sizeForTruncate which should be available)
reassemblyProgressLogger.logProgress(
- current: Double(currentOffset) / Double(sizeForTruncate),
- context: "Reassembling Cache")
-
- try outputHandle.synchronize() // Explicitly synchronize after each chunk
+ current: Double(currentOffset) / Double(sizeForTruncate),
+ context: "Reassembling Cache")
+
+ try outputHandle.synchronize() // Explicitly synchronize after each chunk
}
// Finalize progress, close handle (done by defer)
@@ -1806,13 +1618,13 @@ class ImageContainerRegistry: @unchecked Sendable {
try outputHandle.seek(toOffset: sizeForTruncate - UInt64(testPattern.count))
try outputHandle.write(contentsOf: testPattern)
try outputHandle.synchronize()
-
+
// Ensure handle is properly synchronized before closing
try outputHandle.synchronize()
-
+
// Close handle explicitly instead of relying on defer
try outputHandle.close()
-
+
// Verify final size
let finalSize =
(try? FileManager.default.attributesOfItem(atPath: outputURL.path)[.size]
@@ -1829,44 +1641,49 @@ class ImageContainerRegistry: @unchecked Sendable {
}
Logger.info("Disk image reassembly completed")
-
+
// Optimize sparseness for cached reassembly if on macOS
if FileManager.default.fileExists(atPath: "/bin/cp") {
Logger.info("Optimizing sparse file representation for cached reassembly...")
let optimizedPath = outputURL.path + ".optimized"
-
+
let process = Process()
process.executableURL = URL(fileURLWithPath: "/bin/cp")
process.arguments = ["-c", outputURL.path, optimizedPath]
-
+
do {
try process.run()
process.waitUntilExit()
-
+
if process.terminationStatus == 0 {
// Get size of optimized file
- let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[.size] as? UInt64) ?? 0
+ let optimizedSize =
+ (try? FileManager.default.attributesOfItem(atPath: optimizedPath)[
+ .size] as? UInt64) ?? 0
let originalUsage = getActualDiskUsage(path: outputURL.path)
let optimizedUsage = getActualDiskUsage(path: optimizedPath)
-
+
Logger.info(
"Sparse optimization results for cache: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))"
)
-
+
// Replace the original with the optimized version
try FileManager.default.removeItem(at: outputURL)
- try FileManager.default.moveItem(at: URL(fileURLWithPath: optimizedPath), to: outputURL)
+ try FileManager.default.moveItem(
+ at: URL(fileURLWithPath: optimizedPath), to: outputURL)
Logger.info("Replaced cached reassembly with optimized sparse version")
} else {
Logger.info("Sparse optimization failed for cache, using original file")
try? FileManager.default.removeItem(atPath: optimizedPath)
}
} catch {
- Logger.info("Error during sparse optimization for cache: \(error.localizedDescription)")
+ Logger.info(
+ "Error during sparse optimization for cache: \(error.localizedDescription)"
+ )
try? FileManager.default.removeItem(atPath: optimizedPath)
}
}
-
+
// Set permissions to ensure consistency
let chmodProcess = Process()
chmodProcess.executableURL = URL(fileURLWithPath: "/bin/chmod")
@@ -1880,12 +1697,16 @@ class ImageContainerRegistry: @unchecked Sendable {
}
private func getToken(repository: String) async throws -> String {
- let encodedRepo = repository.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? repository
+ let encodedRepo =
+ repository.addingPercentEncoding(withAllowedCharacters: .urlQueryAllowed) ?? repository
// Request both pull and push scope for uploads
- let url = URL(string: "https://\(self.registry)/token?scope=repository:\(encodedRepo):pull,push&service=\(self.registry)")!
-
+ let url = URL(
+ string:
+ "https://\(self.registry)/token?scope=repository:\(encodedRepo):pull,push&service=\(self.registry)"
+ )!
+
var request = URLRequest(url: url)
- request.httpMethod = "GET" // Token endpoint uses GET
+ request.httpMethod = "GET" // Token endpoint uses GET
request.setValue("application/json", forHTTPHeaderField: "Accept")
// *** Add Basic Authentication Header if credentials exist ***
@@ -1906,26 +1727,31 @@ class ImageContainerRegistry: @unchecked Sendable {
// *** End Basic Auth addition ***
let (data, response) = try await URLSession.shared.data(for: request)
-
+
// Check response status code *before* parsing JSON
guard let httpResponse = response as? HTTPURLResponse else {
- throw PushError.authenticationFailed // Or a more generic network error
+ throw PushError.authenticationFailed // Or a more generic network error
}
-
+
guard httpResponse.statusCode == 200 else {
// Log detailed error including status code and potentially response body
let responseBody = String(data: data, encoding: .utf8) ?? "(Could not decode body)"
- Logger.error("Token request failed with status code: \(httpResponse.statusCode). Response: \(responseBody)")
+ Logger.error(
+ "Token request failed with status code: \(httpResponse.statusCode). Response: \(responseBody)"
+ )
// Throw specific error based on status if needed (e.g., 401 for unauthorized)
- throw PushError.authenticationFailed
+ throw PushError.authenticationFailed
}
-
+
let jsonResponse = try JSONSerialization.jsonObject(with: data) as? [String: Any]
- guard let token = jsonResponse?["token"] as? String ?? jsonResponse?["access_token"] as? String else {
+ guard
+ let token = jsonResponse?["token"] as? String ?? jsonResponse?["access_token"]
+ as? String
+ else {
Logger.error("Token not found in registry response.")
throw PushError.missingToken
}
-
+
return token
}
@@ -2673,9 +2499,9 @@ class ImageContainerRegistry: @unchecked Sendable {
// New push method
public func push(
- vmDirPath: String,
- imageName: String,
- tags: [String],
+ vmDirPath: String,
+ imageName: String,
+ tags: [String],
chunkSizeMb: Int = 512,
verbose: Bool = false,
dryRun: Bool = false,
@@ -2686,18 +2512,18 @@ class ImageContainerRegistry: @unchecked Sendable {
metadata: [
"vm_path": vmDirPath,
"imageName": imageName,
- "tags": "\(tags.joined(separator: ", "))", // Log all tags
+ "tags": "\(tags.joined(separator: ", "))", // Log all tags
"registry": registry,
"organization": organization,
"chunk_size": "\(chunkSizeMb)MB",
"dry_run": "\(dryRun)",
- "reassemble": "\(reassemble)"
+ "reassemble": "\(reassemble)",
])
-
+
// Remove tag parsing here, imageName is now passed directly
// let components = image.split(separator: ":") ...
// let imageTag = String(tag)
-
+
// Get authentication token only if not in dry-run mode
var token: String = ""
if !dryRun {
@@ -2706,17 +2532,17 @@ class ImageContainerRegistry: @unchecked Sendable {
} else {
Logger.info("Dry run mode: skipping authentication token request")
}
-
+
// Create working directory inside the VM folder for caching/resuming
let workDir = URL(fileURLWithPath: vmDirPath).appendingPathComponent(".lume_push_cache")
try FileManager.default.createDirectory(at: workDir, withIntermediateDirectories: true)
Logger.info("Using push cache directory: \(workDir.path)")
-
+
// Get VM files that need to be pushed using vmDirPath
let diskPath = URL(fileURLWithPath: vmDirPath).appendingPathComponent("disk.img")
let configPath = URL(fileURLWithPath: vmDirPath).appendingPathComponent("config.json")
let nvramPath = URL(fileURLWithPath: vmDirPath).appendingPathComponent("nvram.bin")
-
+
var layers: [OCIManifestLayer] = []
var uncompressedDiskSize: UInt64? = nil
@@ -2724,7 +2550,7 @@ class ImageContainerRegistry: @unchecked Sendable {
let cachedConfigPath = workDir.appendingPathComponent("config.json")
var configDigest: String? = nil
var configSize: Int? = nil
-
+
if FileManager.default.fileExists(atPath: cachedConfigPath.path) {
Logger.info("Using cached config.json")
do {
@@ -2734,7 +2560,8 @@ class ImageContainerRegistry: @unchecked Sendable {
// Try to get uncompressed disk size from cached config
if let vmConfig = try? JSONDecoder().decode(VMConfig.self, from: configData) {
uncompressedDiskSize = vmConfig.diskSize
- Logger.info("Found disk size in cached config: \(uncompressedDiskSize ?? 0) bytes")
+ Logger.info(
+ "Found disk size in cached config: \(uncompressedDiskSize ?? 0) bytes")
}
} catch {
Logger.error("Failed to read cached config.json: \(error). Will re-process.")
@@ -2745,20 +2572,22 @@ class ImageContainerRegistry: @unchecked Sendable {
let configData = try Data(contentsOf: configPath)
configDigest = "sha256:" + configData.sha256String()
configSize = configData.count
- try configData.write(to: cachedConfigPath) // Save to cache
+ try configData.write(to: cachedConfigPath) // Save to cache
// Try to get uncompressed disk size from original config
if let vmConfig = try? JSONDecoder().decode(VMConfig.self, from: configData) {
uncompressedDiskSize = vmConfig.diskSize
Logger.info("Found disk size in config: \(uncompressedDiskSize ?? 0) bytes")
}
}
-
- if var digest = configDigest, let size = configSize { // Use 'var' to modify if uploaded
- if !dryRun {
+
+ if var digest = configDigest, let size = configSize { // Use 'var' to modify if uploaded
+ if !dryRun {
// Upload only if not in dry-run mode and blob doesn't exist
- if !(try await blobExists(repository: "\(self.organization)/\(imageName)", digest: digest, token: token)) {
+ if !(try await blobExists(
+ repository: "\(self.organization)/\(imageName)", digest: digest, token: token))
+ {
Logger.info("Uploading config.json blob")
- let configData = try Data(contentsOf: cachedConfigPath) // Read from cache for upload
+ let configData = try Data(contentsOf: cachedConfigPath) // Read from cache for upload
digest = try await uploadBlobFromData(
repository: "\(self.organization)/\(imageName)",
data: configData,
@@ -2769,13 +2598,14 @@ class ImageContainerRegistry: @unchecked Sendable {
}
}
// Add config layer
- layers.append(OCIManifestLayer(
- mediaType: "application/vnd.oci.image.config.v1+json",
- size: size,
- digest: digest
- ))
+ layers.append(
+ OCIManifestLayer(
+ mediaType: "application/vnd.oci.image.config.v1+json",
+ size: size,
+ digest: digest
+ ))
}
-
+
// Process nvram.bin
let cachedNvramPath = workDir.appendingPathComponent("nvram.bin")
var nvramDigest: String? = nil
@@ -2788,47 +2618,56 @@ class ImageContainerRegistry: @unchecked Sendable {
nvramDigest = "sha256:" + nvramData.sha256String()
nvramSize = nvramData.count
} catch {
- Logger.error("Failed to read cached nvram.bin: \(error). Will re-process.")
+ Logger.error("Failed to read cached nvram.bin: \(error). Will re-process.")
}
} else if FileManager.default.fileExists(atPath: nvramPath.path) {
Logger.info("Processing nvram.bin")
let nvramData = try Data(contentsOf: nvramPath)
nvramDigest = "sha256:" + nvramData.sha256String()
nvramSize = nvramData.count
- try nvramData.write(to: cachedNvramPath) // Save to cache
+ try nvramData.write(to: cachedNvramPath) // Save to cache
}
-
- if var digest = nvramDigest, let size = nvramSize { // Use 'var'
- if !dryRun {
- // Upload only if not in dry-run mode and blob doesn't exist
- if !(try await blobExists(repository: "\(self.organization)/\(imageName)", digest: digest, token: token)) {
+
+ if var digest = nvramDigest, let size = nvramSize { // Use 'var'
+ if !dryRun {
+ // Upload only if not in dry-run mode and blob doesn't exist
+ if !(try await blobExists(
+ repository: "\(self.organization)/\(imageName)", digest: digest, token: token))
+ {
Logger.info("Uploading nvram.bin blob")
- let nvramData = try Data(contentsOf: cachedNvramPath) // Read from cache
+ let nvramData = try Data(contentsOf: cachedNvramPath) // Read from cache
digest = try await uploadBlobFromData(
repository: "\(self.organization)/\(imageName)",
data: nvramData,
token: token
)
} else {
- Logger.info("NVRAM blob already exists on registry")
+ Logger.info("NVRAM blob already exists on registry")
}
}
// Add nvram layer
- layers.append(OCIManifestLayer(
- mediaType: "application/octet-stream",
- size: size,
- digest: digest
- ))
+ layers.append(
+ OCIManifestLayer(
+ mediaType: "application/octet-stream",
+ size: size,
+ digest: digest
+ ))
}
-
+
// Process disk.img
if FileManager.default.fileExists(atPath: diskPath.path) {
let diskAttributes = try FileManager.default.attributesOfItem(atPath: diskPath.path)
let diskSize = diskAttributes[.size] as? UInt64 ?? 0
let actualDiskSize = uncompressedDiskSize ?? diskSize
- Logger.info("Processing disk.img in chunks", metadata: ["disk_path": diskPath.path, "disk_size": "\(diskSize) bytes", "actual_size": "\(actualDiskSize) bytes", "chunk_size": "\(chunkSizeMb)MB"])
+ Logger.info(
+ "Processing disk.img in chunks",
+ metadata: [
+ "disk_path": diskPath.path, "disk_size": "\(diskSize) bytes",
+ "actual_size": "\(actualDiskSize) bytes", "chunk_size": "\(chunkSizeMb)MB",
+ ])
let chunksDir = workDir.appendingPathComponent("disk.img.parts")
- try FileManager.default.createDirectory(at: chunksDir, withIntermediateDirectories: true)
+ try FileManager.default.createDirectory(
+ at: chunksDir, withIntermediateDirectories: true)
let chunkSizeBytes = chunkSizeMb * 1024 * 1024
let totalChunks = Int((diskSize + UInt64(chunkSizeBytes) - 1) / UInt64(chunkSizeBytes))
Logger.info("Splitting disk into \(totalChunks) chunks")
@@ -2836,58 +2675,125 @@ class ImageContainerRegistry: @unchecked Sendable {
defer { try? fileHandle.close() }
var pushedDiskLayers: [(index: Int, layer: OCIManifestLayer)] = []
var diskChunks: [(index: Int, path: URL, digest: String)] = []
-
- try await withThrowingTaskGroup(of: (Int, OCIManifestLayer, URL, String).self) { group in
+
+ try await withThrowingTaskGroup(of: (Int, OCIManifestLayer, URL, String).self) {
+ group in
let maxConcurrency = 4
for chunkIndex in 0..<totalChunks {
- if chunkIndex >= maxConcurrency { if let res = try await group.next() { pushedDiskLayers.append((res.0, res.1)); diskChunks.append((res.0, res.2, res.3)) } }
+ if chunkIndex >= maxConcurrency {
+ if let res = try await group.next() {
+ pushedDiskLayers.append((res.0, res.1))
+ diskChunks.append((res.0, res.2, res.3))
+ }
+ }
group.addTask { [token, verbose, dryRun, organization, imageName] in
let chunkIndex = chunkIndex
let chunkPath = chunksDir.appendingPathComponent("chunk.\(chunkIndex)")
- let metadataPath = chunksDir.appendingPathComponent("chunk_metadata.\(chunkIndex).json")
+ let metadataPath = chunksDir.appendingPathComponent(
+ "chunk_metadata.\(chunkIndex).json")
var layer: OCIManifestLayer? = nil
var finalCompressedDigest: String? = nil
- if FileManager.default.fileExists(atPath: metadataPath.path), FileManager.default.fileExists(atPath: chunkPath.path) {
+ if FileManager.default.fileExists(atPath: metadataPath.path),
+ FileManager.default.fileExists(atPath: chunkPath.path)
+ {
do {
let metadataData = try Data(contentsOf: metadataPath)
- let metadata = try JSONDecoder().decode(ChunkMetadata.self, from: metadataData)
- Logger.info("Resuming chunk \(chunkIndex + 1)/\(totalChunks) from cache")
+ let metadata = try JSONDecoder().decode(
+ ChunkMetadata.self, from: metadataData)
+ Logger.info(
+ "Resuming chunk \(chunkIndex + 1)/\(totalChunks) from cache")
finalCompressedDigest = metadata.compressedDigest
- if !dryRun { if !(try await self.blobExists(repository: "\(organization)/\(imageName)", digest: metadata.compressedDigest, token: token)) { Logger.info("Uploading cached chunk \(chunkIndex + 1) blob"); _ = try await self.uploadBlobFromPath(repository: "\(organization)/\(imageName)", path: chunkPath, digest: metadata.compressedDigest, token: token) } else { Logger.info("Chunk \(chunkIndex + 1) blob already exists on registry") } }
- layer = OCIManifestLayer(mediaType: "application/octet-stream+lz4", size: metadata.compressedSize, digest: metadata.compressedDigest, uncompressedSize: metadata.uncompressedSize, uncompressedContentDigest: metadata.uncompressedDigest)
- } catch { Logger.info("Failed to load cached metadata/chunk for index \(chunkIndex): \(error). Re-processing."); finalCompressedDigest = nil; layer = nil }
+ if !dryRun {
+ if !(try await self.blobExists(
+ repository: "\(organization)/\(imageName)",
+ digest: metadata.compressedDigest, token: token))
+ {
+ Logger.info("Uploading cached chunk \(chunkIndex + 1) blob")
+ _ = try await self.uploadBlobFromPath(
+ repository: "\(organization)/\(imageName)",
+ path: chunkPath, digest: metadata.compressedDigest,
+ token: token)
+ } else {
+ Logger.info(
+ "Chunk \(chunkIndex + 1) blob already exists on registry"
+ )
+ }
+ }
+ layer = OCIManifestLayer(
+ mediaType: "application/octet-stream+lz4",
+ size: metadata.compressedSize,
+ digest: metadata.compressedDigest,
+ uncompressedSize: metadata.uncompressedSize,
+ uncompressedContentDigest: metadata.uncompressedDigest)
+ } catch {
+ Logger.info(
+ "Failed to load cached metadata/chunk for index \(chunkIndex): \(error). Re-processing."
+ )
+ finalCompressedDigest = nil
+ layer = nil
+ }
}
if layer == nil {
Logger.info("Processing chunk \(chunkIndex + 1)/\(totalChunks)")
let localFileHandle = try FileHandle(forReadingFrom: diskPath)
defer { try? localFileHandle.close() }
try localFileHandle.seek(toOffset: UInt64(chunkIndex * chunkSizeBytes))
- let chunkData = try localFileHandle.read(upToCount: chunkSizeBytes) ?? Data()
+ let chunkData =
+ try localFileHandle.read(upToCount: chunkSizeBytes) ?? Data()
let uncompressedSize = UInt64(chunkData.count)
let uncompressedDigest = "sha256:" + chunkData.sha256String()
- let compressedData = try (chunkData as NSData).compressed(using: .lz4) as Data
+ let compressedData =
+ try (chunkData as NSData).compressed(using: .lz4) as Data
let compressedSize = compressedData.count
let compressedDigest = "sha256:" + compressedData.sha256String()
try compressedData.write(to: chunkPath)
- let metadata = ChunkMetadata(uncompressedDigest: uncompressedDigest, uncompressedSize: uncompressedSize, compressedDigest: compressedDigest, compressedSize: compressedSize)
+ let metadata = ChunkMetadata(
+ uncompressedDigest: uncompressedDigest,
+ uncompressedSize: uncompressedSize,
+ compressedDigest: compressedDigest, compressedSize: compressedSize)
let metadataData = try JSONEncoder().encode(metadata)
try metadataData.write(to: metadataPath)
finalCompressedDigest = compressedDigest
- if !dryRun { if !(try await self.blobExists(repository: "\(organization)/\(imageName)", digest: compressedDigest, token: token)) { Logger.info("Uploading processed chunk \(chunkIndex + 1) blob"); _ = try await self.uploadBlobFromPath(repository: "\(organization)/\(imageName)", path: chunkPath, digest: compressedDigest, token: token) } else { Logger.info("Chunk \(chunkIndex + 1) blob already exists on registry (processed fresh)") } }
- layer = OCIManifestLayer(mediaType: "application/octet-stream+lz4", size: compressedSize, digest: compressedDigest, uncompressedSize: uncompressedSize, uncompressedContentDigest: uncompressedDigest)
+ if !dryRun {
+ if !(try await self.blobExists(
+ repository: "\(organization)/\(imageName)",
+ digest: compressedDigest, token: token))
+ {
+ Logger.info("Uploading processed chunk \(chunkIndex + 1) blob")
+ _ = try await self.uploadBlobFromPath(
+ repository: "\(organization)/\(imageName)", path: chunkPath,
+ digest: compressedDigest, token: token)
+ } else {
+ Logger.info(
+ "Chunk \(chunkIndex + 1) blob already exists on registry (processed fresh)"
+ )
+ }
+ }
+ layer = OCIManifestLayer(
+ mediaType: "application/octet-stream+lz4", size: compressedSize,
+ digest: compressedDigest, uncompressedSize: uncompressedSize,
+ uncompressedContentDigest: uncompressedDigest)
+ }
+ guard let finalLayer = layer, let finalDigest = finalCompressedDigest else {
+ throw PushError.blobUploadFailed
+ }
+ if verbose {
+ Logger.info("Finished chunk \(chunkIndex + 1)/\(totalChunks)")
}
- guard let finalLayer = layer, let finalDigest = finalCompressedDigest else { throw PushError.blobUploadFailed }
- if verbose { Logger.info("Finished chunk \(chunkIndex + 1)/\(totalChunks)") }
return (chunkIndex, finalLayer, chunkPath, finalDigest)
}
}
- for try await (index, layer, path, digest) in group { pushedDiskLayers.append((index, layer)); diskChunks.append((index, path, digest)) }
+ for try await (index, layer, path, digest) in group {
+ pushedDiskLayers.append((index, layer))
+ diskChunks.append((index, path, digest))
+ }
}
- layers.append(contentsOf: pushedDiskLayers.sorted { $0.index < $1.index }.map { $0.layer })
+ layers.append(
+ contentsOf: pushedDiskLayers.sorted { $0.index < $1.index }.map { $0.layer })
diskChunks.sort { $0.index < $1.index }
Logger.info("All disk chunks processed successfully")
- // --- Calculate Total Upload Size & Initialize Tracker ---
+ // --- Calculate Total Upload Size & Initialize Tracker ---
if !dryRun {
var totalUploadSizeBytes: Int64 = 0
var totalUploadFiles: Int = 0
@@ -2898,49 +2804,60 @@ class ImageContainerRegistry: @unchecked Sendable {
}
// Add nvram size if it exists
if let size = nvramSize {
- totalUploadSizeBytes += Int64(size)
- totalUploadFiles += 1
+ totalUploadSizeBytes += Int64(size)
+ totalUploadFiles += 1
}
// Add sizes of all compressed disk chunks
- let allChunkSizes = diskChunks.compactMap { try? FileManager.default.attributesOfItem(atPath: $0.path.path)[.size] as? Int64 ?? 0 }
+ let allChunkSizes = diskChunks.compactMap {
+ try? FileManager.default.attributesOfItem(atPath: $0.path.path)[.size] as? Int64
+ ?? 0
+ }
totalUploadSizeBytes += allChunkSizes.reduce(0, +)
- totalUploadFiles += totalChunks // Use totalChunks calculated earlier
-
+ totalUploadFiles += totalChunks // Use totalChunks calculated earlier
+
if totalUploadSizeBytes > 0 {
- Logger.info("Initializing upload progress: \(totalUploadFiles) files, total size: \(ByteCountFormatter.string(fromByteCount: totalUploadSizeBytes, countStyle: .file))")
+ Logger.info(
+ "Initializing upload progress: \(totalUploadFiles) files, total size: \(ByteCountFormatter.string(fromByteCount: totalUploadSizeBytes, countStyle: .file))"
+ )
await uploadProgress.setTotal(totalUploadSizeBytes, files: totalUploadFiles)
// Print initial progress bar
- print("[░░░░░░░░░░░░░░░░░░░░] 0% (0/\(totalUploadFiles)) | Initializing upload... | ETA: calculating... ")
- fflush(stdout)
- } else {
- Logger.info("No files marked for upload.")
- }
+ print(
+ "[░░░░░░░░░░░░░░░░░░░░] 0% (0/\(totalUploadFiles)) | Initializing upload... | ETA: calculating... "
+ )
+ fflush(stdout)
+ } else {
+ Logger.info("No files marked for upload.")
+ }
}
- // --- End Size Calculation & Init ---
+ // --- End Size Calculation & Init ---
// Perform reassembly verification if requested in dry-run mode
if dryRun && reassemble {
Logger.info("=== REASSEMBLY MODE ===")
Logger.info("Reassembling chunks to verify integrity...")
let reassemblyDir = workDir.appendingPathComponent("reassembly")
- try FileManager.default.createDirectory(at: reassemblyDir, withIntermediateDirectories: true)
+ try FileManager.default.createDirectory(
+ at: reassemblyDir, withIntermediateDirectories: true)
let reassembledFile = reassemblyDir.appendingPathComponent("reassembled_disk.img")
-
+
// Pre-allocate a sparse file with the correct size
- Logger.info("Pre-allocating sparse file of \(ByteCountFormatter.string(fromByteCount: Int64(actualDiskSize), countStyle: .file))...")
+ Logger.info(
+ "Pre-allocating sparse file of \(ByteCountFormatter.string(fromByteCount: Int64(actualDiskSize), countStyle: .file))..."
+ )
if FileManager.default.fileExists(atPath: reassembledFile.path) {
try FileManager.default.removeItem(at: reassembledFile)
}
- guard FileManager.default.createFile(atPath: reassembledFile.path, contents: nil) else {
+ guard FileManager.default.createFile(atPath: reassembledFile.path, contents: nil)
+ else {
throw PushError.fileCreationFailed(reassembledFile.path)
}
-
+
let outputHandle = try FileHandle(forWritingTo: reassembledFile)
defer { try? outputHandle.close() }
-
+
// Set the file size without writing data (creates a sparse file)
try outputHandle.truncate(atOffset: actualDiskSize)
-
+
// Add test patterns at start and end to verify writability
let testPattern = "LUME_TEST_PATTERN".data(using: .utf8)!
try outputHandle.seek(toOffset: 0)
@@ -2948,217 +2865,266 @@ class ImageContainerRegistry: @unchecked Sendable {
try outputHandle.seek(toOffset: actualDiskSize - UInt64(testPattern.count))
try outputHandle.write(contentsOf: testPattern)
try outputHandle.synchronize()
-
+
Logger.info("Test patterns written to sparse file. File is ready for writing.")
-
+
// Track reassembly progress
var reassemblyProgressLogger = ProgressLogger(threshold: 0.05)
var currentOffset: UInt64 = 0
-
+
// Process each chunk in order
for (index, cachedChunkPath, _) in diskChunks.sorted(by: { $0.index < $1.index }) {
- Logger.info("Decompressing & writing part \(index + 1)/\(diskChunks.count): \(cachedChunkPath.lastPathComponent) at offset \(currentOffset)...")
-
+ Logger.info(
+ "Decompressing & writing part \(index + 1)/\(diskChunks.count): \(cachedChunkPath.lastPathComponent) at offset \(currentOffset)..."
+ )
+
// Always seek to the correct position
try outputHandle.seek(toOffset: currentOffset)
-
+
// Decompress and write the chunk
let decompressedBytesWritten = try decompressChunkAndWriteSparse(
inputPath: cachedChunkPath.path,
outputHandle: outputHandle,
startOffset: currentOffset
)
-
+
currentOffset += decompressedBytesWritten
reassemblyProgressLogger.logProgress(
current: Double(currentOffset) / Double(actualDiskSize),
context: "Reassembling"
)
-
+
// Ensure data is written before processing next part
try outputHandle.synchronize()
}
-
+
// Finalize progress
reassemblyProgressLogger.logProgress(current: 1.0, context: "Reassembly Complete")
Logger.info("") // Newline
-
+
// Close handle before post-processing
try outputHandle.close()
-
+
// Optimize sparseness if on macOS
let optimizedFile = reassemblyDir.appendingPathComponent("optimized_disk.img")
if FileManager.default.fileExists(atPath: "/bin/cp") {
Logger.info("Optimizing sparse file representation...")
-
+
let process = Process()
process.executableURL = URL(fileURLWithPath: "/bin/cp")
process.arguments = ["-c", reassembledFile.path, optimizedFile.path]
-
+
do {
try process.run()
process.waitUntilExit()
-
+
if process.terminationStatus == 0 {
// Get sizes of original and optimized files
- let optimizedSize = (try? FileManager.default.attributesOfItem(atPath: optimizedFile.path)[.size] as? UInt64) ?? 0
+ let optimizedSize =
+ (try? FileManager.default.attributesOfItem(
+ atPath: optimizedFile.path)[.size] as? UInt64) ?? 0
let originalUsage = getActualDiskUsage(path: reassembledFile.path)
let optimizedUsage = getActualDiskUsage(path: optimizedFile.path)
-
+
Logger.info(
"Sparse optimization results: Before: \(ByteCountFormatter.string(fromByteCount: Int64(originalUsage), countStyle: .file)) actual usage, After: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedUsage), countStyle: .file)) actual usage (Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(optimizedSize), countStyle: .file)))"
)
-
+
// Replace original with optimized version
try FileManager.default.removeItem(at: reassembledFile)
try FileManager.default.moveItem(at: optimizedFile, to: reassembledFile)
Logger.info("Using sparse-optimized file for verification")
} else {
- Logger.info("Sparse optimization failed, using original file for verification")
+ Logger.info(
+ "Sparse optimization failed, using original file for verification")
try? FileManager.default.removeItem(at: optimizedFile)
}
} catch {
- Logger.info("Error during sparse optimization: \(error.localizedDescription)")
+ Logger.info(
+ "Error during sparse optimization: \(error.localizedDescription)")
try? FileManager.default.removeItem(at: optimizedFile)
}
}
-
+
// Verification step
Logger.info("Verifying reassembled file...")
let originalSize = diskSize
let originalDigest = calculateSHA256(filePath: diskPath.path)
- let reassembledAttributes = try FileManager.default.attributesOfItem(atPath: reassembledFile.path)
+ let reassembledAttributes = try FileManager.default.attributesOfItem(
+ atPath: reassembledFile.path)
let reassembledSize = reassembledAttributes[.size] as? UInt64 ?? 0
let reassembledDigest = calculateSHA256(filePath: reassembledFile.path)
-
+
// Check actual disk usage
let originalActualSize = getActualDiskUsage(path: diskPath.path)
let reassembledActualSize = getActualDiskUsage(path: reassembledFile.path)
-
+
// Report results
Logger.info("Results:")
- Logger.info(" Original size: \(ByteCountFormatter.string(fromByteCount: Int64(originalSize), countStyle: .file)) (\(originalSize) bytes)")
- Logger.info(" Reassembled size: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledSize), countStyle: .file)) (\(reassembledSize) bytes)")
+ Logger.info(
+ " Original size: \(ByteCountFormatter.string(fromByteCount: Int64(originalSize), countStyle: .file)) (\(originalSize) bytes)"
+ )
+ Logger.info(
+ " Reassembled size: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledSize), countStyle: .file)) (\(reassembledSize) bytes)"
+ )
Logger.info(" Original digest: \(originalDigest)")
Logger.info(" Reassembled digest: \(reassembledDigest)")
- Logger.info(" Original: Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(originalSize), countStyle: .file)), Actual disk usage: \(ByteCountFormatter.string(fromByteCount: Int64(originalActualSize), countStyle: .file))")
- Logger.info(" Reassembled: Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledSize), countStyle: .file)), Actual disk usage: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledActualSize), countStyle: .file))")
-
+ Logger.info(
+ " Original: Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(originalSize), countStyle: .file)), Actual disk usage: \(ByteCountFormatter.string(fromByteCount: Int64(originalActualSize), countStyle: .file))"
+ )
+ Logger.info(
+ " Reassembled: Apparent size: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledSize), countStyle: .file)), Actual disk usage: \(ByteCountFormatter.string(fromByteCount: Int64(reassembledActualSize), countStyle: .file))"
+ )
+
// Determine if verification was successful
if originalDigest == reassembledDigest {
Logger.info("✅ VERIFICATION SUCCESSFUL: Files are identical")
} else {
Logger.info("❌ VERIFICATION FAILED: Files differ")
-
+
if originalSize != reassembledSize {
- Logger.info(" Size mismatch: Original \(originalSize) bytes, Reassembled \(reassembledSize) bytes")
+ Logger.info(
+ " Size mismatch: Original \(originalSize) bytes, Reassembled \(reassembledSize) bytes"
+ )
}
-
+
// Check sparse file characteristics
Logger.info("Attempting to identify differences...")
- Logger.info("NOTE: This might be a sparse file issue. The content may be identical, but sparse regions")
- Logger.info(" may be handled differently between the original and reassembled files.")
-
+ Logger.info(
+ "NOTE: This might be a sparse file issue. The content may be identical, but sparse regions"
+ )
+ Logger.info(
+ " may be handled differently between the original and reassembled files."
+ )
+
if originalActualSize > 0 {
- let diffPercentage = ((Double(reassembledActualSize) - Double(originalActualSize)) / Double(originalActualSize)) * 100.0
- Logger.info(" Disk usage difference: \(String(format: "%.2f", diffPercentage))%")
-
+ let diffPercentage =
+ ((Double(reassembledActualSize) - Double(originalActualSize))
+ / Double(originalActualSize)) * 100.0
+ Logger.info(
+ " Disk usage difference: \(String(format: "%.2f", diffPercentage))%")
+
if diffPercentage < -40 {
- Logger.info(" ⚠️ WARNING: Reassembled disk uses significantly less space (>40% difference).")
- Logger.info(" This indicates sparse regions weren't properly preserved and may affect VM functionality.")
+ Logger.info(
+ " ⚠️ WARNING: Reassembled disk uses significantly less space (>40% difference)."
+ )
+ Logger.info(
+ " This indicates sparse regions weren't properly preserved and may affect VM functionality."
+ )
} else if diffPercentage < -10 {
- Logger.info(" ⚠️ WARNING: Reassembled disk uses less space (10-40% difference).")
- Logger.info(" Some sparse regions may not be properly preserved but VM might still function correctly.")
+ Logger.info(
+ " ⚠️ WARNING: Reassembled disk uses less space (10-40% difference)."
+ )
+ Logger.info(
+ " Some sparse regions may not be properly preserved but VM might still function correctly."
+ )
} else if diffPercentage > 10 {
- Logger.info(" ⚠️ WARNING: Reassembled disk uses more space (>10% difference).")
- Logger.info(" This is unusual and may indicate improper sparse file handling.")
+ Logger.info(
+ " ⚠️ WARNING: Reassembled disk uses more space (>10% difference).")
+ Logger.info(
+ " This is unusual and may indicate improper sparse file handling.")
} else {
- Logger.info(" ✓ Disk usage difference is minimal (<10%). VM likely to function correctly.")
+ Logger.info(
+ " ✓ Disk usage difference is minimal (<10%). VM likely to function correctly."
+ )
}
}
-
+
// Offer recovery option
if originalDigest != reassembledDigest {
Logger.info("")
Logger.info("===== ATTEMPTING RECOVERY ACTION =====")
- Logger.info("Since verification failed, trying direct copy as a fallback method.")
-
+ Logger.info(
+ "Since verification failed, trying direct copy as a fallback method.")
+
let fallbackFile = reassemblyDir.appendingPathComponent("fallback_disk.img")
Logger.info("Creating fallback disk image at: \(fallbackFile.path)")
-
+
// Try rsync first
let rsyncProcess = Process()
rsyncProcess.executableURL = URL(fileURLWithPath: "/usr/bin/rsync")
- rsyncProcess.arguments = ["-aS", "--progress", diskPath.path, fallbackFile.path]
-
+ rsyncProcess.arguments = [
+ "-aS", "--progress", diskPath.path, fallbackFile.path,
+ ]
+
do {
try rsyncProcess.run()
rsyncProcess.waitUntilExit()
-
+
if rsyncProcess.terminationStatus == 0 {
- Logger.info("Direct copy completed with rsync. Fallback image available at: \(fallbackFile.path)")
+ Logger.info(
+ "Direct copy completed with rsync. Fallback image available at: \(fallbackFile.path)"
+ )
} else {
// Try cp -c as fallback
Logger.info("Rsync failed. Attempting with cp -c command...")
let cpProcess = Process()
cpProcess.executableURL = URL(fileURLWithPath: "/bin/cp")
cpProcess.arguments = ["-c", diskPath.path, fallbackFile.path]
-
+
try cpProcess.run()
cpProcess.waitUntilExit()
-
+
if cpProcess.terminationStatus == 0 {
- Logger.info("Direct copy completed with cp -c. Fallback image available at: \(fallbackFile.path)")
+ Logger.info(
+ "Direct copy completed with cp -c. Fallback image available at: \(fallbackFile.path)"
+ )
} else {
Logger.info("All recovery attempts failed.")
}
}
} catch {
- Logger.info("Error during recovery attempts: \(error.localizedDescription)")
+ Logger.info(
+ "Error during recovery attempts: \(error.localizedDescription)")
Logger.info("All recovery attempts failed.")
}
}
}
-
+
Logger.info("Reassembled file is available at: \(reassembledFile.path)")
}
}
- // --- Manifest Creation & Push ---
+ // --- Manifest Creation & Push ---
let manifest = createManifest(
layers: layers,
- configLayerIndex: layers.firstIndex(where: { $0.mediaType == "application/vnd.oci.image.config.v1+json" }),
+ configLayerIndex: layers.firstIndex(where: {
+ $0.mediaType == "application/vnd.oci.image.config.v1+json"
+ }),
uncompressedDiskSize: uncompressedDiskSize
)
// Push manifest only if not in dry-run mode
if !dryRun {
- Logger.info("Pushing manifest(s)") // Updated log
+ Logger.info("Pushing manifest(s)") // Updated log
// Serialize the manifest dictionary to Data first
- let manifestData = try JSONSerialization.data(withJSONObject: manifest, options: [.prettyPrinted, .sortedKeys])
+ let manifestData = try JSONSerialization.data(
+ withJSONObject: manifest, options: [.prettyPrinted, .sortedKeys])
// Loop through tags to push the same manifest data
for tag in tags {
- Logger.info("Pushing manifest for tag: \(tag)")
- try await pushManifest(
- repository: "\(self.organization)/\(imageName)",
- tag: tag, // Use the current tag from the loop
- manifest: manifestData, // Pass the serialized Data
- token: token // Token should be in scope here now
- )
+ Logger.info("Pushing manifest for tag: \(tag)")
+ try await pushManifest(
+ repository: "\(self.organization)/\(imageName)",
+ tag: tag, // Use the current tag from the loop
+ manifest: manifestData, // Pass the serialized Data
+ token: token // Token should be in scope here now
+ )
}
}
// Print final upload summary if not dry run
if !dryRun {
let stats = await uploadProgress.getUploadStats()
- Logger.info("\n\(stats.formattedSummary())") // Add newline for separation
+ Logger.info("\n\(stats.formattedSummary())") // Add newline for separation
}
// Clean up cache directory only on successful non-dry-run push
}
-
- private func createManifest(layers: [OCIManifestLayer], configLayerIndex: Int?, uncompressedDiskSize: UInt64?) -> [String: Any] {
+
+ private func createManifest(
+ layers: [OCIManifestLayer], configLayerIndex: Int?, uncompressedDiskSize: UInt64?
+ ) -> [String: Any] {
var manifest: [String: Any] = [
"schemaVersion": 2,
"mediaType": "application/vnd.oci.image.manifest.v1+json",
@@ -3166,221 +3132,244 @@ class ImageContainerRegistry: @unchecked Sendable {
var layerDict: [String: Any] = [
"mediaType": layer.mediaType,
"size": layer.size,
- "digest": layer.digest
+ "digest": layer.digest,
]
-
+
if let uncompressedSize = layer.uncompressedSize {
var annotations: [String: String] = [:]
- annotations["org.trycua.lume.uncompressed-size"] = "\(uncompressedSize)" // Updated prefix
-
+ annotations["org.trycua.lume.uncompressed-size"] = "\(uncompressedSize)" // Updated prefix
+
if let digest = layer.uncompressedContentDigest {
- annotations["org.trycua.lume.uncompressed-content-digest"] = digest // Updated prefix
+ annotations["org.trycua.lume.uncompressed-content-digest"] = digest // Updated prefix
}
-
+
layerDict["annotations"] = annotations
}
-
+
return layerDict
- }
+ },
]
-
+
// Add config reference if available
if let configIndex = configLayerIndex {
let configLayer = layers[configIndex]
manifest["config"] = [
"mediaType": configLayer.mediaType,
"size": configLayer.size,
- "digest": configLayer.digest
+ "digest": configLayer.digest,
]
}
-
+
// Add annotations
var annotations: [String: String] = [:]
- annotations["org.trycua.lume.upload-time"] = ISO8601DateFormatter().string(from: Date()) // Updated prefix
-
+ annotations["org.trycua.lume.upload-time"] = ISO8601DateFormatter().string(from: Date()) // Updated prefix
+
if let diskSize = uncompressedDiskSize {
- annotations["org.trycua.lume.uncompressed-disk-size"] = "\(diskSize)" // Updated prefix
+ annotations["org.trycua.lume.uncompressed-disk-size"] = "\(diskSize)" // Updated prefix
}
-
+
manifest["annotations"] = annotations
-
+
return manifest
}
-
- private func uploadBlobFromData(repository: String, data: Data, token: String) async throws -> String {
+
+ private func uploadBlobFromData(repository: String, data: Data, token: String) async throws
+ -> String
+ {
// Calculate digest
let digest = "sha256:" + data.sha256String()
-
+
// Check if blob already exists
if try await blobExists(repository: repository, digest: digest, token: token) {
Logger.info("Blob already exists: \(digest)")
return digest
}
-
+
// Initiate upload
let uploadURL = try await startBlobUpload(repository: repository, token: token)
-
+
// Upload blob
try await uploadBlob(url: uploadURL, data: data, digest: digest, token: token)
-
+
// Report progress
await uploadProgress.addProgress(Int64(data.count))
-
+
return digest
}
-
- private func uploadBlobFromPath(repository: String, path: URL, digest: String, token: String) async throws -> String {
+
+ private func uploadBlobFromPath(repository: String, path: URL, digest: String, token: String)
+ async throws -> String
+ {
// Check if blob already exists
if try await blobExists(repository: repository, digest: digest, token: token) {
Logger.info("Blob already exists: \(digest)")
return digest
}
-
+
// Initiate upload
let uploadURL = try await startBlobUpload(repository: repository, token: token)
-
+
// Load data from file
let data = try Data(contentsOf: path)
-
+
// Upload blob
try await uploadBlob(url: uploadURL, data: data, digest: digest, token: token)
-
+
// Report progress
await uploadProgress.addProgress(Int64(data.count))
-
+
return digest
}
-
- private func blobExists(repository: String, digest: String, token: String) async throws -> Bool {
+
+ private func blobExists(repository: String, digest: String, token: String) async throws -> Bool
+ {
let url = URL(string: "https://\(registry)/v2/\(repository)/blobs/\(digest)")!
var request = URLRequest(url: url)
request.httpMethod = "HEAD"
request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization")
-
+
let (_, response) = try await URLSession.shared.data(for: request)
-
+
if let httpResponse = response as? HTTPURLResponse {
return httpResponse.statusCode == 200
}
-
+
return false
}
-
+
private func startBlobUpload(repository: String, token: String) async throws -> URL {
let url = URL(string: "https://\(registry)/v2/\(repository)/blobs/uploads/")!
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization")
- request.setValue("0", forHTTPHeaderField: "Content-Length") // Explicitly set Content-Length to 0 for POST
-
+ request.setValue("0", forHTTPHeaderField: "Content-Length") // Explicitly set Content-Length to 0 for POST
+
let (_, response) = try await URLSession.shared.data(for: request)
-
- guard let httpResponse = response as? HTTPURLResponse,
- httpResponse.statusCode == 202,
- let locationString = httpResponse.value(forHTTPHeaderField: "Location") else {
+
+ guard let httpResponse = response as? HTTPURLResponse,
+ httpResponse.statusCode == 202,
+ let locationString = httpResponse.value(forHTTPHeaderField: "Location")
+ else {
// Log response details on failure
- let responseBody = String(data: (try? await URLSession.shared.data(for: request).0) ?? Data(), encoding: .utf8) ?? "(No Body)"
- Logger.error("Failed to initiate blob upload. Status: \( (response as? HTTPURLResponse)?.statusCode ?? 0 ). Headers: \( (response as? HTTPURLResponse)?.allHeaderFields ?? [:] ). Body: \(responseBody)")
+ let responseBody =
+ String(
+ data: (try? await URLSession.shared.data(for: request).0) ?? Data(),
+ encoding: .utf8) ?? "(No Body)"
+ Logger.error(
+ "Failed to initiate blob upload. Status: \( (response as? HTTPURLResponse)?.statusCode ?? 0 ). Headers: \( (response as? HTTPURLResponse)?.allHeaderFields ?? [:] ). Body: \(responseBody)"
+ )
throw PushError.uploadInitiationFailed
}
-
+
// Construct the base URL for the registry
guard let baseRegistryURL = URL(string: "https://\(registry)") else {
Logger.error("Failed to create base registry URL from: \(registry)")
- throw PushError.invalidURL
+ throw PushError.invalidURL
}
-
+
// Create the final upload URL, resolving the location against the base URL
guard let uploadURL = URL(string: locationString, relativeTo: baseRegistryURL) else {
- Logger.error("Failed to create absolute upload URL from location: \(locationString) relative to base: \(baseRegistryURL.absoluteString)")
+ Logger.error(
+ "Failed to create absolute upload URL from location: \(locationString) relative to base: \(baseRegistryURL.absoluteString)"
+ )
throw PushError.invalidURL
}
-
+
Logger.info("Blob upload initiated. Upload URL: \(uploadURL.absoluteString)")
- return uploadURL.absoluteURL // Ensure it's absolute
+ return uploadURL.absoluteURL // Ensure it's absolute
}
-
+
private func uploadBlob(url: URL, data: Data, digest: String, token: String) async throws {
var components = URLComponents(url: url, resolvingAgainstBaseURL: true)!
-
+
// Add digest parameter
var queryItems = components.queryItems ?? []
queryItems.append(URLQueryItem(name: "digest", value: digest))
components.queryItems = queryItems
-
+
guard let uploadURL = components.url else {
throw PushError.invalidURL
}
-
+
var request = URLRequest(url: uploadURL)
request.httpMethod = "PUT"
request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization")
request.setValue("application/octet-stream", forHTTPHeaderField: "Content-Type")
request.setValue("\(data.count)", forHTTPHeaderField: "Content-Length")
request.httpBody = data
-
+
let (_, response) = try await URLSession.shared.data(for: request)
-
+
guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 201 else {
throw PushError.blobUploadFailed
}
}
-
- private func pushManifest(repository: String, tag: String, manifest: Data, token: String) async throws {
+
+ private func pushManifest(repository: String, tag: String, manifest: Data, token: String)
+ async throws
+ {
let url = URL(string: "https://\(registry)/v2/\(repository)/manifests/\(tag)")!
var request = URLRequest(url: url)
request.httpMethod = "PUT"
request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization")
- request.setValue("application/vnd.oci.image.manifest.v1+json", forHTTPHeaderField: "Content-Type")
+ request.setValue(
+ "application/vnd.oci.image.manifest.v1+json", forHTTPHeaderField: "Content-Type")
request.httpBody = manifest
-
+
let (_, response) = try await URLSession.shared.data(for: request)
-
+
guard let httpResponse = response as? HTTPURLResponse, httpResponse.statusCode == 201 else {
throw PushError.manifestPushFailed
}
}
-
+
private func getCredentialsFromEnvironment() -> (String?, String?) {
- let username = ProcessInfo.processInfo.environment["GITHUB_USERNAME"] ??
- ProcessInfo.processInfo.environment["GHCR_USERNAME"]
- let password = ProcessInfo.processInfo.environment["GITHUB_TOKEN"] ??
- ProcessInfo.processInfo.environment["GHCR_TOKEN"]
+ let username =
+ ProcessInfo.processInfo.environment["GITHUB_USERNAME"]
+ ?? ProcessInfo.processInfo.environment["GHCR_USERNAME"]
+ let password =
+ ProcessInfo.processInfo.environment["GITHUB_TOKEN"]
+ ?? ProcessInfo.processInfo.environment["GHCR_TOKEN"]
return (username, password)
}
// Add these helper methods for dry-run and reassemble implementation
-
+
// NEW Helper function using Compression framework and sparse writing
- private func decompressChunkAndWriteSparse(inputPath: String, outputHandle: FileHandle, startOffset: UInt64) throws -> UInt64 {
+ private func decompressChunkAndWriteSparse(
+ inputPath: String, outputHandle: FileHandle, startOffset: UInt64
+ ) throws -> UInt64 {
guard FileManager.default.fileExists(atPath: inputPath) else {
Logger.error("Compressed chunk not found at: \(inputPath)")
- return 0 // Or throw an error
+ return 0 // Or throw an error
}
- let sourceData = try Data(contentsOf: URL(fileURLWithPath: inputPath), options: .alwaysMapped)
+ let sourceData = try Data(
+ contentsOf: URL(fileURLWithPath: inputPath), options: .alwaysMapped)
var currentWriteOffset = startOffset
var totalDecompressedBytes: UInt64 = 0
- var sourceReadOffset = 0 // Keep track of how much compressed data we've provided
+ var sourceReadOffset = 0 // Keep track of how much compressed data we've provided
// Use the initializer with the readingFrom closure
let filter = try InputFilter(.decompress, using: .lz4) { (length: Int) -> Data? in
let bytesAvailable = sourceData.count - sourceReadOffset
if bytesAvailable == 0 {
- return nil // No more data
+ return nil // No more data
}
let bytesToRead = min(length, bytesAvailable)
- let chunk = sourceData.subdata(in: sourceReadOffset ..< sourceReadOffset + bytesToRead)
+ let chunk = sourceData.subdata(in: sourceReadOffset..<sourceReadOffset + bytesToRead)
sourceReadOffset += bytesToRead
return chunk
}
// Process the decompressed output by reading from the filter
while let decompressedData = try filter.readData(ofLength: Self.holeGranularityBytes) {
- if decompressedData.isEmpty { break } // End of stream
+ if decompressedData.isEmpty { break } // End of stream
// Check if the chunk is all zeros
- if decompressedData.count == Self.holeGranularityBytes && decompressedData == Self.zeroChunk {
+ if decompressedData.count == Self.holeGranularityBytes
+ && decompressedData == Self.zeroChunk
+ {
// It's a zero chunk, just advance the offset, don't write
currentWriteOffset += UInt64(decompressedData.count)
} else {
@@ -3391,7 +3380,7 @@ class ImageContainerRegistry: @unchecked Sendable {
}
totalDecompressedBytes += UInt64(decompressedData.count)
}
-
+
// No explicit finalize needed when initialized with source data
return totalDecompressedBytes
@@ -3402,36 +3391,37 @@ class ImageContainerRegistry: @unchecked Sendable {
guard FileManager.default.fileExists(atPath: filePath) else {
return "file-not-found"
}
-
+
let process = Process()
process.executableURL = URL(fileURLWithPath: "/usr/bin/shasum")
process.arguments = ["-a", "256", filePath]
-
+
let outputPipe = Pipe()
process.standardOutput = outputPipe
-
+
do {
try process.run()
process.waitUntilExit()
-
+
if let data = try outputPipe.fileHandleForReading.readToEnd(),
- let output = String(data: data, encoding: .utf8) {
+ let output = String(data: data, encoding: .utf8)
+ {
return output.components(separatedBy: " ").first ?? "hash-calculation-failed"
}
} catch {
Logger.error("SHA256 calculation failed: \(error)")
}
-
+
return "hash-calculation-failed"
}
}
actor UploadProgressTracker {
private var totalBytes: Int64 = 0
- private var uploadedBytes: Int64 = 0 // Renamed
- private var progressLogger = ProgressLogger(threshold: 0.01)
- private var totalFiles: Int = 0 // Keep track of total items
- private var completedFiles: Int = 0 // Keep track of completed items
+ private var uploadedBytes: Int64 = 0 // Renamed
+ private var progressLogger = ProgressLogger(threshold: 0.01)
+ private var totalFiles: Int = 0 // Keep track of total items
+ private var completedFiles: Int = 0 // Keep track of completed items
// Upload speed tracking
private var startTime: Date = Date()
@@ -3450,8 +3440,8 @@ actor UploadProgressTracker {
totalFiles = files
startTime = Date()
lastUpdateTime = startTime
- uploadedBytes = 0 // Reset uploaded bytes
- completedFiles = 0 // Reset completed files
+ uploadedBytes = 0 // Reset uploaded bytes
+ completedFiles = 0 // Reset completed files
smoothedSpeed = 0
speedSamples = []
peakSpeed = 0
@@ -3460,14 +3450,15 @@ actor UploadProgressTracker {
func addProgress(_ bytes: Int64) {
uploadedBytes += bytes
- completedFiles += 1 // Increment completed files count
+ completedFiles += 1 // Increment completed files count
let now = Date()
let elapsed = now.timeIntervalSince(lastUpdateTime)
// Show first progress update immediately, then throttle updates
- let shouldUpdate = (uploadedBytes <= bytes) || (elapsed >= 0.5) || (completedFiles == totalFiles)
+ let shouldUpdate =
+ (uploadedBytes <= bytes) || (elapsed >= 0.5) || (completedFiles == totalFiles)
- if shouldUpdate && totalBytes > 0 { // Ensure totalBytes is set
+ if shouldUpdate && totalBytes > 0 { // Ensure totalBytes is set
let currentSpeed = Double(uploadedBytes - lastUpdateBytes) / max(elapsed, 0.001)
speedSamples.append(currentSpeed)
@@ -3479,14 +3470,17 @@ actor UploadProgressTracker {
peakSpeed = max(peakSpeed, currentSpeed)
// Apply exponential smoothing
- if smoothedSpeed == 0 { smoothedSpeed = currentSpeed }
- else { smoothedSpeed = speedSmoothing * currentSpeed + (1 - speedSmoothing) * smoothedSpeed }
+ if smoothedSpeed == 0 {
+ smoothedSpeed = currentSpeed
+ } else {
+ smoothedSpeed = speedSmoothing * currentSpeed + (1 - speedSmoothing) * smoothedSpeed
+ }
let recentAvgSpeed = calculateAverageSpeed()
let totalElapsed = now.timeIntervalSince(startTime)
let overallAvgSpeed = totalElapsed > 0 ? Double(uploadedBytes) / totalElapsed : 0
- let progress = totalBytes > 0 ? Double(uploadedBytes) / Double(totalBytes) : 1.0 // Avoid division by zero
+ let progress = totalBytes > 0 ? Double(uploadedBytes) / Double(totalBytes) : 1.0 // Avoid division by zero
logSpeedProgress(
current: progress,
currentSpeed: currentSpeed,
@@ -3494,7 +3488,7 @@ actor UploadProgressTracker {
smoothedSpeed: smoothedSpeed,
overallSpeed: overallAvgSpeed,
peakSpeed: peakSpeed,
- context: "Uploading Image" // Changed context
+ context: "Uploading Image" // Changed context
)
lastUpdateTime = now
@@ -3521,7 +3515,7 @@ actor UploadProgressTracker {
let avgSpeed = totalElapsedTime > 0 ? Double(uploadedBytes) / totalElapsedTime : 0
return UploadStats(
totalBytes: totalBytes,
- uploadedBytes: uploadedBytes, // Renamed
+ uploadedBytes: uploadedBytes, // Renamed
elapsedTime: totalElapsedTime,
averageSpeed: avgSpeed,
peakSpeed: peakSpeed
@@ -3546,10 +3540,10 @@ actor UploadProgressTracker {
let etaSeconds = speedForEta > 0 ? Double(remainingBytes) / speedForEta : 0
let etaStr = formatTimeRemaining(etaSeconds)
let progressBar = createProgressBar(progress: current)
- let fileProgress = "(\(completedFiles)/\(totalFiles))" // Add file count
+ let fileProgress = "(\(completedFiles)/\(totalFiles))" // Add file count
print(
- "\r\(progressBar) \(progressPercent)% \(fileProgress) | Speed: \(avgSpeedStr) (Avg) | ETA: \(etaStr) ", // Simplified output
+ "\r\(progressBar) \(progressPercent)% \(fileProgress) | Speed: \(avgSpeedStr) (Avg) | ETA: \(etaStr) ", // Simplified output
terminator: "")
fflush(stdout)
}
@@ -3566,7 +3560,10 @@ actor UploadProgressTracker {
let units = ["B/s", "KB/s", "MB/s", "GB/s"]
var speed = bytesPerSecond
var unitIndex = 0
- while speed > 1024 && unitIndex < units.count - 1 { speed /= 1024; unitIndex += 1 }
+ while speed > 1024 && unitIndex < units.count - 1 {
+ speed /= 1024
+ unitIndex += 1
+ }
return String(format: "%.1f %@", speed, units[unitIndex])
}
private func formatTimeRemaining(_ seconds: Double) -> String {
@@ -3574,8 +3571,10 @@ actor UploadProgressTracker {
let hours = Int(seconds) / 3600
let minutes = (Int(seconds) % 3600) / 60
let secs = Int(seconds) % 60
- if hours > 0 { return String(format: "%d:%02d:%02d", hours, minutes, secs) }
- else { return String(format: "%d:%02d", minutes, secs) }
+ if hours > 0 {
+ return String(format: "%d:%02d:%02d", hours, minutes, secs)
+ } else {
+ return String(format: "%d:%02d", minutes, secs)
+ }
}
}
-