Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

Remove push, pull old scripts

f-trycua8dfbb0c4a907
diff --git a/libs/lume/scripts/ghcr/pull-ghcr.sh b/libs/lume/scripts/ghcr/pull-ghcr.sh
deleted file mode 100755
index 8b10fae116..0000000000
--- a/libs/lume/scripts/ghcr/pull-ghcr.sh
+++ /dev/null
@@ -1,205 +0,0 @@
-#!/bin/bash
-
-# Exit immediately if a command exits with a non-zero status
-set -e
-
-# Default parameters
-organization=""
-image_name=""
-image_version=""
-target_folder_path=""
-
-# Parse the command line arguments
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --organization)
- organization="$2"
- shift 2
- ;;
- --image-name)
- image_name="$2"
- shift 2
- ;;
- --image-version)
- image_version="$2"
- shift 2
- ;;
- --target-folder-path)
- target_folder_path="$2"
- shift 2
- ;;
- --help)
- echo "Usage: $0 [options]"
- echo "Options:"
- echo " --organization <organization> : GitHub organization (required)"
- echo " --image-name <name> : Name of the image to pull (required)"
- echo " --image-version <version> : Version of the image to pull (required)"
- echo " --target-folder-path <path> : Path where to extract the files (required)"
- exit 0
- ;;
- *)
- echo "Unknown option: $1"
- exit 1
- ;;
- esac
-done
-
-# Ensure required arguments
-if [[ -z "$organization" || -z "$image_name" || -z "$image_version" || -z "$target_folder_path" ]]; then
- echo "Error: Missing required arguments. Use --help for usage."
- exit 1
-fi
-
-# Check and install required tools
-for tool in "jq" "pv" "parallel"; do
- if ! command -v "$tool" &> /dev/null; then
- echo "$tool is not installed. Installing using Homebrew..."
- if ! command -v brew &> /dev/null; then
- echo "Homebrew is not installed. Please install Homebrew first: https://brew.sh/"
- exit 1
- fi
- brew install "$tool"
- fi
-done
-
-# Create target folder if it doesn't exist
-mkdir -p "$target_folder_path"
-
-# Create a temporary directory for processing files
-work_dir=$(mktemp -d)
-echo "Working directory: $work_dir"
-trap 'rm -rf "$work_dir"' EXIT
-
-# Registry details
-REGISTRY="ghcr.io"
-REPOSITORY="$organization/$image_name"
-TAG="$image_version"
-
-# Get anonymous token
-echo "Getting authentication token..."
-curl -s "https://$REGISTRY/token?service=ghcr.io&scope=repository:$REPOSITORY:pull" -o "$work_dir/token.json"
-TOKEN=$(cat "$work_dir/token.json" | jq -r ".token")
-
-if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
- echo "Failed to obtain token"
- exit 1
-fi
-
-echo "Token obtained successfully"
-
-# Fetch manifest
-echo "Fetching manifest..."
-MANIFEST_RESPONSE=$(curl -s \
- -H "Authorization: Bearer $TOKEN" \
- -H "Accept: application/vnd.oci.image.manifest.v1+json" \
- "https://$REGISTRY/v2/$REPOSITORY/manifests/$TAG")
-
-echo "Processing manifest..."
-
-# Create a directory for all files
-cd "$work_dir"
-
-# Create a download function for parallel execution
-download_layer() {
- local media_type="$1"
- local digest="$2"
- local output_file="$3"
-
- echo "Downloading $output_file..."
- curl -s -L \
- -H "Authorization: Bearer $TOKEN" \
- -H "Accept: $media_type" \
- "https://$REGISTRY/v2/$REPOSITORY/blobs/$digest" | \
- pv > "$output_file"
-}
-export -f download_layer
-export TOKEN REGISTRY REPOSITORY
-
-# Process layers and create download jobs
-echo "$MANIFEST_RESPONSE" | jq -c '.layers[]' | while read -r layer; do
- media_type=$(echo "$layer" | jq -r '.mediaType')
- digest=$(echo "$layer" | jq -r '.digest')
-
- # Skip empty layers
- if [[ "$media_type" == "application/vnd.oci.empty.v1+json" ]]; then
- continue
- fi
-
- # Extract part information if present
- if [[ $media_type =~ part\.number=([0-9]+)\;part\.total=([0-9]+) ]]; then
- part_num="${BASH_REMATCH[1]}"
- total_parts="${BASH_REMATCH[2]}"
- echo "Found part $part_num of $total_parts"
- output_file="disk.img.part.$part_num"
- else
- case "$media_type" in
- "application/vnd.oci.image.layer.v1.tar")
- output_file="disk.img"
- ;;
- "application/vnd.oci.image.config.v1+json")
- output_file="config.json"
- ;;
- "application/octet-stream")
- output_file="nvram.bin"
- ;;
- *)
- echo "Unknown media type: $media_type"
- continue
- ;;
- esac
- fi
-
- # Add to download queue
- echo "$media_type"$'\t'"$digest"$'\t'"$output_file" >> download_queue.txt
-done
-
-# Download all files in parallel
-echo "Downloading files in parallel..."
-parallel --colsep $'\t' -a download_queue.txt download_layer {1} {2} {3}
-
-# Check if we have disk parts to reassemble
-if ls disk.img.part.* 1> /dev/null 2>&1; then
- echo "Found disk parts, reassembling..."
-
- # Get total parts from the first part's filename
- first_part=$(ls disk.img.part.* | head -n 1)
- total_parts=$(echo "$MANIFEST_RESPONSE" | jq -r '.layers[] | select(.mediaType | contains("part.total")) | .mediaType' | grep -o 'part\.total=[0-9]*' | cut -d= -f2 | head -n 1)
-
- echo "Total parts to reassemble: $total_parts"
-
- # Concatenate parts in order
- echo "Reassembling disk image..."
- {
- for i in $(seq 1 "$total_parts"); do
- part_file="disk.img.part.$i"
- if [ -f "$part_file" ]; then
- cat "$part_file"
- else
- echo "Error: Missing part $i"
- exit 1
- fi
- done
- } | pv > "$target_folder_path/disk.img"
-
- echo "Disk image reassembled successfully"
-else
- # If no parts found, just copy disk.img if it exists
- if [ -f disk.img ]; then
- echo "Copying disk image..."
- pv disk.img > "$target_folder_path/disk.img"
- fi
-fi
-
-# Copy config.json if it exists
-if [ -f config.json ]; then
- echo "Copying config.json..."
- cp config.json "$target_folder_path/"
-fi
-
-# Copy nvram.bin if it exists
-if [ -f nvram.bin ]; then
- echo "Copying nvram.bin..."
- cp nvram.bin "$target_folder_path/"
-fi
-
-echo "Download complete: Files extracted to $target_folder_path"
\ No newline at end of file
diff --git a/libs/lume/scripts/ghcr/push-ghcr.sh b/libs/lume/scripts/ghcr/push-ghcr.sh
deleted file mode 100755
index d279be6654..0000000000
--- a/libs/lume/scripts/ghcr/push-ghcr.sh
+++ /dev/null
@@ -1,745 +0,0 @@
-#!/bin/bash
-
-# Exit immediately if a command exits with a non-zero status
-set -e
-
-# Default parameters
-organization=""
-folder_path=""
-image_name=""
-image_versions=""
-chunk_size="512M" # Default chunk size for splitting large files
-dry_run=true # Default: actually push to registry
-reassemble=true # Default: don't reassemble in dry-run mode
-# Define the OCI media type for the compressed disk layer
-oci_layer_media_type="application/octet-stream+lz4" # LZ4 compression format
-
-# Parse the command line arguments
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --organization)
- organization="$2"
- shift 2
- ;;
- --folder-path)
- folder_path="$2"
- shift 2
- ;;
- --image-name)
- image_name="$2"
- shift 2
- ;;
- --image-versions)
- image_versions="$2"
- shift 2
- ;;
- --chunk-size)
- chunk_size="$2"
- shift 2
- ;;
- --dry-run)
- dry_run=true
- shift 1
- ;;
- --reassemble)
- reassemble=true
- dry_run=true # Reassemble implies dry-run
- shift 1
- ;;
- --help)
- echo "Usage: $0 [options]"
- echo "Options:"
- echo " --organization <organization> : GitHub organization (required if not using token)"
- echo " --folder-path <path> : Path to the folder to upload (required)"
- echo " --image-name <name> : Name of the image to publish (required)"
- echo " --image-versions <versions> : Comma separated list of versions of the image to publish (required)"
- echo " --chunk-size <size> : Size of chunks for large files (e.g., 512M, default: 512M)"
- echo " --dry-run : Prepare files but don't upload to registry"
- echo " --reassemble : In dry-run mode, also reassemble chunks to verify integrity"
- echo "Note: The script will automatically resume from the last attempt if available"
- exit 0
- ;;
- *)
- echo "Unknown option: $1"
- exit 1
- ;;
- esac
-done
-
-# Ensure required arguments
-if [[ -z "$folder_path" ]]; then
- echo "Error: Missing required folder-path argument. Use --help for usage."
- exit 1
-fi
-
-# Only check organization and other push parameters if not in dry-run mode
-if [[ "$dry_run" = false ]]; then
- if [[ -z "$organization" || -z "$image_name" || -z "$image_versions" ]]; then
- echo "Error: Missing required arguments for push. Use --help for usage."
- exit 1
- fi
-
- # Check if the GITHUB_TOKEN variable is set
- if [[ -z "$GITHUB_TOKEN" ]]; then
- echo "Error: GITHUB_TOKEN is not set."
- exit 1
- fi
-fi
-
-# Ensure the folder exists
-if [[ ! -d "$folder_path" ]]; then
- echo "Error: Folder $folder_path does not exist."
- exit 1
-fi
-
-# Check and install required tools
-for tool in "oras" "split" "pv" "jq" "lz4"; do
- if ! command -v "$tool" &> /dev/null; then
- echo "$tool is not installed. Installing using Homebrew..."
- if ! command -v brew &> /dev/null; then
- echo "Homebrew is not installed. Please install Homebrew first: https://brew.sh/"
- exit 1
- fi
- brew install "$tool"
- fi
-done
-
-echo "LZ4 detected - will use for efficient compression and decompression"
-compressed_ext=".lz4"
-
-# Authenticate with GitHub Container Registry if not in dry-run mode
-if [[ "$dry_run" = false ]]; then
- echo "$GITHUB_TOKEN" | oras login ghcr.io -u "$organization" --password-stdin
-fi
-
-# Use the source folder path as the working directory and get its absolute path
-work_dir=$(cd "$folder_path" && pwd)
-echo "Working directory: $work_dir"
-
-# Function to find the most recent cache directory
-find_latest_cache() {
- local latest_cache=$(ls -td "$work_dir"/.ghcr_cache_* 2>/dev/null | head -n1)
- if [ -n "$latest_cache" ]; then
- echo "$latest_cache"
- else
- echo ""
- fi
-}
-
-# Function to check if a cache directory is valid for resuming
-is_valid_cache() {
- local cache_dir="$1"
- # Check if it contains the necessary files
- [ -f "$cache_dir/config.json" ] || [ -f "$cache_dir/nvram.bin" ] || \
- [ -f "$cache_dir/disk.img.lz4" ] || ls "$cache_dir"/disk.img.part.* 1>/dev/null 2>&1
-}
-
-# Always try to find and use an existing cache
-existing_cache=$(find_latest_cache)
-if [ -n "$existing_cache" ] && is_valid_cache "$existing_cache"; then
- cache_dir="$existing_cache"
-
- # Check if the cache contains old compressed format
- if [ -f "$cache_dir/disk.img.gz" ] || [ -f "$cache_dir/disk.img.aa" ] || ls "$cache_dir"/disk.img.*.part.* 1>/dev/null 2>&1; then
- echo "Error: Found legacy compressed format in cache. This script uses improved LZ4 format."
- echo "Please delete the cache directory and start fresh: $cache_dir"
- exit 1
- fi
-
- echo "Resuming from existing cache: $cache_dir"
-else
- echo "No valid cache found. Starting fresh."
- cache_dir="$work_dir/.ghcr_cache_$(date +%Y%m%d_%H%M%S)"
- mkdir -p "$cache_dir"
-fi
-
-echo "Using cache directory: $cache_dir"
-
-# Display space information
-echo "=== DISK SPACE INFORMATION ==="
-df -h "$cache_dir" | head -1
-df -h "$cache_dir" | grep -v "Filesystem"
-echo
-
-# Change to the cache directory
-cd "$cache_dir"
-files=() # Initialize files array here
-
-# Function to check if a version was already pushed
-version_pushed() {
- local version="$1"
- local version_file="$cache_dir/.pushed_$version"
- [ -f "$version_file" ]
-}
-
-# Function to mark a version as pushed
-mark_version_pushed() {
- local version="$1"
- touch "$cache_dir/.pushed_$version"
-}
-
-# Function to calculate sha256 hash
-calculate_sha256() {
- local file="$1"
- if command -v shasum &> /dev/null; then
- shasum -a 256 "$file" | awk '{print "sha256:" $1}'
- else
- echo "sha256:$(openssl dgst -sha256 -binary "$file" | xxd -p | tr -d '\n')"
- fi
-}
-
-# Copy config.json if it exists and not already in cache
-config_json_source="$folder_path/config.json"
-config_json_dest="$cache_dir/config.json"
-if [ -f "$config_json_source" ]; then
- if [ ! -f "$config_json_dest" ]; then
- echo "Copying config.json..."
- # Copy config.json as is - we'll add annotations later
- cp "$config_json_source" "$config_json_dest"
- fi
-fi
-if [ -f "$config_json_dest" ]; then
- files+=("config.json:application/vnd.oci.image.config.v1+json")
-fi
-
-# Copy nvram.bin if it exists and not already in cache
-if [ -f "$folder_path/nvram.bin" ] && [ ! -f "$cache_dir/nvram.bin" ]; then
- echo "Copying nvram.bin..."
- cp "$folder_path/nvram.bin" nvram.bin
-fi
-if [ -f "$cache_dir/nvram.bin" ]; then
- files+=("nvram.bin:application/octet-stream")
-fi
-
-# Process disk.img if it exists
-disk_img_orig="$folder_path/disk.img"
-original_disk_size=""
-if [ -f "$disk_img_orig" ]; then
- # Get original size *before* compression
- original_disk_size=$(stat -f%z "$disk_img_orig")
-
- # Get real (non-sparse) size
- real_size=$(du -k "$disk_img_orig" | cut -f1)
- real_size_bytes=$((real_size * 1024))
- sparseness_ratio=$(echo "scale=2; $original_disk_size / $real_size_bytes" | bc)
- echo "Disk image: $disk_img_orig"
- echo " Logical size: $original_disk_size bytes ($(du -h "$disk_img_orig" | cut -f1))"
- echo " Actual disk usage: $((real_size_bytes / 1073741824)) GB"
- echo " Sparseness ratio: ${sparseness_ratio}:1"
-
- # If we have config.json, update it with the uncompressed disk size annotation
- if [ -f "$config_json_dest" ] && command -v jq &> /dev/null; then
- echo "Adding uncompressed disk size annotation: $original_disk_size bytes"
- jq --arg size "$original_disk_size" '.annotations = (.annotations // {}) + {"com.trycua.lume.disk.uncompressed_size": $size}' "$config_json_dest" > "$config_json_dest.tmp"
- mv "$config_json_dest.tmp" "$config_json_dest"
- fi
-
- # Create a temporary directory for disk processing
- tmp_dir="$cache_dir/tmp_processing"
- mkdir -p "$tmp_dir"
-
- # Split the disk image into chunks first (before compression)
- split_parts_dir="$tmp_dir/split_parts"
- mkdir -p "$split_parts_dir"
-
- # Check if we already have split parts
- if [ -z "$(ls -A "$split_parts_dir" 2>/dev/null)" ]; then
- echo "Splitting disk image into chunks of $chunk_size..."
- cd "$split_parts_dir"
- pv "$disk_img_orig" | split -b "$chunk_size" - "chunk."
- cd "$cache_dir"
- else
- echo "Using existing split chunks from previous run"
- fi
-
- # Process each chunk (compress, calculate digest, etc.)
- compressed_parts_dir="$tmp_dir/compressed_parts"
- mkdir -p "$compressed_parts_dir"
-
- # Store layer information in an array
- layers=()
- part_num=0
- total_parts=$(ls "$split_parts_dir"/chunk.* | wc -l)
-
- for chunk_file in "$split_parts_dir"/chunk.*; do
- part_basename=$(basename "$chunk_file")
- part_num=$((part_num + 1))
- compressed_file="$compressed_parts_dir/${part_basename}${compressed_ext}"
-
- if [ ! -f "$compressed_file" ]; then
- echo "Compressing chunk $part_num of $total_parts: $part_basename"
-
- # Calculate uncompressed content digest before compression
- uncompressed_digest=$(calculate_sha256 "$chunk_file")
-
- # Get uncompressed size
- uncompressed_size=$(stat -f%z "$chunk_file")
-
- # Compress the chunk with LZ4
- lz4 -9 "$chunk_file" "$compressed_file"
-
- # Get compressed size
- compressed_size=$(stat -f%z "$compressed_file")
-
- echo "Chunk $part_num: Original size: $(du -h "$chunk_file" | cut -f1), Compressed: $(du -h "$compressed_file" | cut -f1)"
- else
- echo "Using existing compressed chunk $part_num of $total_parts"
-
- # Need to calculate these values for existing files
- uncompressed_digest=$(calculate_sha256 "$chunk_file")
- uncompressed_size=$(stat -f%z "$chunk_file")
- compressed_size=$(stat -f%z "$compressed_file")
- fi
-
- # Store layer information
- layer_info="$compressed_file:${oci_layer_media_type};uncompressed_size=$uncompressed_size;uncompressed_digest=$uncompressed_digest;part.number=$part_num;part.total=$total_parts"
- layers+=("$layer_info")
- done
-
- # Generate the files array for ORAS push
- for layer_info in "${layers[@]}"; do
- files+=("$layer_info")
- done
-
- # --- Reassembly in dry-run mode ---
- if [[ "$reassemble" = true ]]; then
- echo "=== REASSEMBLY MODE ==="
- echo "Reassembling chunks to verify integrity..."
-
- # Create a directory for reassembly
- reassembly_dir="$cache_dir/reassembly"
- mkdir -p "$reassembly_dir"
-
- # Prepare the reassembled file - create a properly sized sparse file first
- reassembled_file="$reassembly_dir/reassembled_disk.img"
- if [ -f "$reassembled_file" ]; then
- echo "Removing previous reassembled file..."
- rm -f "$reassembled_file"
- fi
-
- # Get the original disk size from config annotation or directly from image
- if [ -f "$config_json_dest" ] && command -v jq &> /dev/null; then
- config_size=$(jq -r '.annotations."com.trycua.lume.disk.uncompressed_size" // empty' "$config_json_dest")
- if [ -n "$config_size" ]; then
- original_disk_size_bytes=$config_size
- echo "Using uncompressed size from config: $original_disk_size_bytes bytes"
- fi
- fi
-
- # Create a sparse file of the exact original size
- echo "Pre-allocating sparse file of $(du -h "$disk_img_orig" | cut -f1)..."
- dd if=/dev/zero of="$reassembled_file" bs=1 count=0 seek=$original_disk_size
-
- # Make sure filesystem recognizes this as a sparse file
- if [[ "$OSTYPE" == "darwin"* ]]; then
- # On macOS, we can use a better sparse file creation method if mkfile is available
- if command -v mkfile &> /dev/null; then
- rm -f "$reassembled_file"
- mkfile -n ${original_disk_size}b "$reassembled_file"
- echo "Created sparse file using mkfile"
- fi
- else
- # On Linux systems, ensure sparseness with truncate if available
- if command -v truncate &> /dev/null; then
- rm -f "$reassembled_file"
- truncate -s $original_disk_size "$reassembled_file"
- echo "Created sparse file using truncate"
- fi
- fi
-
- # Create an offset tracker to keep track of where each chunk should go
- current_offset=0
-
- # Decompress each chunk and write it at the correct offset
- for ((i=1; i<=total_parts; i++)); do
- # Find the chunk file for part number i
- chunk_pattern=""
- chunk_uncompressed_size=""
-
- for layer_info in "${layers[@]}"; do
- if [[ "$layer_info" == *";part.number=$i;"* ]]; then
- chunk_pattern="${layer_info%%:*}"
- # Extract the uncompressed size from metadata
- if [[ "$layer_info" =~ uncompressed_size=([0-9]+) ]]; then
- chunk_uncompressed_size="${BASH_REMATCH[1]}"
- fi
- break
- fi
- done
-
- if [ -z "$chunk_pattern" ]; then
- echo "Error: Could not find chunk for part $i"
- exit 1
- fi
-
- echo "Processing part $i/$total_parts: $(basename "$chunk_pattern") at offset $current_offset..."
-
- # Create temp decompressed file
- temp_decompressed="$reassembly_dir/temp_part_$i"
- lz4 -d -f "$chunk_pattern" "$temp_decompressed" || {
- echo "Error decompressing part $i"
- exit 1
- }
-
- # Check if this chunk is all zeros (sparse data)
- # Only check the first 1MB for efficiency
- is_likely_sparse=false
- if command -v hexdump &> /dev/null; then
- # Use hexdump to check a sample of the file for non-zero content
- sparse_check=$(hexdump -n 1048576 -v "$temp_decompressed" | grep -v "0000 0000 0000 0000 0000 0000 0000 0000" | head -n 1)
- if [ -z "$sparse_check" ]; then
- echo "Chunk appears to be all zeros (sparse data)"
- is_likely_sparse=true
- fi
- fi
-
- # Use dd to write the chunk at the correct offset with sparse file handling
- if [ "$is_likely_sparse" = true ]; then
- # For sparse chunks, we don't need to write anything - leave as zeros
- echo "Skipping write for all-zero chunk (preserving sparseness)"
- elif [[ "$OSTYPE" == "darwin"* ]]; then
- # macOS dd doesn't support conv=sparse, use standard approach
- dd if="$temp_decompressed" of="$reassembled_file" bs=1M conv=notrunc seek=$((current_offset / 1024 / 1024)) status=progress || {
- echo "Error writing part $i at offset $current_offset"
- exit 1
- }
- else
- # On Linux, use conv=sparse to preserve sparseness during the write
- dd if="$temp_decompressed" of="$reassembled_file" bs=1M conv=sparse,notrunc seek=$((current_offset / 1024 / 1024)) status=progress || {
- echo "Error writing part $i at offset $current_offset"
- exit 1
- }
- fi
-
- # Clean up the temporary file
- rm -f "$temp_decompressed"
-
- # Update the offset for the next chunk
- current_offset=$((current_offset + chunk_uncompressed_size))
- done
-
- # After all chunks are processed, ensure sparseness is preserved
- if command -v cp &> /dev/null && [[ "$OSTYPE" == "darwin"* ]]; then
- echo "Copying disk image to maintain sparseness..."
- final_sparse_file="$reassembly_dir/final_disk.img"
- rm -f "$final_sparse_file" 2>/dev/null
-
- # On macOS, use cp with the clone flag to preserve sparseness
- cp -c "$reassembled_file" "$final_sparse_file"
-
- # Use the sparse-optimized file for verification
- echo "Using sparse-optimized copy for verification"
- mv "$final_sparse_file" "$reassembled_file"
- sync
- elif command -v cp &> /dev/null && command -v file &> /dev/null; then
- # For Linux systems
- echo "Optimizing file sparseness..."
- final_sparse_file="$reassembly_dir/final_disk.img"
- rm -f "$final_sparse_file" 2>/dev/null
-
- # Use cp --sparse=always on Linux
- cp --sparse=always "$reassembled_file" "$final_sparse_file"
-
- # Use the sparse-optimized file for verification
- echo "Using sparse-optimized copy for verification"
- mv "$final_sparse_file" "$reassembled_file"
- sync
- fi
-
- # Make sure to sync to disk
- sync
-
- # Calculate digests for comparison
- echo "Verifying reassembled file..."
- original_digest=$(calculate_sha256 "$disk_img_orig")
- reassembled_digest=$(calculate_sha256 "$reassembled_file")
-
- # Compare the original and reassembled file sizes
- original_size=$(stat -f%z "$disk_img_orig")
- reassembled_size=$(stat -f%z "$reassembled_file")
-
- echo "Results:"
- echo " Original size: $(du -h "$disk_img_orig" | cut -f1) ($original_size bytes)"
- echo " Reassembled size: $(du -h "$reassembled_file" | cut -f1) ($reassembled_size bytes)"
- echo " Original digest: ${original_digest#sha256:}"
- echo " Reassembled digest: ${reassembled_digest#sha256:}"
-
- # Check if the disk is sparse
- original_apparent_size=$(du -h "$disk_img_orig" | cut -f1)
- original_actual_size=$(du -sh "$disk_img_orig" | cut -f1)
- reassembled_apparent_size=$(du -h "$reassembled_file" | cut -f1)
- reassembled_actual_size=$(du -sh "$reassembled_file" | cut -f1)
-
- echo " Original: Apparent size: $original_apparent_size, Actual disk usage: $original_actual_size"
- echo " Reassembled: Apparent size: $reassembled_apparent_size, Actual disk usage: $reassembled_actual_size"
-
- if [ "$original_digest" = "$reassembled_digest" ]; then
- echo "✅ VERIFICATION SUCCESSFUL: Files are identical"
- else
- echo "❌ VERIFICATION FAILED: Files differ"
- if [ "$original_size" != "$reassembled_size" ]; then
- echo " Size mismatch: Original $original_size bytes, Reassembled $reassembled_size bytes"
- fi
-
- # Try to identify where they differ
- echo "Attempting to identify differences..."
- if command -v cmp &> /dev/null; then
- cmp_output=$(cmp -l "$disk_img_orig" "$reassembled_file" 2>&1 | head -5)
- if [[ "$cmp_output" == *"differ"* ]]; then
- echo " First few differences:"
- echo "$cmp_output"
- fi
- fi
-
- # Check if the virtual machine will still boot despite differences
- echo "NOTE: This might be a sparse file issue. The content may be identical, but sparse regions"
- echo " may be handled differently between the original and reassembled files."
-
- # Calculate a percentage comparison of used blocks
- # This helps determine if the sparse issues are severe or minor
- original_used_kb=$(du -k "$disk_img_orig" | cut -f1)
- reassembled_used_kb=$(du -k "$reassembled_file" | cut -f1)
-
- # Calculate percentage difference in used space
- if [ "$original_used_kb" -ne 0 ]; then
- diff_percentage=$(echo "scale=2; ($reassembled_used_kb - $original_used_kb) * 100 / $original_used_kb" | bc)
- echo " Disk usage difference: $diff_percentage% ($reassembled_used_kb KB vs $original_used_kb KB)"
-
- # If reassembled is much smaller, this likely indicates sparse regions weren't preserved
- if (( $(echo "$diff_percentage < -40" | bc -l) )); then
- echo " ⚠️ WARNING: Reassembled disk uses significantly less space (>40% difference)."
- echo " This indicates sparse regions weren't properly preserved and may affect VM functionality."
- echo " The VM might boot but could be missing applications or data."
- elif (( $(echo "$diff_percentage < -10" | bc -l) )); then
- echo " ⚠️ WARNING: Reassembled disk uses less space (10-40% difference)."
- echo " Some sparse regions may not be properly preserved but VM might still function correctly."
- elif (( $(echo "$diff_percentage > 10" | bc -l) )); then
- echo " ⚠️ WARNING: Reassembled disk uses more space (>10% difference)."
- echo " This is unusual and may indicate improper sparse file handling."
- else
- echo " ✓ Disk usage difference is minimal (<10%). VM likely to function correctly."
- fi
- fi
- fi
-
- echo "Reassembled file is available at: $reassembled_file"
-
- # If verification failed and difference is significant, try a direct copy as fallback
- if [ "$original_digest" != "$reassembled_digest" ] && [ -n "$diff_percentage" ] && (( $(echo "$diff_percentage < -20" | bc -l) )); then
- echo
- echo "===== ATTEMPTING RECOVERY ACTION ====="
- echo "Since verification failed with significant disk usage difference,"
- echo "trying direct copy of disk image as a fallback method."
- echo
-
- fallback_file="$reassembly_dir/fallback_disk.img"
- echo "Creating fallback disk image at: $fallback_file"
-
- # Use rsync with sparse option if available
- if command -v rsync &> /dev/null; then
- echo "Using rsync with sparse option for direct copy..."
- rsync -aS --progress "$disk_img_orig" "$fallback_file"
- else
- # Direct cp with sparse option if available
- if [[ "$OSTYPE" == "darwin"* ]]; then
- echo "Using cp -c (clone) for direct copy..."
- cp -c "$disk_img_orig" "$fallback_file"
- else
- echo "Using cp --sparse=always for direct copy..."
- cp --sparse=always "$disk_img_orig" "$fallback_file"
- fi
- fi
-
- echo "Direct copy completed. You may want to try using this fallback disk image"
- echo "instead if the reassembled one has issues: $fallback_file"
- fi
- fi
-
- # --- Push Logic ---
- if [[ "$dry_run" = true ]]; then
- echo "=== DRY RUN MODE ==="
- echo "The following files would be pushed to the registry:"
- for file_info in "${files[@]}"; do
- file_path="${file_info%%:*}"
- file_metadata="${file_info#*:}"
- file_size=$(du -h "$file_path" | cut -f1)
- echo " - $file_path ($file_size) with metadata: $file_metadata"
- done
-
- if [[ -n "$image_versions" ]]; then
- echo "Would push to the following versions:"
- IFS=',' read -ra versions <<< "$image_versions"
- for version in "${versions[@]}"; do
- version=$(echo "$version" | xargs)
- if [[ -z "$version" ]]; then continue; fi
- echo " - ghcr.io/$organization/$image_name:$version"
- done
- else
- echo "No versions specified for dry run. Processing completed successfully."
- fi
-
- echo "All processing tasks completed. No actual push performed."
- echo "Cache directory: $cache_dir"
- exit 0
- fi
-
- # Regular push logic (non-dry-run)
- push_pids=()
- IFS=',' read -ra versions <<< "$image_versions"
- for version in "${versions[@]}"; do
- # Trim whitespace if any from version splitting
- version=$(echo "$version" | xargs)
- if [[ -z "$version" ]]; then continue; fi
-
- # Skip if version was already pushed
- if version_pushed "$version"; then
- echo "Version $version was already pushed, skipping..."
- continue
- fi
-
- echo "Pushing version $version..."
- (
- # Use process substitution to feed file list safely if it gets long
- oras push --disable-path-validation \
- "ghcr.io/$organization/$image_name:$version" \
- "${files[@]}"
- echo "Completed push for version $version"
- mark_version_pushed "$version"
- ) &
- push_pids+=($!)
- done
-
- # Wait for all pushes to complete
- for pid in "${push_pids[@]}"; do
- wait "$pid"
- done
-
- # --- Cleanup only if all versions were pushed successfully ---
- all_versions_pushed=true
- for version in "${versions[@]}"; do
- version=$(echo "$version" | xargs)
- if [[ -z "$version" ]]; then continue; fi
- if ! version_pushed "$version"; then
- all_versions_pushed=false
- break
- fi
- done
-
- if [ "$all_versions_pushed" = true ]; then
- echo "All versions pushed successfully, cleaning up cache directory..."
- cd "$work_dir"
- rm -rf "$cache_dir"
- else
- echo "Some versions failed to push. Cache directory preserved at: $cache_dir"
- echo "Run again to resume from this point"
- fi
-
-else
- echo "Warning: $disk_img_orig not found."
-
- # If in dry run mode, just show what would happen
- if [[ "$dry_run" = true ]]; then
- echo "=== DRY RUN MODE ==="
- if [ ${#files[@]} -gt 0 ]; then
- echo "The following non-disk files would be pushed:"
- for file_info in "${files[@]}"; do
- file_path="${file_info%%:*}"
- file_metadata="${file_info#*:}"
- file_size=$(du -h "$file_path" | cut -f1)
- echo " - $file_path ($file_size) with metadata: $file_metadata"
- done
- else
- echo "No files found to push."
- fi
- echo "All processing tasks completed. No actual push performed."
- exit 0
- fi
-
- # Push only config/nvram if they exist
- if [ ${#files[@]} -gt 0 ]; then
- echo "Pushing non-disk files..."
- push_pids=()
- IFS=',' read -ra versions <<< "$image_versions"
- for version in "${versions[@]}"; do
- # Trim whitespace if any from version splitting
- version=$(echo "$version" | xargs)
- if [[ -z "$version" ]]; then continue; fi
-
- # Skip if version was already pushed
- if version_pushed "$version"; then
- echo "Version $version was already pushed, skipping..."
- continue
- fi
-
- echo "Pushing version $version (config/nvram only)..."
- (
- oras push --disable-path-validation \
- "ghcr.io/$organization/$image_name:$version" \
- "${files[@]}"
- echo "Completed push for version $version"
- mark_version_pushed "$version"
- ) &
- push_pids+=($!)
- done
-
- # Wait for all pushes to complete
- for pid in "${push_pids[@]}"; do
- wait "$pid"
- done
-
- # --- Cleanup only if all versions were pushed successfully ---
- all_versions_pushed=true
- for version in "${versions[@]}"; do
- version=$(echo "$version" | xargs)
- if [[ -z "$version" ]]; then continue; fi
- if ! version_pushed "$version"; then
- all_versions_pushed=false
- break
- fi
- done
-
- if [ "$all_versions_pushed" = true ]; then
- echo "All non-disk versions pushed successfully, cleaning up cache directory..."
- cd "$work_dir"
- rm -rf "$cache_dir"
- else
- echo "Some non-disk versions failed to push. Cache directory preserved at: $cache_dir"
- echo "Run again to resume from this point"
- fi
- else
- echo "No files found to push."
- cd "$work_dir"
- rm -rf "$cache_dir"
- exit 1
- fi
-fi
-
-# Skip final status check in dry-run mode
-if [[ "$dry_run" = true ]]; then
- exit 0
-fi
-
-# Determine final status based on the success check *before* potential cleanup
-echo # Add a newline for better readability
-if [ "$all_versions_pushed" = true ]; then
- echo "All versions pushed successfully:"
- for version in "${versions[@]}"; do
- version=$(echo "$version" | xargs)
- if [[ -z "$version" ]]; then continue; fi
- echo " Upload complete: ghcr.io/$organization/$image_name:$version"
- done
-else
- echo "Final upload status:"
- for version in "${versions[@]}"; do
- version=$(echo "$version" | xargs)
- if [[ -z "$version" ]]; then continue; fi
- # Check the marker file only if the overall process failed (cache preserved)
- if version_pushed "$version"; then
- echo " Upload complete: ghcr.io/$organization/$image_name:$version"
- else
- echo " Upload failed: ghcr.io/$organization/$image_name:$version"
- fi
- done
- # Exit with error code if any version failed
- exit 1
-fi