Skip to content

Agent Mesh

Convert cloud-init to standard GCP startup-script

Antonio Ojea94a469bc857d
diff --git a/scripts/build-rootfs.sh b/scripts/build-rootfs.sh
index 759a6bbe..78214b37 100755
--- a/scripts/build-rootfs.sh
+++ b/scripts/build-rootfs.sh
@@ -16,8 +16,7 @@ RUN curl -sSL -o /usr/local/bin/tun2proxy https://github.com/tun2proxy/tun2proxy
COPY cmd/chaos-agent /app/chaos-agent
RUN pip3 install --no-cache-dir -r /app/chaos-agent/requirements.txt --break-system-packages
-COPY scripts/microvm-init.sh /sbin/init
-RUN chmod +x /sbin/init
+COPY --chmod=755 scripts/microvm-init.sh /sbin/init
EOF
# Build the docker container
@@ -31,10 +30,13 @@ dd if=/dev/zero of=rootfs.ext4 bs=1M count=500
mkfs.ext4 rootfs.ext4
mkdir -p /tmp/rootfs
-# Use sudo to mount and untar, since we need root permissions to create files with correct ownership
-sudo mount rootfs.ext4 /tmp/rootfs
-sudo tar -xf rootfs.tar -C /tmp/rootfs
-sudo umount /tmp/rootfs
+# Use a privileged docker container to mount and populate the ext4 image without requiring sudo on the host
+docker run --rm --privileged -v $(pwd):/work alpine sh -c '
+ mkdir -p /mnt/rootfs
+ mount /work/rootfs.ext4 /mnt/rootfs
+ tar -xf /work/rootfs.tar -C /mnt/rootfs
+ umount /mnt/rootfs
+'
rm rootfs.tar Dockerfile
echo "Rootfs built successfully at rootfs.ext4"
diff --git a/scripts/cloud-init.yaml b/scripts/cloud-init.yaml
deleted file mode 100644
index 10049be6..00000000
--- a/scripts/cloud-init.yaml
+++ /dev/null
@@ -1,68 +0,0 @@
-#cloud-config
-package_update: true
-package_upgrade: true
-packages:
- - curl
- - wget
- - git
- - iptables
- - iproute2
- - socat
- - jq
- - build-essential
- - docker.io
- - acl
-
-runcmd:
- # Check if nested virtualization is enabled
- - |
- if [ ! -e /dev/kvm ]; then
- echo "ERROR: /dev/kvm not found! Nested virtualization is not enabled."
- exit 1
- fi
-
- # Ensure KVM permissions allow execution
- - chmod 0666 /dev/kvm
-
- # Install Firecracker v1.16.1
- - curl -LO https://github.com/firecracker-microvm/firecracker/releases/download/v1.16.1/firecracker-v1.16.1-x86_64.tgz
- - tar -xzf firecracker-v1.16.1-x86_64.tgz
- - mv release-v1.16.1-x86_64/firecracker-v1.16.1-x86_64 /usr/local/bin/firecracker
- - chmod +x /usr/local/bin/firecracker
- - rm -rf release-v1.16.1-x86_64 firecracker-v1.16.1-x86_64.tgz
-
- # Enable IP forwarding (just in case we need it for TAP/NAT later)
- - sysctl -w net.ipv4.ip_forward=1
- - echo "net.ipv4.ip_forward=1" >> /etc/sysctl.d/99-ipforward.conf
-
- # Install SAM Mesh binaries & MicroVM files
- # Check if custom binaries/rootfs were injected via GCP metadata (GCS URL)
- - |
- BIN_URL=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/sam-binaries-url" || true)
-
- if [ -n "$BIN_URL" ] && [ "$BIN_URL" != "null" ]; then
- echo "Found custom artifacts URL: $BIN_URL. Downloading from GCS..."
- # Download SAM binaries
- gcloud storage cp "$BIN_URL/sam-node" /usr/local/bin/
- gcloud storage cp "$BIN_URL/sam-box" /usr/local/bin/
- chmod +x /usr/local/bin/sam-*
-
- # Download the pre-built MicroVM rootfs
- mkdir -p /opt/microvm
- gcloud storage cp "$BIN_URL/rootfs.ext4" /opt/microvm/rootfs.ext4
- else
- echo "No custom binaries specified. Installing latest release from github..."
- export SAM_INSTALL_DIR=/usr/local/bin
- curl -sL https://sam-mesh.dev/install.sh | bash
- fi
-
- # Download the uncompressed Linux kernel for Firecracker
- - mkdir -p /opt/microvm
- - curl -fsSL -o /opt/microvm/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin
-
- # Download the launch script from the repo (since it's public) or we could embed it here.
- # Let's download it from the main branch since it's self-contained.
- - curl -fsSL -o /opt/microvm/launch-microvms.sh https://raw.githubusercontent.com/google/sam/main/scripts/launch-microvms.sh || echo "Failed to download launch script"
- - chmod +x /opt/microvm/launch-microvms.sh
-
- - echo "SAM Host cloud-init bootstrap finished successfully."
diff --git a/scripts/provision-scale-vm.sh b/scripts/provision-scale-vm.sh
index 460e5502..d8fb520d 100755
--- a/scripts/provision-scale-vm.sh
+++ b/scripts/provision-scale-vm.sh
@@ -6,6 +6,7 @@ ZONE="us-central1-a"
PROJECT="ipv6-project-379110"
GCS_URL=""
COUNT=1
+MACHINE_TYPE="n2-standard-64"
while [[ "$#" -gt 0 ]]; do
case $1 in
@@ -14,6 +15,7 @@ while [[ "$#" -gt 0 ]]; do
--project) PROJECT="$2"; shift ;;
--local-binaries) GCS_URL="$2"; shift ;;
--count) COUNT="$2"; shift ;;
+ --machine-type) MACHINE_TYPE="$2"; shift ;;
*) echo "Unknown parameter passed: $1"; exit 1 ;;
esac
shift
@@ -41,12 +43,12 @@ if [ "$COUNT" -eq 1 ]; then
gcloud beta compute instances create "${VM_NAME_PREFIX}-1" \
--project="${PROJECT}" \
--zone="${ZONE}" \
- --machine-type=n2-standard-64 \
+ --machine-type="${MACHINE_TYPE}" \
--enable-nested-virtualization \
--min-cpu-platform="Intel Ice Lake" \
--network-interface=network-tier=PREMIUM,stack-type=IPV4_ONLY,subnet=default \
--metadata="${METADATA}" \
- --metadata-from-file=user-data=scripts/cloud-init.yaml \
+ --metadata-from-file=startup-script=scripts/startup-script.sh \
--no-restart-on-failure \
--maintenance-policy=TERMINATE \
--provisioning-model=SPOT \
@@ -68,12 +70,12 @@ else
--count="${COUNT}" \
--project="${PROJECT}" \
--zone="${ZONE}" \
- --machine-type=n2-standard-64 \
+ --machine-type="${MACHINE_TYPE}" \
--enable-nested-virtualization \
--min-cpu-platform="Intel Ice Lake" \
--network-interface=network-tier=PREMIUM,stack-type=IPV4_ONLY,subnet=default \
--metadata="${METADATA}" \
- --metadata-from-file=user-data=scripts/cloud-init.yaml \
+ --metadata-from-file=startup-script=scripts/startup-script.sh \
--no-restart-on-failure \
--maintenance-policy=TERMINATE \
--provisioning-model=SPOT \
diff --git a/scripts/startup-script.sh b/scripts/startup-script.sh
new file mode 100644
index 00000000..1bc0c9e8
--- /dev/null
+++ b/scripts/startup-script.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+set -e
+
+# Redirect all output to log file
+exec > >(tee -a /var/log/startup-script.log) 2>&1
+echo "Starting SAM Host VM bootstrap..."
+
+# 1. Update and install packages
+apt-get update
+apt-get upgrade -y
+apt-get install -y curl wget git iptables iproute2 socat jq build-essential docker.io acl
+
+# 2. Check if nested virtualization is enabled
+if [ ! -e /dev/kvm ]; then
+ echo "ERROR: /dev/kvm not found! Nested virtualization is not enabled."
+ exit 1
+fi
+
+# Ensure KVM permissions allow execution
+chmod 0666 /dev/kvm
+
+# 3. Install Firecracker v1.16.1
+curl -LO https://github.com/firecracker-microvm/firecracker/releases/download/v1.16.1/firecracker-v1.16.1-x86_64.tgz
+tar -xzf firecracker-v1.16.1-x86_64.tgz
+mv release-v1.16.1-x86_64/firecracker-v1.16.1-x86_64 /usr/local/bin/firecracker
+chmod +x /usr/local/bin/firecracker
+rm -rf release-v1.16.1-x86_64 firecracker-v1.16.1-x86_64.tgz
+
+# Enable IP forwarding (just in case we need it for TAP/NAT later)
+sysctl -w net.ipv4.ip_forward=1
+echo "net.ipv4.ip_forward=1" >> /etc/sysctl.d/99-ipforward.conf
+
+# 4. Install SAM Mesh binaries & MicroVM files
+# Check if custom binaries/rootfs were injected via GCP metadata (GCS URL)
+BIN_URL=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/sam-binaries-url" || true)
+
+if [ -n "$BIN_URL" ] && [ "$BIN_URL" != "null" ]; then
+ echo "Found custom artifacts URL: $BIN_URL. Downloading from GCS..."
+ # Download SAM binaries
+ gcloud storage cp "$BIN_URL/sam-node" /usr/local/bin/
+ gcloud storage cp "$BIN_URL/sam-box" /usr/local/bin/
+ chmod +x /usr/local/bin/sam-*
+
+ # Download the pre-built MicroVM rootfs
+ mkdir -p /opt/microvm
+ gcloud storage cp "$BIN_URL/rootfs.ext4" /opt/microvm/rootfs.ext4
+else
+ echo "No custom binaries specified. Installing latest release from github..."
+ export SAM_INSTALL_DIR=/usr/local/bin
+ curl -sL https://sam-mesh.dev/install.sh | bash
+fi
+
+# Download the uncompressed Linux kernel for Firecracker
+mkdir -p /opt/microvm
+curl -fsSL -o /opt/microvm/vmlinux.bin https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/x86_64/kernels/vmlinux.bin
+
+# Download the launch script from the repo
+curl -fsSL -o /opt/microvm/launch-microvms.sh https://raw.githubusercontent.com/google/sam/main/scripts/launch-microvms.sh || echo "Failed to download launch script"
+chmod +x /opt/microvm/launch-microvms.sh
+
+echo "SAM Host bootstrap finished successfully."