Light, fluffy, and always free - The AWS Local Emulator alternative
chore(build): make the native image build locally and give the Makefile a development workflow (#4271)
Hector Ventura96264cde7052
diff --git a/.gitignore b/.gitignoreindex 5a2b5bed2e..69be4a6b1e 100644--- a/.gitignore+++ b/.gitignore@@ -1,4 +1,6 @@target/+# make native stages the container-built binary here for docker/Dockerfile.native-package+native/data/*.class*.jardiff --git a/AGENTS.md b/AGENTS.mdindex 69b8ebf8ed..f88bf47175 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -180,6 +180,11 @@ Critical areas:./mvnw clean package./mvnw clean package -DskipTests+`make help` lists the Makefile shortcuts. The native binary and image: `make native`+then `make native-image` (CI's flags, staged in `native/<arch>/`), `make native-host` for a+binary built on this machine with the installed GraalVM, `make native-up` to run the image,+`make compat SUITES="sdk-test-java compat-cdk"` for compatibility suites against it.+### Focused tests./mvnw test -Dtest=SsmIntegrationTestdiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex a30b601dae..df772fcf91 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -39,6 +39,12 @@ cd flociIf you prefer to use your own Maven installation (3.9+), you can use `mvn` instead of `./mvnw`.+`make help` lists the repository's shortcuts: `make dev`, `make test T=SsmIntegrationTest`,+`make native` and `make native-image` for the native binary and its Docker image with the flags CI+uses, `make native-host` and `make run-native` for a binary built and run on your machine without+Docker, `make native-up` to run the image, and `make compat SUITES="sdk-test-java"` to run compatibility+suites in Docker against it.+### Run Tests```bashdiff --git a/Makefile b/Makefileindex 238f18aa7d..4c192df2d5 100644--- a/Makefile+++ b/Makefile@@ -8,7 +8,107 @@PYTHON ?= python3-.PHONY: docs-sync docs-check docs-test+# Native builds. The binary is built inside the Quarkus builder container (works on macOS,+# yields a Linux binary for the host arch) with the flags CI's compatibility workflow uses:+# -Ob is the quick profile, and -H:-AOTSingleCallsiteInline is required on Mandrel 25.0.4+,+# where Quarkus 3.39 turns single-callsite inlining on and the generated REST invokers for+# AwsQueryController.dispatch and AwsJson11Controller.handle then outgrow the aarch64 branch+# range (BranchTargetOutOfBoundsException). The flag exists only on Mandrel, so it lives here+# and in the workflows rather than in application.yml.+ARCH := $(shell uname -m | sed -e 's/x86_64/amd64/' -e 's/aarch64/arm64/')+NATIVE_BUILDER ?= quay.io/quarkus/ubi9-quarkus-mandrel-builder-image:jdk-25+NATIVE_ARGS ?= -Ob,-H:-AOTSingleCallsiteInline+NATIVE_IMAGE ?= floci:local-native+COMPOSE_NATIVE := docker compose -f docker-compose.yml -f docker/compose.native.yml+SUITES ?= sdk-test-java++# Host builds (make native-host): the installed GraalVM or Mandrel, found the way Quarkus finds+# it, GRAALVM_HOME first, then JAVA_HOME, then the PATH. The inlining flag above is unknown to+# GraalVM, so it is added only when that native-image is Mandrel.+comma := ,+NATIVE_IMAGE_BIN = $(or $(wildcard $(GRAALVM_HOME)/bin/native-image),$(wildcard $(JAVA_HOME)/bin/native-image),$(shell command -v native-image 2>/dev/null))+NATIVE_HOST_ARGS ?= -Ob+NATIVE_HOST_FLAGS = $(NATIVE_HOST_ARGS)$(if $(findstring andrel,$(shell "$(NATIVE_IMAGE_BIN)" --version 2>/dev/null)),$(comma)-H:-AOTSingleCallsiteInline)+NATIVE_BIN = $(firstword $(wildcard target/*-runner))+PREFIX ?= $(HOME)/.local++.DEFAULT_GOAL := help+.PHONY: help dev build test native native-host run-native native-install native-image native-up native-down native-logs clean-sidecars clean-volumes compat docker-tests \+ docs-sync docs-check docs-test++help: ## List the targets below+ @grep -hE '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " %-14s %s\n", $$1, $$2}'++dev: ## Run Floci in Quarkus dev mode (hot reload on port 4566)+ ./mvnw quarkus:dev++build: ## Build the JVM package without running tests+ ./mvnw clean package -DskipTests++test: ## Run the test suite; T=Class or T=Class#method narrows it (make test T=SsmIntegrationTest)+ ./mvnw test $(if $(T),-Dtest='$(T)' -Dsurefire.failIfNoSpecifiedTests=false,)++native: ## Build the native binary with CI's flags and stage it in native/$(ARCH)/ (about 4 min)+ ./mvnw clean package -Dnative -DskipTests -B \+ -Dquarkus.native.container-build=true \+ -Dquarkus.native.builder-image=$(NATIVE_BUILDER) \+ -Dquarkus.native.additional-build-args-append="$(NATIVE_ARGS)"+ mkdir -p native/$(ARCH)+ cp target/*-runner native/$(ARCH)/application+# The builder image has no zlib-static, so a container-built binary links zlib dynamically and+# ubi9-micro ships no libz.so.1; stage it next to the binary, where the runtime image's+# LD_LIBRARY_PATH finds it. CI builds with a host Mandrel that links zlib statically.+ docker run --rm --entrypoint cat $(NATIVE_BUILDER) /usr/lib64/libz.so.1 > native/$(ARCH)/libz.so.1++native-host: ## Build a native binary for this machine with the installed GraalVM or Mandrel, no Docker: target/floci-<version>-runner+ @test -n "$(NATIVE_IMAGE_BIN)" || { echo "native-image not found: install GraalVM or Mandrel (for example 'sdk install java 25.0.3-graal') or set GRAALVM_HOME"; exit 1; }+ @echo "native-image: $(NATIVE_IMAGE_BIN) (flags $(NATIVE_HOST_FLAGS))"+ ./mvnw clean package -Dnative -DskipTests -B \+ -Dquarkus.native.additional-build-args-append="$(NATIVE_HOST_FLAGS)"+ @echo "built $(NATIVE_BIN) for $$(uname -sm)"++run-native: ## Run the host binary from native-host on port 4566, state under ./data+ @test -n "$(NATIVE_BIN)" || { echo "no target/*-runner: run 'make native-host' first"; exit 1; }+ @file "$(NATIVE_BIN)" | grep -q ELF && [ "$$(uname -s)" != Linux ] && { echo "$(NATIVE_BIN) is a Linux binary from 'make native'; run 'make native-host' for this machine"; exit 1; } || true+ $(NATIVE_BIN)++native-install: ## Copy the host binary to $(PREFIX)/bin/floci (PREFIX defaults to ~/.local)+ @test -n "$(NATIVE_BIN)" || { echo "no target/*-runner: run 'make native-host' first"; exit 1; }+ mkdir -p $(PREFIX)/bin+ cp "$(NATIVE_BIN)" $(PREFIX)/bin/floci+ @echo "installed $(PREFIX)/bin/floci"++native-image: ## Package native/$(ARCH)/ as $(NATIVE_IMAGE) with docker/Dockerfile.native-package+ @test -f native/$(ARCH)/application || { echo "native/$(ARCH)/application is missing: run 'make native' first"; exit 1; }+ docker build -f docker/Dockerfile.native-package --build-arg VERSION=local -t $(NATIVE_IMAGE) .++native-up: ## Start $(NATIVE_IMAGE) with docker compose (docker/compose.native.yml overlay) and wait for health+ $(COMPOSE_NATIVE) up -d --no-build+ @for i in $$(seq 1 60); do \+ curl -fsS -o /dev/null http://localhost:4566/_floci/health 2>/dev/null && { echo "Floci is healthy on http://localhost:4566"; exit 0; }; \+ sleep 2; \+ done; echo "Floci did not become healthy in time"; $(COMPOSE_NATIVE) logs --tail=40 floci; exit 1++native-down: ## Stop the compose stack started by native-up, then the sidecar containers Floci left behind+ $(COMPOSE_NATIVE) down+ $(MAKE) clean-sidecars++clean-sidecars: ## Remove the containers Floci started (label floci=true) that outlived it: ECR registry, EC2 instances, RDS, sidecars+ @ids=$$(docker ps -aq --filter label=floci=true --filter label=floci_emulator=floci-aws); \+ if [ -n "$$ids" ]; then docker rm -f $$ids; else echo "no Floci-managed containers left"; fi++clean-volumes: ## Remove the named volumes Floci created (label floci=true): ECR registry, RDS, OpenSearch and MSK data. Gives the next start a fresh state+ @vols=$$(docker volume ls -q --filter label=floci=true --filter label=floci_emulator=floci-aws); \+ if [ -n "$$vols" ]; then docker volume rm $$vols; else echo "no Floci-managed volumes left"; fi++native-logs: ## Tail the running Floci container's log+ $(COMPOSE_NATIVE) logs -f --tail=100 floci++compat: ## Run compatibility suites against the running Floci: SUITES="sdk-test-java compat-cdk". The suites assume a fresh state: make clean-sidecars clean-volumes before native-up+ FLOCI_START=0 COMPAT_SUITES="$(SUITES)" docker/run-docker-tests.sh++docker-tests: ## Start the JVM image with compose and run every compatibility suite in Docker+ docker/run-docker-tests.shdocs-sync: ## Regenerate the action tables in docs/services from handler source (in place)$(PYTHON) tools/docs/regen_action_docs.pydiff --git a/docker/Dockerfile.native-package b/docker/Dockerfile.native-packageindex eabea62c0f..8266a24cfe 100644--- a/docker/Dockerfile.native-package+++ b/docker/Dockerfile.native-package@@ -19,6 +19,11 @@ COPY --chmod=755 docker/localstack-parity.sh docker/healthcheck.sh /usr/local/biARG VERSION=latestENV FLOCI_VERSION=${VERSION}ENV FLOCI_STORAGE_PERSISTENT_PATH=/app/data+# A binary built in the Quarkus builder container (make native) links zlib dynamically, since+# that image has no zlib-static, and ubi9-micro ships no libz.so.1; the Makefile stages the+# library next to the binary and this is what lets the loader find it. The workflows build+# with a host Mandrel that links zlib statically, so their images carry no such file.+ENV LD_LIBRARY_PATH=/appCOPY --chmod=755 --chown=1001:root native/${TARGETARCH}/ /app/LABEL org.opencontainers.image.title="Floci" \diff --git a/docker/compose.native.yml b/docker/compose.native.ymlnew file mode 100644index 0000000000..424bb13a9c--- /dev/null+++ b/docker/compose.native.yml@@ -0,0 +1,19 @@+# Overlay for running the locally packaged native image (make native-image) instead of the+# JVM build docker-compose.yml describes:+#+# docker compose -f docker-compose.yml -f docker/compose.native.yml up -d+#+# or `make native-up`. The environment block adds what .github/workflows/compatibility.yml+# passes to the native container beyond docker-compose.yml's own settings, so a local run+# exercises CI's configuration: the SSO device page accepts only the configured principal+# (the Java portal test signs in as this id), and the RDS and Redshift endpoint hosts are+# what the JDBC suites dial.+services:+ floci:+ image: floci:local-native+ environment:+ FLOCI_SERVICES_IOT_MQTT_ENABLED: "true"+ FLOCI_SERVICES_EC2_MOCK: "true"+ FLOCI_SERVICES_REDSHIFT_ENDPOINT_HOST: floci+ FLOCI_SERVICES_RDS_ENDPOINT_HOST: floci+ FLOCI_SERVICES_SSOOIDC_LOCAL_PRINCIPAL_ID: 11111111-2222-3333-4444-555555555555diff --git a/docker/run-docker-tests.sh b/docker/run-docker-tests.shindex a8ebdeab84..6e463c7d70 100755--- a/docker/run-docker-tests.sh+++ b/docker/run-docker-tests.sh@@ -1,25 +1,35 @@#!/bin/bashset -e+# Environment:+# FLOCI_START=0 skip starting Floci; use the container already running under compose+# (for example one started by `make native-up`)+# COMPAT_SUITES="..." space-separated subset of the suite directories below to run+# (default: all of them), for example "sdk-test-java compat-cdk"+# 1. Start Floci-echo "=== Starting Floci with docker-compose ==="-docker compose up -d --build+if [ "${FLOCI_START:-1}" != "0" ]; then+ echo "=== Starting Floci with docker-compose ==="+ docker compose up -d --build-# Wait for healthy-echo "Waiting for Floci to be healthy..."-# Portable wait without 'timeout' command-MAX_RETRIES=60-COUNT=0-until curl -sf http://localhost:4566/_floci/health >/dev/null 2>&1; do- if [ $COUNT -ge $MAX_RETRIES ]; then- echo "Floci failed to become healthy in time"- exit 1- fi- sleep 1- COUNT=$((COUNT + 1))- echo -n "."-done-echo " Floci is up!"+ # Wait for healthy+ echo "Waiting for Floci to be healthy..."+ # Portable wait without 'timeout' command+ MAX_RETRIES=60+ COUNT=0+ until curl -sf http://localhost:4566/_floci/health >/dev/null 2>&1; do+ if [ $COUNT -ge $MAX_RETRIES ]; then+ echo "Floci failed to become healthy in time"+ exit 1+ fi+ sleep 1+ COUNT=$((COUNT + 1))+ echo -n "."+ done+ echo " Floci is up!"+else+ echo "=== Using the Floci already running under compose (FLOCI_START=0) ==="+fi# 2. Network setup (Floci uses floci_default from compose)NETWORK="floci_default"@@ -34,7 +44,7 @@ FLOCI_CONTAINER=$(docker compose ps -q floci 2>/dev/null | head -1)FLOCI_IP=$(docker inspect -f "{{.NetworkSettings.Networks.${NETWORK}.IPAddress}}" "$FLOCI_CONTAINER" 2>/dev/null || true)# 3. Test suites-SUITES=(+ALL_SUITES=("sdk-test-python""sdk-test-node""sdk-test-java"@@ -44,6 +54,17 @@ SUITES=("compat-terraform""compat-opentofu")+if [ -n "${COMPAT_SUITES:-}" ]; then+ read -r -a SUITES <<< "$COMPAT_SUITES"+ for suite in "${SUITES[@]}"; do+ case " ${ALL_SUITES[*]} " in+ *" $suite "*) ;;+ *) echo "unknown suite: $suite (expected one of: ${ALL_SUITES[*]})"; exit 2 ;;+ esac+ done+else+ SUITES=("${ALL_SUITES[@]}")+fi# results dirmkdir -p test-resultsdiff --git a/docker/test-run-docker-tests.sh b/docker/test-run-docker-tests.shindex 4854d38883..c9a290b8dc 100755--- a/docker/test-run-docker-tests.sh+++ b/docker/test-run-docker-tests.sh@@ -12,6 +12,7 @@ export FAIL_BUILD_SUITE=docker() {if [[ "$1" == compose && "$2" == up ]]; then+ printf 'compose-up\n' >> "$DOCKER_TEST_LOG"return 0fi@@ -115,3 +116,42 @@ for suite in "${expected_suites[@]}"; doexit 1}done++# A subset with FLOCI_START=0 runs only the named suites and never starts compose.+: > "$LOG_FILE"+export FAIL_BUILD_SUITE=++FLOCI_START=0 COMPAT_SUITES="sdk-test-java compat-cdk" bash "$REPO_ROOT/docker/run-docker-tests.sh" || {+ echo "expected the selected suites to pass" >&2+ exit 1+}++grep -q "compose-up" "$LOG_FILE" && {+ echo "FLOCI_START=0 must not start compose" >&2+ exit 1+}+for suite in sdk-test-java compat-cdk; do+ grep -qx "run:$suite" "$LOG_FILE" || {+ echo "expected selected suite to run: $suite" >&2+ exit 1+ }+done+for suite in sdk-test-python sdk-test-node sdk-test-go sdk-test-awscli compat-terraform compat-opentofu; do+ grep -q "run:$suite" "$LOG_FILE" && {+ echo "unselected suite must not run: $suite" >&2+ exit 1+ }+done++# An unknown suite name is refused before anything runs.+: > "$LOG_FILE"+if COMPAT_SUITES="sdk-test-rust" bash "$REPO_ROOT/docker/run-docker-tests.sh" 2>/dev/null; then+ echo "expected an unknown suite name to be refused" >&2+ exit 1+fi+grep -q "run:" "$LOG_FILE" && {+ echo "an unknown suite name must not run anything" >&2+ exit 1+}++echo "run-docker-tests.sh: all checks passed"diff --git a/docs/configuration/docker-images.md b/docs/configuration/docker-images.mdindex 62395ce640..ffe6a8991f 100644--- a/docs/configuration/docker-images.md+++ b/docs/configuration/docker-images.md@@ -174,3 +174,12 @@ build:context: .dockerfile: docker/Dockerfile.native # or docker/Dockerfile for fast JVM dev build```++Faster for iteration: `make native native-image` builds the binary once and packages it as+`floci:local-native`, and `make native-up` runs that image through the `docker/compose.native.yml`+overlay, which also sets the variables the compatibility workflow passes. `make compat+SUITES="sdk-test-java compat-cdk"` then runs the named suites from `compatibility-tests/` in Docker+against it. The suites expect a fresh emulator: `make clean-sidecars clean-volumes` first removes+the containers and named volumes Floci left behind (the ECR registry's volume in particular, whose+repositories Floci re-adopts on the next start), and `make native-down` removes the containers+after a run.diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.mdindex f71a91dd99..3eee9e538c 100644--- a/docs/getting-started/installation.md+++ b/docs/getting-started/installation.md@@ -76,10 +76,29 @@ java -jar target/quarkus-app/quarkus-run.jar### Build a native executable+The Makefile builds the binary inside the Quarkus builder container, so nothing but Docker is+needed on the host, and packages it the way the release workflow does:++```bash+make native # Linux binary for the host architecture, staged in native/<arch>/+make native-image # packaged as floci:local-native with docker/Dockerfile.native-package+make native-up # started with docker compose, waits for /_floci/health+```++To run Floci as a plain binary on your machine, install GraalVM or Mandrel with `native-image`+(for example `sdk install java 25.0.3-graal`, or set `GRAALVM_HOME`) and build on the host:+```bash-mvn clean package -Pnative -DskipTests-./target/floci-runner+make native-host # target/floci-<version>-runner for this machine, no Docker+make run-native # start it on port 4566, state under ./data+make native-install # copy it to ~/.local/bin/floci (PREFIX=/usr/local to change)```+On Mandrel 25.0.4 and later the build needs `-H:-AOTSingleCallsiteInline`: Quarkus 3.39 turns+single-callsite inlining on and the generated REST invokers then outgrow the aarch64 branch range.+`make native-host` adds the flag when the installed `native-image` is Mandrel; GraalVM does not+know it and does not need it.+!!! note- Native compilation requires GraalVM or Mandrel with the `native-image` tool on your PATH. Build time is typically 2–5 minutes.\ No newline at end of file+ A native build takes about four minutes with the quick profile the Makefile uses (`-Ob`); a+ full optimisation build takes longer. `make help` lists every target.\ No newline at end of file