Light, fluffy, and always free - The AWS Local Emulator alternative
ci: run focused native compatibility tests (#1375)
David Phillips01b38bb4bbe5
diff --git a/.github/workflows/compatibility.yml b/.github/workflows/compatibility.ymlindex 857bf64b1c..71ca378225 100644--- a/.github/workflows/compatibility.yml+++ b/.github/workflows/compatibility.yml@@ -5,6 +5,7 @@ on:paths:- 'src/**'- 'pom.xml'+ - 'docker/Dockerfile.native'- 'docker/Dockerfile.jvm-package'- 'compatibility-tests/**'- '.github/workflows/compatibility.yml'@@ -18,7 +19,7 @@ concurrency:jobs:build:- name: Build floci image+ name: Build JVM floci imageruns-on: ubuntu-latesttimeout-minutes: 20steps:@@ -56,7 +57,7 @@ jobs:retention-days: 1compat-test:- name: ${{ matrix.test }}+ name: JVM / ${{ matrix.test }}needs: buildruns-on: ubuntu-latesttimeout-minutes: 20@@ -161,3 +162,121 @@ jobs:- name: Dump floci logsif: failure()run: docker logs floci++ build-native:+ name: Build native floci image+ runs-on: ubuntu-latest+ timeout-minutes: 45+ steps:+ - uses: actions/[email protected]++ - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0++ - name: Build native Docker image+ uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0+ with:+ context: .+ file: docker/Dockerfile.native+ tags: floci:test-native+ outputs: type=docker,dest=/tmp/floci-native-image.tar+ cache-from: type=gha,scope=floci-native+ cache-to: type=gha,scope=floci-native,mode=max++ - name: Compress native image+ run: gzip /tmp/floci-native-image.tar++ - name: Upload native floci image+ uses: actions/upload-artifact@v7+ with:+ name: floci-native-image+ path: /tmp/floci-native-image.tar.gz+ retention-days: 1++ native-compat-test:+ name: native / ${{ matrix.test }}+ needs: build-native+ runs-on: ubuntu-latest+ timeout-minutes: 20+ strategy:+ fail-fast: false+ matrix:+ test:+ - sdk-test-java+ - sdk-test-awscli++ steps:+ - name: Download native floci image+ uses: actions/download-artifact@v8+ with:+ name: floci-native-image+ path: /tmp++ - name: Load native floci image+ run: gunzip -c /tmp/floci-native-image.tar.gz | docker load++ - name: Create Docker network+ run: docker network create compat-net++ - name: Start floci+ run: |+ DOCKER_GID=$(stat -c '%g' /var/run/docker.sock)+ docker run -d --name floci --network compat-net \+ -p 4566:4566 \+ -v /var/run/docker.sock:/var/run/docker.sock \+ --group-add "$DOCKER_GID" \+ -e FLOCI_BASE_URL=http://floci:4566 \+ -e FLOCI_SERVICES_DOCKER_NETWORK=compat-net \+ -e FLOCI_HOSTNAME=floci \+ -e FLOCI_SERVICES_LAMBDA_HOT_RELOAD_ENABLED=true \+ -e FLOCI_TLS_ENABLED=true \+ -e FLOCI_SERVICES_EC2_MOCK=true \+ floci:test-native++ - name: Wait for floci to be ready+ run: timeout 60 bash -c 'until curl -sf http://localhost:4566/ >/dev/null 2>&1; do sleep 1; done'++ - name: Checkout repository+ uses: actions/[email protected]+ with:+ sparse-checkout: compatibility-tests++ - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0++ - name: Build test image+ uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0+ with:+ context: compatibility-tests/${{ matrix.test }}+ load: true+ tags: compat-${{ matrix.test }}+ cache-from: type=gha,scope=${{ matrix.test }}+ cache-to: type=gha,scope=${{ matrix.test }},mode=max++ - name: Run tests+ id: tests+ run: |+ mkdir -p test-results+ FLOCI_IP=$(docker inspect -f '{{(index .NetworkSettings.Networks "compat-net").IPAddress}}' floci)++ EXTRA_ARGS=""+ if [ "${{ matrix.test }}" = "sdk-test-java" ]; then+ mkdir -p /tmp/floci-hot-reload+ EXTRA_ARGS="$EXTRA_ARGS -v /tmp/floci-hot-reload:/tmp/floci-hot-reload -e HOT_RELOAD_BASE_DIR=/tmp/floci-hot-reload"+ fi++ docker run --rm --network compat-net \+ --dns "${FLOCI_IP}" \+ -e FLOCI_ENDPOINT=http://floci:4566 \+ -e FLOCI_S3_VHOST_ENDPOINT=http://floci:4566 \+ -v "$(pwd)/test-results:/results" \+ $EXTRA_ARGS \+ compat-${{ matrix.test }}++ - name: Generate test summary+ if: always() && steps.tests.outcome != 'skipped'+ uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # v2+ with:+ paths: test-results/*.xml++ - name: Dump floci logs+ if: failure()+ run: docker logs flocidiff --git a/src/main/java/io/github/hectorvent/floci/services/apigateway/VtlTemplateEngine.java b/src/main/java/io/github/hectorvent/floci/services/apigateway/VtlTemplateEngine.javaindex 20cc982d03..df657dbb8e 100644--- a/src/main/java/io/github/hectorvent/floci/services/apigateway/VtlTemplateEngine.java+++ b/src/main/java/io/github/hectorvent/floci/services/apigateway/VtlTemplateEngine.java@@ -5,9 +5,27 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;+import io.quarkus.runtime.annotations.RegisterForReflection;import org.apache.velocity.VelocityContext;import org.apache.velocity.app.VelocityEngine;+import org.apache.velocity.runtime.ParserPoolImpl;import org.apache.velocity.runtime.RuntimeConstants;+import org.apache.velocity.runtime.directive.Break;+import org.apache.velocity.runtime.directive.Define;+import org.apache.velocity.runtime.directive.Evaluate;+import org.apache.velocity.runtime.directive.Foreach;+import org.apache.velocity.runtime.directive.Include;+import org.apache.velocity.runtime.directive.Macro;+import org.apache.velocity.runtime.directive.Parse;+import org.apache.velocity.runtime.directive.Stop;+import org.apache.velocity.runtime.parser.StandardParser;+import org.apache.velocity.runtime.resource.ResourceCacheImpl;+import org.apache.velocity.runtime.resource.ResourceManagerImpl;+import org.apache.velocity.runtime.resource.loader.FileResourceLoader;+import org.apache.velocity.runtime.resource.loader.StringResourceLoader;+import org.apache.velocity.runtime.resource.util.StringResourceRepositoryImpl;+import org.apache.velocity.util.introspection.TypeConversionHandlerImpl;+import org.apache.velocity.util.introspection.UberspectImpl;import java.io.StringWriter;import java.net.URLDecoder;@@ -24,6 +42,28 @@ import java.util.Map;* {@code $input}, {@code $util}, {@code $context}, {@code $stageVariables}.*/@ApplicationScoped+@RegisterForReflection(targets = {+ VtlTemplateEngine.InputVariable.class,+ VtlTemplateEngine.UtilVariable.class,+ VtlTemplateEngine.ResponseOverride.class,+ UberspectImpl.class,+ TypeConversionHandlerImpl.class,+ ResourceManagerImpl.class,+ ResourceCacheImpl.class,+ ParserPoolImpl.class,+ FileResourceLoader.class,+ StringResourceLoader.class,+ StringResourceRepositoryImpl.class,+ Foreach.class,+ Include.class,+ Parse.class,+ Macro.class,+ Evaluate.class,+ Break.class,+ Define.class,+ Stop.class,+ StandardParser.class,+})public class VtlTemplateEngine {private final VelocityEngine engine;diff --git a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.javaindex 0e80c5f78e..59b4ae5304 100644--- a/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java+++ b/src/main/java/io/github/hectorvent/floci/services/glue/GlueService.java@@ -18,6 +18,7 @@ import io.github.hectorvent.floci.services.glue.schemaregistry.SchemaToColumnsCoimport io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaId;import io.github.hectorvent.floci.services.glue.schemaregistry.model.SchemaVersion;import io.github.hectorvent.floci.services.resourcegroupstagging.ResourceGroupsTaggingService;+import io.quarkus.runtime.annotations.RegisterForReflection;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -876,18 +877,22 @@ public class GlueService {@JsonProperty("TableName") String tableName,@JsonProperty("ErrorDetail") ErrorDetail errorDetail) {}+ @RegisterForReflectionpublic record BatchCreatePartitionError(@JsonProperty("PartitionValues") List<String> partitionValues,@JsonProperty("ErrorDetail") ErrorDetail errorDetail) {}+ @RegisterForReflectionpublic record BatchUpdatePartitionEntry(@JsonProperty("PartitionValueList") List<String> partitionValueList,@JsonProperty("PartitionInput") Partition partitionInput) {}+ @RegisterForReflectionpublic record BatchUpdatePartitionError(@JsonProperty("PartitionValueList") List<String> partitionValueList,@JsonProperty("ErrorDetail") ErrorDetail errorDetail) {}+ @RegisterForReflectionpublic record ErrorDetail(@JsonProperty("ErrorCode") String errorCode,@JsonProperty("ErrorMessage") String errorMessage) {}@@ -896,6 +901,7 @@ public class GlueService {@JsonProperty("ColumnStatisticsList") List<Map<String, Object>> columnStatisticsList,@JsonProperty("Errors") List<ColumnError> errors) {}+ @RegisterForReflectionpublic record ColumnError(@JsonProperty("ColumnName") String columnName,@JsonProperty("Error") ErrorDetail error) {}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 608778c040..e5fc08f951 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -21,12 +21,19 @@ quarkus:delay-enabled: truenative:resources:- # The Pricing service reads its bundled snapshot, and EC2 reads its image- # catalog (ec2/image-catalog.yaml), at runtime via+ # The Pricing service reads its bundled snapshot, EC2 reads its image+ # catalog (ec2/image-catalog.yaml), AppSync's GraphQL parser reads its+ # message bundles, and API Gateway's Velocity engine reads its default+ # properties at runtime via# ClassLoader#getResourceAsStream. Quarkus's static resource analysis# cannot detect these dynamically-built paths, so they are registered# explicitly to ensure GraalVM includes every file in the native image.- includes: pricing-snapshots/**,ui/*.html,ec2/*.yaml+ includes:+ - pricing-snapshots/**+ - ui/*.html+ - ec2/*.yaml+ - i18n/*.properties+ - org/apache/velocity/runtime/defaults/*.propertiesadditional-build-args:- --report-unsupported-elements-at-runtime- --allow-incomplete-classpath