Light, fluffy, and always free - The AWS Local Emulator alternative
test(compat): run the Java SDK suite nightly in cn-north-1 against an allowlist of known failures (#4790)
Hector Venturaea06896984d7
diff --git a/.github/ci/compat-allowlist-check.py b/.github/ci/compat-allowlist-check.pynew file mode 100644index 0000000000..effef358de--- /dev/null+++ b/.github/ci/compat-allowlist-check.py@@ -0,0 +1,94 @@+#!/usr/bin/env python3+"""Compare a compat suite's JUnit results with an allowlist of known failures.++Usage: compat-allowlist-check.py <results-dir> <allowlist> [--test-sources <dir>]++Reads every TEST-*.xml under <results-dir> and collects the test cases that+failed or errored. The allowlist names the failures that are known and+accepted, one per line: `Class#method` for one test or `Class` for every test+in a class, by simple class name (no package). `#`-comments and blank lines+are ignored. A missing allowlist file is an empty one.++Fails when:+ - a test fails that the allowlist does not name (a regression), or+ - an allowlist entry matches no failing test (the gap is closed: delete the+ line, so the list only ever shrinks), or+ - no results were found (the suite did not run), or+ - with --test-sources, there are fewer result files than test classes in that+ source tree (the suite stopped partway: Maven exits non-zero for allowlisted+ failures too, so its status cannot tell the two apart).+"""+import argparse+import re+import sys+import xml.etree.ElementTree as ET+from pathlib import Path++TEST_ANNOTATION = re.compile(r'@(?:Test|ParameterizedTest|RepeatedTest|TestFactory|TestTemplate)\b')+++def load_allowlist(path):+ if not Path(path).exists():+ return []+ # `Class#method` carries a `#`, so a comment is a line starting with `#` or a ` #` suffix.+ entries = []+ for raw in Path(path).read_text().splitlines():+ line = raw.split(' #', 1)[0].strip()+ if line and not line.startswith('#'):+ entries.append(line)+ return entries+++def failing_tests(results_dir):+ failed = set()+ files = sorted(Path(results_dir).glob('TEST-*.xml'))+ for path in files:+ root = ET.parse(path).getroot()+ for case in root.iter('testcase'):+ if case.find('failure') is not None or case.find('error') is not None:+ simple_class = case.get('classname', '').rsplit('.', 1)[-1]+ failed.add(f"{simple_class}#{case.get('name', '')}")+ return files, failed+++def test_class_count(source_dir):+ return sum(1 for path in Path(source_dir).rglob('*.java')+ if TEST_ANNOTATION.search(path.read_text(encoding='utf-8')))+++def matches(entry, test_id):+ return test_id == entry if '#' in entry else test_id.split('#', 1)[0] == entry+++def main(argv):+ parser = argparse.ArgumentParser(description='Compare JUnit results with an allowlist.')+ parser.add_argument('results_dir')+ parser.add_argument('allowlist')+ parser.add_argument('--test-sources', help='test source tree; requires a report per test class')+ args = parser.parse_args(argv[1:])+ files, failed = failing_tests(args.results_dir)+ if not files:+ print(f'::error::No TEST-*.xml results under {args.results_dir}: the suite did not run.')+ return 1+ if args.test_sources:+ expected = test_class_count(args.test_sources)+ if len(files) < expected:+ print(f'::error::{len(files)} result files for {expected} test classes: the suite stopped partway.')+ return 1+ allowlist = load_allowlist(args.allowlist)++ unexpected = sorted(t for t in failed if not any(matches(e, t) for e in allowlist))+ stale = [e for e in allowlist if not any(matches(e, t) for t in failed)]+ expected = sorted(t for t in failed if t not in unexpected)++ print(f'{len(files)} result files, {len(failed)} failing tests, '+ f'{len(expected)} allowlisted, {len(allowlist)} allowlist entries.')+ for test_id in unexpected:+ print(f'::error::Not in the allowlist: {test_id}')+ for entry in stale:+ print(f'::error::Allowlist entry no longer fails, delete it: {entry}')+ return 1 if unexpected or stale else 0+++if __name__ == '__main__':+ sys.exit(main(sys.argv))diff --git a/.github/ci/compat-partition-allowlist-cn-north-1.txt b/.github/ci/compat-partition-allowlist-cn-north-1.txtnew file mode 100644index 0000000000..68a8763c77--- /dev/null+++ b/.github/ci/compat-partition-allowlist-cn-north-1.txt@@ -0,0 +1,137 @@+# Known failures of the Java SDK suite against a Floci deployed in cn-north-1, with the+# fixture clients signing for cn-north-1 (.github/workflows/compat-partitions.yml).+# Checked by .github/ci/compat-allowlist-check.py: one `Class#method` per line; a failure+# missing from this list, or an entry that no longer fails, fails the run. Delete an entry+# as soon as its test passes. Measured 2026-10-01 on main 231655b70 plus #4787, JVM image;+# failures that also occur in the commercial control run are environmental and not listed.++# The test pins commercial values: an ARN built as arn:aws:<svc>:us-east-1:..., an+# expected arn:aws: prefix, a us-east-1 availability zone or host, or a client created with+# Region.US_EAST_1 next to the cn-north-1 fixture clients. Floci answers correctly; the+# test should take the partition and region from TestFixtures.+AcmTest#testImportCertificate+AcmTest#testRequestCertificate+ApiGatewayOpenApiImportCompatibilityTest#sdkImportAndPutResolveAwsVariables+ApiGatewayV2ManagementTest#createApi+ApiGatewayV2WebSocketAndExtendedOpsTest#getTagsEmpty+ApiGatewayV2WebSocketAndExtendedOpsTest#tagging+AppSyncTest#createDataSource_hasDataSourceArn+AppSyncTest#createFunction_hasFunctionArn+AppSyncTest#createGraphqlApi+AppSyncTest#createResolver_hasResolverArn+AthenaTest#listTagsForResourceReturnsTagsSetOnCreation+CloudFormationAcmCertificateTest#certificateOptionsFollowTheTemplateThroughCreateAndUpdate+CloudFormationEventBusTest#createsValidatesUpdatesAndDeletesEventBusThroughAwsSdk+CloudFormationEventSourceMappingTest#createDynamoDbStreamStack_withRetryAndDestinationOptions+CloudFormationIamUserTest#createsValidatesUpdatesAndDeletesIamUserThroughAwsSdk+CloudFormationRedshiftClusterTest#provisionsRedshiftClusterAndConnectsViaJdbc+CloudHsmV2Test#testDeleteHsmSelectors+CloudHsmV2Test#testFullLifecycle+CloudTrailTest#describeTrailsByName+CloudWatchLogsCrossAccountPolicyTest#destinationAndAccountPolicyOperationsUseAwsSdk+CloudWatchLogsDescribeLogGroupsTest#returnsAwsLogGroupArnFields+CloudWatchLogsMetricFilterTest#quietDefaultsDriveASumAlarmWithoutLaterIngestion+CloudWatchLogsMetricFilterTest#recordedDefaultsAndQuietTrafficReadIdenticallyOverJsonAndQuery+CloudWatchLogsMetricFilterTest#recordedExtractionAndOrdinaryDimensionSeriesReadIdenticallyOverJsonAndQuery+CloudWatchLogsMetricFilterTest#recordedSystemDimensionsDoNotAttachToPatternNonmatchDefaults+CodeDeployTest#tagAndListTags+CodeDeployTest#untagResource+DmsTest#replicationSubnetGroupLifecycleUsesAwsSdkWireContract+Ec2DescribeSpotPriceHistoryTest#describeSpotPriceHistoryFiltersByInstanceTypeAvailabilityZoneAndProductDescription+Ec2DescribeSpotPriceHistoryTest#describeSpotPriceHistoryFiltersUsingGenericFilters+Ec2DescribeSpotPriceHistoryTest#describeSpotPriceHistoryPriceIsBelowOnDemandAndMonotonicWithCapacity+Ec2DescribeSpotPriceHistoryTest#describeSpotPriceHistoryReturnsPricesForWindowsPlatform+Ec2DescribeSpotPriceHistoryTest#describeSpotPriceHistoryReturnsStableIdenticalPrices+EcrTest#createRepository+EksTest#fargateProfileLifecycle+GlueCatalogTest#createDatabaseWithTags_tagsReturnedByResourceGroupsTaggingApi+GuardDutyTest#detectorLifecycleUsesAwsSdkWireShapes+IamTest#generateAndGetServiceLastAccessedDetails+IamTest#getServiceLastAccessedDetailsWithEntitiesDescribesReachingEntities+IamTest#listPoliciesGrantingServiceAccessEchoesEachRequestedNamespace+IotTest#domainConfigurationLifecycle+MacieOrganizationAdministrationTest#organizationAdministrationUsesAwsSdk+NeptuneTest#createCluster+NeptuneTest#createInstance+OrganizationsTest#testFullLifecycle+PipesTest#batchSizeInSourceParameters+PipesTest#filterCriteriaFiltersMessages+PipesTest#sqsToSqsForwarding+ResourceExplorer2Test#viewFiltersFilterStringDeserializes+Route53Test#crossAccountVpcAssociationLifecycleUsesAwsSdkWireContract+S3TablesTest#createsAndGetsTableBucket+S3VectorsTest#createIndex+S3VectorsTest#createVectorBucket+SecretsManagerTest#rotateSecretStub+SesConfigurationSetEventDestinationTest#v2_eventBridgeDefaultBusAcceptedAndInvalidBusRejected(String, String)[1]+SesConfigurationSetEventDestinationTest#v2_eventBridgeDefaultBusAcceptedAndInvalidBusRejected(String, String)[2]+SesConfigurationSetEventDestinationTest#v2_eventBridgeDefaultBusAcceptedAndInvalidBusRejected(String, String)[3]+SesEventPublishingTest#+SesTagResourceTest#contactList_createWithTags_visibleViaListTagsAndGet+SesTagResourceTest#contactList_tagAndUntag_lifecycle+SesTagResourceTest#customVerificationEmailTemplate_createWithTags_tagAndUntag_lifecycle+SesTagResourceTest#dedicatedIpPool_createWithTags_tagAndUntag_lifecycle+SesTagResourceTest#emailIdentity_createWithTags_visibleViaListTagsAndGet+SesTagResourceTest#emailIdentity_tagAndUntag_lifecycle+SesTagResourceTest#emailTemplate_createWithTags_visibleViaListTagsAndGet+SesTagResourceTest#emailTemplate_tagAndUntag_lifecycle+SesTagResourceTest#tagResource_addsTagsAndListReflectsThem+SesTagResourceTest#tagResource_existingKeyReplacesValue+SesTagResourceTest#tagResource_unknownConfigurationSet_throwsNotFound+SesTagResourceTest#untagResource_removesSpecifiedKeys+SsoAdminAccountAssignmentTest#createApplicationAssignmentUsesAwsSdk+SsoAdminAccountAssignmentTest#createApplicationUsesAwsSdk+SsoAdminAccountAssignmentTest#createInstanceUsesAwsSdk+SsoAdminAccountAssignmentTest#createTrustedTokenIssuerUsesAwsSdk+SsoAdminAccountAssignmentTest#deleteApplicationAccessScopeUsesAwsSdk+SsoAdminAccountAssignmentTest#deleteApplicationAssignmentUsesAwsSdk+SsoAdminAccountAssignmentTest#deleteApplicationAuthenticationMethodUsesAwsSdk+SsoAdminAccountAssignmentTest#deleteApplicationGrantUsesAwsSdk+SsoAdminAccountAssignmentTest#deleteApplicationUsesAwsSdk+SsoAdminAccountAssignmentTest#deleteInstanceUsesAwsSdk+SsoOidcTest#createTokenWithIamUsesAwsSdk++# The test exercises something that does not exist in China, and Floci answers as AWS+# does there (Marketplace and Lightsail are commercial-only, an eu-central-1 bucket cannot be+# created through a China endpoint, no global STS token version, China interface endpoint+# names, the China SAML sign-on URL). These need a commercial-only guard in the test.+Ec2Tests#describeVpcEndpointServices+IamTest#getAccountSummary+LightsailTest#sdkCanCreateAndManageCoreLightsailResources+MarketplaceAgreementTest#usesAwsSdkWireContract+MarketplaceCatalogTest#usesAwsSdkWireContract+MarketplaceDeploymentTest#usesAwsSdkWireContract+MarketplaceDiscoveryTest#usesAwsSdkWireContract+MarketplaceEntitlementTest#usesAwsSdkWireContract+S3Test#createBucketWithLocationConstraint+S3Test#deleteEuBucket+S3Test#getBucketLocationEuCentral1+StsTest#assumeRoleWithSaml+StsTest#assumeRoleWithSamlAssumedRoleUser++# Floci bugs: RdsService checks a parsed ARN against the literal partition "aws", so RDS+# resources created in China are not found after creation; the seeded IAM Identity Center+# instance ARN is always arn:aws:. CloudFormationRdsDynamicReferenceTest and SsoPortalTest were+# reproduced alone (they pass under us-east-1); the other RDS entries fail the same way. Fixed in+# a follow-up PR.+CloudFormationRdsDynamicReferenceTest#resolvesSecureSsmMasterPassword+RdsControlPlaneTest#sdkCopiesModifiesAndDeletesManualSnapshots+RdsControlPlaneTest#sdkCreatesCopiesRestoresAndDeletesClusterSnapshots+RdsControlPlaneTest#sdkFaultsDeletingAnOptionGroupStillAttachedToAnInstance+RdsControlPlaneTest#sdkRegistersDescribesAndDeregistersDbProxyInstanceTarget+RdsControlPlaneTest#sdkResizesAndUpgradesAStandaloneInstance+RdsControlPlaneTest#sdkRoundTripsAuroraServerlessV2ScalingConfiguration+RdsControlPlaneTest#sdkRoundTripsDbProxyAndItsDefaultTargetGroup+RdsControlPlaneTest#sdkRoundTripsIamDefaultAuthAndProxyUpdates+RdsControlPlaneTest#sdkStopsAndStartsAStandaloneInstance+RdsControlPlaneTest#sdkUnmarshalsDbSubnetGroupSubnets+RdsIamTokenCompatTest#sdkGeneratedTokenIsAccepted+RdsJdbcCompatTest#connectWithIamAuthToken+RdsJdbcCompatTest#createDbInstanceAndConnectWithPassword+RdsJdbcCompatTest#modifyKeepsProxyReachableAndDeleteReleasesPort+RdsJdbcCompatTest#toggleIamViaModifyOnRunningInstance+SsoPortalTest#listAccountsUsesAwsSdk++# Passes alone: in the full run OrganizationsTest#testFullLifecycle fails its arn:aws: assertion+# before cleanup, leaving the caller as the Organizations management account.+SsoAdminAccountAssignmentTest#createInstanceQuotaUsesAwsSdkdiff --git a/.github/workflows/compat-partitions.yml b/.github/workflows/compat-partitions.ymlnew file mode 100644index 0000000000..b9fcaadfd5--- /dev/null+++ b/.github/workflows/compat-partitions.yml@@ -0,0 +1,160 @@+name: Partition Compatibility++# The Java SDK suite against a Floci deployed in a non-commercial partition, so+# the SDK's own partition logic (arn:aws-cn: ARNs, China signing scopes, the+# LocationConstraint S3 CreateBucket sends) has to round-trip end to end. Part+# of #4272.+#+# Runs after each Nightly against the image it just published, not per PR: it+# is one more ~15 minute native suite, and the per-PR legs already queue for+# the shared runner pools (see compatibility.yml). Known failures live in+# .github/ci/compat-partition-allowlist-<region>.txt (a region without one has+# none); a failure missing from it, or an entry that no longer fails, fails the+# run, so the list only shrinks.++on:+ workflow_run:+ workflows: [Nightly]+ types: [completed]+ workflow_dispatch:+ inputs:+ image:+ description: Floci image to test+ default: floci/floci:nightly+ region:+ description: Region Floci and the SDK clients default to+ default: cn-north-1++permissions:+ contents: read++concurrency:+ group: ${{ github.workflow }}-${{ github.ref }}+ cancel-in-progress: true++jobs:+ sdk-test-java:+ name: sdk-test-java / ${{ inputs.region || 'cn-north-1' }}+ if: >-+ !github.event.repository.fork+ && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success')+ runs-on: ubuntu-24.04-arm+ timeout-minutes: 30+ env:+ IMAGE: ${{ inputs.image || 'floci/floci:nightly' }}+ REGION: ${{ inputs.region || 'cn-north-1' }}++ steps:+ # The tag is mutable, so the suite and allowlist are checked out at the commit this exact+ # image was built from (its OCI revision label, which Nightly sets), never a guessed one.+ - name: Pull the image and read the commit it was built from+ id: image+ run: |+ docker pull "$IMAGE"+ revision=$(docker inspect -f '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$IMAGE")+ if [ -z "$revision" ] || [ "$revision" = "<no value>" ]; then+ echo "::error::$IMAGE has no org.opencontainers.image.revision label to match the tests to"+ exit 1+ fi+ echo "revision=$revision" >> "$GITHUB_OUTPUT"++ - name: Checkout repository+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ ref: ${{ steps.image.outputs.revision }}+ sparse-checkout: |+ compatibility-tests+ .github/ci++ - name: Create Docker network+ run: docker network create compat-net++ - name: Start floci+ run: |+ DOCKER_GID=$(stat -c '%g' /var/run/docker.sock)+ docker run -d --name floci --network compat-net \+ -p 4566:4566 \+ -v /var/run/docker.sock:/var/run/docker.sock \+ --group-add "$DOCKER_GID" \+ -e FLOCI_DEFAULT_REGION="$REGION" \+ -e FLOCI_BASE_URL=http://floci:4566 \+ -e FLOCI_SERVICES_DOCKER_NETWORK=compat-net \+ -e FLOCI_HOSTNAME=floci \+ -e FLOCI_SERVICES_IOT_MQTT_ENABLED=true \+ -e FLOCI_SERVICES_LAMBDA_HOT_RELOAD_ENABLED=true \+ -e FLOCI_TLS_ENABLED=true \+ -e FLOCI_SERVICES_EC2_MOCK=true \+ -e FLOCI_SERVICES_REDSHIFT_ENDPOINT_HOST=floci \+ -e FLOCI_SERVICES_RDS_ENDPOINT_HOST=floci \+ -e FLOCI_SERVICES_SSOOIDC_LOCAL_PRINCIPAL_ID=11111111-2222-3333-4444-555555555555 \+ "$IMAGE"++ - name: Wait for floci to be healthy+ run: |+ health=starting+ for _ in $(seq 1 60); do+ health=$(docker inspect -f '{{.State.Health.Status}}' floci)+ if [ "$health" = healthy ]; then+ exit 0+ fi+ if [ "$health" = unhealthy ]; then+ break+ fi+ sleep 1+ done+ echo "::error::floci did not report healthy (last status: $health)"+ docker logs --tail 50 floci+ exit 1++ - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0++ - name: Build test image+ uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0+ with:+ context: compatibility-tests/sdk-test-java+ load: true+ tags: compat-sdk-test-java+ cache-from: type=gha,scope=sdk-test-java++ # The suite's exit status is not the verdict here: the allowlist check below is.+ - name: Run tests+ id: tests+ continue-on-error: true+ run: |+ mkdir -p test-results /tmp/floci-hot-reload+ FLOCI_IP=$(docker inspect -f '{{(index .NetworkSettings.Networks "compat-net").IPAddress}}' floci)+ docker run --rm --network compat-net \+ --dns "${FLOCI_IP}" \+ -e AWS_REGION="$REGION" \+ -e FLOCI_ENDPOINT=http://floci:4566 \+ -e FLOCI_S3_VHOST_ENDPOINT=http://floci:4566 \+ -e HOT_RELOAD_BASE_DIR=/tmp/floci-hot-reload \+ -v /tmp/floci-hot-reload:/tmp/floci-hot-reload \+ -v "$(pwd)/test-results:/results" \+ compat-sdk-test-java++ # Maven exits non-zero for the allowlisted failures too, so its status cannot tell a+ # finished run from an aborted one; the checker requires a report for every test class.+ - name: Compare failures with the allowlist+ run: >-+ python3 .github/ci/compat-allowlist-check.py test-results+ ".github/ci/compat-partition-allowlist-$REGION.txt"+ --test-sources compatibility-tests/sdk-test-java/src/test/java++ - name: Generate test summary+ if: always()+ uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # v2+ with:+ paths: test-results/*.xml++ - name: Upload test results+ if: always()+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: compat-partition-results-${{ env.REGION }}+ path: test-results/+ if-no-files-found: warn++ - name: Dump floci logs+ if: failure() || cancelled()+ run: docker logs --tail 300 flocidiff --git a/compatibility-tests/sdk-test-java/README.md b/compatibility-tests/sdk-test-java/README.mdindex f8d3da318f..5789c06497 100644--- a/compatibility-tests/sdk-test-java/README.md+++ b/compatibility-tests/sdk-test-java/README.md@@ -51,11 +51,16 @@ just test-java## Configuration-| Variable | Default | Description |-| ---------------- | ----------------------- | ----------------------- |-| `FLOCI_ENDPOINT` | `http://localhost:4566` | Floci emulator endpoint |--AWS credentials are always `test` / `test` / `us-east-1`.+| Variable | Default | Description |+| ---------------- | ----------------------- | -------------------------------------------------- |+| `FLOCI_ENDPOINT` | `http://localhost:4566` | Floci emulator endpoint |+| `AWS_REGION` | `us-east-1` | Region the `TestFixtures` clients sign for |++AWS credentials are always `test` / `test`. Setting `AWS_REGION=cn-north-1` (with Floci started+with `FLOCI_DEFAULT_REGION=cn-north-1`) runs the suite in the China partition; the nightly+Partition Compatibility workflow does exactly that, and tolerates only the failures listed in+`.github/ci/compat-partition-allowlist-cn-north-1.txt`. Tests that build their own client with an+explicit region keep that region.## Metric filter publishing checksdiff --git a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.javaindex afa787a256..79fb8e0a7d 100644--- a/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java+++ b/compatibility-tests/sdk-test-java/src/main/java/com/floci/test/TestFixtures.java@@ -181,13 +181,23 @@ public final class TestFixtures {ENDPOINT = URI.create(endpointStr);}- private static final Region REGION = Region.US_EAST_1;+ /**+ * The region every fixture client signs for: {@code AWS_REGION} when set (the variable the+ * SDK itself reads), else {@code us-east-1}. The China compat run sets {@code cn-north-1}, so+ * the same suite exercises a non-commercial partition end to end.+ */+ private static final Region REGION = resolveRegion();private static final StaticCredentialsProvider CREDENTIALS =StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test"));private TestFixtures() {}+ private static Region resolveRegion() {+ String region = System.getenv("AWS_REGION");+ return region == null || region.isBlank() ? Region.US_EAST_1 : Region.of(region.trim());+ }+/*** Returns true when running against real AWS (no endpoint override).*/diff --git a/docs/configuration/partitions.md b/docs/configuration/partitions.mdindex 936357a81a..ba2d2a4b3d 100644--- a/docs/configuration/partitions.md+++ b/docs/configuration/partitions.md@@ -151,6 +151,15 @@ names the caller's. ARNs with nothing storedbehind them, the `root` fallback of `GetCallerIdentity` and `federated-user`, take the request'spartition.+## How it is tested++Every partition-dependent rule has unit and integration tests over all eight partitions. On top+of that, the nightly Partition Compatibility workflow runs the Java SDK compatibility suite+against a Floci deployed with `FLOCI_DEFAULT_REGION=cn-north-1`, with the SDK clients signing for+`cn-north-1`, so the SDK's own China partition handling has to round-trip end to end. Its known+failures are listed in `.github/ci/compat-partition-allowlist-cn-north-1.txt`; the list only+shrinks.+## Open questionsThese values have no published source Floci can cite, so it does not guess them; they keep