Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): serialize concurrent mutations of shared resources per resource (#1786)
Hector Ventura342bb52df474
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex 2377ac4f88..99a113e8c0 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -439,47 +439,76 @@ public class Ec2Service {return true;}- public synchronized void createNetworkAclEntry(String region, String networkAclId, int ruleNumber, String protocol,+ public void createNetworkAclEntry(String region, String networkAclId, int ruleNumber, String protocol,String ruleAction, boolean egress, String cidrBlock, Integer from, Integer to,boolean replace) {- NetworkAcl acl = getRequiredNetworkAcl(region, networkAclId);- boolean exists = acl.getEntries().stream()- .anyMatch(e -> e.getRuleNumber() == ruleNumber && e.isEgress() == egress);- if (!replace && exists) {- throw new AwsException("NetworkAclEntryAlreadyExists",- "The network acl entry identified by " + ruleNumber + " already exists.", 400);- }- acl.getEntries().removeIf(e -> e.getRuleNumber() == ruleNumber && e.isEgress() == egress);- NetworkAclEntry entry = naclEntry(ruleNumber, protocol, ruleAction, egress, cidrBlock);- entry.setPortRangeFrom(from);- entry.setPortRangeTo(to);- acl.getEntries().add(entry);- networkAcls.put(key(region, networkAclId), acl);+ synchronized (lockFor(key(region, networkAclId))) {+ NetworkAcl acl = getRequiredNetworkAcl(region, networkAclId);+ boolean exists = acl.getEntries().stream()+ .anyMatch(e -> e.getRuleNumber() == ruleNumber && e.isEgress() == egress);+ if (!replace && exists) {+ throw new AwsException("NetworkAclEntryAlreadyExists",+ "The network acl entry identified by " + ruleNumber + " already exists.", 400);+ }+ List<NetworkAclEntry> next = new ArrayList<>(acl.getEntries());+ next.removeIf(e -> e.getRuleNumber() == ruleNumber && e.isEgress() == egress);+ NetworkAclEntry entry = naclEntry(ruleNumber, protocol, ruleAction, egress, cidrBlock);+ entry.setPortRangeFrom(from);+ entry.setPortRangeTo(to);+ next.add(entry);+ acl.setEntries(next);+ networkAcls.put(key(region, networkAclId), acl);+ }}- public synchronized void deleteNetworkAclEntry(String region, String networkAclId, int ruleNumber, boolean egress) {- NetworkAcl acl = getRequiredNetworkAcl(region, networkAclId);- acl.getEntries().removeIf(e -> e.getRuleNumber() == ruleNumber && e.isEgress() == egress);- networkAcls.put(key(region, networkAclId), acl);+ public void deleteNetworkAclEntry(String region, String networkAclId, int ruleNumber, boolean egress) {+ synchronized (lockFor(key(region, networkAclId))) {+ NetworkAcl acl = getRequiredNetworkAcl(region, networkAclId);+ List<NetworkAclEntry> next = new ArrayList<>(acl.getEntries());+ next.removeIf(e -> e.getRuleNumber() == ruleNumber && e.isEgress() == egress);+ acl.setEntries(next);+ networkAcls.put(key(region, networkAclId), acl);+ }}- public synchronized NetworkAclAssociation replaceNetworkAclAssociation(String region, String associationId, String networkAclId) {+ public NetworkAclAssociation replaceNetworkAclAssociation(String region, String associationId, String networkAclId) {NetworkAcl target = getRequiredNetworkAcl(region, networkAclId);for (NetworkAcl acl : networkAcls.scan(k -> true)) {- if (!region.equals(acl.getRegion())) {+ if (!region.equals(acl.getRegion())+ || acl.getAssociations().stream()+ .noneMatch(a -> a.getNetworkAclAssociationId().equals(associationId))) {continue;}- for (NetworkAclAssociation existing : acl.getAssociations()) {- if (existing.getNetworkAclAssociationId().equals(associationId)) {- String subnetId = existing.getSubnetId();- acl.getAssociations().remove(existing);- networkAcls.put(key(region, acl.getNetworkAclId()), acl);+ String sourceKey = key(region, acl.getNetworkAclId());+ String targetKey = key(region, networkAclId);+ // The move must be atomic across both ACLs, or a describe could observe the subnet+ // associated with neither. Locks are taken in stripe order so two callers moving+ // associations in opposite directions cannot deadlock; one stripe re-enters.+ synchronized (lowerLockOf(sourceKey, targetKey)) {+ synchronized (higherLockOf(sourceKey, targetKey)) {+ List<NetworkAclAssociation> remaining = new ArrayList<>(acl.getAssociations());+ NetworkAclAssociation claimed = remaining.stream()+ .filter(a -> a.getNetworkAclAssociationId().equals(associationId))+ .findFirst()+ .orElse(null);+ // The scan above ran unlocked, so a concurrent replace of the same association+ // may already have moved it. That caller minted the new id; this one sees the+ // requested id no longer exist.+ if (claimed == null) {+ break;+ }+ remaining.remove(claimed);+ acl.setAssociations(remaining);+ networkAcls.put(sourceKey, acl);+NetworkAclAssociation moved = new NetworkAclAssociation();moved.setNetworkAclAssociationId("aclassoc-" + randomHex(17));moved.setNetworkAclId(networkAclId);- moved.setSubnetId(subnetId);- target.getAssociations().add(moved);- networkAcls.put(key(region, networkAclId), target);+ moved.setSubnetId(claimed.getSubnetId());+ List<NetworkAclAssociation> next = new ArrayList<>(target.getAssociations());+ next.add(moved);+ target.setAssociations(next);+ networkAcls.put(targetKey, target);return moved;}}@@ -524,6 +553,42 @@ public class Ec2Service {return region + "::" + id;}+ // Per-resource mutation locks (#1464): storage get() returns the live stored object, so+ // unsynchronized list mutations race under parallel clients (Terraform runs 10-wide) and+ // drop entries. Mutators take the resource's stripe and swap collections copy-on-write so+ // concurrent describes only ever see a complete list. A fixed stripe array keeps this+ // bounded — a lock per storage key would never evict — at the cost of unrelated resources+ // sharing a monitor on hash collision.+ private static final int LOCK_STRIPES = 512;+ private final Object[] resourceLocks = newLockStripes();++ private static Object[] newLockStripes() {+ Object[] stripes = new Object[LOCK_STRIPES];+ for (int i = 0; i < stripes.length; i++) {+ stripes[i] = new Object();+ }+ return stripes;+ }++ private int stripeOf(String storeKey) {+ return Math.floorMod(storeKey.hashCode(), LOCK_STRIPES);+ }++ private Object lockFor(String storeKey) {+ return resourceLocks[stripeOf(storeKey)];+ }++ // Stripe index, not key order, is the total order two-lock callers must agree on: distinct+ // keys can share a stripe, so ordering by key could have two callers take the same pair of+ // monitors in opposite orders.+ private Object lowerLockOf(String keyA, String keyB) {+ return resourceLocks[Math.min(stripeOf(keyA), stripeOf(keyB))];+ }++ private Object higherLockOf(String keyA, String keyB) {+ return resourceLocks[Math.max(stripeOf(keyA), stripeOf(keyB))];+ }+private String randomHex(int len) {StringBuilder sb = new StringBuilder(len);Random rand = new Random();@@ -1326,28 +1391,34 @@ public class Ec2Service {public List<SecurityGroupRule> authorizeSecurityGroupIngress(String region, String groupId, List<IpPermission> permissions) {ensureDefaultResources(region);- SecurityGroup sg = getRequiredSecurityGroup(region, groupId);-List<SecurityGroupRule> rules = new ArrayList<>();- for (IpPermission perm : permissions) {- sg.getIpPermissions().add(perm);- rules.addAll(createRules(region, groupId, perm, false));+ synchronized (lockFor(key(region, groupId))) {+ SecurityGroup sg = getRequiredSecurityGroup(region, groupId);+ List<IpPermission> next = new ArrayList<>(sg.getIpPermissions());+ for (IpPermission perm : permissions) {+ next.add(perm);+ rules.addAll(createRules(region, groupId, perm, false));+ }+ sg.setIpPermissions(next);+ securityGroups.put(key(region, groupId), sg);}- securityGroups.put(key(region, groupId), sg);reconcilePublishedPortsForGroup(region, groupId);return rules;}public List<SecurityGroupRule> authorizeSecurityGroupEgress(String region, String groupId, List<IpPermission> permissions) {ensureDefaultResources(region);- SecurityGroup sg = getRequiredSecurityGroup(region, groupId);-List<SecurityGroupRule> rules = new ArrayList<>();- for (IpPermission perm : permissions) {- sg.getIpPermissionsEgress().add(perm);- rules.addAll(createRules(region, groupId, perm, true));+ synchronized (lockFor(key(region, groupId))) {+ SecurityGroup sg = getRequiredSecurityGroup(region, groupId);+ List<IpPermission> next = new ArrayList<>(sg.getIpPermissionsEgress());+ for (IpPermission perm : permissions) {+ next.add(perm);+ rules.addAll(createRules(region, groupId, perm, true));+ }+ sg.setIpPermissionsEgress(next);+ securityGroups.put(key(region, groupId), sg);}- securityGroups.put(key(region, groupId), sg);return rules;}@@ -1386,19 +1457,25 @@ public class Ec2Service {public void revokeSecurityGroupIngress(String region, String groupId, List<IpPermission> permissions) {ensureDefaultResources(region);- SecurityGroup sg = getRequiredSecurityGroup(region, groupId);-- sg.getIpPermissions().removeIf(p -> matchesAnyPermission(p, permissions));- securityGroups.put(key(region, groupId), sg);+ synchronized (lockFor(key(region, groupId))) {+ SecurityGroup sg = getRequiredSecurityGroup(region, groupId);+ List<IpPermission> next = new ArrayList<>(sg.getIpPermissions());+ next.removeIf(p -> matchesAnyPermission(p, permissions));+ sg.setIpPermissions(next);+ securityGroups.put(key(region, groupId), sg);+ }reconcilePublishedPortsForGroup(region, groupId);}public void revokeSecurityGroupEgress(String region, String groupId, List<IpPermission> permissions) {ensureDefaultResources(region);- SecurityGroup sg = getRequiredSecurityGroup(region, groupId);-- sg.getIpPermissionsEgress().removeIf(p -> matchesAnyPermission(p, permissions));- securityGroups.put(key(region, groupId), sg);+ synchronized (lockFor(key(region, groupId))) {+ SecurityGroup sg = getRequiredSecurityGroup(region, groupId);+ List<IpPermission> next = new ArrayList<>(sg.getIpPermissionsEgress());+ next.removeIf(p -> matchesAnyPermission(p, permissions));+ sg.setIpPermissionsEgress(next);+ securityGroups.put(key(region, groupId), sg);+ }}private SecurityGroup getRequiredSecurityGroup(String region, String groupId) {@@ -2027,28 +2104,33 @@ public class Ec2Service {public void createTags(String region, List<String> resourceIds, List<Tag> tagList) {ensureDefaultResources(region);for (String resourceId : resourceIds) {- List<Tag> existing = tags.get(resourceId).orElse(new ArrayList<>());- for (Tag tag : tagList) {- existing.removeIf(t -> t.getKey().equals(tag.getKey()));- existing.add(tag);+ synchronized (lockFor(key(region, resourceId))) {+ List<Tag> existing = new ArrayList<>(tags.get(resourceId).orElse(List.of()));+ for (Tag tag : tagList) {+ existing.removeIf(t -> t.getKey().equals(tag.getKey()));+ existing.add(tag);+ }+ tags.put(resourceId, existing);+ // Update resource objects+ updateResourceTags(region, resourceId, existing);}- tags.put(resourceId, existing);- // Update resource objects- updateResourceTags(region, resourceId, existing);}}public void deleteTags(String region, List<String> resourceIds, List<Tag> tagList) {ensureDefaultResources(region);for (String resourceId : resourceIds) {- List<Tag> existing = tags.get(resourceId).orElse(null);- if (existing != null) {- for (Tag tag : tagList) {- existing.removeIf(t -> t.getKey().equals(tag.getKey())- && (tag.getValue() == null || tag.getValue().equals(t.getValue())));+ synchronized (lockFor(key(region, resourceId))) {+ List<Tag> stored = tags.get(resourceId).orElse(null);+ if (stored != null) {+ List<Tag> existing = new ArrayList<>(stored);+ for (Tag tag : tagList) {+ existing.removeIf(t -> t.getKey().equals(tag.getKey())+ && (tag.getValue() == null || tag.getValue().equals(t.getValue())));+ }+ tags.put(resourceId, existing);+ updateResourceTags(region, resourceId, existing);}- tags.put(resourceId, existing);- updateResourceTags(region, resourceId, existing);}}}@@ -2240,37 +2322,55 @@ public class Ec2Service {assoc.setSubnetId(subnetId);assoc.setMain(false);assoc.setAssociationState("associated");- rt.getAssociations().add(assoc);- routeTables.put(key(region, routeTableId), rt);+ synchronized (lockFor(key(region, routeTableId))) {+ RouteTable current = getRequiredRouteTable(region, routeTableId);+ List<RouteTableAssociation> next = new ArrayList<>(current.getAssociations());+ next.add(assoc);+ current.setAssociations(next);+ routeTables.put(key(region, routeTableId), current);+ }return assoc;}public void disassociateRouteTable(String region, String associationId) {ensureDefaultResources(region);for (RouteTable rt : routeTables.scan(k -> true)) {- if (rt.getRegion().equals(region)) {- rt.getAssociations().removeIf(a -> a.getRouteTableAssociationId().equals(associationId));- routeTables.put(key(region, rt.getRouteTableId()), rt);+ if (rt.getRegion().equals(region)+ && rt.getAssociations().stream()+ .anyMatch(a -> a.getRouteTableAssociationId().equals(associationId))) {+ synchronized (lockFor(key(region, rt.getRouteTableId()))) {+ RouteTable current = getRequiredRouteTable(region, rt.getRouteTableId());+ List<RouteTableAssociation> next = new ArrayList<>(current.getAssociations());+ next.removeIf(a -> a.getRouteTableAssociationId().equals(associationId));+ current.setAssociations(next);+ routeTables.put(key(region, current.getRouteTableId()), current);+ }}}}public void createRoute(String region, String routeTableId, String destinationCidrBlock, String gatewayId, String natGatewayId) {ensureDefaultResources(region);- RouteTable rt = getRequiredRouteTable(region, routeTableId);-- Route route = new Route(destinationCidrBlock, gatewayId, "CreateRoute");- route.setNatGatewayId(natGatewayId);- rt.getRoutes().add(route);- routeTables.put(key(region, routeTableId), rt);+ synchronized (lockFor(key(region, routeTableId))) {+ RouteTable current = getRequiredRouteTable(region, routeTableId);+ List<Route> next = new ArrayList<>(current.getRoutes());+ Route route = new Route(destinationCidrBlock, gatewayId, "CreateRoute");+ route.setNatGatewayId(natGatewayId);+ next.add(route);+ current.setRoutes(next);+ routeTables.put(key(region, routeTableId), current);+ }}public void deleteRoute(String region, String routeTableId, String destinationCidrBlock) {ensureDefaultResources(region);- RouteTable rt = getRequiredRouteTable(region, routeTableId);-- rt.getRoutes().removeIf(r -> r.getDestinationCidrBlock().equals(destinationCidrBlock));- routeTables.put(key(region, routeTableId), rt);+ synchronized (lockFor(key(region, routeTableId))) {+ RouteTable current = getRequiredRouteTable(region, routeTableId);+ List<Route> next = new ArrayList<>(current.getRoutes());+ next.removeIf(r -> r.getDestinationCidrBlock().equals(destinationCidrBlock));+ current.setRoutes(next);+ routeTables.put(key(region, routeTableId), current);+ }}private RouteTable getRequiredRouteTable(String region, String routeTableId) {diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceConcurrencyTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceConcurrencyTest.javanew file mode 100644index 0000000000..b843cccf0c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceConcurrencyTest.java@@ -0,0 +1,267 @@+package io.github.hectorvent.floci.services.ec2;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.ec2.portforward.Ec2PortForwardManager;+import io.github.hectorvent.floci.services.ec2.model.IpPermission;+import io.github.hectorvent.floci.services.ec2.model.IpRange;+import io.github.hectorvent.floci.services.ec2.model.NetworkAcl;+import io.github.hectorvent.floci.services.ec2.model.RouteTable;+import io.github.hectorvent.floci.services.ec2.model.SecurityGroup;+import io.github.hectorvent.floci.services.ec2.model.Tag;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.stream.Collectors;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * Concurrent mutations of shared EC2 resources must not lose updates (issue #1464):+ * storage get() returns the live stored object, so unsynchronized list mutations under+ * parallel clients (Terraform applies 10-wide by default) dropped route table+ * associations, security group rules, network ACL entries, and tags. Each scenario+ * repeats over several trials so a regression surfaces reliably.+ */+class Ec2ServiceConcurrencyTest {++ private static final int TRIALS = 30;+ private static final int N = 24;+ private static final int THREADS = 8;++ private static Ec2Service newService() {+ return new Ec2Service(mockConfig(), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class),+ mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class), new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory());+ }++ @Test+ void concurrentAssociateRouteTableKeepsEveryAssociation() throws Exception {+ for (int trial = 0; trial < TRIALS; trial++) {+ String region = "race-rtb-" + trial;+ Ec2Service service = newService();+ String vpcId = service.createVpc(region, "10.0.0.0/16", false).getVpcId();+ String rtbId = service.createRouteTable(region, vpcId).getRouteTableId();++ Set<String> returnedIds = runRace(i ->+ service.associateRouteTable(region, rtbId, "subnet-" + i).getRouteTableAssociationId());++ RouteTable rt = service.describeRouteTables(region, List.of(rtbId), Map.of()).getFirst();+ assertEquals(N, rt.getAssociations().size(),+ "trial " + trial + ": lost route table associations");+ Set<String> storedIds = rt.getAssociations().stream()+ .map(a -> a.getRouteTableAssociationId())+ .collect(Collectors.toSet());+ assertEquals(returnedIds, storedIds, "trial " + trial + ": returned ids must all be stored");+ assertTrue(rt.getAssociations().stream()+ .allMatch(a -> "associated".equals(a.getAssociationState())),+ "every association must report associated");+ }+ }++ @Test+ void concurrentAuthorizeIngressKeepsEveryRule() throws Exception {+ for (int trial = 0; trial < TRIALS; trial++) {+ String region = "race-sg-" + trial;+ Ec2Service service = newService();+ String vpcId = service.createVpc(region, "10.0.0.0/16", false).getVpcId();+ String groupId = service.createSecurityGroup(region, "sg-race", "d", vpcId).getGroupId();++ runRace(i -> {+ IpPermission perm = new IpPermission();+ perm.setIpProtocol("tcp");+ perm.setFromPort(1000 + i);+ perm.setToPort(1000 + i);+ IpRange range = new IpRange();+ range.setCidrIp("10.0." + i + ".0/24");+ perm.setIpRanges(List.of(range));+ return service.authorizeSecurityGroupIngress(region, groupId, List.of(perm))+ .getFirst().getSecurityGroupRuleId();+ });++ SecurityGroup sg = service.describeSecurityGroups(region, List.of(groupId), List.of(), Map.of()).getFirst();+ assertEquals(N, sg.getIpPermissions().size(),+ "trial " + trial + ": lost security group ingress permissions");+ long ingressRules = service.describeSecurityGroupRules(region, List.of(groupId), List.of()).stream()+ .filter(r -> !r.isEgress())+ .count();+ assertEquals(N, ingressRules, "trial " + trial + ": lost security group rule records");+ }+ }++ @Test+ void concurrentNetworkAclEntriesKeptWhileReadable() throws Exception {+ for (int trial = 0; trial < TRIALS; trial++) {+ String region = "race-nacl-" + trial;+ Ec2Service service = newService();+ String vpcId = service.createVpc(region, "10.0.0.0/16", false).getVpcId();+ String aclId = service.createNetworkAcl(region, vpcId).getNetworkAclId();+ int seeded = service.describeNetworkAcls(region, List.of(aclId), Map.of())+ .getFirst().getEntries().size();++ Thread reader = new Thread(() -> {+ while (!Thread.currentThread().isInterrupted()) {+ // Must never throw ConcurrentModificationException or see a corrupt list.+ service.describeNetworkAcls(region, List.of(aclId), Map.of())+ .getFirst().getEntries().forEach(e -> e.getRuleNumber());+ }+ });+ reader.start();+ try {+ runRace(i -> {+ service.createNetworkAclEntry(region, aclId, 100 + i, "6", "allow", false,+ "10.0." + i + ".0/24", 80, 80, false);+ return "entry-" + i;+ });+ } finally {+ reader.interrupt();+ reader.join(TimeUnit.SECONDS.toMillis(5));+ }++ NetworkAcl acl = service.describeNetworkAcls(region, List.of(aclId), Map.of()).getFirst();+ assertEquals(seeded + N, acl.getEntries().size(),+ "trial " + trial + ": lost network ACL entries");+ }+ }++ @Test+ void concurrentCreateTagsKeepsEveryTag() throws Exception {+ for (int trial = 0; trial < TRIALS; trial++) {+ String region = "race-tag-" + trial;+ Ec2Service service = newService();+ String vpcId = service.createVpc(region, "10.0.0.0/16", false).getVpcId();++ runRace(i -> {+ service.createTags(region, List.of(vpcId), List.of(new Tag("k" + i, "v" + i)));+ return "k" + i;+ });++ Map<String, List<String>> filters = Map.of("resource-id", List.of(vpcId));+ assertEquals(N, service.describeTags(region, filters).size(),+ "trial " + trial + ": lost tags");+ }+ }++ @Test+ void concurrentReplaceNetworkAclAssociationMovesTheSubnetOnce() throws Exception {+ for (int trial = 0; trial < TRIALS; trial++) {+ String region = "race-aclassoc-" + trial;+ Ec2Service service = newService();+ String vpcId = service.createVpc(region, "10.0.0.0/16", false).getVpcId();+ String subnetId = service.createSubnet(region, vpcId, "10.0.1.0/24", null).getSubnetId();+ String targetAclId = service.createNetworkAcl(region, vpcId).getNetworkAclId();+ String associationId = service.describeNetworkAcls(region, List.of(), Map.of()).stream()+ .flatMap(a -> a.getAssociations().stream())+ .filter(a -> subnetId.equals(a.getSubnetId()))+ .findFirst()+ .orElseThrow()+ .getNetworkAclAssociationId();++ AtomicInteger replaced = new AtomicInteger();+ AtomicInteger notFound = new AtomicInteger();+ runRaceAllowing(i -> {+ try {+ service.replaceNetworkAclAssociation(region, associationId, targetAclId);+ replaced.incrementAndGet();+ } catch (AwsException e) {+ assertEquals("InvalidAssociationID.NotFound", e.getErrorCode());+ notFound.incrementAndGet();+ }+ });++ assertEquals(1, replaced.get(), "trial " + trial + ": exactly one caller may claim the association");+ assertEquals(N - 1, notFound.get(), "trial " + trial + ": losers must see InvalidAssociationID.NotFound");++ NetworkAcl target = service.describeNetworkAcls(region, List.of(targetAclId), Map.of()).getFirst();+ assertEquals(1, target.getAssociations().size(),+ "trial " + trial + ": subnet must be associated to the target ACL exactly once");+ assertEquals(subnetId, target.getAssociations().getFirst().getSubnetId());+ long subnetAssociations = service.describeNetworkAcls(region, List.of(), Map.of()).stream()+ .flatMap(a -> a.getAssociations().stream())+ .filter(a -> subnetId.equals(a.getSubnetId()))+ .count();+ assertEquals(1, subnetAssociations, "trial " + trial + ": a subnet has exactly one NACL association");+ }+ }++ /** Runs N concurrent invocations of the same op; the op absorbs its own expected failures. */+ private static void runRaceAllowing(java.util.function.IntConsumer op) throws Exception {+ ExecutorService pool = Executors.newFixedThreadPool(THREADS);+ CountDownLatch start = new CountDownLatch(1);+ List<java.util.concurrent.Future<?>> futures = new java.util.ArrayList<>();+ for (int i = 0; i < N; i++) {+ int idx = i;+ futures.add(pool.submit(() -> {+ start.await();+ op.accept(idx);+ return null;+ }));+ }+ start.countDown();+ for (var f : futures) {+ f.get(30, TimeUnit.SECONDS);+ }+ pool.shutdownNow();+ assertTrue(pool.awaitTermination(10, TimeUnit.SECONDS));+ }++ /** Runs N indexed mutations across THREADS with a common start gate; returns their results. */+ private static Set<String> runRace(java.util.function.IntFunction<String> op) throws Exception {+ ExecutorService pool = Executors.newFixedThreadPool(THREADS);+ CountDownLatch start = new CountDownLatch(1);+ List<java.util.concurrent.Future<String>> futures = new java.util.ArrayList<>();+ for (int i = 0; i < N; i++) {+ int idx = i;+ futures.add(pool.submit(() -> {+ start.await();+ return op.apply(idx);+ }));+ }+ start.countDown();+ Set<String> results = new java.util.HashSet<>();+ for (var f : futures) {+ results.add(f.get(30, TimeUnit.SECONDS));+ }+ pool.shutdownNow();+ assertTrue(pool.awaitTermination(10, TimeUnit.SECONDS));+ return results;+ }++ private static EmulatorConfig mockConfig() {+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.Ec2ServiceConfig ec2 = mock(EmulatorConfig.Ec2ServiceConfig.class);+ when(config.defaultAccountId()).thenReturn("000000000000");+ when(config.services()).thenReturn(services);+ when(services.ec2()).thenReturn(ec2);+ when(ec2.mock()).thenReturn(true);+ return config;+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ public <V> StorageBackend<String, V> create(String serviceName, String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return new InMemoryStorage<>();+ }+ }+}