Light, fluffy, and always free - The AWS Local Emulator alternative
fix(iam): make the server certificate reference check atomic with the write (#5368)
Priamo Germosen477e3ef813b5
diff --git a/docs/services/iam.md b/docs/services/iam.mdindex a6403490ba..7dce59ef1b 100644--- a/docs/services/iam.md+++ b/docs/services/iam.md@@ -635,13 +635,36 @@ certificate body, certificate chain, or private key".`ServerCertificateId` uses AWS's `ASCA` prefix for certificates. `GetAccountSummary`'s`ServerCertificates` count is backed by this store rather than reporting zero.-`DeleteServerCertificate` returns `DeleteConflict` while the certificate is in use, as AWS does.+`DeleteServerCertificate` returns `DeleteConflict` while the certificate is in use, which is an+intentional deviation rather than a recorded AWS answer. `DeleteConflict` is modeled on the+operation, but the API Reference describes it only as "attached subordinate entities" without saying+what counts as one for a certificate, and the operation's own page warns that if Elastic Load+Balancing "doesn't detect the deletion of bound certificates, it may continue to use the+certificates", which reads as the delete going through. The refusal is kept because a caller is+better served by it than by a dangling reference, not because AWS is known to answer it.Services that reference a certificate (ELB Classic listeners, ELBv2 listeners, CloudFrontdistributions) report it through `ServerCertificateReferenceProvider`, which IAM consults withoutdepending on them. In the other direction, ELB Classic rejects a listener whose `SSLCertificateId`names no certificate with `CertificateNotFound`, and CloudFront rejects an unknown`ViewerCertificate.IAMCertificateId` with `InvalidViewerCertificate`.+Those two directions have to be one step, not two. A referring service checks the certificate and+then saves the resource that names it, so a delete running in between sees no reference, removes the+certificate, and leaves the resource holding something that is gone. CloudFront and ELB Classic+therefore perform that check and the write while holding the lock the delete takes, which leaves+only the two orders that make sense: the delete loses and answers `DeleteConflict`, or it wins and+the create is refused for a certificate that no longer exists. ELBv2 does not validate certificate+references at all, so a listener there can still name a certificate that was never present, even+though `CertificateNotFound` is modeled on `CreateListener`, `ModifyListener` and+`AddListenerCertificates`.++Only a write that names an IAM certificate takes the lock. A listener with no certificate, a+listener served by ACM, a removal, or a distribution without an `IAMCertificateId` cannot leave a+resource holding a deleted certificate, and making those wait on a certificate delete, or it on+them, would buy nothing. The delete reads a snapshot of each load balancer's listeners rather than+the live list, so an unguarded write cannot end its walk in `ConcurrentModificationException`+either, which no mapper covers and which would surface as an unmapped 500.+### SSH Public Keys| Action | Description |diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.javaindex 55262ed638..f38956331f 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java@@ -48,6 +48,7 @@ import java.util.SequencedSet;import java.util.Set;import java.util.UUID;import java.util.function.Function;+import java.util.function.Supplier;import java.util.regex.Pattern;@ApplicationScoped@@ -191,20 +192,22 @@ public class CloudFrontService implements ServerCertificateReferenceProvider {validateOriginCustomHeaders(dist.getConfig());validateResponseHeadersPolicyReferences(dist.getConfig(), null);validateTrustedKeyGroups(dist.getConfig());- validateViewerCertificate(dist.getConfig());- String id = generateDistributionId();- dist.setId(id);- dist.setArn(arn("distribution/" + id));- dist.setDomainName(domainNameFor(id));- dist.setStatus("Deployed");- dist.setLastModifiedTime(Instant.now());- dist.setEtag(UUID.randomUUID().toString());- if (tags != null && !tags.isEmpty()) {- dist.setTags(tags);- tagStore.put("distribution/" + id, tags);- }- distStore.put(id, dist);- return dist;+ return whileCertificatesHeld(dist.getConfig(), () -> {+ validateViewerCertificate(dist.getConfig());+ String id = generateDistributionId();+ dist.setId(id);+ dist.setArn(arn("distribution/" + id));+ dist.setDomainName(domainNameFor(id));+ dist.setStatus("Deployed");+ dist.setLastModifiedTime(Instant.now());+ dist.setEtag(UUID.randomUUID().toString());+ if (tags != null && !tags.isEmpty()) {+ dist.setTags(tags);+ tagStore.put("distribution/" + id, tags);+ }+ distStore.put(id, dist);+ return dist;+ });}public Distribution getDistribution(String id) {@@ -222,16 +225,18 @@ public class CloudFrontService implements ServerCertificateReferenceProvider {validateOriginCustomHeaders(updated.getConfig());validateResponseHeadersPolicyReferences(updated.getConfig(), id);validateTrustedKeyGroups(updated.getConfig());- validateViewerCertificate(updated.getConfig());- updated.setId(id);- updated.setArn(existing.getArn());- updated.setDomainName(existing.getDomainName());- updated.setStatus("Deployed");- updated.setLastModifiedTime(Instant.now());- updated.setEtag(UUID.randomUUID().toString());- updated.setTags(existing.getTags());- distStore.put(id, updated);- return updated;+ return whileCertificatesHeld(updated.getConfig(), () -> {+ validateViewerCertificate(updated.getConfig());+ updated.setId(id);+ updated.setArn(existing.getArn());+ updated.setDomainName(existing.getDomainName());+ updated.setStatus("Deployed");+ updated.setLastModifiedTime(Instant.now());+ updated.setEtag(UUID.randomUUID().toString());+ updated.setTags(existing.getTags());+ distStore.put(id, updated);+ return updated;+ });}@Override@@ -264,6 +269,24 @@ public class CloudFrontService implements ServerCertificateReferenceProvider {}}+ /**+ * Runs a write that may record an IAM server certificate reference while IAM holds the lock+ * {@code DeleteServerCertificate} takes, so the certificate cannot be deleted between+ * {@link #validateViewerCertificate} and the store write. Without the guard the delete sees no+ * reference, because the distribution is not saved yet, and the distribution is left holding a+ * certificate that is gone.+ */+ private <T> T whileCertificatesHeld(DistributionConfig config, Supplier<T> action) {+ if (iamService == null || iamCertificateId(config) == null) {+ // Nothing to order. A distribution that names no IAM certificate cannot be left+ // holding one, and the scan reads a copy of the store rather than a live collection,+ // so this write cannot disturb it either. Taking the lock anyway would make every+ // distribution write wait on a certificate delete, and the other way round.+ return action.get();+ }+ return iamService.supplyWithServerCertificatesHeld(action);+ }+private static void validateOriginCustomHeaders(DistributionConfig config) {if (config == null || config.getOrigins() == null) {return;diff --git a/src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicService.java b/src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicService.javaindex 241fc6e8c7..f9d770758e 100644--- a/src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicService.java@@ -68,6 +68,9 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {// region → load balancer name → recordprivate Map<String, Map<String, ClassicLoadBalancer>> loadBalancers = new ConcurrentHashMap<>();+ /** Serializes listener writes against each other; see {@link #createLoadBalancerListeners}. */+ private final Object listenerLock = new Object();+public ElbClassicService(Ec2Service ec2Service,ElbClassicHealthChecker healthChecker,StorageFactory storageFactory,@@ -165,9 +168,7 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {Map<String, ClassicLoadBalancer> regionLbs =loadBalancers.computeIfAbsent(region, k -> new ConcurrentHashMap<>());if (regionLbs.containsKey(name)) {- throw new AwsException("DuplicateLoadBalancerName",- "Load balancer named " + name + " already exists and is not compatible with the "- + "requested configuration.", 400);+ throw duplicateLoadBalancerName(name);}String id = randomHex();@@ -204,8 +205,18 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {lb.setAvailabilityZones(new ArrayList<>(availabilityZones));}- regionLbs.put(name, lb);- loadBalancers.put(region, regionLbs);+ whileCertificatesHeld(listeners, () -> {+ // The certificate check again, and this is the one that counts. The one above it+ // fails fast and keeps AWS's error precedence, but a check made before the lock says+ // nothing about the moment of the write. The unique name is a separate problem that+ // putIfAbsent settles on its own, lock or no lock: checked and then written as two+ // steps, two creates of one name both pass and only the second survives.+ requireCertificates(region, listeners);+ if (regionLbs.putIfAbsent(name, lb) != null) {+ throw duplicateLoadBalancerName(name);+ }+ loadBalancers.put(region, regionLbs);+ });healthChecker.startMonitoring(lb);return lb;}@@ -282,31 +293,46 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {validateListener(listener);}requireCertificates(region, listeners);- for (ClassicListener listener : listeners) {- ClassicListener existing = lb.getListeners().stream()- .filter(l -> l.getLoadBalancerPort().equals(listener.getLoadBalancerPort()))- .findFirst()- .orElse(null);- if (existing != null) {- // AWS accepts a repeat of an identical listener and rejects a conflicting one.- if (!sameListener(existing, listener)) {- throw new AwsException("DuplicateListener",- "A listener already exists for " + name + " with LoadBalancerPort "- + listener.getLoadBalancerPort()- + ", but with a different InstancePort, Protocol, or "- + "SSLCertificateId", 400);+ whileCertificatesHeld(listeners, () -> {+ requireCertificates(region, listeners);+ // The port of an existing listener decides whether this one is added, so the walk and+ // the add are one step. Two threads adding the same port would otherwise both find it+ // absent and both add it, and a removal landing in between would drop a listener this+ // call has already decided to keep.+ synchronized (listenerLock) {+ for (ClassicListener listener : listeners) {+ ClassicListener existing = lb.getListeners().stream()+ .filter(l -> l.getLoadBalancerPort().equals(listener.getLoadBalancerPort()))+ .findFirst()+ .orElse(null);+ if (existing != null) {+ // AWS accepts a repeat of an identical listener and rejects a conflicting one.+ if (!sameListener(existing, listener)) {+ throw new AwsException("DuplicateListener",+ "A listener already exists for " + name + " with LoadBalancerPort "+ + listener.getLoadBalancerPort()+ + ", but with a different InstancePort, Protocol, or "+ + "SSLCertificateId", 400);+ }+ continue;+ }+ lb.getListeners().add(listener);}- continue;+ persist(region);}- lb.getListeners().add(listener);- }- persist(region);+ });}public void deleteLoadBalancerListeners(String region, String name, List<Integer> loadBalancerPorts) {ClassicLoadBalancer lb = requireLoadBalancer(region, name);- lb.getListeners().removeIf(l -> loadBalancerPorts.contains(l.getLoadBalancerPort()));- persist(region);+ // Under the same monitor as the add, so a removal cannot land between that method's look+ // at the port and its decision. No certificate lock: removing a listener can only drop a+ // reference to a certificate, never record one, and a delete that reads the list while+ // this runs reads a snapshot of it.+ synchronized (listenerLock) {+ lb.getListeners().removeIf(l -> loadBalancerPorts.contains(l.getLoadBalancerPort()));+ persist(region);+ }}// ── Health ────────────────────────────────────────────────────────────────@@ -558,6 +584,13 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {}}+ /**+ * {@inheritDoc}+ *+ * <p>Takes no monitor of this service, which is what keeps the ordering with IAM one way: the+ * delete calls this while holding its certificate lock. Safe without one because both maps+ * walked here are concurrent and the listener list iterates a snapshot.+ */@Overridepublic List<Reference> serverCertificateReferences() {List<Reference> references = new ArrayList<>();@@ -575,6 +608,50 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {return references;}+ private static AwsException duplicateLoadBalancerName(String name) {+ return new AwsException("DuplicateLoadBalancerName",+ "Load balancer named " + name + " already exists and is not compatible with the "+ + "requested configuration.", 400);+ }++ /**+ * Runs a write that records a listener's IAM server certificate while IAM holds the lock+ * {@code DeleteServerCertificate} takes, so the certificate cannot be deleted between the+ * check and the write. The callers re-check inside the guard: the fail-fast check above it+ * keeps the error precedence AWS has, and only the one under the guard is atomic with the+ * commit.+ *+ * <p>Only for a write that names an IAM certificate. A listener with no certificate, or one+ * served by ACM, cannot leave a load balancer holding a deleted IAM certificate, and the+ * delete reads a snapshot of the listener list rather than the live one, so an unguarded+ * write cannot disturb its walk either. Taking the lock for every write would instead make a+ * certificate delete wait on an unrelated load balancer, listener or zone write, and the+ * other way round, for writes that cannot affect it.+ *+ * <p>This class's own invariants do not ride on this lock. The unique load balancer name is+ * carried by {@code putIfAbsent} and the listener list by {@link #listenerLock}, both of+ * which hold whether or not IAM is present, which is what the unit tests build.+ */+ private void whileCertificatesHeld(List<ClassicListener> listeners, Runnable action) {+ if (iamService == null || !namesIamCertificate(listeners)) {+ action.run();+ return;+ }+ iamService.runWithServerCertificatesHeld(action);+ }++ /** Whether any of these listeners names a certificate that IAM serves, rather than ACM. */+ private static boolean namesIamCertificate(List<ClassicListener> listeners) {+ for (ClassicListener listener : listeners) {+ String certificate = listener.getSslCertificateId();+ if (certificate != null && !certificate.isBlank() && AwsArnUtils.isArn(certificate)+ && "iam".equals(AwsArnUtils.parse(certificate).service())) {+ return true;+ }+ }+ return false;+ }+private void requireCertificates(String region, List<ClassicListener> listeners) {for (ClassicListener listener : listeners) {String certificateId = listener.getSslCertificateId();diff --git a/src/main/java/io/github/hectorvent/floci/services/elb/model/ClassicLoadBalancer.java b/src/main/java/io/github/hectorvent/floci/services/elb/model/ClassicLoadBalancer.javaindex 702c0e6868..75306d76f6 100644--- a/src/main/java/io/github/hectorvent/floci/services/elb/model/ClassicLoadBalancer.java+++ b/src/main/java/io/github/hectorvent/floci/services/elb/model/ClassicLoadBalancer.java@@ -8,6 +8,7 @@ import java.util.ArrayList;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;+import java.util.concurrent.CopyOnWriteArrayList;/*** A Classic (2012-06-01) load balancer — the {@code LoadBalancerDescription} shape.@@ -30,7 +31,15 @@ public class ClassicLoadBalancer {private String accountId;private Instant createdTime;- private List<ClassicListener> listeners = new ArrayList<>();+ /**+ * Copy on write because IAM reads it from another thread. {@code DeleteServerCertificate}+ * walks the listeners of every load balancer, through+ * {@code ElbClassicService.serverCertificateReferences()}, looking for the certificate it is+ * about to remove, and a listener write may be changing this list while it does. A plain list+ * ends that walk in {@link java.util.ConcurrentModificationException}; iterating this one+ * reads a snapshot, which also keeps the response renderer and Jackson off the live list.+ */+ private List<ClassicListener> listeners = new CopyOnWriteArrayList<>();private List<String> availabilityZones = new ArrayList<>();private List<String> subnets = new ArrayList<>();private List<String> securityGroups = new ArrayList<>();@@ -73,7 +82,9 @@ public class ClassicLoadBalancer {public void setCreatedTime(Instant v) { this.createdTime = v; }public List<ClassicListener> getListeners() { return listeners; }- public void setListeners(List<ClassicListener> v) { this.listeners = v; }+ public void setListeners(List<ClassicListener> v) {+ this.listeners = new CopyOnWriteArrayList<>(v != null ? v : List.of());+ }public List<String> getAvailabilityZones() { return availabilityZones; }public void setAvailabilityZones(List<String> v) { this.availabilityZones = v; }diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java b/src/main/java/io/github/hectorvent/floci/services/iam/IamService.javaindex 62d36dbd68..6010235bc9 100644--- a/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java+++ b/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java@@ -81,6 +81,7 @@ import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ThreadLocalRandom;import java.util.function.Function;+import java.util.function.Supplier;import java.util.regex.Pattern;import java.util.stream.Collectors;import java.util.stream.Stream;@@ -2698,6 +2699,32 @@ public class IamService implements SessionAccountLookup, ResourceProvider {deleteServerCertificate(name, List.of());}+ /**+ * Runs an action that checks a server certificate and then records a reference to it, holding+ * the lock {@link #deleteServerCertificate} takes so the certificate cannot be deleted in+ * between. Without it the check and the write straddle the delete: the delete sees no+ * reference because the referring resource is not saved yet, removes the certificate, and the+ * resource is then saved pointing at something that is gone.+ *+ * <p>Safe to call from inside a provider's own monitor, because the ordering only ever runs+ * one way. {@code deleteServerCertificate} holds this lock while calling+ * {@link ServerCertificateReferenceProvider#serverCertificateReferences()} on each provider,+ * and those are plain reads over the provider's own store that take no provider monitor, so+ * IAM never waits on a provider while a provider waits on IAM.+ */+ public <T> T supplyWithServerCertificatesHeld(Supplier<T> action) {+ synchronized (serverCertificateLock) {+ return action.get();+ }+ }++ /** {@link #supplyWithServerCertificatesHeld} for an action with no result. */+ public void runWithServerCertificatesHeld(Runnable action) {+ synchronized (serverCertificateLock) {+ action.run();+ }+ }+/*** Deletes a server certificate unless one of the providers reports it in use, in which case* AWS answers {@code DeleteConflict} and the certificate stays.diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCertificateDeleteRaceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCertificateDeleteRaceTest.javanew file mode 100644index 0000000000..1386a886d2--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCertificateDeleteRaceTest.java@@ -0,0 +1,257 @@+package io.github.hectorvent.floci.services.cloudfront;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.acm.CertificateGenerator;+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;+import io.github.hectorvent.floci.services.cloudfront.model.DefaultCacheBehavior;+import io.github.hectorvent.floci.services.cloudfront.model.Distribution;+import io.github.hectorvent.floci.services.cloudfront.model.DistributionConfig;+import io.github.hectorvent.floci.services.cloudfront.model.Origin;+import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.iam.ServerCertificateReferenceProvider;+import io.github.hectorvent.floci.services.iam.model.ServerCertificate;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * A distribution and the certificate it names cannot end up disagreeing about whether the+ * certificate exists.+ *+ * <p>{@code DeleteServerCertificate} checks the reference providers under its own lock, and+ * CloudFront validates the certificate and then saves the distribution. With the two critical+ * sections unrelated, either order was reachable and so was a third state that neither order+ * should produce: a saved distribution naming a certificate the delete had already removed. That+ * reproduced in well over half of the trials below, so this is a tight assertion rather than a+ * hopeful one.+ */+@QuarkusTest+class CloudFrontCertificateDeleteRaceTest {++ private static final CertificateGenerator GENERATOR = new CertificateGenerator();+ /** The window was wide, so a few dozen trials is already decisive. */+ private static final int TRIALS = 120;++ private final CloudFrontService cloudFrontService;+ private final IamService iamService;++ CloudFrontCertificateDeleteRaceTest(CloudFrontService cloudFrontService, IamService iamService) {+ this.cloudFrontService = cloudFrontService;+ this.iamService = iamService;+ }++ private ServerCertificate uploadCertificate(String name) {+ CertificateGenerator.GeneratedCertificate pair = GENERATOR.generateSelfSignedCertificate(+ "race.certs.test.local", List.of(), KeyAlgorithm.EC_prime256v1);+ return iamService.uploadServerCertificate(name, "/race/", pair.certificatePem(),+ pair.privateKeyPem(), null, Map.of());+ }++ private Distribution distributionWithCertificate(String certificateId) {+ Origin origin = new Origin();+ origin.setId("race-origin");+ origin.setDomainName("example.com");+ Map<String, Object> customOriginConfig = new LinkedHashMap<>();+ customOriginConfig.put("HTTPPort", "80");+ customOriginConfig.put("HTTPSPort", "443");+ customOriginConfig.put("OriginProtocolPolicy", "http-only");+ origin.setCustomOriginConfig(customOriginConfig);++ DefaultCacheBehavior behavior = new DefaultCacheBehavior();+ behavior.setTargetOriginId("race-origin");+ behavior.setViewerProtocolPolicy("allow-all");++ DistributionConfig config = new DistributionConfig();+ config.setEnabled(false);+ config.setComment("certificate-delete-race");+ config.setOrigins(List.of(origin));+ config.setDefaultCacheBehavior(behavior);+ Map<String, String> viewerCertificate = new LinkedHashMap<>();+ viewerCertificate.put("IAMCertificateId", certificateId);+ viewerCertificate.put("SSLSupportMethod", "sni-only");+ config.setViewerCertificate(viewerCertificate);++ Distribution distribution = new Distribution();+ distribution.setConfig(config);+ return distribution;+ }++ @Test+ void aDistributionIsNeverLeftHoldingADeletedCertificate() throws Exception {+ List<String> dangling = new ArrayList<>();+ List<ServerCertificateReferenceProvider> providers = List.of(cloudFrontService);++ for (int trial = 0; trial < TRIALS; trial++) {+ String name = "race-cert-" + trial + "-" + Long.toString(System.nanoTime(), 36);+ ServerCertificate certificate = uploadCertificate(name);+ String certificateId = certificate.getServerCertificateId();++ ExecutorService pool = Executors.newFixedThreadPool(2);+ CountDownLatch start = new CountDownLatch(1);+ AtomicReference<Distribution> created = new AtomicReference<>();+ AtomicBoolean deleted = new AtomicBoolean(false);+ try {+ List<Future<?>> racers = List.of(pool.submit(() -> {+ start.await();+ try {+ created.set(cloudFrontService.createDistribution(+ distributionWithCertificate(certificateId), Map.of()));+ } catch (AwsException e) {+ // InvalidViewerCertificate when the delete got there first. Any other+ // code is a real failure, and swallowing it here would read as a lost+ // race and leave the trial looking clean.+ if (!"InvalidViewerCertificate".equals(e.getErrorCode())) {+ throw e;+ }+ }+ return null;+ }), pool.submit(() -> {+ start.await();+ try {+ iamService.deleteServerCertificate(name, providers);+ deleted.set(true);+ } catch (AwsException e) {+ // DeleteConflict when the distribution was already saved. Any other code+ // is a real failure and has to reach the future.+ if (!"DeleteConflict".equals(e.getErrorCode())) {+ throw e;+ }+ }+ return null;+ }));+ start.countDown();+ pool.shutdown();+ assertTrue(pool.awaitTermination(30, TimeUnit.SECONDS), "trial did not finish");+ reportAnythingUnexpected(racers);++ if (created.get() != null && deleted.get()+ && iamService.findServerCertificateById(certificateId).isEmpty()) {+ dangling.add("trial " + trial + ": distribution " + created.get().getId()+ + " holds deleted certificate " + certificateId);+ }+ } finally {+ pool.shutdownNow();+ cleanUpTrial(created.get(), name, certificateId);+ }+ }++ assertTrue(dangling.isEmpty(),+ "a distribution was left holding a deleted certificate in "+ + dangling.size() + " of " + TRIALS + " trials: " + dangling);+ }++ /** Whichever order wins, the pair stays consistent: both present, or neither. */+ @Test+ void oneOfTheTwoOrdersAlwaysWins() throws Exception {+ List<ServerCertificateReferenceProvider> providers = List.of(cloudFrontService);+ int createWon = 0;+ int deleteWon = 0;++ for (int trial = 0; trial < 40; trial++) {+ String name = "race-order-" + trial + "-" + Long.toString(System.nanoTime(), 36);+ String certificateId = uploadCertificate(name).getServerCertificateId();++ ExecutorService pool = Executors.newFixedThreadPool(2);+ CountDownLatch start = new CountDownLatch(1);+ AtomicReference<Distribution> created = new AtomicReference<>();+ AtomicBoolean deleted = new AtomicBoolean(false);+ try {+ List<Future<?>> racers = List.of(pool.submit(() -> {+ start.await();+ try {+ created.set(cloudFrontService.createDistribution(+ distributionWithCertificate(certificateId), Map.of()));+ } catch (AwsException e) {+ // the delete won; anything else is a failure, not an order+ if (!"InvalidViewerCertificate".equals(e.getErrorCode())) {+ throw e;+ }+ }+ return null;+ }), pool.submit(() -> {+ start.await();+ try {+ iamService.deleteServerCertificate(name, providers);+ deleted.set(true);+ } catch (AwsException e) {+ // the create won; anything else is a failure, not an order+ if (!"DeleteConflict".equals(e.getErrorCode())) {+ throw e;+ }+ }+ return null;+ }));+ start.countDown();+ pool.shutdown();+ assertTrue(pool.awaitTermination(30, TimeUnit.SECONDS), "trial did not finish");+ reportAnythingUnexpected(racers);++ boolean certificateGone =+ iamService.findServerCertificateById(certificateId).isEmpty();+ if (created.get() != null) {+ createWon++;+ assertTrue(!certificateGone,+ "trial " + trial + ": the distribution exists so the certificate "+ + "must too");+ } else {+ deleteWon++;+ assertTrue(certificateGone || !deleted.get(),+ "trial " + trial + ": no distribution, so a successful delete must "+ + "have removed the certificate");+ }+ } finally {+ pool.shutdownNow();+ cleanUpTrial(created.get(), name, certificateId);+ }+ }++ assertTrue(createWon + deleteWon == 40, "every trial must resolve one way or the other");+ }++ /**+ * Surfaces anything a racing task threw that it was not written to expect. The throwable+ * otherwise stays in the task's {@link Future}, the trial reads as the other order having+ * won, and the test passes without the race it exists to measure ever having run.+ */+ private static void reportAnythingUnexpected(List<Future<?>> racers) throws Exception {+ for (Future<?> racer : racers) {+ racer.get();+ }+ }++ /**+ * Takes the trial's own fixtures back out of the application, whatever happened above. Other+ * test classes share it, and the account's certificate quota is 20, so a trial that failed+ * part way through must not leave either resource behind. Nothing is asserted and either+ * resource may already be gone, since which one survives is exactly what the race decides.+ */+ private void cleanUpTrial(Distribution created, String name, String certificateId) {+ if (created != null) {+ try {+ cloudFrontService.removeDistribution(created.getId());+ } catch (AwsException ignored) {+ // Already removed, which is one of the outcomes this test measures.+ }+ }+ if (iamService.findServerCertificateById(certificateId).isPresent()) {+ try {+ iamService.deleteServerCertificate(name, List.of());+ } catch (AwsException ignored) {+ // Deleted between that check and here by the racing task.+ }+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicCertificateDeleteRaceTest.java b/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicCertificateDeleteRaceTest.javanew file mode 100644index 0000000000..707ef84aed--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicCertificateDeleteRaceTest.java@@ -0,0 +1,175 @@+package io.github.hectorvent.floci.services.elb;++import io.github.hectorvent.floci.services.acm.CertificateGenerator;+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;+import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * A Classic load balancer and the certificate its listener names cannot end up disagreeing about+ * whether the certificate exists.+ *+ * <p>Same hazard as the CloudFront one: {@code DeleteServerCertificate} checks the reference+ * providers under its own lock, while {@code CreateLoadBalancer} checks the certificate and then+ * commits the load balancer. Driven over the wire here, because the listener's certificate arrives+ * as a request parameter and the check lives behind the handler.+ */+@QuarkusTest+class ElbClassicCertificateDeleteRaceTest {++ private static final String ELB_AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20260427/us-east-1/elasticloadbalancing/aws4_request";+ private static final String IAM_AUTH =+ "AWS4-HMAC-SHA256 Credential=test/20260227/us-east-1/iam/aws4_request";+ private static final String CLASSIC_VERSION = "2012-06-01";+ private static final CertificateGenerator GENERATOR = new CertificateGenerator();+ private static final int TRIALS = 60;++ private static String suffix() {+ return Long.toString(System.nanoTime(), 36);+ }++ private static String uploadCertificate(String name) {+ CertificateGenerator.GeneratedCertificate pair = GENERATOR.generateSelfSignedCertificate(+ "elb.race.test.local", List.of(), KeyAlgorithm.EC_prime256v1);+ return given().header("Authorization", IAM_AUTH)+ .formParam("Action", "UploadServerCertificate")+ .formParam("ServerCertificateName", name)+ .formParam("CertificateBody", pair.certificatePem())+ .formParam("PrivateKey", pair.privateKeyPem())+ .when().post("/").then().statusCode(200)+ .extract().path("UploadServerCertificateResponse.UploadServerCertificateResult"+ + ".ServerCertificateMetadata.Arn");+ }++ private static int createLoadBalancer(String lbName, String certificateArn) {+ return given().header("Authorization", ELB_AUTH)+ .formParam("Action", "CreateLoadBalancer")+ .formParam("Version", CLASSIC_VERSION)+ .formParam("LoadBalancerName", lbName)+ .formParam("Subnets.member.1", Ec2Service.defaultSubnetId("us-east-1", "a"))+ .formParam("Listeners.member.1.Protocol", "HTTPS")+ .formParam("Listeners.member.1.LoadBalancerPort", "443")+ .formParam("Listeners.member.1.InstanceProtocol", "HTTP")+ .formParam("Listeners.member.1.InstancePort", "8080")+ .formParam("Listeners.member.1.SSLCertificateId", certificateArn)+ .when().post("/").then().extract().statusCode();+ }++ private static int deleteCertificate(String name) {+ return given().header("Authorization", IAM_AUTH)+ .formParam("Action", "DeleteServerCertificate")+ .formParam("ServerCertificateName", name)+ .when().post("/").then().extract().statusCode();+ }++ private static boolean certificateExists(String name) {+ return given().header("Authorization", IAM_AUTH)+ .formParam("Action", "GetServerCertificate")+ .formParam("ServerCertificateName", name)+ .when().post("/").then().extract().statusCode() == 200;+ }++ private static int deleteLoadBalancer(String lbName) {+ return given().header("Authorization", ELB_AUTH)+ .formParam("Action", "DeleteLoadBalancer")+ .formParam("Version", CLASSIC_VERSION)+ .formParam("LoadBalancerName", lbName)+ .when().post("/").then().extract().statusCode();+ }++ @Test+ void aLoadBalancerIsNeverLeftHoldingADeletedCertificate() throws Exception {+ List<String> dangling = new ArrayList<>();+ List<String> unresolved = new ArrayList<>();++ for (int trial = 0; trial < TRIALS; trial++) {+ String name = "elb-race-cert-" + trial + "-" + suffix();+ String arn = uploadCertificate(name);+ String lbName = "elb-race-" + trial + "-" + suffix();++ ExecutorService pool = Executors.newFixedThreadPool(2);+ CountDownLatch start = new CountDownLatch(1);+ AtomicInteger createStatus = new AtomicInteger();+ AtomicInteger deleteStatus = new AtomicInteger();+ try {+ List<Future<?>> racers = List.of(pool.submit(() -> {+ start.await();+ createStatus.set(createLoadBalancer(lbName, arn));+ return null;+ }), pool.submit(() -> {+ start.await();+ deleteStatus.set(deleteCertificate(name));+ return null;+ }));+ start.countDown();+ pool.shutdown();+ assertTrue(pool.awaitTermination(60, TimeUnit.SECONDS), "trial did not finish");+ reportAnythingUnexpected(racers);++ int create = createStatus.get();+ int delete = deleteStatus.get();+ if (create == 200 && delete == 200 && !certificateExists(name)) {+ dangling.add("trial " + trial + ": load balancer " + lbName+ + " holds deleted certificate " + name);+ }+ // Exactly one order wins, and the loser says why: the load balancer commits and+ // the delete answers DeleteConflict, or the certificate goes and the listener is+ // refused with CertificateNotFound. Two failures would leave the race untested+ // while the trial still looked clean, so they are collected rather than ignored.+ if (!(create == 200 && delete == 409) && !(create == 400 && delete == 200)) {+ unresolved.add("trial " + trial + ": create=" + create + " delete=" + delete);+ }+ } finally {+ pool.shutdownNow();+ cleanUpTrial(createStatus.get() == 200, lbName, name);+ }+ }++ assertTrue(dangling.isEmpty(),+ "a load balancer was left holding a deleted certificate in "+ + dangling.size() + " of " + TRIALS + " trials: " + dangling);+ assertTrue(unresolved.isEmpty(),+ "every trial must resolve as one order winning and the other being told why, "+ + "these did not: " + unresolved);+ }++ /**+ * Surfaces anything a racing task threw. The throwable otherwise stays in the task's+ * {@link Future}, the status stays at its initial zero, and the trial reads as the other+ * order having won without the race ever having run.+ */+ private static void reportAnythingUnexpected(List<Future<?>> racers) throws Exception {+ for (Future<?> racer : racers) {+ racer.get();+ }+ }++ /**+ * Takes the trial's own fixtures back out of the application, whatever happened above. Other+ * test classes share it, and the account's certificate quota is 20, so a trial that failed+ * part way through must not leave either resource behind. The status codes are ignored: which+ * of the two is still there is exactly what the race decides.+ */+ private static void cleanUpTrial(boolean created, String lbName, String name) {+ if (created) {+ deleteLoadBalancer(lbName);+ }+ if (certificateExists(name)) {+ deleteCertificate(name);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicServiceTest.javaindex 17cae367f6..e2b70258a5 100644--- a/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicServiceTest.java@@ -16,6 +16,13 @@ import org.mockito.junit.jupiter.MockitoExtension;import java.util.List;import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutionException;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNull;@@ -147,6 +154,109 @@ class ElbClassicServiceTest {assertEquals("DuplicateLoadBalancerName", e.getErrorCode());}+ /**+ * The name check and the write have to be one step. Checked before the write and nothing+ * else, two creates of one name both pass the check and both commit, and the caller of the+ * first is told it succeeded while holding a load balancer that was replaced.+ */+ @Test+ void twoConcurrentCreatesOfOneNameLeaveOnlyOneWinner() throws Exception {+ for (int attempt = 0; attempt < 8; attempt++) {+ String name = "concurrent-elb-" + attempt;+ ExecutorService pool = Executors.newFixedThreadPool(2);+ CountDownLatch start = new CountDownLatch(1);+ try {+ List<Future<?>> creates = List.of(+ pool.submit(() -> {+ start.await();+ // Zones rather than subnets, and no security group, so that nothing+ // in here reaches a mock from two threads at once.+ return service.createLoadBalancer(REGION, name,+ List.of(httpListener()), List.of(REGION + "a"), List.of(),+ List.of(), null, Map.of());+ }),+ pool.submit(() -> {+ start.await();+ return service.createLoadBalancer(REGION, name,+ List.of(httpListener()), List.of(REGION + "a"), List.of(),+ List.of(), null, Map.of());+ }));+ start.countDown();+ pool.shutdown();+ assertTrue(pool.awaitTermination(30, TimeUnit.SECONDS), "attempt did not finish");++ int won = 0;+ int refused = 0;+ for (Future<?> create : creates) {+ try {+ create.get();+ won++;+ } catch (ExecutionException e) {+ assertTrue(e.getCause() instanceof AwsException aws+ && "DuplicateLoadBalancerName".equals(aws.getErrorCode()),+ "the loser must be refused by name, got " + e.getCause());+ refused++;+ }+ }+ assertEquals(1, won, "exactly one create must succeed for attempt " + attempt);+ assertEquals(1, refused, "exactly one create must be refused for attempt " + attempt);+ } finally {+ pool.shutdownNow();+ }+ }+ }++ /**+ * IAM's {@code DeleteServerCertificate} walks these listener lists while deciding whether the+ * certificate it is about to remove is still in use. It holds its own lock for that and not+ * this service's, by design, so listener writes run alongside the walk and the list has to+ * tolerate them: on a plain list the walk ends in ConcurrentModificationException, which no+ * mapper covers, and the delete answers an unmapped 500.+ */+ @Test+ void theCertificateScanSurvivesListenerWritesOnAnotherThread() throws Exception {+ create("scanned-elb");+ ClassicListener https = new ClassicListener();+ https.setProtocol("HTTPS");+ https.setLoadBalancerPort(443);+ https.setInstanceProtocol("HTTP");+ https.setInstancePort(80);+ https.setSslCertificateId("arn:aws:iam::000000000000:server-certificate/scanned");++ ExecutorService pool = Executors.newFixedThreadPool(2);+ CountDownLatch start = new CountDownLatch(1);+ AtomicBoolean writing = new AtomicBoolean(true);+ try {+ Future<?> writer = pool.submit(() -> {+ start.await();+ try {+ for (int i = 0; i < 400; i++) {+ service.createLoadBalancerListeners(REGION, "scanned-elb", List.of(https));+ service.deleteLoadBalancerListeners(REGION, "scanned-elb", List.of(443));+ }+ } finally {+ writing.set(false);+ }+ return null;+ });+ Future<Integer> scanner = pool.submit(() -> {+ start.await();+ int scans = 0;+ while (writing.get()) {+ service.serverCertificateReferences();+ scans++;+ }+ return scans;+ });+ start.countDown();++ writer.get(30, TimeUnit.SECONDS);+ assertTrue(scanner.get(30, TimeUnit.SECONDS) > 0, "the scan never ran");+ } finally {+ pool.shutdownNow();+ }+ }+@Testvoid loadBalancersAreScopedByRegion() {create("my-elb");