Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(iam): make the server certificate reference check atomic with the write (#5368)

Priamo Germosen477e3ef813b5
diff --git a/docs/services/iam.md b/docs/services/iam.md
index a6403490ba..7dce59ef1b 100644
--- a/docs/services/iam.md
+++ b/docs/services/iam.md
@@ -635,13 +635,36 @@ certificate body, certificate chain, or private key".
`ServerCertificateId` uses AWS's `ASCA` prefix for certificates. `GetAccountSummary`'s
`ServerCertificates` count is backed by this store rather than reporting zero.
-`DeleteServerCertificate` returns `DeleteConflict` while the certificate is in use, as AWS does.
+`DeleteServerCertificate` returns `DeleteConflict` while the certificate is in use, which is an
+intentional deviation rather than a recorded AWS answer. `DeleteConflict` is modeled on the
+operation, but the API Reference describes it only as "attached subordinate entities" without saying
+what counts as one for a certificate, and the operation's own page warns that if Elastic Load
+Balancing "doesn't detect the deletion of bound certificates, it may continue to use the
+certificates", which reads as the delete going through. The refusal is kept because a caller is
+better served by it than by a dangling reference, not because AWS is known to answer it.
Services that reference a certificate (ELB Classic listeners, ELBv2 listeners, CloudFront
distributions) report it through `ServerCertificateReferenceProvider`, which IAM consults without
depending on them. In the other direction, ELB Classic rejects a listener whose `SSLCertificateId`
names no certificate with `CertificateNotFound`, and CloudFront rejects an unknown
`ViewerCertificate.IAMCertificateId` with `InvalidViewerCertificate`.
+Those two directions have to be one step, not two. A referring service checks the certificate and
+then saves the resource that names it, so a delete running in between sees no reference, removes the
+certificate, and leaves the resource holding something that is gone. CloudFront and ELB Classic
+therefore perform that check and the write while holding the lock the delete takes, which leaves
+only the two orders that make sense: the delete loses and answers `DeleteConflict`, or it wins and
+the create is refused for a certificate that no longer exists. ELBv2 does not validate certificate
+references at all, so a listener there can still name a certificate that was never present, even
+though `CertificateNotFound` is modeled on `CreateListener`, `ModifyListener` and
+`AddListenerCertificates`.
+
+Only a write that names an IAM certificate takes the lock. A listener with no certificate, a
+listener served by ACM, a removal, or a distribution without an `IAMCertificateId` cannot leave a
+resource holding a deleted certificate, and making those wait on a certificate delete, or it on
+them, would buy nothing. The delete reads a snapshot of each load balancer's listeners rather than
+the live list, so an unguarded write cannot end its walk in `ConcurrentModificationException`
+either, which no mapper covers and which would surface as an unmapped 500.
+
### SSH Public Keys
| Action | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java
index 55262ed638..f38956331f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java
@@ -48,6 +48,7 @@ import java.util.SequencedSet;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.regex.Pattern;
@ApplicationScoped
@@ -191,20 +192,22 @@ public class CloudFrontService implements ServerCertificateReferenceProvider {
validateOriginCustomHeaders(dist.getConfig());
validateResponseHeadersPolicyReferences(dist.getConfig(), null);
validateTrustedKeyGroups(dist.getConfig());
- validateViewerCertificate(dist.getConfig());
- String id = generateDistributionId();
- dist.setId(id);
- dist.setArn(arn("distribution/" + id));
- dist.setDomainName(domainNameFor(id));
- dist.setStatus("Deployed");
- dist.setLastModifiedTime(Instant.now());
- dist.setEtag(UUID.randomUUID().toString());
- if (tags != null && !tags.isEmpty()) {
- dist.setTags(tags);
- tagStore.put("distribution/" + id, tags);
- }
- distStore.put(id, dist);
- return dist;
+ return whileCertificatesHeld(dist.getConfig(), () -> {
+ validateViewerCertificate(dist.getConfig());
+ String id = generateDistributionId();
+ dist.setId(id);
+ dist.setArn(arn("distribution/" + id));
+ dist.setDomainName(domainNameFor(id));
+ dist.setStatus("Deployed");
+ dist.setLastModifiedTime(Instant.now());
+ dist.setEtag(UUID.randomUUID().toString());
+ if (tags != null && !tags.isEmpty()) {
+ dist.setTags(tags);
+ tagStore.put("distribution/" + id, tags);
+ }
+ distStore.put(id, dist);
+ return dist;
+ });
}
public Distribution getDistribution(String id) {
@@ -222,16 +225,18 @@ public class CloudFrontService implements ServerCertificateReferenceProvider {
validateOriginCustomHeaders(updated.getConfig());
validateResponseHeadersPolicyReferences(updated.getConfig(), id);
validateTrustedKeyGroups(updated.getConfig());
- validateViewerCertificate(updated.getConfig());
- updated.setId(id);
- updated.setArn(existing.getArn());
- updated.setDomainName(existing.getDomainName());
- updated.setStatus("Deployed");
- updated.setLastModifiedTime(Instant.now());
- updated.setEtag(UUID.randomUUID().toString());
- updated.setTags(existing.getTags());
- distStore.put(id, updated);
- return updated;
+ return whileCertificatesHeld(updated.getConfig(), () -> {
+ validateViewerCertificate(updated.getConfig());
+ updated.setId(id);
+ updated.setArn(existing.getArn());
+ updated.setDomainName(existing.getDomainName());
+ updated.setStatus("Deployed");
+ updated.setLastModifiedTime(Instant.now());
+ updated.setEtag(UUID.randomUUID().toString());
+ updated.setTags(existing.getTags());
+ distStore.put(id, updated);
+ return updated;
+ });
}
@Override
@@ -264,6 +269,24 @@ public class CloudFrontService implements ServerCertificateReferenceProvider {
}
}
+ /**
+ * Runs a write that may record an IAM server certificate reference while IAM holds the lock
+ * {@code DeleteServerCertificate} takes, so the certificate cannot be deleted between
+ * {@link #validateViewerCertificate} and the store write. Without the guard the delete sees no
+ * reference, because the distribution is not saved yet, and the distribution is left holding a
+ * certificate that is gone.
+ */
+ private <T> T whileCertificatesHeld(DistributionConfig config, Supplier<T> action) {
+ if (iamService == null || iamCertificateId(config) == null) {
+ // Nothing to order. A distribution that names no IAM certificate cannot be left
+ // holding one, and the scan reads a copy of the store rather than a live collection,
+ // so this write cannot disturb it either. Taking the lock anyway would make every
+ // distribution write wait on a certificate delete, and the other way round.
+ return action.get();
+ }
+ return iamService.supplyWithServerCertificatesHeld(action);
+ }
+
private static void validateOriginCustomHeaders(DistributionConfig config) {
if (config == null || config.getOrigins() == null) {
return;
diff --git a/src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicService.java b/src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicService.java
index 241fc6e8c7..f9d770758e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elb/ElbClassicService.java
@@ -68,6 +68,9 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {
// region → load balancer name → record
private Map<String, Map<String, ClassicLoadBalancer>> loadBalancers = new ConcurrentHashMap<>();
+ /** Serializes listener writes against each other; see {@link #createLoadBalancerListeners}. */
+ private final Object listenerLock = new Object();
+
public ElbClassicService(Ec2Service ec2Service,
ElbClassicHealthChecker healthChecker,
StorageFactory storageFactory,
@@ -165,9 +168,7 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {
Map<String, ClassicLoadBalancer> regionLbs =
loadBalancers.computeIfAbsent(region, k -> new ConcurrentHashMap<>());
if (regionLbs.containsKey(name)) {
- throw new AwsException("DuplicateLoadBalancerName",
- "Load balancer named " + name + " already exists and is not compatible with the "
- + "requested configuration.", 400);
+ throw duplicateLoadBalancerName(name);
}
String id = randomHex();
@@ -204,8 +205,18 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {
lb.setAvailabilityZones(new ArrayList<>(availabilityZones));
}
- regionLbs.put(name, lb);
- loadBalancers.put(region, regionLbs);
+ whileCertificatesHeld(listeners, () -> {
+ // The certificate check again, and this is the one that counts. The one above it
+ // fails fast and keeps AWS's error precedence, but a check made before the lock says
+ // nothing about the moment of the write. The unique name is a separate problem that
+ // putIfAbsent settles on its own, lock or no lock: checked and then written as two
+ // steps, two creates of one name both pass and only the second survives.
+ requireCertificates(region, listeners);
+ if (regionLbs.putIfAbsent(name, lb) != null) {
+ throw duplicateLoadBalancerName(name);
+ }
+ loadBalancers.put(region, regionLbs);
+ });
healthChecker.startMonitoring(lb);
return lb;
}
@@ -282,31 +293,46 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {
validateListener(listener);
}
requireCertificates(region, listeners);
- for (ClassicListener listener : listeners) {
- ClassicListener existing = lb.getListeners().stream()
- .filter(l -> l.getLoadBalancerPort().equals(listener.getLoadBalancerPort()))
- .findFirst()
- .orElse(null);
- if (existing != null) {
- // AWS accepts a repeat of an identical listener and rejects a conflicting one.
- if (!sameListener(existing, listener)) {
- throw new AwsException("DuplicateListener",
- "A listener already exists for " + name + " with LoadBalancerPort "
- + listener.getLoadBalancerPort()
- + ", but with a different InstancePort, Protocol, or "
- + "SSLCertificateId", 400);
+ whileCertificatesHeld(listeners, () -> {
+ requireCertificates(region, listeners);
+ // The port of an existing listener decides whether this one is added, so the walk and
+ // the add are one step. Two threads adding the same port would otherwise both find it
+ // absent and both add it, and a removal landing in between would drop a listener this
+ // call has already decided to keep.
+ synchronized (listenerLock) {
+ for (ClassicListener listener : listeners) {
+ ClassicListener existing = lb.getListeners().stream()
+ .filter(l -> l.getLoadBalancerPort().equals(listener.getLoadBalancerPort()))
+ .findFirst()
+ .orElse(null);
+ if (existing != null) {
+ // AWS accepts a repeat of an identical listener and rejects a conflicting one.
+ if (!sameListener(existing, listener)) {
+ throw new AwsException("DuplicateListener",
+ "A listener already exists for " + name + " with LoadBalancerPort "
+ + listener.getLoadBalancerPort()
+ + ", but with a different InstancePort, Protocol, or "
+ + "SSLCertificateId", 400);
+ }
+ continue;
+ }
+ lb.getListeners().add(listener);
}
- continue;
+ persist(region);
}
- lb.getListeners().add(listener);
- }
- persist(region);
+ });
}
public void deleteLoadBalancerListeners(String region, String name, List<Integer> loadBalancerPorts) {
ClassicLoadBalancer lb = requireLoadBalancer(region, name);
- lb.getListeners().removeIf(l -> loadBalancerPorts.contains(l.getLoadBalancerPort()));
- persist(region);
+ // Under the same monitor as the add, so a removal cannot land between that method's look
+ // at the port and its decision. No certificate lock: removing a listener can only drop a
+ // reference to a certificate, never record one, and a delete that reads the list while
+ // this runs reads a snapshot of it.
+ synchronized (listenerLock) {
+ lb.getListeners().removeIf(l -> loadBalancerPorts.contains(l.getLoadBalancerPort()));
+ persist(region);
+ }
}
// ── Health ────────────────────────────────────────────────────────────────
@@ -558,6 +584,13 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {
}
}
+ /**
+ * {@inheritDoc}
+ *
+ * <p>Takes no monitor of this service, which is what keeps the ordering with IAM one way: the
+ * delete calls this while holding its certificate lock. Safe without one because both maps
+ * walked here are concurrent and the listener list iterates a snapshot.
+ */
@Override
public List<Reference> serverCertificateReferences() {
List<Reference> references = new ArrayList<>();
@@ -575,6 +608,50 @@ public class ElbClassicService implements ServerCertificateReferenceProvider {
return references;
}
+ private static AwsException duplicateLoadBalancerName(String name) {
+ return new AwsException("DuplicateLoadBalancerName",
+ "Load balancer named " + name + " already exists and is not compatible with the "
+ + "requested configuration.", 400);
+ }
+
+ /**
+ * Runs a write that records a listener's IAM server certificate while IAM holds the lock
+ * {@code DeleteServerCertificate} takes, so the certificate cannot be deleted between the
+ * check and the write. The callers re-check inside the guard: the fail-fast check above it
+ * keeps the error precedence AWS has, and only the one under the guard is atomic with the
+ * commit.
+ *
+ * <p>Only for a write that names an IAM certificate. A listener with no certificate, or one
+ * served by ACM, cannot leave a load balancer holding a deleted IAM certificate, and the
+ * delete reads a snapshot of the listener list rather than the live one, so an unguarded
+ * write cannot disturb its walk either. Taking the lock for every write would instead make a
+ * certificate delete wait on an unrelated load balancer, listener or zone write, and the
+ * other way round, for writes that cannot affect it.
+ *
+ * <p>This class's own invariants do not ride on this lock. The unique load balancer name is
+ * carried by {@code putIfAbsent} and the listener list by {@link #listenerLock}, both of
+ * which hold whether or not IAM is present, which is what the unit tests build.
+ */
+ private void whileCertificatesHeld(List<ClassicListener> listeners, Runnable action) {
+ if (iamService == null || !namesIamCertificate(listeners)) {
+ action.run();
+ return;
+ }
+ iamService.runWithServerCertificatesHeld(action);
+ }
+
+ /** Whether any of these listeners names a certificate that IAM serves, rather than ACM. */
+ private static boolean namesIamCertificate(List<ClassicListener> listeners) {
+ for (ClassicListener listener : listeners) {
+ String certificate = listener.getSslCertificateId();
+ if (certificate != null && !certificate.isBlank() && AwsArnUtils.isArn(certificate)
+ && "iam".equals(AwsArnUtils.parse(certificate).service())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private void requireCertificates(String region, List<ClassicListener> listeners) {
for (ClassicListener listener : listeners) {
String certificateId = listener.getSslCertificateId();
diff --git a/src/main/java/io/github/hectorvent/floci/services/elb/model/ClassicLoadBalancer.java b/src/main/java/io/github/hectorvent/floci/services/elb/model/ClassicLoadBalancer.java
index 702c0e6868..75306d76f6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elb/model/ClassicLoadBalancer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elb/model/ClassicLoadBalancer.java
@@ -8,6 +8,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
/**
* A Classic (2012-06-01) load balancer — the {@code LoadBalancerDescription} shape.
@@ -30,7 +31,15 @@ public class ClassicLoadBalancer {
private String accountId;
private Instant createdTime;
- private List<ClassicListener> listeners = new ArrayList<>();
+ /**
+ * Copy on write because IAM reads it from another thread. {@code DeleteServerCertificate}
+ * walks the listeners of every load balancer, through
+ * {@code ElbClassicService.serverCertificateReferences()}, looking for the certificate it is
+ * about to remove, and a listener write may be changing this list while it does. A plain list
+ * ends that walk in {@link java.util.ConcurrentModificationException}; iterating this one
+ * reads a snapshot, which also keeps the response renderer and Jackson off the live list.
+ */
+ private List<ClassicListener> listeners = new CopyOnWriteArrayList<>();
private List<String> availabilityZones = new ArrayList<>();
private List<String> subnets = new ArrayList<>();
private List<String> securityGroups = new ArrayList<>();
@@ -73,7 +82,9 @@ public class ClassicLoadBalancer {
public void setCreatedTime(Instant v) { this.createdTime = v; }
public List<ClassicListener> getListeners() { return listeners; }
- public void setListeners(List<ClassicListener> v) { this.listeners = v; }
+ public void setListeners(List<ClassicListener> v) {
+ this.listeners = new CopyOnWriteArrayList<>(v != null ? v : List.of());
+ }
public List<String> getAvailabilityZones() { return availabilityZones; }
public void setAvailabilityZones(List<String> v) { this.availabilityZones = v; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java b/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java
index 62d36dbd68..6010235bc9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/iam/IamService.java
@@ -81,6 +81,7 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Function;
+import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -2698,6 +2699,32 @@ public class IamService implements SessionAccountLookup, ResourceProvider {
deleteServerCertificate(name, List.of());
}
+ /**
+ * Runs an action that checks a server certificate and then records a reference to it, holding
+ * the lock {@link #deleteServerCertificate} takes so the certificate cannot be deleted in
+ * between. Without it the check and the write straddle the delete: the delete sees no
+ * reference because the referring resource is not saved yet, removes the certificate, and the
+ * resource is then saved pointing at something that is gone.
+ *
+ * <p>Safe to call from inside a provider's own monitor, because the ordering only ever runs
+ * one way. {@code deleteServerCertificate} holds this lock while calling
+ * {@link ServerCertificateReferenceProvider#serverCertificateReferences()} on each provider,
+ * and those are plain reads over the provider's own store that take no provider monitor, so
+ * IAM never waits on a provider while a provider waits on IAM.
+ */
+ public <T> T supplyWithServerCertificatesHeld(Supplier<T> action) {
+ synchronized (serverCertificateLock) {
+ return action.get();
+ }
+ }
+
+ /** {@link #supplyWithServerCertificatesHeld} for an action with no result. */
+ public void runWithServerCertificatesHeld(Runnable action) {
+ synchronized (serverCertificateLock) {
+ action.run();
+ }
+ }
+
/**
* Deletes a server certificate unless one of the providers reports it in use, in which case
* AWS answers {@code DeleteConflict} and the certificate stays.
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCertificateDeleteRaceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCertificateDeleteRaceTest.java
new file mode 100644
index 0000000000..1386a886d2
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCertificateDeleteRaceTest.java
@@ -0,0 +1,257 @@
+package io.github.hectorvent.floci.services.cloudfront;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.acm.CertificateGenerator;
+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;
+import io.github.hectorvent.floci.services.cloudfront.model.DefaultCacheBehavior;
+import io.github.hectorvent.floci.services.cloudfront.model.Distribution;
+import io.github.hectorvent.floci.services.cloudfront.model.DistributionConfig;
+import io.github.hectorvent.floci.services.cloudfront.model.Origin;
+import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.iam.ServerCertificateReferenceProvider;
+import io.github.hectorvent.floci.services.iam.model.ServerCertificate;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * A distribution and the certificate it names cannot end up disagreeing about whether the
+ * certificate exists.
+ *
+ * <p>{@code DeleteServerCertificate} checks the reference providers under its own lock, and
+ * CloudFront validates the certificate and then saves the distribution. With the two critical
+ * sections unrelated, either order was reachable and so was a third state that neither order
+ * should produce: a saved distribution naming a certificate the delete had already removed. That
+ * reproduced in well over half of the trials below, so this is a tight assertion rather than a
+ * hopeful one.
+ */
+@QuarkusTest
+class CloudFrontCertificateDeleteRaceTest {
+
+ private static final CertificateGenerator GENERATOR = new CertificateGenerator();
+ /** The window was wide, so a few dozen trials is already decisive. */
+ private static final int TRIALS = 120;
+
+ private final CloudFrontService cloudFrontService;
+ private final IamService iamService;
+
+ CloudFrontCertificateDeleteRaceTest(CloudFrontService cloudFrontService, IamService iamService) {
+ this.cloudFrontService = cloudFrontService;
+ this.iamService = iamService;
+ }
+
+ private ServerCertificate uploadCertificate(String name) {
+ CertificateGenerator.GeneratedCertificate pair = GENERATOR.generateSelfSignedCertificate(
+ "race.certs.test.local", List.of(), KeyAlgorithm.EC_prime256v1);
+ return iamService.uploadServerCertificate(name, "/race/", pair.certificatePem(),
+ pair.privateKeyPem(), null, Map.of());
+ }
+
+ private Distribution distributionWithCertificate(String certificateId) {
+ Origin origin = new Origin();
+ origin.setId("race-origin");
+ origin.setDomainName("example.com");
+ Map<String, Object> customOriginConfig = new LinkedHashMap<>();
+ customOriginConfig.put("HTTPPort", "80");
+ customOriginConfig.put("HTTPSPort", "443");
+ customOriginConfig.put("OriginProtocolPolicy", "http-only");
+ origin.setCustomOriginConfig(customOriginConfig);
+
+ DefaultCacheBehavior behavior = new DefaultCacheBehavior();
+ behavior.setTargetOriginId("race-origin");
+ behavior.setViewerProtocolPolicy("allow-all");
+
+ DistributionConfig config = new DistributionConfig();
+ config.setEnabled(false);
+ config.setComment("certificate-delete-race");
+ config.setOrigins(List.of(origin));
+ config.setDefaultCacheBehavior(behavior);
+ Map<String, String> viewerCertificate = new LinkedHashMap<>();
+ viewerCertificate.put("IAMCertificateId", certificateId);
+ viewerCertificate.put("SSLSupportMethod", "sni-only");
+ config.setViewerCertificate(viewerCertificate);
+
+ Distribution distribution = new Distribution();
+ distribution.setConfig(config);
+ return distribution;
+ }
+
+ @Test
+ void aDistributionIsNeverLeftHoldingADeletedCertificate() throws Exception {
+ List<String> dangling = new ArrayList<>();
+ List<ServerCertificateReferenceProvider> providers = List.of(cloudFrontService);
+
+ for (int trial = 0; trial < TRIALS; trial++) {
+ String name = "race-cert-" + trial + "-" + Long.toString(System.nanoTime(), 36);
+ ServerCertificate certificate = uploadCertificate(name);
+ String certificateId = certificate.getServerCertificateId();
+
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicReference<Distribution> created = new AtomicReference<>();
+ AtomicBoolean deleted = new AtomicBoolean(false);
+ try {
+ List<Future<?>> racers = List.of(pool.submit(() -> {
+ start.await();
+ try {
+ created.set(cloudFrontService.createDistribution(
+ distributionWithCertificate(certificateId), Map.of()));
+ } catch (AwsException e) {
+ // InvalidViewerCertificate when the delete got there first. Any other
+ // code is a real failure, and swallowing it here would read as a lost
+ // race and leave the trial looking clean.
+ if (!"InvalidViewerCertificate".equals(e.getErrorCode())) {
+ throw e;
+ }
+ }
+ return null;
+ }), pool.submit(() -> {
+ start.await();
+ try {
+ iamService.deleteServerCertificate(name, providers);
+ deleted.set(true);
+ } catch (AwsException e) {
+ // DeleteConflict when the distribution was already saved. Any other code
+ // is a real failure and has to reach the future.
+ if (!"DeleteConflict".equals(e.getErrorCode())) {
+ throw e;
+ }
+ }
+ return null;
+ }));
+ start.countDown();
+ pool.shutdown();
+ assertTrue(pool.awaitTermination(30, TimeUnit.SECONDS), "trial did not finish");
+ reportAnythingUnexpected(racers);
+
+ if (created.get() != null && deleted.get()
+ && iamService.findServerCertificateById(certificateId).isEmpty()) {
+ dangling.add("trial " + trial + ": distribution " + created.get().getId()
+ + " holds deleted certificate " + certificateId);
+ }
+ } finally {
+ pool.shutdownNow();
+ cleanUpTrial(created.get(), name, certificateId);
+ }
+ }
+
+ assertTrue(dangling.isEmpty(),
+ "a distribution was left holding a deleted certificate in "
+ + dangling.size() + " of " + TRIALS + " trials: " + dangling);
+ }
+
+ /** Whichever order wins, the pair stays consistent: both present, or neither. */
+ @Test
+ void oneOfTheTwoOrdersAlwaysWins() throws Exception {
+ List<ServerCertificateReferenceProvider> providers = List.of(cloudFrontService);
+ int createWon = 0;
+ int deleteWon = 0;
+
+ for (int trial = 0; trial < 40; trial++) {
+ String name = "race-order-" + trial + "-" + Long.toString(System.nanoTime(), 36);
+ String certificateId = uploadCertificate(name).getServerCertificateId();
+
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicReference<Distribution> created = new AtomicReference<>();
+ AtomicBoolean deleted = new AtomicBoolean(false);
+ try {
+ List<Future<?>> racers = List.of(pool.submit(() -> {
+ start.await();
+ try {
+ created.set(cloudFrontService.createDistribution(
+ distributionWithCertificate(certificateId), Map.of()));
+ } catch (AwsException e) {
+ // the delete won; anything else is a failure, not an order
+ if (!"InvalidViewerCertificate".equals(e.getErrorCode())) {
+ throw e;
+ }
+ }
+ return null;
+ }), pool.submit(() -> {
+ start.await();
+ try {
+ iamService.deleteServerCertificate(name, providers);
+ deleted.set(true);
+ } catch (AwsException e) {
+ // the create won; anything else is a failure, not an order
+ if (!"DeleteConflict".equals(e.getErrorCode())) {
+ throw e;
+ }
+ }
+ return null;
+ }));
+ start.countDown();
+ pool.shutdown();
+ assertTrue(pool.awaitTermination(30, TimeUnit.SECONDS), "trial did not finish");
+ reportAnythingUnexpected(racers);
+
+ boolean certificateGone =
+ iamService.findServerCertificateById(certificateId).isEmpty();
+ if (created.get() != null) {
+ createWon++;
+ assertTrue(!certificateGone,
+ "trial " + trial + ": the distribution exists so the certificate "
+ + "must too");
+ } else {
+ deleteWon++;
+ assertTrue(certificateGone || !deleted.get(),
+ "trial " + trial + ": no distribution, so a successful delete must "
+ + "have removed the certificate");
+ }
+ } finally {
+ pool.shutdownNow();
+ cleanUpTrial(created.get(), name, certificateId);
+ }
+ }
+
+ assertTrue(createWon + deleteWon == 40, "every trial must resolve one way or the other");
+ }
+
+ /**
+ * Surfaces anything a racing task threw that it was not written to expect. The throwable
+ * otherwise stays in the task's {@link Future}, the trial reads as the other order having
+ * won, and the test passes without the race it exists to measure ever having run.
+ */
+ private static void reportAnythingUnexpected(List<Future<?>> racers) throws Exception {
+ for (Future<?> racer : racers) {
+ racer.get();
+ }
+ }
+
+ /**
+ * Takes the trial's own fixtures back out of the application, whatever happened above. Other
+ * test classes share it, and the account's certificate quota is 20, so a trial that failed
+ * part way through must not leave either resource behind. Nothing is asserted and either
+ * resource may already be gone, since which one survives is exactly what the race decides.
+ */
+ private void cleanUpTrial(Distribution created, String name, String certificateId) {
+ if (created != null) {
+ try {
+ cloudFrontService.removeDistribution(created.getId());
+ } catch (AwsException ignored) {
+ // Already removed, which is one of the outcomes this test measures.
+ }
+ }
+ if (iamService.findServerCertificateById(certificateId).isPresent()) {
+ try {
+ iamService.deleteServerCertificate(name, List.of());
+ } catch (AwsException ignored) {
+ // Deleted between that check and here by the racing task.
+ }
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicCertificateDeleteRaceTest.java b/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicCertificateDeleteRaceTest.java
new file mode 100644
index 0000000000..707ef84aed
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicCertificateDeleteRaceTest.java
@@ -0,0 +1,175 @@
+package io.github.hectorvent.floci.services.elb;
+
+import io.github.hectorvent.floci.services.acm.CertificateGenerator;
+import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * A Classic load balancer and the certificate its listener names cannot end up disagreeing about
+ * whether the certificate exists.
+ *
+ * <p>Same hazard as the CloudFront one: {@code DeleteServerCertificate} checks the reference
+ * providers under its own lock, while {@code CreateLoadBalancer} checks the certificate and then
+ * commits the load balancer. Driven over the wire here, because the listener's certificate arrives
+ * as a request parameter and the check lives behind the handler.
+ */
+@QuarkusTest
+class ElbClassicCertificateDeleteRaceTest {
+
+ private static final String ELB_AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20260427/us-east-1/elasticloadbalancing/aws4_request";
+ private static final String IAM_AUTH =
+ "AWS4-HMAC-SHA256 Credential=test/20260227/us-east-1/iam/aws4_request";
+ private static final String CLASSIC_VERSION = "2012-06-01";
+ private static final CertificateGenerator GENERATOR = new CertificateGenerator();
+ private static final int TRIALS = 60;
+
+ private static String suffix() {
+ return Long.toString(System.nanoTime(), 36);
+ }
+
+ private static String uploadCertificate(String name) {
+ CertificateGenerator.GeneratedCertificate pair = GENERATOR.generateSelfSignedCertificate(
+ "elb.race.test.local", List.of(), KeyAlgorithm.EC_prime256v1);
+ return given().header("Authorization", IAM_AUTH)
+ .formParam("Action", "UploadServerCertificate")
+ .formParam("ServerCertificateName", name)
+ .formParam("CertificateBody", pair.certificatePem())
+ .formParam("PrivateKey", pair.privateKeyPem())
+ .when().post("/").then().statusCode(200)
+ .extract().path("UploadServerCertificateResponse.UploadServerCertificateResult"
+ + ".ServerCertificateMetadata.Arn");
+ }
+
+ private static int createLoadBalancer(String lbName, String certificateArn) {
+ return given().header("Authorization", ELB_AUTH)
+ .formParam("Action", "CreateLoadBalancer")
+ .formParam("Version", CLASSIC_VERSION)
+ .formParam("LoadBalancerName", lbName)
+ .formParam("Subnets.member.1", Ec2Service.defaultSubnetId("us-east-1", "a"))
+ .formParam("Listeners.member.1.Protocol", "HTTPS")
+ .formParam("Listeners.member.1.LoadBalancerPort", "443")
+ .formParam("Listeners.member.1.InstanceProtocol", "HTTP")
+ .formParam("Listeners.member.1.InstancePort", "8080")
+ .formParam("Listeners.member.1.SSLCertificateId", certificateArn)
+ .when().post("/").then().extract().statusCode();
+ }
+
+ private static int deleteCertificate(String name) {
+ return given().header("Authorization", IAM_AUTH)
+ .formParam("Action", "DeleteServerCertificate")
+ .formParam("ServerCertificateName", name)
+ .when().post("/").then().extract().statusCode();
+ }
+
+ private static boolean certificateExists(String name) {
+ return given().header("Authorization", IAM_AUTH)
+ .formParam("Action", "GetServerCertificate")
+ .formParam("ServerCertificateName", name)
+ .when().post("/").then().extract().statusCode() == 200;
+ }
+
+ private static int deleteLoadBalancer(String lbName) {
+ return given().header("Authorization", ELB_AUTH)
+ .formParam("Action", "DeleteLoadBalancer")
+ .formParam("Version", CLASSIC_VERSION)
+ .formParam("LoadBalancerName", lbName)
+ .when().post("/").then().extract().statusCode();
+ }
+
+ @Test
+ void aLoadBalancerIsNeverLeftHoldingADeletedCertificate() throws Exception {
+ List<String> dangling = new ArrayList<>();
+ List<String> unresolved = new ArrayList<>();
+
+ for (int trial = 0; trial < TRIALS; trial++) {
+ String name = "elb-race-cert-" + trial + "-" + suffix();
+ String arn = uploadCertificate(name);
+ String lbName = "elb-race-" + trial + "-" + suffix();
+
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicInteger createStatus = new AtomicInteger();
+ AtomicInteger deleteStatus = new AtomicInteger();
+ try {
+ List<Future<?>> racers = List.of(pool.submit(() -> {
+ start.await();
+ createStatus.set(createLoadBalancer(lbName, arn));
+ return null;
+ }), pool.submit(() -> {
+ start.await();
+ deleteStatus.set(deleteCertificate(name));
+ return null;
+ }));
+ start.countDown();
+ pool.shutdown();
+ assertTrue(pool.awaitTermination(60, TimeUnit.SECONDS), "trial did not finish");
+ reportAnythingUnexpected(racers);
+
+ int create = createStatus.get();
+ int delete = deleteStatus.get();
+ if (create == 200 && delete == 200 && !certificateExists(name)) {
+ dangling.add("trial " + trial + ": load balancer " + lbName
+ + " holds deleted certificate " + name);
+ }
+ // Exactly one order wins, and the loser says why: the load balancer commits and
+ // the delete answers DeleteConflict, or the certificate goes and the listener is
+ // refused with CertificateNotFound. Two failures would leave the race untested
+ // while the trial still looked clean, so they are collected rather than ignored.
+ if (!(create == 200 && delete == 409) && !(create == 400 && delete == 200)) {
+ unresolved.add("trial " + trial + ": create=" + create + " delete=" + delete);
+ }
+ } finally {
+ pool.shutdownNow();
+ cleanUpTrial(createStatus.get() == 200, lbName, name);
+ }
+ }
+
+ assertTrue(dangling.isEmpty(),
+ "a load balancer was left holding a deleted certificate in "
+ + dangling.size() + " of " + TRIALS + " trials: " + dangling);
+ assertTrue(unresolved.isEmpty(),
+ "every trial must resolve as one order winning and the other being told why, "
+ + "these did not: " + unresolved);
+ }
+
+ /**
+ * Surfaces anything a racing task threw. The throwable otherwise stays in the task's
+ * {@link Future}, the status stays at its initial zero, and the trial reads as the other
+ * order having won without the race ever having run.
+ */
+ private static void reportAnythingUnexpected(List<Future<?>> racers) throws Exception {
+ for (Future<?> racer : racers) {
+ racer.get();
+ }
+ }
+
+ /**
+ * Takes the trial's own fixtures back out of the application, whatever happened above. Other
+ * test classes share it, and the account's certificate quota is 20, so a trial that failed
+ * part way through must not leave either resource behind. The status codes are ignored: which
+ * of the two is still there is exactly what the race decides.
+ */
+ private static void cleanUpTrial(boolean created, String lbName, String name) {
+ if (created) {
+ deleteLoadBalancer(lbName);
+ }
+ if (certificateExists(name)) {
+ deleteCertificate(name);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicServiceTest.java
index 17cae367f6..e2b70258a5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elb/ElbClassicServiceTest.java
@@ -16,6 +16,13 @@ import org.mockito.junit.jupiter.MockitoExtension;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -147,6 +154,109 @@ class ElbClassicServiceTest {
assertEquals("DuplicateLoadBalancerName", e.getErrorCode());
}
+ /**
+ * The name check and the write have to be one step. Checked before the write and nothing
+ * else, two creates of one name both pass the check and both commit, and the caller of the
+ * first is told it succeeded while holding a load balancer that was replaced.
+ */
+ @Test
+ void twoConcurrentCreatesOfOneNameLeaveOnlyOneWinner() throws Exception {
+ for (int attempt = 0; attempt < 8; attempt++) {
+ String name = "concurrent-elb-" + attempt;
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ CountDownLatch start = new CountDownLatch(1);
+ try {
+ List<Future<?>> creates = List.of(
+ pool.submit(() -> {
+ start.await();
+ // Zones rather than subnets, and no security group, so that nothing
+ // in here reaches a mock from two threads at once.
+ return service.createLoadBalancer(REGION, name,
+ List.of(httpListener()), List.of(REGION + "a"), List.of(),
+ List.of(), null, Map.of());
+ }),
+ pool.submit(() -> {
+ start.await();
+ return service.createLoadBalancer(REGION, name,
+ List.of(httpListener()), List.of(REGION + "a"), List.of(),
+ List.of(), null, Map.of());
+ }));
+ start.countDown();
+ pool.shutdown();
+ assertTrue(pool.awaitTermination(30, TimeUnit.SECONDS), "attempt did not finish");
+
+ int won = 0;
+ int refused = 0;
+ for (Future<?> create : creates) {
+ try {
+ create.get();
+ won++;
+ } catch (ExecutionException e) {
+ assertTrue(e.getCause() instanceof AwsException aws
+ && "DuplicateLoadBalancerName".equals(aws.getErrorCode()),
+ "the loser must be refused by name, got " + e.getCause());
+ refused++;
+ }
+ }
+ assertEquals(1, won, "exactly one create must succeed for attempt " + attempt);
+ assertEquals(1, refused, "exactly one create must be refused for attempt " + attempt);
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+ }
+
+ /**
+ * IAM's {@code DeleteServerCertificate} walks these listener lists while deciding whether the
+ * certificate it is about to remove is still in use. It holds its own lock for that and not
+ * this service's, by design, so listener writes run alongside the walk and the list has to
+ * tolerate them: on a plain list the walk ends in ConcurrentModificationException, which no
+ * mapper covers, and the delete answers an unmapped 500.
+ */
+ @Test
+ void theCertificateScanSurvivesListenerWritesOnAnotherThread() throws Exception {
+ create("scanned-elb");
+ ClassicListener https = new ClassicListener();
+ https.setProtocol("HTTPS");
+ https.setLoadBalancerPort(443);
+ https.setInstanceProtocol("HTTP");
+ https.setInstancePort(80);
+ https.setSslCertificateId("arn:aws:iam::000000000000:server-certificate/scanned");
+
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicBoolean writing = new AtomicBoolean(true);
+ try {
+ Future<?> writer = pool.submit(() -> {
+ start.await();
+ try {
+ for (int i = 0; i < 400; i++) {
+ service.createLoadBalancerListeners(REGION, "scanned-elb", List.of(https));
+ service.deleteLoadBalancerListeners(REGION, "scanned-elb", List.of(443));
+ }
+ } finally {
+ writing.set(false);
+ }
+ return null;
+ });
+ Future<Integer> scanner = pool.submit(() -> {
+ start.await();
+ int scans = 0;
+ while (writing.get()) {
+ service.serverCertificateReferences();
+ scans++;
+ }
+ return scans;
+ });
+ start.countDown();
+
+ writer.get(30, TimeUnit.SECONDS);
+ assertTrue(scanner.get(30, TimeUnit.SECONDS) > 0, "the scan never ran");
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
@Test
void loadBalancersAreScopedByRegion() {
create("my-elb");