Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudfront): honor origin connection timeouts and attempts, answer 504 (#5802)
Matheus Oliveira5306e0e470c9
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.javaindex c1bde596f4..7f9cb33ccc 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java@@ -42,14 +42,12 @@ public class CloudFrontController {/*** Origin timeout constraints from the CloudFront {@code CustomOriginConfig} and- * {@code Origin} API references. {@code OriginReadTimeout} accepts 1-120 seconds, default 30.- * {@code OriginKeepaliveTimeout} accepts 1-300 seconds, default 5. {@code ResponseCompletionTimeout}- * has no enforced maximum when unset (represented here as 0), but when set must be at least- * {@code OriginReadTimeout}.+ * {@code Origin} API references. {@code OriginKeepaliveTimeout} accepts 1-300 seconds,+ * default 5. {@code ResponseCompletionTimeout} has no enforced maximum when unset+ * (represented here as 0), but when set must be at least {@code OriginReadTimeout}.+ * The read and connection timeouts live in {@link CloudFrontOriginTimeouts}, shared with+ * the serving layer that applies them.*/- private static final int DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS = 30;- private static final int MIN_ORIGIN_READ_TIMEOUT_SECONDS = 1;- private static final int MAX_ORIGIN_READ_TIMEOUT_SECONDS = 120;private static final int DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS = 5;private static final int MIN_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS = 1;private static final int MAX_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS = 300;@@ -1961,8 +1959,8 @@ public class CloudFrontController {.elem("HTTPSPort", coc.getOrDefault("HTTPSPort", "443").toString()).elem("OriginProtocolPolicy",coc.getOrDefault("OriginProtocolPolicy", "https-only").toString())- .elem("OriginReadTimeout",- coc.getOrDefault("OriginReadTimeout", DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS).toString())+ .elem("OriginReadTimeout", coc.getOrDefault("OriginReadTimeout",+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS).toString()).elem("OriginKeepaliveTimeout",coc.getOrDefault("OriginKeepaliveTimeout", DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS).toString())@@ -2840,8 +2838,8 @@ public class CloudFrontController {if (inCustomOriginConfig && customConfig != null) {originReadTimeoutSeconds = parseBoundedTimeout("InvalidOriginReadTimeout", "OriginReadTimeout",- r.getElementText(), MIN_ORIGIN_READ_TIMEOUT_SECONDS,- MAX_ORIGIN_READ_TIMEOUT_SECONDS);+ r.getElementText(), CloudFrontOriginTimeouts.MIN_ORIGIN_READ_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.MAX_ORIGIN_READ_TIMEOUT_SECONDS);customConfig.put("OriginReadTimeout", originReadTimeoutSeconds);}}@@ -2956,7 +2954,7 @@ public class CloudFrontController {Integer completionTimeout = current.getResponseCompletionTimeout();int effectiveReadTimeout = originReadTimeoutSeconds != null? originReadTimeoutSeconds- : DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS;+ : CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS;if (completionTimeout != null && completionTimeout > 0&& completionTimeout < effectiveReadTimeout) {throw new AwsException(diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.javaindex f457afdd92..76ec0ca340 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.cloudfront;import io.github.hectorvent.floci.core.common.SsrfProtection;+import org.apache.hc.client5.http.ConnectTimeoutException;import org.apache.hc.client5.http.DnsResolver;import org.apache.hc.client5.http.SystemDefaultDnsResolver;import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase;@@ -31,7 +32,9 @@ import java.io.FilterInputStream;import java.io.IOException;import java.io.InputStream;import java.io.InterruptedIOException;+import java.net.ConnectException;import java.net.InetAddress;+import java.net.SocketTimeoutException;import java.net.UnknownHostException;import java.net.http.HttpHeaders;import java.net.http.HttpRequest;@@ -39,6 +42,7 @@ import java.time.Duration;import java.util.ArrayList;import java.util.Collection;import java.util.List;+import java.util.Locale;import java.util.Map;import java.util.Set;import java.util.TreeMap;@@ -54,8 +58,12 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {private static final Logger LOG = Logger.getLogger(CloudFrontOriginHttpClient.class);- private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(10);- private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(30);+ private static final String ORIGIN_TIMEOUT_MESSAGE =+ "The origin didn't respond before the request expired.";+ private static final Duration CONNECT_TIMEOUT = CloudFrontOriginTimeouts.DEFAULT_CONNECTION_TIMEOUT;+ private static final Duration RESPONSE_TIMEOUT = CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT;+ /** CloudFront retries a response timeout only for the methods it treats as idempotent. */+ private static final Set<String> IDEMPOTENT_METHODS = Set.of("GET", "HEAD");static final int MAX_CONNECTIONS = 100;private final CloseableHttpClient client;@@ -119,6 +127,14 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {.build();}+ /**+ * The connection settings of a distribution's origin that one request must honor:+ * {@code Origin.ConnectionAttempts}, {@code Origin.ConnectionTimeout} and the custom origin's+ * {@code OriginReadTimeout}. Read per request, since the client and its pool are shared.+ */+ record OriginTimeouts(Duration connectTimeout, Duration responseTimeout, int connectionAttempts) {+ }+/*** An origin response whose body has not been read. The caller reads {@code body} as it needs it* and closes it, which releases the connection. {@code contentLength} is the length the body@@ -139,12 +155,64 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {List<CloudFrontOriginRequestBuilder.Header> forwardedHeaders,Map<String, String> originHeaders, InputStream requestBody, long requestBodyLength)throws IOException, InterruptedException {+ return open(request, forwardedHeaders, originHeaders, requestBody, requestBodyLength, null);+ }++ /**+ * Sends {@code request}, honoring {@code timeouts}. A connection attempt counts for every+ * method, so a connect timeout or a refused connection is retried up to the configured+ * connection attempts whatever the method is. A response timeout is retried only for an+ * idempotent method (GET, HEAD), whose request has no body to replay; every other method is+ * attempted once for it, as CloudFront does not retry it. When the attempts run out, a timeout+ * raises {@link CloudFrontOriginTimeoutException} so the viewer is answered 504 rather than+ * 502, while a refused connection propagates unchanged, as a bad gateway.+ */+ OpenResponse open(HttpRequest request,+ List<CloudFrontOriginRequestBuilder.Header> forwardedHeaders,+ Map<String, String> originHeaders, InputStream requestBody, long requestBodyLength,+ OriginTimeouts timeouts)+ throws IOException, InterruptedException {+ OriginTimeouts effective = timeouts != null+ ? timeouts+ : new OriginTimeouts(CONNECT_TIMEOUT, request.timeout().orElse(RESPONSE_TIMEOUT), 1);+ int attempts = Math.max(1, effective.connectionAttempts());+ IOException lastFailure = null;+ for (int attempt = 1; attempt <= attempts; attempt++) {+ try {+ return send(request, forwardedHeaders, originHeaders, requestBody, requestBodyLength, effective);+ } catch (ConnectTimeoutException e) {+ lastFailure = e;+ } catch (SocketTimeoutException e) {+ if (!idempotent(request.method())) {+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, e);+ }+ lastFailure = e;+ } catch (ConnectException e) {+ lastFailure = e;+ }+ if (attempt < attempts) {+ LOG.debugv("CloudFront origin connection attempt {0} of {1} failed; retrying",+ attempt, attempts);+ }+ }+ if (lastFailure instanceof SocketTimeoutException timeout) {+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, timeout);+ }+ throw lastFailure;+ }++ private OpenResponse send(HttpRequest request,+ List<CloudFrontOriginRequestBuilder.Header> forwardedHeaders,+ Map<String, String> originHeaders, InputStream requestBody, long requestBodyLength,+ OriginTimeouts timeouts)+ throws IOException, InterruptedException {// No entity content type: the caller's Content-Type request header is sent as is.HttpUriRequestBase originRequest = originRequest(request, forwardedHeaders, originHeaders,requestBody != null ? new InputStreamEntity(requestBody, requestBodyLength, null) : null);ClassicHttpResponse response;try {- response = client.executeOpen(RoutingSupport.determineHost(originRequest), originRequest, context(request));+ response = client.executeOpen(RoutingSupport.determineHost(originRequest), originRequest,+ context(timeouts));} catch (HttpException e) {throw new IOException("CloudFront origin request has no target host", e);} catch (InterruptedIOException e) {@@ -169,6 +237,10 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {}}+ private static boolean idempotent(String method) {+ return method != null && IDEMPOTENT_METHODS.contains(method.toUpperCase(Locale.ROOT));+ }+/*** An origin response body that releases its connection when it is closed. Closed at its end, the* connection goes back to the pool. Closed before its end, as when a viewer goes away or a custom@@ -271,12 +343,14 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {return builder;}- private static HttpClientContext context(HttpRequest request) {+ private static HttpClientContext context(OriginTimeouts timeouts) {HttpClientContext context = HttpClientContext.create();- Duration responseTimeout = request.timeout().orElse(RESPONSE_TIMEOUT);context.setRequestConfig(RequestConfig.custom()+ .setConnectTimeout(Timeout.of(timeouts.connectTimeout()))+ // Waiting for a free pooled connection is emulator-side backpressure, not an origin+ // setting, so it keeps the fixed default rather than the origin's connect timeout..setConnectionRequestTimeout(Timeout.of(CONNECT_TIMEOUT))- .setResponseTimeout(Timeout.of(responseTimeout))+ .setResponseTimeout(Timeout.of(timeouts.responseTimeout())).setRedirectsEnabled(false).setContentCompressionEnabled(false).setProtocolUpgradeEnabled(false)diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.javanew file mode 100644index 0000000000..6dc0df51b3--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.java@@ -0,0 +1,15 @@+package io.github.hectorvent.floci.services.cloudfront;++import java.io.IOException;++/**+ * Signals that a custom origin did not answer before the timeout configured on the origin expired.+ * CloudFront answers the viewer with a 504 Gateway Timeout for this case, distinct from the 502 it+ * answers for a genuine bad gateway, so the serving layer must be able to tell the two apart.+ */+final class CloudFrontOriginTimeoutException extends IOException {++ CloudFrontOriginTimeoutException(String message, Throwable cause) {+ super(message, cause);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.javanew file mode 100644index 0000000000..8260e0f41b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.java@@ -0,0 +1,30 @@+package io.github.hectorvent.floci.services.cloudfront;++import java.time.Duration;++/**+ * The origin-timeout settings of a CloudFront distribution, shared by the distribution API (which+ * validates and emits them) and the serving layer (which applies them to origin requests), so both+ * sides agree on the values the {@code Origin} and {@code CustomOriginConfig} API references+ * document. {@code OriginReadTimeout} accepts 1-120 seconds, default 30; {@code ConnectionTimeout}+ * accepts 1-10 seconds, default 10; {@code ConnectionAttempts} accepts 1-3, default 3.+ */+final class CloudFrontOriginTimeouts {++ static final int DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS = 30;+ static final int MIN_ORIGIN_READ_TIMEOUT_SECONDS = 1;+ static final int MAX_ORIGIN_READ_TIMEOUT_SECONDS = 120;+ static final int DEFAULT_CONNECTION_TIMEOUT_SECONDS = 10;+ static final int MIN_CONNECTION_TIMEOUT_SECONDS = 1;+ static final int MAX_CONNECTION_TIMEOUT_SECONDS = 10;+ static final int DEFAULT_CONNECTION_ATTEMPTS = 3;+ static final int MAX_CONNECTION_ATTEMPTS = 3;++ static final Duration DEFAULT_ORIGIN_READ_TIMEOUT =+ Duration.ofSeconds(DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS);+ static final Duration DEFAULT_CONNECTION_TIMEOUT =+ Duration.ofSeconds(DEFAULT_CONNECTION_TIMEOUT_SECONDS);++ private CloudFrontOriginTimeouts() {+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.javaindex fa1b44b332..e403736bfd 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java@@ -101,6 +101,8 @@ public class CloudFrontServingController {java.util.Set.of("Expires", "Signature", "Key-Pair-Id", "Policy", "Hash-Algorithm");/** Viewer methods that can carry a request body and that CloudFront never caches. */private static final Set<String> BODY_METHODS = Set.of("POST", "PUT", "PATCH", "DELETE");+ /** The 504 body CloudFront returns when the origin did not answer in time. */+ static final String ORIGIN_TIMEOUT_MESSAGE = "The origin didn't respond before the request expired.";private final CloudFrontService service;private final S3Service s3Service;@@ -760,8 +762,7 @@ public class CloudFrontServingController {URI target = buildCustomOriginUri(origin, viewerScheme, forwardUri, originRequest.rawQuery());HttpRequest.Builder rb = HttpRequest.newBuilder()- .uri(target)- .timeout(Duration.ofSeconds(30));+ .uri(target);// Origin custom headers: CloudFront adds these to every request it forwards to the origin// (overriding any same-named viewer header), which is how a distribution restricts an origin// to CloudFront-only traffic via a shared secret header.@@ -782,7 +783,7 @@ public class CloudFrontServingController {ViewerBody requestBody = originRequestBody(method, viewerBody);CloudFrontOriginHttpClient.OpenResponse resp = httpClient.open(rb.build(), originRequest.headers(),originHeaders, requestBody != null ? requestBody.stream() : null,- requestBody != null ? requestBody.length() : -1);+ requestBody != null ? requestBody.length() : -1, originTimeouts(origin));String ct = resp.headers().firstValue("content-type").orElse(DEFAULT_CONTENT_TYPE);long contentLength = resp.contentLength();if (!includeBody) {@@ -795,6 +796,9 @@ public class CloudFrontServingController {LOG.warnv("CloudFront custom origin fetch was interrupted for host {0}",safeOriginName(origin));return OriginResponse.error(502, "Bad Gateway.");+ } catch (CloudFrontOriginTimeoutException e) {+ LOG.warnv("CloudFront custom origin timed out for host {0}", safeOriginName(origin));+ return OriginResponse.error(504, ORIGIN_TIMEOUT_MESSAGE);} catch (Exception e) {// Do not log the target URI: forwarded query strings can contain credentials or tokens.LOG.warnv("CloudFront custom origin fetch failed for host {0}: {1}",@@ -803,6 +807,34 @@ public class CloudFrontServingController {}}+ /**+ * The origin's connection settings for one request: {@code Origin.ConnectionTimeout},+ * {@code CustomOriginConfig.OriginReadTimeout} and {@code Origin.ConnectionAttempts}. A value+ * outside the range AWS accepts falls back to the AWS default, since the model can be built+ * programmatically or loaded from persisted state without the API validation.+ */+ private static CloudFrontOriginHttpClient.OriginTimeouts originTimeouts(Origin origin) {+ int connectSeconds = clampToDefault(origin.getConnectionTimeout(),+ CloudFrontOriginTimeouts.MIN_CONNECTION_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.MAX_CONNECTION_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.DEFAULT_CONNECTION_TIMEOUT_SECONDS);+ int readSeconds = clampToDefault(+ intOrDefault(origin.getCustomOriginConfig(), "OriginReadTimeout",+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS),+ CloudFrontOriginTimeouts.MIN_ORIGIN_READ_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.MAX_ORIGIN_READ_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS);+ int attempts = clampToDefault(origin.getConnectionAttempts(),+ 1, CloudFrontOriginTimeouts.MAX_CONNECTION_ATTEMPTS,+ CloudFrontOriginTimeouts.DEFAULT_CONNECTION_ATTEMPTS);+ return new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(connectSeconds), Duration.ofSeconds(readSeconds), attempts);+ }++ private static int clampToDefault(int value, int min, int max, int fallback) {+ return value < min || value > max ? fallback : value;+ }+/*** The body CloudFront sends to the origin: POST, PUT and PATCH always carry one (possibly empty, so* the origin still sees {@code Content-Length: 0}); DELETE carries one only when the viewer sent it.diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.javaindex 2de8ae7dd9..7aef81e48a 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java@@ -45,6 +45,7 @@ import java.util.Random;import java.util.concurrent.BlockingQueue;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CountDownLatch;+import java.util.concurrent.Executors;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;@@ -82,9 +83,14 @@ class CloudFrontCustomOriginServingTest {final PartitionCleanup cleanup = new PartitionCleanup();private HttpServer originServer;+ private CountDownLatch originRelease;@AfterEachvoid stopOrigin() {+ if (originRelease != null) {+ originRelease.countDown();+ originRelease = null;+ }if (originServer != null) {originServer.stop(0);}@@ -340,6 +346,56 @@ class CloudFrontCustomOriginServingTest {assertEquals(5, hits.get());}+ @Test+ void answers504AndStopsWaitingWhenTheOriginExceedsTheReadTimeout() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ startHoldingOrigin(hits);+ Distribution created = cloudFrontService.createDistribution(+ customOriginDistribution(allMethodsBehavior(), null, 1, 1), Map.of());++ long started = System.nanoTime();+ given().header("Host", created.getDomainName()).when().get("/page")+ .then().statusCode(504)+ .body(equalTo(CloudFrontServingController.ORIGIN_TIMEOUT_MESSAGE));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertEquals(1, hits.get());+ assertTrue(elapsedMillis < 10000,+ "expected the 1 s origin read timeout to end the request, took " + elapsedMillis + " ms");+ }++ @Test+ void retriesTheReadTimeoutForGetUpToConnectionAttempts() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ startHoldingOnlyTheFirstRequest(hits);+ Distribution created = cloudFrontService.createDistribution(+ customOriginDistribution(allMethodsBehavior(), null, 1, 2), Map.of());++ given().header("Host", created.getDomainName()).when().get("/page")+ .then().statusCode(200)+ .body(equalTo("origin-body"));++ assertEquals(2, hits.get(), "a GET that timed out should be retried once");+ }++ @Test+ void doesNotRetryTheReadTimeoutForPost() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ startHoldingOrigin(hits);+ Distribution created = cloudFrontService.createDistribution(+ customOriginDistribution(allMethodsBehavior(), null, 1, 2), Map.of());++ long started = System.nanoTime();+ given().header("Host", created.getDomainName()).when().post("/page")+ .then().statusCode(504)+ .body(equalTo(CloudFrontServingController.ORIGIN_TIMEOUT_MESSAGE));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertEquals(1, hits.get(), "a POST that hit a read timeout must not be retried");+ assertTrue(elapsedMillis < 10000,+ "expected the 1 s origin read timeout to end the request, took " + elapsedMillis + " ms");+ }+@Testvoid streamsALargeOriginResponseWithItsLength() throws Exception {byte[] video = randomBytes(3 * 1024 * 1024 + 1);@@ -932,6 +988,64 @@ class CloudFrontCustomOriginServingTest {originServer.start();}+ /**+ * An origin that counts its hits and holds every request open until the test releases it, so an+ * attempt ends only by the client's own read timeout rather than by racing a fixed origin delay.+ */+ private void startHoldingOrigin(AtomicInteger hits) throws IOException {+ originRelease = new CountDownLatch(1);+ CountDownLatch release = originRelease;+ originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ originServer.setExecutor(Executors.newVirtualThreadPerTaskExecutor());+ originServer.createContext("/", exchange -> {+ hits.incrementAndGet();+ holdUntilReleased(release);+ answerOriginBody(exchange);+ });+ originServer.start();+ }++ /** An origin that holds its first request until the retry arrives, then answers that one. */+ private void startHoldingOnlyTheFirstRequest(AtomicInteger hits) throws IOException {+ originRelease = new CountDownLatch(1);+ CountDownLatch retryArrived = originRelease;+ originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ originServer.setExecutor(Executors.newVirtualThreadPerTaskExecutor());+ originServer.createContext("/", exchange -> {+ if (hits.incrementAndGet() == 1) {+ holdUntilReleased(retryArrived);+ } else {+ retryArrived.countDown();+ }+ answerOriginBody(exchange);+ });+ originServer.start();+ }++ /**+ * Holds the request open until {@code release} opens, with a generous cap so a client that never+ * times out fails the assertion instead of hanging the suite.+ */+ private static void holdUntilReleased(CountDownLatch release) {+ try {+ release.await(30, TimeUnit.SECONDS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }++ private static void answerOriginBody(HttpExchange exchange) {+ byte[] body = "origin-body".getBytes(StandardCharsets.UTF_8);+ try {+ exchange.sendResponseHeaders(200, body.length);+ exchange.getResponseBody().write(body);+ } catch (IOException expected) {+ // The viewer gave up on this attempt after its read timeout, which the test triggers.+ } finally {+ exchange.close();+ }+ }+/** An origin that answers every GET with {@code body}, with its length or chunked. */private void startServingOrigin(byte[] body, boolean withLength) throws IOException {originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);@@ -1007,11 +1121,25 @@ class CloudFrontCustomOriginServingTest {private Distribution customOriginDistribution(DefaultCacheBehavior behavior,List<Map<String, String>> customHeaders) {+ return customOriginDistribution(behavior, customHeaders, null, null);+ }++ private Distribution customOriginDistribution(DefaultCacheBehavior behavior,+ List<Map<String, String>> customHeaders,+ Integer readTimeoutSeconds,+ Integer connectionAttempts) {+ Map<String, Object> originConfig = customOriginConfig(originServer.getAddress().getPort());+ if (readTimeoutSeconds != null) {+ originConfig.put("OriginReadTimeout", readTimeoutSeconds);+ }Origin customOrigin = new Origin();customOrigin.setId("custom-origin");customOrigin.setDomainName("127.0.0.1");- customOrigin.setCustomOriginConfig(customOriginConfig(originServer.getAddress().getPort()));+ customOrigin.setCustomOriginConfig(originConfig);customOrigin.setCustomHeaders(customHeaders);+ if (connectionAttempts != null) {+ customOrigin.setConnectionAttempts(connectionAttempts);+ }DistributionConfig config = new DistributionConfig();config.setEnabled(true);config.setOrigins(List.of(customOrigin));diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.javaindex eff729e66b..ec130dd612 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java@@ -21,8 +21,10 @@ import java.io.ByteArrayInputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;+import java.net.ConnectException;import java.net.InetAddress;import java.net.InetSocketAddress;+import java.net.ServerSocket;import java.net.URI;import java.net.UnknownHostException;import java.net.http.HttpRequest;@@ -36,6 +38,7 @@ import java.util.Map;import java.util.Random;import java.util.concurrent.CopyOnWriteArrayList;import java.util.concurrent.CountDownLatch;+import java.util.concurrent.Executors;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;import java.util.concurrent.atomic.AtomicLong;@@ -361,6 +364,185 @@ class CloudFrontOriginHttpClientTest {}}+ @Test+ void retriesAnIdempotentRequestUntilTheAttemptsAreExhausted() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ CountDownLatch release = new CountDownLatch(1);+ HttpServer server = holdingServer(hits, release);+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofMillis(200), 3);+ HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");++ long started = System.nanoTime();+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertEquals(3, hits.get());+ assertTrue(elapsedMillis < 5000,+ "expected the 200 ms read timeout to end every attempt on an origin that never "+ + "answers, took " + elapsedMillis + " ms");+ } finally {+ release.countDown();+ server.stop(0);+ }+ }++ @Test+ void doesNotRetryAnIdempotentRequestWhenOnlyOneAttemptIsConfigured() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ CountDownLatch release = new CountDownLatch(1);+ HttpServer server = holdingServer(hits, release);+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofMillis(200), 1);+ HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");++ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));++ assertEquals(1, hits.get());+ } finally {+ release.countDown();+ server.stop(0);+ }+ }++ @Test+ void doesNotRetryAResponseTimeoutForANonIdempotentRequest() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ CountDownLatch release = new CountDownLatch(1);+ HttpServer server = holdingServer(hits, release);+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofMillis(200), 3);+ HttpRequest request = HttpRequest.newBuilder(URI.create(+ "http://origin.invalid:" + server.getAddress().getPort() + "/"))+ .method("POST", HttpRequest.BodyPublishers.noBody())+ .build();++ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));++ assertEquals(1, hits.get(), "a POST that hit a response timeout must not be retried");+ } finally {+ release.countDown();+ server.stop(0);+ }+ }++ @Test+ void retriesAConnectTimeoutForANonIdempotentRequest() throws Exception {+ AtomicInteger resolutions = new AtomicInteger();+ DnsResolver resolver = countingResolver(resolutions, InetAddress.getByName("192.0.2.1"));+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver, List.of("blackhole.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofMillis(100), Duration.ofSeconds(5), 3);+ HttpRequest request = HttpRequest.newBuilder(URI.create("http://blackhole.invalid:81/"))+ .method("POST", HttpRequest.BodyPublishers.noBody())+ .build();++ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));++ assertEquals(3, resolutions.get(),+ "a POST whose connect times out counts as a connection attempt and is retried");+ }+ }++ @Test+ void retriesARefusedConnectionForANonIdempotentRequest() throws Exception {+ int refusedPort;+ try (ServerSocket socket = new ServerSocket(0)) {+ refusedPort = socket.getLocalPort();+ }+ AtomicInteger resolutions = new AtomicInteger();+ DnsResolver resolver = countingResolver(resolutions, InetAddress.getByName("127.0.0.1"));+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver, List.of("refused.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofSeconds(5), 3);+ HttpRequest request = HttpRequest.newBuilder(URI.create(+ "http://refused.invalid:" + refusedPort + "/"))+ .method("POST", HttpRequest.BodyPublishers.noBody())+ .build();++ assertThrows(ConnectException.class, () -> get(client, request, timeouts));++ assertEquals(3, resolutions.get(),+ "a POST whose connection is refused counts as a connection attempt and is retried");+ }+ }++ @Test+ void honorsTheConfiguredReadTimeout() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ CountDownLatch release = new CountDownLatch(1);+ HttpServer server = holdingServer(hits, release);+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofMillis(150), 1);+ HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");++ long started = System.nanoTime();+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertTrue(elapsedMillis < 5000,+ "expected the configured 150 ms read timeout to end the request, not the 30 s "+ + "default, took " + elapsedMillis + " ms");+ } finally {+ release.countDown();+ server.stop(0);+ }+ }++ private static CloudFrontOriginHttpClient.OpenResponse get(+ CloudFrontOriginHttpClient client, HttpRequest request,+ CloudFrontOriginHttpClient.OriginTimeouts timeouts) throws Exception {+ return client.open(request, List.of(), Map.of(), null, -1, timeouts);+ }++ /**+ * An origin that counts its hits and holds every request open until {@code release} opens, so a+ * client that answers nothing never races a fixed origin delay: only the client's own timeout+ * can end an attempt.+ */+ private static HttpServer holdingServer(AtomicInteger hits, CountDownLatch release) throws IOException {+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ server.setExecutor(Executors.newVirtualThreadPerTaskExecutor());+ server.createContext("/", exchange -> {+ hits.incrementAndGet();+ try {+ exchange.getRequestBody().readAllBytes();+ release.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ } catch (IOException expected) {+ // The client gave up on the attempt, which is what the read timeout under test causes.+ }+ try {+ byte[] body = "origin-ok".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(200, body.length);+ exchange.getResponseBody().write(body);+ } catch (IOException expected) {+ // The client is gone, so there is nobody left to answer.+ } finally {+ exchange.close();+ }+ });+ server.start();+ return server;+ }+private static CloudFrontOriginHttpClient.OpenResponse get(CloudFrontOriginHttpClient client,HttpRequest request) throws Exception {return client.open(request, List.of(), Map.of(), null, -1);@@ -404,6 +586,22 @@ class CloudFrontOriginHttpClientTest {};}+ /** A resolver that answers every lookup with {@code address}, counting the lookups. */+ private static DnsResolver countingResolver(AtomicInteger resolutions, InetAddress address) {+ return new DnsResolver() {+ @Override+ public InetAddress[] resolve(String host) {+ resolutions.incrementAndGet();+ return new InetAddress[] { address };+ }++ @Override+ public String resolveCanonicalHostname(String host) {+ return host;+ }+ };+ }+private static SSLContext serverSslContext(X509Certificate certificate, java.security.PrivateKey privateKey) throws Exception {char[] password = "changeit".toCharArray();