Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudfront): honor origin connection timeouts and attempts, answer 504 (#5802)

Matheus Oliveira5306e0e470c9
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java
index c1bde596f4..7f9cb33ccc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java
@@ -42,14 +42,12 @@ public class CloudFrontController {
/**
* Origin timeout constraints from the CloudFront {@code CustomOriginConfig} and
- * {@code Origin} API references. {@code OriginReadTimeout} accepts 1-120 seconds, default 30.
- * {@code OriginKeepaliveTimeout} accepts 1-300 seconds, default 5. {@code ResponseCompletionTimeout}
- * has no enforced maximum when unset (represented here as 0), but when set must be at least
- * {@code OriginReadTimeout}.
+ * {@code Origin} API references. {@code OriginKeepaliveTimeout} accepts 1-300 seconds,
+ * default 5. {@code ResponseCompletionTimeout} has no enforced maximum when unset
+ * (represented here as 0), but when set must be at least {@code OriginReadTimeout}.
+ * The read and connection timeouts live in {@link CloudFrontOriginTimeouts}, shared with
+ * the serving layer that applies them.
*/
- private static final int DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS = 30;
- private static final int MIN_ORIGIN_READ_TIMEOUT_SECONDS = 1;
- private static final int MAX_ORIGIN_READ_TIMEOUT_SECONDS = 120;
private static final int DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS = 5;
private static final int MIN_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS = 1;
private static final int MAX_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS = 300;
@@ -1961,8 +1959,8 @@ public class CloudFrontController {
.elem("HTTPSPort", coc.getOrDefault("HTTPSPort", "443").toString())
.elem("OriginProtocolPolicy",
coc.getOrDefault("OriginProtocolPolicy", "https-only").toString())
- .elem("OriginReadTimeout",
- coc.getOrDefault("OriginReadTimeout", DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS).toString())
+ .elem("OriginReadTimeout", coc.getOrDefault("OriginReadTimeout",
+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS).toString())
.elem("OriginKeepaliveTimeout",
coc.getOrDefault("OriginKeepaliveTimeout", DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS)
.toString())
@@ -2840,8 +2838,8 @@ public class CloudFrontController {
if (inCustomOriginConfig && customConfig != null) {
originReadTimeoutSeconds = parseBoundedTimeout(
"InvalidOriginReadTimeout", "OriginReadTimeout",
- r.getElementText(), MIN_ORIGIN_READ_TIMEOUT_SECONDS,
- MAX_ORIGIN_READ_TIMEOUT_SECONDS);
+ r.getElementText(), CloudFrontOriginTimeouts.MIN_ORIGIN_READ_TIMEOUT_SECONDS,
+ CloudFrontOriginTimeouts.MAX_ORIGIN_READ_TIMEOUT_SECONDS);
customConfig.put("OriginReadTimeout", originReadTimeoutSeconds);
}
}
@@ -2956,7 +2954,7 @@ public class CloudFrontController {
Integer completionTimeout = current.getResponseCompletionTimeout();
int effectiveReadTimeout = originReadTimeoutSeconds != null
? originReadTimeoutSeconds
- : DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS;
+ : CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS;
if (completionTimeout != null && completionTimeout > 0
&& completionTimeout < effectiveReadTimeout) {
throw new AwsException(
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java
index f457afdd92..76ec0ca340 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.cloudfront;
import io.github.hectorvent.floci.core.common.SsrfProtection;
+import org.apache.hc.client5.http.ConnectTimeoutException;
import org.apache.hc.client5.http.DnsResolver;
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase;
@@ -31,7 +32,9 @@ import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
+import java.net.ConnectException;
import java.net.InetAddress;
+import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.net.http.HttpHeaders;
import java.net.http.HttpRequest;
@@ -39,6 +42,7 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
@@ -54,8 +58,12 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {
private static final Logger LOG = Logger.getLogger(CloudFrontOriginHttpClient.class);
- private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(10);
- private static final Duration RESPONSE_TIMEOUT = Duration.ofSeconds(30);
+ private static final String ORIGIN_TIMEOUT_MESSAGE =
+ "The origin didn't respond before the request expired.";
+ private static final Duration CONNECT_TIMEOUT = CloudFrontOriginTimeouts.DEFAULT_CONNECTION_TIMEOUT;
+ private static final Duration RESPONSE_TIMEOUT = CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT;
+ /** CloudFront retries a response timeout only for the methods it treats as idempotent. */
+ private static final Set<String> IDEMPOTENT_METHODS = Set.of("GET", "HEAD");
static final int MAX_CONNECTIONS = 100;
private final CloseableHttpClient client;
@@ -119,6 +127,14 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {
.build();
}
+ /**
+ * The connection settings of a distribution's origin that one request must honor:
+ * {@code Origin.ConnectionAttempts}, {@code Origin.ConnectionTimeout} and the custom origin's
+ * {@code OriginReadTimeout}. Read per request, since the client and its pool are shared.
+ */
+ record OriginTimeouts(Duration connectTimeout, Duration responseTimeout, int connectionAttempts) {
+ }
+
/**
* An origin response whose body has not been read. The caller reads {@code body} as it needs it
* and closes it, which releases the connection. {@code contentLength} is the length the body
@@ -139,12 +155,64 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {
List<CloudFrontOriginRequestBuilder.Header> forwardedHeaders,
Map<String, String> originHeaders, InputStream requestBody, long requestBodyLength)
throws IOException, InterruptedException {
+ return open(request, forwardedHeaders, originHeaders, requestBody, requestBodyLength, null);
+ }
+
+ /**
+ * Sends {@code request}, honoring {@code timeouts}. A connection attempt counts for every
+ * method, so a connect timeout or a refused connection is retried up to the configured
+ * connection attempts whatever the method is. A response timeout is retried only for an
+ * idempotent method (GET, HEAD), whose request has no body to replay; every other method is
+ * attempted once for it, as CloudFront does not retry it. When the attempts run out, a timeout
+ * raises {@link CloudFrontOriginTimeoutException} so the viewer is answered 504 rather than
+ * 502, while a refused connection propagates unchanged, as a bad gateway.
+ */
+ OpenResponse open(HttpRequest request,
+ List<CloudFrontOriginRequestBuilder.Header> forwardedHeaders,
+ Map<String, String> originHeaders, InputStream requestBody, long requestBodyLength,
+ OriginTimeouts timeouts)
+ throws IOException, InterruptedException {
+ OriginTimeouts effective = timeouts != null
+ ? timeouts
+ : new OriginTimeouts(CONNECT_TIMEOUT, request.timeout().orElse(RESPONSE_TIMEOUT), 1);
+ int attempts = Math.max(1, effective.connectionAttempts());
+ IOException lastFailure = null;
+ for (int attempt = 1; attempt <= attempts; attempt++) {
+ try {
+ return send(request, forwardedHeaders, originHeaders, requestBody, requestBodyLength, effective);
+ } catch (ConnectTimeoutException e) {
+ lastFailure = e;
+ } catch (SocketTimeoutException e) {
+ if (!idempotent(request.method())) {
+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, e);
+ }
+ lastFailure = e;
+ } catch (ConnectException e) {
+ lastFailure = e;
+ }
+ if (attempt < attempts) {
+ LOG.debugv("CloudFront origin connection attempt {0} of {1} failed; retrying",
+ attempt, attempts);
+ }
+ }
+ if (lastFailure instanceof SocketTimeoutException timeout) {
+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, timeout);
+ }
+ throw lastFailure;
+ }
+
+ private OpenResponse send(HttpRequest request,
+ List<CloudFrontOriginRequestBuilder.Header> forwardedHeaders,
+ Map<String, String> originHeaders, InputStream requestBody, long requestBodyLength,
+ OriginTimeouts timeouts)
+ throws IOException, InterruptedException {
// No entity content type: the caller's Content-Type request header is sent as is.
HttpUriRequestBase originRequest = originRequest(request, forwardedHeaders, originHeaders,
requestBody != null ? new InputStreamEntity(requestBody, requestBodyLength, null) : null);
ClassicHttpResponse response;
try {
- response = client.executeOpen(RoutingSupport.determineHost(originRequest), originRequest, context(request));
+ response = client.executeOpen(RoutingSupport.determineHost(originRequest), originRequest,
+ context(timeouts));
} catch (HttpException e) {
throw new IOException("CloudFront origin request has no target host", e);
} catch (InterruptedIOException e) {
@@ -169,6 +237,10 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {
}
}
+ private static boolean idempotent(String method) {
+ return method != null && IDEMPOTENT_METHODS.contains(method.toUpperCase(Locale.ROOT));
+ }
+
/**
* An origin response body that releases its connection when it is closed. Closed at its end, the
* connection goes back to the pool. Closed before its end, as when a viewer goes away or a custom
@@ -271,12 +343,14 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {
return builder;
}
- private static HttpClientContext context(HttpRequest request) {
+ private static HttpClientContext context(OriginTimeouts timeouts) {
HttpClientContext context = HttpClientContext.create();
- Duration responseTimeout = request.timeout().orElse(RESPONSE_TIMEOUT);
context.setRequestConfig(RequestConfig.custom()
+ .setConnectTimeout(Timeout.of(timeouts.connectTimeout()))
+ // Waiting for a free pooled connection is emulator-side backpressure, not an origin
+ // setting, so it keeps the fixed default rather than the origin's connect timeout.
.setConnectionRequestTimeout(Timeout.of(CONNECT_TIMEOUT))
- .setResponseTimeout(Timeout.of(responseTimeout))
+ .setResponseTimeout(Timeout.of(timeouts.responseTimeout()))
.setRedirectsEnabled(false)
.setContentCompressionEnabled(false)
.setProtocolUpgradeEnabled(false)
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.java
new file mode 100644
index 0000000000..6dc0df51b3
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.java
@@ -0,0 +1,15 @@
+package io.github.hectorvent.floci.services.cloudfront;
+
+import java.io.IOException;
+
+/**
+ * Signals that a custom origin did not answer before the timeout configured on the origin expired.
+ * CloudFront answers the viewer with a 504 Gateway Timeout for this case, distinct from the 502 it
+ * answers for a genuine bad gateway, so the serving layer must be able to tell the two apart.
+ */
+final class CloudFrontOriginTimeoutException extends IOException {
+
+ CloudFrontOriginTimeoutException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.java
new file mode 100644
index 0000000000..8260e0f41b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.java
@@ -0,0 +1,30 @@
+package io.github.hectorvent.floci.services.cloudfront;
+
+import java.time.Duration;
+
+/**
+ * The origin-timeout settings of a CloudFront distribution, shared by the distribution API (which
+ * validates and emits them) and the serving layer (which applies them to origin requests), so both
+ * sides agree on the values the {@code Origin} and {@code CustomOriginConfig} API references
+ * document. {@code OriginReadTimeout} accepts 1-120 seconds, default 30; {@code ConnectionTimeout}
+ * accepts 1-10 seconds, default 10; {@code ConnectionAttempts} accepts 1-3, default 3.
+ */
+final class CloudFrontOriginTimeouts {
+
+ static final int DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS = 30;
+ static final int MIN_ORIGIN_READ_TIMEOUT_SECONDS = 1;
+ static final int MAX_ORIGIN_READ_TIMEOUT_SECONDS = 120;
+ static final int DEFAULT_CONNECTION_TIMEOUT_SECONDS = 10;
+ static final int MIN_CONNECTION_TIMEOUT_SECONDS = 1;
+ static final int MAX_CONNECTION_TIMEOUT_SECONDS = 10;
+ static final int DEFAULT_CONNECTION_ATTEMPTS = 3;
+ static final int MAX_CONNECTION_ATTEMPTS = 3;
+
+ static final Duration DEFAULT_ORIGIN_READ_TIMEOUT =
+ Duration.ofSeconds(DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS);
+ static final Duration DEFAULT_CONNECTION_TIMEOUT =
+ Duration.ofSeconds(DEFAULT_CONNECTION_TIMEOUT_SECONDS);
+
+ private CloudFrontOriginTimeouts() {
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java
index fa1b44b332..e403736bfd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java
@@ -101,6 +101,8 @@ public class CloudFrontServingController {
java.util.Set.of("Expires", "Signature", "Key-Pair-Id", "Policy", "Hash-Algorithm");
/** Viewer methods that can carry a request body and that CloudFront never caches. */
private static final Set<String> BODY_METHODS = Set.of("POST", "PUT", "PATCH", "DELETE");
+ /** The 504 body CloudFront returns when the origin did not answer in time. */
+ static final String ORIGIN_TIMEOUT_MESSAGE = "The origin didn't respond before the request expired.";
private final CloudFrontService service;
private final S3Service s3Service;
@@ -760,8 +762,7 @@ public class CloudFrontServingController {
URI target = buildCustomOriginUri(
origin, viewerScheme, forwardUri, originRequest.rawQuery());
HttpRequest.Builder rb = HttpRequest.newBuilder()
- .uri(target)
- .timeout(Duration.ofSeconds(30));
+ .uri(target);
// Origin custom headers: CloudFront adds these to every request it forwards to the origin
// (overriding any same-named viewer header), which is how a distribution restricts an origin
// to CloudFront-only traffic via a shared secret header.
@@ -782,7 +783,7 @@ public class CloudFrontServingController {
ViewerBody requestBody = originRequestBody(method, viewerBody);
CloudFrontOriginHttpClient.OpenResponse resp = httpClient.open(rb.build(), originRequest.headers(),
originHeaders, requestBody != null ? requestBody.stream() : null,
- requestBody != null ? requestBody.length() : -1);
+ requestBody != null ? requestBody.length() : -1, originTimeouts(origin));
String ct = resp.headers().firstValue("content-type").orElse(DEFAULT_CONTENT_TYPE);
long contentLength = resp.contentLength();
if (!includeBody) {
@@ -795,6 +796,9 @@ public class CloudFrontServingController {
LOG.warnv("CloudFront custom origin fetch was interrupted for host {0}",
safeOriginName(origin));
return OriginResponse.error(502, "Bad Gateway.");
+ } catch (CloudFrontOriginTimeoutException e) {
+ LOG.warnv("CloudFront custom origin timed out for host {0}", safeOriginName(origin));
+ return OriginResponse.error(504, ORIGIN_TIMEOUT_MESSAGE);
} catch (Exception e) {
// Do not log the target URI: forwarded query strings can contain credentials or tokens.
LOG.warnv("CloudFront custom origin fetch failed for host {0}: {1}",
@@ -803,6 +807,34 @@ public class CloudFrontServingController {
}
}
+ /**
+ * The origin's connection settings for one request: {@code Origin.ConnectionTimeout},
+ * {@code CustomOriginConfig.OriginReadTimeout} and {@code Origin.ConnectionAttempts}. A value
+ * outside the range AWS accepts falls back to the AWS default, since the model can be built
+ * programmatically or loaded from persisted state without the API validation.
+ */
+ private static CloudFrontOriginHttpClient.OriginTimeouts originTimeouts(Origin origin) {
+ int connectSeconds = clampToDefault(origin.getConnectionTimeout(),
+ CloudFrontOriginTimeouts.MIN_CONNECTION_TIMEOUT_SECONDS,
+ CloudFrontOriginTimeouts.MAX_CONNECTION_TIMEOUT_SECONDS,
+ CloudFrontOriginTimeouts.DEFAULT_CONNECTION_TIMEOUT_SECONDS);
+ int readSeconds = clampToDefault(
+ intOrDefault(origin.getCustomOriginConfig(), "OriginReadTimeout",
+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS),
+ CloudFrontOriginTimeouts.MIN_ORIGIN_READ_TIMEOUT_SECONDS,
+ CloudFrontOriginTimeouts.MAX_ORIGIN_READ_TIMEOUT_SECONDS,
+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS);
+ int attempts = clampToDefault(origin.getConnectionAttempts(),
+ 1, CloudFrontOriginTimeouts.MAX_CONNECTION_ATTEMPTS,
+ CloudFrontOriginTimeouts.DEFAULT_CONNECTION_ATTEMPTS);
+ return new CloudFrontOriginHttpClient.OriginTimeouts(
+ Duration.ofSeconds(connectSeconds), Duration.ofSeconds(readSeconds), attempts);
+ }
+
+ private static int clampToDefault(int value, int min, int max, int fallback) {
+ return value < min || value > max ? fallback : value;
+ }
+
/**
* The body CloudFront sends to the origin: POST, PUT and PATCH always carry one (possibly empty, so
* the origin still sees {@code Content-Length: 0}); DELETE carries one only when the viewer sent it.
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java
index 2de8ae7dd9..7aef81e48a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java
@@ -45,6 +45,7 @@ import java.util.Random;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -82,9 +83,14 @@ class CloudFrontCustomOriginServingTest {
final PartitionCleanup cleanup = new PartitionCleanup();
private HttpServer originServer;
+ private CountDownLatch originRelease;
@AfterEach
void stopOrigin() {
+ if (originRelease != null) {
+ originRelease.countDown();
+ originRelease = null;
+ }
if (originServer != null) {
originServer.stop(0);
}
@@ -340,6 +346,56 @@ class CloudFrontCustomOriginServingTest {
assertEquals(5, hits.get());
}
+ @Test
+ void answers504AndStopsWaitingWhenTheOriginExceedsTheReadTimeout() throws Exception {
+ AtomicInteger hits = new AtomicInteger();
+ startHoldingOrigin(hits);
+ Distribution created = cloudFrontService.createDistribution(
+ customOriginDistribution(allMethodsBehavior(), null, 1, 1), Map.of());
+
+ long started = System.nanoTime();
+ given().header("Host", created.getDomainName()).when().get("/page")
+ .then().statusCode(504)
+ .body(equalTo(CloudFrontServingController.ORIGIN_TIMEOUT_MESSAGE));
+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;
+
+ assertEquals(1, hits.get());
+ assertTrue(elapsedMillis < 10000,
+ "expected the 1 s origin read timeout to end the request, took " + elapsedMillis + " ms");
+ }
+
+ @Test
+ void retriesTheReadTimeoutForGetUpToConnectionAttempts() throws Exception {
+ AtomicInteger hits = new AtomicInteger();
+ startHoldingOnlyTheFirstRequest(hits);
+ Distribution created = cloudFrontService.createDistribution(
+ customOriginDistribution(allMethodsBehavior(), null, 1, 2), Map.of());
+
+ given().header("Host", created.getDomainName()).when().get("/page")
+ .then().statusCode(200)
+ .body(equalTo("origin-body"));
+
+ assertEquals(2, hits.get(), "a GET that timed out should be retried once");
+ }
+
+ @Test
+ void doesNotRetryTheReadTimeoutForPost() throws Exception {
+ AtomicInteger hits = new AtomicInteger();
+ startHoldingOrigin(hits);
+ Distribution created = cloudFrontService.createDistribution(
+ customOriginDistribution(allMethodsBehavior(), null, 1, 2), Map.of());
+
+ long started = System.nanoTime();
+ given().header("Host", created.getDomainName()).when().post("/page")
+ .then().statusCode(504)
+ .body(equalTo(CloudFrontServingController.ORIGIN_TIMEOUT_MESSAGE));
+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;
+
+ assertEquals(1, hits.get(), "a POST that hit a read timeout must not be retried");
+ assertTrue(elapsedMillis < 10000,
+ "expected the 1 s origin read timeout to end the request, took " + elapsedMillis + " ms");
+ }
+
@Test
void streamsALargeOriginResponseWithItsLength() throws Exception {
byte[] video = randomBytes(3 * 1024 * 1024 + 1);
@@ -932,6 +988,64 @@ class CloudFrontCustomOriginServingTest {
originServer.start();
}
+ /**
+ * An origin that counts its hits and holds every request open until the test releases it, so an
+ * attempt ends only by the client's own read timeout rather than by racing a fixed origin delay.
+ */
+ private void startHoldingOrigin(AtomicInteger hits) throws IOException {
+ originRelease = new CountDownLatch(1);
+ CountDownLatch release = originRelease;
+ originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ originServer.setExecutor(Executors.newVirtualThreadPerTaskExecutor());
+ originServer.createContext("/", exchange -> {
+ hits.incrementAndGet();
+ holdUntilReleased(release);
+ answerOriginBody(exchange);
+ });
+ originServer.start();
+ }
+
+ /** An origin that holds its first request until the retry arrives, then answers that one. */
+ private void startHoldingOnlyTheFirstRequest(AtomicInteger hits) throws IOException {
+ originRelease = new CountDownLatch(1);
+ CountDownLatch retryArrived = originRelease;
+ originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ originServer.setExecutor(Executors.newVirtualThreadPerTaskExecutor());
+ originServer.createContext("/", exchange -> {
+ if (hits.incrementAndGet() == 1) {
+ holdUntilReleased(retryArrived);
+ } else {
+ retryArrived.countDown();
+ }
+ answerOriginBody(exchange);
+ });
+ originServer.start();
+ }
+
+ /**
+ * Holds the request open until {@code release} opens, with a generous cap so a client that never
+ * times out fails the assertion instead of hanging the suite.
+ */
+ private static void holdUntilReleased(CountDownLatch release) {
+ try {
+ release.await(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private static void answerOriginBody(HttpExchange exchange) {
+ byte[] body = "origin-body".getBytes(StandardCharsets.UTF_8);
+ try {
+ exchange.sendResponseHeaders(200, body.length);
+ exchange.getResponseBody().write(body);
+ } catch (IOException expected) {
+ // The viewer gave up on this attempt after its read timeout, which the test triggers.
+ } finally {
+ exchange.close();
+ }
+ }
+
/** An origin that answers every GET with {@code body}, with its length or chunked. */
private void startServingOrigin(byte[] body, boolean withLength) throws IOException {
originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
@@ -1007,11 +1121,25 @@ class CloudFrontCustomOriginServingTest {
private Distribution customOriginDistribution(DefaultCacheBehavior behavior,
List<Map<String, String>> customHeaders) {
+ return customOriginDistribution(behavior, customHeaders, null, null);
+ }
+
+ private Distribution customOriginDistribution(DefaultCacheBehavior behavior,
+ List<Map<String, String>> customHeaders,
+ Integer readTimeoutSeconds,
+ Integer connectionAttempts) {
+ Map<String, Object> originConfig = customOriginConfig(originServer.getAddress().getPort());
+ if (readTimeoutSeconds != null) {
+ originConfig.put("OriginReadTimeout", readTimeoutSeconds);
+ }
Origin customOrigin = new Origin();
customOrigin.setId("custom-origin");
customOrigin.setDomainName("127.0.0.1");
- customOrigin.setCustomOriginConfig(customOriginConfig(originServer.getAddress().getPort()));
+ customOrigin.setCustomOriginConfig(originConfig);
customOrigin.setCustomHeaders(customHeaders);
+ if (connectionAttempts != null) {
+ customOrigin.setConnectionAttempts(connectionAttempts);
+ }
DistributionConfig config = new DistributionConfig();
config.setEnabled(true);
config.setOrigins(List.of(customOrigin));
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java
index eff729e66b..ec130dd612 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java
@@ -21,8 +21,10 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.net.ConnectException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.ServerSocket;
import java.net.URI;
import java.net.UnknownHostException;
import java.net.http.HttpRequest;
@@ -36,6 +38,7 @@ import java.util.Map;
import java.util.Random;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -361,6 +364,185 @@ class CloudFrontOriginHttpClientTest {
}
}
+ @Test
+ void retriesAnIdempotentRequestUntilTheAttemptsAreExhausted() throws Exception {
+ AtomicInteger hits = new AtomicInteger();
+ CountDownLatch release = new CountDownLatch(1);
+ HttpServer server = holdingServer(hits, release);
+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {
+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =
+ new CloudFrontOriginHttpClient.OriginTimeouts(
+ Duration.ofSeconds(1), Duration.ofMillis(200), 3);
+ HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");
+
+ long started = System.nanoTime();
+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));
+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;
+
+ assertEquals(3, hits.get());
+ assertTrue(elapsedMillis < 5000,
+ "expected the 200 ms read timeout to end every attempt on an origin that never "
+ + "answers, took " + elapsedMillis + " ms");
+ } finally {
+ release.countDown();
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void doesNotRetryAnIdempotentRequestWhenOnlyOneAttemptIsConfigured() throws Exception {
+ AtomicInteger hits = new AtomicInteger();
+ CountDownLatch release = new CountDownLatch(1);
+ HttpServer server = holdingServer(hits, release);
+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {
+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =
+ new CloudFrontOriginHttpClient.OriginTimeouts(
+ Duration.ofSeconds(1), Duration.ofMillis(200), 1);
+ HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");
+
+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));
+
+ assertEquals(1, hits.get());
+ } finally {
+ release.countDown();
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void doesNotRetryAResponseTimeoutForANonIdempotentRequest() throws Exception {
+ AtomicInteger hits = new AtomicInteger();
+ CountDownLatch release = new CountDownLatch(1);
+ HttpServer server = holdingServer(hits, release);
+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {
+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =
+ new CloudFrontOriginHttpClient.OriginTimeouts(
+ Duration.ofSeconds(1), Duration.ofMillis(200), 3);
+ HttpRequest request = HttpRequest.newBuilder(URI.create(
+ "http://origin.invalid:" + server.getAddress().getPort() + "/"))
+ .method("POST", HttpRequest.BodyPublishers.noBody())
+ .build();
+
+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));
+
+ assertEquals(1, hits.get(), "a POST that hit a response timeout must not be retried");
+ } finally {
+ release.countDown();
+ server.stop(0);
+ }
+ }
+
+ @Test
+ void retriesAConnectTimeoutForANonIdempotentRequest() throws Exception {
+ AtomicInteger resolutions = new AtomicInteger();
+ DnsResolver resolver = countingResolver(resolutions, InetAddress.getByName("192.0.2.1"));
+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+ resolver, List.of("blackhole.invalid"))) {
+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =
+ new CloudFrontOriginHttpClient.OriginTimeouts(
+ Duration.ofMillis(100), Duration.ofSeconds(5), 3);
+ HttpRequest request = HttpRequest.newBuilder(URI.create("http://blackhole.invalid:81/"))
+ .method("POST", HttpRequest.BodyPublishers.noBody())
+ .build();
+
+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));
+
+ assertEquals(3, resolutions.get(),
+ "a POST whose connect times out counts as a connection attempt and is retried");
+ }
+ }
+
+ @Test
+ void retriesARefusedConnectionForANonIdempotentRequest() throws Exception {
+ int refusedPort;
+ try (ServerSocket socket = new ServerSocket(0)) {
+ refusedPort = socket.getLocalPort();
+ }
+ AtomicInteger resolutions = new AtomicInteger();
+ DnsResolver resolver = countingResolver(resolutions, InetAddress.getByName("127.0.0.1"));
+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+ resolver, List.of("refused.invalid"))) {
+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =
+ new CloudFrontOriginHttpClient.OriginTimeouts(
+ Duration.ofSeconds(1), Duration.ofSeconds(5), 3);
+ HttpRequest request = HttpRequest.newBuilder(URI.create(
+ "http://refused.invalid:" + refusedPort + "/"))
+ .method("POST", HttpRequest.BodyPublishers.noBody())
+ .build();
+
+ assertThrows(ConnectException.class, () -> get(client, request, timeouts));
+
+ assertEquals(3, resolutions.get(),
+ "a POST whose connection is refused counts as a connection attempt and is retried");
+ }
+ }
+
+ @Test
+ void honorsTheConfiguredReadTimeout() throws Exception {
+ AtomicInteger hits = new AtomicInteger();
+ CountDownLatch release = new CountDownLatch(1);
+ HttpServer server = holdingServer(hits, release);
+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(
+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {
+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =
+ new CloudFrontOriginHttpClient.OriginTimeouts(
+ Duration.ofSeconds(1), Duration.ofMillis(150), 1);
+ HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");
+
+ long started = System.nanoTime();
+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));
+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;
+
+ assertTrue(elapsedMillis < 5000,
+ "expected the configured 150 ms read timeout to end the request, not the 30 s "
+ + "default, took " + elapsedMillis + " ms");
+ } finally {
+ release.countDown();
+ server.stop(0);
+ }
+ }
+
+ private static CloudFrontOriginHttpClient.OpenResponse get(
+ CloudFrontOriginHttpClient client, HttpRequest request,
+ CloudFrontOriginHttpClient.OriginTimeouts timeouts) throws Exception {
+ return client.open(request, List.of(), Map.of(), null, -1, timeouts);
+ }
+
+ /**
+ * An origin that counts its hits and holds every request open until {@code release} opens, so a
+ * client that answers nothing never races a fixed origin delay: only the client's own timeout
+ * can end an attempt.
+ */
+ private static HttpServer holdingServer(AtomicInteger hits, CountDownLatch release) throws IOException {
+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ server.setExecutor(Executors.newVirtualThreadPerTaskExecutor());
+ server.createContext("/", exchange -> {
+ hits.incrementAndGet();
+ try {
+ exchange.getRequestBody().readAllBytes();
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } catch (IOException expected) {
+ // The client gave up on the attempt, which is what the read timeout under test causes.
+ }
+ try {
+ byte[] body = "origin-ok".getBytes(StandardCharsets.UTF_8);
+ exchange.sendResponseHeaders(200, body.length);
+ exchange.getResponseBody().write(body);
+ } catch (IOException expected) {
+ // The client is gone, so there is nobody left to answer.
+ } finally {
+ exchange.close();
+ }
+ });
+ server.start();
+ return server;
+ }
+
private static CloudFrontOriginHttpClient.OpenResponse get(CloudFrontOriginHttpClient client,
HttpRequest request) throws Exception {
return client.open(request, List.of(), Map.of(), null, -1);
@@ -404,6 +586,22 @@ class CloudFrontOriginHttpClientTest {
};
}
+ /** A resolver that answers every lookup with {@code address}, counting the lookups. */
+ private static DnsResolver countingResolver(AtomicInteger resolutions, InetAddress address) {
+ return new DnsResolver() {
+ @Override
+ public InetAddress[] resolve(String host) {
+ resolutions.incrementAndGet();
+ return new InetAddress[] { address };
+ }
+
+ @Override
+ public String resolveCanonicalHostname(String host) {
+ return host;
+ }
+ };
+ }
+
private static SSLContext serverSslContext(
X509Certificate certificate, java.security.PrivateKey privateKey) throws Exception {
char[] password = "changeit".toCharArray();