Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudfront): honor the origin completion and keep-alive timeouts (#6198)
Matheus Oliveira8d25cd91a618
diff --git a/docs/services/cloudfront.md b/docs/services/cloudfront.mdindex 898c205cdd..f83738ac05 100644--- a/docs/services/cloudfront.md+++ b/docs/services/cloudfront.md@@ -6,10 +6,10 @@ access identities, public keys, trusted key groups, CloudFront Functions, invaliviewer delivery from S3 or custom origins for every CloudFront viewer method (GET, HEAD, OPTIONS, POST,PUT, PATCH and DELETE).-**Protocol:** REST XML-**API version:** `2020-05-31`-**Endpoint prefix:** `cloudfront`-**Namespace:** `http://cloudfront.amazonaws.com/doc/2020-05-31/`+**Protocol:** REST XML+**API version:** `2020-05-31`+**Endpoint prefix:** `cloudfront`+**Namespace:** `http://cloudfront.amazonaws.com/doc/2020-05-31/`**Global service** — ARNs contain no region segment.## Supported Operations@@ -246,6 +246,19 @@ PUT, PATCH and DELETE).whole. A custom error page is served whole, whatever the `Range` of the request it replaces.- Origin `Set-Cookie` headers always reach the viewer. With legacy `Forward=none`, AWS strips themfrom the response; Floci does not.+- Custom-origin requests honor the origin's connection settings. `Origin.ConnectionAttempts` (1 to 3,+ default 3) is how many times CloudFront tries to reach the origin, and `Origin.ConnectionTimeout`+ (1 to 10 seconds, default 10) how long it waits for the connection. `CustomOriginConfig.OriginReadTimeout`+ (1 to 120 seconds, default 30) bounds the wait for the origin's response, and a read timeout on a+ GET or HEAD is retried up to the connection attempts, as CloudFront does, while a POST, PUT, PATCH+ or DELETE is not retried. `Origin.ResponseCompletionTimeout` bounds the time the origin takes,+ connecting, answering and streaming the body included, over every attempt, and ends the connection+ when it expires; unset, or 0, leaves the origin as long as it wants to take, and a set value must+ be at least the read timeout. The time a slow viewer takes to read what the origin already sent is+ not part of it: the origin answered in time, so the response is not expired.+ `CustomOriginConfig.OriginKeepaliveTimeout` (1 to 300 seconds, default 5) is how long CloudFront+ keeps a connection to the origin after the last packet of a response. An origin that does not+ answer in time is answered 504, "The origin didn't respond before the request expired", as on AWS.- Custom origins that resolve to loopback, private, link-local, carrier-grade NAT, or other non-routable addresses are rejected by default. Development-only private origins must be explicitly allowlisted by exact hostname.- A custom origin whose domain is a [Lambda Web Function](lambda-web-functions.md) endpoint of this Floci isserved by Floci itself, at the address the viewer reached. The origin names the endpoint'sdiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CloudFrontCfnProvisioner.java b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CloudFrontCfnProvisioner.javaindex 1535d386b7..87914f0309 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CloudFrontCfnProvisioner.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CloudFrontCfnProvisioner.java@@ -565,6 +565,10 @@ public class CloudFrontCfnProvisioner implements CfnResourceProvisioner {if (connectionTimeout != null) {origin.setConnectionTimeout(connectionTimeout);}+ Integer responseCompletionTimeout = cfnInt(node, "ResponseCompletionTimeout", engine);+ if (responseCompletionTimeout != null) {+ origin.setResponseCompletionTimeout(responseCompletionTimeout);+ }JsonNode originCustomHeaders = node.path("OriginCustomHeaders");if (originCustomHeaders.isArray()) {List<Map<String, String>> customHeaders = new ArrayList<>();@@ -588,6 +592,10 @@ public class CloudFrontCfnProvisioner implements CfnResourceProvisioner {if (originReadTimeout != null) {coc.put("OriginReadTimeout", originReadTimeout);}+ Integer originKeepaliveTimeout = cfnInt(custom, "OriginKeepaliveTimeout", engine);+ if (originKeepaliveTimeout != null) {+ coc.put("OriginKeepaliveTimeout", originKeepaliveTimeout);+ }origin.setCustomOriginConfig(coc);} else {// No CustomOriginConfig => S3 origin (S3OriginConfig may be present or defaulted).diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.javaindex 202c522fcc..de0852cb90 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontController.java@@ -2942,18 +2942,6 @@ public class CloudFrontController {}case "Origin" -> {if (inOrigin && current != null) {- Integer completionTimeout = current.getResponseCompletionTimeout();- int effectiveReadTimeout = originReadTimeoutSeconds != null- ? originReadTimeoutSeconds- : CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS;- if (completionTimeout != null && completionTimeout > 0- && completionTimeout < effectiveReadTimeout) {- throw new AwsException(- "InvalidArgument",- "The parameter ResponseCompletionTimeout must be greater than or "- + "equal to OriginReadTimeout.",- 400);- }result.add(current);}inOrigin = false;diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.javaindex eb1b95f6da..46834c2a7b 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClient.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.cloudfront;import io.github.hectorvent.floci.core.common.SsrfProtection;import org.apache.hc.client5.http.ConnectTimeoutException;+import org.apache.hc.client5.http.ConnectionKeepAliveStrategy;import org.apache.hc.client5.http.DnsResolver;import org.apache.hc.client5.http.SystemDefaultDnsResolver;import org.apache.hc.client5.http.classic.methods.HttpUriRequestBase;@@ -19,11 +20,16 @@ import org.apache.hc.client5.http.routing.RoutingSupport;import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder;import org.apache.hc.core5.http.ClassicHttpResponse;import org.apache.hc.core5.http.Header;+import org.apache.hc.core5.http.HeaderElement;import org.apache.hc.core5.http.HttpEntity;import org.apache.hc.core5.http.HttpException;+import org.apache.hc.core5.http.HttpResponse;import org.apache.hc.core5.http.HttpVersion;import org.apache.hc.core5.http.io.entity.InputStreamEntity;+import org.apache.hc.core5.http.message.MessageSupport;+import org.apache.hc.core5.http.protocol.HttpContext;import org.apache.hc.core5.http2.HttpVersionPolicy;+import org.apache.hc.core5.util.TimeValue;import org.apache.hc.core5.util.Timeout;import org.jboss.logging.Logger;@@ -41,12 +47,14 @@ import java.net.http.HttpRequest;import java.time.Duration;import java.util.ArrayList;import java.util.Collection;+import java.util.Iterator;import java.util.List;import java.util.Locale;import java.util.Map;import java.util.Set;import java.util.TreeMap;import java.util.TreeSet;+import java.util.function.LongSupplier;import java.util.stream.Collectors;/**@@ -60,37 +68,68 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {private static final String ORIGIN_TIMEOUT_MESSAGE ="The origin didn't respond before the request expired.";+ /** The context attribute the origin's keep-alive timeout travels in, to the pool's strategy. */+ static final String KEEP_ALIVE_TIMEOUT_ATTRIBUTE = "floci.cloudfront.originKeepAliveTimeout";private static final Duration CONNECT_TIMEOUT = CloudFrontOriginTimeouts.DEFAULT_CONNECTION_TIMEOUT;private static final Duration RESPONSE_TIMEOUT = CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT;/** CloudFront retries a response timeout only for the methods it treats as idempotent. */private static final Set<String> IDEMPOTENT_METHODS = Set.of("GET", "HEAD");static final int MAX_CONNECTIONS = 100;+ /**+ * The connect timeout of the request being sent, for the TLS handshake its connection makes.+ * The pool resolves the TLS configuration of a connection while the request that opens it runs,+ * on the same thread, so the handshake answers within the connect timeout of that request.+ */+ private static final ThreadLocal<Duration> HANDSHAKE_TIMEOUT = new ThreadLocal<>();+private final CloseableHttpClient client;+ /** The clock the completion budget reads, {@link System#nanoTime} unless a test supplies one. */+ private final LongSupplier nanoClock;CloudFrontOriginHttpClient(Collection<String> allowedPrivateOriginHosts) {- this(SystemDefaultDnsResolver.INSTANCE, allowedPrivateOriginHosts, null);+ this(SystemDefaultDnsResolver.INSTANCE, allowedPrivateOriginHosts, (SSLContext) null);}CloudFrontOriginHttpClient(DnsResolver delegate, Collection<String> allowedPrivateOriginHosts) {- this(delegate, allowedPrivateOriginHosts, null);+ this(delegate, allowedPrivateOriginHosts, (SSLContext) null);}/** Reaches Floci's own listener, whose address the private address check refuses. */static CloudFrontOriginHttpClient toFloci() {- return new CloudFrontOriginHttpClient(SystemDefaultDnsResolver.INSTANCE, (SSLContext) null);+ return new CloudFrontOriginHttpClient(SystemDefaultDnsResolver.INSTANCE, (SSLContext) null,+ System::nanoTime);}CloudFrontOriginHttpClient(DnsResolver delegate,Collection<String> allowedPrivateOriginHosts,SSLContext sslContext) {+ this(delegate, allowedPrivateOriginHosts, sslContext, System::nanoTime);+ }++ /** The same client, reading the clock from {@code nanoClock} rather than {@link System#nanoTime}. */+ CloudFrontOriginHttpClient(+ DnsResolver delegate,+ Collection<String> allowedPrivateOriginHosts,+ LongSupplier nanoClock) {+ this(delegate, allowedPrivateOriginHosts, null, nanoClock);+ }++ private CloudFrontOriginHttpClient(+ DnsResolver delegate,+ Collection<String> allowedPrivateOriginHosts,+ SSLContext sslContext,+ LongSupplier nanoClock) {this(new ValidatingDnsResolver(delegate, allowedPrivateOriginHosts.stream().map(CloudFrontServingController::normalizeHost)- .collect(Collectors.toUnmodifiableSet())), sslContext);+ .collect(Collectors.toUnmodifiableSet())), sslContext, nanoClock);}- private CloudFrontOriginHttpClient(DnsResolver resolver, SSLContext sslContext) {+ private CloudFrontOriginHttpClient(+ DnsResolver resolver, SSLContext sslContext, LongSupplier nanoClock) {+ this.nanoClock = nanoClock;+ConnectionConfig connectionConfig = ConnectionConfig.custom().setConnectTimeout(Timeout.of(CONNECT_TIMEOUT)).setSocketTimeout(Timeout.of(RESPONSE_TIMEOUT))@@ -104,6 +143,9 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {.setDnsResolver(resolver).setDefaultConnectionConfig(connectionConfig).setDefaultTlsConfig(tlsConfig)+ // The handshake of a connection answers within the connect timeout of the+ // request that opens it, so it cannot outlive the completion timeout.+ .setTlsConfigResolver(host -> tlsConfig(handshakeTimeout()))// CloudFront has no per-origin connection limit, and a streamed response holds its// connection for the whole transfer, so one origin may use the whole pool..setMaxConnTotal(MAX_CONNECTIONS)@@ -125,6 +167,12 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {this.client = HttpClients.custom().setConnectionManager(connectionManager).setDefaultRequestConfig(requestConfig)+ .setKeepAliveStrategy(new KeepAliveStrategy())+ // A connection leaves the pool when its own keep-alive expires, so the evictor only+ // has to clear the connections no origin will ever reuse again: the longest+ // keep-alive an origin can ask for.+ .evictIdleConnections(TimeValue.ofSeconds(+ CloudFrontOriginTimeouts.MAX_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS)).disableAutomaticRetries().disableRedirectHandling().disableContentCompression()@@ -135,10 +183,12 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {/*** The connection settings of a distribution's origin that one request must honor:- * {@code Origin.ConnectionAttempts}, {@code Origin.ConnectionTimeout} and the custom origin's- * {@code OriginReadTimeout}. Read per request, since the client and its pool are shared.+ * {@code Origin.ConnectionAttempts}, {@code Origin.ConnectionTimeout}, the custom origin's+ * {@code OriginReadTimeout}, {@code OriginKeepaliveTimeout} and {@code ResponseCompletionTimeout}.+ * Read per request, since the client and its pool are shared.*/- record OriginTimeouts(Duration connectTimeout, Duration responseTimeout, int connectionAttempts) {+ record OriginTimeouts(Duration connectTimeout, Duration responseTimeout, int connectionAttempts,+ Duration responseCompletionTimeout, Duration keepAliveTimeout) {}/**@@ -171,7 +221,10 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {* idempotent method (GET, HEAD), whose request has no body to replay; every other method is* attempted once for it, as CloudFront does not retry it. When the attempts run out, a timeout* raises {@link CloudFrontOriginTimeoutException} so the viewer is answered 504 rather than- * 502, while a refused connection propagates unchanged, as a bad gateway.+ * 502, while a refused connection propagates unchanged, as a bad gateway. The completion+ * timeout bounds the time the origin takes, connecting, answering and streaming the body+ * included, over every attempt, and is never retried. The time a slow viewer takes to receive+ * the response is not part of it: the origin answered in time, so the request did not expire.*/OpenResponse open(HttpRequest request,List<CloudFrontOriginRequestBuilder.Header> forwardedHeaders,@@ -180,22 +233,40 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {throws IOException, InterruptedException {OriginTimeouts effective = timeouts != null? timeouts- : new OriginTimeouts(CONNECT_TIMEOUT, request.timeout().orElse(RESPONSE_TIMEOUT), 1);+ : new OriginTimeouts(CONNECT_TIMEOUT, request.timeout().orElse(RESPONSE_TIMEOUT), 1,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);int attempts = Math.max(1, effective.connectionAttempts());+ CompletionBudget budget = new CompletionBudget(effective.responseCompletionTimeout(), nanoClock);IOException lastFailure = null;for (int attempt = 1; attempt <= attempts; attempt++) {+ if (budget.expired()) {+ // The origin already took the time the completion timeout allows: another attempt+ // would only push the viewer further past it.+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE);+ }+ boolean responseTimeout = false;try {- return send(request, forwardedHeaders, originHeaders, requestBody, requestBodyLength, effective);+ return send(request, forwardedHeaders, originHeaders, requestBody, requestBodyLength,+ effective, budget);} catch (ConnectTimeoutException e) {+ // A connection attempt counts for every method, so a connect timeout is retried.lastFailure = e;} catch (SocketTimeoutException e) {- if (!idempotent(request.method())) {- throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, e);- }lastFailure = e;+ responseTimeout = true;} catch (ConnectException e) {lastFailure = e;}+ if (budget.expired()) {+ // The origin took the time the completion timeout allows, so the request is+ // expired rather than merely slow.+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, lastFailure);+ }+ if (responseTimeout && !idempotent(request.method())) {+ // A response timeout on a method CloudFront does not retry ends the request.+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, lastFailure);+ }if (attempt < attempts) {LOG.debugv("CloudFront origin connection attempt {0} of {1} failed; retrying",attempt, attempts);@@ -207,18 +278,69 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {throw lastFailure;}+ /**+ * The time CloudFront waits for the origin, bounded by the completion timeout. The budget+ * counts the time the origin itself takes: connecting, sending the response, and streaming the+ * body. The time a slow viewer takes to receive what the origin sent is not part of it, so a+ * viewer that reads slowly never expires a response the origin finished on time. A budget with+ * no limit is the origin's own pace.+ */+ private static final class CompletionBudget {++ /** The time the origin may take, in nanoseconds, or 0 when it may take as long as it wants. */+ private final long limitNanos;+ /** The clock the budget reads, so a test can move time without waiting for it. */+ private final LongSupplier nanoClock;+ /** The time the origin has taken so far, in nanoseconds. */+ private long spentNanos;++ CompletionBudget(Duration responseCompletionTimeout, LongSupplier nanoClock) {+ this.nanoClock = nanoClock;+ this.limitNanos = responseCompletionTimeout == null || responseCompletionTimeout.isZero()+ || responseCompletionTimeout.isNegative() ? 0 : responseCompletionTimeout.toNanos();+ }++ /** The clock's reading, in nanoseconds. */+ long now() {+ return nanoClock.getAsLong();+ }++ /** Whether the origin may take as long as it wants. */+ boolean unlimited() {+ return limitNanos == 0;+ }++ /** Adds the time the origin just took. */+ void spend(long nanos) {+ spentNanos += nanos;+ }++ /** Whether the origin has taken the whole time the completion timeout allows. */+ boolean expired() {+ return limitNanos > 0 && spentNanos >= limitNanos;+ }++ /** How much of the budget is left, in nanoseconds. */+ long remainingNanos() {+ return limitNanos > 0 ? Math.max(0L, limitNanos - spentNanos) : 0L;+ }+ }+private OpenResponse send(HttpRequest request,List<CloudFrontOriginRequestBuilder.Header> forwardedHeaders,Map<String, String> originHeaders, InputStream requestBody, long requestBodyLength,- OriginTimeouts timeouts)+ OriginTimeouts timeouts, CompletionBudget budget)throws IOException, InterruptedException {// No entity content type: the caller's Content-Type request header is sent as is.HttpUriRequestBase originRequest = originRequest(request, forwardedHeaders, originHeaders,requestBody != null ? new InputStreamEntity(requestBody, requestBodyLength, null) : null);ClassicHttpResponse response;+ OriginTimeouts effective = effectiveTimeouts(timeouts, budget);+ long attemptStart = budget.now();+ HANDSHAKE_TIMEOUT.set(effective.connectTimeout());try {response = client.executeOpen(RoutingSupport.determineHost(originRequest), originRequest,- context(timeouts));+ context(effective));} catch (HttpException e) {throw new IOException("CloudFront origin request has no target host", e);} catch (InterruptedIOException e) {@@ -226,6 +348,11 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {throw new InterruptedException("CloudFront origin request interrupted");}throw e;+ } finally {+ HANDSHAKE_TIMEOUT.remove();+ // The time the origin took to connect and to answer is part of the time CloudFront+ // waits for it.+ budget.spend(budget.now() - attemptStart);}try {HttpEntity entity = response.getEntity();@@ -235,6 +362,14 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {// The entity's length follows the framing the body arrives with; with no entity, as for a// HEAD, the header describes the body a GET would get.long contentLength = entity != null ? entity.getContentLength() : headerContentLength(response);+ if (budget.expired()) {+ // The origin answered, but too late: the request has already expired.+ body.close();+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE);+ }+ if (!budget.unlimited()) {+ body = new CompletionTimeoutInputStream(body, budget);+ }return new OpenResponse(response.getCode(), responseHeaders(response), contentLength, body);} catch (IOException | RuntimeException e) {originRequest.cancel();@@ -247,6 +382,128 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {return method != null && IDEMPOTENT_METHODS.contains(method.toUpperCase(Locale.ROOT));}+ /**+ * How long CloudFront keeps a pooled origin connection, from the origin's+ * {@code OriginKeepaliveTimeout}. The duration the origin asks for in its own {@code Keep-Alive}+ * header only shortens it: the setting is the longest CloudFront holds a connection to that+ * origin, and a connection the origin is about to close fails the next request.+ */+ static final class KeepAliveStrategy implements ConnectionKeepAliveStrategy {+ @Override+ public TimeValue getKeepAliveDuration(HttpResponse response, HttpContext context) {+ TimeValue keepAlive = TimeValue.ofSeconds(keepAliveSeconds(context));+ TimeValue fromOrigin = originKeepAliveHeader(response);+ if (fromOrigin != null && fromOrigin.getDuration() > 0 && fromOrigin.compareTo(keepAlive) < 0) {+ return fromOrigin;+ }+ return keepAlive;+ }+ }++ /** The keep-alive the request to the origin asked for, in seconds. */+ private static int keepAliveSeconds(HttpContext context) {+ Object attribute = context != null ? context.getAttribute(KEEP_ALIVE_TIMEOUT_ATTRIBUTE) : null;+ if (attribute instanceof Duration duration && !duration.isNegative()) {+ return (int) Math.min(duration.toSeconds(), Integer.MAX_VALUE);+ }+ return CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS;+ }++ /** The keep-alive the origin asks for in its {@code Keep-Alive} header, or null when it asks for none. */+ private static TimeValue originKeepAliveHeader(HttpResponse response) {+ Iterator<HeaderElement> elements = MessageSupport.iterate(response, "keep-alive");+ while (elements.hasNext()) {+ HeaderElement element = elements.next();+ String name = element.getName();+ String value = element.getValue();+ if (value != null && name != null && name.equalsIgnoreCase("timeout")) {+ try {+ return TimeValue.ofSeconds(Long.parseLong(value.trim()));+ } catch (NumberFormatException e) {+ LOG.debugv("Ignoring an invalid CloudFront origin Keep-Alive timeout: {0}", value);+ return null;+ }+ }+ }+ return null;+ }++ /**+ * An origin response body that gives up when the origin's completion timeout expires, so a+ * response that trickles in forever does not hold the viewer request open. The origin+ * connection goes with it: CloudFront ends the connection when the origin does not complete+ * the response in time. Only the time the origin takes counts against the timeout, so a viewer+ * that reads slowly never loses a response the origin finished on time.+ */+ private static final class CompletionTimeoutInputStream extends FilterInputStream {++ private final CompletionBudget budget;+ private boolean closed;++ CompletionTimeoutInputStream(InputStream delegate, CompletionBudget budget) {+ super(delegate);+ this.budget = budget;+ }++ private void checkExpired() throws IOException {+ if (budget.expired()) {+ close();+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE);+ }+ }++ @Override+ public int read() throws IOException {+ checkExpired();+ long start = budget.now();+ try {+ int value = super.read();+ budget.spend(budget.now() - start);+ if (value >= 0) {+ checkExpired();+ }+ return value;+ } catch (SocketTimeoutException e) {+ budget.spend(budget.now() - start);+ if (budget.expired()) {+ close();+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, e);+ }+ throw e;+ }+ }++ @Override+ public int read(byte[] buffer, int offset, int length) throws IOException {+ checkExpired();+ long start = budget.now();+ try {+ int read = super.read(buffer, offset, length);+ budget.spend(budget.now() - start);+ if (read >= 0) {+ checkExpired();+ }+ return read;+ } catch (SocketTimeoutException e) {+ budget.spend(budget.now() - start);+ if (budget.expired()) {+ close();+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE, e);+ }+ throw e;+ }+ }++ @Override+ public void close() throws IOException {+ if (closed) {+ return;+ }+ closed = true;+ super.close();+ }+ }+/*** An origin response body that releases its connection when it is closed. Closed at its end, the* connection goes back to the pool. Closed before its end, as when a viewer goes away or a custom@@ -349,7 +606,45 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {return builder;}- private static HttpClientContext context(OriginTimeouts timeouts) {+ /**+ * The TLS configuration of a connection: the handshake answers within {@code handshakeTimeout},+ * so it cannot outlive the request that opens the connection.+ */+ private static TlsConfig tlsConfig(Duration handshakeTimeout) {+ return TlsConfig.custom()+ .setHandshakeTimeout(Timeout.of(handshakeTimeout))+ .setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_1)+ .build();+ }++ /** The handshake timeout of the request being sent, or the default connect timeout. */+ private static Duration handshakeTimeout() {+ Duration timeout = HANDSHAKE_TIMEOUT.get();+ return timeout != null ? timeout : CONNECT_TIMEOUT;+ }++ /**+ * The timeouts of an attempt: neither the connection nor the response outlives the completion+ * timeout, so the origin runs out of time before the request does.+ */+ private static OriginTimeouts effectiveTimeouts(OriginTimeouts timeouts, CompletionBudget budget)+ throws CloudFrontOriginTimeoutException {+ if (budget.unlimited()) {+ return timeouts;+ }+ if (budget.expired()) {+ throw new CloudFrontOriginTimeoutException(ORIGIN_TIMEOUT_MESSAGE);+ }+ long remainingMillis = Math.max(1L, budget.remainingNanos() / 1_000_000L);+ return new OriginTimeouts(+ Duration.ofMillis(Math.min(timeouts.connectTimeout().toMillis(), remainingMillis)),+ Duration.ofMillis(Math.min(timeouts.responseTimeout().toMillis(), remainingMillis)),+ timeouts.connectionAttempts(),+ timeouts.responseCompletionTimeout(),+ timeouts.keepAliveTimeout());+ }++ static HttpClientContext context(OriginTimeouts timeouts) {HttpClientContext context = HttpClientContext.create();context.setRequestConfig(RequestConfig.custom().setConnectTimeout(Timeout.of(timeouts.connectTimeout()))@@ -361,6 +656,7 @@ final class CloudFrontOriginHttpClient implements AutoCloseable {.setContentCompressionEnabled(false).setProtocolUpgradeEnabled(false).build());+ context.setAttribute(KEEP_ALIVE_TIMEOUT_ATTRIBUTE, timeouts.keepAliveTimeout());return context;}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.javaindex 6dc0df51b3..5dbc580039 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeoutException.java@@ -9,6 +9,10 @@ import java.io.IOException;*/final class CloudFrontOriginTimeoutException extends IOException {+ CloudFrontOriginTimeoutException(String message) {+ super(message);+ }+CloudFrontOriginTimeoutException(String message, Throwable cause) {super(message, cause);}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.javaindex c19e1d8de1..ac14a84ebd 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginTimeouts.java@@ -9,6 +9,8 @@ import java.time.Duration;* API references document. {@code OriginReadTimeout} accepts 1-120 seconds, default 30;* {@code OriginKeepaliveTimeout} accepts 1-300 seconds, default 5; {@code ConnectionTimeout}* accepts 1-10 seconds, default 10; {@code ConnectionAttempts} accepts 1-3, default 3.+ * {@code ResponseCompletionTimeout} has no maximum: unset, or 0, leaves the origin as long as it+ * wants to take, and a set value bounds the whole request to the origin.*/final class CloudFrontOriginTimeouts {@@ -29,6 +31,10 @@ final class CloudFrontOriginTimeouts {Duration.ofSeconds(DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS);static final Duration DEFAULT_CONNECTION_TIMEOUT =Duration.ofSeconds(DEFAULT_CONNECTION_TIMEOUT_SECONDS);+ /** No completion timeout: the origin may take as long as it wants to answer. */+ static final Duration DEFAULT_RESPONSE_COMPLETION_TIMEOUT = Duration.ZERO;+ static final Duration DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT =+ Duration.ofSeconds(DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS);private CloudFrontOriginTimeouts() {}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.javaindex d8c9d4e6a3..e9db02dc64 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontService.java@@ -315,17 +315,45 @@ public class CloudFrontService implements ServerCertificateReferenceProvider {CloudFrontOriginTimeouts.MIN_CONNECTION_TIMEOUT_SECONDS,CloudFrontOriginTimeouts.MAX_CONNECTION_TIMEOUT_SECONDS, " seconds");Map<String, Object> customOriginConfig = origin.getCustomOriginConfig();- if (customOriginConfig == null) {- continue;+ int originReadTimeout = CloudFrontOriginTimeouts.DEFAULT_ORIGIN_READ_TIMEOUT_SECONDS;+ if (customOriginConfig != null) {+ requireRange("InvalidOriginReadTimeout", "OriginReadTimeout",+ asIntOrNull(customOriginConfig.get("OriginReadTimeout")),+ CloudFrontOriginTimeouts.MIN_ORIGIN_READ_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.MAX_ORIGIN_READ_TIMEOUT_SECONDS, " seconds");+ requireRange("InvalidOriginKeepaliveTimeout", "OriginKeepaliveTimeout",+ asIntOrNull(customOriginConfig.get("OriginKeepaliveTimeout")),+ CloudFrontOriginTimeouts.MIN_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.MAX_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS, " seconds");+ Integer readTimeout = asIntOrNull(customOriginConfig.get("OriginReadTimeout"));+ if (readTimeout != null) {+ originReadTimeout = readTimeout;+ }}- requireRange("InvalidOriginReadTimeout", "OriginReadTimeout",- asIntOrNull(customOriginConfig.get("OriginReadTimeout")),- CloudFrontOriginTimeouts.MIN_ORIGIN_READ_TIMEOUT_SECONDS,- CloudFrontOriginTimeouts.MAX_ORIGIN_READ_TIMEOUT_SECONDS, " seconds");- requireRange("InvalidOriginKeepaliveTimeout", "OriginKeepaliveTimeout",- asIntOrNull(customOriginConfig.get("OriginKeepaliveTimeout")),- CloudFrontOriginTimeouts.MIN_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS,- CloudFrontOriginTimeouts.MAX_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS, " seconds");+ validateResponseCompletionTimeout(origin.getResponseCompletionTimeout(), originReadTimeout);+ }+ }++ /**+ * The completion timeout bounds the whole request to the origin, so it cannot be shorter than+ * the read timeout it contains. Create and update both run this, whichever path the request+ * came from, so a distribution created through the XML API and one created through+ * CloudFormation are held to the same rule.+ */+ private static void validateResponseCompletionTimeout(Integer completionTimeout, int originReadTimeout) {+ if (completionTimeout == null) {+ return;+ }+ if (completionTimeout < 0) {+ throw new AwsException("InvalidArgument",+ "The parameter ResponseCompletionTimeout must be a non-negative integer.", 400);+ }+ if (completionTimeout > 0 && completionTimeout < originReadTimeout) {+ throw new AwsException(+ "InvalidArgument",+ "The parameter ResponseCompletionTimeout must be greater than or "+ + "equal to OriginReadTimeout.",+ 400);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.javaindex 86fd64c64d..7353e634a7 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServingController.java@@ -41,6 +41,7 @@ import org.jboss.resteasy.reactive.server.core.CurrentRequestManager;import java.io.IOException;import java.io.InputStream;+import java.io.OutputStream;import java.io.PushbackInputStream;import java.net.IDN;import java.net.URI;@@ -863,7 +864,8 @@ public class CloudFrontServingController {/*** The origin's connection settings for one request: {@code Origin.ConnectionTimeout},- * {@code CustomOriginConfig.OriginReadTimeout} and {@code Origin.ConnectionAttempts}. A value+ * {@code CustomOriginConfig.OriginReadTimeout}, {@code CustomOriginConfig.OriginKeepaliveTimeout},+ * {@code Origin.ConnectionAttempts} and {@code Origin.ResponseCompletionTimeout}. A value* outside the range AWS accepts falls back to the AWS default, since the model can be built* programmatically or loaded from persisted state without the API validation.*/@@ -882,8 +884,19 @@ public class CloudFrontServingController {CloudFrontOriginTimeouts.MIN_CONNECTION_ATTEMPTS,CloudFrontOriginTimeouts.MAX_CONNECTION_ATTEMPTS,CloudFrontOriginTimeouts.DEFAULT_CONNECTION_ATTEMPTS);+ int keepAliveSeconds = clampToDefault(+ intOrDefault(origin.getCustomOriginConfig(), "OriginKeepaliveTimeout",+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS),+ CloudFrontOriginTimeouts.MIN_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.MAX_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS);+ Integer completionSeconds = origin.getResponseCompletionTimeout();+ Duration completionTimeout = (completionSeconds == null || completionSeconds <= 0)+ ? CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT+ : Duration.ofSeconds(completionSeconds);return new CloudFrontOriginHttpClient.OriginTimeouts(- Duration.ofSeconds(connectSeconds), Duration.ofSeconds(readSeconds), attempts);+ Duration.ofSeconds(connectSeconds), Duration.ofSeconds(readSeconds), attempts,+ completionTimeout, Duration.ofSeconds(keepAliveSeconds));}private static int clampToDefault(int value, int min, int max, int fallback) {@@ -1116,7 +1129,7 @@ public class CloudFrontServingController {InputStream body = origin.stream();StreamingOutput entity = output -> {try (InputStream in = body) {- in.transferTo(output);+ streamToViewer(in, output);}};rb.entity(entity);@@ -1136,6 +1149,21 @@ public class CloudFrontServingController {}}+ /**+ * Copies the origin's body to the viewer, flushing every chunk, so the viewer's response+ * starts before the body is over, as CloudFront's does. A response the origin never completes+ * ends with the bytes that did arrive and a closed connection, the way CloudFront ends it,+ * instead of an error page that hides what the viewer already received.+ */+ private static void streamToViewer(InputStream in, OutputStream out) throws IOException {+ byte[] buffer = new byte[8192];+ int read;+ while ((read = in.read(buffer)) >= 0) {+ out.write(buffer, 0, read);+ out.flush();+ }+ }+/*** Collects the origin's end-to-end response headers case-insensitively while preserving every* value (notably multiple {@code Set-Cookie} fields). Hop-by-hop fields and every field named bydiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CloudFrontCfnProvisionerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CloudFrontCfnProvisionerTest.javaindex 853a39c00b..9ad10647b7 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CloudFrontCfnProvisionerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/provisioners/CloudFrontCfnProvisionerTest.java@@ -184,8 +184,10 @@ class CloudFrontCfnProvisionerTest {{"DistributionConfig": {"Origins": [{"Id": "o1", "DomainName": "origin.example.com","ConnectionAttempts": 2, "ConnectionTimeout": 5,+ "ResponseCompletionTimeout": 90,"CustomOriginConfig": {"OriginProtocolPolicy": "https-only",- "OriginReadTimeout": 45}}],+ "OriginReadTimeout": 45,+ "OriginKeepaliveTimeout": 120}}],"DefaultCacheBehavior": {"TargetOriginId": "o1", "ViewerProtocolPolicy": "redirect-to-https"}}}"""), distributionCtx(null));@@ -195,8 +197,11 @@ class CloudFrontCfnProvisionerTest {Origin origin = sent.getValue().getConfig().getOrigins().getFirst();assertEquals(2, origin.getConnectionAttempts());assertEquals(5, origin.getConnectionTimeout());+ // ResponseCompletionTimeout lives on the origin, where the serving layer reads it.+ assertEquals(Integer.valueOf(90), origin.getResponseCompletionTimeout());// OriginReadTimeout lives on CustomOriginConfig, where the serving layer reads it.assertEquals(Integer.valueOf(45), origin.getCustomOriginConfig().get("OriginReadTimeout"));+ assertEquals(Integer.valueOf(120), origin.getCustomOriginConfig().get("OriginKeepaliveTimeout"));}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontControllerTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontControllerTest.javaindex 28482f13bb..c878859762 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontControllerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontControllerTest.java@@ -476,23 +476,6 @@ class CloudFrontControllerTest {}}- @Test- void responseCompletionTimeoutBelowOriginReadTimeoutIsRejected() {- CloudFrontService service = mock(CloudFrontService.class);- CloudFrontController controller = new CloudFrontController(service);-- String body = distributionConfigBody("").replace(- "<OriginProtocolPolicy>https-only</OriginProtocolPolicy></CustomOriginConfig>",- "<OriginProtocolPolicy>https-only</OriginProtocolPolicy>"- + "<OriginReadTimeout>60</OriginReadTimeout></CustomOriginConfig>"- + "<ResponseCompletionTimeout>30</ResponseCompletionTimeout>");-- try (Response created = controller.createDistribution(null, body)) {- assertEquals(400, created.getStatus());- assertTrue(((String) created.getEntity()).contains("InvalidArgument"));- }- }-@Testvoid responseCompletionTimeoutAtLeastOriginReadTimeoutIsAccepted() {CloudFrontService service = mock(CloudFrontService.class);@@ -510,22 +493,6 @@ class CloudFrontControllerTest {}}- @Test- void responseCompletionTimeoutBelowTheImplicitOriginReadTimeoutDefaultIsRejected() {- CloudFrontService service = mock(CloudFrontService.class);- CloudFrontController controller = new CloudFrontController(service);-- String body = distributionConfigBody("").replace(- "<OriginProtocolPolicy>https-only</OriginProtocolPolicy></CustomOriginConfig>",- "<OriginProtocolPolicy>https-only</OriginProtocolPolicy></CustomOriginConfig>"- + "<ResponseCompletionTimeout>10</ResponseCompletionTimeout>");-- try (Response created = controller.createDistribution(null, body)) {- assertEquals(400, created.getStatus());- assertTrue(((String) created.getEntity()).contains("InvalidArgument"));- }- }-@Testvoid defaultCacheBehaviorRoundTripsForwardedValues() {CloudFrontService service = mock(CloudFrontService.class);diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.javaindex 7aef81e48a..5b8e4ac6a3 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontCustomOriginServingTest.java@@ -396,6 +396,50 @@ class CloudFrontCustomOriginServingTest {"expected the 1 s origin read timeout to end the request, took " + elapsedMillis + " ms");}+ @Test+ void endsTheViewerResponseWhenTheOriginDoesNotCompleteItInTime() throws Exception {+ // The origin answers with its headers, then trickles a body in for longer than the+ // completion timeout allows. The viewer has its status line by then, so the response ends+ // with the body the origin managed to send.+ int bodyBytes = 100;+ originServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ originServer.setExecutor(Executors.newVirtualThreadPerTaskExecutor());+ originServer.createContext("/", exchange -> {+ exchange.sendResponseHeaders(200, bodyBytes);+ try (OutputStream out = exchange.getResponseBody()) {+ for (int sent = 0; sent < bodyBytes; sent++) {+ out.write('x');+ out.flush();+ Thread.sleep(100);+ }+ } catch (IOException | InterruptedException expected) {+ // The origin connection ended, which is what the completion timeout does.+ } finally {+ exchange.close();+ }+ });+ originServer.start();+ Distribution created = createDistribution(+ customOriginDistribution(allMethodsBehavior(), null, 2, 1, 2));++ long started = System.nanoTime();+ byte[] response = readViewerResponse(created.getDomainName(), "/page");+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertTrue(new String(response, StandardCharsets.ISO_8859_1).startsWith("HTTP/1.1 200"),+ "expected the origin's status line, got " + new String(response, StandardCharsets.ISO_8859_1));+ int sent = 0;+ for (byte value : response) {+ if (value == 'x') {+ sent++;+ }+ }+ assertTrue(sent < bodyBytes,+ "expected the response to end before the whole body, got " + sent + " of " + bodyBytes + " bytes");+ assertTrue(elapsedMillis < 5000,+ "expected the 2 s completion timeout to end the response, took " + elapsedMillis + " ms");+ }+@Testvoid streamsALargeOriginResponseWithItsLength() throws Exception {byte[] video = randomBytes(3 * 1024 * 1024 + 1);@@ -1128,6 +1172,14 @@ class CloudFrontCustomOriginServingTest {List<Map<String, String>> customHeaders,Integer readTimeoutSeconds,Integer connectionAttempts) {+ return customOriginDistribution(behavior, customHeaders, readTimeoutSeconds, connectionAttempts, null);+ }++ private Distribution customOriginDistribution(DefaultCacheBehavior behavior,+ List<Map<String, String>> customHeaders,+ Integer readTimeoutSeconds,+ Integer connectionAttempts,+ Integer completionTimeoutSeconds) {Map<String, Object> originConfig = customOriginConfig(originServer.getAddress().getPort());if (readTimeoutSeconds != null) {originConfig.put("OriginReadTimeout", readTimeoutSeconds);@@ -1140,6 +1192,9 @@ class CloudFrontCustomOriginServingTest {if (connectionAttempts != null) {customOrigin.setConnectionAttempts(connectionAttempts);}+ if (completionTimeoutSeconds != null) {+ customOrigin.setResponseCompletionTimeout(completionTimeoutSeconds);+ }DistributionConfig config = new DistributionConfig();config.setEnabled(true);config.setOrigins(List.of(customOrigin));@@ -1149,6 +1204,20 @@ class CloudFrontCustomOriginServingTest {return distribution;}+ /**+ * The viewer's whole response, read from a raw socket, since a response the origin ends+ * midway is not one RestAssured can read.+ */+ private static byte[] readViewerResponse(String host, String path) throws Exception {+ try (Socket viewer = new Socket("127.0.0.1", RestAssured.port)) {+ viewer.setSoTimeout(15000);+ viewer.getOutputStream().write(("GET " + path + " HTTP/1.1\r\nHost: " + host+ + "\r\nConnection: close\r\n\r\n").getBytes(StandardCharsets.US_ASCII));+ viewer.getOutputStream().flush();+ return viewer.getInputStream().readAllBytes();+ }+ }+private static List<Map<String, String>> originVerifyHeader() {return List.of(new LinkedHashMap<>(Map.of("HeaderName", "X-Origin-Verify", "HeaderValue", "s3cr3t")));diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.javaindex ec130dd612..660d1c2b38 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontOriginHttpClientTest.java@@ -8,6 +8,9 @@ import io.github.hectorvent.floci.core.common.Pem;import io.github.hectorvent.floci.services.acm.CertificateGenerator;import io.github.hectorvent.floci.services.acm.model.KeyAlgorithm;import org.apache.hc.client5.http.DnsResolver;+import org.apache.hc.client5.http.protocol.HttpClientContext;+import org.apache.hc.core5.http.message.BasicHttpResponse;+import org.apache.hc.core5.util.TimeValue;import org.bouncycastle.jce.provider.BouncyCastleProvider;import org.junit.jupiter.api.Test;@@ -18,6 +21,7 @@ import javax.net.ssl.SNIServerName;import javax.net.ssl.SSLContext;import javax.net.ssl.TrustManagerFactory;import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;@@ -25,6 +29,8 @@ import java.net.ConnectException;import java.net.InetAddress;import java.net.InetSocketAddress;import java.net.ServerSocket;+import java.net.Socket;+import java.net.SocketTimeoutException;import java.net.URI;import java.net.UnknownHostException;import java.net.http.HttpRequest;@@ -48,6 +54,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assumptions.assumeTrue;class CloudFrontOriginHttpClientTest {@@ -373,7 +380,9 @@ class CloudFrontOriginHttpClientTest {resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {CloudFrontOriginHttpClient.OriginTimeouts timeouts =new CloudFrontOriginHttpClient.OriginTimeouts(- Duration.ofSeconds(1), Duration.ofMillis(200), 3);+ Duration.ofSeconds(1), Duration.ofMillis(200), 3,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");long started = System.nanoTime();@@ -399,7 +408,9 @@ class CloudFrontOriginHttpClientTest {resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {CloudFrontOriginHttpClient.OriginTimeouts timeouts =new CloudFrontOriginHttpClient.OriginTimeouts(- Duration.ofSeconds(1), Duration.ofMillis(200), 1);+ Duration.ofSeconds(1), Duration.ofMillis(200), 1,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));@@ -420,7 +431,9 @@ class CloudFrontOriginHttpClientTest {resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {CloudFrontOriginHttpClient.OriginTimeouts timeouts =new CloudFrontOriginHttpClient.OriginTimeouts(- Duration.ofSeconds(1), Duration.ofMillis(200), 3);+ Duration.ofSeconds(1), Duration.ofMillis(200), 3,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);HttpRequest request = HttpRequest.newBuilder(URI.create("http://origin.invalid:" + server.getAddress().getPort() + "/")).method("POST", HttpRequest.BodyPublishers.noBody())@@ -443,7 +456,9 @@ class CloudFrontOriginHttpClientTest {resolver, List.of("blackhole.invalid"))) {CloudFrontOriginHttpClient.OriginTimeouts timeouts =new CloudFrontOriginHttpClient.OriginTimeouts(- Duration.ofMillis(100), Duration.ofSeconds(5), 3);+ Duration.ofMillis(100), Duration.ofSeconds(5), 3,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);HttpRequest request = HttpRequest.newBuilder(URI.create("http://blackhole.invalid:81/")).method("POST", HttpRequest.BodyPublishers.noBody()).build();@@ -467,7 +482,9 @@ class CloudFrontOriginHttpClientTest {resolver, List.of("refused.invalid"))) {CloudFrontOriginHttpClient.OriginTimeouts timeouts =new CloudFrontOriginHttpClient.OriginTimeouts(- Duration.ofSeconds(1), Duration.ofSeconds(5), 3);+ Duration.ofSeconds(1), Duration.ofSeconds(5), 3,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);HttpRequest request = HttpRequest.newBuilder(URI.create("http://refused.invalid:" + refusedPort + "/")).method("POST", HttpRequest.BodyPublishers.noBody())@@ -489,7 +506,9 @@ class CloudFrontOriginHttpClientTest {resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {CloudFrontOriginHttpClient.OriginTimeouts timeouts =new CloudFrontOriginHttpClient.OriginTimeouts(- Duration.ofSeconds(1), Duration.ofMillis(150), 1);+ Duration.ofSeconds(1), Duration.ofMillis(150), 1,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");long started = System.nanoTime();@@ -505,6 +524,332 @@ class CloudFrontOriginHttpClientTest {}}+ @Test+ void theCompletionTimeoutBoundsTheWaitForTheResponseHeaders() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ CountDownLatch release = new CountDownLatch(1);+ HttpServer server = holdingServer(hits, release);+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ // The read timeout is far longer than the completion timeout, so the completion timeout+ // is the one that has to end the request.+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofSeconds(30), 1,+ Duration.ofMillis(500),+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);+ HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");++ long started = System.nanoTime();+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertTrue(elapsedMillis < 5000,+ "expected the 500 ms completion timeout to end the request, not the 30 s read "+ + "timeout, took " + elapsedMillis + " ms");+ } finally {+ release.countDown();+ server.stop(0);+ }+ }++ @Test+ void theCompletionTimeoutBoundsTheRetriesToo() throws Exception {+ AtomicInteger hits = new AtomicInteger();+ CountDownLatch release = new CountDownLatch(1);+ HttpServer server = holdingServer(hits, release);+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofSeconds(30), 3,+ Duration.ofMillis(600),+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);+ HttpRequest request = request("http://origin.invalid:" + server.getAddress().getPort() + "/");++ long started = System.nanoTime();+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertTrue(elapsedMillis < 3000,+ "expected the completion timeout to bound the retries as well, took "+ + elapsedMillis + " ms");+ } finally {+ release.countDown();+ server.stop(0);+ }+ }++ @Test+ void theCompletionTimeoutEndsAResponseThatNeverCompletes() throws Exception {+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ server.setExecutor(Executors.newVirtualThreadPerTaskExecutor());+ server.createContext("/", exchange -> {+ exchange.sendResponseHeaders(200, 0);+ try (OutputStream out = exchange.getResponseBody()) {+ // A body that never ends: it keeps sending, one byte at a time, past the deadline.+ for (int sent = 0; sent < 200; sent++) {+ out.write('x');+ out.flush();+ Thread.sleep(100);+ }+ } catch (IOException | InterruptedException expected) {+ // The client ended the connection, which is what the completion timeout does.+ } finally {+ exchange.close();+ }+ });+ server.start();++ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofSeconds(30), 1,+ Duration.ofMillis(700),+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);+ CloudFrontOriginHttpClient.OpenResponse response =+ get(client, request("http://origin.invalid:" + server.getAddress().getPort() + "/"),+ timeouts);++ long started = System.nanoTime();+ assertThrows(CloudFrontOriginTimeoutException.class, () -> readAll(response));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertTrue(elapsedMillis < 5000,+ "expected the 700 ms completion timeout to end the body, took " + elapsedMillis+ + " ms");+ } finally {+ server.stop(0);+ }+ }++ @Test+ void keepsTheOriginConnectionForTheConfiguredKeepAlive() throws Exception {+ List<InetSocketAddress> peers = new CopyOnWriteArrayList<>();+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ server.createContext("/", exchange -> {+ peers.add(exchange.getRemoteAddress());+ byte[] body = "origin-ok".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(200, body.length);+ exchange.getResponseBody().write(body);+ exchange.close();+ });+ server.start();++ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ String uri = "http://origin.invalid:" + server.getAddress().getPort() + "/";+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofSeconds(5), 1,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ Duration.ofSeconds(300));+ readAll(get(client, request(uri), timeouts));+ readAll(get(client, request(uri), timeouts));++ assertEquals(2, peers.size());+ assertEquals(peers.get(0), peers.get(1),+ "the connection was kept for the configured keep-alive and reused");+ } finally {+ server.stop(0);+ }+ }++ @Test+ void theKeepAliveStrategyReturnsTheOriginsKeepAlive() {+ CloudFrontOriginHttpClient.KeepAliveStrategy strategy =+ new CloudFrontOriginHttpClient.KeepAliveStrategy();+ // The context the client sends, so the test fails if it stops carrying the keep-alive.+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofSeconds(30), 1,+ CloudFrontOriginTimeouts.DEFAULT_RESPONSE_COMPLETION_TIMEOUT,+ Duration.ofSeconds(2));+ HttpClientContext context = CloudFrontOriginHttpClient.context(timeouts);++ assertEquals(TimeValue.ofSeconds(2),+ strategy.getKeepAliveDuration(new BasicHttpResponse(200), context));++ BasicHttpResponse originAsksForLess = new BasicHttpResponse(200);+ originAsksForLess.addHeader("Keep-Alive", "timeout=1");+ assertEquals(TimeValue.ofSeconds(1),+ strategy.getKeepAliveDuration(originAsksForLess, context),+ "the origin's own Keep-Alive header shortens the setting");++ BasicHttpResponse originAsksForMore = new BasicHttpResponse(200);+ originAsksForMore.addHeader("Keep-Alive", "timeout=30");+ assertEquals(TimeValue.ofSeconds(2),+ strategy.getKeepAliveDuration(originAsksForMore, context),+ "the setting caps what the origin asks for");++ assertEquals(+ TimeValue.ofSeconds(CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT_SECONDS),+ strategy.getKeepAliveDuration(new BasicHttpResponse(200), HttpClientContext.create()),+ "a request with no keep-alive setting uses the default");+ }++ @Test+ void theCompletionTimeoutEndsAStalledBody() throws Exception {+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ server.setExecutor(Executors.newVirtualThreadPerTaskExecutor());+ server.createContext("/", exchange -> {+ exchange.sendResponseHeaders(200, 1024);+ try (OutputStream out = exchange.getResponseBody()) {+ out.write('x');+ out.flush();+ // The body then stalls: the headers arrived, the rest of the body never does.+ Thread.sleep(30_000);+ } catch (IOException | InterruptedException expected) {+ // The client ended the connection, which is what the completion timeout does.+ } finally {+ exchange.close();+ }+ });+ server.start();++ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"))) {+ // The read timeout is far longer than the completion timeout, so the completion timeout+ // is the one that has to end the stalled body.+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofSeconds(30), 1,+ Duration.ofMillis(700),+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);+ CloudFrontOriginHttpClient.OpenResponse response = get(client,+ request("http://origin.invalid:" + server.getAddress().getPort() + "/"), timeouts);++ long started = System.nanoTime();+ CloudFrontOriginTimeoutException failure = assertThrows(CloudFrontOriginTimeoutException.class,+ () -> readAll(response));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertEquals(CloudFrontServingController.ORIGIN_TIMEOUT_MESSAGE, failure.getMessage());+ assertTrue(elapsedMillis < 5000,+ "expected the 700 ms completion timeout to end the stalled body, took "+ + elapsedMillis + " ms");+ } finally {+ server.stop(0);+ }+ }++ @Test+ void aSlowViewerDoesNotExpireAResponseTheOriginFinishedInTime() throws Exception {+ byte[] body = "a body the origin answered in full".getBytes(StandardCharsets.UTF_8);+ HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);+ server.createContext("/", exchange -> {+ exchange.sendResponseHeaders(200, body.length);+ exchange.getResponseBody().write(body);+ exchange.close();+ });+ server.start();++ AtomicLong clock = new AtomicLong();+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(InetAddress.getByName("127.0.0.1")), List.of("origin.invalid"), clock::get)) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(1), Duration.ofSeconds(30), 1,+ Duration.ofMillis(500),+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);+ CloudFrontOriginHttpClient.OpenResponse response = get(client,+ request("http://origin.invalid:" + server.getAddress().getPort() + "/"), timeouts);++ // The viewer reads the body far slower than the completion timeout allows: a second+ // passes between reads. The origin answered in time, so the response is not expired.+ ByteArrayOutputStream received = new ByteArrayOutputStream();+ try (InputStream origin = response.body()) {+ byte[] chunk = new byte[4];+ int read;+ while ((read = origin.read(chunk)) != -1) {+ received.write(chunk, 0, read);+ clock.addAndGet(Duration.ofSeconds(1).toNanos());+ }+ }+ assertArrayEquals(body, received.toByteArray());+ } finally {+ server.stop(0);+ }+ }++ @Test+ void theCompletionTimeoutBoundsTheConnectionToTheOrigin() throws Exception {+ // An address that blackholes the connection: the attempt is never answered.+ InetAddress blackhole = InetAddress.getByName("10.255.255.1");+ assumeTrue(connectionHangs(blackhole),+ "the environment answers the connection attempt instead of dropping it");+ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(blackhole), List.of("origin.invalid"))) {+ // The connect timeout is far longer than the completion timeout, so the completion+ // timeout is the one that has to end the connection attempt.+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(10), Duration.ofSeconds(30), 1,+ Duration.ofMillis(500),+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);+ HttpRequest request = request("http://origin.invalid:80/");++ long started = System.nanoTime();+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertTrue(elapsedMillis < 5000,+ "expected the 500 ms completion timeout to end the connection attempt, not the "+ + "10 s connect timeout, took " + elapsedMillis + " ms");+ }+ }++ @Test+ void theCompletionTimeoutBoundsTheTlsHandshake() throws Exception {+ // An origin that accepts the connection and never answers the handshake.+ InetAddress loopback = InetAddress.getByName("127.0.0.1");+ try (ServerSocket socket = new ServerSocket(0, 1, loopback)) {+ Thread acceptor = new Thread(() -> {+ try (Socket accepted = socket.accept()) {+ Thread.sleep(30_000);+ } catch (IOException | InterruptedException expected) {+ // The client gave up on the handshake.+ }+ });+ acceptor.setDaemon(true);+ acceptor.start();++ try (CloudFrontOriginHttpClient client = new CloudFrontOriginHttpClient(+ resolver(loopback), List.of("origin.invalid"))) {+ CloudFrontOriginHttpClient.OriginTimeouts timeouts =+ new CloudFrontOriginHttpClient.OriginTimeouts(+ Duration.ofSeconds(10), Duration.ofSeconds(30), 1,+ Duration.ofMillis(500),+ CloudFrontOriginTimeouts.DEFAULT_ORIGIN_KEEPALIVE_TIMEOUT);+ HttpRequest request = request("https://origin.invalid:" + socket.getLocalPort() + "/");++ long started = System.nanoTime();+ assertThrows(CloudFrontOriginTimeoutException.class, () -> get(client, request, timeouts));+ long elapsedMillis = (System.nanoTime() - started) / 1_000_000;++ assertTrue(elapsedMillis < 5000,+ "expected the 500 ms completion timeout to end the handshake, not the 10 s "+ + "connect timeout, took " + elapsedMillis + " ms");+ }+ }+ }++ /**+ * Whether a connection to {@code address} hangs, as a blackholed address does: the attempt has+ * to be dropped rather than answered for the origin connect timeout to be the one under test.+ */+ private static boolean connectionHangs(InetAddress address) {+ try (Socket socket = new Socket()) {+ socket.connect(new InetSocketAddress(address, 80), 500);+ return false;+ } catch (SocketTimeoutException expected) {+ return true;+ } catch (IOException expected) {+ return false;+ }+ }+private static CloudFrontOriginHttpClient.OpenResponse get(CloudFrontOriginHttpClient client, HttpRequest request,CloudFrontOriginHttpClient.OriginTimeouts timeouts) throws Exception {diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServiceTest.javaindex 1780cd4216..118e76aebc 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudfront/CloudFrontServiceTest.java@@ -780,6 +780,42 @@ class CloudFrontServiceTest {}), Map.of()));}+ @Test+ void createDistributionRejectsAResponseCompletionTimeoutShorterThanTheReadTimeout() {+ CloudFrontService service = serviceWithDomainSuffix("cloudfront.net");++ AwsException below = assertAws("InvalidArgument",+ () -> service.createDistribution(distributionWithCustomOrigin(origin -> {+ origin.getCustomOriginConfig().put("OriginReadTimeout", 30);+ origin.setResponseCompletionTimeout(29);+ }), Map.of()));+ assertEquals("The parameter ResponseCompletionTimeout must be greater than or equal to"+ + " OriginReadTimeout.", below.getMessage());+ assertEquals(400, below.getHttpStatus());++ AwsException negative = assertAws("InvalidArgument",+ () -> service.createDistribution(distributionWithCustomOrigin(origin ->+ origin.setResponseCompletionTimeout(-1)), Map.of()));+ assertEquals("The parameter ResponseCompletionTimeout must be a non-negative integer.",+ negative.getMessage());+ assertEquals(400, negative.getHttpStatus());++ // A completion timeout as long as the read timeout, and none at all, are both accepted.+ assertDoesNotThrow(() -> service.createDistribution(distributionWithCustomOrigin(origin -> {+ origin.getCustomOriginConfig().put("OriginReadTimeout", 30);+ origin.setResponseCompletionTimeout(30);+ }), Map.of()));+ assertDoesNotThrow(() -> service.createDistribution(distributionWithCustomOrigin(origin ->+ origin.setResponseCompletionTimeout(0)), Map.of()));++ // No OriginReadTimeout means the 30 s default, which a shorter completion timeout breaks.+ AwsException belowDefault = assertAws("InvalidArgument",+ () -> service.createDistribution(distributionWithCustomOrigin(origin ->+ origin.setResponseCompletionTimeout(10)), Map.of()));+ assertEquals("The parameter ResponseCompletionTimeout must be greater than or equal to"+ + " OriginReadTimeout.", belowDefault.getMessage());+ }+@Testvoid updateDistributionRejectsOriginSettingsOutsideTheAwsRange() {CloudFrontService service = serviceWithDomainSuffix("cloudfront.net");@@ -797,6 +833,34 @@ class CloudFrontServiceTest {origin.getCustomOriginConfig().put("OriginReadTimeout", 121))));}+ @Test+ void updateDistributionRejectsAResponseCompletionTimeoutShorterThanTheReadTimeout() {+ CloudFrontService service = serviceWithDomainSuffix("cloudfront.net");+ Distribution existing = service.createDistribution(+ distributionWithCustomOrigin(origin -> { }), Map.of());++ AwsException below = assertAws("InvalidArgument", () -> service.updateDistribution(+ existing.getId(), existing.getEtag(),+ distributionWithCustomOrigin(origin -> {+ origin.getCustomOriginConfig().put("OriginReadTimeout", 30);+ origin.setResponseCompletionTimeout(29);+ })));+ assertEquals("The parameter ResponseCompletionTimeout must be greater than or equal to"+ + " OriginReadTimeout.", below.getMessage());+ assertEquals(400, below.getHttpStatus());++ AwsException negative = assertAws("InvalidArgument", () -> service.updateDistribution(+ existing.getId(), existing.getEtag(),+ distributionWithCustomOrigin(origin -> origin.setResponseCompletionTimeout(-1))));+ assertEquals("The parameter ResponseCompletionTimeout must be a non-negative integer.",+ negative.getMessage());+ assertEquals(400, negative.getHttpStatus());++ // The rejected updates left the distribution as it was.+ Origin origin = service.getDistribution(existing.getId()).getConfig().getOrigins().getFirst();+ assertNull(origin.getResponseCompletionTimeout());+ }+@Testvoid createDistributionAcceptsOriginSettingsWithinTheAwsRange() {CloudFrontService service = serviceWithDomainSuffix("cloudfront.net");