Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): resolve virtual-hosted buckets whose names contain dots (#2493)
Allen Sanbornea2b62fb955e
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsRegions.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsRegions.javaindex 2a94e46b54..4ad0947847 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsRegions.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsRegions.java@@ -1,6 +1,8 @@package io.github.hectorvent.floci.core.common;import java.util.List;+import java.util.Locale;+import java.util.Set;/*** Catalog of AWS regions the emulator advertises. Single source of truth for@@ -15,6 +17,43 @@ public final class AwsRegions {"ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2","ap-south-1", "sa-east-1", "ca-central-1");+ /**+ * Every published AWS region id, across the commercial, GovCloud and China partitions.+ *+ * <p>Distinct from {@link #ALL}, and deliberately a superset of it. {@code ALL} is what+ * this emulator <em>advertises</em> — what DescribeRegions returns. {@code KNOWN_IDS} is+ * for <em>recognising</em> a region id that a client wrote, most importantly inside a+ * hostname, where the alternative is a shape like {@code [a-z]{2}-[a-z-]+-\d+} that also+ * matches perfectly ordinary strings ({@code my-cd-1}, {@code eu-team-2}).+ *+ * <p>Sourced from the AWS General Reference endpoint tables. It needs an entry when AWS+ * launches a region; the cost of a missing one is documented at each call site, because it+ * differs per caller.+ */+ public static final Set<String> KNOWN_IDS = Set.of(+ "us-east-1", "us-east-2", "us-west-1", "us-west-2",+ "af-south-1",+ "ap-east-1", "ap-east-2",+ "ap-south-1", "ap-south-2",+ "ap-northeast-1", "ap-northeast-2", "ap-northeast-3",+ "ap-southeast-1", "ap-southeast-2", "ap-southeast-3", "ap-southeast-4",+ "ap-southeast-5", "ap-southeast-6", "ap-southeast-7",+ "ca-central-1", "ca-west-1",+ "eu-central-1", "eu-central-2",+ "eu-west-1", "eu-west-2", "eu-west-3",+ "eu-north-1", "eu-south-1", "eu-south-2",+ "il-central-1",+ "me-central-1", "me-south-1",+ "mx-central-1",+ "sa-east-1",+ "us-gov-east-1", "us-gov-west-1",+ "cn-north-1", "cn-northwest-1");++ /** True when {@code label} is exactly an AWS region id, case-insensitively. */+ public static boolean isRegionId(String label) {+ return label != null && KNOWN_IDS.contains(label.toLowerCase(Locale.ROOT));+ }+private AwsRegions() {}}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilter.javaindex 80498920d7..e8120cf6b2 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilter.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilter.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.s3;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;import io.github.hectorvent.floci.services.cloudfront.CloudFrontDistributionFilter;@@ -13,6 +14,7 @@ import jakarta.ws.rs.core.UriBuilder;import jakarta.ws.rs.ext.Provider;import java.net.URI;+import java.util.Arrays;import java.util.HashSet;import java.util.List;import java.util.Optional;@@ -35,6 +37,29 @@ public class S3VirtualHostFilter implements ContainerRequestFilter {*/private final Set<String> serviceHostSuffixes;+ /**+ * Host labels belonging to other AWS services that virtual-host under the same endpoint+ * host as S3. A hostname carrying one of these is that service's, never an S3 bucket+ * whose name happens to contain dots — see {@link #bucketFromPrefix}.+ *+ * <p>Every entry names a service that Floci itself routes by {@code Host} in a+ * <em>regionless</em> form, which {@link #isForeignRegionalHost} cannot catch. A label is only+ * justified when some filter in this repository claims that hostname; a speculative entry+ * costs real bucket names, because {@code my.elb.logs} — AWS's own load-balancer+ * access-log naming convention — is a perfectly legal bucket.+ */+ private static final Set<String> NON_S3_SERVICE_LABELS = Set.of(+ "execute-api", // ApiGatewayExecuteApiHostFilter: <api-id>.execute-api[.<region>].<host>+ "lambda-url", // LambdaUrlRoutingFilter: <url-id>.lambda-url.<region>.<host>+ "emr-serverless", // EmrServerlessRouteFilter: emr-serverless.<host>+ "cloudfront"); // CloudFrontDistributionFilter: <dist-id>.cloudfront.net++ /**+ * The label that makes an S3 endpoint host IPv6-capable: {@code s3.dualstack.<region>},+ * {@code s3-fips.dualstack.<region>}, {@code s3-accelerate.dualstack}.+ */+ private static final String DUALSTACK = "dualstack";+@Injectpublic S3VirtualHostFilter(EmulatorConfig config, ContainerDetector containerDetector) {this.baseHostname = config.hostname()@@ -142,20 +167,45 @@ public class S3VirtualHostFilter implements ContainerRequestFilter {/*** Extracts a bucket name from a virtual-hosted-style Host header.*- * A request is considered virtual-hosted-style when the hostname's remainder- * after the first label matches the configured Floci base hostname, or when it- * matches a well-known AWS S3 domain pattern (for DNS-redirect setups).+ * <p>A request is considered virtual-hosted-style when the hostname <em>ends with</em> a+ * known endpoint host — the configured Floci base hostname, plain {@code localhost}, a+ * configured DNS suffix, or a well-known AWS S3 domain (for DNS-redirect setups). The+ * bucket is everything in front of that suffix, minus an optional S3 qualifier+ * ({@code s3}, {@code s3.<region>}, {@code s3-website-<region>}, …).+ *+ * <p><strong>The bucket is not just the first label.</strong> S3 bucket names may contain+ * dots, and naming a website bucket after a domain is the conventional pattern, so+ * {@code www.example.com.localhost} is the bucket {@code www.example.com} — not the bucket+ * {@code www} with an unrecognised remainder. Splitting at the first dot made every dotted+ * bucket fall through to path-style, where {@code HEAD /} answers 200 and the AWS provider+ * reads that as "bucket already exists".** Examples with baseHostname="localhost":* my-bucket.localhost:4566 -> "my-bucket"* my-bucket.localhost -> "my-bucket"+ * www.example.com.localhost -> "www.example.com" (dotted bucket)+ * my.bucket-logs.localhost:4566 -> "my.bucket-logs" (dots and hyphens)+ * my.s3.archive.localhost -> "my.s3.archive" (an s3 label followed by a+ * non-qualifier is bucket text)+ * data.my-cd-1.localhost -> "data.my-cd-1" (region-SHAPED, not a region id)+ * my-bucket.s3.us-east-1.localhost -> "my-bucket" (region-qualified)+ * www.example.com.s3.us-east-1.amazonaws.com -> "www.example.com"+ * s3.localhost -> null (service endpoint, not a bucket named "s3")+ * s3.us-east-1.localhost -> null (region-qualified service endpoint)* floci.svc.cluster.local -> null (no bucket prefix, path-style)- * my-svc.floci.svc.cluster.local -> null (remainder doesn't match "localhost")+ * my-svc.floci.svc.cluster.local -> null (does not end with a known endpoint host)** Examples with baseHostname="floci.svc.cluster.local":* my-bucket.floci.svc.cluster.local -> "my-bucket"* floci.svc.cluster.local -> null (no bucket prefix, path-style)*+ * <p>Inherent ambiguity: a bucket whose <em>own</em> name ends in a complete qualifier+ * ({@code foo.s3}, {@code foo.s3.us-east-1}) is indistinguishable from a qualified reference+ * to {@code foo} and is read as the latter. That is the whole of the ambiguity — a bucket+ * whose name merely <em>contains</em> {@code s3} ({@code my.s3.archive}) is not ambiguous and+ * is not truncated. The service endpoint always wins over the bucket reading, so+ * {@code s3.localhost} stays bucketless.+ ** Returns null if the host does not match a virtual-hosted pattern.*/static String extractBucket(String host, String baseHostname, Set<String> serviceHostSuffixes) {@@ -177,54 +227,268 @@ public class S3VirtualHostFilter implements ContainerRequestFilter {return null;}+ // The service-endpoint guard runs first and wins over every suffix match below, so a+ // bare s3.<endpoint> host is never mistaken for a bucket literally named "s3".String firstLabel = hostname.substring(0, firstDot);String remainder = hostname.substring(firstDot + 1);-if (isS3ServiceEndpointHost(firstLabel, remainder, baseHostname, serviceHostSuffixes)) {return null;}- // Primary: remainder must match the configured base hostname,- // either directly or in the AWS region-qualified s3.<region>.<host> form.- if (baseHostname != null && matchesEndpointHost(remainder, baseHostname)) {- return firstLabel;+ // Primary: the longest known endpoint host this hostname ends with — the configured+ // base hostname, always-on localhost, or a configured Floci DNS suffix (builtins ++ // floci.dns.extra-suffixes). Longest wins, so bucket.localhost.floci.io yields+ // "bucket" rather than "bucket.localhost" when both suffixes are known.+ String prefix = longestEndpointPrefix(hostname, baseHostname, serviceHostSuffixes);+ if (prefix != null) {+ return bucketFromPrefix(prefix, false);}- // Fallback: well-known AWS S3 domains, for users who route AWS DNS to Floci- if (isAwsS3Domain(remainder)) {- return firstLabel;+ // Fallback: well-known AWS S3 domains, for users who route AWS DNS to Floci. An S3+ // qualifier is required here, so a plain foo.amazonaws.com is not an S3 bucket.+ String awsPrefix = stripHostSuffix(hostname, "amazonaws.com");+ if (awsPrefix != null) {+ return bucketFromPrefix(awsPrefix, true);}- // Configured Floci DNS suffixes (builtins + floci.dns.extra-suffixes): route- // bucket.<suffix>, bucket.s3.<suffix> and bucket.s3.<region>.<suffix> the same- // way the always-on wildcard-DNS builtins resolve.- if (matchesConfiguredSuffixHost(remainder, serviceHostSuffixes)) {- return firstLabel;+ // Website endpoints keep their historical tail-agnostic handling: any+ // bucket.s3-website-<region>.<anything> is virtual-hosted.+ return bucketBeforeWebsiteQualifier(hostname);+ }++ /**+ * Returns the part of {@code hostname} preceding the longest known endpoint host it ends+ * with, or {@code null} if it ends with none of them. Candidates are the configured base+ * hostname, plain {@code localhost} (which always resolves to 127.0.0.1 regardless of+ * {@code FLOCI_HOSTNAME}), and every configured DNS suffix.+ */+ private static String longestEndpointPrefix(String hostname, String baseHostname,+ Set<String> serviceHostSuffixes) {+ String best = baseHostname == null ? null : stripHostSuffix(hostname, baseHostname);+ best = shorter(best, stripHostSuffix(hostname, "localhost"));+ for (String suffix : serviceHostSuffixes) {+ best = shorter(best, stripHostSuffix(hostname, suffix));}+ return best;+ }- return null;+ /** The shorter of two candidate prefixes is the one that matched the longer suffix. */+ private static String shorter(String a, String b) {+ if (a == null) {+ return b;+ }+ if (b == null) {+ return a;+ }+ return b.length() < a.length() ? b : a;}/**- * Matches the virtual-hosted bucket forms for a configured DNS suffix {@code S}- * (a builtin, {@code floci.hostname}, or a {@code floci.dns.extra-suffix}): {@code bucket.S},- * {@code bucket.s3.S}, and the region-qualified {@code bucket.s3.<region>.S}.- * Plain {@code localhost} is skipped here — it keeps its dedicated- * {@link #matchesEndpointHost} handling via {@link #isAwsS3Domain}.+ * Returns everything before {@code "." + endpointHost} (case-insensitive), or {@code null}+ * when {@code hostname} does not end with that suffix or has nothing in front of it.*/- private static boolean matchesConfiguredSuffixHost(String remainder, Set<String> serviceHostSuffixes) {- String lower = remainder.toLowerCase();- for (String suffix : serviceHostSuffixes) {- if ("localhost".equals(suffix)) {- continue;+ private static String stripHostSuffix(String hostname, String endpointHost) {+ if (endpointHost == null || endpointHost.isEmpty()) {+ return null;+ }+ String suffix = "." + endpointHost;+ if (hostname.length() <= suffix.length()) {+ return null;+ }+ int start = hostname.length() - suffix.length();+ if (!hostname.regionMatches(true, start, suffix, 0, suffix.length())) {+ return null;+ }+ return hostname.substring(0, start);+ }++ /**+ * Reduces the part of the hostname in front of the endpoint host to a bucket name by+ * removing a trailing S3 endpoint qualifier when one is present.+ *+ * <p>Everything before the qualifier is the bucket, so dots inside the bucket name survive.+ * A prefix that <em>is</em> a qualifier ({@code s3}, {@code s3.us-east-1}) is the bucketless+ * service endpoint and yields {@code null}.+ *+ * <p><strong>A qualifier is a tail, not a label.</strong> The split point is the first index+ * from which the remaining labels form a <em>complete</em> qualifier — see+ * {@link #isS3QualifierTail}. Testing labels one at a time truncates dotted buckets whose name+ * happens to contain {@code s3}: for {@code my.s3.archive.localhost} a per-label test finds+ * {@code s3} at index 1 and hands back the bucket {@code my}, so a request for+ * {@code my.s3.archive} silently reads and writes a different bucket. Nothing legal follows+ * {@code s3} except a region, {@code dualstack.<region>}, or the end of the prefix, so+ * {@code s3.archive} is bucket text and the whole prefix is the bucket.+ *+ * @param requireQualifier when true, a prefix with no qualifier is not a bucket at all+ * (used for {@code amazonaws.com}, where only s3-qualified hosts count)+ */+ private static String bucketFromPrefix(String prefix, boolean requireQualifier) {+ if (prefix == null || prefix.isEmpty()) {+ return null;+ }+ String[] labels = prefix.split("\\.", -1);++ // Other AWS services virtual-host under the same endpoint host as S3 does+ // (<api-id>.execute-api.<endpoint>, <url-id>.lambda-url.<region>.<endpoint>, …).+ // Suffix matching would otherwise read those as a dotted bucket named+ // "<api-id>.execute-api" and hijack the request away from their own filters.+ // Index 0 is exempt: a host whose *first* label is the service label+ // (emr-serverless.<endpoint>) was already read as a bucket before this change.+ for (int i = 1; i < labels.length; i++) {+ if (NON_S3_SERVICE_LABELS.contains(labels[i].toLowerCase())) {+ return null;+ }+ }++ int qualifier = -1;+ for (int i = 0; i < labels.length; i++) {+ if (isS3QualifierTail(labels, i)) {+ qualifier = i;+ break;+ }+ }+ if (qualifier < 0) {+ if (requireQualifier) {+ return null;+ }+ // No S3 qualifier anywhere, and an AWS region id appears in a non-leading position:+ // this is another service's regional virtual host, not a dotted bucket. See+ // isForeignRegionalHost — this is what keeps <acct>.dkr.ecr.<region>.localhost and+ // <domain>.<region>.es.localhost from being served as buckets.+ if (isForeignRegionalHost(labels)) {+ return null;+ }+ return prefix;+ }+ if (qualifier == 0) {+ // Bare service endpoint: s3.<host>, s3.<region>.<host>, s3-website-<region>.<host>+ return null;+ }+ return String.join(".", Arrays.copyOfRange(labels, 0, qualifier));+ }++ /**+ * True when {@code labels[from..]} is exactly an S3 endpoint qualifier — that is, when the+ * labels from {@code from} onwards account for the <em>whole</em> remainder of the prefix and+ * spell one of the endpoint forms AWS actually publishes:+ *+ * <pre>+ * s3 s3-fips.<region>+ * s3.<region> s3-fips.dualstack.<region>+ * s3.dualstack.<region> s3-accelerate+ * s3-<region> (legacy) s3-accelerate.dualstack+ * s3-website s3-website.<region> (newer regions)+ * s3-website-<region> (older regions)+ * </pre>+ *+ * <p>Forms taken from the Amazon S3 endpoint tables in the AWS General Reference. Note that+ * the website endpoint spells its region with a dot in newer regions and a dash in older ones+ * ({@code s3-website.eu-west-2} vs {@code s3-website-eu-west-1}), so both are accepted.+ *+ * <p>The "whole remainder" requirement is the point of this method. It is what makes an+ * {@code s3} label that is followed by something else — {@code my.s3.archive} — bucket text+ * rather than a qualifier, and it is what stops the bucket being truncated at it.+ *+ * <p>{@code s3-accesspoint} and {@code s3-control} are deliberately absent: those hosts are+ * prefixed by an access point name or an account id, not a bucket, and {@code filter} already+ * routes S3 Control away by path.+ */+ private static boolean isS3QualifierTail(String[] labels, int from) {+ int n = labels.length;+ if (from >= n) {+ return false;+ }+ String head = labels[from].toLowerCase();+ int i = from + 1;++ // Transfer acceleration is global: it names no region, only an optional dualstack.+ if ("s3-accelerate".equals(head)) {+ if (i < n && DUALSTACK.equals(labels[i].toLowerCase())) {+ i++;}- if (lower.equals(suffix) || (lower.startsWith("s3.") && lower.endsWith("." + suffix))) {+ return i == n;+ }++ // Legacy dash forms carry the region inside the head label, so nothing may follow:+ // s3-website-us-east-1, s3-us-west-2.+ if (head.startsWith("s3-website-") && AwsRegions.isRegionId(head.substring("s3-website-".length()))) {+ return i == n;+ }+ if (head.startsWith("s3-") && AwsRegions.isRegionId(head.substring("s3-".length()))) {+ return i == n;+ }++ // Dotted forms: <head>[.dualstack][.<region>]+ if (!"s3".equals(head) && !"s3-fips".equals(head) && !"s3-website".equals(head)) {+ return false;+ }+ boolean dualstack = i < n && DUALSTACK.equals(labels[i].toLowerCase());+ if (dualstack) {+ i++;+ }+ if (i < n && AwsRegions.isRegionId(labels[i])) {+ i++;+ } else if (dualstack) {+ // "dualstack" only appears in front of a region; a bare s3.dualstack is not an+ // endpoint, so treat the whole thing as bucket text rather than truncating.+ return false;+ }+ return i == n;+ }++ /**+ * True when an unqualified prefix carries an AWS region id in a non-leading label — the shape+ * every <em>other</em> regional service virtual-hosts under: {@code <acct>.dkr.ecr.<region>},+ * {@code <domain>.<region>.es}, {@code <id>.iot.<region>}. A real S3 virtual host always+ * carries an {@code s3} qualifier instead, and when it does the region is read from there, so+ * a bucket whose own name contains a region stays addressable as {@code logs.us-east-1.s3.<host>}.+ * One structural rule covers every regional service at once, including services Floci does not+ * model yet, where a label denylist would have to enumerate them and would silently hijack the+ * first one it missed.+ *+ * <p><strong>Region ids, not a region-shaped pattern.</strong> This used to match+ * {@code [a-z]{2}-[a-z-]+-\d+}, which is also the shape of ordinary bucket labels:+ * {@code data.my-cd-1} is a legal bucket, and matching by shape rejected it, dropped it back to+ * path-style, and restored the false "bucket exists" 200 this filter exists to prevent. The+ * finite list in {@link AwsRegions#KNOWN_IDS} is what separates "another service's regional+ * virtual host" from "a bucket whose label happens to look region-shaped".+ *+ * <p>Two residual costs, both accepted deliberately:+ * <ul>+ * <li>A bucket whose name contains a <em>real</em> region id in a non-leading position+ * ({@code data.us-east-1}) is still not addressable in the unqualified virtual-hosted+ * form. It remains addressable qualified ({@code data.us-east-1.s3.<host>}) and+ * path-style, and it was equally unaddressable before this filter existed.</li>+ * <li>A regional service host in a region AWS launches after {@link AwsRegions#KNOWN_IDS}+ * was last updated would be read as a bucket. That is the reason to keep the list+ * current; the failure is a wrong route rather than data loss, and the+ * {@code NON_S3_SERVICE_LABELS} entries still cover the services Floci itself routes.</li>+ * </ul>+ */+ private static boolean isForeignRegionalHost(String[] labels) {+ // Index 0 is exempt so a bucket literally named "us-east-1" keeps working.+ for (int i = 1; i < labels.length; i++) {+ if (AwsRegions.isRegionId(labels[i])) {return true;}}return false;}+ /**+ * Historical tail-agnostic website handling: {@code bucket.s3-website-<region>.<anything>}+ * is virtual-hosted even when the tail is not a configured endpoint host.+ */+ private static String bucketBeforeWebsiteQualifier(String hostname) {+ String[] labels = hostname.split("\\.", -1);+ for (int i = labels.length - 2; i >= 1; i--) {+ if (labels[i].toLowerCase().startsWith("s3-website")) {+ return String.join(".", Arrays.copyOfRange(labels, 0, i));+ }+ }+ return null;+ }+static boolean isS3ServiceEndpointHost(String firstLabel, String remainder, String baseHostname,Set<String> serviceHostSuffixes) {if (!"s3".equalsIgnoreCase(firstLabel)) {@@ -288,32 +552,4 @@ public class S3VirtualHostFilter implements ContainerRequestFilter {return true;}- /**- * Returns true for well-known AWS S3 domains and the always-on {@code localhost}- * endpoint. The Floci wildcard-DNS builtins ({@code localhost.floci.io},- * {@code localhost.localstack.cloud}) and any configured {@code floci.dns.extra-suffixes}- * are handled by {@link #matchesConfiguredSuffixHost} instead, so they stay derived- * from the DNS source of truth rather than hardcoded here.- */- private static boolean isAwsS3Domain(String remainder) {- if ("s3.amazonaws.com".equals(remainder)) {- return true;- }- // s3.<region>.amazonaws.com- if (remainder.startsWith("s3.") && remainder.endsWith(".amazonaws.com")) {- return true;- }- // localhost always resolves to 127.0.0.1 — accept regardless of FLOCI_HOSTNAME config.- // Fixes: SDK with endpointOverride("http://localhost:4566") without forcePathStyle sends- // Host: my-bucket.localhost:4566, which must be recognized even when baseHostname=floci.- // Also accept the region-qualified s3.<region>.localhost form (e.g. bucket.s3.us-east-1.localhost).- if (matchesEndpointHost(remainder, "localhost")) {- return true;- }- // S3 website endpoints: bucket.s3-website-<region>.amazonaws.com, bucket.s3-website-<region>.localhost, etc.- if (remainder.startsWith("s3-website")) {- return true;- }- return false;- }}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/AwsRegionsTest.java b/src/test/java/io/github/hectorvent/floci/core/common/AwsRegionsTest.javanew file mode 100644index 0000000000..5a55f07fab--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/AwsRegionsTest.java@@ -0,0 +1,47 @@+package io.github.hectorvent.floci.core.common;++import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;++class AwsRegionsTest {++ /**+ * {@code ALL} is what the emulator advertises; {@code KNOWN_IDS} is what it recognises. The+ * second must contain the first, or DescribeRegions could name a region that hostname parsing+ * refuses to read back.+ */+ @Test+ void everyAdvertisedRegionIsAKnownRegionId() {+ for (String region : AwsRegions.ALL) {+ assertTrue(AwsRegions.KNOWN_IDS.contains(region),+ region + " is advertised by DescribeRegions but is not a known region id");+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"us-east-1", "eu-west-1", "ap-northeast-3", "us-gov-west-1",+ "cn-northwest-1", "il-central-1", "US-EAST-1"})+ void realRegionIdsAreRecognised(String label) {+ assertTrue(AwsRegions.isRegionId(label));+ }++ /**+ * The point of an id list over a pattern: these all match the region <em>shape</em>+ * {@code [a-z]{2}-[a-z-]+-\d+} and none of them is a region. Treating them as regions is what+ * made {@code data.my-cd-1} unreachable as an S3 bucket.+ */+ @ParameterizedTest+ @ValueSource(strings = {"my-cd-1", "eu-team-2", "us-west-9", "ap-corp-1", "no-such-region-12"})+ void regionShapedStringsThatAreNotRegionsAreRejected(String label) {+ assertFalse(AwsRegions.isRegionId(label));+ }++ @Test+ void nullIsNotARegionId() {+ assertFalse(AwsRegions.isRegionId(null));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilterTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilterTest.javaindex c916ed8fca..d7f496139b 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilterTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilterTest.java@@ -15,6 +15,7 @@ import java.util.Set;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotEquals;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.Mockito.mock;@@ -307,4 +308,390 @@ class S3VirtualHostFilterTest {// ...but the s3-website authority survives, so downstream website detection still fires.assertEquals("my-bucket.s3-website-us-east-1.localhost:4566", newUri.getAuthority());}++ // --- Dotted bucket names: the bucket is everything before the endpoint suffix (#s3-vhost) ---++ @ParameterizedTest+ @CsvSource({+ // Single-label bucket — unchanged behaviour+ "my-bucket.localhost, localhost, my-bucket",+ "my-bucket.localhost:4566, localhost, my-bucket",+ // Dotted bucket: a website bucket named after a domain+ "www.example.com.localhost, localhost, www.example.com",+ "www.example.com.localhost:4566, localhost, www.example.com",+ // Deeply dotted bucket+ "a.b.c.d.localhost, localhost, a.b.c.d",+ "a.b.c.d.localhost:4611, localhost, a.b.c.d",+ // Dots AND hyphens — the exact shape from the Terraform BucketAlreadyExists report+ "r5e817e.floci.example.com-logs.localhost, localhost, r5e817e.floci.example.com-logs",+ "one.dot-nonexistent.localhost, localhost, one.dot-nonexistent",+ "a.b.c-nonexistent.localhost, localhost, a.b.c-nonexistent",+ // Dotted bucket against a multi-label configured base hostname+ "www.example.com.floci.internal, floci.internal, www.example.com",+ "www.example.com.floci.default.svc.cluster.local, floci.default.svc.cluster.local, www.example.com",+ // Dotted bucket, region-qualified vhost form+ "www.example.com.s3.us-east-1.localhost, localhost, www.example.com",+ "www.example.com.s3.us-east-1.localhost:4566, localhost, www.example.com",+ "www.example.com.s3.amazonaws.com, localhost, www.example.com",+ "www.example.com.s3.us-east-1.amazonaws.com, localhost, www.example.com",+ "www.example.com.s3.dualstack.us-east-1.amazonaws.com, localhost, www.example.com",+ // Dotted bucket on the wildcard-DNS builtins+ "www.example.com.localhost.floci.io, localhost, www.example.com",+ "www.example.com.s3.localhost.floci.io, localhost, www.example.com",+ "www.example.com.s3.us-east-1.localhost.localstack.cloud, localhost, www.example.com",+ // Dotted bucket on a website endpoint+ "www.example.com.s3-website-us-east-1.localhost, localhost, www.example.com",+ "www.example.com.s3-website.localhost, localhost, www.example.com",+ // Case-insensitive suffix matching keeps the bucket's own case+ "www.Example.com.LOCALHOST, localhost, www.Example.com",+ })+ void extractsDottedBucketNames(String host, String baseHostname, String expectedBucket) {+ assertEquals(expectedBucket, S3VirtualHostFilter.extractBucket(host, baseHostname, DEFAULT_SUFFIXES));+ }++ @Test+ void headOnDottedBucketIsNotDegradedToPathStyle() {+ // Regression: extractBucket split at the FIRST dot, so a dotted bucket's remainder+ // ("floci.example.com-logs.localhost") did not match baseHostname "localhost" and the+ // request degraded to path-style "/" — where HEAD answers 200. The Terraform AWS+ // provider HEADs a bucket before creating it and read that 200 as "already exists",+ // failing with BucketAlreadyExists for a bucket that never existed.+ assertEquals("floci.example.com-logs", S3VirtualHostFilter.extractBucket(+ "floci.example.com-logs.localhost", "localhost", DEFAULT_SUFFIXES));+ assertEquals("floci.example.com-logs", S3VirtualHostFilter.extractBucket(+ "floci.example.com-logs.localhost:4566", "localhost", DEFAULT_SUFFIXES));+ }++ // --- The service-endpoint reading always wins over the new suffix matching ---++ @ParameterizedTest+ @CsvSource({+ // s3.<endpoint> is Floci's own S3 endpoint, never a bucket named "s3"+ "s3.localhost, localhost",+ "s3.localhost:4566, localhost",+ "s3.floci.internal, floci.internal",+ // ...including the region-qualified service endpoint, which the guard alone misses+ "s3.us-east-1.localhost, localhost",+ "s3.us-east-1.localhost:4566, localhost",+ "s3.eu-west-2.localhost.floci.io, localhost",+ "s3.dualstack.us-east-1.amazonaws.com, localhost",+ "s3.amazonaws.com, localhost",+ "s3.us-east-1.amazonaws.com, localhost",+ // Bare website endpoint with no bucket in front+ "s3-website-us-east-1.localhost, localhost",+ // A non-S3 amazonaws.com host is not a bucket+ "foo.amazonaws.com, localhost",+ "queue.amazonaws.com:443, localhost",+ // Dotted hosts that still end in nothing we know+ "www.example.com.custom.internal, localhost",+ "a.b.c.d, localhost",+ })+ void serviceEndpointsAndUnknownSuffixesStayBucketless(String host, String baseHostname) {+ assertNull(S3VirtualHostFilter.extractBucket(host, baseHostname, DEFAULT_SUFFIXES));+ }++ @Test+ void longestKnownSuffixWinsSoBucketDoesNotAbsorbTheEndpoint() {+ // "localhost" and "localhost.floci.io" are both known; the longer one must match,+ // otherwise the bucket would come out as "my-bucket.localhost".+ assertEquals("my-bucket", S3VirtualHostFilter.extractBucket(+ "my-bucket.localhost.floci.io", "localhost", DEFAULT_SUFFIXES));+ assertEquals("www.example.com", S3VirtualHostFilter.extractBucket(+ "www.example.com.localhost.localstack.cloud", "localhost", DEFAULT_SUFFIXES));+ }++ @Test+ void dottedBucketsRouteOnConfiguredExtraSuffixes() {+ Set<String> withExtra = Set.of(+ "localhost", "localhost.floci.io", "localhost.localstack.cloud", "localhost.example.internal");+ assertEquals("www.example.com", S3VirtualHostFilter.extractBucket(+ "www.example.com.localhost.example.internal", "localhost", withExtra));+ assertEquals("www.example.com", S3VirtualHostFilter.extractBucket(+ "www.example.com.s3.localhost.example.internal:4566", "localhost", withExtra));+ assertEquals("www.example.com", S3VirtualHostFilter.extractBucket(+ "www.example.com.s3.us-east-1.localhost.example.internal", "localhost", withExtra));+ // Still bucketless for the bare service endpoint on that suffix+ assertNull(S3VirtualHostFilter.extractBucket(+ "s3.us-east-1.localhost.example.internal", "localhost", withExtra));+ // And no accidental match when the suffix is not configured+ assertNull(S3VirtualHostFilter.extractBucket(+ "www.example.com.localhost.example.internal", "localhost", DEFAULT_SUFFIXES));+ }++ @Test+ void ipv4AuthorityIsNeverABucketEvenWithADottedShape() {+ assertNull(S3VirtualHostFilter.extractBucket("192.168.1.1", "localhost", DEFAULT_SUFFIXES));+ assertNull(S3VirtualHostFilter.extractBucket("127.0.0.1:4566", "localhost", DEFAULT_SUFFIXES));+ }++ @Test+ void filterRewritesPathForADottedBucket() {+ URI requestUri = URI.create("http://www.example.com.localhost:4566/index.html");+ UriInfo uriInfo = mock(UriInfo.class);+ when(uriInfo.getRequestUri()).thenReturn(requestUri);+ ContainerRequestContext ctx = mock(ContainerRequestContext.class);+ when(ctx.getUriInfo()).thenReturn(uriInfo);+ when(ctx.getHeaderString("Host")).thenReturn("www.example.com.localhost:4566");++ new S3VirtualHostFilter().filter(ctx);++ ArgumentCaptor<URI> rewritten = ArgumentCaptor.forClass(URI.class);+ verify(ctx).setRequestUri(rewritten.capture());+ assertEquals("/www.example.com/index.html", rewritten.getValue().getRawPath());+ }++ // --- Other services' virtual-host schemes must not be swallowed as dotted buckets ---++ @ParameterizedTest+ @CsvSource({+ // API Gateway: <api-id>.execute-api.<region>.<endpoint> and the wildcard-DNS forms+ "abc123.execute-api.localhost, localhost",+ "abc123.execute-api.localhost:4566, localhost",+ "abc123.execute-api.ap-northeast-2.localhost:4566, localhost",+ "abc123.execute-api.localhost.floci.io, localhost",+ "abc123.execute-api.localhost.localstack.cloud, localhost",+ "AbC123.execute-api.localhost.floci.io, localhost",+ "abc123.execute-api.us-east-1.amazonaws.com, localhost",+ // Lambda function URLs+ "url-id.lambda-url.us-east-1.localhost, localhost",+ "url-id.lambda-url.localhost:4566, localhost",+ // CloudFront distribution domains routed at the Floci endpoint+ "e123.cloudfront.localhost, localhost",+ })+ void otherServicesVirtualHostsAreNotBuckets(String host, String baseHostname) {+ assertNull(S3VirtualHostFilter.extractBucket(host, baseHostname, DEFAULT_SUFFIXES));+ }++ @Test+ void serviceLabelInFirstPositionKeepsItsPreExistingBucketReading() {+ // emr-serverless.<endpoint> has no id in front; it was read as a bucket before the+ // dotted-bucket fix and still is, so this change alters nothing for it.+ assertEquals("emr-serverless",+ S3VirtualHostFilter.extractBucket("emr-serverless.localhost", "localhost", DEFAULT_SUFFIXES));+ }++ /**+ * Every regional AWS service virtual-hosts as {@code <id>.<service>.<region>.<endpoint>}, and+ * suffix matching reads all of them as a dotted bucket unless something stops it. A label+ * denylist cannot: it has to name each service, and the first one it misses is silently served+ * as an S3 bucket. The structural rule is that a real S3 virtual host never puts a bare region+ * label against the endpoint host — {@code s3}, {@code s3.<region>} or {@code s3-website-<region>}+ * is always there instead.+ *+ * <p>{@code <account>.dkr.ecr.<region>.localhost} is not hypothetical: it is the registry URI+ * {@code EcrRegistryManager} hands out.+ */+ @ParameterizedTest+ @CsvSource({+ // ECR registry, as EcrRegistryManager emits it+ "123456789012.dkr.ecr.us-east-1.localhost, localhost",+ "123456789012.dkr.ecr.us-east-1.localhost:5100, localhost",+ "123456789012.dkr.ecr.eu-central-1.localhost.floci.io, localhost",+ // OpenSearch / Elasticsearch domain endpoints+ "search-mydomain-abc.us-east-1.es.localhost, localhost",+ // IoT data endpoints+ "abc123.iot.us-east-1.localhost, localhost",+ // AppSync+ "myapp.appsync-api.us-east-1.localhost, localhost",+ // Cognito hosted UI+ "mydomain.auth.eu-west-2.localhost, localhost",+ // A service Floci does not model yet still must not be swallowed+ "anything.some-future-service.ap-northeast-2.localhost, localhost",+ })+ void regionalServiceVirtualHostsAreNotDottedBuckets(String host, String baseHostname) {+ assertNull(S3VirtualHostFilter.extractBucket(host, baseHostname, DEFAULT_SUFFIXES));+ }++ /**+ * The other direction: the guard must not cost legal bucket names. {@code my.elb.logs} is+ * AWS's own load-balancer access-log naming convention, and no filter in this repository+ * routes {@code *.elb.<endpoint>} or {@code *.mwaa.<endpoint>} by Host, so neither label+ * belongs in the denylist.+ */+ @ParameterizedTest+ @CsvSource({+ "my.elb.logs.localhost, my.elb.logs",+ "my.elb.logs.localhost:4566, my.elb.logs",+ "airflow.mwaa.dags.localhost, airflow.mwaa.dags",+ "my.elb.logs.s3.us-east-1.localhost, my.elb.logs",+ })+ void bucketNamesCarryingServiceWordsStillResolve(String host, String expectedBucket) {+ assertEquals(expectedBucket, S3VirtualHostFilter.extractBucket(host, "localhost", DEFAULT_SUFFIXES));+ }++ /**+ * The region rule only applies when no S3 qualifier is present, so a bucket whose own name+ * ends in a region label is still addressable in the qualified forms.+ */+ @Test+ void aBucketWhoseNameEndsInARegionLabelIsStillAddressableWhenQualified() {+ assertEquals("logs.us-east-1", S3VirtualHostFilter.extractBucket(+ "logs.us-east-1.s3.localhost", "localhost", DEFAULT_SUFFIXES));+ assertEquals("logs.us-east-1", S3VirtualHostFilter.extractBucket(+ "logs.us-east-1.s3.us-east-1.amazonaws.com", "localhost", DEFAULT_SUFFIXES));+ // Unqualified, it reads as another service's regional host — the documented trade-off.+ assertNull(S3VirtualHostFilter.extractBucket(+ "logs.us-east-1.localhost", "localhost", DEFAULT_SUFFIXES));+ }++ /** A single-label bucket is unaffected by the region rule, region-shaped or not. */+ @Test+ void singleLabelBucketsAreUntouchedByTheRegionRule() {+ assertEquals("us-east-1", S3VirtualHostFilter.extractBucket(+ "us-east-1.localhost", "localhost", DEFAULT_SUFFIXES));+ assertEquals("my-bucket", S3VirtualHostFilter.extractBucket(+ "my-bucket.localhost", "localhost", DEFAULT_SUFFIXES));+ }++ /**+ * The cost of the four remaining denylist labels, pinned rather than left implicit: a bucket+ * whose name carries one of them in a non-leading position is not reachable virtual-hosted and+ * falls back to path-style, which is what it did before this change too.+ *+ * <p>That is the deliberate side of the trade. The alternative failure — validating each+ * service's hostname grammar and letting anything that does not match through — is a silent+ * cross-service hijack: S3 answering a request meant for API Gateway. A bucket that is+ * awkward to address virtual-hosted is recoverable by the client; an API that intermittently+ * returns S3 responses is not. Each label here names a filter in this repository that claims+ * that hostname, so the set stays small and justified.+ */+ @ParameterizedTest+ @CsvSource({+ "my.execute-api.archive.localhost, localhost",+ "my.lambda-url.archive.localhost, localhost",+ "my.emr-serverless.archive.localhost, localhost",+ "my.cloudfront.archive.localhost, localhost",+ })+ void bucketNamesCarryingADenylistedServiceLabelAreDeliberatelyNotReachable(String host, String baseHostname) {+ assertNull(S3VirtualHostFilter.extractBucket(host, baseHostname, DEFAULT_SUFFIXES));+ }++ // --- Regression: an internal "s3" label must not truncate the bucket ---++ /**+ * A qualifier is a complete tail, not a label that may appear anywhere.+ *+ * <p>Scanning labels one at a time and splitting at the last {@code s3} reads+ * {@code my.s3.archive.localhost} as the bucket {@code my}: a request for+ * {@code my.s3.archive} silently reads and writes a <em>different, existing</em> bucket.+ * That is cross-bucket misrouting — strictly worse than the wrong 200 this filter was+ * written to fix, because the client gets a plausible success against the wrong data.+ *+ * <p>Nothing legal follows {@code s3} in an endpoint host except a region,+ * {@code dualstack.<region>}, or the end of the name, so an {@code s3} label followed by an+ * arbitrary label is part of the bucket.+ */+ @ParameterizedTest+ @CsvSource({+ "my.s3.archive.localhost, my.s3.archive",+ "my.s3.archive.localhost:4566, my.s3.archive",+ "www.s3.example.com.localhost, www.s3.example.com",+ "my.s3.archive.localhost.floci.io, my.s3.archive",+ // s3-website is a qualifier head too, and gets the same treatment+ "data.s3-website.archive.localhost, data.s3-website.archive",+ // ...and the region-shaped tail is not a region id, so it is bucket text as well+ "my.s3.not-a-region.localhost, my.s3.not-a-region",+ })+ void anInternalS3LabelDoesNotTruncateTheBucket(String host, String expectedBucket) {+ assertEquals(expectedBucket, S3VirtualHostFilter.extractBucket(host, "localhost", DEFAULT_SUFFIXES));+ }++ /** The same host, stated as the misrouting it would otherwise be. */+ @Test+ void aDottedBucketContainingS3IsNeverResolvedToAShorterBucket() {+ String bucket = S3VirtualHostFilter.extractBucket(+ "my.s3.archive.localhost", "localhost", DEFAULT_SUFFIXES);+ assertNotEquals("my", bucket, "bucket my.s3.archive must not resolve to the bucket my");+ assertEquals("my.s3.archive", bucket);+ }++ /**+ * A real qualifier still splits, including for a dotted bucket that itself contains an+ * {@code s3} label — {@code my.s3.archive} addressed region-qualified.+ */+ @ParameterizedTest+ @CsvSource({+ "my.s3.localhost, my",+ "my.s3.us-east-1.localhost, my",+ "my.s3.archive.s3.localhost, my.s3.archive",+ "my.s3.archive.s3.us-east-1.localhost, my.s3.archive",+ "my.s3.archive.s3.dualstack.eu-west-1.localhost, my.s3.archive",+ })+ void aCompleteQualifierTailStillSplitsTheBucket(String host, String expectedBucket) {+ assertEquals(expectedBucket, S3VirtualHostFilter.extractBucket(host, "localhost", DEFAULT_SUFFIXES));+ }++ /**+ * The published S3 endpoint qualifier forms, from the Amazon S3 endpoint tables in the AWS+ * General Reference. The website endpoint spells its region with a dot in newer regions and a+ * dash in older ones, and both are in service, so both are accepted.+ */+ @ParameterizedTest+ @CsvSource({+ "www.example.com.s3-website.eu-west-2.localhost, www.example.com",+ "www.example.com.s3-website-eu-west-1.localhost, www.example.com",+ "my.bucket.s3-fips.us-east-1.localhost, my.bucket",+ "my.bucket.s3-fips.dualstack.us-east-1.amazonaws.com, my.bucket",+ "my.bucket.s3-accelerate.amazonaws.com, my.bucket",+ "my.bucket.s3-accelerate.dualstack.amazonaws.com, my.bucket",+ "my.bucket.s3-us-west-2.amazonaws.com, my.bucket",+ "my.bucket.s3.dualstack.ap-northeast-3.amazonaws.com, my.bucket",+ })+ void everyPublishedQualifierFormResolvesTheBucket(String host, String expectedBucket) {+ assertEquals(expectedBucket, S3VirtualHostFilter.extractBucket(host, "localhost", DEFAULT_SUFFIXES));+ }++ // --- Regression: the region guard must reject region IDS, not region SHAPES ---++ /**+ * {@code [a-z]{2}-[a-z-]+-\d+} is the shape of an AWS region id, but it is also the shape of+ * perfectly ordinary bucket labels. Matching by shape rejected {@code data.my-cd-1} — a legal+ * bucket — dropped it back to path-style, and restored the exact false "bucket exists" 200+ * this filter exists to prevent. The guard has to test against the finite list of real region+ * ids instead.+ */+ @ParameterizedTest+ @CsvSource({+ "data.my-cd-1.localhost, data.my-cd-1",+ "data.my-cd-1.localhost:4566, data.my-cd-1",+ "reports.eu-team-2.localhost, reports.eu-team-2",+ "assets.us-west-9.localhost, assets.us-west-9",+ "logs.ap-corp-1.archive.localhost, logs.ap-corp-1.archive",+ "backup.no-such-region-12.localhost, backup.no-such-region-12",+ })+ void regionShapedBucketLabelsThatAreNotRegionIdsStillResolve(String host, String expectedBucket) {+ assertEquals(expectedBucket, S3VirtualHostFilter.extractBucket(host, "localhost", DEFAULT_SUFFIXES));+ }++ /**+ * The guard it replaces is still doing its job: a real region id in a non-leading label, with+ * no S3 qualifier, is another service's regional virtual host. Tightening to real ids must not+ * weaken this — including for regions outside the set the emulator advertises.+ */+ @ParameterizedTest+ @CsvSource({+ "123456789012.dkr.ecr.us-east-1.localhost, localhost",+ "search-x.eu-north-1.es.localhost, localhost",+ "abc.iot.me-central-1.localhost, localhost",+ "abc.transfer.us-gov-west-1.localhost, localhost",+ "abc.service.cn-northwest-1.localhost, localhost",+ })+ void realRegionIdsStillMarkAnotherServicesRegionalHost(String host, String baseHostname) {+ assertNull(S3VirtualHostFilter.extractBucket(host, baseHostname, DEFAULT_SUFFIXES));+ }++ /**+ * The residual cost of the region rule, pinned rather than left implicit: a bucket whose name+ * carries a <em>real</em> region id in a non-leading position is still not reachable in the+ * unqualified virtual-hosted form. It stays reachable qualified and path-style, and it was+ * equally unreachable before this filter existed — so the cost is a recoverable one, paid to+ * avoid an unrecoverable cross-service hijack.+ */+ @Test+ void aBucketNamedAfterARealRegionIsStillTheDocumentedCost() {+ assertNull(S3VirtualHostFilter.extractBucket(+ "data.us-east-1.localhost", "localhost", DEFAULT_SUFFIXES));+ assertEquals("data.us-east-1", S3VirtualHostFilter.extractBucket(+ "data.us-east-1.s3.localhost", "localhost", DEFAULT_SUFFIXES));+ }}