Light, fluffy, and always free - The AWS Local Emulator alternative
fix(codeartifact): harden the pypi upload preflight's URI building and index match (#5029)
Priamo Germosen8776c5fc3ac5
diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.javaindex 3f129ca385..7c2263bb36 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java@@ -29,6 +29,7 @@ import java.util.Arrays;import java.util.Locale;import java.util.Optional;import java.util.concurrent.ConcurrentHashMap;+import java.util.regex.Pattern;/*** Serves the real PyPI simple-repository wire protocol behind the URL@@ -64,6 +65,9 @@ public class CodeArtifactPypiController {private static final String SIMPLE_PATH = "/simple/";+ /** Matches an {@code href="..."} attribute in pypiserver's own simple-index HTML. */+ private static final Pattern SIMPLE_INDEX_HREF = Pattern.compile("href=\"([^\"]*)\"");+private final CodeArtifactService service;private final PypiserverSidecarManager pypiserver;private final HttpClient httpClient;@@ -103,7 +107,8 @@ public class CodeArtifactPypiController {@PathParam("repository") String repository,@PathParam("package") String packageName) {RepositoryHandle handle = requireRepository(headers, domain, repository);- Response response = proxyGet(handle.baseUrl(), SIMPLE_PATH + packageName + "/");+ Response response = proxyGet(handle.baseUrl(), SIMPLE_PATH + SidecarUriUtils.encodeSegment(packageName)+ + "/");if (response.getStatus() != 200) {return response;}@@ -120,7 +125,7 @@ public class CodeArtifactPypiController {@PathParam("repository") String repository,@PathParam("filename") String filename) {String baseUrl = requireRepository(headers, domain, repository).baseUrl();- return proxyGet(baseUrl, "/packages/" + filename);+ return proxyGet(baseUrl, "/packages/" + SidecarUriUtils.encodeSegment(filename));}/**@@ -191,11 +196,18 @@ public class CodeArtifactPypiController {* check couldn't complete" the same as "nothing published yet" would let an upload with* different content through to pypiserver (which has no conflict check of its own) during* exactly the moment this proxy's own protection is unavailable, the opposite of fail-safe.+ *+ * <p>{@code packageName} and {@code filename} are percent-encoded via {@link SidecarUriUtils#+ * encodeSegment} before being placed in either request path: without that, a {@code /} in+ * either one would be indistinguishable from a real path separator, and a {@code #} or+ * {@code ?} would be reinterpreted by {@link URI#create} as a fragment or query string,+ * silently fetching a different path than the one actually requested.*/private Optional<byte[]> existingFileContent(String baseUrl, String packageName, String filename) {String normalized = normalizePackageName(packageName);- HttpRequest indexRequest = HttpRequest.newBuilder(URI.create(baseUrl + SIMPLE_PATH + normalized + "/"))- .timeout(Duration.ofSeconds(5)).GET().build();+ URI indexUri = SidecarUriUtils.combine(URI.create(baseUrl),+ SIMPLE_PATH + SidecarUriUtils.encodeSegment(normalized) + "/");+ HttpRequest indexRequest = HttpRequest.newBuilder(indexUri).timeout(Duration.ofSeconds(5)).GET().build();HttpResponse<String> indexResponse;try {indexResponse = httpClient.send(indexRequest, BodyHandlers.ofString());@@ -212,12 +224,13 @@ public class CodeArtifactPypiController {"Could not check pypi package " + packageName + " for an existing " + filename+ ": upstream returned " + indexResponse.statusCode());}- if (!indexResponse.body().contains(filename)) {+ if (!indexListsFilename(indexResponse.body(), filename)) {return Optional.empty();}- HttpRequest fileRequest = HttpRequest.newBuilder(URI.create(baseUrl + "/packages/" + filename))- .timeout(Duration.ofSeconds(10)).GET().build();+ URI fileUri = SidecarUriUtils.combine(URI.create(baseUrl), "/packages/"+ + SidecarUriUtils.encodeSegment(filename));+ HttpRequest fileRequest = HttpRequest.newBuilder(fileUri).timeout(Duration.ofSeconds(10)).GET().build();HttpResponse<byte[]> fileResponse;try {fileResponse = httpClient.send(fileRequest, BodyHandlers.ofByteArray());@@ -237,6 +250,22 @@ public class CodeArtifactPypiController {return name.toLowerCase(Locale.ROOT).replaceAll("[-_.]+", "-");}+ /**+ * An exact entry match, not a substring search: a shorter real filename that happens to be a+ * substring of a different, longer real filename (or of surrounding markup) must not pass+ * this check.+ */+ private static boolean indexListsFilename(String indexBody, String filename) {+ return SIMPLE_INDEX_HREF.matcher(indexBody).results()+ .map(match -> {+ String href = match.group(1);+ String name = href.substring(href.lastIndexOf('/') + 1);+ int fragment = name.indexOf('#');+ return fragment < 0 ? name : name.substring(0, fragment);+ })+ .anyMatch(filename::equals);+ }+private Response forwardUpload(String baseUrl, String contentType, byte[] content) {HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + "/")).timeout(Duration.ofSeconds(30))@@ -252,14 +281,20 @@ public class CodeArtifactPypiController {return Response.status(response.statusCode()).build();}- private Response proxyGet(String baseUrl, String path) {- HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + path))+ /**+ * {@code encodedPath} must already be fully percent-encoded, built from {@link+ * SidecarUriUtils#encodeSegment}-encoded, caller-supplied segments: see {@link+ * SidecarUriUtils#combine} for why a raw {@code URI.create(baseUrl + path)} concatenation+ * isn't safe here.+ */+ private Response proxyGet(String baseUrl, String encodedPath) {+ HttpRequest request = HttpRequest.newBuilder(SidecarUriUtils.combine(URI.create(baseUrl), encodedPath)).timeout(Duration.ofSeconds(10)).GET().build();HttpResponse<byte[]> response;try {response = httpClient.send(request, BodyHandlers.ofByteArray());} catch (Exception e) {- throw new IllegalStateException("Could not reach the pypi sidecar at " + path, e);+ throw new IllegalStateException("Could not reach the pypi sidecar at " + encodedPath, e);}if (response.statusCode() == 404) {return Response.status(404).build();diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiControllerTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiControllerTest.javaindex 866d144a91..f33d942b3e 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiControllerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiControllerTest.java@@ -133,6 +133,67 @@ class CodeArtifactPypiControllerTest {assertEquals(404, response.getStatus());}+ /**+ * The fix for a real finding: building the request URI with a plain string concatenation+ * would let a {@code #} in a caller-supplied filename get interpreted as a URI fragment and+ * silently dropped from the actual wire request, fetching a different, shorter real filename+ * instead of the one actually requested. Checks the raw path the backend actually receives,+ * not just the response, via a root fallback context rather than registering the context at+ * the encoded literal path: {@code com.sun.net.httpserver.HttpServer} matches contexts on the+ * decoded path, so registering one at the raw percent-encoded string wouldn't reliably match.+ */+ @Test+ void downloadFileTreatsAHashInTheFilenameAsLiteralNotAUriFragment() {+ AtomicReference<String> observedRawPath = new AtomicReference<>();+ backend.createContext("/", exchange -> {+ observedRawPath.set(exchange.getRequestURI().getRawPath());+ respond(exchange, 404, "text/plain", "");+ });++ Response response = controller.downloadFile(headers("Bearer good"), DOMAIN, REPOSITORY, "pkg-1.0.jar#forged");++ assertEquals(404, response.getStatus());+ assertEquals("/packages/pkg-1.0.jar%23forged", observedRawPath.get());+ }++ /**+ * Same finding, the path-separator half of it: a {@code /} in a caller-supplied filename+ * would be indistinguishable from a real path separator and could splice in extra path+ * segments.+ */+ @Test+ void downloadFileTreatsASlashInTheFilenameAsLiteralNotAStructuralSeparator() {+ AtomicReference<String> observedRawPath = new AtomicReference<>();+ backend.createContext("/", exchange -> {+ observedRawPath.set(exchange.getRequestURI().getRawPath());+ respond(exchange, 404, "text/plain", "");+ });++ Response response = controller.downloadFile(headers("Bearer good"), DOMAIN, REPOSITORY, "../secret.tar.gz");++ assertEquals(404, response.getStatus());+ assertEquals("/packages/..%2Fsecret.tar.gz", observedRawPath.get());+ }++ /**+ * Same finding again, via the other wire-protocol route that shares the same {@code proxyGet}+ * helper: the package name in {@code GET /simple/{package}/} is just as caller-supplied as+ * the filename in {@code GET /packages/{filename}}.+ */+ @Test+ void packageIndexTreatsASlashInThePackageNameAsLiteralNotAStructuralSeparator() {+ AtomicReference<String> observedRawPath = new AtomicReference<>();+ backend.createContext("/", exchange -> {+ observedRawPath.set(exchange.getRequestURI().getRawPath());+ respond(exchange, 404, "text/plain", "");+ });++ Response response = controller.packageIndex(headers("Bearer good"), DOMAIN, REPOSITORY, "../other-pkg");++ assertEquals(404, response.getStatus());+ assertEquals("/simple/..%2Fother-pkg/", observedRawPath.get());+ }+@Testvoid uploadRejectsABodyWithNoDeterminableBoundary() {Response response = controller.upload(headers("Bearer good", "application/octet-stream"), DOMAIN, REPOSITORY,@@ -171,6 +232,85 @@ class CodeArtifactPypiControllerTest {assertEquals(false, uploadReceived.get());}+ /**+ * The fix for a real finding: a plain substring search on the whole index response body+ * would wrongly treat the uploaded filename as already published if it happened to be a+ * substring of a different, longer real entry, e.g. a detached {@code .asc} signature file+ * whose own name contains the base file's name as a prefix.+ */+ @Test+ void uploadDoesNotTreatAShorterFilenameAsAlreadyListedWhenItIsOnlyASubstringOfALongerOne() {+ backend.createContext("/simple/my-pkg/", exchange -> respond(exchange, 200, "text/html",+ "<a href=\"my_pkg-1.0.0.tar.gz.asc\">my_pkg-1.0.0.tar.gz.asc</a>"));+ AtomicReference<Boolean> uploadReceived = new AtomicReference<>(false);+ backend.createContext("/", exchange -> {+ if ("POST".equals(exchange.getRequestMethod())) {+ uploadReceived.set(true);+ }+ respond(exchange, 200, "text/plain", "");+ });+ byte[] body = multipartWithFile("boundary123", "my-pkg", "my_pkg-1.0.0.tar.gz", "new content");++ Response response = controller.upload(+ headers("Bearer good", "multipart/form-data; boundary=boundary123"), DOMAIN, REPOSITORY, body);++ assertEquals(200, response.getStatus());+ assertEquals(true, uploadReceived.get());+ }++ /**+ * The fix for a real finding: the upload preflight's own index-check request (as opposed to+ * {@code packageIndex}'s, already covered above) built its path the same unsafe way, and+ * nothing exercised it with a reserved character.+ */+ @Test+ void uploadExistenceCheckTreatsAHashInThePackageNameAsLiteralNotAUriFragment() {+ AtomicReference<String> observedRawPath = new AtomicReference<>();+ backend.createContext("/", exchange -> {+ if ("GET".equals(exchange.getRequestMethod())) {+ observedRawPath.set(exchange.getRequestURI().getRawPath());+ respond(exchange, 404, "text/plain", "");+ } else {+ respond(exchange, 200, "text/plain", "");+ }+ });+ byte[] body = multipartWithFile("boundary123", "my#pkg", "my_pkg-1.0.0.tar.gz", "content");++ Response response = controller.upload(+ headers("Bearer good", "multipart/form-data; boundary=boundary123"), DOMAIN, REPOSITORY, body);++ assertEquals(200, response.getStatus());+ assertEquals("/simple/my%23pkg/", observedRawPath.get());+ }++ /**+ * Same finding, the preflight's other request: once the simple index confirms a match, it+ * fetches the existing file's bytes for the content comparison, at a path built from the+ * same caller-supplied filename. Uses {@code ?}, not {@code #} or {@code /}: both of those+ * are already meaningful to the index's own href parsing (a root-relative directory prefix+ * and a hash-fragment suffix, both stripped the way a real pypiserver response uses them),+ * so a filename containing either could never round-trip through the exact-match check at+ * all, real bug or not; {@code ?} isn't special to that parsing, only to building the URI.+ */+ @Test+ void uploadExistenceCheckTreatsAQuestionMarkInTheFilenameAsLiteralNotAUriQueryStringWhenFetchingTheExistingFile() {+ String filename = "my_pkg-1.0.0?sig.tar.gz";+ backend.createContext("/simple/my-pkg/", exchange -> respond(exchange, 200, "text/html",+ "<a href=\"" + filename + "\">" + filename + "</a>"));+ AtomicReference<String> observedRawPath = new AtomicReference<>();+ backend.createContext("/", exchange -> {+ observedRawPath.set(exchange.getRequestURI().getRawPath());+ respond(exchange, 200, "application/octet-stream", "different content");+ });+ byte[] body = multipartWithFile("boundary123", "my-pkg", filename, "new content");++ Response response = controller.upload(+ headers("Bearer good", "multipart/form-data; boundary=boundary123"), DOMAIN, REPOSITORY, body);++ assertEquals(409, response.getStatus());+ assertEquals("/packages/my_pkg-1.0.0%3Fsig.tar.gz", observedRawPath.get());+ }+@Testvoid uploadOfByteIdenticalContentSucceedsIdempotentlyWithoutForwardingToTheBackend() {String sameContent = "same content";