Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(codeartifact): harden the pypi upload preflight's URI building and index match (#5029)

Priamo Germosen8776c5fc3ac5
diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java
index 3f129ca385..7c2263bb36 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java
@@ -29,6 +29,7 @@ import java.util.Arrays;
import java.util.Locale;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
/**
* Serves the real PyPI simple-repository wire protocol behind the URL
@@ -64,6 +65,9 @@ public class CodeArtifactPypiController {
private static final String SIMPLE_PATH = "/simple/";
+ /** Matches an {@code href="..."} attribute in pypiserver's own simple-index HTML. */
+ private static final Pattern SIMPLE_INDEX_HREF = Pattern.compile("href=\"([^\"]*)\"");
+
private final CodeArtifactService service;
private final PypiserverSidecarManager pypiserver;
private final HttpClient httpClient;
@@ -103,7 +107,8 @@ public class CodeArtifactPypiController {
@PathParam("repository") String repository,
@PathParam("package") String packageName) {
RepositoryHandle handle = requireRepository(headers, domain, repository);
- Response response = proxyGet(handle.baseUrl(), SIMPLE_PATH + packageName + "/");
+ Response response = proxyGet(handle.baseUrl(), SIMPLE_PATH + SidecarUriUtils.encodeSegment(packageName)
+ + "/");
if (response.getStatus() != 200) {
return response;
}
@@ -120,7 +125,7 @@ public class CodeArtifactPypiController {
@PathParam("repository") String repository,
@PathParam("filename") String filename) {
String baseUrl = requireRepository(headers, domain, repository).baseUrl();
- return proxyGet(baseUrl, "/packages/" + filename);
+ return proxyGet(baseUrl, "/packages/" + SidecarUriUtils.encodeSegment(filename));
}
/**
@@ -191,11 +196,18 @@ public class CodeArtifactPypiController {
* check couldn't complete" the same as "nothing published yet" would let an upload with
* different content through to pypiserver (which has no conflict check of its own) during
* exactly the moment this proxy's own protection is unavailable, the opposite of fail-safe.
+ *
+ * <p>{@code packageName} and {@code filename} are percent-encoded via {@link SidecarUriUtils#
+ * encodeSegment} before being placed in either request path: without that, a {@code /} in
+ * either one would be indistinguishable from a real path separator, and a {@code #} or
+ * {@code ?} would be reinterpreted by {@link URI#create} as a fragment or query string,
+ * silently fetching a different path than the one actually requested.
*/
private Optional<byte[]> existingFileContent(String baseUrl, String packageName, String filename) {
String normalized = normalizePackageName(packageName);
- HttpRequest indexRequest = HttpRequest.newBuilder(URI.create(baseUrl + SIMPLE_PATH + normalized + "/"))
- .timeout(Duration.ofSeconds(5)).GET().build();
+ URI indexUri = SidecarUriUtils.combine(URI.create(baseUrl),
+ SIMPLE_PATH + SidecarUriUtils.encodeSegment(normalized) + "/");
+ HttpRequest indexRequest = HttpRequest.newBuilder(indexUri).timeout(Duration.ofSeconds(5)).GET().build();
HttpResponse<String> indexResponse;
try {
indexResponse = httpClient.send(indexRequest, BodyHandlers.ofString());
@@ -212,12 +224,13 @@ public class CodeArtifactPypiController {
"Could not check pypi package " + packageName + " for an existing " + filename
+ ": upstream returned " + indexResponse.statusCode());
}
- if (!indexResponse.body().contains(filename)) {
+ if (!indexListsFilename(indexResponse.body(), filename)) {
return Optional.empty();
}
- HttpRequest fileRequest = HttpRequest.newBuilder(URI.create(baseUrl + "/packages/" + filename))
- .timeout(Duration.ofSeconds(10)).GET().build();
+ URI fileUri = SidecarUriUtils.combine(URI.create(baseUrl), "/packages/"
+ + SidecarUriUtils.encodeSegment(filename));
+ HttpRequest fileRequest = HttpRequest.newBuilder(fileUri).timeout(Duration.ofSeconds(10)).GET().build();
HttpResponse<byte[]> fileResponse;
try {
fileResponse = httpClient.send(fileRequest, BodyHandlers.ofByteArray());
@@ -237,6 +250,22 @@ public class CodeArtifactPypiController {
return name.toLowerCase(Locale.ROOT).replaceAll("[-_.]+", "-");
}
+ /**
+ * An exact entry match, not a substring search: a shorter real filename that happens to be a
+ * substring of a different, longer real filename (or of surrounding markup) must not pass
+ * this check.
+ */
+ private static boolean indexListsFilename(String indexBody, String filename) {
+ return SIMPLE_INDEX_HREF.matcher(indexBody).results()
+ .map(match -> {
+ String href = match.group(1);
+ String name = href.substring(href.lastIndexOf('/') + 1);
+ int fragment = name.indexOf('#');
+ return fragment < 0 ? name : name.substring(0, fragment);
+ })
+ .anyMatch(filename::equals);
+ }
+
private Response forwardUpload(String baseUrl, String contentType, byte[] content) {
HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + "/"))
.timeout(Duration.ofSeconds(30))
@@ -252,14 +281,20 @@ public class CodeArtifactPypiController {
return Response.status(response.statusCode()).build();
}
- private Response proxyGet(String baseUrl, String path) {
- HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + path))
+ /**
+ * {@code encodedPath} must already be fully percent-encoded, built from {@link
+ * SidecarUriUtils#encodeSegment}-encoded, caller-supplied segments: see {@link
+ * SidecarUriUtils#combine} for why a raw {@code URI.create(baseUrl + path)} concatenation
+ * isn't safe here.
+ */
+ private Response proxyGet(String baseUrl, String encodedPath) {
+ HttpRequest request = HttpRequest.newBuilder(SidecarUriUtils.combine(URI.create(baseUrl), encodedPath))
.timeout(Duration.ofSeconds(10)).GET().build();
HttpResponse<byte[]> response;
try {
response = httpClient.send(request, BodyHandlers.ofByteArray());
} catch (Exception e) {
- throw new IllegalStateException("Could not reach the pypi sidecar at " + path, e);
+ throw new IllegalStateException("Could not reach the pypi sidecar at " + encodedPath, e);
}
if (response.statusCode() == 404) {
return Response.status(404).build();
diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiControllerTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiControllerTest.java
index 866d144a91..f33d942b3e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiControllerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiControllerTest.java
@@ -133,6 +133,67 @@ class CodeArtifactPypiControllerTest {
assertEquals(404, response.getStatus());
}
+ /**
+ * The fix for a real finding: building the request URI with a plain string concatenation
+ * would let a {@code #} in a caller-supplied filename get interpreted as a URI fragment and
+ * silently dropped from the actual wire request, fetching a different, shorter real filename
+ * instead of the one actually requested. Checks the raw path the backend actually receives,
+ * not just the response, via a root fallback context rather than registering the context at
+ * the encoded literal path: {@code com.sun.net.httpserver.HttpServer} matches contexts on the
+ * decoded path, so registering one at the raw percent-encoded string wouldn't reliably match.
+ */
+ @Test
+ void downloadFileTreatsAHashInTheFilenameAsLiteralNotAUriFragment() {
+ AtomicReference<String> observedRawPath = new AtomicReference<>();
+ backend.createContext("/", exchange -> {
+ observedRawPath.set(exchange.getRequestURI().getRawPath());
+ respond(exchange, 404, "text/plain", "");
+ });
+
+ Response response = controller.downloadFile(headers("Bearer good"), DOMAIN, REPOSITORY, "pkg-1.0.jar#forged");
+
+ assertEquals(404, response.getStatus());
+ assertEquals("/packages/pkg-1.0.jar%23forged", observedRawPath.get());
+ }
+
+ /**
+ * Same finding, the path-separator half of it: a {@code /} in a caller-supplied filename
+ * would be indistinguishable from a real path separator and could splice in extra path
+ * segments.
+ */
+ @Test
+ void downloadFileTreatsASlashInTheFilenameAsLiteralNotAStructuralSeparator() {
+ AtomicReference<String> observedRawPath = new AtomicReference<>();
+ backend.createContext("/", exchange -> {
+ observedRawPath.set(exchange.getRequestURI().getRawPath());
+ respond(exchange, 404, "text/plain", "");
+ });
+
+ Response response = controller.downloadFile(headers("Bearer good"), DOMAIN, REPOSITORY, "../secret.tar.gz");
+
+ assertEquals(404, response.getStatus());
+ assertEquals("/packages/..%2Fsecret.tar.gz", observedRawPath.get());
+ }
+
+ /**
+ * Same finding again, via the other wire-protocol route that shares the same {@code proxyGet}
+ * helper: the package name in {@code GET /simple/{package}/} is just as caller-supplied as
+ * the filename in {@code GET /packages/{filename}}.
+ */
+ @Test
+ void packageIndexTreatsASlashInThePackageNameAsLiteralNotAStructuralSeparator() {
+ AtomicReference<String> observedRawPath = new AtomicReference<>();
+ backend.createContext("/", exchange -> {
+ observedRawPath.set(exchange.getRequestURI().getRawPath());
+ respond(exchange, 404, "text/plain", "");
+ });
+
+ Response response = controller.packageIndex(headers("Bearer good"), DOMAIN, REPOSITORY, "../other-pkg");
+
+ assertEquals(404, response.getStatus());
+ assertEquals("/simple/..%2Fother-pkg/", observedRawPath.get());
+ }
+
@Test
void uploadRejectsABodyWithNoDeterminableBoundary() {
Response response = controller.upload(headers("Bearer good", "application/octet-stream"), DOMAIN, REPOSITORY,
@@ -171,6 +232,85 @@ class CodeArtifactPypiControllerTest {
assertEquals(false, uploadReceived.get());
}
+ /**
+ * The fix for a real finding: a plain substring search on the whole index response body
+ * would wrongly treat the uploaded filename as already published if it happened to be a
+ * substring of a different, longer real entry, e.g. a detached {@code .asc} signature file
+ * whose own name contains the base file's name as a prefix.
+ */
+ @Test
+ void uploadDoesNotTreatAShorterFilenameAsAlreadyListedWhenItIsOnlyASubstringOfALongerOne() {
+ backend.createContext("/simple/my-pkg/", exchange -> respond(exchange, 200, "text/html",
+ "<a href=\"my_pkg-1.0.0.tar.gz.asc\">my_pkg-1.0.0.tar.gz.asc</a>"));
+ AtomicReference<Boolean> uploadReceived = new AtomicReference<>(false);
+ backend.createContext("/", exchange -> {
+ if ("POST".equals(exchange.getRequestMethod())) {
+ uploadReceived.set(true);
+ }
+ respond(exchange, 200, "text/plain", "");
+ });
+ byte[] body = multipartWithFile("boundary123", "my-pkg", "my_pkg-1.0.0.tar.gz", "new content");
+
+ Response response = controller.upload(
+ headers("Bearer good", "multipart/form-data; boundary=boundary123"), DOMAIN, REPOSITORY, body);
+
+ assertEquals(200, response.getStatus());
+ assertEquals(true, uploadReceived.get());
+ }
+
+ /**
+ * The fix for a real finding: the upload preflight's own index-check request (as opposed to
+ * {@code packageIndex}'s, already covered above) built its path the same unsafe way, and
+ * nothing exercised it with a reserved character.
+ */
+ @Test
+ void uploadExistenceCheckTreatsAHashInThePackageNameAsLiteralNotAUriFragment() {
+ AtomicReference<String> observedRawPath = new AtomicReference<>();
+ backend.createContext("/", exchange -> {
+ if ("GET".equals(exchange.getRequestMethod())) {
+ observedRawPath.set(exchange.getRequestURI().getRawPath());
+ respond(exchange, 404, "text/plain", "");
+ } else {
+ respond(exchange, 200, "text/plain", "");
+ }
+ });
+ byte[] body = multipartWithFile("boundary123", "my#pkg", "my_pkg-1.0.0.tar.gz", "content");
+
+ Response response = controller.upload(
+ headers("Bearer good", "multipart/form-data; boundary=boundary123"), DOMAIN, REPOSITORY, body);
+
+ assertEquals(200, response.getStatus());
+ assertEquals("/simple/my%23pkg/", observedRawPath.get());
+ }
+
+ /**
+ * Same finding, the preflight's other request: once the simple index confirms a match, it
+ * fetches the existing file's bytes for the content comparison, at a path built from the
+ * same caller-supplied filename. Uses {@code ?}, not {@code #} or {@code /}: both of those
+ * are already meaningful to the index's own href parsing (a root-relative directory prefix
+ * and a hash-fragment suffix, both stripped the way a real pypiserver response uses them),
+ * so a filename containing either could never round-trip through the exact-match check at
+ * all, real bug or not; {@code ?} isn't special to that parsing, only to building the URI.
+ */
+ @Test
+ void uploadExistenceCheckTreatsAQuestionMarkInTheFilenameAsLiteralNotAUriQueryStringWhenFetchingTheExistingFile() {
+ String filename = "my_pkg-1.0.0?sig.tar.gz";
+ backend.createContext("/simple/my-pkg/", exchange -> respond(exchange, 200, "text/html",
+ "<a href=\"" + filename + "\">" + filename + "</a>"));
+ AtomicReference<String> observedRawPath = new AtomicReference<>();
+ backend.createContext("/", exchange -> {
+ observedRawPath.set(exchange.getRequestURI().getRawPath());
+ respond(exchange, 200, "application/octet-stream", "different content");
+ });
+ byte[] body = multipartWithFile("boundary123", "my-pkg", filename, "new content");
+
+ Response response = controller.upload(
+ headers("Bearer good", "multipart/form-data; boundary=boundary123"), DOMAIN, REPOSITORY, body);
+
+ assertEquals(409, response.getStatus());
+ assertEquals("/packages/my_pkg-1.0.0%3Fsig.tar.gz", observedRawPath.get());
+ }
+
@Test
void uploadOfByteIdenticalContentSucceedsIdempotentlyWithoutForwardingToTheBackend() {
String sameContent = "same content";