Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(codeartifact): make npm republish of identical content idempotent (#5163)

Priamo Germosen2cf011f7264c
diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlane.java
index 0c8846391b..421865f737 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlane.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlane.java
@@ -1,25 +1,44 @@
package io.github.hectorvent.floci.services.codeartifact;
-import io.github.hectorvent.floci.config.EmulatorConfig;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.ServiceConfigAccess;
import io.github.hectorvent.floci.services.codeartifact.CodeArtifactService.AuthorizationTokenScope;
+import io.vertx.core.Context;
import io.vertx.core.Vertx;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.file.AsyncFile;
+import io.vertx.core.file.OpenOptions;
import io.vertx.core.http.HttpClient;
import io.vertx.core.http.HttpClientOptions;
import io.vertx.core.http.HttpClientRequest;
import io.vertx.core.http.HttpClientResponse;
import io.vertx.core.http.HttpMethod;
+import io.vertx.core.http.HttpServerRequest;
import io.vertx.core.http.RequestOptions;
+import io.vertx.core.streams.ReadStream;
import io.vertx.ext.web.Router;
import io.vertx.ext.web.RoutingContext;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
/**
* Proxies the real npm registry protocol through Floci to the per-repository Verdaccio container
@@ -37,9 +56,12 @@ import java.util.Set;
@ApplicationScoped
public class CodeArtifactNpmDataPlane {
+ private static final Logger LOG = Logger.getLogger(CodeArtifactNpmDataPlane.class);
private static final String PREFIX = "/codeartifact/npm/";
private static final String SERVICE_KEY = "codeartifact";
private static final String BEARER_PREFIX = "Bearer ";
+ private static final int MAX_CONCURRENT_PREFLIGHTS = 4;
+ private static final int MAX_QUEUED_PREFLIGHTS = 64;
private static final Set<String> HOP_BY_HOP_HEADERS = Set.of(
"connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
"te", "trailers", "transfer-encoding", "upgrade", "authorization", "host");
@@ -47,21 +69,23 @@ public class CodeArtifactNpmDataPlane {
private final CodeArtifactService service;
private final VerdaccioSidecarManager verdaccioManager;
private final ServiceConfigAccess serviceConfigAccess;
- private final EmulatorConfig config;
private final Vertx vertx;
private final HttpClient proxyClient;
+ private final ObjectMapper mapper;
+ /** Bounds concurrent spooled publishes, which each hold a temp file and a scan in progress. */
+ private final PreflightGate preflights = new PreflightGate(MAX_CONCURRENT_PREFLIGHTS, MAX_QUEUED_PREFLIGHTS);
@Inject
public CodeArtifactNpmDataPlane(CodeArtifactService service, VerdaccioSidecarManager verdaccioManager,
- ServiceConfigAccess serviceConfigAccess, EmulatorConfig config, Vertx vertx) {
+ ServiceConfigAccess serviceConfigAccess, Vertx vertx, ObjectMapper mapper) {
this.service = service;
this.verdaccioManager = verdaccioManager;
this.serviceConfigAccess = serviceConfigAccess;
- this.config = config;
this.vertx = vertx;
this.proxyClient = vertx.createHttpClient(new HttpClientOptions()
.setConnectTimeout(5_000)
.setKeepAlive(true));
+ this.mapper = mapper;
}
void register(@Observes Router router) {
@@ -103,11 +127,25 @@ public class CodeArtifactNpmDataPlane {
return;
}
+ if (isPublish(context.request().method(), npmRequest.rest())) {
+ context.request().pause();
+ Context eventLoop = vertx.getOrCreateContext();
+ AtomicReference<PreflightGate.Waiter> queued = new AtomicReference<>();
+ context.response().closeHandler(ignored -> preflights.abandon(queued.get()));
+ PreflightGate.Waiter waiter = preflights.acquire(() -> eventLoop.runOnContext(ignored ->
+ handlePublish(context, npmRequest, npmRepositoryId)));
+ if (waiter == null) {
+ context.request().resume();
+ context.response().putHeader("Retry-After", "1").setStatusCode(429).end();
+ return;
+ }
+ queued.set(waiter);
+ return;
+ }
+
context.request().pause();
- String publicUrl = config.effectiveBaseUrl() + PREFIX + npmRequest.domain() + "/" + npmRequest.repository()
- + "/";
String repoId = npmRepositoryId;
- vertx.<String>executeBlocking(promise -> promise.complete(verdaccioManager.ensureReady(repoId, publicUrl)))
+ vertx.<String>executeBlocking(promise -> promise.complete(ensureBackendReady(repoId, npmRequest)))
.onComplete(result -> {
if (result.failed()) {
context.request().resume();
@@ -118,6 +156,339 @@ public class CodeArtifactNpmDataPlane {
});
}
+ /**
+ * A real {@code npm publish} is a {@code PUT} to the bare package path (no further path
+ * segments): {@code /<name>} unscoped, or {@code /@scope%2Fname} scoped, still one segment
+ * since the slash between scope and name arrives URL-encoded. {@code npm deprecate} is a
+ * {@code PUT} at this same bare-package shape (a dist-tag update is not: that always goes to
+ * {@code /-/package/<name>/dist-tags/<tag>}, a different, multi-segment path {@link #isPublish}
+ * already excludes), but {@link #confirmedRepublish} only ever short-circuits when it can
+ * positively confirm every version in the body already exists with byte-identical content, so
+ * a deprecate call (which carries no new {@code _attachments}) falls through to the normal
+ * proxy path unaffected.
+ */
+ private static boolean isPublish(HttpMethod method, String rest) {
+ return method == HttpMethod.PUT && rest.length() > 1 && rest.indexOf('/', 1) < 0;
+ }
+
+ /**
+ * Real npm registries and the pinned Verdaccio image both reject a republish of an existing
+ * version with 409, even for byte-identical content. The AWS CodeArtifact User Guide states the
+ * opposite for package assets in general: republishing an existing asset with identical content
+ * succeeds because the operation is idempotent. This applies that rule to npm, which has one
+ * tarball asset per version. The envelope is spooled to disk in bounded chunks and scanned as a
+ * stream, so its size never costs heap. Only when every version and dist-tag matches does this
+ * answer 200 without reaching Verdaccio; anything else forwards the spooled body unchanged.
+ */
+ private void handlePublish(RoutingContext context, NpmRequest npmRequest, String npmRepositoryId) {
+ AtomicBoolean finished = new AtomicBoolean();
+ AtomicReference<String> spool = new AtomicReference<>();
+ AtomicReference<AsyncFile> reading = new AtomicReference<>();
+ Runnable finish = () -> {
+ if (finished.compareAndSet(false, true)) {
+ if (reading.get() != null) {
+ reading.get().close();
+ }
+ if (spool.get() != null) {
+ vertx.fileSystem().delete(spool.get());
+ }
+ preflights.release();
+ }
+ };
+ if (context.response().ended() || context.response().closed()) {
+ finish.run();
+ return;
+ }
+ vertx.fileSystem().createTempFile("npm-publish-", ".json").onComplete(created -> {
+ if (created.failed()) {
+ finish.run();
+ respondUnlessGone(context, 500);
+ return;
+ }
+ String path = created.result();
+ spool.set(path);
+ if (finished.get()) {
+ vertx.fileSystem().delete(path);
+ return;
+ }
+ vertx.fileSystem().open(path, new OpenOptions().setWrite(true)).onComplete(opened -> {
+ if (opened.failed()) {
+ finish.run();
+ respondUnlessGone(context, 500);
+ return;
+ }
+ context.request().pipeTo(opened.result()).onComplete(piped -> {
+ if (piped.failed()) {
+ finish.run();
+ respondUnlessGone(context, 400);
+ return;
+ }
+ vertx.<String>executeBlocking(promise ->
+ promise.complete(ensureBackendReady(npmRepositoryId, npmRequest)), false)
+ .onComplete(readyResult -> {
+ if (readyResult.failed()) {
+ finish.run();
+ respondUnlessGone(context, 503);
+ return;
+ }
+ String backendBaseUrl = readyResult.result();
+ vertx.<Boolean>executeBlocking(promise ->
+ promise.complete(confirmedRepublish(backendBaseUrl, npmRequest, path)),
+ false)
+ .onComplete(matchResult -> {
+ if (context.response().closed()) {
+ finish.run();
+ return;
+ }
+ if (matchResult.succeeded() && matchResult.result()) {
+ // "ok" is a message string in a real Verdaccio response
+ // (e.g. "created new package"), not a boolean; matching
+ // that shape in case any client inspects it for display
+ // rather than just checking truthiness.
+ context.response().putHeader("Content-Type", "application/json")
+ .setStatusCode(200)
+ .end("{\"ok\":\"accepted, content already published\","
+ + "\"success\":true}");
+ finish.run();
+ return;
+ }
+ vertx.fileSystem().open(path, new OpenOptions().setRead(true))
+ .onComplete(reopened -> {
+ if (reopened.failed()) {
+ finish.run();
+ respondUnlessGone(context, 500);
+ return;
+ }
+ reading.set(reopened.result());
+ context.addEndHandler(ended -> finish.run());
+ context.response().closeHandler(ignored -> finish.run());
+ forwardSpooled(context, backendBaseUrl,
+ npmRequest.rest() + querySuffix(context),
+ reopened.result());
+ });
+ });
+ });
+ });
+ });
+ });
+ }
+
+ private static void respondUnlessGone(RoutingContext context, int status) {
+ if (!context.response().ended() && !context.response().closed()) {
+ context.response().setStatusCode(status).end();
+ }
+ }
+
+ /**
+ * Admits at most {@code permits} publish preflights at once. Waiters queue here rather than
+ * blocking a worker thread, and each one resumes on the event loop it arrived on.
+ */
+ private static final class PreflightGate {
+ private final int maxQueued;
+ private final Deque<Waiter> waiting = new ArrayDeque<>();
+ private int available;
+
+ PreflightGate(int permits, int maxQueued) {
+ this.available = permits;
+ this.maxQueued = maxQueued;
+ }
+
+ /** The waiter to abandon if the client leaves; {@code null} when the queue is already full. */
+ Waiter acquire(Runnable onAcquired) {
+ Waiter waiter = new Waiter(onAcquired);
+ synchronized (this) {
+ if (available > 0) {
+ available--;
+ } else if (waiting.size() < maxQueued) {
+ waiting.add(waiter);
+ return waiter;
+ } else {
+ return null;
+ }
+ }
+ onAcquired.run();
+ return waiter;
+ }
+
+ void abandon(Waiter waiter) {
+ if (waiter == null) {
+ return;
+ }
+ synchronized (this) {
+ waiting.remove(waiter);
+ }
+ }
+
+ void release() {
+ Waiter next;
+ synchronized (this) {
+ next = waiting.poll();
+ if (next == null) {
+ available++;
+ return;
+ }
+ }
+ next.onAcquired.run();
+ }
+
+ private static final class Waiter {
+ private final Runnable onAcquired;
+
+ Waiter(Runnable onAcquired) {
+ this.onAcquired = onAcquired;
+ }
+ }
+ }
+
+ private String ensureBackendReady(String npmRepositoryId, NpmRequest npmRequest) {
+ return verdaccioManager.ensureReady(npmRepositoryId,
+ verdaccioManager.publicUrl(npmRequest.domain(), npmRequest.repository()));
+ }
+
+ /**
+ * {@code true} only when every version in the publish envelope already exists on Verdaccio with
+ * the same integrity, every dist-tag the envelope sends already points where it says, and every
+ * attachment in the envelope is one of those versions' tarballs. Client bytes are checked against
+ * the integrity the client itself claims, so a mismatch between the two falls through too.
+ * Anything unreadable or inconsistent is not confirmable and forwards unchanged.
+ */
+ private boolean confirmedRepublish(String backendBaseUrl, NpmRequest npmRequest, String spoolPath) {
+ PackageIdentity identity = packageIdentityFromPath(npmRequest.rest());
+ NpmPublishEnvelopeScanner.Envelope envelope;
+ try (InputStream in = new BufferedInputStream(Files.newInputStream(Path.of(spoolPath)))) {
+ envelope = NpmPublishEnvelopeScanner.scan(in);
+ } catch (IOException e) {
+ LOG.debugv(e, "Publish for {0}/{1} is not a confirmable republish: {2}",
+ npmRequest.domain(), npmRequest.repository(), e.getMessage());
+ return false;
+ }
+ Optional<JsonNode> storedDocument;
+ try {
+ storedDocument = verdaccioManager.fetchPackageDocument(backendBaseUrl, identity.namespace(),
+ identity.packageName());
+ } catch (IllegalStateException e) {
+ LOG.warnv(e, "Could not check {0}/{1} for an existing match before publishing: {2}",
+ npmRequest.domain(), npmRequest.repository(), e.getMessage());
+ return false;
+ }
+ if (storedDocument.isEmpty() || !isConfirmedRepublish(envelope, storedDocument.get(), identity)) {
+ return false;
+ }
+ for (NpmPublishEnvelopeScanner.Version version : envelope.versions().values()) {
+ try {
+ Optional<String> stored = verdaccioManager.storedTarballIntegrity(backendBaseUrl,
+ identity.namespace(), identity.packageName(), tarballFilename(version.tarball()));
+ if (stored.isEmpty() || !stored.get().equals(version.integrity())) {
+ return false;
+ }
+ } catch (IllegalStateException e) {
+ LOG.warnv(e, "Could not verify the stored tarball of {0}/{1} before publishing: {2}",
+ npmRequest.domain(), npmRequest.repository(), e.getMessage());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean isConfirmedRepublish(NpmPublishEnvelopeScanner.Envelope envelope, JsonNode stored,
+ PackageIdentity identity) {
+ if (envelope.name() != null && !envelope.name().equals(identity.fullName())) {
+ return false;
+ }
+ if (envelope.versions().isEmpty()) {
+ return false;
+ }
+ for (Map.Entry<String, String> tag : envelope.distTags().entrySet()) {
+ JsonNode storedTag = stored.path("dist-tags").get(tag.getKey());
+ if (storedTag == null || !storedTag.asText("").equals(tag.getValue())) {
+ return false;
+ }
+ }
+ Set<String> tarballs = new HashSet<>();
+ for (Map.Entry<String, NpmPublishEnvelopeScanner.Version> entry : envelope.versions().entrySet()) {
+ NpmPublishEnvelopeScanner.Version sent = entry.getValue();
+ if (sent.tarball() == null || sent.integrity() == null) {
+ return false;
+ }
+ String filename = tarballFilename(sent.tarball());
+ JsonNode storedVersion = stored.path("versions").get(entry.getKey());
+ if (storedVersion == null
+ || !filename.equals(tarballFilename(storedVersion.path("dist").path("tarball").asText("")))
+ || !sent.integrity().equals(storedVersion.path("dist").path("integrity").asText(""))) {
+ return false;
+ }
+ if (!sent.integrity().equals(envelope.attachmentIntegrity().get(filename))) {
+ return false;
+ }
+ tarballs.add(filename);
+ }
+ for (String attachment : envelope.attachmentIntegrity().keySet()) {
+ if (!tarballs.contains(attachment)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static String tarballFilename(String tarballUrl) {
+ return tarballUrl.substring(tarballUrl.lastIndexOf('/') + 1);
+ }
+
+ /**
+ * {@code namespace} is the npm scope without its leading {@code @}, or {@code null} for an
+ * unscoped package, the same shape {@link RepositorySidecarManager#fetchPackageVersionAsset}
+ * takes everywhere else. {@code rest} is a bare-package publish path, exactly one segment
+ * ({@link #isPublish} already confirmed that): {@code /<name>} unscoped, or {@code
+ * /@scope%2Fname} scoped, with the scope separator arriving percent-encoded since Netty
+ * leaves a {@code %2F} undecoded in {@link HttpServerRequest#path()}
+ * specifically so it is never confused with a real path separator. The one, targeted decode
+ * here undoes exactly that, and only that: every other npm package-name character is already
+ * unreserved and needs no decoding.
+ */
+ private static PackageIdentity packageIdentityFromPath(String rest) {
+ String decoded = rest.substring(1).replace("%2F", "/").replace("%2f", "/");
+ if (decoded.startsWith("@")) {
+ int slash = decoded.indexOf('/');
+ if (slash > 0) {
+ return new PackageIdentity(decoded.substring(1, slash), decoded.substring(slash + 1));
+ }
+ }
+ return new PackageIdentity(null, decoded);
+ }
+
+ private record PackageIdentity(String namespace, String packageName) {
+ String fullName() {
+ return namespace == null ? packageName : "@" + namespace + "/" + packageName;
+ }
+ }
+
+ private void forwardSpooled(RoutingContext context, String backendBaseUrl, String backendPath,
+ ReadStream<Buffer> body) {
+ URI backend = URI.create(backendBaseUrl);
+ RequestOptions options = new RequestOptions()
+ .setHost(backend.getHost())
+ .setPort(backend.getPort())
+ .setURI(backendPath)
+ .setMethod(context.request().method());
+ proxyClient.request(options).onComplete(upstreamRequest -> {
+ if (upstreamRequest.failed()) {
+ respondUnlessGone(context, 503);
+ return;
+ }
+ HttpClientRequest clientReq = upstreamRequest.result();
+ copyRequestHeaders(context, clientReq);
+ clientReq.response().onComplete(upstreamResponse -> {
+ if (upstreamResponse.failed()) {
+ respondUnlessGone(context, 502);
+ return;
+ }
+ copyResponseHeaders(context, upstreamResponse.result());
+ upstreamResponse.result().pipeTo(context.response());
+ });
+ clientReq.send(body).onFailure(ignored -> respondUnlessGone(context, 502));
+ });
+ }
+
private void proxy(RoutingContext context, String backendBaseUrl, String backendPath) {
URI backend = URI.create(backendBaseUrl);
RequestOptions options = new RequestOptions()
diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScanner.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScanner.java
new file mode 100644
index 0000000000..364f971db8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScanner.java
@@ -0,0 +1,165 @@
+package io.github.hectorvent.floci.services.codeartifact;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.StreamReadConstraints;
+import com.fasterxml.jackson.core.StreamReadFeature;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Reads an npm publish envelope as a stream and keeps only what the republish check needs. Jackson
+ * decodes each attachment's base64 {@code data} straight into a SHA-512 digest, so the tarball is never
+ * held in memory. Any structure this scanner does not expect throws, and callers treat a throw as not
+ * confirmable and forward the request unchanged.
+ */
+final class NpmPublishEnvelopeScanner {
+
+ record Version(String tarball, String integrity) {}
+
+ record Envelope(String name, Map<String, String> distTags, Map<String, Version> versions,
+ Map<String, String> attachmentIntegrity) {}
+
+ private static final JsonFactory FACTORY = JsonFactory.builder()
+ .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION)
+ .streamReadConstraints(StreamReadConstraints.builder()
+ .maxNestingDepth(32)
+ .maxNumberLength(1_000_000)
+ .maxStringLength(1_000_000)
+ .build())
+ .build();
+
+ @FunctionalInterface
+ private interface FieldHandler {
+ void accept(String fieldName) throws IOException;
+ }
+
+ private final JsonParser parser;
+ private String name;
+ private final Map<String, String> distTags = new HashMap<>();
+ private final Map<String, Map<String, String>> versionFields = new HashMap<>();
+ private final Map<String, String> attachmentIntegrity = new HashMap<>();
+
+ private NpmPublishEnvelopeScanner(JsonParser parser) {
+ this.parser = parser;
+ }
+
+ static Envelope scan(InputStream in) throws IOException {
+ try (JsonParser parser = FACTORY.createParser(in)) {
+ NpmPublishEnvelopeScanner scanner = new NpmPublishEnvelopeScanner(parser);
+ scanner.parseEnvelope();
+ return scanner.envelope();
+ }
+ }
+
+ private void parseEnvelope() throws IOException {
+ parser.nextToken();
+ requireToken(JsonToken.START_OBJECT);
+ while (parser.nextToken() == JsonToken.FIELD_NAME) {
+ String key = parser.currentName();
+ parser.nextToken();
+ switch (key) {
+ case "name" -> name = stringValue();
+ case "dist-tags" -> forEachField(tag -> distTags.put(tag, stringValue()));
+ case "versions" -> forEachField(this::readVersion);
+ case "_attachments" -> forEachField(filename ->
+ forEachField(attachmentField -> {
+ if (attachmentField.equals("data")) {
+ attachmentIntegrity.put(filename, hashBase64());
+ } else {
+ parser.skipChildren();
+ }
+ }));
+ default -> parser.skipChildren();
+ }
+ }
+ requireToken(JsonToken.END_OBJECT);
+ if (parser.nextToken() != null) {
+ throw invalid("trailing data after the publish envelope");
+ }
+ }
+
+ private void readVersion(String version) throws IOException {
+ Map<String, String> fields = new HashMap<>();
+ forEachField(field -> {
+ if (field.equals("dist")) {
+ forEachField(distField -> {
+ if (distField.equals("tarball") || distField.equals("integrity")) {
+ fields.put(distField, stringValue());
+ } else {
+ parser.skipChildren();
+ }
+ });
+ } else {
+ parser.skipChildren();
+ }
+ });
+ versionFields.put(version, fields);
+ }
+
+ /**
+ * Walks the object the parser is currently on, handing each field name to {@code handler} with the
+ * parser positioned on that field's value. Leaves the parser on the object's END_OBJECT.
+ */
+ private void forEachField(FieldHandler handler) throws IOException {
+ requireToken(JsonToken.START_OBJECT);
+ while (parser.nextToken() == JsonToken.FIELD_NAME) {
+ String fieldName = parser.currentName();
+ parser.nextToken();
+ handler.accept(fieldName);
+ }
+ requireToken(JsonToken.END_OBJECT);
+ }
+
+ private String stringValue() throws IOException {
+ requireToken(JsonToken.VALUE_STRING);
+ return parser.getText();
+ }
+
+ private String hashBase64() throws IOException {
+ requireToken(JsonToken.VALUE_STRING);
+ MessageDigest digest = sha512();
+ try (OutputStream sink = new DigestOutputStream(OutputStream.nullOutputStream(), digest)) {
+ parser.readBinaryValue(sink);
+ } catch (IllegalArgumentException e) {
+ throw invalid("attachment data is not plain base64");
+ }
+ return "sha512-" + Base64.getEncoder().encodeToString(digest.digest());
+ }
+
+ private void requireToken(JsonToken expected) throws IOException {
+ if (parser.currentToken() != expected) {
+ throw invalid("expected " + expected + " but found " + parser.currentToken());
+ }
+ }
+
+ private Envelope envelope() {
+ Map<String, Version> versions = new HashMap<>();
+ for (Map.Entry<String, Map<String, String>> entry : versionFields.entrySet()) {
+ Map<String, String> fields = entry.getValue();
+ versions.put(entry.getKey(), new Version(fields.get("tarball"), fields.get("integrity")));
+ }
+ return new Envelope(name, distTags, versions, attachmentIntegrity);
+ }
+
+ private static MessageDigest sha512() {
+ try {
+ return MessageDigest.getInstance("SHA-512");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("SHA-512 is required by every Java runtime", e);
+ }
+ }
+
+ private static IOException invalid(String reason) {
+ return new IOException("Not a publish envelope this check can confirm: " + reason);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManager.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManager.java
index 718d7a2223..c043bbd3a4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManager.java
@@ -20,13 +20,17 @@ import org.jboss.logging.Logger;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandlers;
import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.time.Duration;
+import java.util.Base64;
import java.util.Optional;
/**
@@ -152,38 +156,99 @@ public class VerdaccioSidecarManager implements RepositorySidecarManager, Contai
@Override
public Optional<byte[]> fetchPackageVersionAsset(String repositoryContainerId, String domain, String repository,
String namespace, String packageName, String version, String assetName) {
- String publicUrl = config.effectiveBaseUrl() + "/codeartifact/npm/" + domain + "/" + repository + "/";
- String baseUrl = ensureReady(repositoryContainerId, publicUrl);
- String packagePath = packagePath(namespace, packageName);
- if (!versionHasAsset(baseUrl, packagePath, version, assetName)) {
+ String baseUrl = ensureReady(repositoryContainerId, publicUrl(domain, repository));
+ if (!versionHasAsset(baseUrl, packagePath(namespace, packageName), version, assetName)) {
return Optional.empty();
}
- URI uri = SidecarUriUtils.combine(URI.create(baseUrl), "/" + packagePath + "/-/"
- + SidecarUriUtils.encodeSegment(assetName));
- HttpRequest request = HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(10)).GET().build();
+ return fetchTarball(baseUrl, namespace, packageName, assetName);
+ }
+
+ /** The public URL a Verdaccio sidecar is told to advertise for one npm repository. */
+ public String publicUrl(String domain, String repository) {
+ return config.effectiveBaseUrl() + "/codeartifact/npm/" + domain + "/" + repository + "/";
+ }
+
+ private static String packagePath(String namespace, String packageName) {
+ return namespace == null
+ ? SidecarUriUtils.encodeSegment(packageName)
+ : "@" + SidecarUriUtils.encodeSegment(namespace) + "/" + SidecarUriUtils.encodeSegment(packageName);
+ }
+
+ /**
+ * The package's metadata document as Verdaccio serves it, or empty when the package isn't
+ * there. {@code baseUrl} is a backend already made ready by {@link #ensureReady}, so repeated
+ * calls for one publish don't each pay another readiness probe.
+ */
+ public Optional<JsonNode> fetchPackageDocument(String baseUrl, String namespace, String packageName) {
+ return packageDocument(baseUrl, packagePath(namespace, packageName));
+ }
+
+ /** One tarball by its filename, from a backend already made ready by {@link #ensureReady}. */
+ public Optional<byte[]> fetchTarball(String baseUrl, String namespace, String packageName, String assetName) {
+ HttpRequest request = tarballRequest(baseUrl, namespace, packageName, assetName);
HttpResponse<byte[]> response;
try {
response = httpClient.send(request, BodyHandlers.ofByteArray());
} catch (Exception e) {
- throw new IllegalStateException("Could not reach the Verdaccio sidecar to fetch " + packagePath, e);
+ throw new IllegalStateException("Could not reach the Verdaccio sidecar to fetch " + assetName, e);
}
if (response.statusCode() == 404) {
return Optional.empty();
}
if (response.statusCode() != 200) {
- throw new IllegalStateException("Could not fetch " + packagePath + "/-/" + assetName
+ throw new IllegalStateException("Could not fetch " + assetName
+ " from the Verdaccio sidecar: upstream returned " + response.statusCode());
}
return Optional.of(response.body());
}
- private static String packagePath(String namespace, String packageName) {
- return namespace == null
- ? SidecarUriUtils.encodeSegment(packageName)
- : "@" + SidecarUriUtils.encodeSegment(namespace) + "/" + SidecarUriUtils.encodeSegment(packageName);
+ /**
+ * The SHA-512 integrity of a stored tarball, computed by streaming its bytes so the file never
+ * sits in memory. Empty when the tarball is not there.
+ */
+ public Optional<String> storedTarballIntegrity(String baseUrl, String namespace, String packageName,
+ String assetName) {
+ HttpResponse<InputStream> response;
+ try {
+ response = httpClient.send(tarballRequest(baseUrl, namespace, packageName, assetName),
+ BodyHandlers.ofInputStream());
+ } catch (Exception e) {
+ throw new IllegalStateException("Could not reach the Verdaccio sidecar to hash " + assetName, e);
+ }
+ try (InputStream body = response.body()) {
+ if (response.statusCode() == 404) {
+ return Optional.empty();
+ }
+ if (response.statusCode() != 200) {
+ throw new IllegalStateException("Could not fetch " + assetName
+ + " from the Verdaccio sidecar: upstream returned " + response.statusCode());
+ }
+ MessageDigest digest = sha512();
+ byte[] chunk = new byte[64 * 1024];
+ for (int read = body.read(chunk); read != -1; read = body.read(chunk)) {
+ digest.update(chunk, 0, read);
+ }
+ return Optional.of("sha512-" + Base64.getEncoder().encodeToString(digest.digest()));
+ } catch (IOException e) {
+ throw new IllegalStateException("Could not read " + assetName + " from the Verdaccio sidecar", e);
+ }
}
- private boolean versionHasAsset(String baseUrl, String packagePath, String version, String assetName) {
+ private HttpRequest tarballRequest(String baseUrl, String namespace, String packageName, String assetName) {
+ URI uri = SidecarUriUtils.combine(URI.create(baseUrl), "/" + packagePath(namespace, packageName) + "/-/"
+ + SidecarUriUtils.encodeSegment(assetName));
+ return HttpRequest.newBuilder(uri).timeout(Duration.ofSeconds(10)).GET().build();
+ }
+
+ private static MessageDigest sha512() {
+ try {
+ return MessageDigest.getInstance("SHA-512");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("SHA-512 is required by every Java runtime", e);
+ }
+ }
+
+ private Optional<JsonNode> packageDocument(String baseUrl, String packagePath) {
HttpRequest request = HttpRequest.newBuilder(SidecarUriUtils.combine(URI.create(baseUrl), "/" + packagePath))
.timeout(Duration.ofSeconds(10)).GET().build();
HttpResponse<String> response;
@@ -193,18 +258,25 @@ public class VerdaccioSidecarManager implements RepositorySidecarManager, Contai
throw new IllegalStateException("Could not reach the Verdaccio sidecar to look up " + packagePath, e);
}
if (response.statusCode() == 404) {
- return false;
+ return Optional.empty();
}
if (response.statusCode() != 200) {
throw new IllegalStateException("Could not look up " + packagePath + " on the Verdaccio sidecar: "
+ "upstream returned " + response.statusCode());
}
- JsonNode versionNode;
try {
- versionNode = mapper.readTree(response.body()).path("versions").path(version);
+ return Optional.of(mapper.readTree(response.body()));
} catch (Exception e) {
throw new IllegalStateException("Could not parse Verdaccio's package metadata for " + packagePath, e);
}
+ }
+
+ private boolean versionHasAsset(String baseUrl, String packagePath, String version, String assetName) {
+ Optional<JsonNode> document = packageDocument(baseUrl, packagePath);
+ if (document.isEmpty()) {
+ return false;
+ }
+ JsonNode versionNode = document.get().path("versions").path(version);
if (versionNode.isMissingNode()) {
return false;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlaneTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlaneTest.java
index 48c4da2c9c..eff3999d84 100644
--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlaneTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlaneTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.codeartifact;
-import io.github.hectorvent.floci.config.EmulatorConfig;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.ServiceConfigAccess;
import io.github.hectorvent.floci.services.codeartifact.CodeArtifactService.AuthorizationTokenScope;
@@ -17,17 +18,32 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Base64;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Random;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;
@@ -41,6 +57,7 @@ import static org.mockito.Mockito.when;
*/
class CodeArtifactNpmDataPlaneTest {
+ private static final ObjectMapper JSON = new ObjectMapper();
private static final String DOMAIN = "dom";
private static final String REPOSITORY = "repo";
private static final String NPM_REPOSITORY_ID = "npm-repo-id-1";
@@ -204,6 +221,513 @@ class CodeArtifactNpmDataPlaneTest {
assertEquals("{\"name\":\"my-pkg\"}", upstreamBody.get());
}
+ /**
+ * Confirmed against the exact pinned Verdaccio image that
+ * republishing a just-published, byte-identical tarball comes back {@code 409}, never a
+ * success, unlike real AWS CodeArtifact's own documented idempotent-republish contract (same
+ * rule Maven and PyPI already implement). Verdaccio must never see this PUT at all: the
+ * preflight has to confirm the match and respond itself before ever reaching the backend.
+ */
+ @Test
+ void aRepublishOfByteIdenticalContentShortCircuitsWithoutReachingTheBackend() throws Exception {
+ byte[] content = "tarball bytes".getBytes(StandardCharsets.UTF_8);
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(500).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ givenStoredVersion(verdaccioManager, null, "my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content, Map.of());
+ startDataPlane(service, verdaccioManager);
+
+ HttpResponse response = put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/my-pkg", "good-token",
+ publishEnvelope("my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content));
+
+ assertEquals(200, response.statusCode());
+ assertEquals(false, upstreamReceivedPut.get());
+ }
+
+ /**
+ * Same finding, the other required half: a version that already exists but with genuinely
+ * different content must still reach the backend and fail the way Verdaccio's own (correctly
+ * strict) check says it should, not be silently treated the same as the identical-content case.
+ */
+ @Test
+ void aRepublishOfDifferentContentStillForwardsToTheBackendAndItsConflictResponseWins() throws Exception {
+ byte[] existingContent = "old tarball bytes".getBytes(StandardCharsets.UTF_8);
+ byte[] newContent = "new tarball bytes".getBytes(StandardCharsets.UTF_8);
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(409).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ givenStoredVersion(verdaccioManager, null, "my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", existingContent, Map.of());
+ startDataPlane(service, verdaccioManager);
+
+ HttpResponse response = put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/my-pkg", "good-token",
+ publishEnvelope("my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", newContent));
+
+ assertEquals(409, response.statusCode());
+ assertEquals(true, upstreamReceivedPut.get());
+ }
+
+ /**
+ * The non-republish baseline: a version Verdaccio has never seen before must always reach the
+ * backend, same as today, since there is nothing yet to compare against.
+ */
+ @Test
+ void aPublishOfAGenuinelyNewVersionStillForwardsToTheBackend() throws Exception {
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(201).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ startDataPlane(service, verdaccioManager);
+
+ HttpResponse response = put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/my-pkg", "good-token",
+ publishEnvelope("my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", "fresh bytes".getBytes(StandardCharsets.UTF_8)));
+
+ assertEquals(201, response.statusCode());
+ assertEquals(true, upstreamReceivedPut.get());
+ }
+
+ /**
+ * The identity used to look up an existing match comes from the URL the publish was sent to,
+ * not the envelope's own {@code name} field. A mismatched body must not be checked against
+ * another package's stored copy; the stub here holds a match for that other package, and the
+ * PUT must still reach the backend.
+ */
+ @Test
+ void aPublishEnvelopeNameThatDisagreesWithTheUrlIsNotTreatedAsAConfirmableRepublish() throws Exception {
+ byte[] content = "tarball bytes".getBytes(StandardCharsets.UTF_8);
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(201).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ // A match exists for "other-pkg", the envelope's own (wrong) name, but the URL is
+ // publishing to "my-pkg": this must never be consulted for this request at all.
+ givenStoredVersion(verdaccioManager, null, "other-pkg", "1.0.0", "other-pkg-1.0.0.tgz", content, Map.of());
+ startDataPlane(service, verdaccioManager);
+
+ HttpResponse response = put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/my-pkg", "good-token",
+ publishEnvelope("other-pkg", "1.0.0", "other-pkg-1.0.0.tgz", content));
+
+ assertEquals(201, response.statusCode());
+ assertEquals(true, upstreamReceivedPut.get());
+ }
+
+ /**
+ * Confirms the URL-derived identity {@link #aPublishEnvelopeNameThatDisagreesWithTheUrlIsNotTreatedAsAConfirmableRepublish}
+ * relies on actually decodes a scoped package's {@code %2F} correctly: the scope separator
+ * arrives percent-encoded in the URL (Netty leaves it that way specifically so it is never
+ * confused with a real path separator), so this exercises the one, targeted decode that
+ * undoes it.
+ */
+ @Test
+ void aRepublishOfAScopedPackageWithByteIdenticalContentShortCircuitsWithoutReachingTheBackend() throws Exception {
+ byte[] content = "scoped tarball bytes".getBytes(StandardCharsets.UTF_8);
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(500).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ givenStoredVersion(verdaccioManager, "myscope", "my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content, Map.of());
+ startDataPlane(service, verdaccioManager);
+
+ HttpResponse response = put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/@myscope%2Fmy-pkg",
+ "good-token", publishEnvelope("@myscope/my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content));
+
+ assertEquals(200, response.statusCode());
+ assertEquals(false, upstreamReceivedPut.get());
+ }
+
+ /**
+ * The metadata can claim an integrity the stored bytes do not have, for instance a tarball that
+ * was damaged or lost behind an intact packument. The retry must not be acknowledged on that claim.
+ */
+ @Test
+ void aRepublishWhoseStoredTarballDoesNotHashToItsIntegrityFallsThroughToTheBackend() throws Exception {
+ byte[] content = "tarball bytes".getBytes(StandardCharsets.UTF_8);
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(409).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ givenStoredVersion(verdaccioManager, null, "my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content, Map.of());
+ when(verdaccioManager.storedTarballIntegrity(anyString(), isNull(), eq("my-pkg"), eq("my-pkg-1.0.0.tgz")))
+ .thenReturn(Optional.of("sha512-" + Base64.getEncoder().encodeToString(new byte[64])));
+ startDataPlane(service, verdaccioManager);
+
+ HttpResponse response = put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/my-pkg", "good-token",
+ publishEnvelope("my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content));
+
+ assertEquals(409, response.statusCode());
+ assertEquals(true, upstreamReceivedPut.get());
+ }
+
+ /**
+ * Every "can't confirm an identical republish" path must fall through to the real backend
+ * rather than short-circuit: malformed JSON, a version whose tarball has no matching attachment,
+ * and a sidecar lookup that fails outright.
+ */
+ @Test
+ void aPublishThatCannotBeConfirmedAsARepublishFallsThroughToTheBackend() throws Exception {
+ assertForwardedWhenNotConfirmable("not json at all", verdaccioManager -> { });
+ }
+
+ @Test
+ void aPublishWhoseVersionHasNoMatchingAttachmentFallsThroughToTheBackend() throws Exception {
+ String envelopeWithoutAttachment = "{\"name\":\"my-pkg\",\"versions\":{\"1.0.0\":{\"name\":\"my-pkg\","
+ + "\"version\":\"1.0.0\",\"dist\":{\"tarball\":\"http://ignored/my-pkg-1.0.0.tgz\"}}},"
+ + "\"_attachments\":{}}";
+ assertForwardedWhenNotConfirmable(envelopeWithoutAttachment, verdaccioManager ->
+ givenStoredVersion(verdaccioManager, null, "my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", new byte[0],
+ Map.of()));
+ }
+
+ @Test
+ void aPublishWhoseSidecarLookupFailsFallsThroughToTheBackend() throws Exception {
+ assertForwardedWhenNotConfirmable(
+ publishEnvelope("my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", "tarball bytes".getBytes(StandardCharsets.UTF_8)),
+ verdaccioManager -> when(verdaccioManager.fetchPackageDocument(anyString(), isNull(), eq("my-pkg")))
+ .thenThrow(new IllegalStateException("sidecar unreachable")));
+ }
+
+ private void assertForwardedWhenNotConfirmable(String body, Consumer<VerdaccioSidecarManager> arrange)
+ throws Exception {
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(201).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ arrange.accept(verdaccioManager);
+ startDataPlane(service, verdaccioManager);
+
+ HttpResponse response = put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/my-pkg", "good-token", body);
+
+ assertEquals(201, response.statusCode());
+ assertEquals(true, upstreamReceivedPut.get());
+ }
+
+ /**
+ * Arranges a package that already exists on the backend: its metadata document (with
+ * {@code storedTags} and one version whose tarball is {@code filename}) and that tarball's bytes.
+ */
+ private static void givenStoredVersion(VerdaccioSidecarManager verdaccioManager, String namespace, String name,
+ String version, String filename, byte[] storedContent, Map<String, String> storedTags) {
+ when(verdaccioManager.fetchPackageDocument(anyString(), eq(namespace), eq(name)))
+ .thenReturn(Optional.of(storedDocument(name, version, filename, storedContent, storedTags)));
+ when(verdaccioManager.storedTarballIntegrity(anyString(), eq(namespace), eq(name), eq(filename)))
+ .thenReturn(Optional.of(integrity(storedContent)));
+ }
+
+ /**
+ * A publish far beyond any in-memory size still gets the idempotent answer: the envelope is
+ * spooled and hashed as a stream, so its size never has to fit on heap.
+ */
+ @Test
+ void aLargeIdenticalRepublishIsStillConfirmedAsIdempotent() throws Exception {
+ byte[] content = new byte[70 * 1024 * 1024];
+ new Random(11).nextBytes(content);
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(409).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ givenStoredVersion(verdaccioManager, null, "big-pkg", "1.0.0", "big-pkg-1.0.0.tgz", content, Map.of());
+ startDataPlane(service, verdaccioManager);
+
+ HttpResponse response = put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/big-pkg", "good-token",
+ publishEnvelope("big-pkg", "1.0.0", "big-pkg-1.0.0.tgz", content));
+
+ assertEquals(200, response.statusCode());
+ assertEquals(false, upstreamReceivedPut.get());
+ }
+
+ private static ObjectNode storedDocument(String name, String version, String filename, byte[] content,
+ Map<String, String> storedTags) {
+ ObjectNode document = JSON.createObjectNode();
+ document.put("name", name);
+ ObjectNode tags = document.putObject("dist-tags");
+ storedTags.forEach(tags::put);
+ ObjectNode dist = document.putObject("versions").putObject(version).putObject("dist");
+ dist.put("tarball", "http://ignored/" + filename);
+ dist.put("integrity", integrity(content));
+ return document;
+ }
+
+ private static String integrity(byte[] content) {
+ try {
+ return "sha512-" + Base64.getEncoder().encodeToString(MessageDigest.getInstance("SHA-512").digest(content));
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private HttpResponse putUnchecked(String path, String body) {
+ try {
+ return put(path, "good-token", body);
+ } catch (Exception e) {
+ throw new CompletionException(e);
+ }
+ }
+
+ /**
+ * Four publishes hold their preflight permits while blocked in the sidecar lookup. A fifth
+ * identical publish must wait for one rather than bypass the check: bypassing would send it
+ * straight to Verdaccio, which answers 409 for an existing version, so the same republish would
+ * succeed or fail depending on load. The upstream answers 409 so that a bypass would show up.
+ */
+ @Test
+ void aPublishArrivingWhilePreflightsAreFullWaitsForAPermitInsteadOfBypassingTheCheck() throws Exception {
+ byte[] content = "tarball bytes".getBytes(StandardCharsets.UTF_8);
+ String envelope = publishEnvelope("my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content);
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(409).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ CountDownLatch entered = new CountDownLatch(4);
+ CountDownLatch release = new CountDownLatch(1);
+ ObjectNode document = storedDocument("my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content, Map.of());
+ when(verdaccioManager.storedTarballIntegrity(anyString(), isNull(), eq("my-pkg"), eq("my-pkg-1.0.0.tgz")))
+ .thenReturn(Optional.of(integrity(content)));
+ when(verdaccioManager.fetchPackageDocument(anyString(), isNull(), eq("my-pkg"))).thenAnswer(invocation -> {
+ entered.countDown();
+ release.await(5, TimeUnit.SECONDS);
+ return Optional.of(document);
+ });
+ startDataPlane(service, verdaccioManager);
+
+ String path = "/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/my-pkg";
+ ExecutorService pool = Executors.newFixedThreadPool(5);
+ try {
+ List<CompletableFuture<HttpResponse>> held = new ArrayList<>();
+ for (int i = 0; i < 4; i++) {
+ held.add(CompletableFuture.supplyAsync(() -> putUnchecked(path, envelope), pool));
+ }
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ CompletableFuture<HttpResponse> overflow = CompletableFuture.supplyAsync(
+ () -> putUnchecked(path, envelope), pool);
+
+ assertThrows(TimeoutException.class, () -> overflow.get(300, TimeUnit.MILLISECONDS));
+ release.countDown();
+
+ for (CompletableFuture<HttpResponse> response : held) {
+ assertEquals(200, response.get(10, TimeUnit.SECONDS).statusCode());
+ }
+ assertEquals(200, overflow.get(10, TimeUnit.SECONDS).statusCode());
+ assertEquals(false, upstreamReceivedPut.get());
+ } finally {
+ release.countDown();
+ pool.shutdownNow();
+ }
+ }
+
+ private static String publishEnvelope(String name, String version, String filename, byte[] content) {
+ return publishEnvelopeWithDistTags(name, version, filename, content, null);
+ }
+
+ private static String publishEnvelopeWithDistTags(String name, String version, String filename, byte[] content,
+ String distTagsJson) {
+ String base64Content = Base64.getEncoder().encodeToString(content);
+ String distTags = distTagsJson == null ? "" : ",\"dist-tags\":" + distTagsJson;
+ return "{\"name\":\"" + name + "\",\"versions\":{\"" + version + "\":{\"name\":\"" + name
+ + "\",\"version\":\"" + version + "\",\"dist\":{\"tarball\":\"http://ignored/" + filename
+ + "\",\"integrity\":\"" + integrity(content) + "\"}}}" + distTags + ",\"_attachments\":{\"" + filename
+ + "\":{\"content_type\":\"application/octet-stream\","
+ + "\"data\":\"" + base64Content + "\",\"length\":" + content.length + "}}}";
+ }
+
+ /**
+ * Republishes byte-identical content whose envelope sends {@code distTagsJson}, against a
+ * package whose tags Verdaccio already has as {@code storedTags}. The backend always answers 409
+ * so a fall-through is visible as that response, and a short-circuit as no PUT reaching it.
+ */
+ private HttpResponse republishWithDistTags(String distTagsJson, Map<String, String> storedTags,
+ AtomicReference<Boolean> upstreamReceivedPut) throws Exception {
+ byte[] content = "tarball bytes".getBytes(StandardCharsets.UTF_8);
+ upstream = vertx.createHttpServer()
+ .requestHandler(request -> {
+ if (request.method() == HttpMethod.PUT) {
+ upstreamReceivedPut.set(true);
+ }
+ request.response().setStatusCode(409).end();
+ })
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CodeArtifactService service = mock(CodeArtifactService.class);
+ when(service.resolveAuthorizationToken("good-token", DOMAIN))
+ .thenReturn(Optional.of(new AuthorizationTokenScope("000000000000", "us-east-1")));
+ when(service.ensureFormatContainerId("npm", "us-east-1", DOMAIN, "000000000000", REPOSITORY))
+ .thenReturn(NPM_REPOSITORY_ID);
+ VerdaccioSidecarManager verdaccioManager = mock(VerdaccioSidecarManager.class);
+ when(verdaccioManager.ensureReady(eq(NPM_REPOSITORY_ID), anyString()))
+ .thenReturn("http://127.0.0.1:" + upstream.actualPort());
+ givenStoredVersion(verdaccioManager, null, "my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content, storedTags);
+ startDataPlane(service, verdaccioManager);
+
+ return put("/codeartifact/npm/" + DOMAIN + "/" + REPOSITORY + "/my-pkg", "good-token",
+ publishEnvelopeWithDistTags("my-pkg", "1.0.0", "my-pkg-1.0.0.tgz", content, distTagsJson));
+ }
+
+ @Test
+ void aRepublishWhoseTagsAlreadyPointWhereItSendsThemStillShortCircuits() throws Exception {
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ HttpResponse response = republishWithDistTags("{\"latest\":\"1.0.0\"}", Map.of("latest", "1.0.0"),
+ upstreamReceivedPut);
+
+ assertEquals(200, response.statusCode());
+ assertEquals(false, upstreamReceivedPut.get());
+ }
+
+ @Test
+ void aRepublishThatWouldMoveATagToAnotherVersionFallsThroughToTheBackend() throws Exception {
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ HttpResponse response = republishWithDistTags("{\"latest\":\"1.0.0\"}", Map.of("latest", "0.9.0"),
+ upstreamReceivedPut);
+
+ assertEquals(409, response.statusCode());
+ assertEquals(true, upstreamReceivedPut.get());
+ }
+
+ @Test
+ void aRepublishThatAddsATagTheServerDoesNotHaveYetFallsThroughToTheBackend() throws Exception {
+ AtomicReference<Boolean> upstreamReceivedPut = new AtomicReference<>(false);
+ HttpResponse response = republishWithDistTags("{\"beta\":\"1.0.0\"}", Map.of("latest", "1.0.0"),
+ upstreamReceivedPut);
+
+ assertEquals(409, response.statusCode());
+ assertEquals(true, upstreamReceivedPut.get());
+ }
+
private void startDataPlane(CodeArtifactService service, VerdaccioSidecarManager verdaccioManager)
throws Exception {
startDataPlane(service, verdaccioManager, true);
@@ -211,12 +735,13 @@ class CodeArtifactNpmDataPlaneTest {
private void startDataPlane(CodeArtifactService service, VerdaccioSidecarManager verdaccioManager,
boolean codeArtifactEnabled) throws Exception {
- EmulatorConfig config = mock(EmulatorConfig.class);
- when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");
+ when(verdaccioManager.publicUrl(DOMAIN, REPOSITORY)).thenReturn("http://localhost:4566/codeartifact/npm/"
+ + DOMAIN + "/" + REPOSITORY + "/");
ServiceConfigAccess serviceConfigAccess = mock(ServiceConfigAccess.class);
when(serviceConfigAccess.isEnabled("codeartifact")).thenReturn(codeArtifactEnabled);
Router router = Router.router(vertx);
- new CodeArtifactNpmDataPlane(service, verdaccioManager, serviceConfigAccess, config, vertx).register(router);
+ new CodeArtifactNpmDataPlane(service, verdaccioManager, serviceConfigAccess, vertx, new ObjectMapper())
+ .register(router);
dataPlane = vertx.createHttpServer().requestHandler(router)
.listen(0, "127.0.0.1")
.toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java
index e17ca59a51..0fe20da784 100644
--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java
@@ -123,8 +123,25 @@ class CodeArtifactNpmDockerIntegrationTest {
assertArrayEquals(expectedTarballBytes, fetchedTarball);
}
+ /**
+ * Confirmed against this exact pinned Verdaccio image that
+ * republishing a just-published, byte-identical tarball comes back {@code 409 "this package
+ * is already present"} on its own, never a success, unlike real AWS CodeArtifact's own
+ * documented idempotent-republish contract (the same rule Maven and PyPI already implement).
+ * Republishing the identical fixture a second time against the real sidecar must still
+ * succeed.
+ */
@Test
@Order(2)
+ void republishingTheSameEnvelopeASecondTimeIsStillIdempotent() {
+ given().header("Authorization", "Bearer " + bearerToken).contentType("application/json")
+ .body(publishFixture)
+ .put("/codeartifact/npm/" + DOMAIN + "/" + REPO + "/" + PACKAGE_NAME)
+ .then().statusCode(anyOf201Or200());
+ }
+
+ @Test
+ @Order(3)
void missingPackageAndMissingRepositoryAreNotFound() {
given().header("Authorization", "Bearer " + bearerToken)
.get("/codeartifact/npm/" + DOMAIN + "/" + REPO + "/does-not-exist")
@@ -136,7 +153,7 @@ class CodeArtifactNpmDockerIntegrationTest {
}
@Test
- @Order(3)
+ @Order(4)
void repositoriesAreIsolatedFromEachOther() {
given().contentType("application/json").header("Authorization", AUTH).body("{}")
.post("/v1/repository?domain=" + DOMAIN + "&repository=other-repo")
@@ -148,7 +165,7 @@ class CodeArtifactNpmDockerIntegrationTest {
}
@Test
- @Order(4)
+ @Order(5)
void missingOrWrongDomainTokensAreUnauthorized() {
given().get("/codeartifact/npm/" + DOMAIN + "/" + REPO + "/" + PACKAGE_NAME)
.then().statusCode(401)
@@ -172,7 +189,7 @@ class CodeArtifactNpmDockerIntegrationTest {
}
@Test
- @Order(5)
+ @Order(6)
void concurrentFirstUseOfANewRepositoryOnlyStartsOneContainer() throws InterruptedException {
given().contentType("application/json").header("Authorization", AUTH).body("{}")
.post("/v1/repository?domain=" + DOMAIN + "&repository=concurrent-repo")
@@ -208,7 +225,7 @@ class CodeArtifactNpmDockerIntegrationTest {
}
@Test
- @Order(6)
+ @Order(7)
void recreatingASameNamedRepositoryDoesNotInheritThePreviousOnesPackages() {
given().contentType("application/json").header("Authorization", AUTH).body("{}")
.post("/v1/repository?domain=" + DOMAIN + "&repository=reused-name")
@@ -231,7 +248,7 @@ class CodeArtifactNpmDockerIntegrationTest {
}
@Test
- @Order(7)
+ @Order(8)
void aDomainCreatedInANonDefaultRegionIsServedThroughTheTokensOwnRegion() {
String nonDefaultRegionAuth =
"AWS4-HMAC-SHA256 Credential=AKID/20260904/us-west-2/codeartifact/aws4_request";
@@ -259,7 +276,7 @@ class CodeArtifactNpmDockerIntegrationTest {
}
@Test
- @Order(8)
+ @Order(9)
void aDomainCreatedUnderANonDefaultAccountIsServedThroughTheTokensOwnAccount() {
String otherAccountAuth = "AWS4-HMAC-SHA256 Credential=111122223333/20260904/us-east-1/codeartifact/aws4_request";
String domain = "npm-sidecar-cross-account-domain";
@@ -293,7 +310,7 @@ class CodeArtifactNpmDockerIntegrationTest {
* already confirmed is really there, through the real sidecar, not a mock.
*/
@Test
- @Order(9)
+ @Order(10)
void getPackageVersionAssetBridgesToTheRealVerdaccioSidecar() throws IOException {
byte[] fetched = given().header("Authorization", AUTH)
.get("/v1/package/version/asset?domain=" + DOMAIN + "&repository=" + REPO + "&format=npm"
@@ -305,7 +322,7 @@ class CodeArtifactNpmDockerIntegrationTest {
}
@Test
- @Order(9)
+ @Order(10)
void getPackageVersionAssetReturns404ForAnNpmAssetThatWasNeverPublished() {
given().header("Authorization", AUTH)
.get("/v1/package/version/asset?domain=" + DOMAIN + "&repository=" + REPO + "&format=npm"
diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScannerTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScannerTest.java
new file mode 100644
index 0000000000..d9fefabcdf
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScannerTest.java
@@ -0,0 +1,178 @@
+package io.github.hectorvent.floci.services.codeartifact;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.util.Base64;
+import java.util.Map;
+import java.util.Random;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class NpmPublishEnvelopeScannerTest {
+
+ @Test
+ void theRealPublishFixtureHashesToItsOwnDistIntegrity() throws Exception {
+ Path fixture = Path.of("src/test/resources/codeartifact/npm-publish-fixture.json");
+ JsonNode raw = new ObjectMapper().readTree(Files.readAllBytes(fixture));
+
+ NpmPublishEnvelopeScanner.Envelope envelope = NpmPublishEnvelopeScanner.scan(Files.newInputStream(fixture));
+
+ assertEquals("floci-verdaccio-spike", envelope.name());
+ assertEquals(Map.of("latest", "1.0.0"), envelope.distTags());
+ NpmPublishEnvelopeScanner.Version version = envelope.versions().get("1.0.0");
+ assertEquals(raw.at("/versions/1.0.0/dist/integrity").asText(), version.integrity());
+ String filename = raw.at("/versions/1.0.0/dist/tarball").asText().replaceAll(".*/", "");
+ assertEquals(version.integrity(), envelope.attachmentIntegrity().get(filename));
+ }
+
+ @Test
+ void escapesInTrackedValuesAreDecoded() throws Exception {
+ String json = "{\"name\":\"my\\u002dpkg\",\"versions\":{}}";
+
+ NpmPublishEnvelopeScanner.Envelope envelope = scan(json);
+
+ assertEquals("my-pkg", envelope.name());
+ }
+
+ @Test
+ void attachmentDataIsHashedAcrossAStreamOfAnyLength() throws Exception {
+ byte[] tarball = new byte[3 * 1024 * 1024 + 1];
+ new Random(7).nextBytes(tarball);
+ String json = "{\"_attachments\":{\"a.tgz\":{\"data\":\"" + Base64.getEncoder().encodeToString(tarball)
+ + "\"}}}";
+
+ NpmPublishEnvelopeScanner.Envelope envelope = scan(json);
+
+ assertEquals("sha512-" + Base64.getEncoder().encodeToString(sha512(tarball)),
+ envelope.attachmentIntegrity().get("a.tgz"));
+ }
+
+ @Test
+ void anAttachmentLargerThanTheStringLimitStillStreamsThroughTheDigest() throws Exception {
+ byte[] tarball = new byte[30 * 1024 * 1024];
+ new Random(9).nextBytes(tarball);
+ String json = "{\"_attachments\":{\"big.tgz\":{\"data\":\"" + Base64.getEncoder().encodeToString(tarball)
+ + "\"}}}";
+
+ NpmPublishEnvelopeScanner.Envelope envelope = scan(json);
+
+ assertEquals("sha512-" + Base64.getEncoder().encodeToString(sha512(tarball)),
+ envelope.attachmentIntegrity().get("big.tgz"));
+ }
+
+ @Test
+ void anUntrackedStringOfAnySizeIsSkippedAndTheTrackedFieldsAfterItStillRead() throws Exception {
+ String huge = "a".repeat(25 * 1024 * 1024);
+
+ NpmPublishEnvelopeScanner.Envelope envelope = scan("{\"readme\":\"" + huge + "\",\"name\":\"a\"}");
+
+ assertEquals("a", envelope.name());
+ }
+
+ @Test
+ void aTrackedStringBeyondTheLimitIsRejected() {
+ assertInvalid("{\"name\":\"" + "a".repeat(2 * 1024 * 1024) + "\"}");
+ }
+
+ @Test
+ void untrackedStringsAreSkippedWithoutBeingKept() throws Exception {
+ String json = "{\"readme\":\"anything \\\" at all\",\"versions\":{}}";
+
+ NpmPublishEnvelopeScanner.Envelope envelope = scan(json);
+
+ assertNull(envelope.name());
+ assertEquals(Map.of(), envelope.versions());
+ }
+
+ @Test
+ void duplicateKeysAreRejected() {
+ assertInvalid("{\"name\":\"a\",\"name\":\"b\"}");
+ }
+
+ @Test
+ void base64WithAnInvalidCharacterIsRejected() {
+ assertInvalid("{\"_attachments\":{\"a.tgz\":{\"data\":\"ab!d\"}}}");
+ }
+
+ @Test
+ void base64WhoseLengthIsNotAMultipleOfFourIsRejected() {
+ assertInvalid("{\"_attachments\":{\"a.tgz\":{\"data\":\"abcde\"}}}");
+ }
+
+ @Test
+ void attachmentDataContainingAnEscapeIsRejected() {
+ assertInvalid("{\"_attachments\":{\"a.tgz\":{\"data\":\"ab\\/cd\"}}}");
+ }
+
+ @Test
+ void trailingGarbageAfterTheEnvelopeIsRejected() {
+ assertInvalid("{\"name\":\"a\"} extra");
+ }
+
+ @Test
+ void anUnrecognizedLiteralIsRejected() {
+ assertInvalid("{\"readme\":tru}");
+ }
+
+ @Test
+ void nestingDeeperThanTheLimitIsRejected() {
+ String json = "{\"x\":" + "[".repeat(40) + "]".repeat(40) + "}";
+
+ assertInvalid(json);
+ }
+
+ @Test
+ void aNonStringDistTagIsRejectedRatherThanDropped() {
+ assertInvalid("{\"dist-tags\":{\"latest\":1}}");
+ }
+
+ @Test
+ void aNonStringNameIsRejected() {
+ assertInvalid("{\"name\":null}");
+ }
+
+ @Test
+ void aValidNumberLongerThanAnyFixedLimitIsAccepted() throws Exception {
+ String digits = "1".repeat(100_000);
+
+ NpmPublishEnvelopeScanner.Envelope envelope = scan("{\"readme\":" + digits + ".5e-10,\"name\":\"a\"}");
+
+ assertEquals("a", envelope.name());
+ }
+
+ @Test
+ void aMalformedNumberIsRejected() {
+ assertInvalid("{\"readme\":1.}");
+ assertInvalid("{\"readme\":01}");
+ assertInvalid("{\"readme\":-}");
+ }
+
+ @Test
+ void aNonObjectEnvelopeIsRejected() {
+ assertInvalid("[1,2,3]");
+ }
+
+ private static NpmPublishEnvelopeScanner.Envelope scan(String json) throws IOException {
+ InputStream in = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));
+ return NpmPublishEnvelopeScanner.scan(in);
+ }
+
+ private static void assertInvalid(String json) {
+ assertThrows(IOException.class, () -> scan(json));
+ }
+
+ private static byte[] sha512(byte[] bytes) throws Exception {
+ return MessageDigest.getInstance("SHA-512").digest(bytes);
+ }
+}