Light, fluffy, and always free - The AWS Local Emulator alternative
feat(codeartifact): implement GetPackageVersionReadme (#5644)
Priamo Germosen8f907023b2dc
diff --git a/docs/services/codeartifact.md b/docs/services/codeartifact.mdindex 55bac0ac63..49d0674c27 100644--- a/docs/services/codeartifact.md+++ b/docs/services/codeartifact.md@@ -43,6 +43,7 @@ the remaining formats (NuGet, etc.) have no real proxy behind them yet.| `PutPackageOriginConfiguration` | Sets a package's `publish`/`upstream` origin-control restrictions; can create the package (with no versions) if it doesn't exist yet. || `DescribePackageVersion` | Returns a package version's status, revision, and origin. || `GetPackageVersionAsset` | Downloads one asset from a package version by name, optionally pinned to a specific revision. |+| `GetPackageVersionReadme` | Returns the readme text for a package version; `npm`, `pypi`, and `nuget` only, matching AWS's own restriction for this action. || `TagResource` | Adds or updates tags on a domain or repository ARN. || `UntagResource` | Removes tags by key from a domain or repository ARN. || `ListTagsForResource` | Lists the tags on a domain or repository ARN. |diff --git a/docs/services/index.md b/docs/services/index.mdindex 3de8c26b1b..7945a4dd3a 100644--- a/docs/services/index.md+++ b/docs/services/index.md@@ -121,7 +121,7 @@ Operation counts are exact. For dispatch-table services (Query and JSON 1.1) eac| [AWS Backup](backup.md) | `/backup-vaults/*`, `/backup/plans/*`, `/backup-jobs/*`, `/supported-resource-types` | REST JSON | 20 || [AWS FIS](fis.md) | `/experimentTemplates/*`, `/experiments/*`, `/actions/*`, `/targetResourceTypes/*`, `/safetyLevers/*`, `/tags/*` | REST JSON | 26 || [CodeGuru Reviewer](codegurureviewer.md) | `/associations`, `/associations/{associationArn}`, `/tags/*` | REST JSON | 7 |-| [CodeArtifact](codeartifact.md) | `/v1/domain*`, `/v1/repository*`, `/v1/package`, `/v1/package/version*`, `/v1/tag*`, `/v1/authorization-token`, `/codeartifact/maven/*`, `/codeartifact/npm/*`, `/codeartifact/pypi/*` | REST JSON | 29 |+| [CodeArtifact](codeartifact.md) | `/v1/domain*`, `/v1/repository*`, `/v1/package`, `/v1/package/version*`, `/v1/tag*`, `/v1/authorization-token`, `/codeartifact/maven/*`, `/codeartifact/npm/*`, `/codeartifact/pypi/*` | REST JSON | 30 || [CloudFront](cloudfront.md) | `/2020-05-31/distribution/*`, `/2020-05-31/cache-policy/*`, `/2020-05-31/function/*` | REST XML | 50 || [Route53](route53.md) | `/2013-04-01/hostedzone/*`, `/2013-04-01/healthcheck/*`, `/2013-04-01/change/*` | REST XML | 27 || [Route 53 Resolver](route53resolver.md) | `POST /` + `X-Amz-Target: Route53Resolver.*` | JSON 1.1 | 18 |diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactController.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactController.javaindex 0adc31ec61..b5e53cc72b 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactController.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactController.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.services.codeartifact.CodeArtifactService.AuthorizationToken;import io.github.hectorvent.floci.services.codeartifact.CodeArtifactService.DomainView;import io.github.hectorvent.floci.services.codeartifact.CodeArtifactService.PackageVersionAssetResult;+import io.github.hectorvent.floci.services.codeartifact.CodeArtifactService.PackageVersionReadmeResult;import io.github.hectorvent.floci.services.codeartifact.CodeArtifactService.PublishPackageVersionResult;import io.github.hectorvent.floci.services.codeartifact.CodeArtifactService.ResourcePolicy;import io.github.hectorvent.floci.services.codeartifact.model.CodeArtifactPackageVersion;@@ -413,6 +414,21 @@ public class CodeArtifactController {.build();}+ @GET+ @Path("/v1/package/version/readme")+ public Response getPackageVersionReadme(@Context HttpHeaders headers, @QueryParam("domain") String domain,+ @QueryParam("domain-owner") String domainOwner,+ @QueryParam("repository") String repository,+ @QueryParam("format") String format,+ @QueryParam("namespace") String namespace,+ @QueryParam("package") String packageName,+ @QueryParam("version") String version) {+ String region = regionResolver.resolveRegion(headers);+ PackageVersionReadmeResult result = service.getPackageVersionReadme(region, domain, domainOwner, repository,+ format, namespace, packageName, version);+ return ok(packageVersionReadme(result));+ }+// -------------------------------------------------------------------- tags@POST@@ -584,6 +600,19 @@ public class CodeArtifactController {return node;}+ private ObjectNode packageVersionReadme(PackageVersionReadmeResult result) {+ ObjectNode node = objectMapper.createObjectNode();+ node.put("format", result.format());+ if (result.namespace() != null) {+ node.put("namespace", result.namespace());+ }+ node.put("package", result.packageName());+ node.put("version", result.version());+ node.put("versionRevision", result.versionRevision());+ node.put("readme", result.readme());+ return node;+ }+private ObjectNode assetSummary(PackageAsset asset) {ObjectNode node = objectMapper.createObjectNode();node.put("name", asset.getName());diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlane.javaindex bbe73b5521..843531a3b2 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlane.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDataPlane.java@@ -33,6 +33,7 @@ import java.nio.file.Files;import java.nio.file.Path;import java.util.ArrayDeque;import java.util.Deque;+import java.util.HashMap;import java.util.HashSet;import java.util.Map;import java.util.Optional;@@ -193,11 +194,22 @@ public class CodeArtifactNpmDataPlane {* {@link CodeArtifactService#forgetContainerBackedPackage}, once Verdaccio's response* confirms the delete actually happened.*- * <p>Unpublishing a single version of a multi-version package is a different, harder-to-detect- * shape (a {@code PUT} carrying an updated packument with that version removed, the same- * method and the same bare-package path a real publish uses), not handled here: this- * service's own accounting is per-package, not per-version, so there is no narrower count a- * single version's removal could subtract from in the first place.+ * <p>Unpublishing a single version of a multi-version package is a different shape, not+ * handled here or anywhere else in this proxy: confirmed directly against npm/cli's own+ * {@code libnpmpublish/lib/unpublish.js} source (not assumed, and not what an earlier version+ * of this comment claimed) that it is a {@code PUT} to this exact same {@code /-rev/<revision>}+ * path this method already matches, carrying the packument with that one version's entry, its+ * now-dangling dist-tags, and {@code _attachments}/{@code _revisions} removed — a metadata-only+ * rewrite of an existing package, not a publish, and a different operation from the real+ * {@code npm deprecate}, which PUTs to the bare package path instead (confirmed the same way).+ * Left unhandled deliberately for this method's whole-package cleanup — {@link+ * CodeArtifactService#forgetContainerBackedPackage} would incorrectly wipe every surviving+ * version's own accounting and readme for a PUT that only ever removes one of several — and+ * for {@code GetPackageVersionReadme}: a real fix needs its own narrower, per-version cleanup+ * fed by this PUT's own request body (the resulting version set), not this DELETE-shaped+ * signal, and is scoped out of this change rather than hand-built without that plumbing. This+ * service's own byte accounting is per-package, not per-version, so there is no narrower count+ * a single version's removal could subtract from there either way.*/private static Optional<PackageIdentity> nativeUnpublishTarget(HttpMethod method, String rest) {if (method != HttpMethod.DELETE) {@@ -282,7 +294,7 @@ public class CodeArtifactNpmDataPlane {}RepublishCheck check = checkResult.succeeded() ? checkResult.result(): new RepublishCheck(false, 0L,- packageIdentityFromPath(npmRequest.rest()));+ packageIdentityFromPath(npmRequest.rest()), Map.of());if (check.confirmed()) {// "ok" is a message string in a real Verdaccio response// (e.g. "created new package"), not a boolean; matching@@ -320,7 +332,8 @@ public class CodeArtifactNpmDataPlane {npmRequest.rest() + querySuffix(context),reopened.result(), scope, npmRequest.domain(),npmRequest.repository(), check.identity(),- check.totalAttachmentBytes(), fenceSnapshot);+ check.totalAttachmentBytes(), fenceSnapshot,+ check.readmeByVersion());});});});@@ -335,6 +348,13 @@ public class CodeArtifactNpmDataPlane {}}+ /** Same guard as the status-only overload, for a response whose status/headers are already set. */+ private static void respondUnlessGone(RoutingContext context, Buffer body) {+ if (!context.response().ended() && !context.response().closed()) {+ context.response().end(body);+ }+ }+/*** Admits at most {@code permits} publish preflights at once. Waiters queue here rather than* blocking a worker thread, and each one resumes on the event loop it arrived on.@@ -403,16 +423,22 @@ public class CodeArtifactNpmDataPlane {/*** {@code confirmed} is {@code true} only when every version in the publish envelope already- * exists on Verdaccio with the same integrity, every dist-tag the envelope sends already points- * where it says, and every attachment in the envelope is one of those versions' tarballs.- * Client bytes are checked against the integrity the client itself claims, so a mismatch- * between the two falls through too. Anything unreadable or inconsistent is not confirmable and- * forwards unchanged. {@code totalAttachmentBytes} is the real decoded byte count of every- * attachment the envelope carries, regardless of {@code confirmed}: the caller only ever uses- * it when forwarding an unconfirmed publish actually succeeds, since a confirmed republish- * writes no new bytes at all.+ * exists on Verdaccio with the same integrity, every dist-tag the envelope sends already+ * points where it says, and every attachment in the envelope is one of those versions'+ * tarballs. Client bytes are checked against the integrity the client itself claims, so a+ * mismatch between the two falls through too. Anything unreadable or inconsistent is not+ * confirmable and forwards unchanged. {@code totalAttachmentBytes} is the real decoded byte+ * count of every attachment the envelope carries, regardless of {@code confirmed}: the caller+ * only ever uses it when forwarding an unconfirmed publish actually succeeds, since a+ * confirmed republish writes no new bytes at all. {@code readmeByVersion} is populated the+ * same way, regardless of {@code confirmed} (a real envelope's own per-version {@code readme}+ * field, see {@link NpmPublishEnvelopeScanner.Version} for why this, not Verdaccio's own+ * packument response, is the only place it is ever available), and the caller only ever+ * records it on that same successful-forward path: a confirmed republish writes nothing new,+ * so there is nothing to capture a readme change for either.*/- private record RepublishCheck(boolean confirmed, long totalAttachmentBytes, PackageIdentity identity) {}+ private record RepublishCheck(boolean confirmed, long totalAttachmentBytes, PackageIdentity identity,+ Map<String, String> readmeByVersion) {}private RepublishCheck checkRepublish(String backendBaseUrl, NpmRequest npmRequest, String spoolPath) {PackageIdentity identity = packageIdentityFromPath(npmRequest.rest());@@ -422,11 +448,17 @@ public class CodeArtifactNpmDataPlane {} catch (IOException e) {LOG.debugv(e, "Publish for {0}/{1} is not a confirmable republish: {2}",npmRequest.domain(), npmRequest.repository(), e.getMessage());- return new RepublishCheck(false, 0L, identity);+ return new RepublishCheck(false, 0L, identity, Map.of());}long totalAttachmentBytes = envelope.attachments().values().stream().mapToLong(NpmPublishEnvelopeScanner.Attachment::size).sum();+ Map<String, String> readmeByVersion = new HashMap<>();+ envelope.versions().forEach((version, v) -> {+ if (v.readme() != null) {+ readmeByVersion.put(version, v.readme());+ }+ });Optional<JsonNode> storedDocument;try {storedDocument = verdaccioClient.fetchPackageDocument(backendBaseUrl, identity.namespace(),@@ -434,25 +466,25 @@ public class CodeArtifactNpmDataPlane {} catch (IllegalStateException e) {LOG.warnv(e, "Could not check {0}/{1} for an existing match before publishing: {2}",npmRequest.domain(), npmRequest.repository(), e.getMessage());- return new RepublishCheck(false, totalAttachmentBytes, identity);+ return new RepublishCheck(false, totalAttachmentBytes, identity, readmeByVersion);}if (storedDocument.isEmpty() || !isConfirmedRepublish(envelope, storedDocument.get(), identity)) {- return new RepublishCheck(false, totalAttachmentBytes, identity);+ return new RepublishCheck(false, totalAttachmentBytes, identity, readmeByVersion);}for (NpmPublishEnvelopeScanner.Version version : envelope.versions().values()) {try {Optional<String> stored = verdaccioClient.storedTarballIntegrity(backendBaseUrl,identity.namespace(), identity.packageName(), tarballFilename(version.tarball()));if (stored.isEmpty() || !stored.get().equals(version.integrity())) {- return new RepublishCheck(false, totalAttachmentBytes, identity);+ return new RepublishCheck(false, totalAttachmentBytes, identity, readmeByVersion);}} catch (IllegalStateException e) {LOG.warnv(e, "Could not verify the stored tarball of {0}/{1} before publishing: {2}",npmRequest.domain(), npmRequest.repository(), e.getMessage());- return new RepublishCheck(false, totalAttachmentBytes, identity);+ return new RepublishCheck(false, totalAttachmentBytes, identity, readmeByVersion);}}- return new RepublishCheck(true, totalAttachmentBytes, identity);+ return new RepublishCheck(true, totalAttachmentBytes, identity, readmeByVersion);}private static boolean isConfirmedRepublish(NpmPublishEnvelopeScanner.Envelope envelope, JsonNode stored,@@ -529,18 +561,25 @@ public class CodeArtifactNpmDataPlane {}/**- * {@code identity}/{@code totalAttachmentBytes} are only used to record real published bytes+ * {@code identity}/{@code totalAttachmentBytes}/{@code readmeByVersion} are only ever recorded* once Verdaccio's own response confirms the publish actually succeeded (200 or 201): a* forward that Verdaccio itself rejects (409 on a genuine conflict, or anything else) writes- * nothing, and must not be counted as though it had. {@code totalAttachmentBytes} is already 0- * when the envelope couldn't be parsed (see {@link #checkRepublish}), so this is a no-op in- * that case regardless of what Verdaccio answers. {@code fenceSnapshot} is whatever {@link- * CodeArtifactService#currentContainerBackedPackageFence} returned at the caller's own earlier- * snapshot point, before this request was ever forwarded.+ * nothing, and must not be counted or recorded as though it had. {@code totalAttachmentBytes}+ * is already 0, and {@code readmeByVersion} already empty,+ * when the envelope couldn't be parsed (see {@link #checkRepublish}), so both become no-ops+ * in that case regardless of what Verdaccio answers. {@code fenceSnapshot} is whatever+ * {@link CodeArtifactService#currentContainerBackedPackageFence} returned at the caller's own+ * earlier snapshot point, before this request was ever forwarded.+ *+ * <p>All of this bookkeeping runs in one {@code executeBlocking} call, and the response is+ * only piped to the client once that call completes, not before: a client that requests this+ * version's readme immediately after seeing its publish succeed must not be able to observe a+ * window where the publish already finished but the readme has not been recorded yet.*/private void forwardSpooled(RoutingContext context, String backendBaseUrl, String backendPath,ReadStream<Buffer> body, AuthorizationTokenScope scope, String domain, String repository,- PackageIdentity identity, long totalAttachmentBytes, long fenceSnapshot) {+ PackageIdentity identity, long totalAttachmentBytes, long fenceSnapshot,+ Map<String, String> readmeByVersion) {URI backend = URI.create(backendBaseUrl);RequestOptions options = new RequestOptions().setHost(backend.getHost())@@ -561,24 +600,54 @@ public class CodeArtifactNpmDataPlane {}HttpClientResponse response = upstreamResponse.result();copyResponseHeaders(context, response);- response.pipeTo(context.response());- // Piped on first, not after: the client is never made to wait on the call below.- // That call is fast and in-memory-only now (no sidecar round trip of its own,- // unlike the one this replaced), but it still runs off the event loop, in- // executeBlocking, on principle: CodeArtifactService is a plain CDI bean Vert.x- // knows nothing about, so nothing here can guarantee it stays that way forever,- // and the cost of getting this wrong (stalling every request on this event loop,- // not just this one) is high enough that "it happens to be cheap today" is not a- // reason to call a synchronized service method straight from a response callback.- if (totalAttachmentBytes > 0 && (response.statusCode() == 200 || response.statusCode() == 201)) {+ // response.body() attaches its handler on this same event-loop tick, same as+ // pipeTo used to: a publish response is always a small JSON acknowledgement, never+ // tarball-sized, so buffering it rather than streaming it costs nothing. Unlike+ // pipeTo, buffering lets the client-visible response wait on the bookkeeping below+ // without risking the response stream auto-ending for having no consumer attached+ // in time (confirmed the hard way: deferring pipeTo itself past this tick threw+ // "Response already ended").+ response.body().onComplete(bodyResult -> {+ if (bodyResult.failed()) {+ respondUnlessGone(context, 502);+ return;+ }+ Buffer responseBody = bodyResult.result();+ boolean forwardSucceeded = response.statusCode() == 200 || response.statusCode() == 201;+ boolean needsBytes = totalAttachmentBytes > 0 && forwardSucceeded;+ boolean needsReadmeCapture = !readmeByVersion.isEmpty() && forwardSucceeded;+ if (!needsBytes && !needsReadmeCapture) {+ respondUnlessGone(context, responseBody);+ return;+ }+ // Each still runs off the event loop, in executeBlocking, on principle:+ // CodeArtifactService is a plain CDI bean Vert.x knows nothing about, so+ // nothing here can guarantee it stays that way forever, and the cost of+ // getting this wrong (stalling every request on this event loop, not just this+ // one) is high enough that "it happens to be cheap today" is not a reason to+ // call a synchronized service method straight from a response callback.vertx.<Void>executeBlocking(promise -> {- service.recordContainerBackedAssetBytes(scope.region(), domain, scope.owner(), repository,- "npm", identity.namespace(), identity.packageName(), totalAttachmentBytes,- fenceSnapshot);+ if (needsBytes) {+ service.recordContainerBackedAssetBytes(scope.region(), domain, scope.owner(),+ repository, "npm", identity.namespace(), identity.packageName(),+ totalAttachmentBytes, fenceSnapshot);+ }+ if (needsReadmeCapture) {+ readmeByVersion.forEach((version, readme) ->+ service.recordPublishedPackageVersionReadme(scope.region(), domain,+ scope.owner(), repository, "npm", identity.namespace(),+ identity.packageName(), version, readme, fenceSnapshot));+ }promise.complete();- }, false).onFailure(e -> LOG.warnv(e, "Could not record npm asset bytes for {0}/{1} package "- + "''{2}'': {3}", domain, repository, identity.packageName(), e.getMessage()));- }+ }, false).onComplete(result -> {+ if (result.failed()) {+ LOG.warnv(result.cause(), "Could not record npm publish bookkeeping for {0}/{1} "+ + "package ''{2}'': {3}", domain, repository, identity.packageName(),+ result.cause().getMessage());+ }+ respondUnlessGone(context, responseBody);+ });+ });});clientReq.send(body).onFailure(ignored -> respondUnlessGone(context, 502));});diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.javaindex dff1c48ac9..5b16f303d3 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiController.java@@ -164,6 +164,7 @@ public class CodeArtifactPypiController {}MultipartFormParser.ParsedForm form = MultipartFormParser.parse(content, boundary);String packageName = form.fields().get("name");+ String version = form.fields().get("version");Optional<MultipartFormParser.FilePart> file = form.file();if (packageName == null || packageName.isBlank() || file.isEmpty()) {return Response.status(400).entity("Upload must include name and content fields.").build();@@ -188,10 +189,22 @@ public class CodeArtifactPypiController {long fenceSnapshot = service.currentContainerBackedPackageFence(handle.region(), domain, handle.owner(),repository, "pypi", null, packageName);Response response = forwardUpload(handle.baseUrl(), contentType, content);- if (response.getStatus() == 200 || response.getStatus() == 201) {+ boolean uploadSucceeded = response.getStatus() == 200 || response.getStatus() == 201;+ if (uploadSucceeded) {service.recordContainerBackedAssetBytes(handle.region(), domain, handle.owner(), repository,"pypi", null, packageName, file.get().content().length, fenceSnapshot);}+ // Only for a real new publish, never the byte-identical-retry short-circuit above+ // (nothing new was written there either): the "description" field is the real PyPI+ // legacy upload API's form field for a distribution's long description, the only place+ // this text is ever available (see CodeArtifactService#getPackageVersionReadme's own+ // javadoc for why pypiserver itself can't answer for it later). A missing/blank version+ // field means twine never named a version for this upload, so there is no package+ // version coordinate to record the readme against.+ if (uploadSucceeded && version != null && !version.isBlank()) {+ service.recordPublishedPackageVersionReadme(handle.region(), domain, handle.owner(), repository,+ "pypi", null, packageName, version, form.fields().get("description"), fenceSnapshot);+ }return response;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.javaindex dbf6d0f5c5..2438420195 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java@@ -79,6 +79,15 @@ public class CodeArtifactService implements Resettable {*/private static final List<String> CONTAINER_BACKED_FORMATS = List.of("maven", "npm", "pypi");private static final Set<String> ORIGIN_RESTRICTION_VALUES = Set.of("ALLOW", "BLOCK");+ /**+ * Formats AWS documents as valid for {@code GetPackageVersionReadme} specifically: the+ * action's own request parameter doc gives "Valid Values: npm | pypi | nuget", narrower than+ * the shared eight-value {@code PackageFormat} wire enum every other package-version action+ * here accepts. Maven, Swift, generic, Ruby, and Cargo package versions carry no readme at+ * all in real CodeArtifact, so a request for one of those formats is rejected outright rather+ * than silently 404ing the way an unimplemented capability elsewhere here might.+ */+ private static final Set<String> README_FORMATS = Set.of("npm", "pypi", "nuget");private static final Logger LOG = Logger.getLogger(CodeArtifactService.class);@@ -107,6 +116,8 @@ public class CodeArtifactService implements Resettable {public record ResourcePolicy(String resourceArn, String revision, String document) {}public record PublishPackageVersionResult(CodeArtifactPackageVersion packageVersion, PackageAsset asset) {}public record PackageVersionAssetResult(PackageAsset asset, String packageVersionRevision) {}+ public record PackageVersionReadmeResult(String format, String namespace, String packageName, String version,+ String versionRevision, String readme) {}public record AuthorizationToken(String token, long expirationEpochSeconds) {}public record AuthorizationTokenScope(String owner, String region) {}private record AuthorizationTokenRecord(String owner, String region, String domain, Instant expiration) {}@@ -144,6 +155,16 @@ public class CodeArtifactService implements Resettable {// a stale total surviving a restart for bytes that no longer exist anywhere, with nothing to// ever naturally clean it up; kept in memory instead, so a restart loses both together.private final ConcurrentHashMap<String, Long> containerBackedAssetBytes = new ConcurrentHashMap<>();+ // Keyed the same way as packageVersions (packageVersionKey): npm's per-version readme (from+ // the publish envelope, see recordPublishedPackageVersionReadme) and pypi's upload+ // "description" form field both land here, since neither sidecar can answer for either after+ // the fact (see getPackageVersionReadme's own javadoc). In memory only, like the fence and+ // containerBackedAssetBytes above and for the exact same reason: the Verdaccio/pypiserver+ // content a readme describes does not survive a Floci restart either (no volume attached,+ // removed on shutdown), so persisting the readme text but not the package version it+ // describes would leave it answering for a version that, after a restart, no longer exists+ // anywhere real.+ private final ConcurrentHashMap<String, String> packageVersionReadmes = new ConcurrentHashMap<>();@Injectpublic CodeArtifactService(StorageFactory storageFactory, RegionResolver regionResolver, EmulatorConfig config,@@ -389,6 +410,15 @@ public class CodeArtifactService implements Resettable {}}}+ // Same reasoning, same prefix: packageVersionReadmes is also keyed by name+ // (region/domain/repository/format/namespace/package/version), not a freshly minted id, so+ // a pypi/npm package republished to a repository recreated under this same name must not+ // inherit a readme captured for a version with the same coordinates under the deleted+ // repository.+ packageVersionReadmes.keySet().stream()+ .filter(k -> k.startsWith(ownerScopedOriginPrefix))+ .toList()+ .forEach(packageVersionReadmes::remove);return r;}@@ -895,6 +925,18 @@ public class CodeArtifactService implements Resettable {containerBackedPackageDeletionFence.put(ownerScopedKey(owner, accountingKey),containerBackedPackageDeletionCounter.incrementAndGet());}+ // Harmless to do unconditionally: without this, a package republished under the same+ // namespace/packageName/version after being deleted would inherit the deleted+ // publish's readme text instead of whatever (or nothing) the new publish itself sent.+ // Built from accountingKey, not the raw originKey above: readme capture/lookup key off+ // accountingPackageName the same way the fence and byte accounting just above do (see+ // accountingPackageName's own javadoc), so a pypi package deleted under one case/+ // punctuation spelling still finds the readme captured under another.+ String ownerScopedReadmePrefix = ownerScopedKey(owner, accountingKey) + "::";+ packageVersionReadmes.keySet().stream()+ .filter(k -> k.startsWith(ownerScopedReadmePrefix))+ .toList()+ .forEach(packageVersionReadmes::remove);} else {List<CodeArtifactPackageVersion> versions = genericPackageVersions(region, domain, owner, repository,format, namespace, packageName);@@ -1069,8 +1111,7 @@ public class CodeArtifactService implements Resettable {throw validation("namespace is required when requesting an asset from a maven package version.");}String repoId = ensureFormatContainerId(format, region, domain, owner, repository);- String revisionKey = repoId + "/" + (namespace == null ? "" : namespace) + "/" + packageName + "/" + version;- String revision = Digests.sha256Hex(revisionKey.getBytes(StandardCharsets.UTF_8));+ String revision = synthesizeContainerBackedRevision(repoId, namespace, packageName, version);if (packageVersionRevision != null && !packageVersionRevision.equals(revision)) {throw notFound("Package version '" + version + "' was not found at revision '"+ packageVersionRevision + "'.", version, "package-version");@@ -1090,6 +1131,142 @@ public class CodeArtifactService implements Resettable {return new PackageVersionAssetResult(asset, revision);}+ /**+ * The same synthesized revision {@link #getContainerBackedPackageVersionAsset} uses, shared+ * so {@link #getPackageVersionReadme}'s npm/pypi answers report a revision consistent with+ * whatever {@code GetPackageVersionAsset} would already report for the same version+ * coordinate, rather than two independently-computed stand-ins that happen to differ.+ */+ private static String synthesizeContainerBackedRevision(String repoId, String namespace, String packageName,+ String version) {+ String revisionKey = repoId + "/" + (namespace == null ? "" : namespace) + "/" + packageName + "/" + version;+ return Digests.sha256Hex(revisionKey.getBytes(StandardCharsets.UTF_8));+ }++ /**+ * Gets the readme text for one package version. AWS documents this action for npm, pypi, and+ * nuget only (see {@link #README_FORMATS}), unlike every other package-version action here,+ * which accepts the full eight-format set: Maven, Swift, generic, Ruby, and Cargo package+ * versions carry no readme in real CodeArtifact, and this rejects them outright with+ * {@code ValidationException} rather than quietly answering not-found.+ *+ * <p>Neither npm nor pypi can be answered live from the sidecar itself, confirmed directly+ * rather than assumed for either one. npm: a real npm client's publish envelope does carry a+ * per-version {@code readme} (confirmed against a real captured publish fixture), but+ * Verdaccio's own packument response does not echo it back — checked against a live Verdaccio+ * container, which hoists {@code readme} to the top level only, for the single+ * most-recently-published version, truncated to 64K, exactly like real npmjs.org's own+ * documented response shape; the per-version field in a GET response simply does not exist.+ * pypi: pypiserver parses only a distribution's filename, never its embedded metadata (also+ * confirmed directly: its own {@code PkgFile} model has no description-shaped field anywhere,+ * and its {@code /json} route's own code comment says outright "we do not presently have any+ * description/summary"). For both, the one place the text exists at all is the original+ * publish/upload itself (npm's per-version {@code readme} field; pypi's twine upload's+ * {@code description} form field, the real PyPI legacy upload API's field for a distribution's+ * long description, in Markdown or reStructuredText per AWS's own doc wording for this very+ * action); see {@link #recordPublishedPackageVersionReadme}, the only place either is ever+ * captured.+ *+ * <p>nuget is accepted, since AWS documents it as valid, but never finds anything: nothing in+ * Floci today implements the real NuGet wire protocol, so no nuget package version can ever+ * actually exist here. This 404s for it exactly the same way as any npm/pypi version nothing+ * ever published, rather than special-casing a format with no data behind it either way.+ */+ public PackageVersionReadmeResult getPackageVersionReadme(String region, String domain, String domainOwner,+ String repository, String format, String namespace, String packageName, String version) {+ if (format == null || !README_FORMATS.contains(format)) {+ throw validation("format must be one of " + README_FORMATS + ".");+ }+ requireNonBlank(domain, "domain");+ requireNonBlank(repository, "repository");+ requireNonBlank(packageName, "package");+ requireNonBlank(version, "packageVersion");+ validatePackageToken("package", packageName);+ validatePackageToken("packageVersion", version);+ if (namespace != null) {+ validatePackageToken("namespace", namespace);+ }+ String owner = effectiveOwner(domainOwner);+ requireRepository(owner, repositoryKey(region, domain, repository), repository);+ // npm, pypi, and nuget alike: none of the three can be answered live (see this method's+ // own javadoc), so a version that was never captured here means it was never published,+ // including every nuget version, since nothing ever publishes one. Checked before+ // ensureFormatContainerId below so a nuget request, which can only ever 404, never mints+ // and persists a sidecar container id for a format with no sidecar at all.+ String key = packageVersionKey(region, domain, repository, format, namespace,+ accountingPackageName(format, packageName), version);+ String readme = Optional.ofNullable(packageVersionReadmes.get(ownerScopedKey(owner, key)))+ .orElseThrow(() -> notFound("Package version '" + version + "' of package '" + packageName+ + "' was not found.", version, "package-version"));+ String repoId = ensureFormatContainerId(format, region, domain, owner, repository);+ String revision = synthesizeContainerBackedRevision(repoId, namespace, packageName, version);+ return new PackageVersionReadmeResult(format, namespace, packageName, version, revision, readme);+ }++ /**+ * Captures the readme text for one published npm or pypi package version at the one moment it+ * is ever available: right after the real upload/publish to the sidecar actually succeeded.+ * See {@link #getPackageVersionReadme}'s own javadoc for why both formats need this; neither+ * sidecar can answer for a specific version after the fact.+ *+ * <p>{@code namespace}/{@code packageName}/{@code version} arrive straight from the wire+ * protocol here (an npm publish envelope's own per-version field, or a twine upload's {@code+ * name}/{@code version} form fields), never through {@link #validatePackageToken} the way+ * every JSON-API package operation's own coordinates already are. {@link #escapeKeySegment}'s+ * whole collision-safety argument rests on no namespace/package/version that reaches it ever+ * containing a raw {@code '#'}; silently skipping one that does (rather than escaping or+ * rejecting outright) keeps that invariant true for this new path into {@link #packageVersionKey}+ * without turning a client's publish, which already succeeded against the real sidecar, into a+ * thrown exception over what is only a best-effort readme capture.+ *+ * <p>{@code synchronized}, and re-checks the repository still exists, for the same reason+ * {@code recordContainerBackedAssetBytes}-style accounting calls elsewhere in this codebase+ * do: the caller's own "does this repository still exist" check ran in a separate,+ * already-released {@code synchronized} call ({@link #ensureFormatContainerId}, inside each+ * proxy's own {@code requireRepository} path), before the real HTTP round trip to the sidecar+ * that this call reports the result of. A {@code DeleteRepository} landing in that window+ * would otherwise resurrect a key {@link #deleteRepository} already cleaned up. Re-checking+ * existence here, inside the same {@code synchronized} call that does the write, closes that+ * window the same way every other mutating method here already does.+ *+ * <p>{@code packageFenceSnapshot} is whatever {@link #currentContainerBackedPackageFence}+ * returned when the proxy called it, right before issuing the real write this call reports the+ * readme of — the exact same snapshot the proxy already takes to call+ * {@link #recordContainerBackedAssetBytes}. Without comparing it here too, a publish's readme+ * capture that runs on a delayed callback (npm's own, off the event loop) after a concurrent+ * {@link #deletePackage} or native unpublish has already removed this package would resurrect+ * a readme for a package {@link #getPackageVersionReadme} should now 404 on, the same window+ * {@code recordContainerBackedAssetBytes} already closes for byte accounting.+ */+ public synchronized void recordPublishedPackageVersionReadme(String region, String domain, String domainOwner,+ String repository, String format, String namespace, String packageName, String version,+ String readme, long packageFenceSnapshot) {+ if (!isValidPackageToken(packageName) || !isValidPackageToken(version)+ || (namespace != null && !isValidPackageToken(namespace))) {+ LOG.warnv("Skipping readme capture for {0}/{1} package ''{2}'' version ''{3}'': the namespace, name, "+ + "or version contains a character ('#', '/', or whitespace) a real CodeArtifact package "+ + "coordinate cannot have.", domain, repository, packageName, version);+ return;+ }+ String owner = effectiveOwner(domainOwner);+ if (repositories.getForAccount(owner, repositoryKey(region, domain, repository)).isEmpty()) {+ LOG.warnv("Skipping readme capture for {0}/{1}: the repository was deleted while this publish was in "+ + "flight.", domain, repository);+ return;+ }+ String accountingName = accountingPackageName(format, packageName);+ String fenceKey = packageKey(region, domain, repository, format, namespace, accountingName);+ long currentFence = containerBackedPackageDeletionFence.getOrDefault(ownerScopedKey(owner, fenceKey), 0L);+ if (currentFence != packageFenceSnapshot) {+ LOG.warnv("Skipping readme capture for {0}/{1} package ''{2}'': a DeletePackage removed it from the "+ + "{3} sidecar (possibly replaced by a different publish) while this publish was in flight.",+ domain, repository, packageName, format);+ return;+ }+ String key = packageVersionKey(region, domain, repository, format, namespace, accountingName, version);+ packageVersionReadmes.put(ownerScopedKey(owner, key), readme == null ? "" : readme);+ }+// ------------------------------------------------------------ authorizationpublic AuthorizationToken getAuthorizationToken(String region, String domain, String domainOwner,@@ -1198,6 +1375,7 @@ public class CodeArtifactService implements Resettable {packageOriginConfigs.clear();containerBackedAssetBytes.clear();containerBackedPackageDeletionFence.clear();+ packageVersionReadmes.clear();authorizationTokens.clear();if (inMemory) {memoryAssetStore.clear();@@ -1385,15 +1563,18 @@ public class CodeArtifactService implements Resettable {}/**- * Cleans up this service's own byte accounting and deletion fence for one container-backed- * package after something other than {@link #deletePackage} removed it directly from the- * sidecar: specifically, a real npm client's native {@code npm unpublish}, which the pinned- * Verdaccio image (configured with {@code unpublish: $all}) honors on its own wire protocol,- * bypassing {@link #deletePackage} entirely (confirmed: {@code CodeArtifactNpmDataPlane}- * forwards any request that is not a publish straight through). {@link #deletePackage}'s own- * container-backed branch already does both of these same two things as part of deleting the- * real sidecar content too; this is that same bookkeeping on its own, for when the sidecar- * content is already gone by a path {@link #deletePackage} never saw.+ * Cleans up this service's own byte accounting, deletion fence, and captured readmes for one+ * container-backed package after something other than {@link #deletePackage} removed it+ * directly from the sidecar: specifically, a real npm client's native {@code npm unpublish},+ * which the pinned Verdaccio image (configured with {@code unpublish: $all}) honors on its own+ * wire protocol, bypassing {@link #deletePackage} entirely (confirmed: {@code+ * CodeArtifactNpmDataPlane} forwards any request that is not a publish straight through).+ * {@link #deletePackage}'s own container-backed branch already does all three of these same+ * things as part of deleting the real sidecar content too; this is that same bookkeeping on+ * its own, for when the sidecar content is already gone by a path {@link #deletePackage} never+ * saw. Without the readme cleanup here, {@link #getPackageVersionReadme} would keep answering+ * for a package a real npm registry (and real CodeArtifact) now 404s on, forever, since nothing+ * else ever revisits that store once it's written.** <p>{@code namespace}/{@code packageName} come straight from the wire protocol here too (the* unpublish request's own path), so this guards the same escape-collision invariant@@ -1413,6 +1594,14 @@ public class CodeArtifactService implements Resettable {String ownerScopedKey = ownerScopedKey(owner, key);containerBackedAssetBytes.remove(ownerScopedKey);containerBackedPackageDeletionFence.put(ownerScopedKey, containerBackedPackageDeletionCounter.incrementAndGet());+ // Same key as the accounting/fence entries above: recordPublishedPackageVersionReadme now+ // normalizes through accountingPackageName the same way (see its own javadoc), so this+ // prefix matches whatever it wrote regardless of format.+ String readmePrefix = ownerScopedKey + "::";+ packageVersionReadmes.keySet().stream()+ .filter(k -> k.startsWith(readmePrefix))+ .toList()+ .forEach(packageVersionReadmes::remove);}/**@@ -1561,6 +1750,15 @@ public class CodeArtifactService implements Resettable {}}+ /**+ * The non-throwing form of {@link #validatePackageToken}, for a caller like+ * {@link #recordContainerBackedAssetBytes} or {@link #recordPublishedPackageVersionReadme}+ * that takes a namespace/package/version straight from a wire protocol rather than the JSON+ * API's own validated request fields, and must skip a value that fails this rather than throw:+ * the real write it guards already succeeded against a live sidecar by the time it runs, so+ * rejecting it here would turn that success into a client-visible error over what is only+ * best-effort bookkeeping.+ */private static boolean isValidPackageToken(String value) {return value != null && !value.isEmpty() && value.length() <= 255 && PACKAGE_TOKEN.matcher(value).matches();}diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScanner.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScanner.javaindex 1ef27b8d9e..6967dd71ff 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScanner.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScanner.java@@ -17,14 +17,26 @@ import java.util.HashMap;import java.util.Map;/**- * Reads an npm publish envelope as a stream and keeps only what the republish check needs. Jackson- * decodes each attachment's base64 {@code data} straight into a SHA-512 digest, so the tarball is never- * held in memory. Any structure this scanner does not expect throws, and callers treat a throw as not- * confirmable and forward the request unchanged.+ * Reads an npm publish envelope as a stream and keeps only what the republish check, plus+ * {@code GetPackageVersionReadme}'s own capture-at-publish-time accounting, needs. Jackson+ * decodes each attachment's base64 {@code data} straight into a SHA-512 digest, so the tarball is+ * never held in memory. Any structure this scanner does not expect throws, and callers treat a+ * throw as not confirmable and forward the request unchanged.*/final class NpmPublishEnvelopeScanner {- record Version(String tarball, String integrity) {}+ /**+ * {@code readme} is each version's own, exactly as a real npm client sends it (confirmed+ * against a real captured publish fixture): a real {@code npm publish} always sends one, even+ * when the package has no README.md at all ({@code "ERROR: No README data found!"}, npm's own+ * literal placeholder text, not an absent field). {@code null} only for a client that omits+ * the field outright. Deliberately not sourced from Verdaccio's own packument later: confirmed+ * directly against a live Verdaccio container that it hoists {@code readme} to the top level+ * only (the latest published version, truncated to 64K, matching real npmjs.org's own+ * documented response shape) and drops the per-version field entirely, so this is the only+ * point where a specific version's own readme is still available at all.+ */+ record Version(String tarball, String integrity, String readme) {}/** {@code size} is the attachment's real, decoded byte count, counted alongside its digest. */record Attachment(String integrity, long size) {}@@ -102,6 +114,18 @@ final class NpmPublishEnvelopeScanner {parser.skipChildren();}});+ } else if (field.equals("readme")) {+ // Best-effort: unlike dist.tarball/integrity, which a confirmable envelope must+ // have, a readme missing or of the wrong shape (null, an object, ...) must not+ // reject the whole envelope the way an actually malformed required field does.+ // Before this field existed at all, any shape here fell to the default branch's+ // plain skipChildren() below; this keeps that same tolerance for this one field+ // now that it is read instead of skipped.+ if (parser.currentToken() == JsonToken.VALUE_STRING) {+ fields.put("readme", parser.getText());+ } else {+ parser.skipChildren();+ }} else {parser.skipChildren();}@@ -175,7 +199,8 @@ final class NpmPublishEnvelopeScanner {Map<String, Version> versions = new HashMap<>();for (Map.Entry<String, Map<String, String>> entry : versionFields.entrySet()) {Map<String, String> fields = entry.getValue();- versions.put(entry.getKey(), new Version(fields.get("tarball"), fields.get("integrity")));+ versions.put(entry.getKey(), new Version(fields.get("tarball"), fields.get("integrity"),+ fields.get("readme")));}return new Envelope(name, distTags, versions, attachments);}diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.javaindex a2da209ab8..01b1910659 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java@@ -435,6 +435,125 @@ class CodeArtifactNpmDockerIntegrationTest {assertEquals(assetSizeBytesBeforePublish, describeDomainAssetSizeBytes());}+ /**+ * A second real finding found alongside the npm readme redesign itself: the native unpublish+ * path above cleans up the byte count and the deletion fence, but a first version of that same+ * fix left {@code packageVersionReadmes} untouched, so {@code GetPackageVersionReadme} would+ * keep answering for a package a real npm registry (and real CodeArtifact) now 404s on,+ * forever, since nothing else ever revisits that store once it's written. A throwaway+ * package, not {@link #PACKAGE_NAME}, for the same reason as the tests above.+ */+ @Test+ @Order(13)+ void nativeNpmUnpublishCleansUpTheCapturedReadmeToo() {+ String packageName = "unpublish-readme-pkg";+ byte[] tarball = "unpublish readme throwaway tarball".getBytes(StandardCharsets.UTF_8);+ String envelope = "{\"name\":\"" + packageName + "\",\"versions\":{\"1.0.0\":{\"name\":\"" + packageName+ + "\",\"version\":\"1.0.0\",\"readme\":\"# Soon to be unpublished\",\"dist\":{\"tarball\":"+ + "\"http://ignored/" + packageName + "-1.0.0.tgz\""+ + "}}},\"_attachments\":{\"" + packageName + "-1.0.0.tgz\":{\"content_type\":"+ + "\"application/octet-stream\",\"data\":\""+ + Base64.getEncoder().encodeToString(tarball) + "\",\"length\":" + tarball.length + "}}}";++ given().header("Authorization", "Bearer " + bearerToken).contentType("application/json").body(envelope)+ .put("/codeartifact/npm/" + DOMAIN + "/" + REPO + "/" + packageName)+ .then().statusCode(anyOf201Or200());++ given().header("Authorization", AUTH)+ .get("/v1/package/version/readme?domain=" + DOMAIN + "&repository=" + REPO + "&format=npm&package="+ + packageName + "&version=1.0.0")+ .then().statusCode(200)+ .body("readme", equalTo("# Soon to be unpublished"));++ String rev = given().header("Authorization", "Bearer " + bearerToken)+ .get("/codeartifact/npm/" + DOMAIN + "/" + REPO + "/" + packageName)+ .then().statusCode(200)+ .extract().jsonPath().getString("_rev");++ given().header("Authorization", "Bearer " + bearerToken)+ .delete("/codeartifact/npm/" + DOMAIN + "/" + REPO + "/" + packageName + "/-rev/" + rev)+ .then().statusCode(anyOf201Or200());++ given().header("Authorization", AUTH)+ .get("/v1/package/version/readme?domain=" + DOMAIN + "&repository=" + REPO + "&format=npm&package="+ + packageName + "&version=1.0.0")+ .then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));+ }++ /**+ * Each of a multi-version npm package's versions is published in its own {@code PUT}, exactly+ * like a real {@code npm publish} (confirmed directly: an earlier draft of this test tried+ * publishing two versions in one envelope and Verdaccio itself rejected it outright, {@code 400+ * unsupported registry call}). Proves readme capture is genuinely per-version, not just+ * per-package: publishing the second version must not disturb the first's already-captured+ * readme.+ *+ * <p>Unpublishing a single version of a multi-version npm package (real wire shape confirmed+ * against npm/cli's own source: a {@code PUT} to the same {@code /-rev/<rev>} path the+ * whole-package {@code DELETE} unpublish already uses, carrying the packument with that one+ * version removed) is not detected anywhere in this proxy, deliberately: see+ * {@code CodeArtifactNpmDataPlane#nativeUnpublishTarget}'s own javadoc for why, left as a+ * documented, scoped-out limitation rather than fixed without the request-body-reading+ * plumbing a correct, narrower fix for it actually needs.+ */+ @Test+ @Order(14)+ void publishingASecondVersionDoesNotDisturbTheFirstVersionsCapturedReadme() {+ String packageName = "multi-version-readme-pkg";+ byte[] tarballV1 = "multi version readme v1".getBytes(StandardCharsets.UTF_8);+ byte[] tarballV2 = "multi version readme v2".getBytes(StandardCharsets.UTF_8);+ String envelopeV1 = "{\"name\":\"" + packageName + "\",\"versions\":{\"1.0.0\":{\"name\":\""+ + packageName + "\",\"version\":\"1.0.0\",\"readme\":\"# v1\",\"dist\":{\"tarball\":"+ + "\"http://ignored/" + packageName + "-1.0.0.tgz\"}}},\"_attachments\":{\"" + packageName+ + "-1.0.0.tgz\":{\"content_type\":\"application/octet-stream\",\"data\":\""+ + Base64.getEncoder().encodeToString(tarballV1) + "\",\"length\":" + tarballV1.length + "}}}";+ String envelopeV2 = "{\"name\":\"" + packageName + "\",\"versions\":{\"2.0.0\":{\"name\":\""+ + packageName + "\",\"version\":\"2.0.0\",\"readme\":\"# v2\",\"dist\":{\"tarball\":"+ + "\"http://ignored/" + packageName + "-2.0.0.tgz\"}}},\"_attachments\":{\"" + packageName+ + "-2.0.0.tgz\":{\"content_type\":\"application/octet-stream\",\"data\":\""+ + Base64.getEncoder().encodeToString(tarballV2) + "\",\"length\":" + tarballV2.length + "}}}";++ given().header("Authorization", "Bearer " + bearerToken).contentType("application/json").body(envelopeV1)+ .put("/codeartifact/npm/" + DOMAIN + "/" + REPO + "/" + packageName)+ .then().statusCode(anyOf201Or200());+ given().header("Authorization", "Bearer " + bearerToken).contentType("application/json").body(envelopeV2)+ .put("/codeartifact/npm/" + DOMAIN + "/" + REPO + "/" + packageName)+ .then().statusCode(anyOf201Or200());++ given().header("Authorization", AUTH)+ .get("/v1/package/version/readme?domain=" + DOMAIN + "&repository=" + REPO + "&format=npm&package="+ + packageName + "&version=1.0.0")+ .then().statusCode(200).body("readme", equalTo("# v1"));+ given().header("Authorization", AUTH)+ .get("/v1/package/version/readme?domain=" + DOMAIN + "&repository=" + REPO + "&format=npm&package="+ + packageName + "&version=2.0.0")+ .then().statusCode(200).body("readme", equalTo("# v2"));+ }++ /**+ * The fix for a real finding: a first implementation answered this live from Verdaccio's own+ * packument, which turned out, checked directly against this same live container, to hoist+ * {@code readme} to the top level only (the single most-recently-published version), dropping+ * the per-version field entirely from its own GET response — even though a real npm client's+ * publish envelope (the real fixture {@link #setUp} loads, captured from an actual {@code npm+ * publish}) genuinely sends one per version. This exercises the real capture-at-publish-time+ * path end to end against that same live container: the exact text the fixture's own+ * {@code versions["1.0.0"].readme} field carries ({@code "ERROR: No README data found!"},+ * npm's own literal placeholder for a package with no real {@code README.md}, not an absent+ * field) must come back unchanged.+ */+ @Test+ @Order(15)+ void getPackageVersionReadmeReturnsWhatTheRealPublishEnvelopeSent() {+ given().header("Authorization", AUTH)+ .get("/v1/package/version/readme?domain=" + DOMAIN + "&repository=" + REPO + "&format=npm&package="+ + PACKAGE_NAME + "&version=1.0.0")+ .then().statusCode(200)+ .body("readme", equalTo("ERROR: No README data found!"))+ .body("package", equalTo(PACKAGE_NAME))+ .body("version", equalTo("1.0.0"));+ }+/*** Proves the actual mechanism, not just that {@code VerdaccioSidecarManager} has a method* named right: {@code ContainerTeardowns.stopAll} runs every {@code ContainerTeardown} ondiff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiDockerIntegrationTest.javaindex 80567c6b78..90b167ee58 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactPypiDockerIntegrationTest.java@@ -403,6 +403,37 @@ class CodeArtifactPypiDockerIntegrationTest {.then().statusCode(404).body("__type", equalTo("ResourceNotFoundException"));}+ /**+ * The real-world gap this closes: the readme-capture call site inside {@code+ * CodeArtifactPypiController#upload} had never once been exercised by a real multipart body+ * against a live pypiserver container (every other readme test either called {@code+ * CodeArtifactService} directly or never sent a {@code description} field at all). A+ * throwaway package, not {@link #PACKAGE_NAME}, so it doesn't disturb the ordered sequence+ * above.+ */+ @Test+ @Order(14)+ void getPackageVersionReadmeReturnsWhatARealTwineUploadSent() {+ String packageName = "floci-pypiserver-readme-spike";+ String filename = "floci_pypiserver_readme_spike-1.0.0.tar.gz";+ String description = "# Real Readme\n\nSent as the real twine upload's own description field.";++ given().header("Authorization", "Bearer " + bearerToken)+ .contentType("multipart/form-data; boundary=boundary123")+ .body(twineUploadBody("boundary123", packageName, filename,+ "readme-spike-bytes".getBytes(StandardCharsets.UTF_8), description))+ .post("/codeartifact/pypi/" + DOMAIN + "/" + REPO + "/")+ .then().statusCode(200);++ given().header("Authorization", AUTH)+ .get("/v1/package/version/readme?domain=" + DOMAIN + "&repository=" + REPO + "&format=pypi&package="+ + packageName + "&version=1.0.0")+ .then().statusCode(200)+ .body("readme", equalTo(description))+ .body("package", equalTo(packageName))+ .body("version", equalTo("1.0.0"));+ }+/*** Proves the actual mechanism, not just that {@code PypiserverSidecarManager} has a method* named right: {@code ContainerTeardowns.stopAll} runs every {@code ContainerTeardown} on@@ -458,6 +489,16 @@ class CodeArtifactPypiDockerIntegrationTest {* {@code name} and the {@code content} file part, so the rest are included for realism only.*/private static byte[] twineUploadBody(String boundary, String packageName, String filename, byte[] content) {+ return twineUploadBody(boundary, packageName, filename, content, null);+ }++ /**+ * Same real twine shape as the 4-argument overload above, with an optional {@code description}+ * field (the real PyPI legacy upload API's field for a distribution's long description) a real+ * {@code twine upload} also sends whenever the project's metadata carries one.+ */+ private static byte[] twineUploadBody(String boundary, String packageName, String filename, byte[] content,+ String description) {String preamble = "--" + boundary + "\r\n"+ "Content-Disposition: form-data; name=\":action\"\r\n\r\n"+ "file_upload\r\n"@@ -473,6 +514,9 @@ class CodeArtifactPypiDockerIntegrationTest {+ "--" + boundary + "\r\n"+ "Content-Disposition: form-data; name=\"filetype\"\r\n\r\n"+ "sdist\r\n"+ + (description == null ? "" : "--" + boundary + "\r\n"+ + "Content-Disposition: form-data; name=\"description\"\r\n\r\n"+ + description + "\r\n")+ "--" + boundary + "\r\n"+ "Content-Disposition: form-data; name=\"content\"; filename=\"" + filename + "\"\r\n"+ "Content-Type: application/octet-stream\r\n\r\n";diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.javaindex e5afef1329..a8f8f89ceb 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java@@ -311,6 +311,15 @@ class CodeArtifactServiceTest {bytes, fence);}+ /** Same snapshot-then-report shape as {@link #recordBytes}, for readme capture instead. */+ private void recordReadme(String domain, String repository, String format, String namespace, String packageName,+ String version, String readme) {+ long fence = service.currentContainerBackedPackageFence(REGION, domain, null, repository, format, namespace,+ packageName);+ service.recordPublishedPackageVersionReadme(REGION, domain, null, repository, format, namespace, packageName,+ version, readme, fence);+ }+/*** {@code recordContainerBackedAssetBytes} is what each of the three sidecar-backed proxies* calls after a real (non-idempotent, non-rejected) write; this is the unit-level proof that@@ -2121,6 +2130,254 @@ class CodeArtifactServiceTest {assertEquals("package-version", wrongRevision.getExtendedData().get("resourceType"));}+ // ---------------------------------------------------------- GetPackageVersionReadme++ @Test+ void getPackageVersionReadmeRejectsAFormatAwsDoesNotSupportForThisAction() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());++ // AWS's own API reference restricts this one action's format parameter to npm/pypi/nuget,+ // narrower than every other package-version action here, which accepts the full 8-value+ // set: maven, swift, generic, ruby, and cargo package versions carry no readme in real+ // CodeArtifact at all, so every one of them is checked here, not just maven.+ for (String format : new String[] {"maven", "swift", "generic", "ruby", "cargo"}) {+ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom",+ null, "repo", format, "com.example", "my-artifact", "1.0.0"), format);+ assertEquals("ValidationException", e.getErrorCode(), format);+ }+ }++ /**+ * npm, like pypi, can only ever be answered from what was captured at publish time: confirmed+ * directly against a live Verdaccio container that it hoists {@code readme} to the top level+ * only (the single most-recently-published version), dropping the per-version field a real+ * npm client's own publish envelope actually sends. See+ * {@link CodeArtifactNpmDataPlane#forwardSpooled} for where this is captured for real.+ */+ @Test+ void getPackageVersionReadmeReturnsWhatWasCapturedAtNpmPublishTime() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ recordReadme("dom", "repo", "npm", "myscope", "my-pkg", "1.0.0", "# My Package");++ CodeArtifactService.PackageVersionReadmeResult result = service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "npm", "myscope", "my-pkg", "1.0.0");++ assertEquals("# My Package", result.readme());+ assertEquals("my-pkg", result.packageName());+ assertEquals("1.0.0", result.version());+ }++ @Test+ void getPackageVersionReadme404sForNpmWhenNeverCaptured() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());++ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "npm", null, "my-pkg", "9.9.9"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ assertEquals("package-version", e.getExtendedData().get("resourceType"));+ }++ @Test+ void getPackageVersionReadmeReturnsWhatWasCapturedAtPypiUploadTime() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ recordReadme("dom", "repo", "pypi", null, "my-pkg", "1.0.0", "Long description in reStructuredText.");++ CodeArtifactService.PackageVersionReadmeResult result = service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "pypi", null, "my-pkg", "1.0.0");++ assertEquals("Long description in reStructuredText.", result.readme());+ }++ @Test+ void getPackageVersionReadme404sForPypiWhenNeverCaptured() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());++ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "pypi", null, "never-published", "1.0.0"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ }++ /**+ * nuget is a documented valid format for this action, but nothing in Floci implements the+ * real NuGet wire protocol, so no nuget package version can ever actually exist here; this+ * must 404 honestly rather than throwing some other error for a format it otherwise accepts.+ */+ @Test+ void getPackageVersionReadme404sForNugetSinceNothingCanEverPublishOne() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());++ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "nuget", null, "my-pkg", "1.0.0"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ }++ @Test+ void recordPublishedPackageVersionReadmeSkipsAPackageNameContainingTheEscapeCharacterWithoutCorruptingOthers() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());++ // The escape character itself can never reach escapeKeySegment's own key building, the+ // same way validatePackageToken already keeps it out of every JSON-API coordinate: this+ // call is just dropped, logged, and never throws back into the sidecar upload it reports.+ recordReadme("dom", "repo", "pypi", null, "bad#name", "1.0.0", "bad text");+ recordReadme("dom", "repo", "pypi", null, "legit", "1.0.0", "legit text");++ CodeArtifactService.PackageVersionReadmeResult result = service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "pypi", null, "legit", "1.0.0");+ assertEquals("legit text", result.readme());+ }++ @Test+ void recordPublishedPackageVersionReadmeAfterTheRepositoryIsAlreadyDeletedIsIgnored() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ service.deleteRepository(REGION, "dom", null, "repo");++ recordReadme("dom", "repo", "pypi", null, "my-pkg", "1.0.0", "late text");++ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "pypi", null, "my-pkg", "1.0.0"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ }++ @Test+ void deleteRepositoryRemovesItsCapturedPypiReadmesSoARecreatedRepositoryDoesNotInheritThem() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ recordReadme("dom", "repo", "pypi", null, "my-pkg", "1.0.0", "old text");++ service.deleteRepository(REGION, "dom", null, "repo");+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());++ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "pypi", null, "my-pkg", "1.0.0"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ }++ @Test+ void deletePackageRemovesItsCapturedPypiReadmesSoARepublishedPackageDoesNotInheritThem() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ recordReadme("dom", "repo", "pypi", null, "my-pkg", "1.0.0", "old text");+ when(pypiserverClient.packageExists(anyString(), eq("dom"), eq("repo"), isNull(), eq("my-pkg")))+ .thenReturn(true);++ service.deletePackage(REGION, "dom", null, "repo", "pypi", null, "my-pkg");++ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "pypi", null, "my-pkg", "1.0.0"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ }++ @Test+ void forgetContainerBackedPackageRemovesItsCapturedNpmReadmeTooSoAReplacedPackageDoesNotInheritIt() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ recordReadme("dom", "repo", "npm", null, "my-pkg", "1.0.0", "old text");++ service.forgetContainerBackedPackage(REGION, "dom", null, "repo", "npm", null, "my-pkg");++ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "npm", null, "my-pkg", "1.0.0"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ }++ /**+ * {@code getPackageVersionReadmeRejectsAFormatAwsDoesNotSupportForThisAction} above only+ * covers the format check; a Qodo review found this method's other required-field validation+ * (domain/repository/package/version) had no regression coverage of its own, nor any proof+ * that a rejected request leaves a previously captured readme untouched.+ */+ @Test+ void getPackageVersionReadmeRejectsMissingRequiredFieldsWithoutChangingStoredState() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ recordReadme("dom", "repo", "npm", null, "my-pkg", "1.0.0", "untouched text");++ AwsException missingDomain = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION,+ null, null, "repo", "npm", null, "my-pkg", "1.0.0"));+ assertEquals("ValidationException", missingDomain.getErrorCode());+ AwsException missingRepository = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(+ REGION, "dom", null, null, "npm", null, "my-pkg", "1.0.0"));+ assertEquals("ValidationException", missingRepository.getErrorCode());+ AwsException missingPackage = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION,+ "dom", null, "repo", "npm", null, null, "1.0.0"));+ assertEquals("ValidationException", missingPackage.getErrorCode());+ AwsException missingVersion = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION,+ "dom", null, "repo", "npm", null, "my-pkg", null));+ assertEquals("ValidationException", missingVersion.getErrorCode());++ CodeArtifactService.PackageVersionReadmeResult result = service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "npm", null, "my-pkg", "1.0.0");+ assertEquals("untouched text", result.readme());+ }++ /**+ * Same race {@code recordContainerBackedAssetBytesAfterAConcurrentDeletePackageIsIgnored}+ * already closes for byte accounting, found again by Qodo review for readme capture: a delayed+ * npm callback's readme capture must not resurrect a readme for a package a concurrent+ * {@code DeletePackage} already removed.+ */+ @Test+ void recordPublishedPackageVersionReadmeAfterAConcurrentDeletePackageIsIgnored() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ long fence = service.currentContainerBackedPackageFence(REGION, "dom", null, "repo", "npm", null, "late");+ when(verdaccioClient.packageExists(anyString(), eq("dom"), eq("repo"), isNull(), eq("late")))+ .thenReturn(true);+ service.deletePackage(REGION, "dom", null, "repo", "npm", null, "late");++ service.recordPublishedPackageVersionReadme(REGION, "dom", null, "repo", "npm", null, "late", "1.0.0",+ "too late", fence);++ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "npm", null, "late", "1.0.0"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ }++ /**+ * Found by Qodo review: pypi readme capture/lookup/deletion all used the package name exactly+ * as the caller spelled it, while {@code accountingPackageName} (and so byte accounting, the+ * fence, and sidecar resolution itself) treats case- and {@code -_.}-equivalent spellings as+ * the same pypi package. A readme captured under one spelling has to be answerable, and+ * deletable, under any PEP 503-equivalent spelling of the same name.+ */+ @Test+ void getPackageVersionReadmeResolvesPypiSpellingVariantsToTheSameReadme() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ recordReadme("dom", "repo", "pypi", null, "My_Package", "1.0.0", "the readme");++ CodeArtifactService.PackageVersionReadmeResult result = service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "pypi", null, "my-package", "1.0.0");++ assertEquals("the readme", result.readme());+ // Echoes the caller's own spelling back, not the normalized one used for the lookup.+ assertEquals("my-package", result.packageName());+ }++ @Test+ void deletePackageRemovesAPypiReadmeCapturedUnderADifferentSpelling() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ recordReadme("dom", "repo", "pypi", null, "My.Package", "1.0.0", "the readme");+ when(pypiserverClient.packageExists(anyString(), eq("dom"), eq("repo"), isNull(), eq("my-package")))+ .thenReturn(true);++ service.deletePackage(REGION, "dom", null, "repo", "pypi", null, "my-package");++ AwsException e = assertThrows(AwsException.class, () -> service.getPackageVersionReadme(REGION, "dom", null,+ "repo", "pypi", null, "My_Package", "1.0.0"));+ assertEquals("ResourceNotFoundException", e.getErrorCode());+ }+@Testvoid concurrentUnfinishedPublishesOfDifferentAssetsBothPersist() throws InterruptedException {service.createDomain(REGION, "dom", null, Map.of());diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScannerTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScannerTest.javaindex 9c404b0c4d..7a2de73a81 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScannerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/NpmPublishEnvelopeScannerTest.java@@ -199,6 +199,36 @@ class NpmPublishEnvelopeScannerTest {assertInvalid("[1,2,3]");}+ /**+ * Found by Qodo review: before this field was read at all, any shape here fell to the default+ * branch's plain {@code skipChildren()}; once it became a tracked field (for readme capture),+ * a value sent as anything but a string (a real client never would, but a hand-built or buggy+ * one might) must keep that same tolerance, not reject the whole envelope over one best-effort+ * field. {@code dist.integrity} on the same version, a field this scanner does require, proves+ * the rest of the version is still read correctly when {@code readme} is skipped.+ */+ @Test+ void aNonStringPerVersionReadmeIsToleratedRatherThanRejectingTheWholeEnvelope() throws Exception {+ String json = "{\"versions\":{\"1.0.0\":{\"readme\":null,\"dist\":{\"tarball\":\"x/y-1.0.0.tgz\","+ + "\"integrity\":\"sha512-abc\"}}}}";++ NpmPublishEnvelopeScanner.Envelope envelope = scan(json);++ NpmPublishEnvelopeScanner.Version version = envelope.versions().get("1.0.0");+ assertNull(version.readme());+ assertEquals("sha512-abc", version.integrity());+ }++ @Test+ void anObjectShapedPerVersionReadmeIsToleratedTheSameWay() throws Exception {+ String json = "{\"versions\":{\"1.0.0\":{\"readme\":{\"not\":\"a string\"},\"dist\":{\"tarball\":"+ + "\"x/y-1.0.0.tgz\",\"integrity\":\"sha512-abc\"}}}}";++ NpmPublishEnvelopeScanner.Envelope envelope = scan(json);++ assertNull(envelope.versions().get("1.0.0").readme());+ }+private static NpmPublishEnvelopeScanner.Envelope scan(String json) throws IOException {InputStream in = new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8));return NpmPublishEnvelopeScanner.scan(in);