Light, fluffy, and always free - The AWS Local Emulator alternative
fix(codeartifact): treat identical-content asset republish as idempotent (#4619)
Priamo Germosen0232d5d22457
diff --git a/docs/services/codeartifact.md b/docs/services/codeartifact.mdindex 7508b94b96..211fde8dea 100644--- a/docs/services/codeartifact.md+++ b/docs/services/codeartifact.md@@ -62,11 +62,14 @@ and `UpdateRepository` also cap direct upstreams at 10, AWS's own repository lim`PublishPackageVersion` creates a package version in the `Unfinished` state when the `unfinished`flag is set, and `Published` otherwise; once `Published`, a repeat publish to the samedomain/repository/package/version fails with `ConflictException`, matching AWS's real rule that a-published version cannot accept additional assets. Every publish returns a fresh-`versionRevision`, and each asset's hashes (`MD5`, `SHA-1`, `SHA-256`, `SHA-512`) are computed from-the bytes Floci actually received, not echoed from the request. Floci enforces AWS's own published-quotas for this action: a 5 GB max asset file size and a 350-asset cap per package version, both-returning `ServiceQuotaExceededException`.+published version cannot accept additional assets. While still `Unfinished`, publishing a new+asset name is always accepted, but republishing an asset name that already exists is only+accepted when the content is byte-identical to what's already stored (AWS's own "Overwriting+package assets" rule: idempotent on a matching retry, `ConflictException` on genuinely different+content); each asset's hashes (`MD5`, `SHA-1`, `SHA-256`, `SHA-512`) are computed from the bytes+Floci actually received, not echoed from the request. Every publish returns a fresh+`versionRevision`. Floci enforces AWS's own published quotas for this action: a 5 GB max asset+file size and a 350-asset cap per package version, both returning `ServiceQuotaExceededException`.## The Maven repository endpointdiff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.javaindex f3902eb68a..0928aec1ff 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java@@ -22,10 +22,12 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.io.IOException;+import java.io.InputStream;import java.io.UncheckedIOException;import java.nio.charset.StandardCharsets;import java.nio.file.AtomicMoveNotSupportedException;import java.nio.file.Files;+import java.nio.file.NoSuchFileException;import java.nio.file.Path;import java.nio.file.StandardCopyOption;import java.security.MessageDigest;@@ -33,6 +35,7 @@ import java.security.NoSuchAlgorithmException;import java.security.SecureRandom;import java.time.Instant;import java.util.ArrayList;+import java.util.Arrays;import java.util.Base64;import java.util.Comparator;import java.util.LinkedHashMap;@@ -581,6 +584,19 @@ public class CodeArtifactService implements Resettable {}Map<String, PackageAsset> assets = new LinkedHashMap<>(pv.getAssets());+ // AWS's own docs ("Overwriting package assets" in CodeArtifact's packages overview): a+ // republish of an asset name that already exists is only rejected when the new content+ // actually differs from what's already there; byte-identical content succeeds because the+ // operation is idempotent, e.g. a client retrying after a dropped response. The in-memory+ // PackageAsset.content field is @JsonIgnore and not reliable after a reload from persisted+ // storage, so the real existing bytes have to come from the asset store, the same place+ // getPackageVersionAsset reads them from. A genuinely missing backing file (metadata says+ // the asset exists but its bytes don't, e.g. a partial restore) reads the same as "nothing+ // to conflict with" rather than a hard failure, so a republish can still repair it.+ if (assets.containsKey(assetName) && !existingAssetMatchesOrIsMissing(owner, key, assetName, assetContent)) {+ throw conflict("Asset '" + assetName + "' already exists for package version '" + version+ + "' of package '" + packageName + "' with different content.", assetName, "asset");+ }if (!assets.containsKey(assetName) && assets.size() >= MAX_ASSETS_PER_PACKAGE_VERSION) {throw new AwsException("ServiceQuotaExceededException","A package version can have a maximum of " + MAX_ASSETS_PER_PACKAGE_VERSION + " assets.", 402,@@ -973,6 +989,43 @@ public class CodeArtifactService implements Resettable {}}+ /**+ * {@code true} when the asset's backing bytes are genuinely missing (e.g. a partial restore+ * left the metadata but not the file, so there is nothing to conflict with) or match+ * {@code newContent}; {@code false} only on a real mismatch. Deliberately never materializes+ * the existing file as a second full byte array: {@code newContent} is already fully resident+ * (RESTEasy Reactive buffers the whole request body before this method ever runs), so loading a+ * second full copy of a large existing asset just to compare it would double peak memory for a+ * same-content retry. Also deliberately separate from {@link #readAssetContent}: a real+ * download through {@code getPackageVersionAsset} needs a missing file to stay a hard failure,+ * not silently read as "nothing published yet".+ */+ private boolean existingAssetMatchesOrIsMissing(String owner, String packageVersionKey, String assetName,+ byte[] newContent) {+ if (inMemory) {+ byte[] existing = memoryAssetStore.get(assetStoreKey(owner, packageVersionKey, assetName));+ return existing == null || Arrays.equals(existing, newContent);+ }+ Path filePath = resolveAssetPath(owner, packageVersionKey, assetName);+ try (InputStream in = Files.newInputStream(filePath)) {+ byte[] buffer = new byte[8192];+ int offset = 0;+ int read;+ while ((read = in.read(buffer)) != -1) {+ if (offset + read > newContent.length || !Arrays.equals(buffer, 0, read,+ newContent, offset, offset + read)) {+ return false;+ }+ offset += read;+ }+ return offset == newContent.length;+ } catch (NoSuchFileException e) {+ return true;+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to read CodeArtifact asset file: " + filePath, e);+ }+ }+private static String assetStoreKey(String owner, String packageVersionKey, String assetName) {return owner + "::" + packageVersionKey + "::" + assetName;}diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactAssetPersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactAssetPersistenceTest.javaindex 29c91046a8..b6e9f54373 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactAssetPersistenceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactAssetPersistenceTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.codeartifact;import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.PersistentStorage;@@ -17,12 +18,15 @@ import java.io.IOException;import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;+import java.util.Arrays;import java.util.Base64;import java.util.List;import java.util.Map;+import java.util.stream.Stream;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.when;@@ -72,6 +76,144 @@ class CodeArtifactAssetPersistenceTest {assertEquals(published.packageVersion().getRevision(), result.packageVersionRevision());}+ /**+ * The in-memory {@code PackageAsset.content} field is {@code @JsonIgnore}, so a package+ * version reloaded from disk (a fresh {@code CodeArtifactService} instance here, standing in+ * for a Floci restart) never has it populated; this only passes if the idempotent-republish+ * check in {@code publishPackageVersion} reads the real bytes from the asset store the same+ * way {@code getPackageVersionAsset} does, not that field.+ */+ @Test+ void republishingAnUnfinishedAssetAfterARestartWithIdenticalContentSucceedsIdempotently(@TempDir Path dir) {+ CodeArtifactService first = newService(dir);+ first.createDomain(REGION, "dom", null, Map.of());+ first.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ byte[] content = "hello world".getBytes(StandardCharsets.UTF_8);+ first.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",+ sha256Hex(content), "true", content);++ CodeArtifactService restarted = newService(dir);+ PublishPackageVersionResult result = restarted.publishPackageVersion(REGION, "dom", null, "repo", "generic",+ null, "my-pkg", "1.0.0", "a.txt", sha256Hex(content), "true", content);++ assertEquals("Unfinished", result.packageVersion().getStatus());+ assertEquals(1, result.packageVersion().getAssets().size());+ }++ @Test+ void republishingAnUnfinishedAssetAfterARestartWithDifferentContentConflicts(@TempDir Path dir) {+ CodeArtifactService first = newService(dir);+ first.createDomain(REGION, "dom", null, Map.of());+ first.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ byte[] content = "hello world".getBytes(StandardCharsets.UTF_8);+ first.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",+ sha256Hex(content), "true", content);++ CodeArtifactService restarted = newService(dir);+ byte[] different = "different bytes".getBytes(StandardCharsets.UTF_8);+ AwsException e = assertThrows(AwsException.class, () -> restarted.publishPackageVersion(REGION, "dom", null,+ "repo", "generic", null, "my-pkg", "1.0.0", "a.txt", sha256Hex(different), "true", different));+ assertEquals("ConflictException", e.getErrorCode());+ assertEquals("a.txt", e.getExtendedData().get("resourceId"));+ }++ /**+ * A package version whose metadata still lists an asset but whose backing file is gone (a+ * partial restore, or anything else that touched the asset store without also touching the+ * metadata) must not turn a republish into a hard failure: that would block the one thing that+ * could actually repair it. Greptile caught this as a real regression risk in the overwrite+ * check above; this is the case that check must not treat as "the asset exists and must match".+ */+ @Test+ void republishingAnAssetWhoseBackingFileIsMissingRepairsItInsteadOfFailing(@TempDir Path dir) throws IOException {+ CodeArtifactService service = newService(dir);+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ byte[] content = "hello world".getBytes(StandardCharsets.UTF_8);+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",+ sha256Hex(content), "true", content);++ Path assetRoot = dir.resolve("codeartifact-assets");+ try (Stream<Path> paths = Files.walk(assetRoot)) {+ for (Path p : paths.filter(Files::isRegularFile).toList()) {+ Files.delete(p);+ }+ }++ byte[] repaired = "repaired content".getBytes(StandardCharsets.UTF_8);+ PublishPackageVersionResult result = service.publishPackageVersion(REGION, "dom", null, "repo", "generic",+ null, "my-pkg", "1.0.0", "a.txt", sha256Hex(repaired), "true", repaired);++ assertEquals("Unfinished", result.packageVersion().getStatus());+ PackageVersionAssetResult fetched = service.getPackageVersionAsset(REGION, "dom", null, "repo", "generic",+ null, "my-pkg", "1.0.0", "a.txt", null);+ assertEquals("repaired content", new String(fetched.asset().getContent(), StandardCharsets.UTF_8));+ }++ /**+ * The overwrite check streams the existing file in fixed-size chunks rather than loading it+ * whole, specifically to avoid holding two full copies of a large asset in memory at once for a+ * same-content retry. These three cases exercise the chunk boundary directly: content spanning+ * several reads of the comparison buffer, a mismatch that only appears in the final chunk, and a+ * length mismatch that only becomes apparent once the shorter side is exhausted.+ */+ @Test+ void republishingALargeAssetSpanningMultipleBufferReadsWithIdenticalContentSucceeds(@TempDir Path dir) {+ CodeArtifactService service = newService(dir);+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ byte[] content = deterministicBytes(20_000);+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.bin",+ sha256Hex(content), "true", content);++ PublishPackageVersionResult result = service.publishPackageVersion(REGION, "dom", null, "repo", "generic",+ null, "my-pkg", "1.0.0", "a.bin", sha256Hex(content), "true", content);++ assertEquals("Unfinished", result.packageVersion().getStatus());+ assertEquals(1, result.packageVersion().getAssets().size());+ }++ @Test+ void republishingALargeAssetWithAMismatchInTheFinalChunkConflicts(@TempDir Path dir) {+ CodeArtifactService service = newService(dir);+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ byte[] original = deterministicBytes(20_000);+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.bin",+ sha256Hex(original), "true", original);++ byte[] almostSame = original.clone();+ almostSame[almostSame.length - 1] ^= 0xFF;++ AwsException e = assertThrows(AwsException.class, () -> service.publishPackageVersion(REGION, "dom", null,+ "repo", "generic", null, "my-pkg", "1.0.0", "a.bin", sha256Hex(almostSame), "true", almostSame));+ assertEquals("ConflictException", e.getErrorCode());+ }++ @Test+ void republishingALargeAssetWithDifferentLengthConflicts(@TempDir Path dir) {+ CodeArtifactService service = newService(dir);+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ byte[] original = deterministicBytes(20_000);+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.bin",+ sha256Hex(original), "true", original);++ byte[] shorter = Arrays.copyOf(original, original.length - 1);++ AwsException e = assertThrows(AwsException.class, () -> service.publishPackageVersion(REGION, "dom", null,+ "repo", "generic", null, "my-pkg", "1.0.0", "a.bin", sha256Hex(shorter), "true", shorter));+ assertEquals("ConflictException", e.getErrorCode());+ }++ private static byte[] deterministicBytes(int size) {+ byte[] bytes = new byte[size];+ for (int i = 0; i < size; i++) {+ bytes[i] = (byte) (i * 31 + 7);+ }+ return bytes;+ }+@Testvoid distinctAssetNamesNeverCollideOnDisk(@TempDir Path dir) {CodeArtifactService service = newService(dir);diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.javaindex dd651df440..9962e29a0d 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java@@ -849,6 +849,37 @@ class CodeArtifactServiceTest {assertTrue(r2.packageVersion().getAssets().containsKey("b.txt"));}+ @Test+ void republishingAnUnfinishedAssetWithIdenticalContentSucceedsIdempotently() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ byte[] content = "same-bytes".getBytes(StandardCharsets.UTF_8);+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",+ sha256Hex(content), "true", content);++ PublishPackageVersionResult result = service.publishPackageVersion(REGION, "dom", null, "repo", "generic",+ null, "my-pkg", "1.0.0", "a.txt", sha256Hex(content), "true", content);++ assertEquals("Unfinished", result.packageVersion().getStatus());+ assertEquals(1, result.packageVersion().getAssets().size());+ }++ @Test+ void republishingAnUnfinishedAssetWithDifferentContentConflicts() {+ service.createDomain(REGION, "dom", null, Map.of());+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());+ byte[] first = "first-bytes".getBytes(StandardCharsets.UTF_8);+ byte[] different = "different-bytes".getBytes(StandardCharsets.UTF_8);+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",+ sha256Hex(first), "true", first);++ AwsException e = assertThrows(AwsException.class, () -> service.publishPackageVersion(REGION, "dom", null,+ "repo", "generic", null, "my-pkg", "1.0.0", "a.txt", sha256Hex(different), "true", different));+ assertEquals("ConflictException", e.getErrorCode());+ assertEquals("a.txt", e.getExtendedData().get("resourceId"));+ assertEquals("asset", e.getExtendedData().get("resourceType"));+ }+@Testvoid publishingToAnAlreadyPublishedVersionConflicts() {service.createDomain(REGION, "dom", null, Map.of());