Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(codeartifact): treat identical-content asset republish as idempotent (#4619)

Priamo Germosen0232d5d22457
diff --git a/docs/services/codeartifact.md b/docs/services/codeartifact.md
index 7508b94b96..211fde8dea 100644
--- a/docs/services/codeartifact.md
+++ b/docs/services/codeartifact.md
@@ -62,11 +62,14 @@ and `UpdateRepository` also cap direct upstreams at 10, AWS's own repository lim
`PublishPackageVersion` creates a package version in the `Unfinished` state when the `unfinished`
flag is set, and `Published` otherwise; once `Published`, a repeat publish to the same
domain/repository/package/version fails with `ConflictException`, matching AWS's real rule that a
-published version cannot accept additional assets. Every publish returns a fresh
-`versionRevision`, and each asset's hashes (`MD5`, `SHA-1`, `SHA-256`, `SHA-512`) are computed from
-the bytes Floci actually received, not echoed from the request. Floci enforces AWS's own published
-quotas for this action: a 5 GB max asset file size and a 350-asset cap per package version, both
-returning `ServiceQuotaExceededException`.
+published version cannot accept additional assets. While still `Unfinished`, publishing a new
+asset name is always accepted, but republishing an asset name that already exists is only
+accepted when the content is byte-identical to what's already stored (AWS's own "Overwriting
+package assets" rule: idempotent on a matching retry, `ConflictException` on genuinely different
+content); each asset's hashes (`MD5`, `SHA-1`, `SHA-256`, `SHA-512`) are computed from the bytes
+Floci actually received, not echoed from the request. Every publish returns a fresh
+`versionRevision`. Floci enforces AWS's own published quotas for this action: a 5 GB max asset
+file size and a 350-asset cap per package version, both returning `ServiceQuotaExceededException`.
## The Maven repository endpoint
diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java
index f3902eb68a..0928aec1ff 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java
@@ -22,10 +22,12 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.io.IOException;
+import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
@@ -33,6 +35,7 @@ import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Base64;
import java.util.Comparator;
import java.util.LinkedHashMap;
@@ -581,6 +584,19 @@ public class CodeArtifactService implements Resettable {
}
Map<String, PackageAsset> assets = new LinkedHashMap<>(pv.getAssets());
+ // AWS's own docs ("Overwriting package assets" in CodeArtifact's packages overview): a
+ // republish of an asset name that already exists is only rejected when the new content
+ // actually differs from what's already there; byte-identical content succeeds because the
+ // operation is idempotent, e.g. a client retrying after a dropped response. The in-memory
+ // PackageAsset.content field is @JsonIgnore and not reliable after a reload from persisted
+ // storage, so the real existing bytes have to come from the asset store, the same place
+ // getPackageVersionAsset reads them from. A genuinely missing backing file (metadata says
+ // the asset exists but its bytes don't, e.g. a partial restore) reads the same as "nothing
+ // to conflict with" rather than a hard failure, so a republish can still repair it.
+ if (assets.containsKey(assetName) && !existingAssetMatchesOrIsMissing(owner, key, assetName, assetContent)) {
+ throw conflict("Asset '" + assetName + "' already exists for package version '" + version
+ + "' of package '" + packageName + "' with different content.", assetName, "asset");
+ }
if (!assets.containsKey(assetName) && assets.size() >= MAX_ASSETS_PER_PACKAGE_VERSION) {
throw new AwsException("ServiceQuotaExceededException",
"A package version can have a maximum of " + MAX_ASSETS_PER_PACKAGE_VERSION + " assets.", 402,
@@ -973,6 +989,43 @@ public class CodeArtifactService implements Resettable {
}
}
+ /**
+ * {@code true} when the asset's backing bytes are genuinely missing (e.g. a partial restore
+ * left the metadata but not the file, so there is nothing to conflict with) or match
+ * {@code newContent}; {@code false} only on a real mismatch. Deliberately never materializes
+ * the existing file as a second full byte array: {@code newContent} is already fully resident
+ * (RESTEasy Reactive buffers the whole request body before this method ever runs), so loading a
+ * second full copy of a large existing asset just to compare it would double peak memory for a
+ * same-content retry. Also deliberately separate from {@link #readAssetContent}: a real
+ * download through {@code getPackageVersionAsset} needs a missing file to stay a hard failure,
+ * not silently read as "nothing published yet".
+ */
+ private boolean existingAssetMatchesOrIsMissing(String owner, String packageVersionKey, String assetName,
+ byte[] newContent) {
+ if (inMemory) {
+ byte[] existing = memoryAssetStore.get(assetStoreKey(owner, packageVersionKey, assetName));
+ return existing == null || Arrays.equals(existing, newContent);
+ }
+ Path filePath = resolveAssetPath(owner, packageVersionKey, assetName);
+ try (InputStream in = Files.newInputStream(filePath)) {
+ byte[] buffer = new byte[8192];
+ int offset = 0;
+ int read;
+ while ((read = in.read(buffer)) != -1) {
+ if (offset + read > newContent.length || !Arrays.equals(buffer, 0, read,
+ newContent, offset, offset + read)) {
+ return false;
+ }
+ offset += read;
+ }
+ return offset == newContent.length;
+ } catch (NoSuchFileException e) {
+ return true;
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read CodeArtifact asset file: " + filePath, e);
+ }
+ }
+
private static String assetStoreKey(String owner, String packageVersionKey, String assetName) {
return owner + "::" + packageVersionKey + "::" + assetName;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactAssetPersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactAssetPersistenceTest.java
index 29c91046a8..b6e9f54373 100644
--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactAssetPersistenceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactAssetPersistenceTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.codeartifact;
import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.PersistentStorage;
@@ -17,12 +18,15 @@ import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Arrays;
import java.util.Base64;
import java.util.List;
import java.util.Map;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -72,6 +76,144 @@ class CodeArtifactAssetPersistenceTest {
assertEquals(published.packageVersion().getRevision(), result.packageVersionRevision());
}
+ /**
+ * The in-memory {@code PackageAsset.content} field is {@code @JsonIgnore}, so a package
+ * version reloaded from disk (a fresh {@code CodeArtifactService} instance here, standing in
+ * for a Floci restart) never has it populated; this only passes if the idempotent-republish
+ * check in {@code publishPackageVersion} reads the real bytes from the asset store the same
+ * way {@code getPackageVersionAsset} does, not that field.
+ */
+ @Test
+ void republishingAnUnfinishedAssetAfterARestartWithIdenticalContentSucceedsIdempotently(@TempDir Path dir) {
+ CodeArtifactService first = newService(dir);
+ first.createDomain(REGION, "dom", null, Map.of());
+ first.createRepository(REGION, "dom", null, "repo", null, null, Map.of());
+ byte[] content = "hello world".getBytes(StandardCharsets.UTF_8);
+ first.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",
+ sha256Hex(content), "true", content);
+
+ CodeArtifactService restarted = newService(dir);
+ PublishPackageVersionResult result = restarted.publishPackageVersion(REGION, "dom", null, "repo", "generic",
+ null, "my-pkg", "1.0.0", "a.txt", sha256Hex(content), "true", content);
+
+ assertEquals("Unfinished", result.packageVersion().getStatus());
+ assertEquals(1, result.packageVersion().getAssets().size());
+ }
+
+ @Test
+ void republishingAnUnfinishedAssetAfterARestartWithDifferentContentConflicts(@TempDir Path dir) {
+ CodeArtifactService first = newService(dir);
+ first.createDomain(REGION, "dom", null, Map.of());
+ first.createRepository(REGION, "dom", null, "repo", null, null, Map.of());
+ byte[] content = "hello world".getBytes(StandardCharsets.UTF_8);
+ first.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",
+ sha256Hex(content), "true", content);
+
+ CodeArtifactService restarted = newService(dir);
+ byte[] different = "different bytes".getBytes(StandardCharsets.UTF_8);
+ AwsException e = assertThrows(AwsException.class, () -> restarted.publishPackageVersion(REGION, "dom", null,
+ "repo", "generic", null, "my-pkg", "1.0.0", "a.txt", sha256Hex(different), "true", different));
+ assertEquals("ConflictException", e.getErrorCode());
+ assertEquals("a.txt", e.getExtendedData().get("resourceId"));
+ }
+
+ /**
+ * A package version whose metadata still lists an asset but whose backing file is gone (a
+ * partial restore, or anything else that touched the asset store without also touching the
+ * metadata) must not turn a republish into a hard failure: that would block the one thing that
+ * could actually repair it. Greptile caught this as a real regression risk in the overwrite
+ * check above; this is the case that check must not treat as "the asset exists and must match".
+ */
+ @Test
+ void republishingAnAssetWhoseBackingFileIsMissingRepairsItInsteadOfFailing(@TempDir Path dir) throws IOException {
+ CodeArtifactService service = newService(dir);
+ service.createDomain(REGION, "dom", null, Map.of());
+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());
+ byte[] content = "hello world".getBytes(StandardCharsets.UTF_8);
+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",
+ sha256Hex(content), "true", content);
+
+ Path assetRoot = dir.resolve("codeartifact-assets");
+ try (Stream<Path> paths = Files.walk(assetRoot)) {
+ for (Path p : paths.filter(Files::isRegularFile).toList()) {
+ Files.delete(p);
+ }
+ }
+
+ byte[] repaired = "repaired content".getBytes(StandardCharsets.UTF_8);
+ PublishPackageVersionResult result = service.publishPackageVersion(REGION, "dom", null, "repo", "generic",
+ null, "my-pkg", "1.0.0", "a.txt", sha256Hex(repaired), "true", repaired);
+
+ assertEquals("Unfinished", result.packageVersion().getStatus());
+ PackageVersionAssetResult fetched = service.getPackageVersionAsset(REGION, "dom", null, "repo", "generic",
+ null, "my-pkg", "1.0.0", "a.txt", null);
+ assertEquals("repaired content", new String(fetched.asset().getContent(), StandardCharsets.UTF_8));
+ }
+
+ /**
+ * The overwrite check streams the existing file in fixed-size chunks rather than loading it
+ * whole, specifically to avoid holding two full copies of a large asset in memory at once for a
+ * same-content retry. These three cases exercise the chunk boundary directly: content spanning
+ * several reads of the comparison buffer, a mismatch that only appears in the final chunk, and a
+ * length mismatch that only becomes apparent once the shorter side is exhausted.
+ */
+ @Test
+ void republishingALargeAssetSpanningMultipleBufferReadsWithIdenticalContentSucceeds(@TempDir Path dir) {
+ CodeArtifactService service = newService(dir);
+ service.createDomain(REGION, "dom", null, Map.of());
+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());
+ byte[] content = deterministicBytes(20_000);
+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.bin",
+ sha256Hex(content), "true", content);
+
+ PublishPackageVersionResult result = service.publishPackageVersion(REGION, "dom", null, "repo", "generic",
+ null, "my-pkg", "1.0.0", "a.bin", sha256Hex(content), "true", content);
+
+ assertEquals("Unfinished", result.packageVersion().getStatus());
+ assertEquals(1, result.packageVersion().getAssets().size());
+ }
+
+ @Test
+ void republishingALargeAssetWithAMismatchInTheFinalChunkConflicts(@TempDir Path dir) {
+ CodeArtifactService service = newService(dir);
+ service.createDomain(REGION, "dom", null, Map.of());
+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());
+ byte[] original = deterministicBytes(20_000);
+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.bin",
+ sha256Hex(original), "true", original);
+
+ byte[] almostSame = original.clone();
+ almostSame[almostSame.length - 1] ^= 0xFF;
+
+ AwsException e = assertThrows(AwsException.class, () -> service.publishPackageVersion(REGION, "dom", null,
+ "repo", "generic", null, "my-pkg", "1.0.0", "a.bin", sha256Hex(almostSame), "true", almostSame));
+ assertEquals("ConflictException", e.getErrorCode());
+ }
+
+ @Test
+ void republishingALargeAssetWithDifferentLengthConflicts(@TempDir Path dir) {
+ CodeArtifactService service = newService(dir);
+ service.createDomain(REGION, "dom", null, Map.of());
+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());
+ byte[] original = deterministicBytes(20_000);
+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.bin",
+ sha256Hex(original), "true", original);
+
+ byte[] shorter = Arrays.copyOf(original, original.length - 1);
+
+ AwsException e = assertThrows(AwsException.class, () -> service.publishPackageVersion(REGION, "dom", null,
+ "repo", "generic", null, "my-pkg", "1.0.0", "a.bin", sha256Hex(shorter), "true", shorter));
+ assertEquals("ConflictException", e.getErrorCode());
+ }
+
+ private static byte[] deterministicBytes(int size) {
+ byte[] bytes = new byte[size];
+ for (int i = 0; i < size; i++) {
+ bytes[i] = (byte) (i * 31 + 7);
+ }
+ return bytes;
+ }
+
@Test
void distinctAssetNamesNeverCollideOnDisk(@TempDir Path dir) {
CodeArtifactService service = newService(dir);
diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java
index dd651df440..9962e29a0d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java
@@ -849,6 +849,37 @@ class CodeArtifactServiceTest {
assertTrue(r2.packageVersion().getAssets().containsKey("b.txt"));
}
+ @Test
+ void republishingAnUnfinishedAssetWithIdenticalContentSucceedsIdempotently() {
+ service.createDomain(REGION, "dom", null, Map.of());
+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());
+ byte[] content = "same-bytes".getBytes(StandardCharsets.UTF_8);
+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",
+ sha256Hex(content), "true", content);
+
+ PublishPackageVersionResult result = service.publishPackageVersion(REGION, "dom", null, "repo", "generic",
+ null, "my-pkg", "1.0.0", "a.txt", sha256Hex(content), "true", content);
+
+ assertEquals("Unfinished", result.packageVersion().getStatus());
+ assertEquals(1, result.packageVersion().getAssets().size());
+ }
+
+ @Test
+ void republishingAnUnfinishedAssetWithDifferentContentConflicts() {
+ service.createDomain(REGION, "dom", null, Map.of());
+ service.createRepository(REGION, "dom", null, "repo", null, null, Map.of());
+ byte[] first = "first-bytes".getBytes(StandardCharsets.UTF_8);
+ byte[] different = "different-bytes".getBytes(StandardCharsets.UTF_8);
+ service.publishPackageVersion(REGION, "dom", null, "repo", "generic", null, "my-pkg", "1.0.0", "a.txt",
+ sha256Hex(first), "true", first);
+
+ AwsException e = assertThrows(AwsException.class, () -> service.publishPackageVersion(REGION, "dom", null,
+ "repo", "generic", null, "my-pkg", "1.0.0", "a.txt", sha256Hex(different), "true", different));
+ assertEquals("ConflictException", e.getErrorCode());
+ assertEquals("a.txt", e.getExtendedData().get("resourceId"));
+ assertEquals("asset", e.getExtendedData().get("resourceType"));
+ }
+
@Test
void publishingToAnAlreadyPublishedVersionConflicts() {
service.createDomain(REGION, "dom", null, Map.of());