Light, fluffy, and always free - The AWS Local Emulator alternative
fix(codeartifact): stop sidecar containers on state reset and nuke (#4411)
Priamo Germosen61e7f07fb650
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/PerKeyContainerPool.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/PerKeyContainerPool.javaindex ef637b8864..e17c6edfe7 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/PerKeyContainerPool.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/PerKeyContainerPool.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.core.common.docker;import org.jboss.logging.Logger;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicLong;/*** Lazily starts, health-checks, and reuses one container per arbitrary key. Pulled out of@@ -33,6 +34,15 @@ public class PerKeyContainerPool {private final String healthPath;private final ConcurrentHashMap<String, StartedContainer> containers = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, Object> startLocks = new ConcurrentHashMap<>();+ /**+ * Bumped by every {@link #stopAll()}. {@link #ensureReady} reads it before and after starting+ * a container: {@code stopAll()} takes no per-key lock (it has no single key to take), so it+ * cannot see a container that only starts registering itself after {@code stopAll()} already+ * iterated the map. A changed generation is how a start still notices that a teardown happened+ * while it was building, so it can stop what it just started instead of leaving it running with+ * nothing left able to ever stop it again.+ */+ private final AtomicLong generation = new AtomicLong();/*** @param healthPath path (e.g. {@code /-/ping}) appended to a container's base URL to probe@@ -64,7 +74,16 @@ public class PerKeyContainerPool {LOG.warnv("Sidecar container for key {0} is no longer healthy; restarting it", key);lifecycleManager.stopAndRemove(existing.containerId(), null);}+ long startedGeneration = generation.get();StartedContainer started = starter.start();+ if (generation.get() != startedGeneration) {+ // A stopAll() ran while this container was being built and could not have seen+ // it: it was not in the map yet. Stop it now rather than register it, so a reset+ // racing a first-use does not leave this one running behind.+ LOG.warnv("Sidecar pool was reset while starting a container for key {0}; stopping it", key);+ lifecycleManager.stopAndRemove(started.containerId(), null);+ throw new IllegalStateException("Sidecar pool was reset while starting a container for key " + key);+ }// Tracked before the health wait, not after: a timeout below must not stop// stopContainer()/stopAll() from ever finding this already-running container again.containers.put(key, started);@@ -96,6 +115,7 @@ public class PerKeyContainerPool {/** Stops and removes every container this pool has started, then forgets them all. */public void stopAll() {+ generation.incrementAndGet();if (containers.isEmpty()) {return;}diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.javaindex 0c9881a87a..f3902eb68a 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactService.java@@ -326,11 +326,7 @@ public class CodeArtifactService implements Resettable {/*** Generic over whatever formats a repository actually has sidecar ids for: adding the next* format's sidecar needs no change here, only a new {@link RepositorySidecarManager}- * implementation that {@link CodeArtifactSidecarRegistry} discovers. Shared by- * {@link #deleteRepository} and {@link #beforeReset}, both of which forget the repository's- * record right after calling this and so must not leave its sidecar storage (a Docker- * container, for a per-repository sidecar such as Verdaccio) running with nothing left able to- * ever release it again.+ * implementation that {@link CodeArtifactSidecarRegistry} discovers.*/private void releaseSidecarStorage(CodeArtifactRepository r, String domain, String repository) {r.getSidecarContainerIds().forEach((format, containerId) ->@@ -759,18 +755,6 @@ public class CodeArtifactService implements Resettable {ref.repository()).getTags();}- /**- * Releases every repository's sidecar storage before {@link #clear} wipes the records that- * name it. Without this, an emulator reset forgets which sidecar container backs a repository- * without ever stopping it: harmless for Maven's one shared Reposilite instance, but a real- * Docker container leak for a per-repository sidecar such as Verdaccio, one abandoned- * container per repository that existed at reset time.- */- @Override- public void beforeReset() {- repositories.scanAllAccounts().forEach(r -> releaseSidecarStorage(r, r.getDomainName(), r.getName()));- }-@Overridepublic void clear() {domains.clear();diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/ReposiliteSidecarManager.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/ReposiliteSidecarManager.javaindex 89c488d481..209a15f4c6 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/ReposiliteSidecarManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/ReposiliteSidecarManager.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.codeartifact;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.ContainerTeardown;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;@@ -8,9 +9,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.Eimport io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.SidecarHealthHelper;-import io.quarkus.runtime.ShutdownEvent;import jakarta.enterprise.context.ApplicationScoped;-import jakarta.enterprise.event.Observes;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -30,9 +29,15 @@ import java.util.Optional;* <p>One repository-agnostic container backs every CodeArtifact domain/repository pair; a* CodeArtifact repository maps to a named Reposilite repository provisioned on first use (see* {@link ReposiliteSidecarClient#ensureRepository}), not to its own container.+ *+ * <p>Implements {@link ContainerTeardown} rather than observing {@code ShutdownEvent} directly:+ * {@code ContainerTeardowns.stopAll} already runs every implementation both at process shutdown+ * and on {@code /state/reset}/{@code /state/nuke}, so a reset now actually stops the managed+ * container instead of leaving it running with every {@code CodeArtifactRepository} record that+ * named it gone.*/@ApplicationScoped-public class ReposiliteSidecarManager {+public class ReposiliteSidecarManager implements ContainerTeardown {static final String IMAGE_ENV = "FLOCI_SERVICES_CODEARTIFACT_MAVEN_IMAGE";static final String URL_ENV = "FLOCI_SERVICES_CODEARTIFACT_MAVEN_URL";static final String MANAGED_TOKEN_NAME = "floci-manager";@@ -161,7 +166,8 @@ public class ReposiliteSidecarManager {LOG.infov("Reposilite sidecar is ready at {0}", resolvedUrl);}- void onStop(@Observes ShutdownEvent event) {+ @Override+ public void stopManagedContainers() {if (containerId == null) {return;}diff --git a/src/main/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManager.java b/src/main/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManager.javaindex f1967e0594..a80c3baf4d 100644--- a/src/main/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManager.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.codeartifact;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.ContainerTeardown;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;@@ -8,9 +9,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.Eimport io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.PerKeyContainerPool;-import io.quarkus.runtime.ShutdownEvent;import jakarta.enterprise.context.ApplicationScoped;-import jakarta.enterprise.event.Observes;import jakarta.inject.Inject;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;@@ -35,9 +34,15 @@ import java.nio.charset.StandardCharsets;* real CodeArtifact authorization check happens once at Floci's proxy layer* ({@code CodeArtifactNpmDataPlane}), before a request ever reaches the container; the container* itself is never reachable directly by a client.+ *+ * <p>Implements {@link ContainerTeardown} rather than observing {@code ShutdownEvent} directly:+ * {@code ContainerTeardowns.stopAll} already runs every implementation both at process shutdown+ * and on {@code /state/reset}/{@code /state/nuke}, so this is what actually stops every+ * repository's container on reset, not the generic {@code CodeArtifactService} repository-record+ * walk, which only knows about repositories that still exist in storage.*/@ApplicationScoped-public class VerdaccioSidecarManager implements RepositorySidecarManager {+public class VerdaccioSidecarManager implements RepositorySidecarManager, ContainerTeardown {private static final Logger LOG = Logger.getLogger(VerdaccioSidecarManager.class);private static final String FORMAT = "npm";@@ -171,7 +176,8 @@ public class VerdaccioSidecarManager implements RepositorySidecarManager {}}- void onStop(@Observes ShutdownEvent ignored) {+ @Override+ public void stopManagedContainers() {pool.stopAll();}}diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/PerKeyContainerPoolTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/PerKeyContainerPoolTest.javaindex 044fcfa09e..08b37085f6 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/PerKeyContainerPoolTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/PerKeyContainerPoolTest.java@@ -20,6 +20,7 @@ import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.verify;@@ -148,6 +149,40 @@ class PerKeyContainerPoolTest {verify(lifecycleManager).stopAndRemove("container-1", null);}+ @Test+ void ensureReadyStopsItsOwnContainerWhenStopAllRunsWhileItWasStarting() throws Exception {+ CountDownLatch starterEntered = new CountDownLatch(1);+ CountDownLatch releaseStarter = new CountDownLatch(1);+ ExecutorService pool2 = Executors.newFixedThreadPool(2);+ try {+ Future<?> startFuture = pool2.submit(() -> assertThrows(IllegalStateException.class,+ () -> pool.ensureReady("key-1", () -> {+ starterEntered.countDown();+ try {+ assertTrue(releaseStarter.await(5, TimeUnit.SECONDS));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ return new StartedContainer("container-1", "http://127.0.0.1:1");+ })));+ // stopAll() has no single key to lock, so it cannot serialize against an in-flight+ // start the way stopContainer() does: this is what forces the exact race that would+ // otherwise let the container this start commits to outlive the reset that was+ // supposed to remove it.+ assertTrue(starterEntered.await(5, TimeUnit.SECONDS));++ pool.stopAll();+ releaseStarter.countDown();++ startFuture.get(5, TimeUnit.SECONDS);+ } finally {+ pool2.shutdown();+ assertTrue(pool2.awaitTermination(30, TimeUnit.SECONDS));+ }++ verify(lifecycleManager).stopAndRemove("container-1", null);+ }+@Testvoid stopAllStopsEveryTrackedContainerAndForgetsThem() throws IOException {String urlA = startFakePing(200);diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.javaindex 81560bf483..bc6ea2945a 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactNpmDockerIntegrationTest.java@@ -276,6 +276,63 @@ class CodeArtifactNpmDockerIntegrationTest {.then().statusCode(200);}+ /**+ * Proves the actual mechanism, not just that {@code VerdaccioSidecarManager} has a method+ * named right: {@code ContainerTeardowns.stopAll} runs every {@code ContainerTeardown} on+ * {@code /state/reset}, and this is what stops a live container, not the repository-record+ * walk {@code CodeArtifactService} used to do (which only knew about repositories that still+ * existed in storage).+ *+ * <p>Ordered before every other test, not after: a reset stops every container this process's+ * pool is tracking, not just the one this test starts, so running it after the other tests+ * have already started their own containers would make this test's own reset sweep those up+ * too and fail the baseline comparison below for a reason that has nothing to do with whether+ * reset-repo's own container was actually stopped. Running first means the only baseline is+ * whatever this namespace already had running before this test class touched it at all (e.g.+ * a container an earlier interrupted run never got the chance to clean up), which this reset+ * cannot be expected to touch either. Self-contained (its own domain, repository and token)+ * since the class's shared domain/token fields are not populated until {@code Order(0)} runs,+ * and a state reset wipes them regardless, so every later test must not depend on anything+ * this one touched.+ */+ @Test+ @Order(-1)+ void stateResetStopsTheRepositorysVerdaccioContainer() throws Exception {+ int baseline = runningVerdaccioTestContainerCount();++ String domain = "npm-sidecar-reset-domain";+ given().contentType("application/json").header("Authorization", AUTH).body("{}")+ .post("/v1/domain?domain=" + domain)+ .then().statusCode(200);+ given().contentType("application/json").header("Authorization", AUTH).body("{}")+ .post("/v1/repository?domain=" + domain + "&repository=reset-repo")+ .then().statusCode(200);+ String token = given().header("Authorization", AUTH)+ .post("/v1/authorization-token?domain=" + domain)+ .then().statusCode(200)+ .extract().jsonPath().getString("authorizationToken");+ // A 404 read still starts the container; only a missing token skips it.+ given().header("Authorization", "Bearer " + token)+ .get("/codeartifact/npm/" + domain + "/reset-repo/does-not-exist")+ .then().statusCode(404);+ assertEquals(baseline + 1, runningVerdaccioTestContainerCount(),+ "expected reset-repo's Verdaccio container to be running before reset");++ given().post("/_floci/state/reset").then().statusCode(200);++ assertEquals(baseline, runningVerdaccioTestContainerCount(),+ "state reset must stop reset-repo's Verdaccio container");+ }++ private static int runningVerdaccioTestContainerCount() throws IOException, InterruptedException {+ Process process = new ProcessBuilder("docker", "ps", "-q",+ "--filter", "name=floci-aws-codeartifact-npm-test-verdaccio-")+ .redirectErrorStream(true).start();+ String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8);+ process.waitFor();+ return output.isBlank() ? 0 : (int) output.lines().count();+ }+private static Matcher<Integer> anyOf201Or200() {return anyOf(is(200), is(201));}diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.javaindex 890862d7c8..dd651df440 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/CodeArtifactServiceTest.java@@ -394,37 +394,6 @@ class CodeArtifactServiceTest {verify(reposiliteClient).release(created.getSidecarContainerIds().get("maven"));}- @Test- void beforeResetReleasesEveryRepositorysSidecarStorageAcrossEveryAccount() {- service.createDomain(REGION, "dom", null, Map.of());- CodeArtifactRepository first = service.createRepository(REGION, "dom", null, "repo-1", null, null, Map.of());- CodeArtifactRepository second = service.createRepository(REGION, "dom", null, "repo-2", null, null, Map.of());-- service.beforeReset();-- verify(verdaccioManager).release(first.getSidecarContainerIds().get("npm"));- verify(reposiliteClient).release(first.getSidecarContainerIds().get("maven"));- verify(verdaccioManager).release(second.getSidecarContainerIds().get("npm"));- verify(reposiliteClient).release(second.getSidecarContainerIds().get("maven"));- }-- @Test- void beforeResetToleratesASidecarReleaseFailureAndStillReleasesTheRest() {- service.createDomain(REGION, "dom", null, Map.of());- CodeArtifactRepository first = service.createRepository(REGION, "dom", null, "repo-1", null, null, Map.of());- CodeArtifactRepository second = service.createRepository(REGION, "dom", null, "repo-2", null, null, Map.of());- doThrow(new IllegalStateException("Reposilite unreachable")).when(reposiliteClient)- .release(first.getSidecarContainerIds().get("maven"));-- // A reset must not abort partway through and leave later repositories' containers running- // just because one repository's release failed.- service.beforeReset();-- verify(reposiliteClient).release(second.getSidecarContainerIds().get("maven"));- verify(verdaccioManager).release(first.getSidecarContainerIds().get("npm"));- verify(verdaccioManager).release(second.getSidecarContainerIds().get("npm"));- }-@Testvoid deleteRepositorySucceedsEvenWhenOneFormatsSidecarReleaseFails() {service.createDomain(REGION, "dom", null, Map.of());diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/ReposiliteSidecarManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/ReposiliteSidecarManagerTest.javaindex 80f5caa832..8bfab3f463 100644--- a/src/test/java/io/github/hectorvent/floci/services/codeartifact/ReposiliteSidecarManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/ReposiliteSidecarManagerTest.java@@ -106,6 +106,25 @@ class ReposiliteSidecarManagerTest {verifyNoInteractions(containerBuilder, lifecycleManager);}+ @Test+ void stopManagedContainersStopsAndForgetsTheManagedContainer() throws Exception {+ ReposiliteSidecarManager manager = manager();+ setField(manager, "containerId", "running-reposilite");++ manager.stopManagedContainers();++ verify(lifecycleManager).stopAndRemove("running-reposilite", null);+ }++ @Test+ void stopManagedContainersIsANoOpWhenNothingWasEverStarted() {+ ReposiliteSidecarManager manager = manager();++ manager.stopManagedContainers();++ verifyNoInteractions(lifecycleManager);+ }+private ReposiliteSidecarManager manager() {return new ReposiliteSidecarManager(containerBuilder, lifecycleManager, config, "unused-managed-secret");}diff --git a/src/test/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManagerTest.javanew file mode 100644index 0000000000..a5d3ca7f3f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/codeartifact/VerdaccioSidecarManagerTest.java@@ -0,0 +1,79 @@+package io.github.hectorvent.floci.services.codeartifact;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.ContainerTeardown;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.PerKeyContainerPool;+import org.junit.jupiter.api.Test;++import java.lang.reflect.Field;+import java.util.concurrent.ConcurrentHashMap;++import static org.junit.jupiter.api.Assertions.assertInstanceOf;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;++/**+ * {@link VerdaccioSidecarManager#stopManagedContainers()} is a thin delegation to+ * {@link PerKeyContainerPool#stopAll()}, whose own behavior (stop every tracked container,+ * tolerate a failure in one, forget them all) is already covered by+ * {@code PerKeyContainerPoolTest}. What matters here, and what a Docker-gated integration test+ * cannot isolate from every other moving part, is the wiring: that the manager genuinely+ * implements {@link ContainerTeardown} (so {@code ContainerTeardowns.stopAll} finds it on+ * {@code /state/reset}, {@code /state/nuke}, and process shutdown) and that it hands the pool+ * container ids it actually started rather than something it invented. A container is seeded+ * directly into the pool's tracking map by reflection, the same way+ * {@code ReposiliteSidecarManagerTest} seeds a stale endpoint, so this needs no Docker daemon.+ */+class VerdaccioSidecarManagerTest {++ private final ContainerBuilder containerBuilder = mock(ContainerBuilder.class);+ private final ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ private final EmulatorConfig config = mock(EmulatorConfig.class);++ @Test+ void implementsContainerTeardownSoStateResetAndNukeCanFindIt() {+ assertInstanceOf(ContainerTeardown.class, manager());+ }++ @Test+ void stopManagedContainersStopsEveryContainerThePoolIsTracking() throws Exception {+ VerdaccioSidecarManager manager = manager();+ seedPooledContainer(manager, "npm-repo-1", "tracked-verdaccio-1");+ seedPooledContainer(manager, "npm-repo-2", "tracked-verdaccio-2");++ manager.stopManagedContainers();++ verify(lifecycleManager).stopAndRemove("tracked-verdaccio-1", null);+ verify(lifecycleManager).stopAndRemove("tracked-verdaccio-2", null);+ }++ @Test+ void stopManagedContainersIsANoOpWhenNothingWasEverStarted() {+ VerdaccioSidecarManager manager = manager();++ manager.stopManagedContainers();++ verifyNoInteractions(lifecycleManager);+ }++ private VerdaccioSidecarManager manager() {+ return new VerdaccioSidecarManager(containerBuilder, lifecycleManager, config);+ }++ @SuppressWarnings("unchecked")+ private static void seedPooledContainer(VerdaccioSidecarManager manager, String key, String containerId)+ throws Exception {+ Field poolField = VerdaccioSidecarManager.class.getDeclaredField("pool");+ poolField.setAccessible(true);+ PerKeyContainerPool pool = (PerKeyContainerPool) poolField.get(manager);++ Field containersField = PerKeyContainerPool.class.getDeclaredField("containers");+ containersField.setAccessible(true);+ ConcurrentHashMap<String, PerKeyContainerPool.StartedContainer> containers =+ (ConcurrentHashMap<String, PerKeyContainerPool.StartedContainer>) containersField.get(pool);+ containers.put(key, new PerKeyContainerPool.StartedContainer(containerId, "http://127.0.0.1:1"));+ }+}