Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

test(lambda): reuse code volume threshold fixture (#6577)

Andre Kurait95127c8370f8
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
index 14176e664e..5ba6a1cf45 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
@@ -469,13 +469,9 @@ class ContainerLauncherTest {
fn.setCodeSha256("large-arm64-code-sha");
fn.setArchitectures(List.of("arm64"));
- long originalThreshold = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ withLoweredCodeVolumeThreshold(() -> {
launcher.launch(fn);
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalThreshold;
- }
+ });
ArgumentCaptor<String> platformCaptor = ArgumentCaptor.forClass(String.class);
verify(lifecycleManager, times(2)).create(any(ContainerSpec.class), platformCaptor.capture());
@@ -1334,14 +1330,9 @@ class ContainerLauncherTest {
fn.setCodeSha256("large-fn-sha-v1");
when(lifecycleManager.create(any())).thenReturn("helper-123", "container-123");
- long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- try {
- // Force the volume path without writing a 32 MiB file.
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024; // 4 KiB
+ withLoweredCodeVolumeThreshold(() -> {
launcher.launch(fn);
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;
- }
+ });
ContainerSpec spec = captureRealContainerSpec();
// The real container mounts /var/task read-only from the code volume...
@@ -1421,13 +1412,9 @@ class ContainerLauncherTest {
// The code-volume populate fails (daemon under load), before any container is created.
doThrow(new RuntimeException("docker daemon busy")).when(lifecycleManager).ensureVolume(any());
- long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024; // force the volume path without a 32 MiB file
+ withLoweredCodeVolumeThreshold(() -> {
assertThrows(RuntimeException.class, () -> launcher.launch(fn));
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;
- }
+ });
// The runtime-api port allocated before the populate is released on this failure path...
verify(runtimeApiServerFactory).release(runtimeApiServer);
@@ -1455,14 +1442,10 @@ class ContainerLauncherTest {
// ensureCodeVolume checks, regardless of what the in-memory flag says.
when(lifecycleManager.volumeExists(anyString())).thenReturn(false);
- long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ withLoweredCodeVolumeThreshold(() -> {
launcher.launch(fn);
launcher.launch(fn);
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = original;
- }
+ });
// Populated twice, once per launch, because the bookkeeping alone is never trusted.
assertEquals(2, capturedRemotePaths.stream().filter("/var/task"::equals).count(),
@@ -1493,27 +1476,26 @@ class ContainerLauncherTest {
v2.setCodeSha256("cleanup-fn-sha-v2");
String volumeV2 = ContainerLauncher.codeVolumeName(v2);
- long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
- launcher.launch(v1);
- launcher.launch(v2);
-
- // The v1 volume is superseded but not yet cleaned up: the grace period hasn't elapsed.
- launcher.cleanupSupersededVolumes();
- verify(lifecycleManager, never()).removeVolume(volumeV1);
-
- // Once the grace period has (trivially) elapsed, the sweep removes exactly the
- // superseded v1 volume, not the current v2 one.
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
- launcher.cleanupSupersededVolumes();
- verify(lifecycleManager, times(1)).removeVolume(volumeV1);
- verify(lifecycleManager, never()).removeVolume(volumeV2);
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
- }
+ withLoweredCodeVolumeThreshold(() -> {
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ launcher.launch(v1);
+ launcher.launch(v2);
+
+ // The v1 volume is superseded but not yet cleaned up: the grace period hasn't elapsed.
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, never()).removeVolume(volumeV1);
+
+ // Once the grace period has (trivially) elapsed, the sweep removes exactly the
+ // superseded v1 volume, not the current v2 one.
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);
+ verify(lifecycleManager, never()).removeVolume(volumeV2);
+ } finally {
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ });
}
@Test
@@ -1546,22 +1528,21 @@ class ContainerLauncherTest {
// fast path actually evaluates volumeExists instead of short-circuiting past it.
when(lifecycleManager.volumeExists(volumeV1)).thenReturn(true);
- long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
- launcher.launch(v1);
- launcher.launch(v2);
- launcher.launch(v1); // roll back
-
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
- launcher.cleanupSupersededVolumes();
- verify(lifecycleManager, never()).removeVolume(volumeV1);
- verify(lifecycleManager, times(1)).removeVolume(volumeV2);
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
- }
+ withLoweredCodeVolumeThreshold(() -> {
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ launcher.launch(v1);
+ launcher.launch(v2);
+ launcher.launch(v1); // roll back
+
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, never()).removeVolume(volumeV1);
+ verify(lifecycleManager, times(1)).removeVolume(volumeV2);
+ } finally {
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ });
}
/**
@@ -1588,23 +1569,22 @@ class ContainerLauncherTest {
when(lifecycleManager.tryVolumeLabels(volumeV1))
.thenReturn(Optional.of(Map.of("floci_namespace", "someone-else")));
- long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
- launcher.launch(v1);
- launcher.launch(v2);
+ withLoweredCodeVolumeThreshold(() -> {
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.launch(v1);
+ launcher.launch(v2);
- launcher.cleanupSupersededVolumes();
- launcher.cleanupSupersededVolumes();
+ launcher.cleanupSupersededVolumes();
+ launcher.cleanupSupersededVolumes();
- verify(lifecycleManager, never()).removeVolume(volumeV1);
- verify(lifecycleManager, times(1)).tryVolumeLabels(volumeV1);
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
- }
+ verify(lifecycleManager, never()).removeVolume(volumeV1);
+ verify(lifecycleManager, times(1)).tryVolumeLabels(volumeV1);
+ } finally {
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ });
}
@Test
@@ -1634,25 +1614,24 @@ class ContainerLauncherTest {
// v1's volume persists (still in use) no matter how many times removal is attempted.
when(lifecycleManager.removeVolume(volumeV1)).thenReturn(false);
- long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
- launcher.launch(v1);
- launcher.launch(v2);
-
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
- launcher.cleanupSupersededVolumes();
- verify(lifecycleManager, times(1)).removeVolume(volumeV1);
-
- // A later sweep must still retry it, not have silently dropped it after the first
- // no-op'd attempt.
- launcher.cleanupSupersededVolumes();
- verify(lifecycleManager, times(2)).removeVolume(volumeV1);
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
- }
+ withLoweredCodeVolumeThreshold(() -> {
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ launcher.launch(v1);
+ launcher.launch(v2);
+
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);
+
+ // A later sweep must still retry it, not have silently dropped it after the first
+ // no-op'd attempt.
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, times(2)).removeVolume(volumeV1);
+ } finally {
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ });
}
@Test
@@ -1694,40 +1673,39 @@ class ContainerLauncherTest {
return null;
}).when(lifecycleManager).removeVolume(volumeV1);
- long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
- launcher.launch(v1);
- launcher.launch(v2);
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
-
- Thread cleanupThread = new Thread(launcher::cleanupSupersededVolumes);
- cleanupThread.start();
- assertTrue(cleanupHoldingLock.await(5, java.util.concurrent.TimeUnit.SECONDS),
- "cleanup never reached removeVolume");
-
- java.util.concurrent.atomic.AtomicBoolean rollbackReturned =
- new java.util.concurrent.atomic.AtomicBoolean(false);
- Thread rollbackThread = new Thread(() -> {
+ withLoweredCodeVolumeThreshold(() -> {
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
launcher.launch(v1);
- rollbackReturned.set(true);
- });
- rollbackThread.start();
-
- // Give the rollback every chance to (wrongly) proceed if the lock weren't shared.
- Thread.sleep(200);
- assertFalse(rollbackReturned.get(),
- "rollback must block while cleanup holds the volume's lock, not race past it");
-
- releaseCleanup.countDown();
- cleanupThread.join(5000);
- rollbackThread.join(5000);
- assertTrue(rollbackReturned.get(), "rollback should complete once cleanup releases the lock");
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
- }
+ launcher.launch(v2);
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+
+ Thread cleanupThread = new Thread(launcher::cleanupSupersededVolumes);
+ cleanupThread.start();
+ assertTrue(cleanupHoldingLock.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "cleanup never reached removeVolume");
+
+ java.util.concurrent.atomic.AtomicBoolean rollbackReturned =
+ new java.util.concurrent.atomic.AtomicBoolean(false);
+ Thread rollbackThread = new Thread(() -> {
+ launcher.launch(v1);
+ rollbackReturned.set(true);
+ });
+ rollbackThread.start();
+
+ // Give the rollback every chance to (wrongly) proceed if the lock weren't shared.
+ Thread.sleep(200);
+ assertFalse(rollbackReturned.get(),
+ "rollback must block while cleanup holds the volume's lock, not race past it");
+
+ releaseCleanup.countDown();
+ cleanupThread.join(5000);
+ rollbackThread.join(5000);
+ assertTrue(rollbackReturned.get(), "rollback should complete once cleanup releases the lock");
+ } finally {
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ });
}
@Test
@@ -1762,59 +1740,58 @@ class ContainerLauncherTest {
// fast path actually evaluates volumeExists instead of short-circuiting past it.
when(lifecycleManager.volumeExists(volumeV1)).thenReturn(true);
- long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
-
- // Pre-populate v1's volume with an ordinary launch, before installing the create()
- // blocking stub below - otherwise that stub would catch populateCodeVolume's own
- // helper-container create() call instead of the real container's.
- launcher.launch(v1);
-
- java.util.concurrent.CountDownLatch launchReachedCreate = new java.util.concurrent.CountDownLatch(1);
- java.util.concurrent.CountDownLatch releaseLaunch = new java.util.concurrent.CountDownLatch(1);
- java.util.concurrent.atomic.AtomicBoolean blockedOnce = new java.util.concurrent.atomic.AtomicBoolean(false);
- doAnswer(inv -> {
- if (blockedOnce.compareAndSet(false, true)) {
- // Only the delayed re-launch of v1 (the first caller after this stub is
- // installed) blocks here; v2's later launch below must not, or the test would
- // deadlock itself waiting on its own main thread to release the latch.
- launchReachedCreate.countDown();
- assertTrue(releaseLaunch.await(5, java.util.concurrent.TimeUnit.SECONDS),
- "test did not release the delayed launch in time");
- }
- return "container-123";
- }).when(lifecycleManager).create(any());
-
- Thread launchThread = new Thread(() -> launcher.launch(v1));
- launchThread.start();
- assertTrue(launchReachedCreate.await(5, java.util.concurrent.TimeUnit.SECONDS),
- "v1's delayed re-launch never reached create()");
-
- assertEquals(1, launcher.inFlightCount(volumeV1),
- "ensureCodeVolume must mark the volume in-flight before create() confirms it");
-
- // A redeploy resolves v2 and supersedes v1's volume while v1's own launch is still stuck.
- launcher.launch(v2);
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1; // grace period elapses instantly
-
- launcher.cleanupSupersededVolumes();
- verify(lifecycleManager, never()).removeVolume(volumeV1);
+ withLoweredCodeVolumeThreshold(() -> {
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
- // Let the delayed launch finish; it releases its in-flight reference on success.
- releaseLaunch.countDown();
- launchThread.join(5000);
- assertEquals(0, launcher.inFlightCount(volumeV1),
- "in-flight count must be released once create() succeeds");
+ // Pre-populate v1's volume with an ordinary launch, before installing the create()
+ // blocking stub below - otherwise that stub would catch populateCodeVolume's own
+ // helper-container create() call instead of the real container's.
+ launcher.launch(v1);
- // Nothing is in flight anymore, so a later sweep is free to actually delete it.
- launcher.cleanupSupersededVolumes();
- verify(lifecycleManager, times(1)).removeVolume(volumeV1);
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
- }
+ java.util.concurrent.CountDownLatch launchReachedCreate = new java.util.concurrent.CountDownLatch(1);
+ java.util.concurrent.CountDownLatch releaseLaunch = new java.util.concurrent.CountDownLatch(1);
+ java.util.concurrent.atomic.AtomicBoolean blockedOnce = new java.util.concurrent.atomic.AtomicBoolean(false);
+ doAnswer(inv -> {
+ if (blockedOnce.compareAndSet(false, true)) {
+ // Only the delayed re-launch of v1 (the first caller after this stub is
+ // installed) blocks here; v2's later launch below must not, or the test would
+ // deadlock itself waiting on its own main thread to release the latch.
+ launchReachedCreate.countDown();
+ assertTrue(releaseLaunch.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "test did not release the delayed launch in time");
+ }
+ return "container-123";
+ }).when(lifecycleManager).create(any());
+
+ Thread launchThread = new Thread(() -> launcher.launch(v1));
+ launchThread.start();
+ assertTrue(launchReachedCreate.await(5, java.util.concurrent.TimeUnit.SECONDS),
+ "v1's delayed re-launch never reached create()");
+
+ assertEquals(1, launcher.inFlightCount(volumeV1),
+ "ensureCodeVolume must mark the volume in-flight before create() confirms it");
+
+ // A redeploy resolves v2 and supersedes v1's volume while v1's own launch is still stuck.
+ launcher.launch(v2);
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1; // grace period elapses instantly
+
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, never()).removeVolume(volumeV1);
+
+ // Let the delayed launch finish; it releases its in-flight reference on success.
+ releaseLaunch.countDown();
+ launchThread.join(5000);
+ assertEquals(0, launcher.inFlightCount(volumeV1),
+ "in-flight count must be released once create() succeeds");
+
+ // Nothing is in flight anymore, so a later sweep is free to actually delete it.
+ launcher.cleanupSupersededVolumes();
+ verify(lifecycleManager, times(1)).removeVolume(volumeV1);
+ } finally {
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ });
}
@Test
@@ -1841,9 +1818,7 @@ class ContainerLauncherTest {
v1.setCodeLocalPath(codePath.toString());
v1.setCodeSha256("xfn-sha-v1");
- long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
+ withLoweredCodeVolumeThreshold(() -> {
Object functionLock = launcher.functionVolumeTransitionLockFor("xfn");
java.util.concurrent.CountDownLatch testHoldingLock = new java.util.concurrent.CountDownLatch(1);
@@ -1880,9 +1855,7 @@ class ContainerLauncherTest {
holderThread.join(5000);
launchThread.join(5000);
assertTrue(launchReturned.get(), "launch should complete once the function lock is released");
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
- }
+ });
}
@Test
@@ -1914,24 +1887,23 @@ class ContainerLauncherTest {
v2.setCodeLocalPath(codePath.toString());
v2.setCodeSha256("lock-persist-fn-sha-v2");
- long originalBytes = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
- long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
- try {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
- launcher.launch(v1);
- assertTrue(launcher.hasCodeVolumeLock(volumeV1), "lock must exist right after populate");
+ withLoweredCodeVolumeThreshold(() -> {
+ long originalGrace = ContainerLauncher.VOLUME_CLEANUP_GRACE_MS;
+ try {
+ launcher.launch(v1);
+ assertTrue(launcher.hasCodeVolumeLock(volumeV1), "lock must exist right after populate");
- launcher.launch(v2); // supersedes v1's volume, queues it for cleanup
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
- launcher.cleanupSupersededVolumes(); // deletes v1's volume for real
+ launcher.launch(v2); // supersedes v1's volume, queues it for cleanup
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = -1;
+ launcher.cleanupSupersededVolumes(); // deletes v1's volume for real
- assertTrue(launcher.hasCodeVolumeLock(volumeV1),
- "lock must still exist even after the volume itself was deleted - "
- + "pruning it here is exactly the bug this test guards against");
- } finally {
- ContainerLauncher.CODE_VOLUME_MIN_BYTES = originalBytes;
- ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
- }
+ assertTrue(launcher.hasCodeVolumeLock(volumeV1),
+ "lock must still exist even after the volume itself was deleted - "
+ + "pruning it here is exactly the bug this test guards against");
+ } finally {
+ ContainerLauncher.VOLUME_CLEANUP_GRACE_MS = originalGrace;
+ }
+ });
}
/** Builds a tar archive matching what {@code docker cp}/{@code copyArchiveFromContainerCmd}
@@ -2267,7 +2239,7 @@ class ContainerLauncherTest {
assertEquals(List.of("/opt/extensions/lambda-adapter", "/opt/extensions/otel-collector"),
capturedLaunchPaths(launchCmds));
}
- private static void withLoweredCodeVolumeThreshold(Runnable action) {
+ private static void withLoweredCodeVolumeThreshold(ThrowingRunnable action) throws Exception {
long original = ContainerLauncher.CODE_VOLUME_MIN_BYTES;
try {
ContainerLauncher.CODE_VOLUME_MIN_BYTES = 4 * 1024;
@@ -2277,4 +2249,8 @@ class ContainerLauncherTest {
}
}
+ @FunctionalInterface
+ private interface ThrowingRunnable {
+ void run() throws Exception;
+ }
}