Light, fluffy, and always free - The AWS Local Emulator alternative
fix(secretsmanager): serialize secret version updates with the rotation lock (#3247)
Kenshin Okinaka172765a8088c
diff --git a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.javaindex 78fbe83783..b2fe2666f3 100644--- a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java@@ -191,99 +191,104 @@ public class SecretsManagerService implements ResourceProvider {throw new AwsException("InvalidParameterException", "ClientRequestToken must be between 32 and 64 characters long.", 400);}- if (clientRequestToken != null && secret.getVersions() != null && secret.getVersions().containsKey(clientRequestToken)) {- SecretVersion existingVersion = secret.getVersions().get(clientRequestToken);- boolean isPendingPlaceholder = existingVersion.getSecretString() == null- && existingVersion.getSecretBinary() == null- && existingVersion.getVersionStages() != null- && existingVersion.getVersionStages().contains("AWSPENDING");- if (!isPendingPlaceholder) {- if (!Objects.equals(existingVersion.getSecretString(), secretString) ||- !Objects.equals(existingVersion.getSecretBinary(), secretBinary)) {- throw new AwsException("ResourceExistsException",- "You can't use ClientRequestToken " + clientRequestToken- + " because that value is already in use for a version of secret " + secret.getArn(), 400);+ synchronized (lockFor(secret.getArn())) {+ if (clientRequestToken != null && secret.getVersions() != null && secret.getVersions().containsKey(clientRequestToken)) {+ SecretVersion existingVersion = secret.getVersions().get(clientRequestToken);+ boolean isPendingPlaceholder = existingVersion.getSecretString() == null+ && existingVersion.getSecretBinary() == null+ && existingVersion.getVersionStages() != null+ && existingVersion.getVersionStages().contains("AWSPENDING");+ if (!isPendingPlaceholder) {+ if (!Objects.equals(existingVersion.getSecretString(), secretString) ||+ !Objects.equals(existingVersion.getSecretBinary(), secretBinary)) {+ throw new AwsException("ResourceExistsException",+ "You can't use ClientRequestToken " + clientRequestToken+ + " because that value is already in use for a version of secret " + secret.getArn(), 400);+ }+ return existingVersion;}- return existingVersion;- }- }-- Instant now = Instant.now();- String newVersionId = clientRequestToken != null ? clientRequestToken : UUID.randomUUID().toString();-- List<String> stages;- if (versionStages != null) {- if (versionStages.isEmpty() || versionStages.size() > 20) {- throw new AwsException("ValidationException", "Invalid length for parameter VersionStages", 400);}- if (versionStages.stream()- .anyMatch(stage -> stage == null- || stage.isEmpty()- || stage.length() > 256)) {- throw new AwsException("ValidationException", "Member must have length less than or equal to 256, Member must have length greater than or equal to 1", 400);- }- stages = versionStages;- } else {- stages = List.of(AWSCURRENT);- }- SecretVersion previousCurrent = stages.contains(AWSCURRENT) ? findVersionByStage(secret, AWSCURRENT) : null;+ Instant now = Instant.now();+ String newVersionId = clientRequestToken != null ? clientRequestToken : UUID.randomUUID().toString();- for (String stage : stages) {- SecretVersion version = findVersionByStage(secret, stage);- if (version == null) {- continue;- }- List<String> newStages = new ArrayList<>(version.getVersionStages());- // if stage is AWSCURRENT, the previous AWSCURRENT will become- // AWSPREVIOUS, and the previous AWSPREVIOUS will drop that stage- // name- if (stage.equals(AWSCURRENT)) {- SecretVersion previous = findVersionByStage(secret, AWSPREVIOUS);- if (previous != null) {- List<String> oldPrevious = new ArrayList<>(previous.getVersionStages());- oldPrevious.remove(AWSPREVIOUS);- previous.setVersionStages(oldPrevious);+ List<String> stages;+ if (versionStages != null) {+ if (versionStages.isEmpty() || versionStages.size() > 20) {+ throw new AwsException("ValidationException", "Invalid length for parameter VersionStages", 400);}- newStages.add(AWSPREVIOUS);+ if (versionStages.stream()+ .anyMatch(stage -> stage == null+ || stage.isEmpty()+ || stage.length() > 256)) {+ throw new AwsException("ValidationException", "Member must have length less than or equal to 256, Member must have length greater than or equal to 1", 400);+ }+ stages = versionStages;+ } else {+ stages = List.of(AWSCURRENT);}- newStages.remove(stage);- version.setVersionStages(newStages);- }+ SecretVersion previousCurrent = stages.contains(AWSCURRENT) ? findVersionByStage(secret, AWSCURRENT) : null;- if (previousCurrent != null) {- for (SecretVersion version : secret.getVersions().values()) {- List<String> assignedStages = version.getVersionStages();- if (assignedStages == null || !assignedStages.contains(AWSPREVIOUS)) {+ for (String stage : stages) {+ SecretVersion version = findVersionByStage(secret, stage);+ // A version carrying the id being written is replaced wholesale below, so moving+ // the stage off it first would only leave that stage briefly unassigned to a+ // reader, which is how a rotation loses track of its own AWSPENDING version.+ if (version == null || newVersionId.equals(version.getVersionId())) {continue;}- List<String> newStages = new ArrayList<>(assignedStages);- newStages.removeIf(AWSPREVIOUS::equals);+ List<String> newStages = new ArrayList<>(version.getVersionStages());+ // if stage is AWSCURRENT, the previous AWSCURRENT will become+ // AWSPREVIOUS, and the previous AWSPREVIOUS will drop that stage+ // name+ if (stage.equals(AWSCURRENT)) {+ SecretVersion previous = findVersionByStage(secret, AWSPREVIOUS);+ if (previous != null) {+ List<String> oldPrevious = new ArrayList<>(previous.getVersionStages());+ oldPrevious.remove(AWSPREVIOUS);+ previous.setVersionStages(oldPrevious);+ }+ newStages.add(AWSPREVIOUS);+ }+ newStages.remove(stage);+version.setVersionStages(newStages);}- List<String> newStages = new ArrayList<>(previousCurrent.getVersionStages());- newStages.add(AWSPREVIOUS);- previousCurrent.setVersionStages(newStages);- }+ if (previousCurrent != null) {+ for (SecretVersion version : secret.getVersions().values()) {+ List<String> assignedStages = version.getVersionStages();+ if (assignedStages == null || !assignedStages.contains(AWSPREVIOUS)) {+ continue;+ }+ List<String> newStages = new ArrayList<>(assignedStages);+ newStages.removeIf(AWSPREVIOUS::equals);+ version.setVersionStages(newStages);+ }- SecretVersion newVersion = new SecretVersion();- newVersion.setVersionId(newVersionId);- newVersion.setSecretString(secretString);- newVersion.setSecretBinary(secretBinary);- newVersion.setVersionStages(stages);- newVersion.setCreatedDate(now);+ List<String> newStages = new ArrayList<>(previousCurrent.getVersionStages());+ newStages.add(AWSPREVIOUS);+ previousCurrent.setVersionStages(newStages);+ }- secret.getVersions().put(newVersionId, newVersion);- if (stages.contains(AWSCURRENT)) {- secret.setCurrentVersionId(newVersionId);- }- secret.setLastChangedDate(now);+ SecretVersion newVersion = new SecretVersion();+ newVersion.setVersionId(newVersionId);+ newVersion.setSecretString(secretString);+ newVersion.setSecretBinary(secretBinary);+ newVersion.setVersionStages(stages);+ newVersion.setCreatedDate(now);- store.put(regionKey(region, secret.getName()), secret);- LOG.infov("Put secret value for: {0}", secret.getName());- return newVersion;+ secret.getVersions().put(newVersionId, newVersion);+ if (stages.contains(AWSCURRENT)) {+ secret.setCurrentVersionId(newVersionId);+ }+ secret.setLastChangedDate(now);++ store.put(regionKey(region, secret.getName()), secret);+ LOG.infov("Put secret value for: {0}", secret.getName());+ return newVersion;+ }}public Secret updateSecret(String secretId, String description, String kmsKeyId, String region) {@@ -677,14 +682,16 @@ public class SecretsManagerService implements ResourceProvider {private void executeRotationLifecycle(String secretArn, String clientRequestToken, String lambdaArn, boolean rotateImmediately, boolean isExistingVersion, String region) {if (!rotateImmediately) {invokeRotationLambda(secretArn, clientRequestToken, lambdaArn, "testSecret", region);-- Secret refreshed = resolveSecret(secretArn, region);- SecretVersion pending = findVersionByStage(refreshed, "AWSPENDING");- if (pending != null) {- List<String> stages = new ArrayList<>(pending.getVersionStages());- stages.remove("AWSPENDING");- pending.setVersionStages(stages);- store.put(regionKey(region, refreshed.getName()), refreshed);++ synchronized (lockFor(secretArn)) {+ Secret refreshed = resolveSecret(secretArn, region);+ SecretVersion pending = findVersionByStage(refreshed, "AWSPENDING");+ if (pending != null) {+ List<String> stages = new ArrayList<>(pending.getVersionStages());+ stages.remove("AWSPENDING");+ pending.setVersionStages(stages);+ store.put(regionKey(region, refreshed.getName()), refreshed);+ }}return;}@@ -868,69 +875,71 @@ public class SecretsManagerService implements ResourceProvider {Secret secret = resolveSecret(secretId, region);throwIfPendingDeletion(secret);- SecretVersion versionByStage = findVersionByStage(secret, versionStage);- String currentVersionId = versionByStage != null- ? versionByStage.getVersionId() : null;-- if (currentVersionId != null) {-- // If the label is attached and you either do not specify- // this parameter, or the version ID does not match, then the- // operation fails.- if (removeFromVersionId == null) {- throw new AwsException("InvalidParameterException",- ("The parameter RemoveFromVersionId can't be empty. Staging label %s is currently attached to "- + "version %s, so you must explicitly reference that version in RemoveFromVersionId.")- .formatted(versionByStage, currentVersionId), 400);- } else if (!Objects.equals(currentVersionId, removeFromVersionId)) {- throw new AwsException("InvalidParameterException",- ("When you move staging label %s, if you specify RemoveFromVersionId, it must be set to the "- + "version that currently has the staging label %s.")- .formatted(versionByStage, currentVersionId), 400);- }+ synchronized (lockFor(secret.getArn())) {+ SecretVersion versionByStage = findVersionByStage(secret, versionStage);+ String currentVersionId = versionByStage != null+ ? versionByStage.getVersionId() : null;++ if (currentVersionId != null) {++ // If the label is attached and you either do not specify+ // this parameter, or the version ID does not match, then the+ // operation fails.+ if (removeFromVersionId == null) {+ throw new AwsException("InvalidParameterException",+ ("The parameter RemoveFromVersionId can't be empty. Staging label %s is currently attached to "+ + "version %s, so you must explicitly reference that version in RemoveFromVersionId.")+ .formatted(versionStage, currentVersionId), 400);+ } else if (!Objects.equals(currentVersionId, removeFromVersionId)) {+ throw new AwsException("InvalidParameterException",+ ("When you move staging label %s, if you specify RemoveFromVersionId, it must be set to the "+ + "version that currently has the staging label %s.")+ .formatted(versionStage, currentVersionId), 400);+ }- List<String> mutableStages = new ArrayList<>(secret.getVersions()- .get(removeFromVersionId).getVersionStages());- mutableStages.remove(versionStage);+ List<String> mutableStages = new ArrayList<>(secret.getVersions()+ .get(removeFromVersionId).getVersionStages());+ mutableStages.remove(versionStage);- if (AWSCURRENT.equals(versionStage)) {- mutableStages.add(AWSPREVIOUS);+ if (AWSCURRENT.equals(versionStage)) {+ mutableStages.add(AWSPREVIOUS);- // remove AWSPREVIOUS tag from the previous SecretVersion- SecretVersion previous = findVersionByStage(secret, AWSPREVIOUS);- if (previous != null) {- List<String> mutablePrevStages =- new ArrayList<>(previous.getVersionStages());- mutablePrevStages.remove(AWSPREVIOUS);- previous.setVersionStages(mutablePrevStages);- }+ // remove AWSPREVIOUS tag from the previous SecretVersion+ SecretVersion previous = findVersionByStage(secret, AWSPREVIOUS);+ if (previous != null) {+ List<String> mutablePrevStages =+ new ArrayList<>(previous.getVersionStages());+ mutablePrevStages.remove(AWSPREVIOUS);+ previous.setVersionStages(mutablePrevStages);+ }- // we will set currentVersionId further down+ // we will set currentVersionId further down+ }+ secret.getVersions().get(removeFromVersionId).setVersionStages(mutableStages);}- secret.getVersions().get(removeFromVersionId).setVersionStages(mutableStages);- }- if (moveToVersionId != null) {- // check whether it exists- if (!secret.getVersions().containsKey(moveToVersionId)) {- throw new AwsException("ResourceNotFoundException",- "Secrets Manager can't find the specified secret value for VersionId: %s.".formatted(moveToVersionId),- 400);- }+ if (moveToVersionId != null) {+ // check whether it exists+ if (!secret.getVersions().containsKey(moveToVersionId)) {+ throw new AwsException("ResourceNotFoundException",+ "Secrets Manager can't find the specified secret value for VersionId: %s.".formatted(moveToVersionId),+ 400);+ }- // we are adding versionStage to this ID- List<String> mutableStages = new ArrayList<>(secret.getVersions().get(moveToVersionId).getVersionStages());- mutableStages.add(versionStage);- secret.getVersions().get(moveToVersionId).setVersionStages(mutableStages);+ // we are adding versionStage to this ID+ List<String> mutableStages = new ArrayList<>(secret.getVersions().get(moveToVersionId).getVersionStages());+ mutableStages.add(versionStage);+ secret.getVersions().get(moveToVersionId).setVersionStages(mutableStages);- if (AWSCURRENT.equals(versionStage)) {- secret.setCurrentVersionId(moveToVersionId);+ if (AWSCURRENT.equals(versionStage)) {+ secret.setCurrentVersionId(moveToVersionId);+ }}- }- store.put(regionKey(region, secret.getName()), secret);+ store.put(regionKey(region, secret.getName()), secret);- return secret;+ return secret;+ }}public record BatchSecretValue(diff --git a/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerRotationLockTest.java b/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerRotationLockTest.javanew file mode 100644index 0000000000..4f0cf934fb--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerRotationLockTest.java@@ -0,0 +1,310 @@+package io.github.hectorvent.floci.services.secretsmanager;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.secretsmanager.model.Secret;+import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertInstanceOf;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;++/**+ * A rotation Lambda calls PutSecretValue and UpdateSecretVersionStage on the secret its own+ * rotation is staging, so those calls race RotateSecret's "a previous rotation isn't complete"+ * guard. These tests park each of them halfway through its update and assert that another caller+ * either waits for it or still sees a coherent set of staging labels.+ */+class SecretsManagerRotationLockTest {++ private static final String REGION = "us-east-1";+ private static final String SECRET_NAME = "rotating-secret";+ private static final String PENDING_TOKEN = "a1b2c3d4-e5f6-7890-abcd-ef1234567890";+ private static final String SECOND_TOKEN = "b1b2c3d4-e5f6-7890-abcd-ef1234567890";+ private static final String THIRD_TOKEN = "c1b2c3d4-e5f6-7890-abcd-ef1234567890";+ private static final String LAMBDA_ARN = "arn:aws:lambda:us-east-1:000000000000:function:rotate";++ private SecretsManagerService service;+ private Secret secret;++ @BeforeEach+ void setUp() {+ service = new SecretsManagerService(new InMemoryStorage<>(), 30,+ new RegionResolver(REGION, "000000000000"), mock(LambdaService.class), new ObjectMapper());+ service.createSecret(SECRET_NAME, "current-value", null, null, null, null, REGION);+ // The version a rotation registers before it invokes the Lambda: staged AWSPENDING, no value yet.+ service.putSecretValue(SECRET_NAME, null, null, PENDING_TOKEN, REGION, List.of("AWSPENDING"));+ secret = service.describeSecret(SECRET_NAME, REGION);+ }++ @Test+ void awsPendingStaysReadableWhilePutSecretValueReplacesThePlaceholder() throws Exception {+ BlockingVersions versions = installBlockingVersions();+ Watched writer = startBlockedPutSecretValue(versions);++ SecretVersion pending = service.getSecretValue(SECRET_NAME, null, "AWSPENDING", REGION);+ assertNotNull(pending);+ assertEquals(PENDING_TOKEN, pending.getVersionId());++ versions.release();+ writer.awaitSuccess();+ }++ @Test+ void rotateSecretStillRejectsASecondRotationWhilePutSecretValueIsMidUpdate() throws Exception {+ BlockingVersions versions = installBlockingVersions();+ Watched writer = startBlockedPutSecretValue(versions);++ Watched rotator = start("rotator", () ->+ service.rotateSecret(SECRET_NAME, SECOND_TOKEN, LAMBDA_ARN, null, true, REGION));+ assertTrue(rotator.awaitBlocked(), "RotateSecret did not wait for the in-flight PutSecretValue");++ versions.release();+ writer.awaitSuccess();+ rotator.awaitTermination();++ AwsException failure = assertInstanceOf(AwsException.class, rotator.thrown.get());+ assertEquals("InvalidRequestException", failure.getErrorCode());+ }++ @Test+ void putSecretValueWaitsForAnInFlightStagingLabelMove() throws Exception {+ ParkingSecretVersion current = installParkingCurrentVersion();+ String currentVersionId = current.getVersionId();++ current.arm();+ // The move a rotation Lambda's finishSecret step makes: AWSCURRENT from the old version+ // onto the version it just staged as AWSPENDING.+ Watched mover = start("stage-mover", () ->+ service.updateSecretVersionStage(SECRET_NAME, PENDING_TOKEN, currentVersionId, "AWSCURRENT", REGION));+ assertTrue(current.awaitEntered(), "UpdateSecretVersionStage never reached the label move");++ Watched writer = start("writer", () ->+ service.putSecretValue(SECRET_NAME, "later-value", null, THIRD_TOKEN, REGION, null));+ assertTrue(writer.awaitBlocked(), "PutSecretValue did not wait for the in-flight label move");++ current.release();+ mover.awaitSuccess();+ writer.awaitSuccess();++ // Serialized, so the put applied on top of the completed move: one AWSCURRENT, and it is+ // the value the put wrote.+ SecretVersion latest = service.getSecretValue(SECRET_NAME, null, "AWSCURRENT", REGION);+ assertEquals(THIRD_TOKEN, latest.getVersionId());+ assertEquals("later-value", latest.getSecretString());+ assertEquals(1, secret.getVersions().values().stream()+ .filter(v -> v.getVersionStages() != null && v.getVersionStages().contains("AWSCURRENT"))+ .count());+ }++ @Test+ void putSecretValueWaitsForTheRotationLifecycleRetiringAwsPending() throws Exception {+ // A rotation started with RotateImmediately=false retires AWSPENDING from its background+ // thread. That service has no Lambda, so the lifecycle runs its steps as no-ops and+ // reaches the retirement directly.+ SecretsManagerService lambdaless = new SecretsManagerService(new InMemoryStorage<>(), 30);+ lambdaless.createSecret(SECRET_NAME, "current-value", null, null, null, null, REGION);+ Secret target = lambdaless.describeSecret(SECRET_NAME, REGION);+ SecretVersion current = target.getVersions().get(target.getCurrentVersionId());+ // The state a finishing rotation leaves behind: one version holding both labels, which+ // RotateSecret's guard accepts.+ current.setVersionStages(List.of("AWSCURRENT", "AWSPENDING"));+ ParkingSecretVersion parking = new ParkingSecretVersion(current);+ target.getVersions().put(parking.getVersionId(), parking);++ parking.arm();+ lambdaless.rotateSecret(SECRET_NAME, SECOND_TOKEN, LAMBDA_ARN, null, false, REGION);+ assertTrue(parking.awaitEntered(), "the rotation lifecycle never retired AWSPENDING");++ Watched writer = start("writer", () ->+ lambdaless.putSecretValue(SECRET_NAME, "staged-value", null, THIRD_TOKEN, REGION, List.of("AWSPENDING")));+ assertTrue(writer.awaitBlocked(), "PutSecretValue did not wait for the rotation lifecycle");++ parking.release();+ writer.awaitSuccess();++ SecretVersion pending = lambdaless.getSecretValue(SECRET_NAME, null, "AWSPENDING", REGION);+ assertEquals(THIRD_TOKEN, pending.getVersionId());+ assertEquals(1, target.getVersions().values().stream()+ .filter(v -> v.getVersionStages() != null && v.getVersionStages().contains("AWSPENDING"))+ .count());+ }++ private BlockingVersions installBlockingVersions() {+ BlockingVersions versions = new BlockingVersions(secret.getVersions());+ secret.setVersions(versions);+ return versions;+ }++ private ParkingSecretVersion installParkingCurrentVersion() {+ SecretVersion current = secret.getVersions().get(secret.getCurrentVersionId());+ ParkingSecretVersion parking = new ParkingSecretVersion(current);+ secret.getVersions().put(parking.getVersionId(), parking);+ return parking;+ }++ private Watched startBlockedPutSecretValue(BlockingVersions versions) throws InterruptedException {+ versions.arm();+ Watched writer = start("writer", () ->+ service.putSecretValue(SECRET_NAME, "rotated-value", null, PENDING_TOKEN, REGION, List.of("AWSPENDING")));+ assertTrue(versions.awaitEntered(), "PutSecretValue never reached the version swap");+ return writer;+ }++ private static Watched start(String name, Runnable body) {+ Watched watched = new Watched(name, body);+ watched.thread.start();+ return watched;+ }++ /** A thread whose failure and whose wait on the secret's monitor the test can assert on. */+ private static final class Watched {++ private final Thread thread;+ private final AtomicReference<Throwable> thrown = new AtomicReference<>();++ Watched(String name, Runnable body) {+ this.thread = new Thread(() -> {+ try {+ body.run();+ } catch (Throwable t) {+ thrown.set(t);+ }+ }, name);+ this.thread.setDaemon(true);+ }++ /** Waits for the thread to sit on a monitor rather than for a fixed interval to pass. */+ boolean awaitBlocked() throws InterruptedException {+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);+ while (System.nanoTime() < deadline) {+ if (thread.getState() == Thread.State.BLOCKED) {+ return true;+ }+ if (!thread.isAlive()) {+ return false;+ }+ Thread.sleep(5);+ }+ return false;+ }++ void awaitTermination() throws InterruptedException {+ thread.join(TimeUnit.SECONDS.toMillis(5));+ assertTrue(!thread.isAlive(), thread.getName() + " did not finish within the join timeout");+ }++ void awaitSuccess() throws InterruptedException {+ awaitTermination();+ assertNull(thrown.get(), () -> thread.getName() + " failed: " + thrown.get());+ }+ }++ /** Parks the first armed {@code put} so another thread can observe the half-applied swap. */+ private static final class BlockingVersions extends HashMap<String, SecretVersion> {++ private final Park park = new Park();++ BlockingVersions(Map<String, SecretVersion> initial) {+ super(initial);+ }++ void arm() {+ park.arm();+ }++ boolean awaitEntered() throws InterruptedException {+ return park.awaitEntered();+ }++ void release() {+ park.release();+ }++ @Override+ public SecretVersion put(String key, SecretVersion value) {+ park.parkIfArmed();+ return super.put(key, value);+ }+ }++ /** Parks after the first armed staging-label change, with that change already applied. */+ private static final class ParkingSecretVersion extends SecretVersion {++ private final Park park = new Park();++ ParkingSecretVersion(SecretVersion source) {+ setVersionId(source.getVersionId());+ setSecretString(source.getSecretString());+ setSecretBinary(source.getSecretBinary());+ setVersionStages(source.getVersionStages());+ setCreatedDate(source.getCreatedDate());+ }++ void arm() {+ park.arm();+ }++ boolean awaitEntered() throws InterruptedException {+ return park.awaitEntered();+ }++ void release() {+ park.release();+ }++ @Override+ public void setVersionStages(List<String> versionStages) {+ super.setVersionStages(versionStages);+ park.parkIfArmed();+ }+ }++ /** One-shot rendezvous: the armed thread stops here until the test releases it. */+ private static final class Park {++ private final CountDownLatch entered = new CountDownLatch(1);+ private final CountDownLatch released = new CountDownLatch(1);+ private volatile boolean armed;++ void arm() {+ armed = true;+ }++ boolean awaitEntered() throws InterruptedException {+ return entered.await(5, TimeUnit.SECONDS);+ }++ void release() {+ released.countDown();+ }++ void parkIfArmed() {+ if (!armed) {+ return;+ }+ armed = false;+ entered.countDown();+ try {+ released.await(5, TimeUnit.SECONDS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ }+}