Light, fluffy, and always free - The AWS Local Emulator alternative
fix(aps): honor workspace idempotency tokens (#6552)
Aniket Shukla2bab3b5cbeda
diff --git a/docs/services/managed-prometheus.md b/docs/services/managed-prometheus.mdindex db2697f8a1..dd272f1f6e 100644--- a/docs/services/managed-prometheus.md+++ b/docs/services/managed-prometheus.md@@ -20,7 +20,13 @@ definitions, scrapers and logging configurations are not implemented.ingestion/query endpoint.- The `alias` parameter of `ListWorkspaces` is a **prefix** filter, matching the provider's`alias_prefix` data-source argument.-- `clientToken` on `CreateWorkspace` is accepted and ignored: creates are not deduplicated.+- `CreateWorkspace` deduplicates requests with the same case-sensitive `clientToken` in+ the calling account and region. Tokens accept 1–64 printable ASCII characters excluding+ spaces. A retry returns the original response; changing the alias, tags or KMS key with+ the same token returns `ConflictException` (409). Receipts use the configured storage+ mode and are committed atomically with the workspace in the same store. They remain+ until that state is cleared, including after workspace deletion; token+ expiry is not emulated. Requests without a token create independently.- `kmsKeyArn` is stored and echoed back but no encryption is performed.- A rule groups namespace is **ACTIVE from birth** for the same reason, so the provider'screate (`CREATING` → `ACTIVE`) and update (`UPDATING` → `ACTIVE`) waiters complete on their@@ -38,8 +44,8 @@ definitions, scrapers and logging configurations are not implemented.- Namespace names are validated against AMP's documented constraints (1 to 128 charactersmatching `.*[0-9A-Za-z][-.0-9A-Z_a-z]*.*`). Floci additionally rejects `/`, because thenamespace is addressed as a single path segment.-- `clientToken` on `CreateRuleGroupsNamespace` is accepted and ignored, the same as on- `CreateWorkspace`: creates are not deduplicated.+- `clientToken` on `CreateRuleGroupsNamespace` is accepted and ignored: namespace creates+ are not deduplicated.- The shared tags dispatcher rejects an ARN whose service, region or account does not match therequest with `ValidationException` (400), rather than resolving it against the request's ownregion.diff --git a/pom.xml b/pom.xmlindex 564145197a..ac51810f64 100644--- a/pom.xml+++ b/pom.xml@@ -366,6 +366,12 @@<version>${aws.sdk.version}</version><scope>test</scope></dependency>+ <dependency>+ <groupId>software.amazon.awssdk</groupId>+ <artifactId>amp</artifactId>+ <version>${aws.sdk.version}</version>+ <scope>test</scope>+ </dependency><dependency><groupId>software.amazon.awssdk</groupId><artifactId>bedrock</artifactId>diff --git a/src/main/java/io/github/hectorvent/floci/services/aps/ApsController.java b/src/main/java/io/github/hectorvent/floci/services/aps/ApsController.javaindex 1638a74979..82896a4313 100644--- a/src/main/java/io/github/hectorvent/floci/services/aps/ApsController.java+++ b/src/main/java/io/github/hectorvent/floci/services/aps/ApsController.java@@ -60,10 +60,10 @@ public class ApsController {String alias = asString(body.get("alias"), "alias");String kmsKeyArn = asString(body.get("kmsKeyArn"), "kmsKeyArn");Map<String, String> tags = asStringMap(body.get("tags"), "tags");- // clientToken is accepted and ignored: creates are not deduplicated.+ String clientToken = asString(body.get("clientToken"), "clientToken");PrometheusWorkspace workspace =- service.createWorkspace(regionResolver.resolveRegion(headers), alias, tags, kmsKeyArn);+ service.createWorkspace(regionResolver.resolveRegion(headers), alias, tags, kmsKeyArn, clientToken);ObjectNode response = objectMapper.createObjectNode();response.put("workspaceId", workspace.getWorkspaceId());diff --git a/src/main/java/io/github/hectorvent/floci/services/aps/ApsService.java b/src/main/java/io/github/hectorvent/floci/services/aps/ApsService.javaindex 0581c9ca2d..23fd927891 100644--- a/src/main/java/io/github/hectorvent/floci/services/aps/ApsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/aps/ApsService.java@@ -19,6 +19,7 @@ import org.jboss.logging.Logger;import java.time.Instant;import java.util.List;import java.util.Map;+import java.util.Objects;import java.util.UUID;import java.util.regex.Pattern;@@ -31,6 +32,7 @@ public class ApsService implements TagHandler {private static final int MAX_PAGE = 1000;private static final int MAX_NAMESPACE_NAME_LENGTH = 128;private static final Pattern NAMESPACE_NAME = Pattern.compile(".*[0-9A-Za-z][-.0-9A-Z_a-z]*.*");+ private static final Pattern CLIENT_TOKEN = Pattern.compile("[!-~]{1,64}");private final StorageBackend<String, PrometheusWorkspace> storage;private final StorageBackend<String, RuleGroupsNamespace> namespaceStorage;@@ -47,6 +49,29 @@ public class ApsService implements TagHandler {public PrometheusWorkspace createWorkspace(String region, String alias, Map<String, String> tags,String kmsKeyArn) {+ return createWorkspace(region, alias, tags, kmsKeyArn, null);+ }++ // ponytail: serialize token claims; use per-token locks if workspace-create throughput matters.+ public synchronized PrometheusWorkspace createWorkspace(String region, String alias, Map<String, String> tags,+ String kmsKeyArn, String clientToken) {+ if (clientToken != null && !CLIENT_TOKEN.matcher(clientToken).matches()) {+ throw new AwsException("ValidationException",+ "clientToken must contain 1 to 64 printable ASCII characters without spaces.", 400);+ }+ Map<String, String> requestedTags = tags != null ? tags : Map.of();+ if (clientToken != null) {+ PrometheusWorkspace receipt = storage.get(workspaceRequestKey(region, clientToken)).orElse(null);+ if (receipt != null) {+ if (!Objects.equals(stripAlias(alias), receipt.getAlias())+ || !Objects.equals(kmsKeyArn, receipt.getKmsKeyArn())+ || !requestedTags.equals(receipt.getTags())) {+ throw new AwsException("ConflictException",+ "clientToken has already been used with different workspace parameters.", 409);+ }+ return copyWorkspace(receipt);+ }+ }String workspaceId = "ws-" + UUID.randomUUID();String arn = regionResolver.buildArn("aps", region, "workspace/" + workspaceId);@@ -66,9 +91,29 @@ public class ApsService implements TagHandler {workspace.getTags().putAll(tags);}- storage.put(key(region, workspaceId), workspace);+ PrometheusWorkspace response = clientToken != null ? copyWorkspace(workspace) : workspace;+ if (clientToken != null) {+ // One commit keeps the receipt and resource together in snapshots and WAL replay.+ storage.putAllAtomically(Map.of(key(region, workspaceId), workspace,+ workspaceRequestKey(region, clientToken), response));+ } else {+ storage.put(key(region, workspaceId), workspace);+ }LOG.infov("Created AMP workspace: {0} in {1}", workspaceId, region);- return workspace;+ return clientToken != null ? copyWorkspace(response) : workspace;+ }++ private static PrometheusWorkspace copyWorkspace(PrometheusWorkspace source) {+ PrometheusWorkspace copy = new PrometheusWorkspace();+ copy.setWorkspaceId(source.getWorkspaceId());+ copy.setAlias(source.getAlias());+ copy.setArn(source.getArn());+ copy.setStatus(source.getStatus());+ copy.setPrometheusEndpoint(source.getPrometheusEndpoint());+ copy.setCreatedAt(source.getCreatedAt());+ copy.setKmsKeyArn(source.getKmsKeyArn());+ copy.setTags(source.getTags());+ return copy;}public PrometheusWorkspace describeWorkspace(String region, String workspaceId) {@@ -263,6 +308,11 @@ public class ApsService implements TagHandler {return region + "::";}+ private static String workspaceRequestKey(String region, String clientToken) {+ // Request regions cannot contain ':', keeping receipts outside every workspace prefix.+ return ":CreateWorkspace::" + region + "::" + clientToken;+ }+private static String namespaceKey(String region, String workspaceId, String name) {return namespaceKeyPrefix(region, workspaceId) + name;}diff --git a/src/test/java/io/github/hectorvent/floci/services/aps/ApsControllerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/aps/ApsControllerIntegrationTest.javaindex 232397d1a0..e17d514f13 100644--- a/src/test/java/io/github/hectorvent/floci/services/aps/ApsControllerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/aps/ApsControllerIntegrationTest.java@@ -5,6 +5,7 @@ import org.junit.jupiter.api.Test;import java.nio.charset.StandardCharsets;import java.util.Base64;+import java.util.UUID;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.containsString;@@ -29,8 +30,8 @@ class ApsControllerIntegrationTest {return given().contentType("application/json").body("""- {"alias": "%s", "tags": {"team": "devops"}, "clientToken": "token-123"}- """.formatted(alias))+ {"alias": "%s", "tags": {"team": "devops"}, "clientToken": "%s"}+ """.formatted(alias, UUID.randomUUID())).when().post("/workspaces").then()diff --git a/src/test/java/io/github/hectorvent/floci/services/aps/ApsCreateWorkspaceIdempotencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/aps/ApsCreateWorkspaceIdempotencyIntegrationTest.javanew file mode 100644index 0000000000..4c116f736e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/aps/ApsCreateWorkspaceIdempotencyIntegrationTest.java@@ -0,0 +1,195 @@+package io.github.hectorvent.floci.services.aps;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.aps.model.PrometheusWorkspace;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.RestAssured;+import io.restassured.response.Response;+import jakarta.inject.Inject;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.MethodSource;+import org.junit.jupiter.params.provider.ValueSource;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.amp.AmpClient;+import software.amazon.awssdk.services.amp.model.ConflictException;+import software.amazon.awssdk.services.amp.model.CreateWorkspaceResponse;+import software.amazon.awssdk.services.amp.model.ValidationException;++import java.net.URI;+import java.util.HashSet;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.Set;+import java.util.UUID;+import java.util.concurrent.ConcurrentLinkedQueue;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.stream.Stream;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++@QuarkusTest+class ApsCreateWorkspaceIdempotencyIntegrationTest {++ private static final String ACCOUNT = "741648800001";+ private static final String OTHER_ACCOUNT = "741648800002";+ private static final Region REGION = Region.EU_CENTRAL_1;+ private final ConcurrentLinkedQueue<Creation> created = new ConcurrentLinkedQueue<>();++ @Inject+ StorageFactory storageFactory;++ @AfterEach+ void cleanUp() {+ AccountAwareStorageBackend<PrometheusWorkspace> workspaces = storageFactory.create("aps", "aps-workspaces.json",+ new TypeReference<Map<String, PrometheusWorkspace>>() {});+ created.forEach(item -> {+ workspaces.deleteForAccount(item.account(), item.region().id() + "::" + item.id());+ if (item.token() != null) {+ workspaces.deleteForAccount(item.account(), ":CreateWorkspace::" + item.region().id() + "::" + item.token());+ }+ });+ }++ @Test+ void sdkReplaysTheOriginalResponseWithoutOverwritingLaterWorkspaceChanges() {+ String token = UUID.randomUUID().toString();+ try (AmpClient client = client(ACCOUNT, REGION)) {+ CreateWorkspaceResponse first = create(client, token, "original", Map.of("team", "one"), null, ACCOUNT, REGION);+ client.updateWorkspaceAlias(request -> request.workspaceId(first.workspaceId()).alias("updated"));+ client.tagResource(request -> request.resourceArn(first.arn()).tags(Map.of("team", "two")));+ CreateWorkspaceResponse replay = create(client, token, "original", Map.of("team", "one"), null, ACCOUNT, REGION);+ assertEquals(first.workspaceId(), replay.workspaceId());+ assertEquals(first.arn(), replay.arn());+ assertEquals(first.tags(), replay.tags());+ assertEquals(first.status(), replay.status());+ assertEquals("updated", client.describeWorkspace(request -> request.workspaceId(first.workspaceId()))+ .workspace().alias());+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"alias", "tags", "kms"})+ void sdkRejectsTokenReuseWithDifferentWorkspaceParameters(String changed) {+ String token = UUID.randomUUID().toString();+ try (AmpClient client = client(ACCOUNT, REGION)) {+ create(client, token, "original", Map.of("team", "one"), null, ACCOUNT, REGION);+ String kms = "kms".equals(changed) ? AwsArnUtils.Arn.of("kms", REGION.id(), ACCOUNT,+ "key/" + UUID.randomUUID()).toString() : null;+ ConflictException error = assertThrows(ConflictException.class, () -> create(client, token,+ "alias".equals(changed) ? "different" : "original",+ "tags".equals(changed) ? Map.of("team", "two") : Map.of("team", "one"), kms, ACCOUNT, REGION));+ assertEquals(409, error.statusCode());+ }+ }++ @ParameterizedTest+ @MethodSource("invalidTokens")+ void sdkRejectsTokensOutsideTheDocumentedShape(String token) {+ try (AmpClient client = client(ACCOUNT, REGION)) {+ assertEquals(400, assertThrows(ValidationException.class,+ () -> create(client, token, "invalid", Map.of(), null, ACCOUNT, REGION)).statusCode());+ }+ }++ static Stream<String> invalidTokens() {+ return Stream.of("", "has space", "line\nbreak", "é", "a".repeat(65));+ }++ @Test+ void sdkAcceptsTheSixtyFourCharacterBoundaryAndKeepsTokensCaseSensitive() {+ try (AmpClient client = client(ACCOUNT, REGION)) {+ String token = "A".repeat(64);+ CreateWorkspaceResponse first = create(client, token, "boundary", Map.of(), null, ACCOUNT, REGION);+ assertEquals(first.workspaceId(), create(client, token, "boundary", Map.of(), null, ACCOUNT, REGION).workspaceId());+ assertNotEquals(first.workspaceId(), create(client, token.toLowerCase(Locale.ROOT),+ "boundary", Map.of(), null, ACCOUNT, REGION).workspaceId());+ }+ }++ @Test+ void sdkScopesTheSameTokenToAccountAndRegion() {+ String token = UUID.randomUUID().toString();+ try (AmpClient client = client(ACCOUNT, REGION);+ AmpClient otherRegion = client(ACCOUNT, Region.EU_WEST_1);+ AmpClient otherAccount = client(OTHER_ACCOUNT, REGION)) {+ CreateWorkspaceResponse first = create(client, token, "scoped", Map.of(), null, ACCOUNT, REGION);+ CreateWorkspaceResponse regional = create(otherRegion, token, "scoped", Map.of(), null, ACCOUNT, Region.EU_WEST_1);+ CreateWorkspaceResponse foreign = create(otherAccount, token, "scoped", Map.of(), null, OTHER_ACCOUNT, REGION);+ assertEquals(3, Set.of(first.workspaceId(), regional.workspaceId(), foreign.workspaceId()).size());+ assertEquals(first.workspaceId(), create(client, token, "scoped", Map.of(), null, ACCOUNT, REGION).workspaceId());+ assertEquals(OTHER_ACCOUNT, AwsArnUtils.parse(foreign.arn()).accountId());+ }+ }++ @Test+ void concurrentSdkRetriesCreateOneWorkspace() throws Exception {+ String token = UUID.randomUUID().toString();+ ExecutorService pool = Executors.newFixedThreadPool(6);+ CountDownLatch start = new CountDownLatch(1);+ try (AmpClient client = client(ACCOUNT, REGION)) {+ List<Future<String>> replies = Stream.generate(() -> pool.submit(() -> {+ start.await();+ return create(client, token, "concurrent", Map.of(), null, ACCOUNT, REGION).workspaceId();+ })).limit(6).toList();+ start.countDown();+ Set<String> ids = new HashSet<>();+ for (Future<String> reply : replies) {+ ids.add(reply.get(10, TimeUnit.SECONDS));+ }+ assertEquals(1, ids.size());+ } finally {+ pool.shutdownNow();+ assertTrue(pool.awaitTermination(10, TimeUnit.SECONDS));+ }+ }++ @Test+ void requestsWithoutATokenCreateIndependentWorkspaces() {+ String first = createWithoutToken();+ String second = createWithoutToken();+ assertNotEquals(first, second);+ }++ private String createWithoutToken() {+ Response response = given().header("Authorization", "AWS4-HMAC-SHA256 Credential=" + ACCOUNT+ + "/20261026/" + REGION.id() + "/aps/aws4_request")+ .contentType("application/json").body(Map.of("alias", "no-token")).post("/workspaces");+ assertEquals(202, response.statusCode());+ String id = response.jsonPath().getString("workspaceId");+ created.add(new Creation(ACCOUNT, REGION, id, null));+ return id;+ }++ private CreateWorkspaceResponse create(AmpClient client, String token, String alias, Map<String, String> tags,+ String kms, String account, Region region) {+ CreateWorkspaceResponse response = client.createWorkspace(request -> request.clientToken(token)+ .alias(alias).tags(tags).kmsKeyArn(kms));+ created.add(new Creation(account, region, response.workspaceId(), token));+ return response;+ }++ private static AmpClient client(String account, Region region) {+ return AmpClient.builder().endpointOverride(URI.create("http://localhost:" + RestAssured.port))+ .httpClientBuilder(UrlConnectionHttpClient.builder()).region(region)+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(account, "test"))).build();+ }++ private record Creation(String account, Region region, String id, String token) {}+}diff --git a/src/test/java/io/github/hectorvent/floci/services/aps/ApsWorkspacePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/aps/ApsWorkspacePersistenceTest.javanew file mode 100644index 0000000000..8eef24f4a0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/aps/ApsWorkspacePersistenceTest.java@@ -0,0 +1,159 @@+package io.github.hectorvent.floci.services.aps;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.HybridStorage;+import io.github.hectorvent.floci.core.storage.PersistentStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.aps.model.PrometheusWorkspace;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import org.junit.jupiter.params.provider.ValueSource;+import org.mockito.stubbing.Answer;++import java.nio.file.Files;+import java.nio.file.Path;+import java.util.HashMap;+import java.util.Map;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.when;++class ApsWorkspacePersistenceTest {++ private static final String ACCOUNT = "741648800003";+ private static final String REGION = "eu-central-1";++ @TempDir+ Path directory;++ @Test+ void aFlushDuringCreateCannotPublishOnlyOneHalfOfTheRequest() throws Exception {+ String name = "aps-workspaces.json";+ Path restored = Files.createDirectory(directory.resolve("restored"));+ HybridStorage<String, PrometheusWorkspace> backend = new HybridStorage<>(directory.resolve(name),+ new TypeReference<Map<String, PrometheusWorkspace>>() {}, 3_600_000);+ AccountAwareStorageBackend<PrometheusWorkspace> storage =+ spy(new AccountAwareStorageBackend<>(backend, null, ACCOUNT));+ AtomicReference<byte[]> firstImage = new AtomicReference<>();+ Answer<Object> capturePublication = call -> {+ Object result = call.callRealMethod();+ backend.flush();+ firstImage.compareAndSet(null, Files.readAllBytes(directory.resolve(name)));+ return result;+ };+ doAnswer(capturePublication).when(storage).put(anyString(), any());+ doAnswer(capturePublication).when(storage).putAllAtomically(any());+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.<PrometheusWorkspace>create(eq("aps"), eq(name), any())).thenReturn(storage);+ Map<String, StorageBackend<String, ?>> restoredStores = new HashMap<>();+ try {+ ApsService first = new ApsService(factory, new RegionResolver(REGION, ACCOUNT));+ PrometheusWorkspace initial = first.createWorkspace(REGION, "original", Map.of(), null, "mid-create");+ Files.write(restored.resolve(name), firstImage.get());+ ApsService restarted = service(restored, "hybrid", restoredStores);+ PrometheusWorkspace replay = restarted.createWorkspace(REGION, "original", Map.of(), null, "mid-create");+ assertEquals(initial.getWorkspaceId(), replay.getWorkspaceId());+ assertDoesNotThrow(() -> restarted.describeWorkspace(REGION, replay.getWorkspaceId()));+ assertEquals(1, restarted.listWorkspaces(REGION, null, null, null).items().size());+ } finally {+ backend.shutdown();+ shutdown(restoredStores);+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"hybrid", "persistent"})+ void savedCreateRestoresBothWorkspaceAndReceipt(String mode) throws Exception {+ Map<String, StorageBackend<String, ?>> originalStores = new HashMap<>();+ Map<String, StorageBackend<String, ?>> restoredStores = new HashMap<>();+ Path original = Files.createDirectory(directory.resolve("original"));+ Path restored = Files.createDirectory(directory.resolve("restored"));+ try {+ ApsService first = service(original, mode, originalStores);+ PrometheusWorkspace initial = first.createWorkspace(REGION, "original", Map.of("team", "one"), null, "retry");+ String savedFile = "aps-workspaces.json";+ originalStores.get(savedFile).flush();+ // Restore the disk image at this flush boundary, before any shutdown flushes.+ Files.copy(original.resolve(savedFile), restored.resolve(savedFile));++ ApsService restarted = service(restored, mode, restoredStores);+ PrometheusWorkspace replay = restarted.createWorkspace(REGION, "original", Map.of("team", "one"), null, "retry");+ assertEquals(initial.getWorkspaceId(), replay.getWorkspaceId(), "a durable create cannot duplicate on retry");+ PrometheusWorkspace described = assertDoesNotThrow(() -> restarted.describeWorkspace(REGION, replay.getWorkspaceId()),+ "a durable receipt cannot point at a missing workspace");+ assertEquals(initial.getCreatedAt(), described.getCreatedAt());+ assertEquals(1, restarted.listWorkspaces(REGION, null, null, null).items().size());+ assertEquals(0, restarted.listWorkspaces("us-west-2", null, null, null).items().size());+ assertEquals(0, restarted.listWorkspaces("CreateWorkspace", null, null, null).items().size());+ assertEquals(0, restarted.listWorkspaces("", null, null, null).items().size());+ } finally {+ shutdown(originalStores);+ shutdown(restoredStores);+ }+ }++ @ParameterizedTest+ @CsvSource({"hybrid,false", "hybrid,true", "persistent,false", "persistent,true"})+ void savedDeletionRetainsReceiptWithoutRestoringTheDeletedWorkspace(String mode, boolean changeAlias) {+ Map<String, StorageBackend<String, ?>> originalStores = new HashMap<>();+ Map<String, StorageBackend<String, ?>> restoredStores = new HashMap<>();+ try {+ ApsService first = service(directory, mode, originalStores);+ PrometheusWorkspace initial = first.createWorkspace(REGION, "original", Map.of("team", "one"), null, "deleted");+ if (changeAlias) {+ first.updateWorkspaceAlias(REGION, initial.getWorkspaceId(), "updated");+ }+ first.deleteWorkspace(REGION, initial.getWorkspaceId());+ originalStores.values().forEach(StorageBackend::flush);++ ApsService restarted = service(directory, mode, restoredStores);+ PrometheusWorkspace replay = restarted.createWorkspace(REGION, "original", Map.of("team", "one"), null, "deleted");+ assertEquals(initial.getWorkspaceId(), replay.getWorkspaceId());+ assertEquals("original", replay.getAlias());+ assertFalse(restarted.listWorkspaces(REGION, null, null, null).items().stream()+ .anyMatch(workspace -> workspace.getWorkspaceId().equals(initial.getWorkspaceId())));+ } finally {+ shutdown(originalStores);+ shutdown(restoredStores);+ }+ }++ private ApsService service(Path path, String mode, Map<String, StorageBackend<String, ?>> stores) {+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.create(anyString(), anyString(), any())).thenAnswer(call ->+ store(path, mode, call.getArgument(1), call.getArgument(2), stores));+ return new ApsService(factory, new RegionResolver(REGION, ACCOUNT));+ }++ private <T> AccountAwareStorageBackend<T> store(Path path, String mode, String name, TypeReference<Map<String, T>> type,+ Map<String, StorageBackend<String, ?>> stores) {+ StorageBackend<String, T> backend = "hybrid".equals(mode)+ ? new HybridStorage<>(path.resolve(name), type, 3_600_000)+ : new PersistentStorage<>(path.resolve(name), type);+ backend.load();+ stores.put(name, backend);+ return new AccountAwareStorageBackend<>(backend, null, ACCOUNT);+ }++ private void shutdown(Map<String, StorageBackend<String, ?>> stores) {+ stores.values().forEach(backend -> {+ if (backend instanceof HybridStorage<?, ?> hybrid) {+ hybrid.shutdown();+ }+ });+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/aps/ApsWorkspaceReceiptTest.java b/src/test/java/io/github/hectorvent/floci/services/aps/ApsWorkspaceReceiptTest.javanew file mode 100644index 0000000000..e7e964c7dc--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/aps/ApsWorkspaceReceiptTest.java@@ -0,0 +1,79 @@+package io.github.hectorvent.floci.services.aps;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.aps.model.PrometheusWorkspace;+import jakarta.ws.rs.core.HttpHeaders;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.when;++class ApsWorkspaceReceiptTest {++ private static final String ACCOUNT = "741648800003";+ private static final String REGION = "eu-central-1";+ private final AccountAwareStorageBackend<PrometheusWorkspace> workspaces =+ spy(AccountAwareStorageBackend.inMemory(ACCOUNT));+ private final StorageFactory factory = mock(StorageFactory.class);+ private final RegionResolver resolver = new RegionResolver(REGION, ACCOUNT);+ private final ObjectMapper mapper = new ObjectMapper().registerModule(new JavaTimeModule());+ private final HttpHeaders headers = mock(HttpHeaders.class);++ @BeforeEach+ void setUp() {+ when(factory.<PrometheusWorkspace>create(eq("aps"), eq("aps-workspaces.json"), any())).thenReturn(workspaces);+ }++ @Test+ void tokenReceiptSurvivesJsonRoundTripAndServiceRecreation() throws Exception {+ String token = "receipt-reload";+ Map<String, Object> request = Map.of("alias", "receipt", "clientToken", token, "tags", Map.of("team", "one"));+ ApsController first = new ApsController(new ApsService(factory, resolver), resolver, mapper);+ JsonNode initial = (JsonNode) first.createWorkspace(headers, request).getEntity();+ String receiptKey = ":CreateWorkspace::" + REGION + "::" + token;+ PrometheusWorkspace receipt = workspaces.get(receiptKey).orElse(null);+ assertNotNull(receipt, "the receipt belongs in StorageFactory storage, not a service-local cache");+ workspaces.put(receiptKey, mapper.readValue(mapper.writeValueAsBytes(receipt), PrometheusWorkspace.class));+ ApsController recreated = new ApsController(new ApsService(factory, resolver), resolver, mapper);+ JsonNode replay = (JsonNode) recreated.createWorkspace(headers, request).getEntity();+ assertEquals(initial, replay);+ assertEquals(1, workspaces.scan(key -> key.startsWith(REGION + "::")).size());+ }++ @Test+ void concurrentChangesAtPublicationCannotAlterTheReceiptOrFirstResponse() {+ doAnswer(call -> {+ call.callRealMethod();+ Map<String, PrometheusWorkspace> entries = call.getArgument(0);+ PrometheusWorkspace published = entries.entrySet().stream()+ .filter(entry -> entry.getKey().startsWith(REGION + "::"))+ .map(Map.Entry::getValue).findFirst().orElseThrow();+ published.setAlias("updated");+ published.getTags().put("team", "two");+ return null;+ }).when(workspaces).putAllAtomically(any());+ Map<String, Object> request = Map.of("alias", "original", "clientToken", "publication", "tags", Map.of("team", "one"));+ ApsController controller = new ApsController(new ApsService(factory, resolver), resolver, mapper);+ JsonNode initial = (JsonNode) controller.createWorkspace(headers, request).getEntity();+ PrometheusWorkspace receipt = workspaces.get(":CreateWorkspace::" + REGION + "::publication").orElseThrow();+ assertEquals("original", receipt.getAlias());+ assertEquals("one", initial.path("tags").path("team").asText());+ assertEquals("updated", workspaces.scan(key -> key.startsWith(REGION + "::")).getFirst().getAlias());+ JsonNode replay = (JsonNode) controller.createWorkspace(headers, request).getEntity();+ assertEquals(initial, replay);+ }+}