Light, fluffy, and always free - The AWS Local Emulator alternative
fix(transcribe): isolate jobs and vocabularies by account and region (#3402)
Niels Kootstrac4ff42b80065
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.javaindex f36b181040..a13b8c1546 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java@@ -308,7 +308,7 @@ public class AwsJson11Controller {case "comprehend" -> comprehendJsonHandler.handle(action, request, region);case "rekognition" -> rekognitionJsonHandler.handle(action, request, region);case "pricing" -> pricingJsonHandler.handle(action, request, region);- case "transcribe" -> transcribeJsonHandler.handle(action, request, region);+ case "transcribe" -> transcribeJsonHandler.handle(action, request);case "translate" -> translateJsonHandler.handle(action, request, region);case "ce" -> costExplorerJsonHandler.handle(action, request, region);case "cur" -> curJsonHandler.handle(action, request, region);diff --git a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.javaindex 97d033c3c1..2d3983255a 100644--- a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.java@@ -33,7 +33,7 @@ public class TranscribeJsonHandler {this.objectMapper = objectMapper;}- public Response handle(String action, JsonNode request, String region) {+ public Response handle(String action, JsonNode request) {LOG.debugv("Transcribe action: {0}", action);return switch (action) {case "StartTranscriptionJob" -> {diff --git a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.javaindex 6269b09988..7c19cb2993 100644--- a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java+++ b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java@@ -2,7 +2,8 @@ package io.github.hectorvent.floci.services.transcribe;import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.core.common.AwsException;-import io.github.hectorvent.floci.core.storage.StorageBackedMap;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.transcribe.model.TranscriptionJob;import io.github.hectorvent.floci.services.transcribe.model.TranscriptionJobSummary;@@ -16,6 +17,7 @@ import java.time.Instant;import java.util.Comparator;import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.concurrent.ConcurrentHashMap;/**@@ -28,35 +30,45 @@ import java.util.concurrent.ConcurrentHashMap;public class TranscribeService implements Resettable {private final StorageFactory storageFactory;+ private final RegionResolver regionResolver;// Transcription jobs are transient (async work deleted after processing); vocabularies are durable.private final ConcurrentHashMap<String, TranscriptionJob> transcriptionJobs = new ConcurrentHashMap<>();- private Map<String, VocabularyInfo> vocabularies = new ConcurrentHashMap<>();+ private AccountAwareStorageBackend<VocabularyInfo> vocabularies;@Inject- public TranscribeService(StorageFactory storageFactory) {+ public TranscribeService(StorageFactory storageFactory, RegionResolver regionResolver) {this.storageFactory = storageFactory;+ this.regionResolver = regionResolver;+ }++ TranscribeService(StorageFactory storageFactory) {+ this(storageFactory, new RegionResolver("us-east-1", "000000000000"));}@PostConstructvoid initializeStorage() {if (storageFactory == null) {- return; // keeps non-CDI unit tests working+ vocabularies = AccountAwareStorageBackend.inMemory(regionResolver.getDefaultAccountId());+ return;}- this.vocabularies = new StorageBackedMap<>(storageFactory.create("transcribe",- "transcribe-vocabularies.json", new TypeReference<Map<String, VocabularyInfo>>() {}));+ this.vocabularies = storageFactory.create("transcribe",+ "transcribe-vocabularies.json", new TypeReference<Map<String, VocabularyInfo>>() {});}public void clear() {transcriptionJobs.clear();- vocabularies.clear();+ if (vocabularies != null) {+ vocabularies.clear();+ }}public TranscriptionJob startTranscriptionJob(String jobName, String mediaFileUri,String languageCode, String mediaFormat) {requireNonBlank(jobName, "TranscriptionJobName");requireNonBlank(mediaFileUri, "Media.MediaFileUri");- if (transcriptionJobs.containsKey(jobName)) {+ String jobKey = jobKey(jobName);+ if (transcriptionJobs.containsKey(jobKey)) {throw new AwsException("ConflictException","The requested job name already exists. Use a different job name.", 400);}@@ -72,13 +84,13 @@ public class TranscribeService implements Resettable {new TranscriptionJob.Transcript("s3://floci-transcribe-output/" + jobName + ".json"),now, now, now);- transcriptionJobs.put(jobName, job);+ transcriptionJobs.put(jobKey, job);return job;}public TranscriptionJob getTranscriptionJob(String jobName) {requireNonBlank(jobName, "TranscriptionJobName");- TranscriptionJob job = transcriptionJobs.get(jobName);+ TranscriptionJob job = transcriptionJobs.get(jobKey(jobName));if (job == null) {throw new AwsException("NotFoundException","The requested job couldn't be found. Check the job name and try your request again.", 400);@@ -90,7 +102,10 @@ public class TranscribeService implements Resettable {Integer maxResults) {int limit = maxResults != null ? Math.min(maxResults, 100) : 100;- List<TranscriptionJobSummary> filtered = transcriptionJobs.values().stream()+ String currentPrefix = currentJobPrefix();+ List<TranscriptionJobSummary> filtered = transcriptionJobs.entrySet().stream()+ .filter(entry -> entry.getKey().startsWith(currentPrefix))+ .map(Map.Entry::getValue).filter(j -> statusFilter == null || statusFilter.equals(j.transcriptionJobStatus())).filter(j -> jobNameContains == null || j.transcriptionJobName().contains(jobNameContains)).sorted(Comparator.comparing(TranscriptionJob::transcriptionJobName))@@ -106,7 +121,7 @@ public class TranscribeService implements Resettable {public void deleteTranscriptionJob(String jobName) {requireNonBlank(jobName, "TranscriptionJobName");- if (transcriptionJobs.remove(jobName) == null) {+ if (transcriptionJobs.remove(jobKey(jobName)) == null) {throw new AwsException("BadRequestException","The requested job couldn't be found. Check the job name and try your request again.", 400);}@@ -115,20 +130,20 @@ public class TranscribeService implements Resettable {public VocabularyInfo createVocabulary(String vocabularyName, String languageCode) {requireNonBlank(vocabularyName, "VocabularyName");requireNonBlank(languageCode, "LanguageCode");- if (vocabularies.containsKey(vocabularyName)) {+ if (getStoredVocabulary(vocabularyName).isPresent()) {throw new AwsException("ConflictException","The requested vocabulary name already exists. Use a different vocabulary name.", 400);}VocabularyInfo vocab = new VocabularyInfo(vocabularyName, languageCode, "READY", Instant.now().getEpochSecond());- vocabularies.put(vocabularyName, vocab);+ vocabularies.putForAccount(regionResolver.getAccountId(), vocabularyKey(vocabularyName), vocab);return vocab;}public VocabularyInfo getVocabulary(String vocabularyName) {requireNonBlank(vocabularyName, "VocabularyName");- VocabularyInfo vocab = vocabularies.get(vocabularyName);+ VocabularyInfo vocab = getStoredVocabulary(vocabularyName).orElse(null);if (vocab == null) {throw new AwsException("NotFoundException","The requested vocabulary couldn't be found. Check the vocabulary name and try your request again.",@@ -141,7 +156,12 @@ public class TranscribeService implements Resettable {Integer maxResults) {int limit = maxResults != null ? Math.min(maxResults, 100) : 100;- List<VocabularyInfo> filtered = vocabularies.values().stream()+ migrateDefaultScopeVocabularies();++ String currentPrefix = currentVocabularyPrefix();+ List<VocabularyInfo> filtered = vocabularies.scanForAccount(regionResolver.getAccountId(),+ key -> key.startsWith(currentPrefix))+ .stream().filter(v -> stateEquals == null || stateEquals.equals(v.vocabularyState())).filter(v -> nameContains == null || v.vocabularyName().contains(nameContains)).sorted(Comparator.comparing(VocabularyInfo::vocabularyName))@@ -156,11 +176,53 @@ public class TranscribeService implements Resettable {public void deleteVocabulary(String vocabularyName) {requireNonBlank(vocabularyName, "VocabularyName");- if (vocabularies.remove(vocabularyName) == null) {+ if (getStoredVocabulary(vocabularyName).isEmpty()) {throw new AwsException("NotFoundException","The requested vocabulary couldn't be found. Check the vocabulary name and try your request again.",400);}+ vocabularies.deleteForAccount(regionResolver.getAccountId(), vocabularyKey(vocabularyName));+ }++ private String jobKey(String jobName) {+ return currentJobPrefix() + jobName;+ }++ private String vocabularyKey(String vocabularyName) {+ return regionResolver.getRegion() + "/" + vocabularyName;+ }++ private String currentJobPrefix() {+ return regionResolver.getAccountId() + "/" + regionResolver.getRegion() + "/";+ }++ private String currentVocabularyPrefix() {+ return regionResolver.getRegion() + "/";+ }++ private Optional<VocabularyInfo> getStoredVocabulary(String vocabularyName) {+ String accountId = regionResolver.getAccountId();+ String region = regionResolver.getRegion();+ boolean isDefaultScope = regionResolver.getDefaultAccountId().equals(accountId)+ && regionResolver.getDefaultRegion().equals(region);+ return vocabularies.getForAccountMigratingLegacyKeys(+ accountId,+ region + "/" + vocabularyName,+ List.of(vocabularyName),+ ignored -> true,+ isDefaultScope);+ }++ private void migrateDefaultScopeVocabularies() {+ boolean isDefaultAccount = regionResolver.getDefaultAccountId().equals(regionResolver.getAccountId());+ if (isDefaultAccount && regionResolver.getDefaultRegion().equals(regionResolver.getRegion())) {+ vocabularies.scanUnscopedLegacy(ignored -> true).stream()+ .map(VocabularyInfo::vocabularyName)+ .forEach(this::getStoredVocabulary);+ }+ vocabularies.keysForAccount(regionResolver.getAccountId()).stream()+ .filter(key -> !key.contains("/"))+ .forEach(this::getStoredVocabulary);}private void requireNonBlank(String value, String fieldName) {diff --git a/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServiceTest.javanew file mode 100644index 0000000000..ff18b26a98--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServiceTest.java@@ -0,0 +1,129 @@+package io.github.hectorvent.floci.services.transcribe;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.transcribe.model.VocabularyInfo;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++class TranscribeServiceTest {++ private static final String DEFAULT_ACCOUNT = "000000000000";+ private static final String DEFAULT_REGION = "us-east-1";++ private final AtomicReference<String> account = new AtomicReference<>(DEFAULT_ACCOUNT);+ private final AtomicReference<String> region = new AtomicReference<>(DEFAULT_REGION);+ private AccountAwareStorageBackend<VocabularyInfo> vocabularyStore;+ private TranscribeService service;++ @BeforeEach+ void setUp() {+ RegionResolver resolver = mock(RegionResolver.class);+ when(resolver.getAccountId()).thenAnswer(ignored -> account.get());+ when(resolver.getRegion()).thenAnswer(ignored -> region.get());+ when(resolver.getDefaultAccountId()).thenReturn(DEFAULT_ACCOUNT);+ when(resolver.getDefaultRegion()).thenReturn(DEFAULT_REGION);++ StorageFactory storageFactory = new StorageFactory(null, null) {+ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,+ com.fasterxml.jackson.core.type.TypeReference<java.util.Map<String, V>> typeReference) {+ @SuppressWarnings("unchecked")+ AccountAwareStorageBackend<V> result =+ (AccountAwareStorageBackend<V>) (AccountAwareStorageBackend<?>) TranscribeServiceTest.this.vocabularyStore;+ return result;+ }+ };+ vocabularyStore = AccountAwareStorageBackend.inMemory(DEFAULT_ACCOUNT);+ service = new TranscribeService(storageFactory, resolver);+ service.initializeStorage();+ }++ @Test+ void sameNameJobsAreIsolatedByAccountAndRegion() {+ service.startTranscriptionJob("shared-job", "s3://bucket/one.wav", "en-US", "wav");++ account.set("111111111111");+ region.set("eu-west-1");+ service.startTranscriptionJob("shared-job", "s3://bucket/two.wav", "en-US", "wav");++ assertEquals("s3://bucket/two.wav",+ service.getTranscriptionJob("shared-job").media().mediaFileUri());+ assertEquals(1, service.listTranscriptionJobs(null, null, null).summaries().size());++ account.set(DEFAULT_ACCOUNT);+ region.set(DEFAULT_REGION);+ assertEquals("s3://bucket/one.wav",+ service.getTranscriptionJob("shared-job").media().mediaFileUri());+ }++ @Test+ void sameNameVocabulariesAreIsolatedByAccountAndRegion() {+ service.createVocabulary("shared-vocabulary", "en-US");++ account.set("111111111111");+ region.set("eu-west-1");+ service.createVocabulary("shared-vocabulary", "de-DE");++ assertEquals("de-DE", service.getVocabulary("shared-vocabulary").languageCode());+ assertEquals(1, service.listVocabularies(null, null, null).vocabularies().size());++ account.set(DEFAULT_ACCOUNT);+ region.set(DEFAULT_REGION);+ assertEquals("en-US", service.getVocabulary("shared-vocabulary").languageCode());+ }++ @Test+ void legacyVocabularyIsVisibleOnlyInDefaultAccountAndRegion() {+ vocabularyStore.put("legacy-vocabulary",+ new VocabularyInfo("legacy-vocabulary", "en-US", "READY", 1L));++ assertEquals(1, service.listVocabularies(null, null, null).vocabularies().size());+ assertEquals("en-US", service.getVocabulary("legacy-vocabulary").languageCode());++ account.set("111111111111");+ region.set("eu-west-1");+ assertThrows(AwsException.class, () -> service.getVocabulary("legacy-vocabulary"));+ assertTrue(service.listVocabularies(null, null, null).vocabularies().isEmpty());++ account.set(DEFAULT_ACCOUNT);+ region.set("eu-west-1");+ assertThrows(AwsException.class, () -> service.getVocabulary("legacy-vocabulary"));+ }++ @Test+ void accountPrefixedLegacyVocabularyMigratesForNonDefaultAccount() {+ vocabularyStore.putForAccount("111111111111", "legacy-vocabulary",+ new VocabularyInfo("legacy-vocabulary", "en-US", "READY", 1L));+ account.set("111111111111");+ region.set("eu-west-1");++ assertEquals("en-US", service.getVocabulary("legacy-vocabulary").languageCode());+ assertEquals(1, service.listVocabularies(null, null, null).vocabularies().size());+ assertTrue(vocabularyStore.getForAccount("111111111111", "legacy-vocabulary").isEmpty());+ assertTrue(vocabularyStore.getForAccount("111111111111", "eu-west-1/legacy-vocabulary").isPresent());++ service.deleteVocabulary("legacy-vocabulary");+ assertThrows(AwsException.class, () -> service.getVocabulary("legacy-vocabulary"));+ }++ @Test+ void nullStorageFactoryUsesAnInMemoryVocabularyStore() {+ TranscribeService serviceWithoutFactory = new TranscribeService(null,+ new RegionResolver(DEFAULT_REGION, DEFAULT_ACCOUNT));+ serviceWithoutFactory.initializeStorage();++ assertEquals("READY", serviceWithoutFactory.createVocabulary("memory-vocabulary", "en-US")+ .vocabularyState());+ }+}