Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(transcribe): isolate jobs and vocabularies by account and region (#3402)

Niels Kootstrac4ff42b80065
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java
index f36b181040..a13b8c1546 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/AwsJson11Controller.java
@@ -308,7 +308,7 @@ public class AwsJson11Controller {
case "comprehend" -> comprehendJsonHandler.handle(action, request, region);
case "rekognition" -> rekognitionJsonHandler.handle(action, request, region);
case "pricing" -> pricingJsonHandler.handle(action, request, region);
- case "transcribe" -> transcribeJsonHandler.handle(action, request, region);
+ case "transcribe" -> transcribeJsonHandler.handle(action, request);
case "translate" -> translateJsonHandler.handle(action, request, region);
case "ce" -> costExplorerJsonHandler.handle(action, request, region);
case "cur" -> curJsonHandler.handle(action, request, region);
diff --git a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.java
index 97d033c3c1..2d3983255a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeJsonHandler.java
@@ -33,7 +33,7 @@ public class TranscribeJsonHandler {
this.objectMapper = objectMapper;
}
- public Response handle(String action, JsonNode request, String region) {
+ public Response handle(String action, JsonNode request) {
LOG.debugv("Transcribe action: {0}", action);
return switch (action) {
case "StartTranscriptionJob" -> {
diff --git a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java
index 6269b09988..7c19cb2993 100644
--- a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java
@@ -2,7 +2,8 @@ package io.github.hectorvent.floci.services.transcribe;
import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.core.common.AwsException;
-import io.github.hectorvent.floci.core.storage.StorageBackedMap;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.transcribe.model.TranscriptionJob;
import io.github.hectorvent.floci.services.transcribe.model.TranscriptionJobSummary;
@@ -16,6 +17,7 @@ import java.time.Instant;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -28,35 +30,45 @@ import java.util.concurrent.ConcurrentHashMap;
public class TranscribeService implements Resettable {
private final StorageFactory storageFactory;
+ private final RegionResolver regionResolver;
// Transcription jobs are transient (async work deleted after processing); vocabularies are durable.
private final ConcurrentHashMap<String, TranscriptionJob> transcriptionJobs = new ConcurrentHashMap<>();
- private Map<String, VocabularyInfo> vocabularies = new ConcurrentHashMap<>();
+ private AccountAwareStorageBackend<VocabularyInfo> vocabularies;
@Inject
- public TranscribeService(StorageFactory storageFactory) {
+ public TranscribeService(StorageFactory storageFactory, RegionResolver regionResolver) {
this.storageFactory = storageFactory;
+ this.regionResolver = regionResolver;
+ }
+
+ TranscribeService(StorageFactory storageFactory) {
+ this(storageFactory, new RegionResolver("us-east-1", "000000000000"));
}
@PostConstruct
void initializeStorage() {
if (storageFactory == null) {
- return; // keeps non-CDI unit tests working
+ vocabularies = AccountAwareStorageBackend.inMemory(regionResolver.getDefaultAccountId());
+ return;
}
- this.vocabularies = new StorageBackedMap<>(storageFactory.create("transcribe",
- "transcribe-vocabularies.json", new TypeReference<Map<String, VocabularyInfo>>() {}));
+ this.vocabularies = storageFactory.create("transcribe",
+ "transcribe-vocabularies.json", new TypeReference<Map<String, VocabularyInfo>>() {});
}
public void clear() {
transcriptionJobs.clear();
- vocabularies.clear();
+ if (vocabularies != null) {
+ vocabularies.clear();
+ }
}
public TranscriptionJob startTranscriptionJob(String jobName, String mediaFileUri,
String languageCode, String mediaFormat) {
requireNonBlank(jobName, "TranscriptionJobName");
requireNonBlank(mediaFileUri, "Media.MediaFileUri");
- if (transcriptionJobs.containsKey(jobName)) {
+ String jobKey = jobKey(jobName);
+ if (transcriptionJobs.containsKey(jobKey)) {
throw new AwsException("ConflictException",
"The requested job name already exists. Use a different job name.", 400);
}
@@ -72,13 +84,13 @@ public class TranscribeService implements Resettable {
new TranscriptionJob.Transcript("s3://floci-transcribe-output/" + jobName + ".json"),
now, now, now);
- transcriptionJobs.put(jobName, job);
+ transcriptionJobs.put(jobKey, job);
return job;
}
public TranscriptionJob getTranscriptionJob(String jobName) {
requireNonBlank(jobName, "TranscriptionJobName");
- TranscriptionJob job = transcriptionJobs.get(jobName);
+ TranscriptionJob job = transcriptionJobs.get(jobKey(jobName));
if (job == null) {
throw new AwsException("NotFoundException",
"The requested job couldn't be found. Check the job name and try your request again.", 400);
@@ -90,7 +102,10 @@ public class TranscribeService implements Resettable {
Integer maxResults) {
int limit = maxResults != null ? Math.min(maxResults, 100) : 100;
- List<TranscriptionJobSummary> filtered = transcriptionJobs.values().stream()
+ String currentPrefix = currentJobPrefix();
+ List<TranscriptionJobSummary> filtered = transcriptionJobs.entrySet().stream()
+ .filter(entry -> entry.getKey().startsWith(currentPrefix))
+ .map(Map.Entry::getValue)
.filter(j -> statusFilter == null || statusFilter.equals(j.transcriptionJobStatus()))
.filter(j -> jobNameContains == null || j.transcriptionJobName().contains(jobNameContains))
.sorted(Comparator.comparing(TranscriptionJob::transcriptionJobName))
@@ -106,7 +121,7 @@ public class TranscribeService implements Resettable {
public void deleteTranscriptionJob(String jobName) {
requireNonBlank(jobName, "TranscriptionJobName");
- if (transcriptionJobs.remove(jobName) == null) {
+ if (transcriptionJobs.remove(jobKey(jobName)) == null) {
throw new AwsException("BadRequestException",
"The requested job couldn't be found. Check the job name and try your request again.", 400);
}
@@ -115,20 +130,20 @@ public class TranscribeService implements Resettable {
public VocabularyInfo createVocabulary(String vocabularyName, String languageCode) {
requireNonBlank(vocabularyName, "VocabularyName");
requireNonBlank(languageCode, "LanguageCode");
- if (vocabularies.containsKey(vocabularyName)) {
+ if (getStoredVocabulary(vocabularyName).isPresent()) {
throw new AwsException("ConflictException",
"The requested vocabulary name already exists. Use a different vocabulary name.", 400);
}
VocabularyInfo vocab = new VocabularyInfo(
vocabularyName, languageCode, "READY", Instant.now().getEpochSecond());
- vocabularies.put(vocabularyName, vocab);
+ vocabularies.putForAccount(regionResolver.getAccountId(), vocabularyKey(vocabularyName), vocab);
return vocab;
}
public VocabularyInfo getVocabulary(String vocabularyName) {
requireNonBlank(vocabularyName, "VocabularyName");
- VocabularyInfo vocab = vocabularies.get(vocabularyName);
+ VocabularyInfo vocab = getStoredVocabulary(vocabularyName).orElse(null);
if (vocab == null) {
throw new AwsException("NotFoundException",
"The requested vocabulary couldn't be found. Check the vocabulary name and try your request again.",
@@ -141,7 +156,12 @@ public class TranscribeService implements Resettable {
Integer maxResults) {
int limit = maxResults != null ? Math.min(maxResults, 100) : 100;
- List<VocabularyInfo> filtered = vocabularies.values().stream()
+ migrateDefaultScopeVocabularies();
+
+ String currentPrefix = currentVocabularyPrefix();
+ List<VocabularyInfo> filtered = vocabularies.scanForAccount(regionResolver.getAccountId(),
+ key -> key.startsWith(currentPrefix))
+ .stream()
.filter(v -> stateEquals == null || stateEquals.equals(v.vocabularyState()))
.filter(v -> nameContains == null || v.vocabularyName().contains(nameContains))
.sorted(Comparator.comparing(VocabularyInfo::vocabularyName))
@@ -156,11 +176,53 @@ public class TranscribeService implements Resettable {
public void deleteVocabulary(String vocabularyName) {
requireNonBlank(vocabularyName, "VocabularyName");
- if (vocabularies.remove(vocabularyName) == null) {
+ if (getStoredVocabulary(vocabularyName).isEmpty()) {
throw new AwsException("NotFoundException",
"The requested vocabulary couldn't be found. Check the vocabulary name and try your request again.",
400);
}
+ vocabularies.deleteForAccount(regionResolver.getAccountId(), vocabularyKey(vocabularyName));
+ }
+
+ private String jobKey(String jobName) {
+ return currentJobPrefix() + jobName;
+ }
+
+ private String vocabularyKey(String vocabularyName) {
+ return regionResolver.getRegion() + "/" + vocabularyName;
+ }
+
+ private String currentJobPrefix() {
+ return regionResolver.getAccountId() + "/" + regionResolver.getRegion() + "/";
+ }
+
+ private String currentVocabularyPrefix() {
+ return regionResolver.getRegion() + "/";
+ }
+
+ private Optional<VocabularyInfo> getStoredVocabulary(String vocabularyName) {
+ String accountId = regionResolver.getAccountId();
+ String region = regionResolver.getRegion();
+ boolean isDefaultScope = regionResolver.getDefaultAccountId().equals(accountId)
+ && regionResolver.getDefaultRegion().equals(region);
+ return vocabularies.getForAccountMigratingLegacyKeys(
+ accountId,
+ region + "/" + vocabularyName,
+ List.of(vocabularyName),
+ ignored -> true,
+ isDefaultScope);
+ }
+
+ private void migrateDefaultScopeVocabularies() {
+ boolean isDefaultAccount = regionResolver.getDefaultAccountId().equals(regionResolver.getAccountId());
+ if (isDefaultAccount && regionResolver.getDefaultRegion().equals(regionResolver.getRegion())) {
+ vocabularies.scanUnscopedLegacy(ignored -> true).stream()
+ .map(VocabularyInfo::vocabularyName)
+ .forEach(this::getStoredVocabulary);
+ }
+ vocabularies.keysForAccount(regionResolver.getAccountId()).stream()
+ .filter(key -> !key.contains("/"))
+ .forEach(this::getStoredVocabulary);
}
private void requireNonBlank(String value, String fieldName) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServiceTest.java
new file mode 100644
index 0000000000..ff18b26a98
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServiceTest.java
@@ -0,0 +1,129 @@
+package io.github.hectorvent.floci.services.transcribe;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.transcribe.model.VocabularyInfo;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class TranscribeServiceTest {
+
+ private static final String DEFAULT_ACCOUNT = "000000000000";
+ private static final String DEFAULT_REGION = "us-east-1";
+
+ private final AtomicReference<String> account = new AtomicReference<>(DEFAULT_ACCOUNT);
+ private final AtomicReference<String> region = new AtomicReference<>(DEFAULT_REGION);
+ private AccountAwareStorageBackend<VocabularyInfo> vocabularyStore;
+ private TranscribeService service;
+
+ @BeforeEach
+ void setUp() {
+ RegionResolver resolver = mock(RegionResolver.class);
+ when(resolver.getAccountId()).thenAnswer(ignored -> account.get());
+ when(resolver.getRegion()).thenAnswer(ignored -> region.get());
+ when(resolver.getDefaultAccountId()).thenReturn(DEFAULT_ACCOUNT);
+ when(resolver.getDefaultRegion()).thenReturn(DEFAULT_REGION);
+
+ StorageFactory storageFactory = new StorageFactory(null, null) {
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ com.fasterxml.jackson.core.type.TypeReference<java.util.Map<String, V>> typeReference) {
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<V> result =
+ (AccountAwareStorageBackend<V>) (AccountAwareStorageBackend<?>) TranscribeServiceTest.this.vocabularyStore;
+ return result;
+ }
+ };
+ vocabularyStore = AccountAwareStorageBackend.inMemory(DEFAULT_ACCOUNT);
+ service = new TranscribeService(storageFactory, resolver);
+ service.initializeStorage();
+ }
+
+ @Test
+ void sameNameJobsAreIsolatedByAccountAndRegion() {
+ service.startTranscriptionJob("shared-job", "s3://bucket/one.wav", "en-US", "wav");
+
+ account.set("111111111111");
+ region.set("eu-west-1");
+ service.startTranscriptionJob("shared-job", "s3://bucket/two.wav", "en-US", "wav");
+
+ assertEquals("s3://bucket/two.wav",
+ service.getTranscriptionJob("shared-job").media().mediaFileUri());
+ assertEquals(1, service.listTranscriptionJobs(null, null, null).summaries().size());
+
+ account.set(DEFAULT_ACCOUNT);
+ region.set(DEFAULT_REGION);
+ assertEquals("s3://bucket/one.wav",
+ service.getTranscriptionJob("shared-job").media().mediaFileUri());
+ }
+
+ @Test
+ void sameNameVocabulariesAreIsolatedByAccountAndRegion() {
+ service.createVocabulary("shared-vocabulary", "en-US");
+
+ account.set("111111111111");
+ region.set("eu-west-1");
+ service.createVocabulary("shared-vocabulary", "de-DE");
+
+ assertEquals("de-DE", service.getVocabulary("shared-vocabulary").languageCode());
+ assertEquals(1, service.listVocabularies(null, null, null).vocabularies().size());
+
+ account.set(DEFAULT_ACCOUNT);
+ region.set(DEFAULT_REGION);
+ assertEquals("en-US", service.getVocabulary("shared-vocabulary").languageCode());
+ }
+
+ @Test
+ void legacyVocabularyIsVisibleOnlyInDefaultAccountAndRegion() {
+ vocabularyStore.put("legacy-vocabulary",
+ new VocabularyInfo("legacy-vocabulary", "en-US", "READY", 1L));
+
+ assertEquals(1, service.listVocabularies(null, null, null).vocabularies().size());
+ assertEquals("en-US", service.getVocabulary("legacy-vocabulary").languageCode());
+
+ account.set("111111111111");
+ region.set("eu-west-1");
+ assertThrows(AwsException.class, () -> service.getVocabulary("legacy-vocabulary"));
+ assertTrue(service.listVocabularies(null, null, null).vocabularies().isEmpty());
+
+ account.set(DEFAULT_ACCOUNT);
+ region.set("eu-west-1");
+ assertThrows(AwsException.class, () -> service.getVocabulary("legacy-vocabulary"));
+ }
+
+ @Test
+ void accountPrefixedLegacyVocabularyMigratesForNonDefaultAccount() {
+ vocabularyStore.putForAccount("111111111111", "legacy-vocabulary",
+ new VocabularyInfo("legacy-vocabulary", "en-US", "READY", 1L));
+ account.set("111111111111");
+ region.set("eu-west-1");
+
+ assertEquals("en-US", service.getVocabulary("legacy-vocabulary").languageCode());
+ assertEquals(1, service.listVocabularies(null, null, null).vocabularies().size());
+ assertTrue(vocabularyStore.getForAccount("111111111111", "legacy-vocabulary").isEmpty());
+ assertTrue(vocabularyStore.getForAccount("111111111111", "eu-west-1/legacy-vocabulary").isPresent());
+
+ service.deleteVocabulary("legacy-vocabulary");
+ assertThrows(AwsException.class, () -> service.getVocabulary("legacy-vocabulary"));
+ }
+
+ @Test
+ void nullStorageFactoryUsesAnInMemoryVocabularyStore() {
+ TranscribeService serviceWithoutFactory = new TranscribeService(null,
+ new RegionResolver(DEFAULT_REGION, DEFAULT_ACCOUNT));
+ serviceWithoutFactory.initializeStorage();
+
+ assertEquals("READY", serviceWithoutFactory.createVocabulary("memory-vocabulary", "en-US")
+ .vocabularyState());
+ }
+}