Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(transcribe): persist vocabularies across restart (#1608)

Hector Ventura239bc52e0625
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 584f51ac5e..f4ea15a04b 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -193,6 +193,7 @@ public interface EmulatorConfig {
CodeBuildStorageConfig codebuild();
ConfigStorageConfig config();
CodeDeployStorageConfig codedeploy();
+ TranscribeStorageConfig transcribe();
}
interface SsmStorageConfig {
@@ -366,6 +367,13 @@ public interface EmulatorConfig {
long flushIntervalMs();
}
+ interface TranscribeStorageConfig {
+ Optional<String> mode();
+
+ @WithDefault("5000")
+ long flushIntervalMs();
+ }
+
interface CodeDeployStorageConfig {
Optional<String> mode();
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
index bc35218c7f..81298d0812 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java
@@ -320,7 +320,8 @@ public class ResolvedServiceCatalog {
protocols(ServiceProtocol.JSON),
Set.of("AWSPriceListService."), Set.of("pricing", "api.pricing"), Set.of(), Set.of()),
descriptor("transcribe", "transcribe", config.services().transcribe().enabled(), true,
- null, null, 5000L, null, ServiceProtocol.JSON,
+ "transcribe", storageMode(config.storage().services().transcribe().mode(), config.storage().mode()),
+ config.storage().services().transcribe().flushIntervalMs(), null, ServiceProtocol.JSON,
protocols(ServiceProtocol.JSON),
Set.of("Transcribe."), Set.of("transcribe"), Set.of(), Set.of()),
descriptor("ce", "ce", config.services().ce().enabled(), true,
diff --git a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java
index 75be594a59..fe9793fcc5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/transcribe/TranscribeService.java
@@ -1,14 +1,20 @@
package io.github.hectorvent.floci.services.transcribe;
+import com.fasterxml.jackson.core.type.TypeReference;
import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.StorageBackedMap;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.transcribe.model.TranscriptionJob;
import io.github.hectorvent.floci.services.transcribe.model.TranscriptionJobSummary;
import io.github.hectorvent.floci.services.transcribe.model.VocabularyInfo;
+import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
import java.time.Instant;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
@@ -20,8 +26,25 @@ import java.util.concurrent.ConcurrentHashMap;
@ApplicationScoped
public class TranscribeService {
+ private final StorageFactory storageFactory;
+
+ // Transcription jobs are transient (async work deleted after processing); vocabularies are durable.
private final ConcurrentHashMap<String, TranscriptionJob> transcriptionJobs = new ConcurrentHashMap<>();
- private final ConcurrentHashMap<String, VocabularyInfo> vocabularies = new ConcurrentHashMap<>();
+ private Map<String, VocabularyInfo> vocabularies = new ConcurrentHashMap<>();
+
+ @Inject
+ public TranscribeService(StorageFactory storageFactory) {
+ this.storageFactory = storageFactory;
+ }
+
+ @PostConstruct
+ void initializeStorage() {
+ if (storageFactory == null) {
+ return; // keeps non-CDI unit tests working
+ }
+ this.vocabularies = new StorageBackedMap<>(storageFactory.create("transcribe",
+ "transcribe-vocabularies.json", new TypeReference<Map<String, VocabularyInfo>>() {}));
+ }
public TranscriptionJob startTranscriptionJob(String jobName, String mediaFileUri,
String languageCode, String mediaFormat) {
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 087c80bf14..dc42dca5fd 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -105,6 +105,8 @@ floci:
flush-interval-ms: 5000
config:
flush-interval-ms: 5000
+ transcribe:
+ flush-interval-ms: 5000
codedeploy:
flush-interval-ms: 5000
rds:
diff --git a/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServicePersistenceTest.java
new file mode 100644
index 0000000000..cd27ac6836
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/transcribe/TranscribeServicePersistenceTest.java
@@ -0,0 +1,74 @@
+package io.github.hectorvent.floci.services.transcribe;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.transcribe.model.VocabularyInfo;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Verifies Transcribe vocabularies survive a restart. Two service instances share the same
+ * {@link StorageFactory} backend; the second simulates a process restart reloading from disk.
+ */
+class TranscribeServicePersistenceTest {
+
+ @Test
+ void vocabulariesSurviveRestart() {
+ SharedStorageFactory storage = new SharedStorageFactory();
+
+ TranscribeService first = serviceWithStorage(storage);
+ first.createVocabulary("medical-terms", "en-US");
+
+ TranscribeService reloaded = serviceWithStorage(storage);
+ VocabularyInfo vocab = reloaded.getVocabulary("medical-terms");
+ assertEquals("en-US", vocab.languageCode());
+ assertEquals("READY", vocab.vocabularyState());
+ }
+
+ @Test
+ void deletedVocabularyDoesNotReappearAfterRestart() {
+ SharedStorageFactory storage = new SharedStorageFactory();
+
+ TranscribeService first = serviceWithStorage(storage);
+ first.createVocabulary("keep", "en-US");
+ first.createVocabulary("drop", "es-ES");
+ first.deleteVocabulary("drop");
+
+ TranscribeService reloaded = serviceWithStorage(storage);
+ assertEquals("en-US", reloaded.getVocabulary("keep").languageCode());
+ assertThrows(AwsException.class, () -> reloaded.getVocabulary("drop"));
+ assertTrue(reloaded.listVocabularies(null, null, null).vocabularies().stream()
+ .noneMatch(v -> v.vocabularyName().equals("drop")));
+ }
+
+ private static TranscribeService serviceWithStorage(StorageFactory storage) {
+ TranscribeService service = new TranscribeService(storage);
+ service.initializeStorage();
+ return service;
+ }
+
+ private static final class SharedStorageFactory extends StorageFactory {
+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();
+
+ private SharedStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public <V> StorageBackend<String, V> create(String serviceName,
+ String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return (StorageBackend<String, V>) stores.computeIfAbsent(fileName, ignored -> new InMemoryStorage<>());
+ }
+ }
+}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 39af0794aa..d52b7ef23b 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -48,6 +48,8 @@ floci:
flush-interval-ms: 60000
config:
flush-interval-ms: 60000
+ transcribe:
+ flush-interval-ms: 60000
codedeploy:
flush-interval-ms: 60000
cloudfront: