Light, fluffy, and always free - The AWS Local Emulator alternative
fix(config): persist config rules, conformance packs, recorders, channels, tags (#1516)
Hector Ventura2b26149ef5cd
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 5dbea683c6..125d65ed9c 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -179,6 +179,7 @@ public interface EmulatorConfig {S3VectorsStorageConfig s3vectors();EcsStorageConfig ecs();CodeBuildStorageConfig codebuild();+ ConfigStorageConfig config();}interface SsmStorageConfig {@@ -345,6 +346,13 @@ public interface EmulatorConfig {long flushIntervalMs();}+ interface ConfigStorageConfig {+ Optional<String> mode();++ @WithDefault("5000")+ long flushIntervalMs();+ }+interface WalConfig {@WithDefault("30000")long compactionIntervalMs();diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.javaindex 2d1246738c..66e6c201bc 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java@@ -272,7 +272,8 @@ public class ResolvedServiceCatalog {protocols(ServiceProtocol.JSON),Set.of("CodePipeline_20150709."), Set.of("codepipeline"), Set.of(), Set.of()),descriptor("config", "configservice", config.services().configservice().enabled(), true,- null, null, 5000L, null, ServiceProtocol.JSON,+ "config", storageMode(config.storage().services().config().mode(), config.storage().mode()),+ config.storage().services().config().flushIntervalMs(), null, ServiceProtocol.JSON,protocols(ServiceProtocol.JSON),Set.of("StarlingDoveService."), Set.of("config"), Set.of(), Set.of()),descriptor("cloudtrail", "cloudtrail", config.services().cloudtrail().enabled(), true,diff --git a/src/main/java/io/github/hectorvent/floci/services/configservice/AwsConfigService.java b/src/main/java/io/github/hectorvent/floci/services/configservice/AwsConfigService.javaindex 8bbd968937..109e5c2379 100644--- a/src/main/java/io/github/hectorvent/floci/services/configservice/AwsConfigService.java+++ b/src/main/java/io/github/hectorvent/floci/services/configservice/AwsConfigService.java@@ -1,8 +1,11 @@package io.github.hectorvent.floci.services.configservice;+import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.StorageBackedMap;+import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.configservice.model.ConfigRule;import io.github.hectorvent.floci.services.configservice.model.ConfigRuleEvaluationStatus;import io.github.hectorvent.floci.services.configservice.model.ConfigRuleSource;@@ -11,6 +14,7 @@ import io.github.hectorvent.floci.services.configservice.model.ConfigurationRecoimport io.github.hectorvent.floci.services.configservice.model.ConformancePack;import io.github.hectorvent.floci.services.configservice.model.ConformancePackStatusDetail;import io.github.hectorvent.floci.services.configservice.model.DeliveryChannel;+import jakarta.annotation.PostConstruct;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -26,33 +30,84 @@ import java.util.stream.Collectors;public class AwsConfigService {private final RegionResolver regionResolver;+ private final StorageFactory storageFactory;- private final ConcurrentHashMap<String, ConcurrentHashMap<String, ConfigRule>> configRules = new ConcurrentHashMap<>();- private final ConcurrentHashMap<String, ConcurrentHashMap<String, ConformancePack>> conformancePacks = new ConcurrentHashMap<>();+ // region -> ruleName -> rule (nested)+ private Map<String, Map<String, ConfigRule>> configRules = new ConcurrentHashMap<>();+ // region -> packName -> pack (nested)+ private Map<String, Map<String, ConformancePack>> conformancePacks = new ConcurrentHashMap<>();- private final ConcurrentHashMap<String, ConfigurationRecorder> configurationRecorders = new ConcurrentHashMap<>();+ // region -> recorder / channel (flat)+ private Map<String, ConfigurationRecorder> configurationRecorders = new ConcurrentHashMap<>();+ private Map<String, DeliveryChannel> deliveryChannels = new ConcurrentHashMap<>();++ // recorder run-state is transient runtime state (not persisted)private final ConcurrentHashMap<String, Boolean> recorderRunning = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, Long> recorderLastStartTime = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, Long> recorderLastStopTime = new ConcurrentHashMap<>();- private final ConcurrentHashMap<String, DeliveryChannel> deliveryChannels = new ConcurrentHashMap<>();-- private final ConcurrentHashMap<String, Map<String, String>> tags = new ConcurrentHashMap<>();+ // resourceArn -> {tagKey -> tagValue} (flat outer, mutable inner)+ private Map<String, Map<String, String>> tags = new ConcurrentHashMap<>();@Inject- public AwsConfigService(RegionResolver regionResolver) {+ public AwsConfigService(RegionResolver regionResolver, StorageFactory storageFactory) {this.regionResolver = regionResolver;+ this.storageFactory = storageFactory;+ }++ @PostConstruct+ void initializeStorage() {+ if (storageFactory == null) {+ return; // keeps non-CDI unit tests working+ }+ this.configRules = storageBacked("config-rules.json",+ new TypeReference<Map<String, Map<String, ConfigRule>>>() {});+ this.conformancePacks = storageBacked("config-conformance-packs.json",+ new TypeReference<Map<String, Map<String, ConformancePack>>>() {});+ this.configurationRecorders = storageBacked("config-recorders.json",+ new TypeReference<Map<String, ConfigurationRecorder>>() {});+ this.deliveryChannels = storageBacked("config-delivery-channels.json",+ new TypeReference<Map<String, DeliveryChannel>>() {});+ this.tags = storageBacked("config-tags.json",+ new TypeReference<Map<String, Map<String, String>>>() {});+ normalizeRegionMaps(configRules);+ normalizeRegionMaps(conformancePacks);+ normalizeRegionMaps(tags);+ }++ private <V> Map<String, V> storageBacked(String fileName, TypeReference<Map<String, V>> typeReference) {+ return new StorageBackedMap<>(storageFactory.create("config", fileName, typeReference));+ }++ /** After load, re-wrap persisted inner maps as {@link ConcurrentHashMap} (Jackson deserializes+ * them as plain maps) so per-key mutation stays thread-safe. */+ private <V> void normalizeRegionMaps(Map<String, Map<String, V>> resources) {+ for (Map.Entry<String, Map<String, V>> entry : new ArrayList<>(resources.entrySet())) {+ if (!(entry.getValue() instanceof ConcurrentHashMap)) {+ resources.put(entry.getKey(), new ConcurrentHashMap<>(entry.getValue()));+ }+ }+ }++ /** {@link StorageBackedMap} only flushes on a top-level put, so an in-place mutation of an+ * inner map must be written back by re-putting the outer entry. */+ private <V> void persistRegion(Map<String, Map<String, V>> resources, String region) {+ Map<String, V> regionResources = resources.get(region);+ if (regionResources != null) {+ resources.put(region, regionResources);+ }}// --- Config Rules ---public ConfigRule putConfigRule(String region, String ruleName, ConfigRuleSource source) {- ConcurrentHashMap<String, ConfigRule> store = rulesFor(region);+ Map<String, ConfigRule> store = rulesFor(region);ConfigRule existing = store.get(ruleName);if (existing != null) {ConfigRule updated = new ConfigRule(existing.configRuleName(), existing.configRuleArn(),existing.configRuleId(), existing.configRuleState(), source);store.put(ruleName, updated);+ persistRegion(configRules, region);return updated;}String ruleId = "config-rule-" + shortId();@@ -60,20 +115,22 @@ public class AwsConfigService {"config-rule/" + ruleId).toString();ConfigRule rule = new ConfigRule(ruleName, ruleArn, ruleId, "ACTIVE", source);store.put(ruleName, rule);+ persistRegion(configRules, region);return rule;}public void deleteConfigRule(String region, String ruleName) {- ConcurrentHashMap<String, ConfigRule> store = rulesFor(region);+ Map<String, ConfigRule> store = rulesFor(region);if (store.remove(ruleName) == null) {throw new AwsException("NoSuchConfigRuleException","The ConfigRule provided in the request is invalid. " +"Please check the configRule name.", 400);}+ persistRegion(configRules, region);}public List<ConfigRule> describeConfigRules(String region, List<String> ruleNames) {- ConcurrentHashMap<String, ConfigRule> store = rulesFor(region);+ Map<String, ConfigRule> store = rulesFor(region);if (ruleNames == null || ruleNames.isEmpty()) {return new ArrayList<>(store.values());}@@ -101,7 +158,7 @@ public class AwsConfigService {}public void startConfigRulesEvaluation(String region, List<String> ruleNames) {- ConcurrentHashMap<String, ConfigRule> store = rulesFor(region);+ Map<String, ConfigRule> store = rulesFor(region);for (String name : ruleNames) {if (!store.containsKey(name)) {throw new AwsException("NoSuchConfigRuleException",@@ -222,12 +279,13 @@ public class AwsConfigService {public ConformancePack putConformancePack(String region, String packName,String templateS3Uri, String templateBody) {- ConcurrentHashMap<String, ConformancePack> store = packsFor(region);+ Map<String, ConformancePack> store = packsFor(region);ConformancePack existing = store.get(packName);if (existing != null) {ConformancePack updated = new ConformancePack(existing.conformancePackName(), existing.conformancePackArn(),existing.conformancePackId(), templateS3Uri, templateBody);store.put(packName, updated);+ persistRegion(conformancePacks, region);return updated;}String packId = "conformance-pack-" + shortId();@@ -235,19 +293,21 @@ public class AwsConfigService {"conformance-pack/" + packName + "/" + packId).toString();ConformancePack pack = new ConformancePack(packName, packArn, packId, templateS3Uri, templateBody);store.put(packName, pack);+ persistRegion(conformancePacks, region);return pack;}public void deleteConformancePack(String region, String packName) {- ConcurrentHashMap<String, ConformancePack> store = packsFor(region);+ Map<String, ConformancePack> store = packsFor(region);if (store.remove(packName) == null) {throw new AwsException("NoSuchConformancePackException","Conformance pack '" + packName + "' does not exist.", 400);}+ persistRegion(conformancePacks, region);}public List<ConformancePack> describeConformancePacks(String region, List<String> names) {- ConcurrentHashMap<String, ConformancePack> store = packsFor(region);+ Map<String, ConformancePack> store = packsFor(region);if (names == null || names.isEmpty()) {return new ArrayList<>(store.values());}@@ -284,12 +344,14 @@ public class AwsConfigService {for (Map<String, String> t : tagList) {tagMap.put(t.get("Key"), t.get("Value"));}+ tags.put(arn, tagMap); // write back the in-place inner-map mutation}public void untagResource(String arn, List<String> tagKeys) {Map<String, String> tagMap = tags.get(arn);if (tagMap != null) {tagKeys.forEach(tagMap::remove);+ tags.put(arn, tagMap); // write back the in-place inner-map mutation}}@@ -302,11 +364,11 @@ public class AwsConfigService {// --- Helpers ---- private ConcurrentHashMap<String, ConfigRule> rulesFor(String region) {+ private Map<String, ConfigRule> rulesFor(String region) {return configRules.computeIfAbsent(region, r -> new ConcurrentHashMap<>());}- private ConcurrentHashMap<String, ConformancePack> packsFor(String region) {+ private Map<String, ConformancePack> packsFor(String region) {return conformancePacks.computeIfAbsent(region, r -> new ConcurrentHashMap<>());}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 2333f74750..e572a3e48d 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -102,6 +102,8 @@ floci:flush-interval-ms: 5000codebuild:flush-interval-ms: 5000+ config:+ flush-interval-ms: 5000rds:# Override storage mode for RDS (memory = no Docker volumes; hybrid/persistent = named volume per instance)# mode: memorydiff --git a/src/test/java/io/github/hectorvent/floci/services/configservice/AwsConfigServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/configservice/AwsConfigServicePersistenceTest.javanew file mode 100644index 0000000000..b235f3da84--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/configservice/AwsConfigServicePersistenceTest.java@@ -0,0 +1,102 @@+package io.github.hectorvent.floci.services.configservice;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.configservice.model.ConfigRule;+import io.github.hectorvent.floci.services.configservice.model.ConfigRuleSource;+import io.github.hectorvent.floci.services.configservice.model.ConfigurationRecorder;+import io.github.hectorvent.floci.services.configservice.model.DeliveryChannel;+import io.github.hectorvent.floci.services.configservice.model.RecordingGroup;+import org.junit.jupiter.api.Test;++import java.util.HashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Verifies AWS Config durable resources survive a restart. Two service instances share the same+ * {@link StorageFactory} backends; the second simulates a process restart reloading from disk.+ */+class AwsConfigServicePersistenceTest {++ private static final String REGION = "us-east-1";++ @Test+ void durableResourcesAndTagsSurviveRestart() {+ SharedStorageFactory storage = new SharedStorageFactory();++ AwsConfigService first = serviceWithStorage(storage);+ ConfigRule rule = first.putConfigRule(REGION, "s3-public-read",+ new ConfigRuleSource("AWS", "S3_BUCKET_PUBLIC_READ_PROHIBITED"));+ first.putConformancePack(REGION, "ops-pack", "s3://bucket/template.yaml", null);+ first.putConfigurationRecorder(REGION, new ConfigurationRecorder("default",+ "arn:aws:iam::000000000000:role/config", new RecordingGroup(true, false, null)));+ first.putDeliveryChannel(REGION,+ new DeliveryChannel("default", "config-bucket", null, null, null, null));+ first.tagResource(rule.configRuleArn(), List.of(Map.of("Key", "env", "Value", "prod")));++ AwsConfigService reloaded = serviceWithStorage(storage);++ assertEquals(List.of("s3-public-read"),+ reloaded.describeConfigRules(REGION, null).stream().map(ConfigRule::configRuleName).toList());+ assertEquals(List.of("ops-pack"),+ reloaded.describeConformancePacks(REGION, null).stream()+ .map(p -> p.conformancePackName()).toList());+ assertEquals("default",+ reloaded.describeConfigurationRecorders(REGION, null).getFirst().name());+ assertEquals("config-bucket",+ reloaded.describeDeliveryChannels(REGION, null).getFirst().s3BucketName());+ assertEquals("prod", reloaded.listTagsForResource(rule.configRuleArn()).getFirst().get("Value"));+ }++ @Test+ void deleteAndUntagArePersistedAfterRestart() {+ SharedStorageFactory storage = new SharedStorageFactory();++ AwsConfigService first = serviceWithStorage(storage);+ ConfigRule rule = first.putConfigRule(REGION, "keep",+ new ConfigRuleSource("AWS", "REQUIRED_TAGS"));+ first.putConfigRule(REGION, "drop", new ConfigRuleSource("AWS", "REQUIRED_TAGS"));+ first.deleteConfigRule(REGION, "drop");+ first.tagResource(rule.configRuleArn(),+ List.of(Map.of("Key", "env", "Value", "prod"), Map.of("Key", "team", "Value", "sec")));+ first.untagResource(rule.configRuleArn(), List.of("team"));++ AwsConfigService reloaded = serviceWithStorage(storage);++ assertEquals(List.of("keep"),+ reloaded.describeConfigRules(REGION, null).stream().map(ConfigRule::configRuleName).toList());+ Map<String, String> tags = reloaded.listTagsForResource(rule.configRuleArn()).stream()+ .collect(java.util.stream.Collectors.toMap(t -> t.get("Key"), t -> t.get("Value")));+ assertEquals("prod", tags.get("env"));+ assertTrue(!tags.containsKey("team"), "untagged key must not reappear after restart");+ }++ private static AwsConfigService serviceWithStorage(StorageFactory storage) {+ AwsConfigService service = new AwsConfigService(new RegionResolver(REGION, "000000000000"), storage);+ service.initializeStorage();+ return service;+ }++ private static final class SharedStorageFactory extends StorageFactory {+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();++ private SharedStorageFactory() {+ super(null, null);+ }++ @Override+ @SuppressWarnings("unchecked")+ public <V> StorageBackend<String, V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return (StorageBackend<String, V>) stores.computeIfAbsent(fileName, ignored -> new InMemoryStorage<>());+ }+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 3be10f8de4..97547a3c45 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -46,6 +46,8 @@ floci:flush-interval-ms: 60000codebuild:flush-interval-ms: 60000+ config:+ flush-interval-ms: 60000cloudfront:auth: