Light, fluffy, and always free - The AWS Local Emulator alternative
fix(codebuild): persist projects, report groups and source credentials (#1515)
Hector Venturae0758cca232c
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 6013a50bb1..5dbea683c6 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -178,6 +178,7 @@ public interface EmulatorConfig {CodePipelineStorageConfig codepipeline();S3VectorsStorageConfig s3vectors();EcsStorageConfig ecs();+ CodeBuildStorageConfig codebuild();}interface SsmStorageConfig {@@ -337,6 +338,13 @@ public interface EmulatorConfig {long flushIntervalMs();}+ interface CodeBuildStorageConfig {+ Optional<String> mode();++ @WithDefault("5000")+ long flushIntervalMs();+ }+interface WalConfig {@WithDefault("30000")long compactionIntervalMs();diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.javaindex 84f7084651..2d1246738c 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ResolvedServiceCatalog.java@@ -252,7 +252,8 @@ public class ResolvedServiceCatalog {protocols(ServiceProtocol.QUERY),Set.of(), Set.of("elasticloadbalancing"), Set.of(), Set.of()),descriptor("codebuild", "codebuild", config.services().codebuild().enabled(), true,- null, null, 5000L, null, ServiceProtocol.JSON,+ "codebuild", storageMode(config.storage().services().codebuild().mode(), config.storage().mode()),+ config.storage().services().codebuild().flushIntervalMs(), null, ServiceProtocol.JSON,protocols(ServiceProtocol.JSON),Set.of("CodeBuild_20161006."), Set.of("codebuild"), Set.of(), Set.of()),descriptor("batch", "batch", config.services().batch().enabled(), true,diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.javaindex 61182b5ea2..2faeaf415c 100644--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildService.java@@ -1,8 +1,11 @@package io.github.hectorvent.floci.services.codebuild;+import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.StorageBackedMap;+import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.codebuild.model.Build;import io.github.hectorvent.floci.services.codebuild.model.BuildPhase;import io.github.hectorvent.floci.services.codebuild.model.Project;@@ -11,6 +14,7 @@ import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;import io.github.hectorvent.floci.services.codebuild.model.ProjectSource;import io.github.hectorvent.floci.services.codebuild.model.ReportGroup;import io.github.hectorvent.floci.services.codebuild.model.SourceCredential;+import jakarta.annotation.PostConstruct;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -28,23 +32,64 @@ import java.util.stream.Collectors;public class CodeBuildService {// key: region -> name -> project- private final ConcurrentHashMap<String, ConcurrentHashMap<String, Project>> projects = new ConcurrentHashMap<>();+ private Map<String, Map<String, Project>> projects = new ConcurrentHashMap<>();// key: region -> arn -> report group- private final ConcurrentHashMap<String, ConcurrentHashMap<String, ReportGroup>> reportGroups = new ConcurrentHashMap<>();+ private Map<String, Map<String, ReportGroup>> reportGroups = new ConcurrentHashMap<>();// key: region -> arn -> source credential (token is stored but never returned)- private final ConcurrentHashMap<String, ConcurrentHashMap<String, SourceCredential>> sourceCredentials = new ConcurrentHashMap<>();- // key: region -> buildId -> build+ private Map<String, Map<String, SourceCredential>> sourceCredentials = new ConcurrentHashMap<>();+ // key: region -> buildId -> build (transient: builds are runtime state)private final ConcurrentHashMap<String, ConcurrentHashMap<String, Build>> builds = new ConcurrentHashMap<>();- // key: region:projectName -> build counter+ // key: region:projectName -> build counter (transient)private final ConcurrentHashMap<String, AtomicLong> buildCounters = new ConcurrentHashMap<>();private final CodeBuildRunner runner;private final EmulatorConfig config;+ private final StorageFactory storageFactory;@Inject- public CodeBuildService(CodeBuildRunner runner, EmulatorConfig config) {+ public CodeBuildService(CodeBuildRunner runner, EmulatorConfig config, StorageFactory storageFactory) {this.runner = runner;this.config = config;+ this.storageFactory = storageFactory;+ }++ @PostConstruct+ void initializeStorage() {+ if (storageFactory == null) {+ return; // keeps non-CDI unit tests working+ }+ this.projects = storageBacked("codebuild-projects.json",+ new TypeReference<Map<String, Map<String, Project>>>() {});+ this.reportGroups = storageBacked("codebuild-report-groups.json",+ new TypeReference<Map<String, Map<String, ReportGroup>>>() {});+ this.sourceCredentials = storageBacked("codebuild-source-credentials.json",+ new TypeReference<Map<String, Map<String, SourceCredential>>>() {});+ normalizeRegionMaps(projects);+ normalizeRegionMaps(reportGroups);+ normalizeRegionMaps(sourceCredentials);+ }++ private <V> Map<String, V> storageBacked(String fileName, TypeReference<Map<String, V>> typeReference) {+ return new StorageBackedMap<>(storageFactory.create("codebuild", fileName, typeReference));+ }++ /** After load, re-wrap the persisted inner maps as {@link ConcurrentHashMap} so per-region+ * mutation stays thread-safe (Jackson deserializes them as plain LinkedHashMaps). */+ private <V> void normalizeRegionMaps(Map<String, Map<String, V>> resources) {+ for (Map.Entry<String, Map<String, V>> entry : new ArrayList<>(resources.entrySet())) {+ if (!(entry.getValue() instanceof ConcurrentHashMap)) {+ resources.put(entry.getKey(), new ConcurrentHashMap<>(entry.getValue()));+ }+ }+ }++ /** {@link StorageBackedMap} only flushes on a top-level put, so an in-place mutation of a+ * region's inner map must be written back by re-putting the region entry. */+ private <V> void persistRegion(Map<String, Map<String, V>> resources, String region) {+ Map<String, V> regionResources = resources.get(region);+ if (regionResources != null) {+ resources.put(region, regionResources);+ }}private Map<String, Project> projectsFor(String region) {@@ -121,6 +166,7 @@ public class CodeBuildService {project.setProjectVisibility("PRIVATE");store.put(name, project);+ persistRegion(projects, region);return project;}@@ -160,6 +206,7 @@ public class CodeBuildService {if (concurrentBuildLimit != null) { project.setConcurrentBuildLimit(concurrentBuildLimit); }project.setLastModified(Instant.now().toEpochMilli() / 1000.0);+ persistRegion(projects, region);return project;}@@ -168,6 +215,7 @@ public class CodeBuildService {if (store.remove(name) == null) {throw new AwsException("ResourceNotFoundException", "Project not found: " + name, 400);}+ persistRegion(projects, region);}public List<Project> batchGetProjects(String region, List<String> names) {@@ -213,6 +261,7 @@ public class CodeBuildService {rg.setStatus("ACTIVE");store.put(arn, rg);+ persistRegion(reportGroups, region);return rg;}@@ -227,6 +276,7 @@ public class CodeBuildService {if (exportConfig != null) { rg.setExportConfig(exportConfig); }if (tags != null) { rg.setTags(tags); }rg.setLastModified(Instant.now().toEpochMilli() / 1000.0);+ persistRegion(reportGroups, region);return rg;}@@ -235,6 +285,7 @@ public class CodeBuildService {if (store.remove(arn) == null) {throw new AwsException("ResourceNotFoundException", "Report group not found: " + arn, 400);}+ persistRegion(reportGroups, region);}public List<ReportGroup> batchGetReportGroups(String region, List<String> arns) {@@ -277,6 +328,7 @@ public class CodeBuildService {cred.setAuthType(authType);// Token is accepted but not stored in plaintext in a returned fieldstore.put(arn, cred);+ persistRegion(sourceCredentials, region);return cred;}@@ -290,6 +342,7 @@ public class CodeBuildService {throw new AwsException("ResourceNotFoundException","Source credentials not found: " + arn, 400);}+ persistRegion(sourceCredentials, region);}// ---- Curated Environment Images ----diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex e2b80da20f..2333f74750 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -100,6 +100,8 @@ floci:flush-interval-ms: 5000ecs:flush-interval-ms: 5000+ codebuild:+ flush-interval-ms: 5000rds:# Override storage mode for RDS (memory = no Docker volumes; hybrid/persistent = named volume per instance)# mode: memorydiff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServicePersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServicePersistenceTest.javanew file mode 100644index 0000000000..b40a271185--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildServicePersistenceTest.java@@ -0,0 +1,119 @@+package io.github.hectorvent.floci.services.codebuild;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.codebuild.model.Project;+import io.github.hectorvent.floci.services.codebuild.model.ProjectArtifacts;+import io.github.hectorvent.floci.services.codebuild.model.ProjectEnvironment;+import io.github.hectorvent.floci.services.codebuild.model.ProjectSource;+import io.github.hectorvent.floci.services.codebuild.model.ReportGroup;+import io.github.hectorvent.floci.services.codebuild.model.SourceCredential;+import org.junit.jupiter.api.Test;++import java.util.HashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;++/**+ * Verifies CodeBuild durable resources survive a restart. Two service instances share the same+ * {@link StorageFactory} backends; the second simulates a process restart reloading from disk.+ */+class CodeBuildServicePersistenceTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";++ @Test+ void durableResourcesSurviveRestart() {+ SharedStorageFactory storage = new SharedStorageFactory();++ CodeBuildService first = serviceWithStorage(storage);+ first.createProject(REGION, ACCOUNT, "build-proj", "demo",+ source("GITHUB"), null, null, artifacts("NO_ARTIFACTS"), null,+ new ProjectEnvironment(), "arn:aws:iam::" + ACCOUNT + ":role/cb",+ 30, null, null, List.of(Map.of("key", "team", "value", "platform")), null, null, null);+ first.createReportGroup(REGION, ACCOUNT, "rg-1", "TEST", null,+ List.of(Map.of("key", "env", "value", "test")));+ SourceCredential cred = first.importSourceCredentials(REGION, ACCOUNT,+ "tok-secret", "GITHUB", "PERSONAL_ACCESS_TOKEN", true);++ CodeBuildService reloaded = serviceWithStorage(storage);++ List<Project> projects = reloaded.batchGetProjects(REGION, List.of("build-proj"));+ assertEquals(1, projects.size());+ assertEquals("platform", projects.getFirst().getTags().getFirst().get("value"));+ assertEquals(List.of("rg-1"), reloaded.listReportGroups(REGION).stream()+ .map(arn -> reloaded.batchGetReportGroups(REGION, List.of(arn)).getFirst().getName()).toList());+ List<SourceCredential> creds = reloaded.listSourceCredentials(REGION);+ assertEquals(1, creds.size());+ assertEquals(cred.getArn(), creds.getFirst().getArn());+ }++ @Test+ void projectUpdateAndDeleteArePersistedAfterRestart() {+ SharedStorageFactory storage = new SharedStorageFactory();++ CodeBuildService first = serviceWithStorage(storage);+ first.createProject(REGION, ACCOUNT, "p1", "original",+ source("GITHUB"), null, null, artifacts("NO_ARTIFACTS"), null,+ new ProjectEnvironment(), "arn:aws:iam::" + ACCOUNT + ":role/cb",+ null, null, null, null, null, null, null);+ // in-place mutation: update must be written back through StorageBackedMap+ first.updateProject(REGION, "p1", "updated", null, null, null, null, null,+ null, null, null, null, null, null, null, null, null);+ first.createProject(REGION, ACCOUNT, "p2", "to-delete",+ source("GITHUB"), null, null, artifacts("NO_ARTIFACTS"), null,+ new ProjectEnvironment(), "arn:aws:iam::" + ACCOUNT + ":role/cb",+ null, null, null, null, null, null, null);+ first.deleteProject(REGION, "p2");++ CodeBuildService reloaded = serviceWithStorage(storage);+ assertEquals("updated", reloaded.batchGetProjects(REGION, List.of("p1")).getFirst().getDescription());+ assertTrue(reloaded.batchGetProjects(REGION, List.of("p2")).isEmpty(),+ "deleted project must not reappear after restart");+ assertNull(reloaded.batchGetProjects(REGION, List.of("p2")).stream().findFirst().orElse(null));+ }++ private static ProjectSource source(String type) {+ ProjectSource s = new ProjectSource();+ s.setType(type);+ return s;+ }++ private static ProjectArtifacts artifacts(String type) {+ ProjectArtifacts a = new ProjectArtifacts();+ a.setType(type);+ return a;+ }++ private static CodeBuildService serviceWithStorage(StorageFactory storage) {+ CodeBuildService service = new CodeBuildService(+ mock(CodeBuildRunner.class), mock(EmulatorConfig.class), storage);+ service.initializeStorage();+ return service;+ }++ private static final class SharedStorageFactory extends StorageFactory {+ private final Map<String, StorageBackend<String, ?>> stores = new HashMap<>();++ private SharedStorageFactory() {+ super(null, null);+ }++ @Override+ @SuppressWarnings("unchecked")+ public <V> StorageBackend<String, V> create(String serviceName,+ String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return (StorageBackend<String, V>) stores.computeIfAbsent(fileName, ignored -> new InMemoryStorage<>());+ }+ }+}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex f587203c38..3be10f8de4 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -44,6 +44,8 @@ floci:flush-interval-ms: 60000ecs:flush-interval-ms: 60000+ codebuild:+ flush-interval-ms: 60000cloudfront:auth: